From 5a365dbc068f2a8cdfa9813c39cc84bb30e15637 Mon Sep 17 00:00:00 2001 From: Dante Niewenhuis Date: Fri, 25 Oct 2024 13:32:41 +0200 Subject: Rewrote the FlowEngine (#256) * Removed unused components. Updated tests. Improved checkpointing model Improved model, started with SimPowerSource implemented FailureModels and Checkpointing First working version midway commit first update All simulation are now run with a single CPU and single MemoryUnit. multi CPUs are combined into one. This is for performance and explainability. * fixed merge conflicts * Updated M3SA paths. * Fixed small typo --- .../kotlin/org/opendc/compute/api/ComputeClient.kt | 126 ---- .../main/kotlin/org/opendc/compute/api/Flavor.kt | 2 +- .../main/kotlin/org/opendc/compute/api/Resource.kt | 5 - .../src/main/kotlin/org/opendc/compute/api/Task.kt | 79 --- .../kotlin/org/opendc/compute/api/TaskState.kt | 29 +- .../kotlin/org/opendc/compute/api/TaskWatcher.kt | 39 -- .../org/opendc/compute/carbon/CarbonTrace.kt | 8 +- .../org/opendc/compute/carbon/CarbonTraceLoader.kt | 4 +- .../opendc-compute-failure/build.gradle.kts | 1 - .../opendc/compute/failure/hostfault/HostFault.kt | 4 +- .../failure/hostfault/StartStopHostFault.kt | 18 +- .../opendc/compute/failure/models/FailureModel.kt | 4 +- .../failure/models/SampleBasedFailureModel.kt | 2 +- .../failure/models/TraceBasedFailureModel.kt | 4 +- .../org/opendc/compute/failure/prefab/G5k06.kt | 2 +- .../org/opendc/compute/failure/prefab/Lanl05.kt | 2 +- .../org/opendc/compute/failure/prefab/Ldns04.kt | 2 +- .../opendc/compute/failure/prefab/Microsoft99.kt | 2 +- .../org/opendc/compute/failure/prefab/Nd07cpu.kt | 2 +- .../org/opendc/compute/failure/prefab/Overnet03.kt | 2 +- .../org/opendc/compute/failure/prefab/Pl05.kt | 2 +- .../failure/prefab/PrefabFailureModelFactory.kt | 2 +- .../org/opendc/compute/failure/prefab/Skype06.kt | 2 +- .../opendc/compute/failure/prefab/Websites02.kt | 2 +- .../victimselector/StochasticVictimSelector.kt | 2 +- .../failure/victimselector/VictimSelector.kt | 2 +- .../opendc-compute-service/build.gradle.kts | 40 -- .../org/opendc/compute/service/ComputeService.java | 637 -------------------- .../java/org/opendc/compute/service/HostView.java | 78 --- .../org/opendc/compute/service/ServiceFlavor.java | 122 ---- .../org/opendc/compute/service/ServiceImage.java | 101 ---- .../org/opendc/compute/service/ServiceTask.java | 267 --------- .../org/opendc/compute/service/driver/Host.java | 137 ----- .../compute/service/driver/HostListener.java | 41 -- .../opendc/compute/service/driver/HostModel.java | 32 - .../opendc/compute/service/driver/HostState.java | 43 -- .../service/driver/telemetry/GuestCpuStats.java | 43 -- .../service/driver/telemetry/GuestSystemStats.java | 35 -- .../service/driver/telemetry/HostCpuStats.java | 46 -- .../service/driver/telemetry/HostSystemStats.java | 50 -- .../compute/service/telemetry/SchedulerStats.java | 45 -- .../compute/service/scheduler/ComputeScheduler.kt | 50 -- .../compute/service/scheduler/ComputeSchedulers.kt | 118 ---- .../compute/service/scheduler/FilterScheduler.kt | 111 ---- .../compute/service/scheduler/ReplayScheduler.kt | 65 -- .../service/scheduler/filters/ComputeFilter.kt | 42 -- .../scheduler/filters/DifferentHostFilter.kt | 41 -- .../service/scheduler/filters/HostFilter.kt | 41 -- .../scheduler/filters/InstanceCountFilter.kt | 42 -- .../compute/service/scheduler/filters/RamFilter.kt | 55 -- .../service/scheduler/filters/SameHostFilter.kt | 41 -- .../scheduler/filters/VCpuCapacityFilter.kt | 43 -- .../service/scheduler/filters/VCpuFilter.kt | 50 -- .../service/scheduler/weights/CoreRamWeigher.kt | 44 -- .../service/scheduler/weights/HostWeigher.kt | 81 --- .../scheduler/weights/InstanceCountWeigher.kt | 40 -- .../service/scheduler/weights/RamWeigher.kt | 43 -- .../scheduler/weights/VCpuCapacityWeigher.kt | 42 -- .../service/scheduler/weights/VCpuWeigher.kt | 46 -- .../opendc/compute/service/ComputeServiceTest.kt | 386 ------------ .../opendc/compute/service/ServiceFlavorTest.kt | 67 --- .../org/opendc/compute/service/ServiceImageTest.kt | 67 --- .../org/opendc/compute/service/ServiceTaskTest.kt | 442 -------------- .../service/scheduler/FilterSchedulerTest.kt | 536 ----------------- .../src/test/resources/log4j2.xml | 38 -- .../opendc-compute-simulator/build.gradle.kts | 7 +- .../compute/simulator/host/HostListener.java | 41 ++ .../opendc/compute/simulator/host/HostModel.java | 32 + .../opendc/compute/simulator/host/HostState.java | 43 ++ .../compute/simulator/service/ComputeService.java | 652 ++++++++++++++++++++ .../opendc/compute/simulator/service/HostView.java | 78 +++ .../compute/simulator/service/ServiceFlavor.java | 107 ++++ .../compute/simulator/service/ServiceImage.java | 95 +++ .../compute/simulator/service/ServiceTask.java | 230 +++++++ .../compute/simulator/telemetry/GuestCpuStats.java | 43 ++ .../simulator/telemetry/GuestSystemStats.java | 35 ++ .../compute/simulator/telemetry/HostCpuStats.java | 46 ++ .../simulator/telemetry/HostSystemStats.java | 50 ++ .../simulator/telemetry/SchedulerStats.java | 45 ++ .../compute/simulator/MutableServiceRegistry.kt | 65 -- .../opendc/compute/simulator/ServiceRegistry.kt | 61 +- .../compute/simulator/ServiceRegistryImpl.kt | 76 --- .../kotlin/org/opendc/compute/simulator/SimHost.kt | 378 ------------ .../compute/simulator/SimMetaWorkloadMapper.kt | 35 -- .../opendc/compute/simulator/SimWorkloadMapper.kt | 37 -- .../org/opendc/compute/simulator/TaskWatcher.kt | 42 ++ .../org/opendc/compute/simulator/host/SimHost.kt | 369 ++++++++++++ .../simulator/internal/DefaultWorkloadMapper.kt | 46 -- .../org/opendc/compute/simulator/internal/Guest.kt | 247 ++++---- .../compute/simulator/internal/GuestListener.kt | 6 +- .../provisioner/ComputeMonitorProvisioningStep.kt | 16 +- .../provisioner/ComputeServiceProvisioningStep.kt | 4 +- .../compute/simulator/provisioner/ComputeSteps.kt | 10 +- .../simulator/provisioner/HostsProvisioningStep.kt | 24 +- .../compute/simulator/provisioner/Provisioner.kt | 4 +- .../simulator/provisioner/ProvisioningContext.kt | 6 +- .../simulator/scheduler/ComputeScheduler.kt | 49 ++ .../simulator/scheduler/ComputeSchedulers.kt | 118 ++++ .../compute/simulator/scheduler/FilterScheduler.kt | 111 ++++ .../compute/simulator/scheduler/ReplayScheduler.kt | 65 ++ .../simulator/scheduler/filters/ComputeFilter.kt | 42 ++ .../scheduler/filters/DifferentHostFilter.kt | 41 ++ .../simulator/scheduler/filters/HostFilter.kt | 40 ++ .../scheduler/filters/InstanceCountFilter.kt | 42 ++ .../simulator/scheduler/filters/RamFilter.kt | 55 ++ .../simulator/scheduler/filters/SameHostFilter.kt | 41 ++ .../scheduler/filters/VCpuCapacityFilter.kt | 42 ++ .../simulator/scheduler/filters/VCpuFilter.kt | 50 ++ .../simulator/scheduler/weights/CoreRamWeigher.kt | 44 ++ .../simulator/scheduler/weights/HostWeigher.kt | 80 +++ .../scheduler/weights/InstanceCountWeigher.kt | 40 ++ .../simulator/scheduler/weights/RamWeigher.kt | 43 ++ .../scheduler/weights/VCpuCapacityWeigher.kt | 42 ++ .../simulator/scheduler/weights/VCpuWeigher.kt | 46 ++ .../simulator/telemetry/ComputeMetricReader.kt | 664 +++++++++++++++++++++ .../compute/simulator/telemetry/ComputeMonitor.kt | 47 ++ .../telemetry/parquet/ComputeExportConfig.kt | 192 ++++++ .../telemetry/parquet/DfltHostExportColumns.kt | 195 ++++++ .../telemetry/parquet/DfltServiceExportColumns.kt | 105 ++++ .../telemetry/parquet/DfltTaskExportColumns.kt | 171 ++++++ .../telemetry/parquet/ParquetComputeMonitor.kt | 125 ++++ .../compute/simulator/telemetry/parquet/README.md | 70 +++ .../compute/simulator/telemetry/table/HostInfo.kt | 35 ++ .../simulator/telemetry/table/HostTableReader.kt | 150 +++++ .../simulator/telemetry/table/ServiceData.kt | 55 ++ .../telemetry/table/ServiceTableReader.kt | 90 +++ .../compute/simulator/telemetry/table/TaskInfo.kt | 35 ++ .../simulator/telemetry/table/TaskTableReader.kt | 125 ++++ .../org/opendc/compute/simulator/SimHostTest.kt | 393 ------------ .../opendc-compute-telemetry/build.gradle.kts | 44 -- .../compute/telemetry/ComputeMetricReader.kt | 594 ------------------ .../org/opendc/compute/telemetry/ComputeMonitor.kt | 47 -- .../export/parquet/ComputeExportConfig.kt | 192 ------ .../export/parquet/DfltHostExportColumns.kt | 195 ------ .../export/parquet/DfltServiceExportColumns.kt | 95 --- .../export/parquet/DfltTaskExportColumns.kt | 161 ----- .../export/parquet/ParquetComputeMonitor.kt | 125 ---- .../compute/telemetry/export/parquet/README.md | 70 --- .../org/opendc/compute/telemetry/table/HostInfo.kt | 35 -- .../compute/telemetry/table/HostTableReader.kt | 150 ----- .../opendc/compute/telemetry/table/ServiceData.kt | 57 -- .../compute/telemetry/table/ServiceTableReader.kt | 85 --- .../org/opendc/compute/telemetry/table/TaskInfo.kt | 37 -- .../compute/telemetry/table/TaskTableReader.kt | 115 ---- .../src/test/resources/log4j2.xml | 38 -- .../opendc/compute/topology/TopologyFactories.kt | 15 +- .../org/opendc/compute/topology/specs/HostSpec.kt | 11 +- .../org/opendc/compute/workload/ComputeWorkload.kt | 4 +- .../compute/workload/ComputeWorkloadLoader.kt | 70 +-- .../kotlin/org/opendc/compute/workload/Task.kt | 51 ++ .../org/opendc/compute/workload/VirtualMachine.kt | 54 -- .../workload/internal/CompositeComputeWorkload.kt | 6 +- .../workload/internal/HpcSampledComputeWorkload.kt | 14 +- .../internal/LoadSampledComputeWorkload.kt | 6 +- .../workload/internal/TraceComputeWorkload.kt | 4 +- 155 files changed, 5325 insertions(+), 7920 deletions(-) delete mode 100644 opendc-compute/opendc-compute-api/src/main/kotlin/org/opendc/compute/api/ComputeClient.kt delete mode 100644 opendc-compute/opendc-compute-api/src/main/kotlin/org/opendc/compute/api/Task.kt delete mode 100644 opendc-compute/opendc-compute-api/src/main/kotlin/org/opendc/compute/api/TaskWatcher.kt delete mode 100644 opendc-compute/opendc-compute-service/build.gradle.kts delete mode 100644 opendc-compute/opendc-compute-service/src/main/java/org/opendc/compute/service/ComputeService.java delete mode 100644 opendc-compute/opendc-compute-service/src/main/java/org/opendc/compute/service/HostView.java delete mode 100644 opendc-compute/opendc-compute-service/src/main/java/org/opendc/compute/service/ServiceFlavor.java delete mode 100644 opendc-compute/opendc-compute-service/src/main/java/org/opendc/compute/service/ServiceImage.java delete mode 100644 opendc-compute/opendc-compute-service/src/main/java/org/opendc/compute/service/ServiceTask.java delete mode 100644 opendc-compute/opendc-compute-service/src/main/java/org/opendc/compute/service/driver/Host.java delete mode 100644 opendc-compute/opendc-compute-service/src/main/java/org/opendc/compute/service/driver/HostListener.java delete mode 100644 opendc-compute/opendc-compute-service/src/main/java/org/opendc/compute/service/driver/HostModel.java delete mode 100644 opendc-compute/opendc-compute-service/src/main/java/org/opendc/compute/service/driver/HostState.java delete mode 100644 opendc-compute/opendc-compute-service/src/main/java/org/opendc/compute/service/driver/telemetry/GuestCpuStats.java delete mode 100644 opendc-compute/opendc-compute-service/src/main/java/org/opendc/compute/service/driver/telemetry/GuestSystemStats.java delete mode 100644 opendc-compute/opendc-compute-service/src/main/java/org/opendc/compute/service/driver/telemetry/HostCpuStats.java delete mode 100644 opendc-compute/opendc-compute-service/src/main/java/org/opendc/compute/service/driver/telemetry/HostSystemStats.java delete mode 100644 opendc-compute/opendc-compute-service/src/main/java/org/opendc/compute/service/telemetry/SchedulerStats.java delete mode 100644 opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/scheduler/ComputeScheduler.kt delete mode 100644 opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/scheduler/ComputeSchedulers.kt delete mode 100644 opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/scheduler/FilterScheduler.kt delete mode 100644 opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/scheduler/ReplayScheduler.kt delete mode 100644 opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/scheduler/filters/ComputeFilter.kt delete mode 100644 opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/scheduler/filters/DifferentHostFilter.kt delete mode 100644 opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/scheduler/filters/HostFilter.kt delete mode 100644 opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/scheduler/filters/InstanceCountFilter.kt delete mode 100644 opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/scheduler/filters/RamFilter.kt delete mode 100644 opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/scheduler/filters/SameHostFilter.kt delete mode 100644 opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/scheduler/filters/VCpuCapacityFilter.kt delete mode 100644 opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/scheduler/filters/VCpuFilter.kt delete mode 100644 opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/scheduler/weights/CoreRamWeigher.kt delete mode 100644 opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/scheduler/weights/HostWeigher.kt delete mode 100644 opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/scheduler/weights/InstanceCountWeigher.kt delete mode 100644 opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/scheduler/weights/RamWeigher.kt delete mode 100644 opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/scheduler/weights/VCpuCapacityWeigher.kt delete mode 100644 opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/scheduler/weights/VCpuWeigher.kt delete mode 100644 opendc-compute/opendc-compute-service/src/test/kotlin/org/opendc/compute/service/ComputeServiceTest.kt delete mode 100644 opendc-compute/opendc-compute-service/src/test/kotlin/org/opendc/compute/service/ServiceFlavorTest.kt delete mode 100644 opendc-compute/opendc-compute-service/src/test/kotlin/org/opendc/compute/service/ServiceImageTest.kt delete mode 100644 opendc-compute/opendc-compute-service/src/test/kotlin/org/opendc/compute/service/ServiceTaskTest.kt delete mode 100644 opendc-compute/opendc-compute-service/src/test/kotlin/org/opendc/compute/service/scheduler/FilterSchedulerTest.kt delete mode 100644 opendc-compute/opendc-compute-service/src/test/resources/log4j2.xml create mode 100644 opendc-compute/opendc-compute-simulator/src/main/java/org/opendc/compute/simulator/host/HostListener.java create mode 100644 opendc-compute/opendc-compute-simulator/src/main/java/org/opendc/compute/simulator/host/HostModel.java create mode 100644 opendc-compute/opendc-compute-simulator/src/main/java/org/opendc/compute/simulator/host/HostState.java create mode 100644 opendc-compute/opendc-compute-simulator/src/main/java/org/opendc/compute/simulator/service/ComputeService.java create mode 100644 opendc-compute/opendc-compute-simulator/src/main/java/org/opendc/compute/simulator/service/HostView.java create mode 100644 opendc-compute/opendc-compute-simulator/src/main/java/org/opendc/compute/simulator/service/ServiceFlavor.java create mode 100644 opendc-compute/opendc-compute-simulator/src/main/java/org/opendc/compute/simulator/service/ServiceImage.java create mode 100644 opendc-compute/opendc-compute-simulator/src/main/java/org/opendc/compute/simulator/service/ServiceTask.java create mode 100644 opendc-compute/opendc-compute-simulator/src/main/java/org/opendc/compute/simulator/telemetry/GuestCpuStats.java create mode 100644 opendc-compute/opendc-compute-simulator/src/main/java/org/opendc/compute/simulator/telemetry/GuestSystemStats.java create mode 100644 opendc-compute/opendc-compute-simulator/src/main/java/org/opendc/compute/simulator/telemetry/HostCpuStats.java create mode 100644 opendc-compute/opendc-compute-simulator/src/main/java/org/opendc/compute/simulator/telemetry/HostSystemStats.java create mode 100644 opendc-compute/opendc-compute-simulator/src/main/java/org/opendc/compute/simulator/telemetry/SchedulerStats.java delete mode 100644 opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/MutableServiceRegistry.kt delete mode 100644 opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/ServiceRegistryImpl.kt delete mode 100644 opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/SimHost.kt delete mode 100644 opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/SimMetaWorkloadMapper.kt delete mode 100644 opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/SimWorkloadMapper.kt create mode 100644 opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/TaskWatcher.kt create mode 100644 opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/host/SimHost.kt delete mode 100644 opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/internal/DefaultWorkloadMapper.kt create mode 100644 opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/ComputeScheduler.kt create mode 100644 opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/ComputeSchedulers.kt create mode 100644 opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/FilterScheduler.kt create mode 100644 opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/ReplayScheduler.kt create mode 100644 opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/filters/ComputeFilter.kt create mode 100644 opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/filters/DifferentHostFilter.kt create mode 100644 opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/filters/HostFilter.kt create mode 100644 opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/filters/InstanceCountFilter.kt create mode 100644 opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/filters/RamFilter.kt create mode 100644 opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/filters/SameHostFilter.kt create mode 100644 opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/filters/VCpuCapacityFilter.kt create mode 100644 opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/filters/VCpuFilter.kt create mode 100644 opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/weights/CoreRamWeigher.kt create mode 100644 opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/weights/HostWeigher.kt create mode 100644 opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/weights/InstanceCountWeigher.kt create mode 100644 opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/weights/RamWeigher.kt create mode 100644 opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/weights/VCpuCapacityWeigher.kt create mode 100644 opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/weights/VCpuWeigher.kt create mode 100644 opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/telemetry/ComputeMetricReader.kt create mode 100644 opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/telemetry/ComputeMonitor.kt create mode 100644 opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/telemetry/parquet/ComputeExportConfig.kt create mode 100644 opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/telemetry/parquet/DfltHostExportColumns.kt create mode 100644 opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/telemetry/parquet/DfltServiceExportColumns.kt create mode 100644 opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/telemetry/parquet/DfltTaskExportColumns.kt create mode 100644 opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/telemetry/parquet/ParquetComputeMonitor.kt create mode 100644 opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/telemetry/parquet/README.md create mode 100644 opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/telemetry/table/HostInfo.kt create mode 100644 opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/telemetry/table/HostTableReader.kt create mode 100644 opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/telemetry/table/ServiceData.kt create mode 100644 opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/telemetry/table/ServiceTableReader.kt create mode 100644 opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/telemetry/table/TaskInfo.kt create mode 100644 opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/telemetry/table/TaskTableReader.kt delete mode 100644 opendc-compute/opendc-compute-simulator/src/test/kotlin/org/opendc/compute/simulator/SimHostTest.kt delete mode 100644 opendc-compute/opendc-compute-telemetry/build.gradle.kts delete mode 100644 opendc-compute/opendc-compute-telemetry/src/main/kotlin/org/opendc/compute/telemetry/ComputeMetricReader.kt delete mode 100644 opendc-compute/opendc-compute-telemetry/src/main/kotlin/org/opendc/compute/telemetry/ComputeMonitor.kt delete mode 100644 opendc-compute/opendc-compute-telemetry/src/main/kotlin/org/opendc/compute/telemetry/export/parquet/ComputeExportConfig.kt delete mode 100644 opendc-compute/opendc-compute-telemetry/src/main/kotlin/org/opendc/compute/telemetry/export/parquet/DfltHostExportColumns.kt delete mode 100644 opendc-compute/opendc-compute-telemetry/src/main/kotlin/org/opendc/compute/telemetry/export/parquet/DfltServiceExportColumns.kt delete mode 100644 opendc-compute/opendc-compute-telemetry/src/main/kotlin/org/opendc/compute/telemetry/export/parquet/DfltTaskExportColumns.kt delete mode 100644 opendc-compute/opendc-compute-telemetry/src/main/kotlin/org/opendc/compute/telemetry/export/parquet/ParquetComputeMonitor.kt delete mode 100644 opendc-compute/opendc-compute-telemetry/src/main/kotlin/org/opendc/compute/telemetry/export/parquet/README.md delete mode 100644 opendc-compute/opendc-compute-telemetry/src/main/kotlin/org/opendc/compute/telemetry/table/HostInfo.kt delete mode 100644 opendc-compute/opendc-compute-telemetry/src/main/kotlin/org/opendc/compute/telemetry/table/HostTableReader.kt delete mode 100644 opendc-compute/opendc-compute-telemetry/src/main/kotlin/org/opendc/compute/telemetry/table/ServiceData.kt delete mode 100644 opendc-compute/opendc-compute-telemetry/src/main/kotlin/org/opendc/compute/telemetry/table/ServiceTableReader.kt delete mode 100644 opendc-compute/opendc-compute-telemetry/src/main/kotlin/org/opendc/compute/telemetry/table/TaskInfo.kt delete mode 100644 opendc-compute/opendc-compute-telemetry/src/main/kotlin/org/opendc/compute/telemetry/table/TaskTableReader.kt delete mode 100644 opendc-compute/opendc-compute-telemetry/src/test/resources/log4j2.xml create mode 100644 opendc-compute/opendc-compute-workload/src/main/kotlin/org/opendc/compute/workload/Task.kt delete mode 100644 opendc-compute/opendc-compute-workload/src/main/kotlin/org/opendc/compute/workload/VirtualMachine.kt (limited to 'opendc-compute') diff --git a/opendc-compute/opendc-compute-api/src/main/kotlin/org/opendc/compute/api/ComputeClient.kt b/opendc-compute/opendc-compute-api/src/main/kotlin/org/opendc/compute/api/ComputeClient.kt deleted file mode 100644 index 9e24a3fd..00000000 --- a/opendc-compute/opendc-compute-api/src/main/kotlin/org/opendc/compute/api/ComputeClient.kt +++ /dev/null @@ -1,126 +0,0 @@ -/* - * Copyright (c) 2021 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.compute.api - -import org.opendc.simulator.compute.workload.SimWorkload -import java.util.UUID - -/** - * A client interface for the OpenDC Compute service. - */ -public interface ComputeClient : AutoCloseable { - /** - * Obtain the list of [Flavor]s accessible by the requesting user. - */ - public fun queryFlavors(): List - - /** - * Obtain a [Flavor] by its unique identifier. - * - * @param id The identifier of the flavor. - */ - public fun findFlavor(id: UUID): Flavor? - - /** - * Create a new [Flavor] instance at this compute service. - * - * @param name The name of the flavor. - * @param cpuCount The amount of CPU cores for this flavor. - * @param memorySize The size of the memory in MB. - * @param labels The identifying labels of the image. - * @param meta The non-identifying meta-data of the image. - */ - public fun newFlavor( - name: String, - cpuCount: Int, - memorySize: Long, - labels: Map = emptyMap(), - meta: Map = emptyMap(), - ): Flavor - - /** - * Obtain the list of [Image]s accessible by the requesting user. - */ - public fun queryImages(): List - - /** - * Obtain a [Image] by its unique identifier. - * - * @param id The identifier of the image. - */ - public fun findImage(id: UUID): Image? - - /** - * Create a new [Image] instance at this compute service. - * - * @param name The name of the image. - * @param labels The identifying labels of the image. - * @param meta The non-identifying meta-data of the image. - */ - public fun newImage( - name: String, - labels: Map = emptyMap(), - meta: Map = emptyMap(), - ): Image - - /** - * Obtain the list of [Task]s accessible by the requesting user. - */ - public fun queryTasks(): List - - /** - * Obtain a [Task] by its unique identifier. - * - * @param id The identifier of the task. - */ - public fun findTask(id: UUID): Task? - - /** - * Create a new [Task] instance at this compute service. - * - * @param name The name of the task to deploy. - * @param image The image to be deployed. - * @param flavor The flavor of the machine instance to run this [image] on. - * @param labels The identifying labels of the task. - * @param meta The non-identifying meta-data of the task. - * @param start A flag to indicate that the task should be started immediately. - */ - public fun newTask( - name: String, - image: Image, - flavor: Flavor, - labels: Map = emptyMap(), - meta: Map = emptyMap(), - start: Boolean = true, - ): Task - - public fun rescheduleTask( - task: Task, - workload: SimWorkload, - ) - - /** - * Release the resources associated with this client, preventing any further API calls. - */ - public override fun close() -} diff --git a/opendc-compute/opendc-compute-api/src/main/kotlin/org/opendc/compute/api/Flavor.kt b/opendc-compute/opendc-compute-api/src/main/kotlin/org/opendc/compute/api/Flavor.kt index 99042c24..e88379f6 100644 --- a/opendc-compute/opendc-compute-api/src/main/kotlin/org/opendc/compute/api/Flavor.kt +++ b/opendc-compute/opendc-compute-api/src/main/kotlin/org/opendc/compute/api/Flavor.kt @@ -23,7 +23,7 @@ package org.opendc.compute.api /** - * Flavors define the compute and memory capacity of [Task] instance. To put it simply, a flavor is an available + * Flavors define the compute and memory capacity of [ServiceTask] instance. To put it simply, a flavor is an available * hardware configuration for a task. It defines the size of a virtual task that can be launched. */ public interface Flavor : Resource { diff --git a/opendc-compute/opendc-compute-api/src/main/kotlin/org/opendc/compute/api/Resource.kt b/opendc-compute/opendc-compute-api/src/main/kotlin/org/opendc/compute/api/Resource.kt index 58082130..2c3822a7 100644 --- a/opendc-compute/opendc-compute-api/src/main/kotlin/org/opendc/compute/api/Resource.kt +++ b/opendc-compute/opendc-compute-api/src/main/kotlin/org/opendc/compute/api/Resource.kt @@ -38,11 +38,6 @@ public interface Resource { */ public val name: String - /** - * The identifying labels attached to the resource. - */ - public val labels: Map - /** * The non-identifying metadata attached to the resource. */ diff --git a/opendc-compute/opendc-compute-api/src/main/kotlin/org/opendc/compute/api/Task.kt b/opendc-compute/opendc-compute-api/src/main/kotlin/org/opendc/compute/api/Task.kt deleted file mode 100644 index 23f2cb91..00000000 --- a/opendc-compute/opendc-compute-api/src/main/kotlin/org/opendc/compute/api/Task.kt +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Copyright (c) 2021 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.compute.api - -import java.time.Instant - -/** - * A stateful object representing a task instance that is running on some physical or virtual machine. - */ -public interface Task : Resource { - /** - * The flavor of the task. - */ - public val flavor: Flavor - - /** - * The image of the task. - */ - public val image: Image - - /** - * The last known state of the task. - */ - public val state: TaskState - - /** - * The number of times a Task has been stopped due to failures - */ - public val numFailures: Int - - /** - * The most recent moment in time when the task was launched. - */ - public val launchedAt: Instant? - - /** - * Request the task to be started. - */ - public fun start() - - /** - * Request the task to be stopped. - */ - public fun stop() - - /** - * Register the specified [TaskWatcher] to watch the state of the task. - * - * @param watcher The watcher to register for the task. - */ - public fun watch(watcher: TaskWatcher) - - /** - * De-register the specified [TaskWatcher] from the task to stop it from receiving events. - * - * @param watcher The watcher to de-register from the task. - */ - public fun unwatch(watcher: TaskWatcher) -} diff --git a/opendc-compute/opendc-compute-api/src/main/kotlin/org/opendc/compute/api/TaskState.kt b/opendc-compute/opendc-compute-api/src/main/kotlin/org/opendc/compute/api/TaskState.kt index a093ff47..f3f2ca6f 100644 --- a/opendc-compute/opendc-compute-api/src/main/kotlin/org/opendc/compute/api/TaskState.kt +++ b/opendc-compute/opendc-compute-api/src/main/kotlin/org/opendc/compute/api/TaskState.kt @@ -27,14 +27,15 @@ package org.opendc.compute.api */ public enum class TaskState { /** - * Resources are being allocated for the instance. The instance is not running yet. + * A static task is created + * */ - PROVISIONING, + CREATED, /** - * A user shut down the instance. + * Resources are being allocated for the instance. The instance is not running yet. */ - TERMINATED, + PROVISIONING, /** * The task instance is booting up or running. @@ -42,12 +43,26 @@ public enum class TaskState { RUNNING, /** - * The task is in an error state. + * The task is in a failed state. + */ + FAILED, + + /** + * The task has been terminated due to too many failures + * + */ + TERMINATED, + + /** + * The task has been completed successfully + * */ - ERROR, + COMPLETED, /** - * The task has been deleted and cannot be started later on. + * Task has been deleted + * + * @constructor Create empty Deleted */ DELETED, } diff --git a/opendc-compute/opendc-compute-api/src/main/kotlin/org/opendc/compute/api/TaskWatcher.kt b/opendc-compute/opendc-compute-api/src/main/kotlin/org/opendc/compute/api/TaskWatcher.kt deleted file mode 100644 index 423d7dec..00000000 --- a/opendc-compute/opendc-compute-api/src/main/kotlin/org/opendc/compute/api/TaskWatcher.kt +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright (c) 2021 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.compute.api - -/** - * An interface used to watch the state of [Task] instances. - */ -public interface TaskWatcher { - /** - * This method is invoked when the state of a [Task] changes. - * - * @param task The task whose state has changed. - * @param newState The new state of the task. - */ - public fun onStateChanged( - task: Task, - newState: TaskState, - ) {} -} diff --git a/opendc-compute/opendc-compute-carbon/src/main/kotlin/org/opendc/compute/carbon/CarbonTrace.kt b/opendc-compute/opendc-compute-carbon/src/main/kotlin/org/opendc/compute/carbon/CarbonTrace.kt index 2ba3e4e3..6eb7a762 100644 --- a/opendc-compute/opendc-compute-carbon/src/main/kotlin/org/opendc/compute/carbon/CarbonTrace.kt +++ b/opendc-compute/opendc-compute-carbon/src/main/kotlin/org/opendc/compute/carbon/CarbonTrace.kt @@ -40,7 +40,7 @@ import java.time.Instant public data class CarbonFragment( var startTime: Long, var endTime: Long, - var carbonIntensity: Double, + var carbonIntensity: Float, ) { init { require(endTime > startTime) { @@ -67,7 +67,7 @@ public class CarbonTrace(reports: List? = null) { return index < numberOfReports } - public fun getCarbonIntensity(timestamp: Instant): Double { + public fun getCarbonIntensity(timestamp: Instant): Float { return getCarbonIntensity(timestamp.toEpochMilli()) } @@ -79,9 +79,9 @@ public class CarbonTrace(reports: List? = null) { * @param timestamp * @return The carbon intensity at the given timestamp in gCO2/kWh */ - public fun getCarbonIntensity(timestamp: Long): Double { + public fun getCarbonIntensity(timestamp: Long): Float { if (reports == null) { - return 0.0 + return 0.0f } var currentFragment: CarbonFragment diff --git a/opendc-compute/opendc-compute-carbon/src/main/kotlin/org/opendc/compute/carbon/CarbonTraceLoader.kt b/opendc-compute/opendc-compute-carbon/src/main/kotlin/org/opendc/compute/carbon/CarbonTraceLoader.kt index b66aedf9..12340adf 100644 --- a/opendc-compute/opendc-compute-carbon/src/main/kotlin/org/opendc/compute/carbon/CarbonTraceLoader.kt +++ b/opendc-compute/opendc-compute-carbon/src/main/kotlin/org/opendc/compute/carbon/CarbonTraceLoader.kt @@ -56,7 +56,7 @@ public class CarbonTraceLoader { try { while (reader.nextRow()) { val startTime = reader.getInstant(startTimeCol)!! - val carbonIntensity = reader.getDouble(carbonIntensityCol) + val carbonIntensity = reader.getFloat(carbonIntensityCol) builder.add(startTime, carbonIntensity) } @@ -106,7 +106,7 @@ public class CarbonTraceLoader { */ fun add( startTime: Instant, - carbonIntensity: Double, + carbonIntensity: Float, ) { fragments.add( CarbonFragment(startTime.toEpochMilli(), Long.MAX_VALUE, carbonIntensity), diff --git a/opendc-compute/opendc-compute-failure/build.gradle.kts b/opendc-compute/opendc-compute-failure/build.gradle.kts index d6ec9116..3bd7af83 100644 --- a/opendc-compute/opendc-compute-failure/build.gradle.kts +++ b/opendc-compute/opendc-compute-failure/build.gradle.kts @@ -32,7 +32,6 @@ dependencies { implementation(projects.opendcCommon) implementation(project(mapOf("path" to ":opendc-trace:opendc-trace-api"))) implementation(project(mapOf("path" to ":opendc-simulator:opendc-simulator-compute"))) - implementation(project(mapOf("path" to ":opendc-compute:opendc-compute-service"))) implementation(project(mapOf("path" to ":opendc-compute:opendc-compute-simulator"))) api(libs.commons.math3) diff --git a/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/hostfault/HostFault.kt b/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/hostfault/HostFault.kt index 4134c58a..a1e4c489 100644 --- a/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/hostfault/HostFault.kt +++ b/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/hostfault/HostFault.kt @@ -22,8 +22,8 @@ package org.opendc.compute.failure.hostfault -import org.opendc.compute.service.ComputeService -import org.opendc.compute.simulator.SimHost +import org.opendc.compute.simulator.host.SimHost +import org.opendc.compute.simulator.service.ComputeService /** * Interface responsible for applying the fault to a host. diff --git a/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/hostfault/StartStopHostFault.kt b/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/hostfault/StartStopHostFault.kt index 0bebca66..c7ddc5b5 100644 --- a/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/hostfault/StartStopHostFault.kt +++ b/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/hostfault/StartStopHostFault.kt @@ -23,10 +23,8 @@ package org.opendc.compute.failure.hostfault import kotlinx.coroutines.delay -import org.opendc.compute.api.ComputeClient -import org.opendc.compute.service.ComputeService -import org.opendc.compute.simulator.SimHost -import org.opendc.simulator.compute.workload.SimWorkload +import org.opendc.compute.simulator.host.SimHost +import org.opendc.compute.simulator.service.ComputeService /** * A type of [HostFault] where the hosts are stopped and recover after a given amount of time. @@ -38,18 +36,20 @@ public class StartStopHostFault( victims: List, faultDuration: Long, ) { - val client: ComputeClient = service.newClient() + val client: ComputeService.ComputeClient = service.newClient() for (host in victims) { - val tasks = host.instances + val guests = host.getGuests() - val sortedTasks = tasks.sortedBy { it.name } - val snapshots = sortedTasks.map { (it.meta["workload"] as SimWorkload).getSnapshot() } + val snapshots = guests.map { it.virtualMachine!!.getActiveWorkload().getSnapshot() } + val tasks = guests.map { it.task } host.fail() - for ((task, snapshot) in sortedTasks.zip(snapshots)) { + for ((task, snapshot) in tasks.zip(snapshots)) { client.rescheduleTask(task, snapshot) } + + print("test") } delay(faultDuration) diff --git a/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/models/FailureModel.kt b/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/models/FailureModel.kt index 5f05d96c..f0cee5f2 100644 --- a/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/models/FailureModel.kt +++ b/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/models/FailureModel.kt @@ -29,8 +29,8 @@ import kotlinx.coroutines.launch import org.opendc.compute.failure.hostfault.HostFault import org.opendc.compute.failure.hostfault.StartStopHostFault import org.opendc.compute.failure.victimselector.StochasticVictimSelector -import org.opendc.compute.service.ComputeService -import org.opendc.compute.simulator.SimHost +import org.opendc.compute.simulator.host.SimHost +import org.opendc.compute.simulator.service.ComputeService import java.time.InstantSource import java.util.random.RandomGenerator import kotlin.coroutines.CoroutineContext diff --git a/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/models/SampleBasedFailureModel.kt b/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/models/SampleBasedFailureModel.kt index 3ae66f6f..ae4077e8 100644 --- a/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/models/SampleBasedFailureModel.kt +++ b/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/models/SampleBasedFailureModel.kt @@ -24,7 +24,7 @@ package org.opendc.compute.failure.models import kotlinx.coroutines.delay import org.apache.commons.math3.distribution.RealDistribution -import org.opendc.compute.service.ComputeService +import org.opendc.compute.simulator.service.ComputeService import java.time.InstantSource import java.util.random.RandomGenerator import kotlin.coroutines.CoroutineContext diff --git a/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/models/TraceBasedFailureModel.kt b/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/models/TraceBasedFailureModel.kt index f1ff09e9..cab96cd8 100644 --- a/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/models/TraceBasedFailureModel.kt +++ b/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/models/TraceBasedFailureModel.kt @@ -23,7 +23,7 @@ package org.opendc.compute.failure.models import kotlinx.coroutines.delay -import org.opendc.compute.service.ComputeService +import org.opendc.compute.simulator.service.ComputeService import org.opendc.trace.Trace import org.opendc.trace.conv.FAILURE_DURATION import org.opendc.trace.conv.FAILURE_INTENSITY @@ -71,7 +71,7 @@ public class TraceBasedFailureModel( service: ComputeService, random: RandomGenerator, pathToTrace: String, - private val repeat: Boolean = true, + private val repeat: Boolean = false, ) : FailureModel(context, clock, service, random) { private val failureList = loadTrace(pathToTrace) diff --git a/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/prefab/G5k06.kt b/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/prefab/G5k06.kt index da58250d..1c1d65d2 100644 --- a/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/prefab/G5k06.kt +++ b/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/prefab/G5k06.kt @@ -29,7 +29,7 @@ import org.apache.commons.math3.distribution.UniformRealDistribution import org.apache.commons.math3.distribution.WeibullDistribution import org.apache.commons.math3.random.Well19937c import org.opendc.compute.failure.models.SampleBasedFailureModel -import org.opendc.compute.service.ComputeService +import org.opendc.compute.simulator.service.ComputeService import java.time.InstantSource import java.util.random.RandomGenerator import kotlin.coroutines.CoroutineContext diff --git a/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/prefab/Lanl05.kt b/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/prefab/Lanl05.kt index 3e722630..d3d4d704 100644 --- a/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/prefab/Lanl05.kt +++ b/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/prefab/Lanl05.kt @@ -29,7 +29,7 @@ import org.apache.commons.math3.distribution.UniformRealDistribution import org.apache.commons.math3.distribution.WeibullDistribution import org.apache.commons.math3.random.Well19937c import org.opendc.compute.failure.models.SampleBasedFailureModel -import org.opendc.compute.service.ComputeService +import org.opendc.compute.simulator.service.ComputeService import java.time.InstantSource import java.util.random.RandomGenerator import kotlin.coroutines.CoroutineContext diff --git a/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/prefab/Ldns04.kt b/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/prefab/Ldns04.kt index 4a8b3c0f..fd4bd351 100644 --- a/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/prefab/Ldns04.kt +++ b/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/prefab/Ldns04.kt @@ -29,7 +29,7 @@ import org.apache.commons.math3.distribution.UniformRealDistribution import org.apache.commons.math3.distribution.WeibullDistribution import org.apache.commons.math3.random.Well19937c import org.opendc.compute.failure.models.SampleBasedFailureModel -import org.opendc.compute.service.ComputeService +import org.opendc.compute.simulator.service.ComputeService import java.time.InstantSource import java.util.random.RandomGenerator import kotlin.coroutines.CoroutineContext diff --git a/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/prefab/Microsoft99.kt b/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/prefab/Microsoft99.kt index 725f6622..db27efb8 100644 --- a/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/prefab/Microsoft99.kt +++ b/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/prefab/Microsoft99.kt @@ -29,7 +29,7 @@ import org.apache.commons.math3.distribution.UniformRealDistribution import org.apache.commons.math3.distribution.WeibullDistribution import org.apache.commons.math3.random.Well19937c import org.opendc.compute.failure.models.SampleBasedFailureModel -import org.opendc.compute.service.ComputeService +import org.opendc.compute.simulator.service.ComputeService import java.time.InstantSource import java.util.random.RandomGenerator import kotlin.coroutines.CoroutineContext diff --git a/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/prefab/Nd07cpu.kt b/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/prefab/Nd07cpu.kt index 100a3a8d..eac2df0c 100644 --- a/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/prefab/Nd07cpu.kt +++ b/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/prefab/Nd07cpu.kt @@ -29,7 +29,7 @@ import org.apache.commons.math3.distribution.UniformRealDistribution import org.apache.commons.math3.distribution.WeibullDistribution import org.apache.commons.math3.random.Well19937c import org.opendc.compute.failure.models.SampleBasedFailureModel -import org.opendc.compute.service.ComputeService +import org.opendc.compute.simulator.service.ComputeService import java.time.InstantSource import java.util.random.RandomGenerator import kotlin.coroutines.CoroutineContext diff --git a/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/prefab/Overnet03.kt b/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/prefab/Overnet03.kt index 4f5e3f84..965e861d 100644 --- a/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/prefab/Overnet03.kt +++ b/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/prefab/Overnet03.kt @@ -29,7 +29,7 @@ import org.apache.commons.math3.distribution.UniformRealDistribution import org.apache.commons.math3.distribution.WeibullDistribution import org.apache.commons.math3.random.Well19937c import org.opendc.compute.failure.models.SampleBasedFailureModel -import org.opendc.compute.service.ComputeService +import org.opendc.compute.simulator.service.ComputeService import java.time.InstantSource import java.util.random.RandomGenerator import kotlin.coroutines.CoroutineContext diff --git a/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/prefab/Pl05.kt b/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/prefab/Pl05.kt index 3e1f1b58..5102bd6b 100644 --- a/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/prefab/Pl05.kt +++ b/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/prefab/Pl05.kt @@ -29,7 +29,7 @@ import org.apache.commons.math3.distribution.UniformRealDistribution import org.apache.commons.math3.distribution.WeibullDistribution import org.apache.commons.math3.random.Well19937c import org.opendc.compute.failure.models.SampleBasedFailureModel -import org.opendc.compute.service.ComputeService +import org.opendc.compute.simulator.service.ComputeService import java.time.InstantSource import java.util.random.RandomGenerator import kotlin.coroutines.CoroutineContext diff --git a/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/prefab/PrefabFailureModelFactory.kt b/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/prefab/PrefabFailureModelFactory.kt index 477f3ac4..361620b0 100644 --- a/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/prefab/PrefabFailureModelFactory.kt +++ b/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/prefab/PrefabFailureModelFactory.kt @@ -25,7 +25,7 @@ package org.opendc.compute.failure.prefab import org.opendc.compute.failure.models.SampleBasedFailureModel -import org.opendc.compute.service.ComputeService +import org.opendc.compute.simulator.service.ComputeService import java.time.InstantSource import java.util.random.RandomGenerator import kotlin.coroutines.CoroutineContext diff --git a/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/prefab/Skype06.kt b/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/prefab/Skype06.kt index 7495bf66..30f7ec1b 100644 --- a/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/prefab/Skype06.kt +++ b/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/prefab/Skype06.kt @@ -29,7 +29,7 @@ import org.apache.commons.math3.distribution.UniformRealDistribution import org.apache.commons.math3.distribution.WeibullDistribution import org.apache.commons.math3.random.Well19937c import org.opendc.compute.failure.models.SampleBasedFailureModel -import org.opendc.compute.service.ComputeService +import org.opendc.compute.simulator.service.ComputeService import java.time.InstantSource import java.util.random.RandomGenerator import kotlin.coroutines.CoroutineContext diff --git a/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/prefab/Websites02.kt b/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/prefab/Websites02.kt index 77bb0d64..df37c27b 100644 --- a/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/prefab/Websites02.kt +++ b/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/prefab/Websites02.kt @@ -29,7 +29,7 @@ import org.apache.commons.math3.distribution.UniformRealDistribution import org.apache.commons.math3.distribution.WeibullDistribution import org.apache.commons.math3.random.Well19937c import org.opendc.compute.failure.models.SampleBasedFailureModel -import org.opendc.compute.service.ComputeService +import org.opendc.compute.simulator.service.ComputeService import java.time.InstantSource import java.util.random.RandomGenerator import kotlin.coroutines.CoroutineContext diff --git a/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/victimselector/StochasticVictimSelector.kt b/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/victimselector/StochasticVictimSelector.kt index fef60eb3..9b92cf33 100644 --- a/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/victimselector/StochasticVictimSelector.kt +++ b/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/victimselector/StochasticVictimSelector.kt @@ -22,7 +22,7 @@ package org.opendc.compute.failure.victimselector -import org.opendc.compute.simulator.SimHost +import org.opendc.compute.simulator.host.SimHost import java.util.SplittableRandom import java.util.random.RandomGenerator import kotlin.math.max diff --git a/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/victimselector/VictimSelector.kt b/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/victimselector/VictimSelector.kt index 955cbced..b276b8b4 100644 --- a/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/victimselector/VictimSelector.kt +++ b/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/victimselector/VictimSelector.kt @@ -22,7 +22,7 @@ package org.opendc.compute.failure.victimselector -import org.opendc.compute.simulator.SimHost +import org.opendc.compute.simulator.host.SimHost /** * Interface responsible for selecting the victim(s) for fault injection. diff --git a/opendc-compute/opendc-compute-service/build.gradle.kts b/opendc-compute/opendc-compute-service/build.gradle.kts deleted file mode 100644 index cd25e05c..00000000 --- a/opendc-compute/opendc-compute-service/build.gradle.kts +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright (c) 2021 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -description = "OpenDC Compute Service implementation" - -// Build configuration -plugins { - `kotlin-library-conventions` -} - -dependencies { - api(projects.opendcCompute.opendcComputeApi) - implementation(projects.opendcCommon) - implementation(libs.kotlin.logging) - implementation(project(mapOf("path" to ":opendc-simulator:opendc-simulator-compute"))) - - testImplementation(projects.opendcSimulator.opendcSimulatorCore) - testImplementation(libs.log4j.slf4j) - testRuntimeOnly(libs.log4j.core) - testRuntimeOnly(libs.log4j.slf4j) -} diff --git a/opendc-compute/opendc-compute-service/src/main/java/org/opendc/compute/service/ComputeService.java b/opendc-compute/opendc-compute-service/src/main/java/org/opendc/compute/service/ComputeService.java deleted file mode 100644 index ad01ee57..00000000 --- a/opendc-compute/opendc-compute-service/src/main/java/org/opendc/compute/service/ComputeService.java +++ /dev/null @@ -1,637 +0,0 @@ -/* - * Copyright (c) 2022 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.compute.service; - -import java.time.Duration; -import java.time.Instant; -import java.time.InstantSource; -import java.util.ArrayDeque; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Deque; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.SplittableRandom; -import java.util.UUID; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.opendc.common.Dispatcher; -import org.opendc.common.util.Pacer; -import org.opendc.compute.api.ComputeClient; -import org.opendc.compute.api.Flavor; -import org.opendc.compute.api.Image; -import org.opendc.compute.api.Task; -import org.opendc.compute.api.TaskState; -import org.opendc.compute.service.driver.Host; -import org.opendc.compute.service.driver.HostListener; -import org.opendc.compute.service.driver.HostModel; -import org.opendc.compute.service.driver.HostState; -import org.opendc.compute.service.scheduler.ComputeScheduler; -import org.opendc.compute.service.telemetry.SchedulerStats; -import org.opendc.simulator.compute.workload.SimWorkload; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * The {@link ComputeService} hosts the API implementation of the OpenDC Compute Engine. - */ -public final class ComputeService implements AutoCloseable { - private static final Logger LOGGER = LoggerFactory.getLogger(ComputeService.class); - - /** - * The {@link InstantSource} representing the clock tracking the (simulation) time. - */ - private final InstantSource clock; - - /** - * The {@link ComputeScheduler} responsible for placing the tasks onto hosts. - */ - private final ComputeScheduler scheduler; - - /** - * The {@link Pacer} used to pace the scheduling requests. - */ - private final Pacer pacer; - - /** - * The {@link SplittableRandom} used to generate the unique identifiers for the service resources. - */ - private final SplittableRandom random = new SplittableRandom(0); - - private final int maxNumFailures; - - /** - * A flag to indicate that the service is closed. - */ - private boolean isClosed; - - /** - * A mapping from host to host view. - */ - private final Map hostToView = new HashMap<>(); - - /** - * The available hypervisors. - */ - private final Set availableHosts = new HashSet<>(); - - /** - * The tasks that should be launched by the service. - */ - private final Deque taskQueue = new ArrayDeque<>(); - - /** - * The active tasks in the system. - */ - private final Map activeTasks = new HashMap<>(); - - /** - * The registered flavors for this compute service. - */ - private final Map flavorById = new HashMap<>(); - - private final List flavors = new ArrayList<>(); - - /** - * The registered images for this compute service. - */ - private final Map imageById = new HashMap<>(); - - private final List images = new ArrayList<>(); - - /** - * The registered tasks for this compute service. - */ - private final Map taskById = new HashMap<>(); - - private final List tasks = new ArrayList<>(); - - /** - * A [HostListener] used to track the active tasks. - */ - private final HostListener hostListener = new HostListener() { - @Override - public void onStateChanged(@NotNull Host host, @NotNull HostState newState) { - LOGGER.debug("Host {} state changed: {}", host, newState); - - final HostView hv = hostToView.get(host); - - if (hv != null) { - if (newState == HostState.UP) { - availableHosts.add(hv); - } else { - availableHosts.remove(hv); - } - } - - // Re-schedule on the new machine - requestSchedulingCycle(); - } - - @Override - public void onStateChanged(@NotNull Host host, @NotNull Task task, @NotNull TaskState newState) { - final ServiceTask serviceTask = (ServiceTask) task; - - if (serviceTask.getHost() != host) { - // This can happen when a task is rescheduled and started on another machine, while being deleted from - // the old machine. - return; - } - - serviceTask.setState(newState); - - // TODO: move the removal of tasks when max Failures are reached to here - if (newState == TaskState.TERMINATED || newState == TaskState.DELETED || newState == TaskState.ERROR) { - LOGGER.info("task {} {} {} finished", task.getUid(), task.getName(), task.getFlavor()); - - if (activeTasks.remove(task) != null) { - tasksActive--; - } - - HostView hv = hostToView.get(host); - final ServiceFlavor flavor = serviceTask.getFlavor(); - if (hv != null) { - hv.provisionedCores -= flavor.getCoreCount(); - hv.instanceCount--; - hv.availableMemory += flavor.getMemorySize(); - } else { - LOGGER.error("Unknown host {}", host); - } - - // Try to reschedule if needed - requestSchedulingCycle(); - } - } - }; - - private int maxCores = 0; - private long maxMemory = 0L; - private long attemptsSuccess = 0L; - private long attemptsFailure = 0L; - private long attemptsError = 0L; - private int tasksPending = 0; - private int tasksActive = 0; - - /** - * Construct a {@link ComputeService} instance. - */ - ComputeService(Dispatcher dispatcher, ComputeScheduler scheduler, Duration quantum, int maxNumFailures) { - this.clock = dispatcher.getTimeSource(); - this.scheduler = scheduler; - this.pacer = new Pacer(dispatcher, quantum.toMillis(), (time) -> doSchedule()); - this.maxNumFailures = maxNumFailures; - } - - /** - * Create a new {@link Builder} instance. - */ - public static Builder builder(Dispatcher dispatcher, ComputeScheduler scheduler) { - return new Builder(dispatcher, scheduler); - } - - /** - * Create a new {@link ComputeClient} to control the compute service. - */ - public ComputeClient newClient() { - if (isClosed) { - throw new IllegalStateException("Service is closed"); - } - return new Client(this); - } - - /** - * Return the {@link Task}s hosted by this service. - */ - public List getTasks() { - return Collections.unmodifiableList(tasks); - } - - /** - * Add a {@link Host} to the scheduling pool of the compute service. - */ - public void addHost(Host host) { - // Check if host is already known - if (hostToView.containsKey(host)) { - return; - } - - HostView hv = new HostView(host); - HostModel model = host.getModel(); - - maxCores = Math.max(maxCores, model.coreCount()); - maxMemory = Math.max(maxMemory, model.memoryCapacity()); - hostToView.put(host, hv); - - if (host.getState() == HostState.UP) { - availableHosts.add(hv); - } - - scheduler.addHost(hv); - host.addListener(hostListener); - } - - /** - * Remove a {@link Host} from the scheduling pool of the compute service. - */ - public void removeHost(Host host) { - HostView view = hostToView.remove(host); - if (view != null) { - availableHosts.remove(view); - scheduler.removeHost(view); - host.removeListener(hostListener); - } - } - - /** - * Lookup the {@link Host} that currently hosts the specified {@link Task}. - */ - public Host lookupHost(Task task) { - if (task instanceof ServiceTask) { - return ((ServiceTask) task).getHost(); - } - - ServiceTask internal = Objects.requireNonNull(taskById.get(task.getUid()), "Invalid task passed to lookupHost"); - return internal.getHost(); - } - - /** - * Return the {@link Host}s that are registered with this service. - */ - public Set getHosts() { - return Collections.unmodifiableSet(hostToView.keySet()); - } - - /** - * Collect the statistics about the scheduler component of this service. - */ - public SchedulerStats getSchedulerStats() { - return new SchedulerStats( - availableHosts.size(), - hostToView.size() - availableHosts.size(), - attemptsSuccess, - attemptsFailure, - attemptsError, - tasks.size(), - tasksPending, - tasksActive); - } - - @Override - public void close() { - if (isClosed) { - return; - } - - isClosed = true; - pacer.cancel(); - } - - /** - * Enqueue the specified [task] to be scheduled onto a host. - */ - SchedulingRequest schedule(ServiceTask task) { - LOGGER.debug("Enqueueing task {} to be assigned to host", task.getUid()); - - long now = clock.millis(); - SchedulingRequest request = new SchedulingRequest(task, now); - - task.launchedAt = Instant.ofEpochMilli(now); - taskQueue.add(request); - tasksPending++; - requestSchedulingCycle(); - return request; - } - - void delete(ServiceFlavor flavor) { - flavorById.remove(flavor.getUid()); - flavors.remove(flavor); - } - - void delete(ServiceImage image) { - imageById.remove(image.getUid()); - images.remove(image); - } - - void delete(ServiceTask task) { - taskById.remove(task.getUid()); - tasks.remove(task); - } - - /** - * Indicate that a new scheduling cycle is needed due to a change to the service's state. - */ - private void requestSchedulingCycle() { - // Bail out in case the queue is empty. - if (taskQueue.isEmpty()) { - return; - } - - pacer.enqueue(); - } - - /** - * Run a single scheduling iteration. - */ - private void doSchedule() { - // reorder tasks - - while (!taskQueue.isEmpty()) { - SchedulingRequest request = taskQueue.peek(); - - if (request.isCancelled) { - taskQueue.poll(); - tasksPending--; - continue; - } - - final ServiceTask task = request.task; - - // Remove task from scheduling if it has failed too many times - if (task.getNumFailures() > maxNumFailures) { - LOGGER.warn("Failed to spawn {}: Task has failed more than the allowed {} times", task, maxNumFailures); - - taskQueue.poll(); - tasksPending--; - task.setState(TaskState.TERMINATED); - continue; - } - - final ServiceFlavor flavor = task.getFlavor(); - final HostView hv = scheduler.select(request.task); - - if (hv == null || !hv.getHost().canFit(task)) { - LOGGER.trace("Task {} selected for scheduling but no capacity available for it at the moment", task); - - if (flavor.getMemorySize() > maxMemory || flavor.getCoreCount() > maxCores) { - // Remove the incoming image - taskQueue.poll(); - tasksPending--; - attemptsFailure++; - - LOGGER.warn("Failed to spawn {}: does not fit", task); - - task.setState(TaskState.TERMINATED); - continue; - } else { - break; - } - } - - Host host = hv.getHost(); - - // Remove request from queue - taskQueue.poll(); - tasksPending--; - - LOGGER.info("Assigned task {} to host {}", task, host); - - try { - task.host = host; - - host.spawn(task); - host.start(task); - - tasksActive++; - attemptsSuccess++; - - hv.instanceCount++; - hv.provisionedCores += flavor.getCoreCount(); - hv.availableMemory -= flavor.getMemorySize(); - - activeTasks.put(task, host); - } catch (Exception cause) { - LOGGER.error("Failed to deploy VM", cause); - attemptsError++; - } - } - } - - /** - * Builder class for a {@link ComputeService}. - */ - public static class Builder { - private final Dispatcher dispatcher; - private final ComputeScheduler computeScheduler; - private Duration quantum = Duration.ofMinutes(5); - private int maxNumFailures = 10; - - Builder(Dispatcher dispatcher, ComputeScheduler computeScheduler) { - this.dispatcher = dispatcher; - this.computeScheduler = computeScheduler; - } - - /** - * Set the scheduling quantum of the service. - */ - public Builder withQuantum(Duration quantum) { - this.quantum = quantum; - return this; - } - - public Builder withMaxNumFailures(int maxNumFailures) { - this.maxNumFailures = maxNumFailures; - return this; - } - - /** - * Build a {@link ComputeService}. - */ - public ComputeService build() { - return new ComputeService(dispatcher, computeScheduler, quantum, maxNumFailures); - } - } - - /** - * Implementation of {@link ComputeClient} using a {@link ComputeService}. - */ - private static class Client implements ComputeClient { - private final ComputeService service; - private boolean isClosed; - - Client(ComputeService service) { - this.service = service; - } - - /** - * Method to check if the client is still open and throw an exception if it is not. - */ - private void checkOpen() { - if (isClosed) { - throw new IllegalStateException("Client is already closed"); - } - } - - @NotNull - @Override - public List queryFlavors() { - checkOpen(); - return new ArrayList<>(service.flavors); - } - - @Override - public Flavor findFlavor(@NotNull UUID id) { - checkOpen(); - - return service.flavorById.get(id); - } - - @NotNull - @Override - public Flavor newFlavor( - @NotNull String name, - int cpuCount, - long memorySize, - @NotNull Map labels, - @NotNull Map meta) { - checkOpen(); - - final ComputeService service = this.service; - UUID uid = new UUID(service.clock.millis(), service.random.nextLong()); - ServiceFlavor flavor = new ServiceFlavor(service, uid, name, cpuCount, memorySize, labels, meta); - - service.flavorById.put(uid, flavor); - service.flavors.add(flavor); - - return flavor; - } - - @NotNull - @Override - public List queryImages() { - checkOpen(); - - return new ArrayList<>(service.images); - } - - @Override - public Image findImage(@NotNull UUID id) { - checkOpen(); - - return service.imageById.get(id); - } - - @NotNull - public Image newImage(@NotNull String name, @NotNull Map labels, @NotNull Map meta) { - checkOpen(); - - final ComputeService service = this.service; - UUID uid = new UUID(service.clock.millis(), service.random.nextLong()); - - ServiceImage image = new ServiceImage(service, uid, name, labels, meta); - - service.imageById.put(uid, image); - service.images.add(image); - - return image; - } - - @NotNull - @Override - public Task newTask( - @NotNull String name, - @NotNull Image image, - @NotNull Flavor flavor, - @NotNull Map labels, - @NotNull Map meta, - boolean start) { - checkOpen(); - - final ComputeService service = this.service; - UUID uid = new UUID(service.clock.millis(), service.random.nextLong()); - - final ServiceFlavor internalFlavor = - Objects.requireNonNull(service.flavorById.get(flavor.getUid()), "Unknown flavor"); - final ServiceImage internalImage = - Objects.requireNonNull(service.imageById.get(image.getUid()), "Unknown image"); - - ServiceTask task = new ServiceTask(service, uid, name, internalFlavor, internalImage, labels, meta); - - service.taskById.put(uid, task); - service.tasks.add(task); - - if (start) { - task.start(); - } - - return task; - } - - @Nullable - @Override - public Task findTask(@NotNull UUID id) { - checkOpen(); - return service.taskById.get(id); - } - - @NotNull - @Override - public List queryTasks() { - checkOpen(); - - return new ArrayList<>(service.tasks); - } - - @Override - public void close() { - isClosed = true; - } - - @Override - public String toString() { - return "ComputeService.Client"; - } - - @Nullable - @Override - public void rescheduleTask(@NotNull Task task, @NotNull SimWorkload workload) { - ServiceTask internalTask = (ServiceTask) findTask(task.getUid()); - Host from = service.lookupHost(internalTask); - - from.delete(internalTask); - - internalTask.host = null; - - internalTask.setWorkload(workload); - internalTask.start(); - } - } - - /** - * A request to schedule a {@link ServiceTask} onto one of the {@link Host}s. - */ - static class SchedulingRequest { - final ServiceTask task; - final long submitTime; - - boolean isCancelled; - - SchedulingRequest(ServiceTask task, long submitTime) { - this.task = task; - this.submitTime = submitTime; - } - } -} diff --git a/opendc-compute/opendc-compute-service/src/main/java/org/opendc/compute/service/HostView.java b/opendc-compute/opendc-compute-service/src/main/java/org/opendc/compute/service/HostView.java deleted file mode 100644 index 6e2cdcb4..00000000 --- a/opendc-compute/opendc-compute-service/src/main/java/org/opendc/compute/service/HostView.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Copyright (c) 2022 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.compute.service; - -import org.opendc.compute.service.driver.Host; - -/** - * A view of a {@link Host} as seen from the {@link ComputeService}. - */ -public class HostView { - private final Host host; - int instanceCount; - long availableMemory; - int provisionedCores; - - /** - * Construct a {@link HostView} instance. - * - * @param host The host to create a view of. - */ - public HostView(Host host) { - this.host = host; - this.availableMemory = host.getModel().memoryCapacity(); - } - - /** - * The {@link Host} this is a view of. - */ - public Host getHost() { - return host; - } - - /** - * Return the number of instances on this host. - */ - public int getInstanceCount() { - return instanceCount; - } - - /** - * Return the available memory of the host. - */ - public long getAvailableMemory() { - return availableMemory; - } - - /** - * Return the provisioned cores on the host. - */ - public int getProvisionedCores() { - return provisionedCores; - } - - @Override - public String toString() { - return "HostView[host=" + host + "]"; - } -} diff --git a/opendc-compute/opendc-compute-service/src/main/java/org/opendc/compute/service/ServiceFlavor.java b/opendc-compute/opendc-compute-service/src/main/java/org/opendc/compute/service/ServiceFlavor.java deleted file mode 100644 index 0f434a6a..00000000 --- a/opendc-compute/opendc-compute-service/src/main/java/org/opendc/compute/service/ServiceFlavor.java +++ /dev/null @@ -1,122 +0,0 @@ -/* - * Copyright (c) 2022 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.compute.service; - -import java.util.Collections; -import java.util.Map; -import java.util.Objects; -import java.util.UUID; -import org.jetbrains.annotations.NotNull; -import org.opendc.compute.api.Flavor; - -/** - * Implementation of {@link Flavor} provided by {@link ComputeService}. - */ -public final class ServiceFlavor implements Flavor { - private final ComputeService service; - private final UUID uid; - private final String name; - private final int coreCount; - private final long memorySize; - private final Map labels; - private final Map meta; - - ServiceFlavor( - ComputeService service, - UUID uid, - String name, - int coreCount, - long memorySize, - Map labels, - Map meta) { - this.service = service; - this.uid = uid; - this.name = name; - this.coreCount = coreCount; - this.memorySize = memorySize; - this.labels = labels; - this.meta = meta; - } - - @Override - public int getCoreCount() { - return coreCount; - } - - @Override - public long getMemorySize() { - return memorySize; - } - - @NotNull - @Override - public UUID getUid() { - return uid; - } - - @NotNull - @Override - public String getName() { - return name; - } - - @NotNull - @Override - public Map getLabels() { - return Collections.unmodifiableMap(labels); - } - - @NotNull - @Override - public Map getMeta() { - return Collections.unmodifiableMap(meta); - } - - @Override - public void reload() { - // No-op: this object is the source-of-truth - } - - @Override - public void delete() { - service.delete(this); - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - ServiceFlavor flavor = (ServiceFlavor) o; - return service.equals(flavor.service) && uid.equals(flavor.uid); - } - - @Override - public int hashCode() { - return Objects.hash(service, uid); - } - - @Override - public String toString() { - return "Flavor[uid=" + uid + ",name=" + name + "]"; - } -} diff --git a/opendc-compute/opendc-compute-service/src/main/java/org/opendc/compute/service/ServiceImage.java b/opendc-compute/opendc-compute-service/src/main/java/org/opendc/compute/service/ServiceImage.java deleted file mode 100644 index 706be483..00000000 --- a/opendc-compute/opendc-compute-service/src/main/java/org/opendc/compute/service/ServiceImage.java +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Copyright (c) 2022 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.compute.service; - -import java.util.Collections; -import java.util.Map; -import java.util.Objects; -import java.util.UUID; -import org.jetbrains.annotations.NotNull; -import org.opendc.compute.api.Image; - -/** - * Implementation of {@link Image} provided by {@link ComputeService}. - */ -public final class ServiceImage implements Image { - private final ComputeService service; - private final UUID uid; - private final String name; - private final Map labels; - private final Map meta; - - ServiceImage(ComputeService service, UUID uid, String name, Map labels, Map meta) { - this.service = service; - this.uid = uid; - this.name = name; - this.labels = labels; - this.meta = meta; - } - - @NotNull - @Override - public UUID getUid() { - return uid; - } - - @NotNull - @Override - public String getName() { - return name; - } - - @NotNull - @Override - public Map getLabels() { - return Collections.unmodifiableMap(labels); - } - - @NotNull - @Override - public Map getMeta() { - return Collections.unmodifiableMap(meta); - } - - @Override - public void reload() { - // No-op: this object is the source-of-truth - } - - @Override - public void delete() { - service.delete(this); - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - ServiceImage image = (ServiceImage) o; - return service.equals(image.service) && uid.equals(image.uid); - } - - @Override - public int hashCode() { - return Objects.hash(service, uid); - } - - @Override - public String toString() { - return "Image[uid=" + uid + ",name=" + name + "]"; - } -} diff --git a/opendc-compute/opendc-compute-service/src/main/java/org/opendc/compute/service/ServiceTask.java b/opendc-compute/opendc-compute-service/src/main/java/org/opendc/compute/service/ServiceTask.java deleted file mode 100644 index f0e2a82e..00000000 --- a/opendc-compute/opendc-compute-service/src/main/java/org/opendc/compute/service/ServiceTask.java +++ /dev/null @@ -1,267 +0,0 @@ -/* - * Copyright (c) 2022 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.compute.service; - -import java.time.Instant; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.UUID; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.opendc.compute.api.Task; -import org.opendc.compute.api.TaskState; -import org.opendc.compute.api.TaskWatcher; -import org.opendc.compute.service.driver.Host; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Implementation of {@link Task} provided by {@link ComputeService}. - */ -public final class ServiceTask implements Task { - private static final Logger LOGGER = LoggerFactory.getLogger(ServiceTask.class); - - private final ComputeService service; - private final UUID uid; - - private final String name; - private final ServiceFlavor flavor; - private final ServiceImage image; - private final Map labels; - private Map meta; - - private final List watchers = new ArrayList<>(); - private TaskState state = TaskState.TERMINATED; - Instant launchedAt = null; - Host host = null; - private ComputeService.SchedulingRequest request = null; - - private int numFailures = 0; - - ServiceTask( - ComputeService service, - UUID uid, - String name, - ServiceFlavor flavor, - ServiceImage image, - Map labels, - Map meta) { - this.service = service; - this.uid = uid; - this.name = name; - this.flavor = flavor; - this.image = image; - this.labels = labels; - this.meta = meta; - } - - @NotNull - @Override - public UUID getUid() { - return uid; - } - - @NotNull - @Override - public String getName() { - return name; - } - - @NotNull - @Override - public ServiceFlavor getFlavor() { - return flavor; - } - - @NotNull - @Override - public ServiceImage getImage() { - return image; - } - - @NotNull - @Override - public Map getLabels() { - return Collections.unmodifiableMap(labels); - } - - @NotNull - @Override - public Map getMeta() { - return Collections.unmodifiableMap(meta); - } - - public void setWorkload(Object _workload) { - Map new_meta = new HashMap(); - new_meta.put("workload", _workload); - - meta = new_meta; - } - - @NotNull - @Override - public TaskState getState() { - return state; - } - - @Nullable - @Override - public Instant getLaunchedAt() { - return launchedAt; - } - - /** - * Return the {@link Host} on which the task is running or null if it is not running on a host. - */ - public Host getHost() { - return host; - } - - @Override - public void start() { - switch (state) { - case PROVISIONING: - LOGGER.debug("User tried to start task but request is already pending: doing nothing"); - case RUNNING: - LOGGER.debug("User tried to start task but task is already running"); - break; - case DELETED: - LOGGER.warn("User tried to start deleted task"); - throw new IllegalStateException("Task is deleted"); - default: - LOGGER.info("User requested to start task {}", uid); - setState(TaskState.PROVISIONING); - assert request == null : "Scheduling request already active"; - request = service.schedule(this); - break; - } - } - - @Override - public void stop() { - switch (state) { - case PROVISIONING: - cancelProvisioningRequest(); - setState(TaskState.TERMINATED); - break; - case RUNNING: - case ERROR: - final Host host = this.host; - if (host == null) { - throw new IllegalStateException("Task not running"); - } - host.stop(this); - break; - } - } - - @Override - public void watch(@NotNull TaskWatcher watcher) { - watchers.add(watcher); - } - - @Override - public void unwatch(@NotNull TaskWatcher watcher) { - watchers.remove(watcher); - } - - @Override - public void reload() { - // No-op: this object is the source-of-truth - } - - @Override - public void delete() { - switch (state) { - case PROVISIONING: - case TERMINATED: - cancelProvisioningRequest(); - service.delete(this); - setState(TaskState.DELETED); - break; - case RUNNING: - case ERROR: - final Host host = this.host; - if (host == null) { - throw new IllegalStateException("Task not running"); - } - host.delete(this); - service.delete(this); - setState(TaskState.DELETED); - break; - } - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - ServiceTask task = (ServiceTask) o; - return service.equals(task.service) && uid.equals(task.uid); - } - - @Override - public int hashCode() { - return Objects.hash(service, uid); - } - - @Override - public String toString() { - return "Task[uid=" + uid + ",name=" + name + ",state=" + state + "]"; - } - - void setState(TaskState newState) { - if (this.state == newState) { - return; - } - - for (TaskWatcher watcher : watchers) { - watcher.onStateChanged(this, newState); - } - if (newState == TaskState.ERROR) { - this.numFailures++; - } - - this.state = newState; - } - - /** - * Cancel the provisioning request if active. - */ - private void cancelProvisioningRequest() { - final ComputeService.SchedulingRequest request = this.request; - if (request != null) { - this.request = null; - request.isCancelled = true; - } - } - - @Override - public int getNumFailures() { - return this.numFailures; - } -} diff --git a/opendc-compute/opendc-compute-service/src/main/java/org/opendc/compute/service/driver/Host.java b/opendc-compute/opendc-compute-service/src/main/java/org/opendc/compute/service/driver/Host.java deleted file mode 100644 index 546f774b..00000000 --- a/opendc-compute/opendc-compute-service/src/main/java/org/opendc/compute/service/driver/Host.java +++ /dev/null @@ -1,137 +0,0 @@ -/* - * Copyright (c) 2022 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.compute.service.driver; - -import java.util.Map; -import java.util.Set; -import java.util.UUID; -import org.opendc.compute.api.Task; -import org.opendc.compute.service.driver.telemetry.GuestCpuStats; -import org.opendc.compute.service.driver.telemetry.GuestSystemStats; -import org.opendc.compute.service.driver.telemetry.HostCpuStats; -import org.opendc.compute.service.driver.telemetry.HostSystemStats; - -/** - * Base interface for representing compute resources that host virtualized {@link Task} instances. - */ -public interface Host { - /** - * Return a unique identifier representing the host. - */ - UUID getUid(); - - /** - * Return the name of this host. - */ - String getName(); - - /** - * Return the machine model of the host. - */ - HostModel getModel(); - - /** - * Return the state of the host. - */ - HostState getState(); - - /** - * Return the meta-data associated with the host. - */ - Map getMeta(); - - /** - * Return the {@link Task} instances known to the host. - */ - Set getInstances(); - - /** - * Determine whether the specified task can still fit on this host. - */ - boolean canFit(Task task); - - /** - * Register the specified task on the host. - */ - void spawn(Task task); - - /** - * Determine whether the specified task exists on the host. - */ - boolean contains(Task task); - - /** - * Start the task if it is currently not running on this host. - * - * @throws IllegalArgumentException if the task is not present on the host. - */ - void start(Task task); - - /** - * Stop the task if it is currently running on this host. - * - * @throws IllegalArgumentException if the task is not present on the host. - */ - void stop(Task task); - - /** - * Delete the specified task on this host and cleanup all resources associated with it. - */ - void delete(Task task); - - /** - * Add a [HostListener] to this host. - */ - void addListener(HostListener listener); - - /** - * Remove a [HostListener] from this host. - */ - void removeListener(HostListener listener); - - /** - * Query the system statistics of the host. - */ - HostSystemStats getSystemStats(); - - /** - * Query the system statistics of a {@link Task} that is located on this host. - * - * @param task The {@link Task} to obtain the system statistics of. - * @throws IllegalArgumentException if the task is not present on the host. - */ - GuestSystemStats getSystemStats(Task task); - - /** - * Query the CPU statistics of the host. - */ - HostCpuStats getCpuStats(); - - /** - * Query the CPU statistics of a {@link Task} that is located on this host. - * - * @param task The {@link Task} to obtain the CPU statistics of. - * @throws IllegalArgumentException if the task is not present on the host. - */ - GuestCpuStats getCpuStats(Task task); -} diff --git a/opendc-compute/opendc-compute-service/src/main/java/org/opendc/compute/service/driver/HostListener.java b/opendc-compute/opendc-compute-service/src/main/java/org/opendc/compute/service/driver/HostListener.java deleted file mode 100644 index 079c6cff..00000000 --- a/opendc-compute/opendc-compute-service/src/main/java/org/opendc/compute/service/driver/HostListener.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright (c) 2022 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.compute.service.driver; - -import org.opendc.compute.api.Task; -import org.opendc.compute.api.TaskState; - -/** - * Listener interface for events originating from a {@link Host}. - */ -public interface HostListener { - /** - * This method is invoked when the state of task on host changes. - */ - default void onStateChanged(Host host, Task task, TaskState newState) {} - - /** - * This method is invoked when the state of a {@link Host} has changed. - */ - default void onStateChanged(Host host, HostState newState) {} -} diff --git a/opendc-compute/opendc-compute-service/src/main/java/org/opendc/compute/service/driver/HostModel.java b/opendc-compute/opendc-compute-service/src/main/java/org/opendc/compute/service/driver/HostModel.java deleted file mode 100644 index 87464fe1..00000000 --- a/opendc-compute/opendc-compute-service/src/main/java/org/opendc/compute/service/driver/HostModel.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright (c) 2022 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.compute.service.driver; - -/** - * Record describing the static machine properties of the host. - * - * @param cpuCapacity The total CPU capacity of the host in MHz. - * @param coreCount The number of logical processing cores available for this host. - * @param memoryCapacity The amount of memory available for this host in MB. - */ -public record HostModel(double cpuCapacity, int coreCount, long memoryCapacity) {} diff --git a/opendc-compute/opendc-compute-service/src/main/java/org/opendc/compute/service/driver/HostState.java b/opendc-compute/opendc-compute-service/src/main/java/org/opendc/compute/service/driver/HostState.java deleted file mode 100644 index ce12a67e..00000000 --- a/opendc-compute/opendc-compute-service/src/main/java/org/opendc/compute/service/driver/HostState.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright (c) 2022 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.compute.service.driver; - -/** - * The state of a host. - */ -public enum HostState { - /** - * The host is up and able to host guests. - */ - UP, - - /** - * The host is in a (forced) down state and unable to host any guests. - */ - DOWN, - - /** - * The host is in an error state and unable to host any guests. - */ - ERROR -} diff --git a/opendc-compute/opendc-compute-service/src/main/java/org/opendc/compute/service/driver/telemetry/GuestCpuStats.java b/opendc-compute/opendc-compute-service/src/main/java/org/opendc/compute/service/driver/telemetry/GuestCpuStats.java deleted file mode 100644 index 0b78c7ea..00000000 --- a/opendc-compute/opendc-compute-service/src/main/java/org/opendc/compute/service/driver/telemetry/GuestCpuStats.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright (c) 2022 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.compute.service.driver.telemetry; - -/** - * Statistics about the CPUs of a guest. - * - * @param activeTime The cumulative time (in seconds) that the CPUs of the guest were actively running. - * @param idleTime The cumulative time (in seconds) the CPUs of the guest were idle. - * @param stealTime The cumulative CPU time (in seconds) that the guest was ready to run, but not granted time by the host. - * @param lostTime The cumulative CPU time (in seconds) that was lost due to interference with other machines. - * @param capacity The available CPU capacity of the guest (in MHz). - * @param usage Amount of CPU resources (in MHz) actually used by the guest. - * @param utilization The utilization of the CPU resources (in %) relative to the total CPU capacity. - */ -public record GuestCpuStats( - long activeTime, - long idleTime, - long stealTime, - long lostTime, - double capacity, - double usage, - double utilization) {} diff --git a/opendc-compute/opendc-compute-service/src/main/java/org/opendc/compute/service/driver/telemetry/GuestSystemStats.java b/opendc-compute/opendc-compute-service/src/main/java/org/opendc/compute/service/driver/telemetry/GuestSystemStats.java deleted file mode 100644 index dbf98dd5..00000000 --- a/opendc-compute/opendc-compute-service/src/main/java/org/opendc/compute/service/driver/telemetry/GuestSystemStats.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright (c) 2022 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.compute.service.driver.telemetry; - -import java.time.Duration; -import java.time.Instant; - -/** - * System-level statistics of a guest. - * - * @param uptime The cumulative uptime of the guest since last boot (in ms). - * @param downtime The cumulative downtime of the guest since last boot (in ms). - * @param bootTime The time at which the guest booted. - */ -public record GuestSystemStats(Duration uptime, Duration downtime, Instant bootTime) {} diff --git a/opendc-compute/opendc-compute-service/src/main/java/org/opendc/compute/service/driver/telemetry/HostCpuStats.java b/opendc-compute/opendc-compute-service/src/main/java/org/opendc/compute/service/driver/telemetry/HostCpuStats.java deleted file mode 100644 index d1c2328b..00000000 --- a/opendc-compute/opendc-compute-service/src/main/java/org/opendc/compute/service/driver/telemetry/HostCpuStats.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright (c) 2022 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.compute.service.driver.telemetry; - -/** - * Statistics about the CPUs of a host. - * - * @param activeTime The cumulative time (in seconds) that the CPUs of the host were actively running. - * @param idleTime The cumulative time (in seconds) the CPUs of the host were idle. - * @param stealTime The cumulative CPU time (in seconds) that virtual machines were ready to run, but were not able to. - * @param lostTime The cumulative CPU time (in seconds) that was lost due to interference between virtual machines. - * @param capacity The available CPU capacity of the host (in MHz). - * @param demand Amount of CPU resources (in MHz) the guests would use if there were no CPU contention or CPU - * limits. - * @param usage Amount of CPU resources (in MHz) actually used by the host. - * @param utilization The utilization of the CPU resources (in %) relative to the total CPU capacity. - */ -public record HostCpuStats( - long activeTime, - long idleTime, - long stealTime, - long lostTime, - double capacity, - double demand, - double usage, - double utilization) {} diff --git a/opendc-compute/opendc-compute-service/src/main/java/org/opendc/compute/service/driver/telemetry/HostSystemStats.java b/opendc-compute/opendc-compute-service/src/main/java/org/opendc/compute/service/driver/telemetry/HostSystemStats.java deleted file mode 100644 index c0713f3c..00000000 --- a/opendc-compute/opendc-compute-service/src/main/java/org/opendc/compute/service/driver/telemetry/HostSystemStats.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright (c) 2022 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.compute.service.driver.telemetry; - -import java.time.Duration; -import java.time.Instant; - -/** - * System-level statistics of a host. - * - * @param uptime The cumulative uptime of the host since last boot (in ms). - * @param downtime The cumulative downtime of the host since last boot (in ms). - * @param bootTime The time at which the task started. - * @param powerDraw Instantaneous power draw of the system (in W). - * @param energyUsage The cumulative energy usage of the system (in J). - * @param guestsTerminated The number of guests that are in a terminated state. - * @param guestsRunning The number of guests that are in a running state. - * @param guestsError The number of guests that are in an error state. - * @param guestsInvalid The number of guests that are in an unknown state. - */ -public record HostSystemStats( - Duration uptime, - Duration downtime, - Instant bootTime, - double powerDraw, - double energyUsage, - int guestsTerminated, - int guestsRunning, - int guestsError, - int guestsInvalid) {} diff --git a/opendc-compute/opendc-compute-service/src/main/java/org/opendc/compute/service/telemetry/SchedulerStats.java b/opendc-compute/opendc-compute-service/src/main/java/org/opendc/compute/service/telemetry/SchedulerStats.java deleted file mode 100644 index fc044d8c..00000000 --- a/opendc-compute/opendc-compute-service/src/main/java/org/opendc/compute/service/telemetry/SchedulerStats.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright (c) 2022 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.compute.service.telemetry; - -/** - * Statistics about the scheduling component of the [ComputeService]. - * - * @param hostsAvailable The number of hosts currently available for scheduling. - * @param hostsUnavailable The number of hosts unavailable for scheduling. - * @param attemptsSuccess Scheduling attempts that resulted into an allocation onto a host. - * @param attemptsFailure The number of failed scheduling attempt due to insufficient capacity at the moment. - * @param attemptsError The number of scheduling attempts that failed due to system error. - * @param tasksTotal The number of tasks registered with the service. - * @param tasksPending The number of tasks that are pending to be scheduled. - * @param tasksActive The number of tasks that are currently managed by the service and running. - */ -public record SchedulerStats( - int hostsAvailable, - int hostsUnavailable, - long attemptsSuccess, - long attemptsFailure, - long attemptsError, - int tasksTotal, - int tasksPending, - int tasksActive) {} diff --git a/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/scheduler/ComputeScheduler.kt b/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/scheduler/ComputeScheduler.kt deleted file mode 100644 index 42de9ebc..00000000 --- a/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/scheduler/ComputeScheduler.kt +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright (c) 2021 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.compute.service.scheduler - -import org.opendc.compute.api.Task -import org.opendc.compute.service.ComputeService -import org.opendc.compute.service.HostView - -/** - * A generic scheduler interface used by the [ComputeService] to select hosts to place [Task]s on. - */ -public interface ComputeScheduler { - /** - * Register the specified [host] to be used for scheduling. - */ - public fun addHost(host: HostView) - - /** - * Remove the specified [host] to be removed from the scheduling pool. - */ - public fun removeHost(host: HostView) - - /** - * Select a host for the specified [task]. - * - * @param task The server to select a host for. - * @return The host to schedule the server on or `null` if no server is available. - */ - public fun select(task: Task): HostView? -} diff --git a/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/scheduler/ComputeSchedulers.kt b/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/scheduler/ComputeSchedulers.kt deleted file mode 100644 index 7fcc670f..00000000 --- a/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/scheduler/ComputeSchedulers.kt +++ /dev/null @@ -1,118 +0,0 @@ -/* - * Copyright (c) 2021 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -@file:JvmName("ComputeSchedulers") - -package org.opendc.compute.service.scheduler - -import org.opendc.compute.service.scheduler.filters.ComputeFilter -import org.opendc.compute.service.scheduler.filters.RamFilter -import org.opendc.compute.service.scheduler.filters.VCpuFilter -import org.opendc.compute.service.scheduler.weights.CoreRamWeigher -import org.opendc.compute.service.scheduler.weights.InstanceCountWeigher -import org.opendc.compute.service.scheduler.weights.RamWeigher -import org.opendc.compute.service.scheduler.weights.VCpuWeigher -import java.util.SplittableRandom -import java.util.random.RandomGenerator - -public enum class ComputeSchedulerEnum { - Mem, - MemInv, - CoreMem, - CoreMemInv, - ActiveServers, - ActiveServersInv, - ProvisionedCores, - ProvisionedCoresInv, - Random, - Replay, -} - -public fun createComputeScheduler( - name: String, - seeder: RandomGenerator, - placements: Map = emptyMap(), -): ComputeScheduler { - return createComputeScheduler(ComputeSchedulerEnum.valueOf(name.uppercase()), seeder, placements) -} - -/** - * Create a [ComputeScheduler] for the experiment. - */ -public fun createComputeScheduler( - name: ComputeSchedulerEnum, - seeder: RandomGenerator, - placements: Map = emptyMap(), -): ComputeScheduler { - val cpuAllocationRatio = 1.0 - val ramAllocationRatio = 1.5 - return when (name) { - ComputeSchedulerEnum.Mem -> - FilterScheduler( - filters = listOf(ComputeFilter(), VCpuFilter(cpuAllocationRatio), RamFilter(ramAllocationRatio)), - weighers = listOf(RamWeigher(multiplier = 1.0)), - ) - ComputeSchedulerEnum.MemInv -> - FilterScheduler( - filters = listOf(ComputeFilter(), VCpuFilter(cpuAllocationRatio), RamFilter(ramAllocationRatio)), - weighers = listOf(RamWeigher(multiplier = -1.0)), - ) - ComputeSchedulerEnum.CoreMem -> - FilterScheduler( - filters = listOf(ComputeFilter(), VCpuFilter(cpuAllocationRatio), RamFilter(ramAllocationRatio)), - weighers = listOf(CoreRamWeigher(multiplier = 1.0)), - ) - ComputeSchedulerEnum.CoreMemInv -> - FilterScheduler( - filters = listOf(ComputeFilter(), VCpuFilter(cpuAllocationRatio), RamFilter(ramAllocationRatio)), - weighers = listOf(CoreRamWeigher(multiplier = -1.0)), - ) - ComputeSchedulerEnum.ActiveServers -> - FilterScheduler( - filters = listOf(ComputeFilter(), VCpuFilter(cpuAllocationRatio), RamFilter(ramAllocationRatio)), - weighers = listOf(InstanceCountWeigher(multiplier = -1.0)), - ) - ComputeSchedulerEnum.ActiveServersInv -> - FilterScheduler( - filters = listOf(ComputeFilter(), VCpuFilter(cpuAllocationRatio), RamFilter(ramAllocationRatio)), - weighers = listOf(InstanceCountWeigher(multiplier = 1.0)), - ) - ComputeSchedulerEnum.ProvisionedCores -> - FilterScheduler( - filters = listOf(ComputeFilter(), VCpuFilter(cpuAllocationRatio), RamFilter(ramAllocationRatio)), - weighers = listOf(VCpuWeigher(cpuAllocationRatio, multiplier = 1.0)), - ) - ComputeSchedulerEnum.ProvisionedCoresInv -> - FilterScheduler( - filters = listOf(ComputeFilter(), VCpuFilter(cpuAllocationRatio), RamFilter(ramAllocationRatio)), - weighers = listOf(VCpuWeigher(cpuAllocationRatio, multiplier = -1.0)), - ) - ComputeSchedulerEnum.Random -> - FilterScheduler( - filters = listOf(ComputeFilter(), VCpuFilter(cpuAllocationRatio), RamFilter(ramAllocationRatio)), - weighers = emptyList(), - subsetSize = Int.MAX_VALUE, - random = SplittableRandom(seeder.nextLong()), - ) - ComputeSchedulerEnum.Replay -> ReplayScheduler(placements) - } -} diff --git a/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/scheduler/FilterScheduler.kt b/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/scheduler/FilterScheduler.kt deleted file mode 100644 index 772a470d..00000000 --- a/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/scheduler/FilterScheduler.kt +++ /dev/null @@ -1,111 +0,0 @@ -/* - * Copyright (c) 2021 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.compute.service.scheduler - -import org.opendc.compute.api.Task -import org.opendc.compute.service.HostView -import org.opendc.compute.service.scheduler.filters.HostFilter -import org.opendc.compute.service.scheduler.weights.HostWeigher -import java.util.SplittableRandom -import java.util.random.RandomGenerator -import kotlin.math.min - -/** - * A [ComputeScheduler] implementation that uses filtering and weighing passes to select - * the host to schedule a [Task] on. - * - * This implementation is based on the filter scheduler from OpenStack Nova. - * See: https://docs.openstack.org/nova/latest/user/filter-scheduler.html - * - * @param filters The list of filters to apply when searching for an appropriate host. - * @param weighers The list of weighers to apply when searching for an appropriate host. - * @param subsetSize The size of the subset of best hosts from which a target is randomly chosen. - * @param random A [RandomGenerator] instance for selecting - */ -public class FilterScheduler( - private val filters: List, - private val weighers: List, - private val subsetSize: Int = 1, - private val random: RandomGenerator = SplittableRandom(0), -) : ComputeScheduler { - /** - * The pool of hosts available to the scheduler. - */ - private val hosts = mutableListOf() - - init { - require(subsetSize >= 1) { "Subset size must be one or greater" } - } - - override fun addHost(host: HostView) { - hosts.add(host) - } - - override fun removeHost(host: HostView) { - hosts.remove(host) - } - - override fun select(task: Task): HostView? { - val hosts = hosts - val filteredHosts = hosts.filter { host -> filters.all { filter -> filter.test(host, task) } } - - val subset = - if (weighers.isNotEmpty()) { - val results = weighers.map { it.getWeights(filteredHosts, task) } - val weights = DoubleArray(filteredHosts.size) - - for (result in results) { - val min = result.min - val range = (result.max - min) - - // Skip result if all weights are the same - if (range == 0.0) { - continue - } - - val multiplier = result.multiplier - val factor = multiplier / range - - for ((i, weight) in result.weights.withIndex()) { - weights[i] += factor * (weight - min) - } - } - - weights.indices - .asSequence() - .sortedByDescending { weights[it] } - .map { filteredHosts[it] } - .take(subsetSize) - .toList() - } else { - filteredHosts - } - - // fixme: currently finding no matching hosts can result in an error - return when (val maxSize = min(subsetSize, subset.size)) { - 0 -> null - 1 -> subset[0] - else -> subset[random.nextInt(maxSize)] - } - } -} diff --git a/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/scheduler/ReplayScheduler.kt b/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/scheduler/ReplayScheduler.kt deleted file mode 100644 index d1690ddf..00000000 --- a/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/scheduler/ReplayScheduler.kt +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Copyright (c) 2020 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.compute.service.scheduler - -import mu.KotlinLogging -import org.opendc.compute.api.Task -import org.opendc.compute.service.HostView - -/** - * Policy replaying VM-cluster assignment. - * - * Within each cluster, the active servers on each node determine which node gets - * assigned the VM image. - */ -public class ReplayScheduler(private val vmPlacements: Map) : ComputeScheduler { - private val logger = KotlinLogging.logger {} - - /** - * The pool of hosts available to the scheduler. - */ - private val hosts = mutableListOf() - - override fun addHost(host: HostView) { - hosts.add(host) - } - - override fun removeHost(host: HostView) { - hosts.remove(host) - } - - override fun select(task: Task): HostView? { - val clusterName = - vmPlacements[task.name] - ?: throw IllegalStateException("Could not find placement data in VM placement file for VM ${task.name}") - val machinesInCluster = hosts.filter { it.host.name.contains(clusterName) } - - if (machinesInCluster.isEmpty()) { - logger.info { "Could not find any machines belonging to cluster $clusterName for image ${task.name}, assigning randomly." } - return hosts.maxByOrNull { it.availableMemory } - } - - return machinesInCluster.maxByOrNull { it.availableMemory } - ?: throw IllegalStateException("Cloud not find any machine and could not randomly assign") - } -} diff --git a/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/scheduler/filters/ComputeFilter.kt b/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/scheduler/filters/ComputeFilter.kt deleted file mode 100644 index 2ad626f3..00000000 --- a/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/scheduler/filters/ComputeFilter.kt +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright (c) 2021 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.compute.service.scheduler.filters - -import org.opendc.compute.api.Task -import org.opendc.compute.service.HostView -import org.opendc.compute.service.driver.HostState - -/** - * A [HostFilter] that filters on active hosts. - */ -public class ComputeFilter : HostFilter { - override fun test( - host: HostView, - task: Task, - ): Boolean { - val result = host.host.state == HostState.UP - return result - } - - override fun toString(): String = "ComputeFilter" -} diff --git a/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/scheduler/filters/DifferentHostFilter.kt b/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/scheduler/filters/DifferentHostFilter.kt deleted file mode 100644 index ffafeaa9..00000000 --- a/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/scheduler/filters/DifferentHostFilter.kt +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright (c) 2022 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.compute.service.scheduler.filters - -import org.opendc.compute.api.Task -import org.opendc.compute.service.HostView -import java.util.UUID - -/** - * A [HostFilter] that ensures an instance is scheduled on a different host from a set of instances. - */ -public class DifferentHostFilter : HostFilter { - override fun test( - host: HostView, - task: Task, - ): Boolean { - @Suppress("UNCHECKED_CAST") - val affinityUUIDs = task.meta["scheduler_hint:different_host"] as? Set ?: return true - return host.host.instances.none { it.uid in affinityUUIDs } - } -} diff --git a/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/scheduler/filters/HostFilter.kt b/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/scheduler/filters/HostFilter.kt deleted file mode 100644 index f506127a..00000000 --- a/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/scheduler/filters/HostFilter.kt +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright (c) 2021 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.compute.service.scheduler.filters - -import org.opendc.compute.api.Task -import org.opendc.compute.service.HostView -import org.opendc.compute.service.scheduler.FilterScheduler - -/** - * A filter used by the [FilterScheduler] to filter hosts. - */ -public fun interface HostFilter { - /** - * Test whether the specified [host] should be included in the selection - * for scheduling the specified [task]. - */ - public fun test( - host: HostView, - task: Task, - ): Boolean -} diff --git a/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/scheduler/filters/InstanceCountFilter.kt b/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/scheduler/filters/InstanceCountFilter.kt deleted file mode 100644 index 7d5eb400..00000000 --- a/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/scheduler/filters/InstanceCountFilter.kt +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright (c) 2021 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.compute.service.scheduler.filters - -import org.opendc.compute.api.Task -import org.opendc.compute.service.HostView - -/** - * A [HostFilter] that filters hosts based on the number of instances on the host. - * - * @param limit The maximum number of instances on the host. - */ -public class InstanceCountFilter(private val limit: Int) : HostFilter { - override fun test( - host: HostView, - task: Task, - ): Boolean { - return host.instanceCount < limit - } - - override fun toString(): String = "InstanceCountFilter[limit=$limit]" -} diff --git a/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/scheduler/filters/RamFilter.kt b/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/scheduler/filters/RamFilter.kt deleted file mode 100644 index 0a28ccc6..00000000 --- a/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/scheduler/filters/RamFilter.kt +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright (c) 2021 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.compute.service.scheduler.filters - -import org.opendc.compute.api.Task -import org.opendc.compute.service.HostView - -/** - * A [HostFilter] that filters hosts based on the memory requirements of a [Task] and the RAM available on the host. - * - * @param allocationRatio Virtual RAM to physical RAM allocation ratio. - */ -public class RamFilter(private val allocationRatio: Double) : HostFilter { - override fun test( - host: HostView, - task: Task, - ): Boolean { - val requestedMemory = task.flavor.memorySize - val availableMemory = host.availableMemory - val memoryCapacity = host.host.model.memoryCapacity - - // Do not allow an instance to overcommit against itself, only against - // other instances. - if (requestedMemory > memoryCapacity) { - return false - } - - val limit = memoryCapacity * allocationRatio - val used = memoryCapacity - availableMemory - val usable = limit - used - - val result = usable >= requestedMemory - return result - } -} diff --git a/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/scheduler/filters/SameHostFilter.kt b/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/scheduler/filters/SameHostFilter.kt deleted file mode 100644 index d8634285..00000000 --- a/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/scheduler/filters/SameHostFilter.kt +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright (c) 2022 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.compute.service.scheduler.filters - -import org.opendc.compute.api.Task -import org.opendc.compute.service.HostView -import java.util.UUID - -/** - * A [HostFilter] that ensures an instance is scheduled on the same host as all other instances in a set of instances. - */ -public class SameHostFilter : HostFilter { - override fun test( - host: HostView, - task: Task, - ): Boolean { - @Suppress("UNCHECKED_CAST") - val affinityUUIDs = task.meta["scheduler_hint:same_host"] as? Set ?: return true - return host.host.instances.any { it.uid in affinityUUIDs } - } -} diff --git a/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/scheduler/filters/VCpuCapacityFilter.kt b/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/scheduler/filters/VCpuCapacityFilter.kt deleted file mode 100644 index 5af7ccf0..00000000 --- a/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/scheduler/filters/VCpuCapacityFilter.kt +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright (c) 2021 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.compute.service.scheduler.filters - -import org.opendc.compute.api.Task -import org.opendc.compute.service.HostView - -/** - * A [HostFilter] that filters hosts based on the vCPU speed requirements of a [Task] and the available - * capacity on the host. - */ -public class VCpuCapacityFilter : HostFilter { - override fun test( - host: HostView, - task: Task, - ): Boolean { - val requiredCapacity = task.flavor.meta["cpu-capacity"] as? Double - val hostModel = host.host.model - val availableCapacity = hostModel.cpuCapacity - - return requiredCapacity == null || availableCapacity >= (requiredCapacity / task.flavor.coreCount) - } -} diff --git a/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/scheduler/filters/VCpuFilter.kt b/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/scheduler/filters/VCpuFilter.kt deleted file mode 100644 index 442e58f6..00000000 --- a/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/scheduler/filters/VCpuFilter.kt +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright (c) 2021 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.compute.service.scheduler.filters - -import org.opendc.compute.api.Task -import org.opendc.compute.service.HostView - -/** - * A [HostFilter] that filters hosts based on the vCPU requirements of a [Task] and the available vCPUs on the host. - * - * @param allocationRatio Virtual CPU to physical CPU allocation ratio. - */ -public class VCpuFilter(private val allocationRatio: Double) : HostFilter { - override fun test( - host: HostView, - task: Task, - ): Boolean { - val requested = task.flavor.coreCount - val totalCores = host.host.model.coreCount - val limit = totalCores * allocationRatio - - // Do not allow an instance to overcommit against itself, only against other instances - if (requested > totalCores) { - return false - } - - val availableCores = limit - host.provisionedCores - return availableCores >= requested - } -} diff --git a/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/scheduler/weights/CoreRamWeigher.kt b/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/scheduler/weights/CoreRamWeigher.kt deleted file mode 100644 index 6e320bf4..00000000 --- a/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/scheduler/weights/CoreRamWeigher.kt +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright (c) 2021 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.compute.service.scheduler.weights - -import org.opendc.compute.api.Task -import org.opendc.compute.service.HostView - -/** - * A [HostWeigher] that weighs the hosts based on the available memory per core on the host. - * - * @param multiplier Weight multiplier ratio. A positive value will result in the scheduler preferring hosts with more - * available core memory, and a negative number will result in the scheduler preferring hosts with less available core - * memory. - */ -public class CoreRamWeigher(override val multiplier: Double = 1.0) : HostWeigher { - override fun getWeight( - host: HostView, - task: Task, - ): Double { - return host.availableMemory.toDouble() - } - - override fun toString(): String = "CoreRamWeigher" -} diff --git a/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/scheduler/weights/HostWeigher.kt b/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/scheduler/weights/HostWeigher.kt deleted file mode 100644 index 3f2c4123..00000000 --- a/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/scheduler/weights/HostWeigher.kt +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Copyright (c) 2021 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.compute.service.scheduler.weights - -import org.opendc.compute.api.Task -import org.opendc.compute.service.HostView -import org.opendc.compute.service.scheduler.FilterScheduler - -/** - * An interface used by the [FilterScheduler] to weigh the pool of host for a scheduling request. - */ -public interface HostWeigher { - /** - * The multiplier for the weigher. - */ - public val multiplier: Double - - /** - * Obtain the weight of the specified [host] when scheduling the specified [task]. - */ - public fun getWeight( - host: HostView, - task: Task, - ): Double - - /** - * Obtain the weights for [hosts] when scheduling the specified [task]. - */ - public fun getWeights( - hosts: List, - task: Task, - ): Result { - val weights = DoubleArray(hosts.size) - var min = Double.MAX_VALUE - var max = Double.MIN_VALUE - - for ((i, host) in hosts.withIndex()) { - val weight = getWeight(host, task) - weights[i] = weight - min = kotlin.math.min(min, weight) - max = kotlin.math.max(max, weight) - } - - return Result(weights, min, max, multiplier) - } - - /** - * A result returned by the weigher. - * - * @param weights The weights returned by the weigher. - * @param min The minimum weight returned. - * @param max The maximum weight returned. - * @param multiplier The weight multiplier to use. - */ - public class Result( - public val weights: DoubleArray, - public val min: Double, - public val max: Double, - public val multiplier: Double, - ) -} diff --git a/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/scheduler/weights/InstanceCountWeigher.kt b/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/scheduler/weights/InstanceCountWeigher.kt deleted file mode 100644 index 0789f109..00000000 --- a/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/scheduler/weights/InstanceCountWeigher.kt +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright (c) 2021 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.compute.service.scheduler.weights - -import org.opendc.compute.api.Task -import org.opendc.compute.service.HostView - -/** - * A [HostWeigher] that weighs the hosts based on the number of instances on the host. - */ -public class InstanceCountWeigher(override val multiplier: Double = 1.0) : HostWeigher { - override fun getWeight( - host: HostView, - task: Task, - ): Double { - return host.instanceCount.toDouble() - } - - override fun toString(): String = "InstanceCountWeigher" -} diff --git a/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/scheduler/weights/RamWeigher.kt b/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/scheduler/weights/RamWeigher.kt deleted file mode 100644 index fb03d064..00000000 --- a/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/scheduler/weights/RamWeigher.kt +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright (c) 2021 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.compute.service.scheduler.weights - -import org.opendc.compute.api.Task -import org.opendc.compute.service.HostView - -/** - * A [HostWeigher] that weighs the hosts based on the available RAM (memory) on the host. - * - * @param multiplier Weight multiplier ratio. A positive value will result in the scheduler preferring hosts with more - * available memory, and a negative number will result in the scheduler preferring hosts with less memory. - */ -public class RamWeigher(override val multiplier: Double = 1.0) : HostWeigher { - override fun getWeight( - host: HostView, - task: Task, - ): Double { - return host.availableMemory.toDouble() - } - - override fun toString(): String = "RamWeigher" -} diff --git a/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/scheduler/weights/VCpuCapacityWeigher.kt b/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/scheduler/weights/VCpuCapacityWeigher.kt deleted file mode 100644 index 5f99cab3..00000000 --- a/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/scheduler/weights/VCpuCapacityWeigher.kt +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright (c) 2021 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.compute.service.scheduler.weights - -import org.opendc.compute.api.Task -import org.opendc.compute.service.HostView - -/** - * A [HostWeigher] that weighs the hosts based on the difference required vCPU capacity and the available CPU capacity. - */ -public class VCpuCapacityWeigher(override val multiplier: Double = 1.0) : HostWeigher { - override fun getWeight( - host: HostView, - task: Task, - ): Double { - val model = host.host.model - val requiredCapacity = task.flavor.meta["cpu-capacity"] as? Double ?: 0.0 - return model.cpuCapacity - requiredCapacity / task.flavor.coreCount - } - - override fun toString(): String = "VCpuWeigher" -} diff --git a/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/scheduler/weights/VCpuWeigher.kt b/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/scheduler/weights/VCpuWeigher.kt deleted file mode 100644 index 0c3d9c21..00000000 --- a/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/scheduler/weights/VCpuWeigher.kt +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright (c) 2021 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.compute.service.scheduler.weights - -import org.opendc.compute.api.Task -import org.opendc.compute.service.HostView - -/** - * A [HostWeigher] that weighs the hosts based on the remaining number of vCPUs available. - * - * @param allocationRatio Virtual CPU to physical CPU allocation ratio. - */ -public class VCpuWeigher(private val allocationRatio: Double, override val multiplier: Double = 1.0) : HostWeigher { - init { - require(allocationRatio > 0.0) { "Allocation ratio must be greater than zero" } - } - - override fun getWeight( - host: HostView, - task: Task, - ): Double { - return allocationRatio - host.provisionedCores - } - - override fun toString(): String = "VCpuWeigher" -} diff --git a/opendc-compute/opendc-compute-service/src/test/kotlin/org/opendc/compute/service/ComputeServiceTest.kt b/opendc-compute/opendc-compute-service/src/test/kotlin/org/opendc/compute/service/ComputeServiceTest.kt deleted file mode 100644 index eb686faf..00000000 --- a/opendc-compute/opendc-compute-service/src/test/kotlin/org/opendc/compute/service/ComputeServiceTest.kt +++ /dev/null @@ -1,386 +0,0 @@ -/* - * Copyright (c) 2021 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.compute.service - -import io.mockk.coEvery -import io.mockk.coVerify -import io.mockk.every -import io.mockk.mockk -import io.mockk.slot -import io.mockk.verify -import kotlinx.coroutines.delay -import org.junit.jupiter.api.Assertions.assertEquals -import org.junit.jupiter.api.Assertions.assertNull -import org.junit.jupiter.api.BeforeEach -import org.junit.jupiter.api.Test -import org.junit.jupiter.api.assertThrows -import org.opendc.compute.api.Flavor -import org.opendc.compute.api.Image -import org.opendc.compute.api.Task -import org.opendc.compute.api.TaskState -import org.opendc.compute.api.TaskWatcher -import org.opendc.compute.service.driver.Host -import org.opendc.compute.service.driver.HostListener -import org.opendc.compute.service.driver.HostModel -import org.opendc.compute.service.driver.HostState -import org.opendc.compute.service.scheduler.FilterScheduler -import org.opendc.compute.service.scheduler.filters.ComputeFilter -import org.opendc.compute.service.scheduler.filters.RamFilter -import org.opendc.compute.service.scheduler.filters.VCpuFilter -import org.opendc.compute.service.scheduler.weights.RamWeigher -import org.opendc.simulator.kotlin.SimulationCoroutineScope -import org.opendc.simulator.kotlin.runSimulation -import java.time.Duration -import java.util.UUID - -/** - * Test suite for the [ComputeService] interface. - */ -internal class ComputeServiceTest { - private lateinit var scope: SimulationCoroutineScope - private lateinit var service: ComputeService - - @BeforeEach - fun setUp() { - scope = SimulationCoroutineScope() - val computeScheduler = - FilterScheduler( - filters = listOf(ComputeFilter(), VCpuFilter(allocationRatio = 1.0), RamFilter(allocationRatio = 1.0)), - weighers = listOf(RamWeigher()), - ) - service = ComputeService(scope.dispatcher, computeScheduler, Duration.ofMinutes(5), 10) - } - - @Test - fun testClientClose() = - scope.runSimulation { - val client = service.newClient() - - assertEquals(emptyList(), client.queryFlavors()) - assertEquals(emptyList(), client.queryImages()) - assertEquals(emptyList(), client.queryTasks()) - - client.close() - - assertThrows { client.queryFlavors() } - assertThrows { client.queryImages() } - assertThrows { client.queryTasks() } - - assertThrows { client.findFlavor(UUID.randomUUID()) } - assertThrows { client.findImage(UUID.randomUUID()) } - assertThrows { client.findTask(UUID.randomUUID()) } - - assertThrows { client.newFlavor("test", 1, 2) } - assertThrows { client.newImage("test") } - assertThrows { client.newTask("test", mockk(), mockk()) } - } - - @Test - fun testClientCreate() = - scope.runSimulation { - val client = service.newClient() - - val flavor = client.newFlavor("test", 1, 1024) - assertEquals(listOf(flavor), client.queryFlavors()) - assertEquals(flavor, client.findFlavor(flavor.uid)) - val image = client.newImage("test") - assertEquals(listOf(image), client.queryImages()) - assertEquals(image, client.findImage(image.uid)) - val server = client.newTask("test", image, flavor, start = false) - assertEquals(listOf(server), client.queryTasks()) - assertEquals(server, client.findTask(server.uid)) - - server.delete() - assertNull(client.findTask(server.uid)) - - image.delete() - assertNull(client.findImage(image.uid)) - - flavor.delete() - assertNull(client.findFlavor(flavor.uid)) - - assertThrows { server.start() } - } - - @Test - fun testClientOnClose() = - scope.runSimulation { - service.close() - assertThrows { - service.newClient() - } - } - - @Test - fun testAddHost() = - scope.runSimulation { - val host = mockk(relaxUnitFun = true) - - every { host.model } returns HostModel(4 * 2600.0, 4, 2048) - every { host.state } returns HostState.UP - - assertEquals(emptySet(), service.hosts) - - service.addHost(host) - - verify(exactly = 1) { host.addListener(any()) } - - assertEquals(1, service.hosts.size) - - service.removeHost(host) - - verify(exactly = 1) { host.removeListener(any()) } - } - - @Test - fun testAddHostDouble() = - scope.runSimulation { - val host = mockk(relaxUnitFun = true) - - every { host.model } returns HostModel(4 * 2600.0, 4, 2048) - every { host.state } returns HostState.DOWN - - assertEquals(emptySet(), service.hosts) - - service.addHost(host) - service.addHost(host) - - verify(exactly = 1) { host.addListener(any()) } - } - - @Test - fun testServerStartWithoutEnoughCpus() = - scope.runSimulation { - val client = service.newClient() - val flavor = client.newFlavor("test", 1, 0) - val image = client.newImage("test") - val server = client.newTask("test", image, flavor, start = false) - - server.start() - delay(5L * 60 * 1000) - server.reload() - assertEquals(TaskState.TERMINATED, server.state) - } - - @Test - fun testServerStartWithoutEnoughMemory() = - scope.runSimulation { - val client = service.newClient() - val flavor = client.newFlavor("test", 0, 1024) - val image = client.newImage("test") - val server = client.newTask("test", image, flavor, start = false) - - server.start() - delay(5L * 60 * 1000) - server.reload() - assertEquals(TaskState.TERMINATED, server.state) - } - - @Test - fun testServerStartWithoutEnoughResources() = - scope.runSimulation { - val client = service.newClient() - val flavor = client.newFlavor("test", 1, 1024) - val image = client.newImage("test") - val server = client.newTask("test", image, flavor, start = false) - - server.start() - delay(5L * 60 * 1000) - server.reload() - assertEquals(TaskState.TERMINATED, server.state) - } - - @Test - fun testServerCancelRequest() = - scope.runSimulation { - val client = service.newClient() - val flavor = client.newFlavor("test", 1, 1024) - val image = client.newImage("test") - val server = client.newTask("test", image, flavor, start = false) - - server.start() - server.stop() - delay(5L * 60 * 1000) - server.reload() - assertEquals(TaskState.TERMINATED, server.state) - } - - @Test - fun testServerCannotFitOnHost() = - scope.runSimulation { - val host = mockk(relaxUnitFun = true) - - every { host.model } returns HostModel(4 * 2600.0, 4, 2048) - every { host.state } returns HostState.UP - every { host.canFit(any()) } returns false - - service.addHost(host) - - val client = service.newClient() - val flavor = client.newFlavor("test", 1, 1024) - val image = client.newImage("test") - val server = client.newTask("test", image, flavor, start = false) - - server.start() - delay(10L * 60 * 1000) - server.reload() - assertEquals(TaskState.PROVISIONING, server.state) - - verify { host.canFit(server) } - } - - @Test - fun testHostAvailableAfterSomeTime() = - scope.runSimulation { - val host = mockk(relaxUnitFun = true) - val listeners = mutableListOf() - - every { host.uid } returns UUID.randomUUID() - every { host.model } returns HostModel(4 * 2600.0, 4, 2048) - every { host.state } returns HostState.DOWN - every { host.addListener(any()) } answers { listeners.add(it.invocation.args[0] as HostListener) } - every { host.canFit(any()) } returns false - - service.addHost(host) - - val client = service.newClient() - val flavor = client.newFlavor("test", 1, 1024) - val image = client.newImage("test") - val server = client.newTask("test", image, flavor, start = false) - - server.start() - delay(5L * 60 * 1000) - - every { host.state } returns HostState.UP - listeners.forEach { it.onStateChanged(host, HostState.UP) } - - delay(5L * 60 * 1000) - server.reload() - assertEquals(TaskState.PROVISIONING, server.state) - - verify { host.canFit(server) } - } - - @Test - fun testHostUnavailableAfterSomeTime() = - scope.runSimulation { - val host = mockk(relaxUnitFun = true) - val listeners = mutableListOf() - - every { host.uid } returns UUID.randomUUID() - every { host.model } returns HostModel(4 * 2600.0, 4, 2048) - every { host.state } returns HostState.UP - every { host.addListener(any()) } answers { listeners.add(it.invocation.args[0] as HostListener) } - every { host.canFit(any()) } returns false - - service.addHost(host) - - val client = service.newClient() - val flavor = client.newFlavor("test", 1, 1024) - val image = client.newImage("test") - val server = client.newTask("test", image, flavor, start = false) - - delay(5L * 60 * 1000) - - every { host.state } returns HostState.DOWN - listeners.forEach { it.onStateChanged(host, HostState.DOWN) } - - server.start() - delay(5L * 60 * 1000) - server.reload() - assertEquals(TaskState.PROVISIONING, server.state) - - verify(exactly = 0) { host.canFit(server) } - } - - @Test - fun testServerDeploy() = - scope.runSimulation { - val host = mockk(relaxUnitFun = true) - val listeners = mutableListOf() - - every { host.uid } returns UUID.randomUUID() - every { host.model } returns HostModel(4 * 2600.0, 4, 2048) - every { host.state } returns HostState.UP - every { host.canFit(any()) } returns true - every { host.addListener(any()) } answers { listeners.add(it.invocation.args[0] as HostListener) } - - service.addHost(host) - - val client = service.newClient() - val flavor = client.newFlavor("test", 1, 1024) - val image = client.newImage("test") - val server = client.newTask("test", image, flavor, start = false) - val slot = slot() - - val watcher = mockk(relaxUnitFun = true) - server.watch(watcher) - - // Start server - server.start() - delay(5L * 60 * 1000) - coVerify { host.spawn(capture(slot)) } - - listeners.forEach { it.onStateChanged(host, slot.captured, TaskState.RUNNING) } - - server.reload() - assertEquals(TaskState.RUNNING, server.state) - - verify { watcher.onStateChanged(server, TaskState.RUNNING) } - - // Stop server - listeners.forEach { it.onStateChanged(host, slot.captured, TaskState.TERMINATED) } - - server.reload() - assertEquals(TaskState.TERMINATED, server.state) - - verify { watcher.onStateChanged(server, TaskState.TERMINATED) } - } - - @Test - fun testServerDeployFailure() = - scope.runSimulation { - val host = mockk(relaxUnitFun = true) - val listeners = mutableListOf() - - every { host.uid } returns UUID.randomUUID() - every { host.model } returns HostModel(4 * 2600.0, 4, 2048) - every { host.state } returns HostState.UP - every { host.canFit(any()) } returns true - every { host.addListener(any()) } answers { listeners.add(it.invocation.args[0] as HostListener) } - coEvery { host.spawn(any()) } throws IllegalStateException() - - service.addHost(host) - - val client = service.newClient() - val flavor = client.newFlavor("test", 1, 1024) - val image = client.newImage("test") - val server = client.newTask("test", image, flavor, start = false) - - server.start() - delay(5L * 60 * 1000) - - server.reload() - assertEquals(TaskState.PROVISIONING, server.state) - } -} diff --git a/opendc-compute/opendc-compute-service/src/test/kotlin/org/opendc/compute/service/ServiceFlavorTest.kt b/opendc-compute/opendc-compute-service/src/test/kotlin/org/opendc/compute/service/ServiceFlavorTest.kt deleted file mode 100644 index 7938f789..00000000 --- a/opendc-compute/opendc-compute-service/src/test/kotlin/org/opendc/compute/service/ServiceFlavorTest.kt +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright (c) 2021 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.compute.service - -import io.mockk.every -import io.mockk.mockk -import org.junit.jupiter.api.Assertions.assertEquals -import org.junit.jupiter.api.Assertions.assertNotEquals -import org.junit.jupiter.api.Test -import org.opendc.compute.api.Flavor -import java.util.UUID - -/** - * Test suite for the [ServiceFlavor] implementation. - */ -class ServiceFlavorTest { - @Test - fun testEquality() { - val service = mockk() - val uid = UUID.randomUUID() - val a = ServiceFlavor(service, uid, "test", 1, 1024, mutableMapOf(), mutableMapOf()) - val b = ServiceFlavor(service, uid, "test", 1, 1024, mutableMapOf(), mutableMapOf()) - - assertEquals(a, b) - } - - @Test - fun testInequalityWithDifferentType() { - val service = mockk() - val uid = UUID.randomUUID() - val a = ServiceFlavor(service, uid, "test", 1, 1024, mutableMapOf(), mutableMapOf()) - - val b = mockk(relaxUnitFun = true) - every { b.uid } returns UUID.randomUUID() - - assertNotEquals(a, b) - } - - @Test - fun testInequalityWithIncorrectType() { - val service = mockk() - val uid = UUID.randomUUID() - val a = ServiceFlavor(service, uid, "test", 1, 1024, mutableMapOf(), mutableMapOf()) - - assertNotEquals(a, Unit) - } -} diff --git a/opendc-compute/opendc-compute-service/src/test/kotlin/org/opendc/compute/service/ServiceImageTest.kt b/opendc-compute/opendc-compute-service/src/test/kotlin/org/opendc/compute/service/ServiceImageTest.kt deleted file mode 100644 index c36d75f4..00000000 --- a/opendc-compute/opendc-compute-service/src/test/kotlin/org/opendc/compute/service/ServiceImageTest.kt +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright (c) 2021 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.compute.service - -import io.mockk.every -import io.mockk.mockk -import org.junit.jupiter.api.Assertions.assertEquals -import org.junit.jupiter.api.Assertions.assertNotEquals -import org.junit.jupiter.api.Test -import org.opendc.compute.api.Image -import java.util.UUID - -/** - * Test suite for the [ServiceFlavor] implementation. - */ -class ServiceImageTest { - @Test - fun testEquality() { - val service = mockk() - val uid = UUID.randomUUID() - val a = ServiceImage(service, uid, "test", mutableMapOf(), mutableMapOf()) - val b = ServiceImage(service, uid, "test", mutableMapOf(), mutableMapOf()) - - assertEquals(a, b) - } - - @Test - fun testInequalityWithDifferentType() { - val service = mockk() - val uid = UUID.randomUUID() - val a = ServiceImage(service, uid, "test", mutableMapOf(), mutableMapOf()) - - val b = mockk(relaxUnitFun = true) - every { b.uid } returns UUID.randomUUID() - - assertNotEquals(a, b) - } - - @Test - fun testInequalityWithIncorrectType() { - val service = mockk() - val uid = UUID.randomUUID() - val a = ServiceImage(service, uid, "test", mutableMapOf(), mutableMapOf()) - - assertNotEquals(a, Unit) - } -} diff --git a/opendc-compute/opendc-compute-service/src/test/kotlin/org/opendc/compute/service/ServiceTaskTest.kt b/opendc-compute/opendc-compute-service/src/test/kotlin/org/opendc/compute/service/ServiceTaskTest.kt deleted file mode 100644 index e77665fe..00000000 --- a/opendc-compute/opendc-compute-service/src/test/kotlin/org/opendc/compute/service/ServiceTaskTest.kt +++ /dev/null @@ -1,442 +0,0 @@ -/* - * Copyright (c) 2021 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.compute.service - -import io.mockk.every -import io.mockk.mockk -import io.mockk.verify -import kotlinx.coroutines.yield -import org.junit.jupiter.api.Assertions.assertEquals -import org.junit.jupiter.api.Assertions.assertNotEquals -import org.junit.jupiter.api.Assertions.assertTrue -import org.junit.jupiter.api.Test -import org.junit.jupiter.api.assertThrows -import org.opendc.compute.api.Task -import org.opendc.compute.api.TaskState -import org.opendc.compute.service.driver.Host -import org.opendc.simulator.kotlin.runSimulation -import java.util.UUID - -/** - * Test suite for the [ServiceTask] implementation. - */ -class ServiceTaskTest { - @Test - fun testEquality() { - val service = mockk() - val uid = UUID.randomUUID() - val flavor = mockFlavor() - val image = mockImage() - - val a = - ServiceTask( - service, - uid, - "test", - flavor, - image, - mutableMapOf(), - mutableMapOf(), - ) - val b = - ServiceTask( - service, - uid, - "test", - flavor, - image, - mutableMapOf(), - mutableMapOf(), - ) - - assertEquals(a, b) - } - - @Test - fun testInequalityWithDifferentType() { - val service = mockk() - val uid = UUID.randomUUID() - val flavor = mockFlavor() - val image = mockImage() - val a = - ServiceTask( - service, - uid, - "test", - flavor, - image, - mutableMapOf(), - mutableMapOf(), - ) - - val b = mockk(relaxUnitFun = true) - every { b.uid } returns UUID.randomUUID() - - assertNotEquals(a, b) - } - - @Test - fun testInequalityWithIncorrectType() { - val service = mockk() - val uid = UUID.randomUUID() - val flavor = mockFlavor() - val image = mockImage() - val a = - ServiceTask( - service, - uid, - "test", - flavor, - image, - mutableMapOf(), - mutableMapOf(), - ) - - assertNotEquals(a, Unit) - } - - @Test - fun testStartTerminatedServer() = - runSimulation { - val service = mockk() - val uid = UUID.randomUUID() - val flavor = mockFlavor() - val image = mockImage() - val server = - ServiceTask( - service, - uid, - "test", - flavor, - image, - mutableMapOf(), - mutableMapOf(), - ) - - every { service.schedule(any()) } answers { ComputeService.SchedulingRequest(it.invocation.args[0] as ServiceTask, 0) } - - server.start() - - verify(exactly = 1) { service.schedule(server) } - assertEquals(TaskState.PROVISIONING, server.state) - } - - @Test - fun testStartDeletedServer() = - runSimulation { - val service = mockk() - val uid = UUID.randomUUID() - val flavor = mockFlavor() - val image = mockImage() - val server = - ServiceTask( - service, - uid, - "test", - flavor, - image, - mutableMapOf(), - mutableMapOf(), - ) - - server.setState(TaskState.DELETED) - - assertThrows { server.start() } - } - - @Test - fun testStartProvisioningServer() = - runSimulation { - val service = mockk() - val uid = UUID.randomUUID() - val flavor = mockFlavor() - val image = mockImage() - val server = - ServiceTask( - service, - uid, - "test", - flavor, - image, - mutableMapOf(), - mutableMapOf(), - ) - - server.setState(TaskState.PROVISIONING) - - server.start() - - assertEquals(TaskState.PROVISIONING, server.state) - } - - @Test - fun testStartRunningServer() = - runSimulation { - val service = mockk() - val uid = UUID.randomUUID() - val flavor = mockFlavor() - val image = mockImage() - val server = - ServiceTask( - service, - uid, - "test", - flavor, - image, - mutableMapOf(), - mutableMapOf(), - ) - - server.setState(TaskState.RUNNING) - - server.start() - - assertEquals(TaskState.RUNNING, server.state) - } - - @Test - fun testStopProvisioningServer() = - runSimulation { - val service = mockk() - val uid = UUID.randomUUID() - val flavor = mockFlavor() - val image = mockImage() - val server = - ServiceTask( - service, - uid, - "test", - flavor, - image, - mutableMapOf(), - mutableMapOf(), - ) - val request = ComputeService.SchedulingRequest(server, 0) - - every { service.schedule(any()) } returns request - - server.start() - server.stop() - - assertTrue(request.isCancelled) - assertEquals(TaskState.TERMINATED, server.state) - } - - @Test - fun testStopTerminatedServer() = - runSimulation { - val service = mockk() - val uid = UUID.randomUUID() - val flavor = mockFlavor() - val image = mockImage() - val server = - ServiceTask( - service, - uid, - "test", - flavor, - image, - mutableMapOf(), - mutableMapOf(), - ) - - server.setState(TaskState.TERMINATED) - server.stop() - - assertEquals(TaskState.TERMINATED, server.state) - } - - @Test - fun testStopDeletedServer() = - runSimulation { - val service = mockk() - val uid = UUID.randomUUID() - val flavor = mockFlavor() - val image = mockImage() - val server = - ServiceTask( - service, - uid, - "test", - flavor, - image, - mutableMapOf(), - mutableMapOf(), - ) - - server.setState(TaskState.DELETED) - server.stop() - - assertEquals(TaskState.DELETED, server.state) - } - - @Test - fun testStopRunningServer() = - runSimulation { - val service = mockk() - val uid = UUID.randomUUID() - val flavor = mockFlavor() - val image = mockImage() - val server = - ServiceTask( - service, - uid, - "test", - flavor, - image, - mutableMapOf(), - mutableMapOf(), - ) - val host = mockk(relaxUnitFun = true) - - server.setState(TaskState.RUNNING) - server.host = host - server.stop() - yield() - - verify { host.stop(server) } - } - - @Test - fun testDeleteProvisioningServer() = - runSimulation { - val service = mockk(relaxUnitFun = true) - val uid = UUID.randomUUID() - val flavor = mockFlavor() - val image = mockImage() - val server = - ServiceTask( - service, - uid, - "test", - flavor, - image, - mutableMapOf(), - mutableMapOf(), - ) - val request = ComputeService.SchedulingRequest(server, 0) - - every { service.schedule(any()) } returns request - - server.start() - server.delete() - - assertTrue(request.isCancelled) - assertEquals(TaskState.DELETED, server.state) - verify { service.delete(server) } - } - - @Test - fun testDeleteTerminatedServer() = - runSimulation { - val service = mockk(relaxUnitFun = true) - val uid = UUID.randomUUID() - val flavor = mockFlavor() - val image = mockImage() - val server = - ServiceTask( - service, - uid, - "test", - flavor, - image, - mutableMapOf(), - mutableMapOf(), - ) - - server.setState(TaskState.TERMINATED) - server.delete() - - assertEquals(TaskState.DELETED, server.state) - - verify { service.delete(server) } - } - - @Test - fun testDeleteDeletedServer() = - runSimulation { - val service = mockk(relaxUnitFun = true) - val uid = UUID.randomUUID() - val flavor = mockFlavor() - val image = mockImage() - val server = - ServiceTask( - service, - uid, - "test", - flavor, - image, - mutableMapOf(), - mutableMapOf(), - ) - - server.setState(TaskState.DELETED) - server.delete() - - assertEquals(TaskState.DELETED, server.state) - } - - @Test - fun testDeleteRunningServer() = - runSimulation { - val service = mockk(relaxUnitFun = true) - val uid = UUID.randomUUID() - val flavor = mockFlavor() - val image = mockImage() - val server = - ServiceTask( - service, - uid, - "test", - flavor, - image, - mutableMapOf(), - mutableMapOf(), - ) - val host = mockk(relaxUnitFun = true) - - server.setState(TaskState.RUNNING) - server.host = host - server.delete() - yield() - - verify { host.delete(server) } - verify { service.delete(server) } - } - - private fun mockFlavor(): ServiceFlavor { - val flavor = mockk() - every { flavor.name } returns "c5.large" - every { flavor.uid } returns UUID.randomUUID() - every { flavor.coreCount } returns 2 - every { flavor.memorySize } returns 4096 - return flavor - } - - private fun mockImage(): ServiceImage { - val image = mockk() - every { image.name } returns "ubuntu-20.04" - every { image.uid } returns UUID.randomUUID() - return image - } -} diff --git a/opendc-compute/opendc-compute-service/src/test/kotlin/org/opendc/compute/service/scheduler/FilterSchedulerTest.kt b/opendc-compute/opendc-compute-service/src/test/kotlin/org/opendc/compute/service/scheduler/FilterSchedulerTest.kt deleted file mode 100644 index add10f8f..00000000 --- a/opendc-compute/opendc-compute-service/src/test/kotlin/org/opendc/compute/service/scheduler/FilterSchedulerTest.kt +++ /dev/null @@ -1,536 +0,0 @@ -/* - * Copyright (c) 2021 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.compute.service.scheduler - -import io.mockk.every -import io.mockk.mockk -import org.junit.jupiter.api.Assertions.assertEquals -import org.junit.jupiter.api.Assertions.assertNull -import org.junit.jupiter.api.Test -import org.junit.jupiter.api.assertAll -import org.junit.jupiter.api.assertThrows -import org.opendc.compute.api.Task -import org.opendc.compute.service.HostView -import org.opendc.compute.service.driver.HostModel -import org.opendc.compute.service.driver.HostState -import org.opendc.compute.service.scheduler.filters.ComputeFilter -import org.opendc.compute.service.scheduler.filters.DifferentHostFilter -import org.opendc.compute.service.scheduler.filters.InstanceCountFilter -import org.opendc.compute.service.scheduler.filters.RamFilter -import org.opendc.compute.service.scheduler.filters.SameHostFilter -import org.opendc.compute.service.scheduler.filters.VCpuCapacityFilter -import org.opendc.compute.service.scheduler.filters.VCpuFilter -import org.opendc.compute.service.scheduler.weights.CoreRamWeigher -import org.opendc.compute.service.scheduler.weights.InstanceCountWeigher -import org.opendc.compute.service.scheduler.weights.RamWeigher -import org.opendc.compute.service.scheduler.weights.VCpuWeigher -import java.util.Random -import java.util.UUID - -/** - * Test suite for the [FilterScheduler]. - */ -internal class FilterSchedulerTest { - @Test - fun testInvalidSubsetSize() { - assertThrows { - FilterScheduler( - filters = emptyList(), - weighers = emptyList(), - subsetSize = 0, - ) - } - - assertThrows { - FilterScheduler( - filters = emptyList(), - weighers = emptyList(), - subsetSize = -2, - ) - } - } - - @Test - fun testNoHosts() { - val scheduler = - FilterScheduler( - filters = emptyList(), - weighers = emptyList(), - ) - - val task = mockk() - every { task.flavor.coreCount } returns 2 - every { task.flavor.memorySize } returns 1024 - - assertNull(scheduler.select(task)) - } - - @Test - fun testNoFiltersAndSchedulers() { - val scheduler = - FilterScheduler( - filters = emptyList(), - weighers = emptyList(), - ) - - val hostA = mockk() - every { hostA.host.state } returns HostState.DOWN - - val hostB = mockk() - every { hostB.host.state } returns HostState.UP - - scheduler.addHost(hostA) - scheduler.addHost(hostB) - - val task = mockk() - every { task.flavor.coreCount } returns 2 - every { task.flavor.memorySize } returns 1024 - - // Make sure we get the first host both times - assertAll( - { assertEquals(hostA, scheduler.select(task)) }, - { assertEquals(hostA, scheduler.select(task)) }, - ) - } - - @Test - fun testNoFiltersAndSchedulersRandom() { - val scheduler = - FilterScheduler( - filters = emptyList(), - weighers = emptyList(), - subsetSize = Int.MAX_VALUE, - random = Random(1), - ) - - val hostA = mockk() - every { hostA.host.state } returns HostState.DOWN - - val hostB = mockk() - every { hostB.host.state } returns HostState.UP - - scheduler.addHost(hostA) - scheduler.addHost(hostB) - - val task = mockk() - every { task.flavor.coreCount } returns 2 - every { task.flavor.memorySize } returns 1024 - - // Make sure we get the first host both times - assertAll( - { assertEquals(hostB, scheduler.select(task)) }, - { assertEquals(hostA, scheduler.select(task)) }, - ) - } - - @Test - fun testHostIsDown() { - val scheduler = - FilterScheduler( - filters = listOf(ComputeFilter()), - weighers = emptyList(), - ) - - val host = mockk() - every { host.host.state } returns HostState.DOWN - - scheduler.addHost(host) - - val task = mockk() - every { task.flavor.coreCount } returns 2 - every { task.flavor.memorySize } returns 1024 - - assertNull(scheduler.select(task)) - } - - @Test - fun testHostIsUp() { - val scheduler = - FilterScheduler( - filters = listOf(ComputeFilter()), - weighers = emptyList(), - ) - - val host = mockk() - every { host.host.state } returns HostState.UP - - scheduler.addHost(host) - - val task = mockk() - every { task.flavor.coreCount } returns 2 - every { task.flavor.memorySize } returns 1024 - - assertEquals(host, scheduler.select(task)) - } - - @Test - fun testRamFilter() { - val scheduler = - FilterScheduler( - filters = listOf(RamFilter(1.0)), - weighers = emptyList(), - ) - - val hostA = mockk() - every { hostA.host.state } returns HostState.UP - every { hostA.host.model } returns HostModel(4 * 2600.0, 4, 2048) - every { hostA.availableMemory } returns 512 - - val hostB = mockk() - every { hostB.host.state } returns HostState.UP - every { hostB.host.model } returns HostModel(4 * 2600.0, 4, 2048) - every { hostB.availableMemory } returns 2048 - - scheduler.addHost(hostA) - scheduler.addHost(hostB) - - val task = mockk() - every { task.flavor.coreCount } returns 2 - every { task.flavor.memorySize } returns 1024 - - assertEquals(hostB, scheduler.select(task)) - } - - @Test - fun testRamFilterOvercommit() { - val scheduler = - FilterScheduler( - filters = listOf(RamFilter(1.5)), - weighers = emptyList(), - ) - - val host = mockk() - every { host.host.state } returns HostState.UP - every { host.host.model } returns HostModel(4 * 2600.0, 4, 2048) - every { host.availableMemory } returns 2048 - - scheduler.addHost(host) - - val task = mockk() - every { task.flavor.coreCount } returns 2 - every { task.flavor.memorySize } returns 2300 - - assertNull(scheduler.select(task)) - } - - @Test - fun testVCpuFilter() { - val scheduler = - FilterScheduler( - filters = listOf(VCpuFilter(1.0)), - weighers = emptyList(), - ) - - val hostA = mockk() - every { hostA.host.state } returns HostState.UP - every { hostA.host.model } returns HostModel(4 * 2600.0, 4, 2048) - every { hostA.provisionedCores } returns 3 - - val hostB = mockk() - every { hostB.host.state } returns HostState.UP - every { hostB.host.model } returns HostModel(4 * 2600.0, 4, 2048) - every { hostB.provisionedCores } returns 0 - - scheduler.addHost(hostA) - scheduler.addHost(hostB) - - val task = mockk() - every { task.flavor.coreCount } returns 2 - every { task.flavor.memorySize } returns 1024 - - assertEquals(hostB, scheduler.select(task)) - } - - @Test - fun testVCpuFilterOvercommit() { - val scheduler = - FilterScheduler( - filters = listOf(VCpuFilter(16.0)), - weighers = emptyList(), - ) - - val host = mockk() - every { host.host.state } returns HostState.UP - every { host.host.model } returns HostModel(4 * 2600.0, 4, 2048) - every { host.provisionedCores } returns 0 - - scheduler.addHost(host) - - val task = mockk() - every { task.flavor.coreCount } returns 8 - every { task.flavor.memorySize } returns 1024 - - assertNull(scheduler.select(task)) - } - -// TODO: fix when schedulers are reworked -// @Test - fun testVCpuCapacityFilter() { - val scheduler = - FilterScheduler( - filters = listOf(VCpuCapacityFilter()), - weighers = emptyList(), - ) - - val hostA = mockk() - every { hostA.host.state } returns HostState.UP - every { hostA.host.model } returns HostModel(8 * 2600.0, 8, 2048) - every { hostA.availableMemory } returns 512 - scheduler.addHost(hostA) - - val hostB = mockk() - every { hostB.host.state } returns HostState.UP - every { hostB.host.model } returns HostModel(4 * 3200.0, 4, 2048) - every { hostB.availableMemory } returns 512 - - scheduler.addHost(hostB) - - val task = mockk() - every { task.flavor.coreCount } returns 2 - every { task.flavor.memorySize } returns 1024 - every { task.flavor.meta } returns mapOf("cpu-capacity" to 2 * 3200.0) - - assertEquals(hostB, scheduler.select(task)) - } - - @Test - fun testInstanceCountFilter() { - val scheduler = - FilterScheduler( - filters = listOf(InstanceCountFilter(limit = 2)), - weighers = emptyList(), - ) - - val hostA = mockk() - every { hostA.host.state } returns HostState.UP - every { hostA.host.model } returns HostModel(4 * 2600.0, 4, 2048) - every { hostA.instanceCount } returns 2 - - val hostB = mockk() - every { hostB.host.state } returns HostState.UP - every { hostB.host.model } returns HostModel(4 * 2600.0, 4, 2048) - every { hostB.instanceCount } returns 0 - - scheduler.addHost(hostA) - scheduler.addHost(hostB) - - val task = mockk() - every { task.flavor.coreCount } returns 2 - every { task.flavor.memorySize } returns 1024 - - assertEquals(hostB, scheduler.select(task)) - } - - @Test - fun testAffinityFilter() { - val scheduler = - FilterScheduler( - filters = listOf(SameHostFilter()), - weighers = emptyList(), - ) - - val taskA = mockk() - every { taskA.uid } returns UUID.randomUUID() - every { taskA.flavor.coreCount } returns 2 - every { taskA.flavor.memorySize } returns 1024 - - val hostA = mockk() - every { hostA.host.state } returns HostState.UP - every { hostA.host.model } returns HostModel(4 * 2600.0, 4, 2048) - every { hostA.host.instances } returns emptySet() - every { hostA.provisionedCores } returns 3 - - val hostB = mockk() - every { hostB.host.state } returns HostState.UP - every { hostB.host.model } returns HostModel(4 * 2600.0, 4, 2048) - every { hostB.host.instances } returns setOf(taskA) - every { hostB.provisionedCores } returns 0 - - scheduler.addHost(hostA) - scheduler.addHost(hostB) - - val taskB = mockk() - every { taskB.flavor.coreCount } returns 2 - every { taskB.flavor.memorySize } returns 1024 - every { taskB.meta } returns emptyMap() - - assertEquals(hostA, scheduler.select(taskB)) - - every { taskB.meta } returns mapOf("scheduler_hint:same_host" to setOf(taskA.uid)) - - assertEquals(hostB, scheduler.select(taskB)) - } - - @Test - fun testAntiAffinityFilter() { - val scheduler = - FilterScheduler( - filters = listOf(DifferentHostFilter()), - weighers = emptyList(), - ) - - val taskA = mockk() - every { taskA.uid } returns UUID.randomUUID() - every { taskA.flavor.coreCount } returns 2 - every { taskA.flavor.memorySize } returns 1024 - - val hostA = mockk() - every { hostA.host.state } returns HostState.UP - every { hostA.host.model } returns HostModel(4 * 2600.0, 4, 2048) - every { hostA.host.instances } returns setOf(taskA) - every { hostA.provisionedCores } returns 3 - - val hostB = mockk() - every { hostB.host.state } returns HostState.UP - every { hostB.host.model } returns HostModel(4 * 2600.0, 4, 2048) - every { hostB.host.instances } returns emptySet() - every { hostB.provisionedCores } returns 0 - - scheduler.addHost(hostA) - scheduler.addHost(hostB) - - val taskB = mockk() - every { taskB.flavor.coreCount } returns 2 - every { taskB.flavor.memorySize } returns 1024 - every { taskB.meta } returns emptyMap() - - assertEquals(hostA, scheduler.select(taskB)) - - every { taskB.meta } returns mapOf("scheduler_hint:different_host" to setOf(taskA.uid)) - - assertEquals(hostB, scheduler.select(taskB)) - } - - @Test - fun testRamWeigher() { - val scheduler = - FilterScheduler( - filters = emptyList(), - weighers = listOf(RamWeigher(1.5)), - ) - - val hostA = mockk() - every { hostA.host.state } returns HostState.UP - every { hostA.host.model } returns HostModel(4 * 2600.0, 4, 2048) - every { hostA.availableMemory } returns 1024 - - val hostB = mockk() - every { hostB.host.state } returns HostState.UP - every { hostB.host.model } returns HostModel(4 * 2600.0, 4, 2048) - every { hostB.availableMemory } returns 512 - - scheduler.addHost(hostA) - scheduler.addHost(hostB) - - val task = mockk() - every { task.flavor.coreCount } returns 2 - every { task.flavor.memorySize } returns 1024 - - assertEquals(hostA, scheduler.select(task)) - } - - // TODO: fix test when updating schedulers -// @Test - fun testCoreRamWeigher() { - val scheduler = - FilterScheduler( - filters = emptyList(), - weighers = listOf(CoreRamWeigher(1.5)), - ) - - val hostA = mockk() - every { hostA.host.state } returns HostState.UP - every { hostA.host.model } returns HostModel(12 * 2600.0, 12, 2048) - every { hostA.availableMemory } returns 1024 - - val hostB = mockk() - every { hostB.host.state } returns HostState.UP - every { hostB.host.model } returns HostModel(4 * 2600.0, 4, 2048) - every { hostB.availableMemory } returns 512 - - scheduler.addHost(hostA) - scheduler.addHost(hostB) - - val task = mockk() - every { task.flavor.coreCount } returns 2 - every { task.flavor.memorySize } returns 1024 - - assertEquals(hostB, scheduler.select(task)) - } - - @Test - fun testVCpuWeigher() { - val scheduler = - FilterScheduler( - filters = emptyList(), - weighers = listOf(VCpuWeigher(16.0)), - ) - - val hostA = mockk() - every { hostA.host.state } returns HostState.UP - every { hostA.host.model } returns HostModel(4 * 2600.0, 4, 2048) - every { hostA.provisionedCores } returns 2 - - val hostB = mockk() - every { hostB.host.state } returns HostState.UP - every { hostB.host.model } returns HostModel(4 * 2600.0, 4, 2048) - every { hostB.provisionedCores } returns 0 - - scheduler.addHost(hostA) - scheduler.addHost(hostB) - - val task = mockk() - every { task.flavor.coreCount } returns 2 - every { task.flavor.memorySize } returns 1024 - - assertEquals(hostB, scheduler.select(task)) - } - - @Test - fun testInstanceCountWeigher() { - val scheduler = - FilterScheduler( - filters = emptyList(), - weighers = listOf(InstanceCountWeigher(multiplier = -1.0)), - ) - - val hostA = mockk() - every { hostA.host.state } returns HostState.UP - every { hostA.host.model } returns HostModel(4 * 2600.0, 4, 2048) - every { hostA.instanceCount } returns 2 - - val hostB = mockk() - every { hostB.host.state } returns HostState.UP - every { hostB.host.model } returns HostModel(4 * 2600.0, 4, 2048) - every { hostB.instanceCount } returns 0 - - scheduler.addHost(hostA) - scheduler.addHost(hostB) - - val task = mockk() - every { task.flavor.coreCount } returns 2 - every { task.flavor.memorySize } returns 1024 - - assertEquals(hostB, scheduler.select(task)) - } -} diff --git a/opendc-compute/opendc-compute-service/src/test/resources/log4j2.xml b/opendc-compute/opendc-compute-service/src/test/resources/log4j2.xml deleted file mode 100644 index 0dfb75f2..00000000 --- a/opendc-compute/opendc-compute-service/src/test/resources/log4j2.xml +++ /dev/null @@ -1,38 +0,0 @@ - - - - - - - - - - - - - - - - - - diff --git a/opendc-compute/opendc-compute-simulator/build.gradle.kts b/opendc-compute/opendc-compute-simulator/build.gradle.kts index 20ceb93e..8cbddb44 100644 --- a/opendc-compute/opendc-compute-simulator/build.gradle.kts +++ b/opendc-compute/opendc-compute-simulator/build.gradle.kts @@ -25,22 +25,25 @@ description = "Simulator for OpenDC Compute" // Build configuration plugins { `kotlin-library-conventions` + kotlin("plugin.serialization") version "1.9.22" } dependencies { - api(projects.opendcCompute.opendcComputeService) api(projects.opendcSimulator.opendcSimulatorCompute) + api(projects.opendcTrace.opendcTraceParquet) api(libs.commons.math3) implementation(projects.opendcCommon) implementation(libs.kotlin.logging) + implementation("org.jetbrains.kotlinx:kotlinx-serialization-json:1.6.0") api(libs.microprofile.config) implementation(project(mapOf("path" to ":opendc-compute:opendc-compute-topology"))) - implementation(project(mapOf("path" to ":opendc-compute:opendc-compute-telemetry"))) implementation(project(mapOf("path" to ":opendc-compute:opendc-compute-carbon"))) implementation(project(mapOf("path" to ":opendc-trace:opendc-trace-api"))) + implementation(project(mapOf("path" to ":opendc-trace:opendc-trace-parquet"))) testImplementation(projects.opendcSimulator.opendcSimulatorCore) testRuntimeOnly(libs.slf4j.simple) + testRuntimeOnly(libs.log4j.slf4j) } diff --git a/opendc-compute/opendc-compute-simulator/src/main/java/org/opendc/compute/simulator/host/HostListener.java b/opendc-compute/opendc-compute-simulator/src/main/java/org/opendc/compute/simulator/host/HostListener.java new file mode 100644 index 00000000..01acfa97 --- /dev/null +++ b/opendc-compute/opendc-compute-simulator/src/main/java/org/opendc/compute/simulator/host/HostListener.java @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2024 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.compute.simulator.host; + +import org.opendc.compute.api.TaskState; +import org.opendc.compute.simulator.service.ServiceTask; + +/** + * Listener interface for events originating from a {@link SimHost}. + */ +public interface HostListener { + /** + * This method is invoked when the state of task on host changes. + */ + default void onStateChanged(SimHost host, ServiceTask task, TaskState newState) {} + + /** + * This method is invoked when the state of a {@link SimHost} has changed. + */ + default void onStateChanged(SimHost host, HostState newState) {} +} diff --git a/opendc-compute/opendc-compute-simulator/src/main/java/org/opendc/compute/simulator/host/HostModel.java b/opendc-compute/opendc-compute-simulator/src/main/java/org/opendc/compute/simulator/host/HostModel.java new file mode 100644 index 00000000..96236c5c --- /dev/null +++ b/opendc-compute/opendc-compute-simulator/src/main/java/org/opendc/compute/simulator/host/HostModel.java @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2022 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.compute.simulator.host; + +/** + * Record describing the static machine properties of the host. + * + * @param cpuCapacity The total CPU capacity of the host in MHz. + * @param coreCount The number of logical processing cores available for this host. + * @param memoryCapacity The amount of memory available for this host in MB. + */ +public record HostModel(float cpuCapacity, int coreCount, long memoryCapacity) {} diff --git a/opendc-compute/opendc-compute-simulator/src/main/java/org/opendc/compute/simulator/host/HostState.java b/opendc-compute/opendc-compute-simulator/src/main/java/org/opendc/compute/simulator/host/HostState.java new file mode 100644 index 00000000..29fc8cb4 --- /dev/null +++ b/opendc-compute/opendc-compute-simulator/src/main/java/org/opendc/compute/simulator/host/HostState.java @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2022 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.compute.simulator.host; + +/** + * The state of a host. + */ +public enum HostState { + /** + * The host is up and able to host guests. + */ + UP, + + /** + * The host is in a (forced) down state and unable to host any guests. + */ + DOWN, + + /** + * The host is in an error state and unable to host any guests. + */ + ERROR +} diff --git a/opendc-compute/opendc-compute-simulator/src/main/java/org/opendc/compute/simulator/service/ComputeService.java b/opendc-compute/opendc-compute-simulator/src/main/java/org/opendc/compute/simulator/service/ComputeService.java new file mode 100644 index 00000000..84e23516 --- /dev/null +++ b/opendc-compute/opendc-compute-simulator/src/main/java/org/opendc/compute/simulator/service/ComputeService.java @@ -0,0 +1,652 @@ +/* + * Copyright (c) 2022 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.compute.simulator.service; + +import java.time.Duration; +import java.time.Instant; +import java.time.InstantSource; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Deque; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.SplittableRandom; +import java.util.UUID; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.opendc.common.Dispatcher; +import org.opendc.common.util.Pacer; +import org.opendc.compute.api.Flavor; +import org.opendc.compute.api.Image; +import org.opendc.compute.api.TaskState; +import org.opendc.compute.simulator.host.HostListener; +import org.opendc.compute.simulator.host.HostModel; +import org.opendc.compute.simulator.host.HostState; +import org.opendc.compute.simulator.host.SimHost; +import org.opendc.compute.simulator.scheduler.ComputeScheduler; +import org.opendc.compute.simulator.telemetry.SchedulerStats; +import org.opendc.simulator.compute.workload.Workload; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * The {@link ComputeService} hosts the API implementation of the OpenDC Compute Engine. + */ +public final class ComputeService implements AutoCloseable { + private static final Logger LOGGER = LoggerFactory.getLogger(ComputeService.class); + + /** + * The {@link InstantSource} representing the clock tracking the (simulation) time. + */ + private final InstantSource clock; + + /** + * The {@link ComputeScheduler} responsible for placing the tasks onto hosts. + */ + private final ComputeScheduler scheduler; + + /** + * The {@link Pacer} used to pace the scheduling requests. + */ + private final Pacer pacer; + + /** + * The {@link SplittableRandom} used to generate the unique identifiers for the service resources. + */ + private final SplittableRandom random = new SplittableRandom(0); + + private final int maxNumFailures; + + /** + * A flag to indicate that the service is closed. + */ + private boolean isClosed; + + /** + * A mapping from host to host view. + */ + private final Map hostToView = new HashMap<>(); + + /** + * The available hypervisors. + */ + private final Set availableHosts = new HashSet<>(); + + /** + * The tasks that should be launched by the service. + */ + private final Deque taskQueue = new ArrayDeque<>(); + + /** + * The active tasks in the system. + */ + private final Map activeTasks = new HashMap<>(); + + /** + * The active tasks in the system. + */ + private final Map completedTasks = new HashMap<>(); + + /** + * The registered flavors for this compute service. + */ + private final Map flavorById = new HashMap<>(); + + private final List flavors = new ArrayList<>(); + + /** + * The registered images for this compute service. + */ + private final Map imageById = new HashMap<>(); + + private final List images = new ArrayList<>(); + + /** + * The registered tasks for this compute service. + */ + private final Map taskById = new HashMap<>(); + + private final List tasks = new ArrayList<>(); + + private final List tasksToRemove = new ArrayList<>(); + + /** + * A [HostListener] used to track the active tasks. + */ + private final HostListener hostListener = new HostListener() { + @Override + public void onStateChanged(@NotNull SimHost host, @NotNull HostState newState) { + LOGGER.debug("Host {} state changed: {}", host, newState); + + final HostView hv = hostToView.get(host); + + if (hv != null) { + if (newState == HostState.UP) { + availableHosts.add(hv); + } else { + availableHosts.remove(hv); + } + } + + // Re-schedule on the new machine + requestSchedulingCycle(); + } + + @Override + public void onStateChanged(@NotNull SimHost host, @NotNull ServiceTask task, @NotNull TaskState newState) { + if (task.getHost() != host) { + // This can happen when a task is rescheduled and started on another machine, while being deleted from + // the old machine. + return; + } + + task.setState(newState); + + if (newState == TaskState.COMPLETED || newState == TaskState.TERMINATED || newState == TaskState.FAILED) { + LOGGER.info("task {} {} {} finished", task.getUid(), task.getName(), task.getFlavor()); + + if (activeTasks.remove(task) != null) { + tasksActive--; + } + + HostView hv = hostToView.get(host); + final ServiceFlavor flavor = task.getFlavor(); + if (hv != null) { + hv.provisionedCores -= flavor.getCoreCount(); + hv.instanceCount--; + hv.availableMemory += flavor.getMemorySize(); + } else { + LOGGER.error("Unknown host {}", host); + } + + task.setHost(null); + host.removeTask(task); + + if (newState == TaskState.COMPLETED) { + tasksCompleted++; + } + if (newState == TaskState.TERMINATED) { + tasksTerminated++; + } + + if (task.getState() == TaskState.COMPLETED || task.getState() == TaskState.TERMINATED) { + tasksToRemove.add(task); + } + + // Try to reschedule if needed + requestSchedulingCycle(); + } + } + }; + + private int maxCores = 0; + private long maxMemory = 0L; + private long attemptsSuccess = 0L; + private long attemptsFailure = 0L; + private int tasksTotal = 0; + private int tasksPending = 0; + private int tasksActive = 0; + private int tasksTerminated = 0; + private int tasksCompleted = 0; + + /** + * Construct a {@link ComputeService} instance. + */ + public ComputeService(Dispatcher dispatcher, ComputeScheduler scheduler, Duration quantum, int maxNumFailures) { + this.clock = dispatcher.getTimeSource(); + this.scheduler = scheduler; + this.pacer = new Pacer(dispatcher, quantum.toMillis(), (time) -> doSchedule()); + this.maxNumFailures = maxNumFailures; + } + + /** + * Create a new {@link Builder} instance. + */ + public static Builder builder(Dispatcher dispatcher, ComputeScheduler scheduler) { + return new Builder(dispatcher, scheduler); + } + + /** + * Create a new {@link ComputeClient} to control the compute service. + */ + public ComputeClient newClient() { + if (isClosed) { + throw new IllegalStateException("Service is closed"); + } + return new ComputeClient(this); + } + + /** + * Return the {@link ServiceTask}s hosted by this service. + */ + public List getTasks() { + return Collections.unmodifiableList(tasks); + } + + /** + * Return the {@link ServiceTask}s hosted by this service. + */ + public List getTasksToRemove() { + return Collections.unmodifiableList(tasksToRemove); + } + + public void clearTasksToRemove() { + this.tasksToRemove.clear(); + } + + /** + * Add a {@link SimHost} to the scheduling pool of the compute service. + */ + public void addHost(SimHost host) { + // Check if host is already known + if (hostToView.containsKey(host)) { + return; + } + + HostView hv = new HostView(host); + HostModel model = host.getModel(); + + maxCores = Math.max(maxCores, model.coreCount()); + maxMemory = Math.max(maxMemory, model.memoryCapacity()); + hostToView.put(host, hv); + + if (host.getState() == HostState.UP) { + availableHosts.add(hv); + } + + scheduler.addHost(hv); + host.addListener(hostListener); + } + + /** + * Remove a {@link SimHost} from the scheduling pool of the compute service. + */ + public void removeHost(SimHost host) { + HostView view = hostToView.remove(host); + if (view != null) { + availableHosts.remove(view); + scheduler.removeHost(view); + host.removeListener(hostListener); + } + } + + /** + * Lookup the {@link SimHost} that currently hosts the specified {@link ServiceTask}. + */ + public SimHost lookupHost(ServiceTask task) { + return task.getHost(); + } + + /** + * Return the {@link SimHost}s that are registered with this service. + */ + public Set getHosts() { + return Collections.unmodifiableSet(hostToView.keySet()); + } + + public InstantSource getClock() { + return this.clock; + } + + /** + * Collect the statistics about the scheduler component of this service. + */ + public SchedulerStats getSchedulerStats() { + return new SchedulerStats( + availableHosts.size(), + hostToView.size() - availableHosts.size(), + attemptsSuccess, + attemptsFailure, + tasksTotal, + tasksPending, + tasksActive, + tasksCompleted, + tasksTerminated); + } + + @Override + public void close() { + if (isClosed) { + return; + } + + isClosed = true; + pacer.cancel(); + } + + /** + * Enqueue the specified [task] to be scheduled onto a host. + */ + SchedulingRequest schedule(ServiceTask task) { + LOGGER.debug("Enqueueing task {} to be assigned to host", task.getUid()); + + long now = clock.millis(); + SchedulingRequest request = new SchedulingRequest(task, now); + + task.launchedAt = Instant.ofEpochMilli(now); + taskQueue.add(request); + tasksPending++; + requestSchedulingCycle(); + return request; + } + + void delete(ServiceFlavor flavor) { + flavorById.remove(flavor.getUid()); + flavors.remove(flavor); + } + + void delete(ServiceImage image) { + imageById.remove(image.getUid()); + images.remove(image); + } + + void delete(ServiceTask task) { + completedTasks.remove(task); + taskById.remove(task.getUid()); + tasks.remove(task); + } + + /** + * Indicate that a new scheduling cycle is needed due to a change to the service's state. + */ + private void requestSchedulingCycle() { + // Bail out in case the queue is empty. + if (taskQueue.isEmpty()) { + return; + } + + pacer.enqueue(); + } + + /** + * Run a single scheduling iteration. + */ + private void doSchedule() { + // reorder tasks + + while (!taskQueue.isEmpty()) { + SchedulingRequest request = taskQueue.peek(); + + if (request.isCancelled) { + taskQueue.poll(); + tasksPending--; + continue; + } + + final ServiceTask task = request.task; + + if (task.getNumFailures() >= maxNumFailures) { + LOGGER.warn("task {} has been terminated because it failed {} times", task, task.getNumFailures()); + + taskQueue.poll(); + tasksPending--; + tasksTerminated++; + task.setState(TaskState.TERMINATED); + tasksToRemove.add(task); + continue; + } + + final ServiceFlavor flavor = task.getFlavor(); + final HostView hv = scheduler.select(request.task); + + if (hv == null || !hv.getHost().canFit(task)) { + LOGGER.trace("Task {} selected for scheduling but no capacity available for it at the moment", task); + + if (flavor.getMemorySize() > maxMemory || flavor.getCoreCount() > maxCores) { + // Remove the incoming image + taskQueue.poll(); + tasksPending--; + + LOGGER.warn("Failed to spawn {}: does not fit", task); + + task.setState(TaskState.FAILED); + continue; + } else { + break; + } + } + + SimHost host = hv.getHost(); + + // Remove request from queue + taskQueue.poll(); + tasksPending--; + + LOGGER.info("Assigned task {} to host {}", task, host); + + try { + task.host = host; + + host.spawn(task); + // host.start(task); + + tasksActive++; + attemptsSuccess++; + + hv.instanceCount++; + hv.provisionedCores += flavor.getCoreCount(); + hv.availableMemory -= flavor.getMemorySize(); + + activeTasks.put(task, host); + } catch (Exception cause) { + LOGGER.error("Failed to deploy VM", cause); + attemptsFailure++; + } + } + } + + /** + * Builder class for a {@link ComputeService}. + */ + public static class Builder { + private final Dispatcher dispatcher; + private final ComputeScheduler computeScheduler; + private Duration quantum = Duration.ofSeconds(1); + private int maxNumFailures = 10; + + Builder(Dispatcher dispatcher, ComputeScheduler computeScheduler) { + this.dispatcher = dispatcher; + this.computeScheduler = computeScheduler; + } + + /** + * Set the scheduling quantum of the service. + */ + public Builder withQuantum(Duration quantum) { + this.quantum = quantum; + return this; + } + + public Builder withMaxNumFailures(int maxNumFailures) { + this.maxNumFailures = maxNumFailures; + return this; + } + + /** + * Build a {@link ComputeService}. + */ + public ComputeService build() { + return new ComputeService(dispatcher, computeScheduler, quantum, maxNumFailures); + } + } + + /** + * Implementation of {@link ComputeClient} using a {@link ComputeService}. + */ + public static class ComputeClient { + private final ComputeService service; + private boolean isClosed; + + ComputeClient(ComputeService service) { + this.service = service; + } + + /** + * Method to check if the client is still open and throw an exception if it is not. + */ + private void checkOpen() { + if (isClosed) { + throw new IllegalStateException("Client is already closed"); + } + } + + @NotNull + public List queryFlavors() { + checkOpen(); + return new ArrayList<>(service.flavors); + } + + public Flavor findFlavor(@NotNull UUID id) { + checkOpen(); + + return service.flavorById.get(id); + } + + @NotNull + public Flavor newFlavor(@NotNull String name, int cpuCount, long memorySize, @NotNull Map meta) { + checkOpen(); + + final ComputeService service = this.service; + UUID uid = new UUID(service.clock.millis(), service.random.nextLong()); + ServiceFlavor flavor = new ServiceFlavor(service, uid, name, cpuCount, memorySize, meta); + + service.flavorById.put(uid, flavor); + service.flavors.add(flavor); + + return flavor; + } + + @NotNull + public List queryImages() { + checkOpen(); + + return new ArrayList<>(service.images); + } + + public Image findImage(@NotNull UUID id) { + checkOpen(); + + return service.imageById.get(id); + } + + public Image newImage(@NotNull String name) { + return newImage(name, Collections.emptyMap(), Collections.emptyMap()); + } + + @NotNull + public Image newImage(@NotNull String name, @NotNull Map labels, @NotNull Map meta) { + checkOpen(); + + final ComputeService service = this.service; + UUID uid = new UUID(service.clock.millis(), service.random.nextLong()); + + ServiceImage image = new ServiceImage(service, uid, name, labels, meta); + + service.imageById.put(uid, image); + service.images.add(image); + + return image; + } + + @NotNull + public ServiceTask newTask( + @NotNull String name, + @NotNull Flavor flavor, + @NotNull Workload workload, + @NotNull Map meta) { + checkOpen(); + + final ComputeService service = this.service; + UUID uid = new UUID(service.clock.millis(), service.random.nextLong()); + + final ServiceFlavor internalFlavor = + Objects.requireNonNull(service.flavorById.get(flavor.getUid()), "Unknown flavor"); + + ServiceTask task = new ServiceTask(service, uid, name, internalFlavor, workload, meta); + + service.taskById.put(uid, task); + service.tasks.add(task); + + service.tasksTotal++; + + task.start(); + + return task; + } + + @Nullable + public ServiceTask findTask(@NotNull UUID id) { + checkOpen(); + return service.taskById.get(id); + } + + @NotNull + public List queryTasks() { + checkOpen(); + + return new ArrayList<>(service.tasks); + } + + public void close() { + isClosed = true; + } + + @Override + public String toString() { + return "ComputeService.Client"; + } + + @Nullable + public void rescheduleTask(@NotNull ServiceTask task, @NotNull Workload workload) { + ServiceTask internalTask = findTask(task.getUid()); + // SimHost from = service.lookupHost(internalTask); + + // from.delete(internalTask); + + internalTask.host = null; + + internalTask.setWorkload(workload); + internalTask.start(); + } + } + + /** + * A request to schedule a {@link ServiceTask} onto one of the {@link SimHost}s. + */ + static class SchedulingRequest { + final ServiceTask task; + final long submitTime; + + boolean isCancelled; + + SchedulingRequest(ServiceTask task, long submitTime) { + this.task = task; + this.submitTime = submitTime; + } + } +} diff --git a/opendc-compute/opendc-compute-simulator/src/main/java/org/opendc/compute/simulator/service/HostView.java b/opendc-compute/opendc-compute-simulator/src/main/java/org/opendc/compute/simulator/service/HostView.java new file mode 100644 index 00000000..f4aa9c70 --- /dev/null +++ b/opendc-compute/opendc-compute-simulator/src/main/java/org/opendc/compute/simulator/service/HostView.java @@ -0,0 +1,78 @@ +/* + * Copyright (c) 2022 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.compute.simulator.service; + +import org.opendc.compute.simulator.host.SimHost; + +/** + * A view of a {@link SimHost} as seen from the {@link ComputeService}. + */ +public class HostView { + private final SimHost host; + int instanceCount; + long availableMemory; + int provisionedCores; + + /** + * Construct a {@link HostView} instance. + * + * @param host The host to create a view of. + */ + public HostView(SimHost host) { + this.host = host; + this.availableMemory = host.getModel().memoryCapacity(); + } + + /** + * The {@link SimHost} this is a view of. + */ + public SimHost getHost() { + return host; + } + + /** + * Return the number of instances on this host. + */ + public int getInstanceCount() { + return instanceCount; + } + + /** + * Return the available memory of the host. + */ + public long getAvailableMemory() { + return availableMemory; + } + + /** + * Return the provisioned cores on the host. + */ + public int getProvisionedCores() { + return provisionedCores; + } + + @Override + public String toString() { + return "HostView[host=" + host + "]"; + } +} diff --git a/opendc-compute/opendc-compute-simulator/src/main/java/org/opendc/compute/simulator/service/ServiceFlavor.java b/opendc-compute/opendc-compute-simulator/src/main/java/org/opendc/compute/simulator/service/ServiceFlavor.java new file mode 100644 index 00000000..eddde87e --- /dev/null +++ b/opendc-compute/opendc-compute-simulator/src/main/java/org/opendc/compute/simulator/service/ServiceFlavor.java @@ -0,0 +1,107 @@ +/* + * Copyright (c) 2022 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.compute.simulator.service; + +import java.util.Collections; +import java.util.Map; +import java.util.Objects; +import java.util.UUID; +import org.jetbrains.annotations.NotNull; +import org.opendc.compute.api.Flavor; + +/** + * Implementation of {@link Flavor} provided by {@link ComputeService}. + */ +public final class ServiceFlavor implements Flavor { + private final ComputeService service; + private final UUID uid; + private final String name; + private final int coreCount; + private final long memorySize; + private final Map meta; + + ServiceFlavor(ComputeService service, UUID uid, String name, int coreCount, long memorySize, Map meta) { + this.service = service; + this.uid = uid; + this.name = name; + this.coreCount = coreCount; + this.memorySize = memorySize; + this.meta = meta; + } + + @Override + public int getCoreCount() { + return coreCount; + } + + @Override + public long getMemorySize() { + return memorySize; + } + + @NotNull + @Override + public UUID getUid() { + return uid; + } + + @NotNull + @Override + public String getName() { + return name; + } + + @NotNull + @Override + public Map getMeta() { + return Collections.unmodifiableMap(meta); + } + + @Override + public void reload() { + // No-op: this object is the source-of-truth + } + + @Override + public void delete() { + service.delete(this); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ServiceFlavor flavor = (ServiceFlavor) o; + return service.equals(flavor.service) && uid.equals(flavor.uid); + } + + @Override + public int hashCode() { + return Objects.hash(service, uid); + } + + @Override + public String toString() { + return "Flavor[uid=" + uid + ",name=" + name + "]"; + } +} diff --git a/opendc-compute/opendc-compute-simulator/src/main/java/org/opendc/compute/simulator/service/ServiceImage.java b/opendc-compute/opendc-compute-simulator/src/main/java/org/opendc/compute/simulator/service/ServiceImage.java new file mode 100644 index 00000000..dffa4356 --- /dev/null +++ b/opendc-compute/opendc-compute-simulator/src/main/java/org/opendc/compute/simulator/service/ServiceImage.java @@ -0,0 +1,95 @@ +/* + * Copyright (c) 2022 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.compute.simulator.service; + +import java.util.Collections; +import java.util.Map; +import java.util.Objects; +import java.util.UUID; +import org.jetbrains.annotations.NotNull; +import org.opendc.compute.api.Image; + +/** + * Implementation of {@link Image} provided by {@link ComputeService}. + */ +public final class ServiceImage implements Image { + private final ComputeService service; + private final UUID uid; + private final String name; + private final Map labels; + private final Map meta; + + ServiceImage(ComputeService service, UUID uid, String name, Map labels, Map meta) { + this.service = service; + this.uid = uid; + this.name = name; + this.labels = labels; + this.meta = meta; + } + + @NotNull + @Override + public UUID getUid() { + return uid; + } + + @NotNull + @Override + public String getName() { + return name; + } + + @NotNull + @Override + public Map getMeta() { + return Collections.unmodifiableMap(meta); + } + + @Override + public void reload() { + // No-op: this object is the source-of-truth + } + + @Override + public void delete() { + service.delete(this); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ServiceImage image = (ServiceImage) o; + return service.equals(image.service) && uid.equals(image.uid); + } + + @Override + public int hashCode() { + return Objects.hash(service, uid); + } + + @Override + public String toString() { + return "Image[uid=" + uid + ",name=" + name + "]"; + } +} diff --git a/opendc-compute/opendc-compute-simulator/src/main/java/org/opendc/compute/simulator/service/ServiceTask.java b/opendc-compute/opendc-compute-simulator/src/main/java/org/opendc/compute/simulator/service/ServiceTask.java new file mode 100644 index 00000000..f39142eb --- /dev/null +++ b/opendc-compute/opendc-compute-simulator/src/main/java/org/opendc/compute/simulator/service/ServiceTask.java @@ -0,0 +1,230 @@ +/* + * Copyright (c) 2022 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.compute.simulator.service; + +import java.time.Instant; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.UUID; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.opendc.compute.api.TaskState; +import org.opendc.compute.simulator.TaskWatcher; +import org.opendc.compute.simulator.host.SimHost; +import org.opendc.simulator.compute.workload.Workload; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Implementation of {@link ServiceTask} provided by {@link ComputeService}. + */ +public class ServiceTask { + private static final Logger LOGGER = LoggerFactory.getLogger(ServiceTask.class); + + private final ComputeService service; + private final UUID uid; + + private final String name; + private final ServiceFlavor flavor; + public Workload workload; + + private Map meta; // TODO: remove this + + private final List watchers = new ArrayList<>(); + private TaskState state = TaskState.CREATED; + Instant launchedAt = null; + Instant createdAt; + Instant finishedAt; + SimHost host = null; + private ComputeService.SchedulingRequest request = null; + + private int numFailures = 0; + + ServiceTask( + ComputeService service, + UUID uid, + String name, + ServiceFlavor flavor, + Workload workload, + Map meta) { + this.service = service; + this.uid = uid; + this.name = name; + this.flavor = flavor; + this.workload = workload; + this.meta = meta; + + this.createdAt = this.service.getClock().instant(); + } + + @NotNull + public UUID getUid() { + return uid; + } + + @NotNull + public String getName() { + return name; + } + + @NotNull + public ServiceFlavor getFlavor() { + return flavor; + } + + @NotNull + public Map getMeta() { + return Collections.unmodifiableMap(meta); + } + + public void setWorkload(Workload newWorkload) { + this.workload = newWorkload; + } + + @NotNull + public TaskState getState() { + return state; + } + + @Nullable + public Instant getLaunchedAt() { + return launchedAt; + } + + @Nullable + public Instant getCreatedAt() { + return createdAt; + } + + @Nullable + public Instant getFinishedAt() { + return finishedAt; + } + + /** + * Return the {@link SimHost} on which the task is running or null if it is not running on a host. + */ + public SimHost getHost() { + return host; + } + + public void setHost(SimHost host) { + this.host = host; + } + + public int getNumFailures() { + return this.numFailures; + } + + public void start() { + switch (state) { + case PROVISIONING: + LOGGER.debug("User tried to start task but request is already pending: doing nothing"); + case RUNNING: + LOGGER.debug("User tried to start task but task is already running"); + break; + case COMPLETED: + case TERMINATED: + LOGGER.warn("User tried to start deleted task"); + throw new IllegalStateException("Task is deleted"); + case CREATED: + LOGGER.info("User requested to start task {}", uid); + setState(TaskState.PROVISIONING); + assert request == null : "Scheduling request already active"; + request = service.schedule(this); + break; + case FAILED: + LOGGER.info("User requested to start task after failure {}", uid); + setState(TaskState.PROVISIONING); + request = service.schedule(this); + break; + } + } + + public void watch(@NotNull TaskWatcher watcher) { + watchers.add(watcher); + } + + public void unwatch(@NotNull TaskWatcher watcher) { + watchers.remove(watcher); + } + + public void delete() { + cancelProvisioningRequest(); + final SimHost host = this.host; + if (host != null) { + host.delete(this); + } + service.delete(this); + + this.setState(TaskState.DELETED); + } + + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ServiceTask task = (ServiceTask) o; + return service.equals(task.service) && uid.equals(task.uid); + } + + public int hashCode() { + return Objects.hash(service, uid); + } + + public String toString() { + return "Task[uid=" + uid + ",name=" + name + ",state=" + state + "]"; + } + + void setState(TaskState newState) { + if (this.state == newState) { + return; + } + + for (TaskWatcher watcher : watchers) { + watcher.onStateChanged(this, newState); + } + if (newState == TaskState.FAILED) { + this.numFailures++; + } + + if ((newState == TaskState.COMPLETED) || newState == TaskState.FAILED) { + this.finishedAt = this.service.getClock().instant(); + } + + this.state = newState; + } + + /** + * Cancel the provisioning request if active. + */ + private void cancelProvisioningRequest() { + final ComputeService.SchedulingRequest request = this.request; + if (request != null) { + this.request = null; + request.isCancelled = true; + } + } +} diff --git a/opendc-compute/opendc-compute-simulator/src/main/java/org/opendc/compute/simulator/telemetry/GuestCpuStats.java b/opendc-compute/opendc-compute-simulator/src/main/java/org/opendc/compute/simulator/telemetry/GuestCpuStats.java new file mode 100644 index 00000000..ea37f5f2 --- /dev/null +++ b/opendc-compute/opendc-compute-simulator/src/main/java/org/opendc/compute/simulator/telemetry/GuestCpuStats.java @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2022 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.compute.simulator.telemetry; + +/** + * Statistics about the CPUs of a guest. + * + * @param activeTime The cumulative time (in seconds) that the CPUs of the guest were actively running. + * @param idleTime The cumulative time (in seconds) the CPUs of the guest were idle. + * @param stealTime The cumulative CPU time (in seconds) that the guest was ready to run, but not granted time by the host. + * @param lostTime The cumulative CPU time (in seconds) that was lost due to interference with other machines. + * @param capacity The available CPU capacity of the guest (in MHz). + * @param usage Amount of CPU resources (in MHz) actually used by the guest. + * @param utilization The utilization of the CPU resources (in %) relative to the total CPU capacity. + */ +public record GuestCpuStats( + long activeTime, + long idleTime, + long stealTime, + long lostTime, + float capacity, + float usage, + float utilization) {} diff --git a/opendc-compute/opendc-compute-simulator/src/main/java/org/opendc/compute/simulator/telemetry/GuestSystemStats.java b/opendc-compute/opendc-compute-simulator/src/main/java/org/opendc/compute/simulator/telemetry/GuestSystemStats.java new file mode 100644 index 00000000..0d51e223 --- /dev/null +++ b/opendc-compute/opendc-compute-simulator/src/main/java/org/opendc/compute/simulator/telemetry/GuestSystemStats.java @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2022 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.compute.simulator.telemetry; + +import java.time.Duration; +import java.time.Instant; + +/** + * System-level statistics of a guest. + * + * @param uptime The cumulative uptime of the guest since last boot (in ms). + * @param downtime The cumulative downtime of the guest since last boot (in ms). + * @param bootTime The time at which the guest booted. + */ +public record GuestSystemStats(Duration uptime, Duration downtime, Instant bootTime) {} diff --git a/opendc-compute/opendc-compute-simulator/src/main/java/org/opendc/compute/simulator/telemetry/HostCpuStats.java b/opendc-compute/opendc-compute-simulator/src/main/java/org/opendc/compute/simulator/telemetry/HostCpuStats.java new file mode 100644 index 00000000..3f2aab78 --- /dev/null +++ b/opendc-compute/opendc-compute-simulator/src/main/java/org/opendc/compute/simulator/telemetry/HostCpuStats.java @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2022 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.compute.simulator.telemetry; + +/** + * Statistics about the CPUs of a host. + * + * @param activeTime The cumulative time (in seconds) that the CPUs of the host were actively running. + * @param idleTime The cumulative time (in seconds) the CPUs of the host were idle. + * @param stealTime The cumulative CPU time (in seconds) that virtual machines were ready to run, but were not able to. + * @param lostTime The cumulative CPU time (in seconds) that was lost due to interference between virtual machines. + * @param capacity The available CPU capacity of the host (in MHz). + * @param demand Amount of CPU resources (in MHz) the guests would use if there were no CPU contention or CPU + * limits. + * @param usage Amount of CPU resources (in MHz) actually used by the host. + * @param utilization The utilization of the CPU resources (in %) relative to the total CPU capacity. + */ +public record HostCpuStats( + long activeTime, + long idleTime, + long stealTime, + long lostTime, + float capacity, + float demand, + float usage, + float utilization) {} diff --git a/opendc-compute/opendc-compute-simulator/src/main/java/org/opendc/compute/simulator/telemetry/HostSystemStats.java b/opendc-compute/opendc-compute-simulator/src/main/java/org/opendc/compute/simulator/telemetry/HostSystemStats.java new file mode 100644 index 00000000..353e62fa --- /dev/null +++ b/opendc-compute/opendc-compute-simulator/src/main/java/org/opendc/compute/simulator/telemetry/HostSystemStats.java @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2022 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.compute.simulator.telemetry; + +import java.time.Duration; +import java.time.Instant; + +/** + * System-level statistics of a host. + * + * @param uptime The cumulative uptime of the host since last boot (in ms). + * @param downtime The cumulative downtime of the host since last boot (in ms). + * @param bootTime The time at which the task started. + * @param powerDraw Instantaneous power draw of the system (in W). + * @param energyUsage The cumulative energy usage of the system (in J). + * @param guestsTerminated The number of guests that are in a terminated state. + * @param guestsRunning The number of guests that are in a running state. + * @param guestsError The number of guests that are in an error state. + * @param guestsInvalid The number of guests that are in an unknown state. + */ +public record HostSystemStats( + Duration uptime, + Duration downtime, + Instant bootTime, + float powerDraw, + float energyUsage, + int guestsTerminated, + int guestsRunning, + int guestsError, + int guestsInvalid) {} diff --git a/opendc-compute/opendc-compute-simulator/src/main/java/org/opendc/compute/simulator/telemetry/SchedulerStats.java b/opendc-compute/opendc-compute-simulator/src/main/java/org/opendc/compute/simulator/telemetry/SchedulerStats.java new file mode 100644 index 00000000..9d44a4b8 --- /dev/null +++ b/opendc-compute/opendc-compute-simulator/src/main/java/org/opendc/compute/simulator/telemetry/SchedulerStats.java @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2022 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.compute.simulator.telemetry; + +/** + * Statistics about the scheduling component of the [ComputeService]. + * + * @param hostsAvailable The number of hosts currently available for scheduling. + * @param hostsUnavailable The number of hosts unavailable for scheduling. + * @param attemptsSuccess Scheduling attempts that resulted into an allocation onto a host. + * @param attemptsFailure The number of failed scheduling attempt due to any reason + * @param tasksTotal The number of tasks registered with the service. + * @param tasksPending The number of tasks that are pending to be scheduled. + * @param tasksActive The number of tasks that are currently managed by the service and running. + */ +public record SchedulerStats( + int hostsAvailable, + int hostsUnavailable, + long attemptsSuccess, + long attemptsFailure, + int tasksTotal, + int tasksPending, + int tasksActive, + int tasksCompleted, + int tasksTerminated) {} diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/MutableServiceRegistry.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/MutableServiceRegistry.kt deleted file mode 100644 index ca72c910..00000000 --- a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/MutableServiceRegistry.kt +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Copyright (c) 2022 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.compute.simulator - -/** - * A mutable [ServiceRegistry]. - */ -public interface MutableServiceRegistry : ServiceRegistry { - /** - * Register [service] for the specified [name] in this registry. - * - * @param name The name of the service to register, which should follow the rules for domain names as defined by - * DNS. - * @param type The interface provided by the service. - * @param service A reference to the actual implementation of the service. - */ - public fun register( - name: String, - type: Class, - service: T, - ) - - /** - * Remove the service with [name] and [type] from this registry. - * - * @param name The name of the service to remove, which should follow the rules for domain names as defined by DNS. - * @param type The type of the service to remove. - */ - public fun remove( - name: String, - type: Class<*>, - ) - - /** - * Remove all services registered with [name]. - * - * @param name The name of the services to remove, which should follow the rules for domain names as defined by DNS. - */ - public fun remove(name: String) - - /** - * Create a copy of the registry. - */ - public override fun clone(): MutableServiceRegistry -} diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/ServiceRegistry.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/ServiceRegistry.kt index 5a4bced1..e2f6c9d0 100644 --- a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/ServiceRegistry.kt +++ b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/ServiceRegistry.kt @@ -23,26 +23,53 @@ package org.opendc.compute.simulator /** - * A read-only registry of services used during experiments to resolve services. - * - * The service registry is similar conceptually to the Domain Name System (DNS), which is a naming system used to - * identify computers reachable via the Internet. The service registry should be used in a similar fashion. + * Implementation of the [ServiceRegistry] interface. */ -public interface ServiceRegistry { - /** - * Lookup the service with the specified [name] and [type]. - * - * @param name The name of the service to resolve, which should follow the rules for domain names as defined by DNS. - * @param type The type of the service to resolve, identified by the interface that is implemented by the service. - * @return The service with specified [name] and implementing [type] or `null` if it does not exist. - */ +public class ServiceRegistry(private val registry: MutableMap, Any>> = mutableMapOf()) { public fun resolve( name: String, type: Class, - ): T? + ): T? { + val servicesForName = registry[name] ?: return null + + @Suppress("UNCHECKED_CAST") + return servicesForName[type] as T? + } + + public fun register( + name: String, + type: Class, + service: T, + ) { + val services = registry.computeIfAbsent(name) { mutableMapOf() } + + if (type in services) { + throw IllegalStateException("Duplicate service $type registered for name $name") + } + + services[type] = service + } + + public fun remove( + name: String, + type: Class<*>, + ) { + val services = registry[name] ?: return + services.remove(type) + } + + public fun remove(name: String) { + registry.remove(name) + } + + public fun clone(): ServiceRegistry { + val res = mutableMapOf, Any>>() + registry.mapValuesTo(res) { (_, v) -> v.toMutableMap() } + return ServiceRegistry(res) + } - /** - * Create a copy of the registry. - */ - public fun clone(): ServiceRegistry + override fun toString(): String { + val entries = registry.map { "${it.key}=${it.value}" }.joinToString() + return "ServiceRegistry{$entries}" + } } diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/ServiceRegistryImpl.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/ServiceRegistryImpl.kt deleted file mode 100644 index bf3ee43f..00000000 --- a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/ServiceRegistryImpl.kt +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Copyright (c) 2022 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.compute.simulator - -/** - * Implementation of the [MutableServiceRegistry] interface. - */ -internal class ServiceRegistryImpl(private val registry: MutableMap, Any>> = mutableMapOf()) : - MutableServiceRegistry { - override fun resolve( - name: String, - type: Class, - ): T? { - val servicesForName = registry[name] ?: return null - - @Suppress("UNCHECKED_CAST") - return servicesForName[type] as T? - } - - override fun register( - name: String, - type: Class, - service: T, - ) { - val services = registry.computeIfAbsent(name) { mutableMapOf() } - - if (type in services) { - throw IllegalStateException("Duplicate service $type registered for name $name") - } - - services[type] = service - } - - override fun remove( - name: String, - type: Class<*>, - ) { - val services = registry[name] ?: return - services.remove(type) - } - - override fun remove(name: String) { - registry.remove(name) - } - - override fun clone(): MutableServiceRegistry { - val res = mutableMapOf, Any>>() - registry.mapValuesTo(res) { (_, v) -> v.toMutableMap() } - return ServiceRegistryImpl(res) - } - - override fun toString(): String { - val entries = registry.map { "${it.key}=${it.value}" }.joinToString() - return "ServiceRegistry{$entries}" - } -} diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/SimHost.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/SimHost.kt deleted file mode 100644 index e681403c..00000000 --- a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/SimHost.kt +++ /dev/null @@ -1,378 +0,0 @@ -/* - * Copyright (c) 2020 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.compute.simulator - -import org.opendc.compute.api.Flavor -import org.opendc.compute.api.Task -import org.opendc.compute.api.TaskState -import org.opendc.compute.service.driver.Host -import org.opendc.compute.service.driver.HostListener -import org.opendc.compute.service.driver.HostModel -import org.opendc.compute.service.driver.HostState -import org.opendc.compute.service.driver.telemetry.GuestCpuStats -import org.opendc.compute.service.driver.telemetry.GuestSystemStats -import org.opendc.compute.service.driver.telemetry.HostCpuStats -import org.opendc.compute.service.driver.telemetry.HostSystemStats -import org.opendc.compute.simulator.internal.DefaultWorkloadMapper -import org.opendc.compute.simulator.internal.Guest -import org.opendc.compute.simulator.internal.GuestListener -import org.opendc.simulator.compute.SimBareMetalMachine -import org.opendc.simulator.compute.SimMachineContext -import org.opendc.simulator.compute.kernel.SimHypervisor -import org.opendc.simulator.compute.model.MachineModel -import org.opendc.simulator.compute.model.MemoryUnit -import org.opendc.simulator.compute.workload.SimWorkload -import org.opendc.simulator.compute.workload.SimWorkloads -import java.time.Duration -import java.time.Instant -import java.time.InstantSource -import java.util.UUID -import java.util.function.Supplier - -/** - * A [Host] implementation that simulates virtual machines on a physical machine using [SimHypervisor]. - * - * @param uid The unique identifier of the host. - * @param name The name of the host. - * @param meta The metadata of the host. - * @param clock The (virtual) clock used to track time. - * @param machine The [SimBareMetalMachine] on which the host runs. - * @param hypervisor The [SimHypervisor] to run on top of the machine. - * @param mapper A [SimWorkloadMapper] to map a [Task] to a [SimWorkload]. - * @param bootModel A [Supplier] providing the [SimWorkload] to execute during the boot procedure of the hypervisor. - * @param optimize A flag to indicate to optimize the machine models of the virtual machines. - */ -public class SimHost( - private val uid: UUID, - private val name: String, - private val meta: Map, - private val clock: InstantSource, - private val machine: SimBareMetalMachine, - private val hypervisor: SimHypervisor, - private val mapper: SimWorkloadMapper = DefaultWorkloadMapper, - private val bootModel: Supplier = Supplier { null }, - private val optimize: Boolean = false, -) : Host, AutoCloseable { - /** - * The event listeners registered with this host. - */ - private val listeners = mutableListOf() - - /** - * The virtual machines running on the hypervisor. - */ - private val guests = HashMap() - private val localGuests = mutableListOf() - - private var localState: HostState = HostState.DOWN - set(value) { - if (value != field) { - listeners.forEach { it.onStateChanged(this, value) } - } - field = value - } - - private val model: HostModel = - HostModel( - machine.model.cpu.totalCapacity, - machine.model.cpu.coreCount, - machine.model.memory.size, - ) - - /** - * The [GuestListener] that listens for guest events. - */ - private val guestListener = - object : GuestListener { - override fun onStart(guest: Guest) { - listeners.forEach { it.onStateChanged(this@SimHost, guest.task, guest.state) } - } - - override fun onStop(guest: Guest) { - listeners.forEach { it.onStateChanged(this@SimHost, guest.task, guest.state) } - } - } - - init { - launch() - } - - override fun getUid(): UUID { - return uid - } - - override fun getName(): String { - return name - } - - override fun getModel(): HostModel { - return model - } - - override fun getMeta(): Map { - return meta - } - - override fun getState(): HostState { - return localState - } - - override fun getInstances(): Set { - return guests.keys - } - - override fun canFit(task: Task): Boolean { - val sufficientMemory = model.memoryCapacity >= task.flavor.memorySize - val enoughCpus = model.coreCount >= task.flavor.coreCount - val canFit = hypervisor.canFit(task.flavor.toMachineModel()) - - return sufficientMemory && enoughCpus && canFit - } - - override fun spawn(task: Task) { - guests.computeIfAbsent(task) { key -> - require(canFit(key)) { "Task does not fit" } - - val machine = hypervisor.newMachine(key.flavor.toMachineModel()) - val newGuest = - Guest( - clock, - this, - hypervisor, - mapper, - guestListener, - task, - machine, - ) - - localGuests.add(newGuest) - newGuest - } - } - - override fun contains(task: Task): Boolean { - return task in guests - } - - override fun start(task: Task) { - val guest = requireNotNull(guests[task]) { "Unknown task ${task.uid} at host $uid" } - guest.start() - } - - override fun stop(task: Task) { - val guest = requireNotNull(guests[task]) { "Unknown task ${task.uid} at host $uid" } - guest.stop() - } - - override fun delete(task: Task) { - val guest = guests[task] ?: return - guest.delete() - - guests.remove(task) - localGuests.remove(guest) - } - - override fun addListener(listener: HostListener) { - listeners.add(listener) - } - - override fun removeListener(listener: HostListener) { - listeners.remove(listener) - } - - override fun close() { - reset(HostState.DOWN) - machine.cancel() - } - - override fun getSystemStats(): HostSystemStats { - updateUptime() - - var terminated = 0 - var running = 0 - var error = 0 - var invalid = 0 - - val guests = localGuests.listIterator() - for (guest in guests) { - when (guest.state) { - TaskState.TERMINATED -> terminated++ - TaskState.RUNNING -> running++ - TaskState.ERROR -> error++ - TaskState.DELETED -> { - // Remove guests that have been deleted - this.guests.remove(guest.task) - guests.remove() - } - else -> invalid++ - } - } - - return HostSystemStats( - Duration.ofMillis(localUptime), - Duration.ofMillis(localDowntime), - localBootTime, - machine.psu.powerDraw, - machine.psu.energyUsage, - terminated, - running, - error, - invalid, - ) - } - - override fun getSystemStats(task: Task): GuestSystemStats { - val guest = requireNotNull(guests[task]) { "Unknown task ${task.uid} at host $uid" } - return guest.getSystemStats() - } - - override fun getCpuStats(): HostCpuStats { - val counters = hypervisor.counters - counters.sync() - - return HostCpuStats( - counters.cpuActiveTime, - counters.cpuIdleTime, - counters.cpuStealTime, - counters.cpuLostTime, - hypervisor.cpuCapacity, - hypervisor.cpuDemand, - hypervisor.cpuUsage, - hypervisor.cpuUsage / localCpuLimit, - ) - } - - override fun getCpuStats(task: Task): GuestCpuStats { - val guest = requireNotNull(guests[task]) { "Unknown task ${task.uid} at host $uid" } - return guest.getCpuStats() - } - - override fun hashCode(): Int = uid.hashCode() - - override fun equals(other: Any?): Boolean { - return other is SimHost && uid == other.uid - } - - override fun toString(): String = "SimHost[uid=$uid,name=$name,model=$model]" - - public fun fail() { - reset(HostState.ERROR) - - for (guest in localGuests) { - guest.fail() - } - } - - public fun recover() { - updateUptime() - - launch() - } - - /** - * The [SimMachineContext] that represents the machine running the hypervisor. - */ - private var ctx: SimMachineContext? = null - - /** - * Launch the hypervisor. - */ - private fun launch() { - check(ctx == null) { "Concurrent hypervisor running" } - - val bootWorkload = bootModel.get() - val hypervisor = hypervisor - val hypervisorWorkload = - object : SimWorkload by hypervisor { - override fun onStart(ctx: SimMachineContext) { - try { - localBootTime = clock.instant() - localState = HostState.UP - hypervisor.onStart(ctx) - - // Recover the guests that were running on the hypervisor. - for (guest in localGuests) { - guest.recover() - } - } catch (cause: Throwable) { - localState = HostState.ERROR - throw cause - } - } - } - - val workload = if (bootWorkload != null) SimWorkloads.chain(bootWorkload, hypervisorWorkload) else hypervisorWorkload - - // Launch hypervisor onto machine - ctx = - machine.startWorkload(workload, emptyMap()) { cause -> - localState = if (cause != null) HostState.ERROR else HostState.DOWN - ctx = null - } - } - - /** - * Reset the machine. - */ - private fun reset(state: HostState) { - updateUptime() - - // Stop the hypervisor - ctx?.shutdown() - localState = state - } - - /** - * Convert flavor to machine model. - */ - private fun Flavor.toMachineModel(): MachineModel { - return MachineModel(machine.model.cpu, MemoryUnit("Generic", "Generic", 3200.0, memorySize)) - } - - private var localLastReport = clock.millis() - private var localUptime = 0L - private var localDowntime = 0L - private var localBootTime: Instant? = null - private val localCpuLimit = machine.model.cpu.totalCapacity - - /** - * Helper function to track the uptime of a machine. - */ - private fun updateUptime() { - val now = clock.millis() - val duration = now - localLastReport - localLastReport = now - - if (localState == HostState.UP) { - localUptime += duration - } else if (localState == HostState.ERROR) { - // Only increment downtime if the machine is in a failure state - localDowntime += duration - } - - val guests = localGuests - for (i in guests.indices) { - guests[i].updateUptime() - } - } -} diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/SimMetaWorkloadMapper.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/SimMetaWorkloadMapper.kt deleted file mode 100644 index 907f6acd..00000000 --- a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/SimMetaWorkloadMapper.kt +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright (c) 2021 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.compute.simulator - -import org.opendc.compute.api.Task -import org.opendc.simulator.compute.workload.SimWorkload - -/** - * A [SimWorkloadMapper] that maps a [Task] to a workload via the meta-data. - */ -public class SimMetaWorkloadMapper(private val key: String = "workload") : SimWorkloadMapper { - override fun createWorkload(task: Task): SimWorkload { - return requireNotNull(task.meta[key] ?: task.image.meta[key]) as SimWorkload - } -} diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/SimWorkloadMapper.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/SimWorkloadMapper.kt deleted file mode 100644 index a85091a0..00000000 --- a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/SimWorkloadMapper.kt +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2021 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.compute.simulator - -import org.opendc.compute.api.Image -import org.opendc.compute.api.Task -import org.opendc.simulator.compute.workload.SimWorkload - -/** - * A [SimWorkloadMapper] is responsible for mapping a [Task] and [Image] to a [SimWorkload] that can be simulated. - */ -public fun interface SimWorkloadMapper { - /** - * Map the specified [task] to a [SimWorkload] that can be simulated. - */ - public fun createWorkload(task: Task): SimWorkload -} diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/TaskWatcher.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/TaskWatcher.kt new file mode 100644 index 00000000..9fe4dff5 --- /dev/null +++ b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/TaskWatcher.kt @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.compute.simulator + +import org.opendc.compute.api.TaskState +import org.opendc.compute.simulator.service.ServiceTask + +/** + * An interface used to watch the state of [ServiceTask] instances. + */ +public interface TaskWatcher { + /** + * This method is invoked when the state of a [ServiceTask] changes. + * + * @param task The task whose state has changed. + * @param newState The new state of the task. + */ + public fun onStateChanged( + task: ServiceTask, + newState: TaskState, + ) {} +} diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/host/SimHost.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/host/SimHost.kt new file mode 100644 index 00000000..31ff384c --- /dev/null +++ b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/host/SimHost.kt @@ -0,0 +1,369 @@ +/* + * Copyright (c) 2020 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.compute.simulator.host + +import org.opendc.compute.api.Flavor +import org.opendc.compute.api.TaskState +import org.opendc.compute.simulator.internal.Guest +import org.opendc.compute.simulator.internal.GuestListener +import org.opendc.compute.simulator.service.ServiceTask +import org.opendc.compute.simulator.telemetry.GuestCpuStats +import org.opendc.compute.simulator.telemetry.GuestSystemStats +import org.opendc.compute.simulator.telemetry.HostCpuStats +import org.opendc.compute.simulator.telemetry.HostSystemStats +import org.opendc.simulator.compute.cpu.CpuPowerModel +import org.opendc.simulator.compute.machine.SimMachine +import org.opendc.simulator.compute.models.MachineModel +import org.opendc.simulator.compute.models.MemoryUnit +import org.opendc.simulator.engine.FlowGraph +import java.time.Duration +import java.time.Instant +import java.time.InstantSource +import java.util.UUID + +/** + * A [Host] implementation that simulates virtual machines on a physical machine. + * + * @param uid The unique identifier of the host. + * @param name The name of the host. + * @param meta The metadata of the host. + * @param clock The (virtual) clock used to track time. + * @param graph The Flow Graph that the Host is part of + * @param machineModel The static model of the host + * @param powerModel The static powerModel of the CPU TODO: can this be combined with machinemodel? + * @constructor Create empty Sim host + */ +public class SimHost( + private val uid: UUID, + private val name: String, + private val meta: Map, + private val clock: InstantSource, + private val graph: FlowGraph, + private val machineModel: MachineModel, + private val powerModel: CpuPowerModel, +) : AutoCloseable { + /** + * The event listeners registered with this host. + */ + private val hostListeners = mutableListOf() + + /** + * The virtual machines running on the hypervisor. + */ + private val taskToGuestMap = HashMap() + private val guests = mutableListOf() + + private var hostState: HostState = HostState.DOWN + set(value) { + if (value != field) { + hostListeners.forEach { it.onStateChanged(this, value) } + } + field = value + } + + private val model: HostModel = + HostModel( + machineModel.cpu.totalCapacity, + machineModel.cpu.coreCount, + machineModel.memory.size, + ) + + private var simMachine: SimMachine? = null + + /** + * The [GuestListener] that listens for guest events. + */ + private val guestListener = + object : GuestListener { + override fun onStart(guest: Guest) { + hostListeners.forEach { it.onStateChanged(this@SimHost, guest.task, guest.state) } + } + + override fun onStop(guest: Guest) { + hostListeners.forEach { it.onStateChanged(this@SimHost, guest.task, guest.state) } + } + } + + private var lastReport = clock.millis() + private var totalUptime = 0L + private var totalDowntime = 0L + private var bootTime: Instant? = null + private val cpuLimit = machineModel.cpu.totalCapacity + + init { + launch() + } + + /** + * Launch the hypervisor. + */ + private fun launch() { + bootTime = this.clock.instant() + hostState = HostState.UP + + if (this.simMachine != null) { + return + } + + this.simMachine = + SimMachine( + this.graph, + this.machineModel, + this.powerModel, + ) { cause -> + hostState = if (cause != null) HostState.ERROR else HostState.DOWN + } + } + + override fun close() { + reset(HostState.DOWN) + } + + public fun fail() { + reset(HostState.ERROR) + + // Fail the guest and delete them + // This weird loop is the only way I have been able to make it work. + while (guests.size > 0) { + val guest = guests[0] + guest.fail() + this.delete(guest.task) + } + } + + public fun recover() { + updateUptime() + + launch() + } + + /** + * Reset the machine. + */ + private fun reset(state: HostState) { + updateUptime() + + // Stop the hypervisor + hostState = state + } + + public fun getUid(): UUID { + return uid + } + + public fun getName(): String { + return name + } + + public fun getModel(): HostModel { + return model + } + + public fun getMeta(): Map { + return meta + } + + public fun getState(): HostState { + return hostState + } + + public fun getInstances(): Set { + return taskToGuestMap.keys + } + + public fun getGuests(): List { + return this.guests + } + + public fun canFit(task: ServiceTask): Boolean { + val sufficientMemory = model.memoryCapacity >= task.flavor.memorySize + val enoughCpus = model.coreCount >= task.flavor.coreCount + val canFit = simMachine!!.canFit(task.flavor.toMachineModel()) + + return sufficientMemory && enoughCpus && canFit + } + + /** + * Spawn A Virtual machine that run the Task and put this Task as a Guest on it + * + * @param task + */ + public fun spawn(task: ServiceTask) { + assert(simMachine != null) { "Tried start task $task while no SimMachine is active" } + + require(canFit(task)) { "Task does not fit" } + + val newGuest = + Guest( + clock, + this, + guestListener, + task, + simMachine!!, + ) + + guests.add(newGuest) + newGuest.start() + + taskToGuestMap.computeIfAbsent(task) { newGuest } + } + + public fun contains(task: ServiceTask): Boolean { + return task in taskToGuestMap + } + + public fun start(task: ServiceTask) { + val guest = requireNotNull(taskToGuestMap[task]) { "Unknown task ${task.uid} at host $uid" } + guest.start() + } + + public fun stop(task: ServiceTask) { + val guest = requireNotNull(taskToGuestMap[task]) { "Unknown task ${task.uid} at host $uid" } + guest.stop() + } + + public fun delete(task: ServiceTask) { + val guest = taskToGuestMap[task] ?: return + guest.delete() + + taskToGuestMap.remove(task) + guests.remove(guest) + task.host = null + } + + public fun removeTask(task: ServiceTask) { + val guest = taskToGuestMap[task] ?: return + guest.delete() + + taskToGuestMap.remove(task) + guests.remove(guest) + } + + public fun addListener(listener: HostListener) { + hostListeners.add(listener) + } + + public fun removeListener(listener: HostListener) { + hostListeners.remove(listener) + } + + public fun getSystemStats(): HostSystemStats { + updateUptime() + this.simMachine!!.psu.updateCounters() + + var terminated = 0 + var running = 0 + var failed = 0 + var invalid = 0 + var completed = 0 + + val guests = guests.listIterator() + for (guest in guests) { + when (guest.state) { + TaskState.RUNNING -> running++ + TaskState.COMPLETED, TaskState.FAILED, TaskState.TERMINATED -> { + failed++ + // Remove guests that have been deleted + this.taskToGuestMap.remove(guest.task) + guests.remove() + } + else -> invalid++ + } + } + + return HostSystemStats( + Duration.ofMillis(totalUptime), + Duration.ofMillis(totalDowntime), + bootTime, + simMachine!!.psu.powerDraw, + simMachine!!.psu.energyUsage, + terminated, + running, + failed, + invalid, + ) + } + + public fun getSystemStats(task: ServiceTask): GuestSystemStats { + val guest = requireNotNull(taskToGuestMap[task]) { "Unknown task ${task.uid} at host $uid" } + return guest.getSystemStats() + } + + public fun getCpuStats(): HostCpuStats { + simMachine!!.cpu.updateCounters(this.clock.millis()) + + val counters = simMachine!!.performanceCounters + + return HostCpuStats( + counters.cpuActiveTime, + counters.cpuIdleTime, + counters.cpuStealTime, + counters.cpuLostTime, + counters.cpuCapacity, + counters.cpuDemand, + counters.cpuSupply, + counters.cpuSupply / cpuLimit, + ) + } + + public fun getCpuStats(task: ServiceTask): GuestCpuStats { + val guest = requireNotNull(taskToGuestMap[task]) { "Unknown task ${task.uid} at host $uid" } + return guest.getCpuStats() + } + + override fun hashCode(): Int = uid.hashCode() + + override fun equals(other: Any?): Boolean { + return other is SimHost && uid == other.uid + } + + override fun toString(): String = "SimHost[uid=$uid,name=$name,model=$model]" + + /** + * Convert flavor to machine model. + */ + private fun Flavor.toMachineModel(): MachineModel { + return MachineModel(simMachine!!.machineModel.cpu, MemoryUnit("Generic", "Generic", 3200.0, memorySize)) + } + + /** + * Helper function to track the uptime of a machine. + */ + private fun updateUptime() { + val now = clock.millis() + val duration = now - lastReport + lastReport = now + + if (hostState == HostState.UP) { + totalUptime += duration + } else if (hostState == HostState.ERROR) { + // Only increment downtime if the machine is in a failure state + totalDowntime += duration + } + + val guests = guests + for (i in guests.indices) { + guests[i].updateUptime() + } + } +} diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/internal/DefaultWorkloadMapper.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/internal/DefaultWorkloadMapper.kt deleted file mode 100644 index 412da37f..00000000 --- a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/internal/DefaultWorkloadMapper.kt +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright (c) 2022 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.compute.simulator.internal - -import org.opendc.compute.api.Task -import org.opendc.compute.simulator.SimMetaWorkloadMapper -import org.opendc.compute.simulator.SimWorkloadMapper -import org.opendc.simulator.compute.workload.SimWorkload -import org.opendc.simulator.compute.workload.SimWorkloads -import java.time.Duration - -/** - * A [SimWorkloadMapper] to introduces a boot delay of 1 ms. This object exists to retain the old behavior while - * introducing the possibility of adding custom boot delays. - */ -internal object DefaultWorkloadMapper : SimWorkloadMapper { - private val delegate = SimMetaWorkloadMapper() - - override fun createWorkload(task: Task): SimWorkload { - val workload = delegate.createWorkload(task) - - // FIXME: look at connecting this to frontend. This does currently not work correctly - val bootWorkload = SimWorkloads.runtime(Duration.ofMillis(0), 1.0, 0L, 0L) - return SimWorkloads.chain(bootWorkload, workload) - } -} diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/internal/Guest.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/internal/Guest.kt index cf6c146a..3a923222 100644 --- a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/internal/Guest.kt +++ b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/internal/Guest.kt @@ -23,15 +23,16 @@ package org.opendc.compute.simulator.internal import mu.KotlinLogging -import org.opendc.compute.api.Task import org.opendc.compute.api.TaskState -import org.opendc.compute.service.driver.telemetry.GuestCpuStats -import org.opendc.compute.service.driver.telemetry.GuestSystemStats -import org.opendc.compute.simulator.SimHost -import org.opendc.compute.simulator.SimWorkloadMapper -import org.opendc.simulator.compute.SimMachineContext -import org.opendc.simulator.compute.kernel.SimHypervisor -import org.opendc.simulator.compute.workload.SimWorkload +import org.opendc.compute.simulator.host.SimHost +import org.opendc.compute.simulator.service.ServiceTask +import org.opendc.compute.simulator.telemetry.GuestCpuStats +import org.opendc.compute.simulator.telemetry.GuestSystemStats +import org.opendc.simulator.compute.machine.SimMachine +import org.opendc.simulator.compute.machine.VirtualMachine +import org.opendc.simulator.compute.workload.ChainWorkload +import org.opendc.simulator.compute.workload.TraceFragment +import org.opendc.simulator.compute.workload.TraceWorkload import java.time.Duration import java.time.Instant import java.time.InstantSource @@ -39,14 +40,12 @@ import java.time.InstantSource /** * A virtual machine instance that is managed by a [SimHost]. */ -internal class Guest( +public class Guest( private val clock: InstantSource, - val host: SimHost, - private val hypervisor: SimHypervisor, - private val mapper: SimWorkloadMapper, + public val host: SimHost, private val listener: GuestListener, - val task: Task, - val machine: SimHypervisor.SimVirtualMachine, + public val task: ServiceTask, + public val simMachine: SimMachine, ) { /** * The state of the [Guest]. @@ -54,50 +53,133 @@ internal class Guest( * [TaskState.PROVISIONING] is an invalid value for a guest, since it applies before the host is selected for * a task. */ - var state: TaskState = TaskState.TERMINATED + public var state: TaskState = TaskState.CREATED private set + /** + * The [VirtualMachine] on which the task is currently running + */ + public var virtualMachine: VirtualMachine? = null + + private var uptime = 0L + private var downtime = 0L + private var lastReport = clock.millis() + private var bootTime: Instant? = null + private val cpuLimit = simMachine.cpu.cpuModel.totalCapacity + /** * Start the guest. */ - fun start() { + public fun start() { when (state) { - TaskState.TERMINATED, TaskState.ERROR -> { + TaskState.CREATED, TaskState.FAILED -> { LOGGER.info { "User requested to start task ${task.uid}" } doStart() } TaskState.RUNNING -> return - TaskState.DELETED -> { - LOGGER.warn { "User tried to start deleted task" } - throw IllegalArgumentException("Task is deleted") + TaskState.COMPLETED, TaskState.TERMINATED -> { + LOGGER.warn { "User tried to start a finished task" } + throw IllegalArgumentException("Task is already finished") } else -> assert(false) { "Invalid state transition" } } } + /** + * Launch the guest on the simulated Virtual machine + */ + private fun doStart() { + assert(virtualMachine == null) { "Concurrent job is already running" } + + onStart() + + val bootworkload = + TraceWorkload( + ArrayList( + listOf( + TraceFragment( + 1000000L, + 100000.0, + 1, + ), + ), + ), + 0, + 0, + 0.0, + ) + val newChainWorkload = + ChainWorkload( + ArrayList(listOf(task.workload)), + task.workload.checkpointInterval, + task.workload.checkpointDuration, + task.workload.checkpointIntervalScaling, + ) + + virtualMachine = + simMachine.startWorkload(newChainWorkload) { cause -> + onStop(if (cause != null) TaskState.FAILED else TaskState.COMPLETED) + } + } + + /** + * This method is invoked when the guest was started on the host and has booted into a running state. + */ + private fun onStart() { + bootTime = clock.instant() + state = TaskState.RUNNING + listener.onStart(this) + } + /** * Stop the guest. */ - fun stop() { + public fun stop() { when (state) { - TaskState.RUNNING -> doStop(TaskState.TERMINATED) - TaskState.ERROR -> doRecover() - TaskState.TERMINATED, TaskState.DELETED -> return + TaskState.RUNNING -> doStop(TaskState.COMPLETED) + TaskState.FAILED -> state = TaskState.TERMINATED + TaskState.COMPLETED, TaskState.TERMINATED -> return else -> assert(false) { "Invalid state transition" } } } + /** + * Attempt to stop the task and put it into [target] state. + */ + private fun doStop(target: TaskState) { + assert(virtualMachine != null) { "Invalid job state" } + val virtualMachine = this.virtualMachine ?: return + if (target == TaskState.FAILED) { + virtualMachine.shutdown(Exception("Task has failed")) + } else { + virtualMachine.shutdown() + } + + this.virtualMachine = null + + this.state = target + } + + /** + * This method is invoked when the guest stopped. + */ + private fun onStop(target: TaskState) { + updateUptime() + + state = target + listener.onStop(this) + } + /** * Delete the guest. * * This operation will stop the guest if it is running on the host and remove all resources associated with the * guest. */ - fun delete() { + public fun delete() { stop() - state = TaskState.DELETED - hypervisor.removeMachine(machine) + state = TaskState.FAILED } /** @@ -105,19 +187,19 @@ internal class Guest( * * This operation forcibly stops the guest and puts the task into an error state. */ - fun fail() { + public fun fail() { if (state != TaskState.RUNNING) { return } - doStop(TaskState.ERROR) + doStop(TaskState.FAILED) } /** * Recover the guest if it is in an error state. */ - fun recover() { - if (state != TaskState.ERROR) { + public fun recover() { + if (state != TaskState.FAILED) { return } @@ -127,117 +209,46 @@ internal class Guest( /** * Obtain the system statistics of this guest. */ - fun getSystemStats(): GuestSystemStats { + public fun getSystemStats(): GuestSystemStats { updateUptime() return GuestSystemStats( - Duration.ofMillis(localUptime), - Duration.ofMillis(localDowntime), - localBootTime, + Duration.ofMillis(uptime), + Duration.ofMillis(downtime), + bootTime, ) } /** * Obtain the CPU statistics of this guest. */ - fun getCpuStats(): GuestCpuStats { - val counters = machine.counters - counters.sync() + public fun getCpuStats(): GuestCpuStats { + virtualMachine!!.updateCounters(this.clock.millis()) + val counters = virtualMachine!!.performanceCounters return GuestCpuStats( counters.cpuActiveTime / 1000L, counters.cpuIdleTime / 1000L, counters.cpuStealTime / 1000L, counters.cpuLostTime / 1000L, - machine.cpuCapacity, - machine.cpuUsage, - machine.cpuUsage / localCpuLimit, + counters.cpuCapacity, + counters.cpuSupply, + counters.cpuSupply / cpuLimit, ) } - /** - * The [SimMachineContext] representing the current active virtual machine instance or `null` if no virtual machine - * is active. - */ - private var ctx: SimMachineContext? = null - - /** - * Launch the guest on the simulated - */ - private fun doStart() { - assert(ctx == null) { "Concurrent job running" } - - onStart() - - val workload: SimWorkload = mapper.createWorkload(task) - workload.setOffset(clock.millis()) - val meta = mapOf("driver" to host, "task" to task) + task.meta - ctx = - machine.startWorkload(workload, meta) { cause -> - onStop(if (cause != null) TaskState.ERROR else TaskState.TERMINATED) - ctx = null - } - } - - /** - * Attempt to stop the task and put it into [target] state. - */ - private fun doStop(target: TaskState) { - assert(ctx != null) { "Invalid job state" } - val ctx = ctx ?: return - if (target == TaskState.ERROR) { - ctx.shutdown(Exception("Stopped because of ERROR")) - } else { - ctx.shutdown() - } - - state = target - } - - /** - * Attempt to recover from an error state. - */ - private fun doRecover() { - state = TaskState.TERMINATED - } - - /** - * This method is invoked when the guest was started on the host and has booted into a running state. - */ - private fun onStart() { - localBootTime = clock.instant() - state = TaskState.RUNNING - listener.onStart(this) - } - - /** - * This method is invoked when the guest stopped. - */ - private fun onStop(target: TaskState) { - updateUptime() - - state = target - listener.onStop(this) - } - - private var localUptime = 0L - private var localDowntime = 0L - private var localLastReport = clock.millis() - private var localBootTime: Instant? = null - private val localCpuLimit = machine.model.cpu.totalCapacity - /** * Helper function to track the uptime and downtime of the guest. */ - fun updateUptime() { + public fun updateUptime() { val now = clock.millis() - val duration = now - localLastReport - localLastReport = now + val duration = now - lastReport + lastReport = now if (state == TaskState.RUNNING) { - localUptime += duration - } else if (state == TaskState.ERROR) { - localDowntime += duration + uptime += duration + } else if (state == TaskState.FAILED) { + downtime += duration } } diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/internal/GuestListener.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/internal/GuestListener.kt index e6d0fdad..895d78f9 100644 --- a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/internal/GuestListener.kt +++ b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/internal/GuestListener.kt @@ -25,14 +25,14 @@ package org.opendc.compute.simulator.internal /** * Helper interface to listen for [Guest] events. */ -internal interface GuestListener { +public interface GuestListener { /** * This method is invoked when the guest machine is running. */ - fun onStart(guest: Guest) + public fun onStart(guest: Guest) /** * This method is invoked when the guest machine is stopped. */ - fun onStop(guest: Guest) + public fun onStop(guest: Guest) } diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/provisioner/ComputeMonitorProvisioningStep.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/provisioner/ComputeMonitorProvisioningStep.kt index f1123742..f295f522 100644 --- a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/provisioner/ComputeMonitorProvisioningStep.kt +++ b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/provisioner/ComputeMonitorProvisioningStep.kt @@ -23,9 +23,9 @@ package org.opendc.compute.simulator.provisioner import org.opendc.compute.carbon.CarbonTrace -import org.opendc.compute.service.ComputeService -import org.opendc.compute.telemetry.ComputeMetricReader -import org.opendc.compute.telemetry.ComputeMonitor +import org.opendc.compute.simulator.service.ComputeService +import org.opendc.compute.simulator.telemetry.ComputeMetricReader +import org.opendc.compute.simulator.telemetry.ComputeMonitor import java.time.Duration /** @@ -44,7 +44,15 @@ public class ComputeMonitorProvisioningStep( requireNotNull( ctx.registry.resolve(serviceDomain, ComputeService::class.java), ) { "Compute service $serviceDomain does not exist" } - val metricReader = ComputeMetricReader(ctx.dispatcher, service, monitor, exportInterval, startTime, carbonTrace) + val metricReader = + ComputeMetricReader( + ctx.dispatcher, + service, + monitor, + exportInterval, + startTime, + carbonTrace, + ) return AutoCloseable { metricReader.close() } } } diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/provisioner/ComputeServiceProvisioningStep.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/provisioner/ComputeServiceProvisioningStep.kt index 645c9d46..6bdb131f 100644 --- a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/provisioner/ComputeServiceProvisioningStep.kt +++ b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/provisioner/ComputeServiceProvisioningStep.kt @@ -22,8 +22,8 @@ package org.opendc.compute.simulator.provisioner -import org.opendc.compute.service.ComputeService -import org.opendc.compute.service.scheduler.ComputeScheduler +import org.opendc.compute.simulator.scheduler.ComputeScheduler +import org.opendc.compute.simulator.service.ComputeService import java.time.Duration /** diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/provisioner/ComputeSteps.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/provisioner/ComputeSteps.kt index afde8219..07db3d26 100644 --- a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/provisioner/ComputeSteps.kt +++ b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/provisioner/ComputeSteps.kt @@ -25,9 +25,8 @@ package org.opendc.compute.simulator.provisioner import org.opendc.compute.carbon.CarbonTrace -import org.opendc.compute.service.ComputeService -import org.opendc.compute.service.scheduler.ComputeScheduler -import org.opendc.compute.telemetry.ComputeMonitor +import org.opendc.compute.simulator.scheduler.ComputeScheduler +import org.opendc.compute.simulator.telemetry.ComputeMonitor import org.opendc.compute.topology.specs.HostSpec import java.time.Duration @@ -41,7 +40,7 @@ import java.time.Duration public fun setupComputeService( serviceDomain: String, scheduler: (ProvisioningContext) -> ComputeScheduler, - schedulingQuantum: Duration = Duration.ofMinutes(5), + schedulingQuantum: Duration = Duration.ofSeconds(1), maxNumFailures: Int = 10, ): ProvisioningStep { return ComputeServiceProvisioningStep(serviceDomain, scheduler, schedulingQuantum, maxNumFailures) @@ -76,7 +75,6 @@ public fun registerComputeMonitor( public fun setupHosts( serviceDomain: String, specs: List, - optimize: Boolean = false, ): ProvisioningStep { - return HostsProvisioningStep(serviceDomain, specs, optimize) + return HostsProvisioningStep(serviceDomain, specs) } diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/provisioner/HostsProvisioningStep.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/provisioner/HostsProvisioningStep.kt index a80be634..19674d5e 100644 --- a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/provisioner/HostsProvisioningStep.kt +++ b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/provisioner/HostsProvisioningStep.kt @@ -22,13 +22,10 @@ package org.opendc.compute.simulator.provisioner -import org.opendc.compute.service.ComputeService -import org.opendc.compute.simulator.SimHost +import org.opendc.compute.simulator.host.SimHost +import org.opendc.compute.simulator.service.ComputeService import org.opendc.compute.topology.specs.HostSpec -import org.opendc.simulator.compute.SimBareMetalMachine -import org.opendc.simulator.compute.kernel.SimHypervisor -import org.opendc.simulator.flow2.FlowEngine -import java.util.SplittableRandom +import org.opendc.simulator.engine.FlowEngine /** * A [ProvisioningStep] that provisions a list of hosts for a [ComputeService]. @@ -40,30 +37,27 @@ import java.util.SplittableRandom public class HostsProvisioningStep internal constructor( private val serviceDomain: String, private val specs: List, - private val optimize: Boolean, ) : ProvisioningStep { override fun apply(ctx: ProvisioningContext): AutoCloseable { val service = requireNotNull( ctx.registry.resolve(serviceDomain, ComputeService::class.java), ) { "Compute service $serviceDomain does not exist" } - val engine = FlowEngine.create(ctx.dispatcher) - val graph = engine.newGraph() val hosts = mutableSetOf() - for (spec in specs) { - val machine = SimBareMetalMachine.create(graph, spec.model, spec.psuFactory) - val hypervisor = SimHypervisor.create(spec.multiplexerFactory, SplittableRandom(ctx.seeder.nextLong())) + val flowEngine = FlowEngine.create(ctx.dispatcher) + val flowGraph = flowEngine.newGraph() + for (spec in specs) { val host = SimHost( spec.uid, spec.name, spec.meta, ctx.dispatcher.timeSource, - machine, - hypervisor, - optimize = optimize, + flowGraph, + spec.model, + spec.cpuPowerModel, ) require(hosts.add(host)) { "Host with uid ${spec.uid} already exists" } diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/provisioner/Provisioner.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/provisioner/Provisioner.kt index 58d3a8c2..2e76478e 100644 --- a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/provisioner/Provisioner.kt +++ b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/provisioner/Provisioner.kt @@ -23,9 +23,7 @@ package org.opendc.compute.simulator.provisioner import org.opendc.common.Dispatcher -import org.opendc.compute.simulator.MutableServiceRegistry import org.opendc.compute.simulator.ServiceRegistry -import org.opendc.compute.simulator.ServiceRegistryImpl import java.util.ArrayDeque import java.util.SplittableRandom @@ -47,7 +45,7 @@ public class Provisioner(dispatcher: Dispatcher, seed: Long) : AutoCloseable { object : ProvisioningContext { override val dispatcher: Dispatcher = dispatcher override val seeder: SplittableRandom = SplittableRandom(seed) - override val registry: MutableServiceRegistry = ServiceRegistryImpl() + override val registry: ServiceRegistry = ServiceRegistry() override fun toString(): String = "Provisioner.ProvisioningContext" } diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/provisioner/ProvisioningContext.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/provisioner/ProvisioningContext.kt index 1788c8e2..20c441c4 100644 --- a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/provisioner/ProvisioningContext.kt +++ b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/provisioner/ProvisioningContext.kt @@ -23,7 +23,7 @@ package org.opendc.compute.simulator.provisioner import org.opendc.common.Dispatcher -import org.opendc.compute.simulator.MutableServiceRegistry +import org.opendc.compute.simulator.ServiceRegistry import java.util.SplittableRandom import java.util.random.RandomGenerator @@ -44,7 +44,7 @@ public interface ProvisioningContext { public val seeder: RandomGenerator /** - * A [MutableServiceRegistry] where the provisioned services are registered. + * A [ServiceRegistry] where the provisioned services are registered. */ - public val registry: MutableServiceRegistry + public val registry: ServiceRegistry } diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/ComputeScheduler.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/ComputeScheduler.kt new file mode 100644 index 00000000..f0a2c3b4 --- /dev/null +++ b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/ComputeScheduler.kt @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.compute.simulator.scheduler + +import org.opendc.compute.simulator.service.HostView +import org.opendc.compute.simulator.service.ServiceTask + +/** + * A generic scheduler interface used by the [ComputeService] to select hosts to place [ServiceTask]s on. + */ +public interface ComputeScheduler { + /** + * Register the specified [host] to be used for scheduling. + */ + public fun addHost(host: HostView) + + /** + * Remove the specified [host] to be removed from the scheduling pool. + */ + public fun removeHost(host: HostView) + + /** + * Select a host for the specified [task]. + * + * @param task The server to select a host for. + * @return The host to schedule the server on or `null` if no server is available. + */ + public fun select(task: ServiceTask): HostView? +} diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/ComputeSchedulers.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/ComputeSchedulers.kt new file mode 100644 index 00000000..ec3aedcb --- /dev/null +++ b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/ComputeSchedulers.kt @@ -0,0 +1,118 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +@file:JvmName("ComputeSchedulers") + +package org.opendc.compute.simulator.scheduler + +import org.opendc.compute.simulator.scheduler.filters.ComputeFilter +import org.opendc.compute.simulator.scheduler.filters.RamFilter +import org.opendc.compute.simulator.scheduler.filters.VCpuFilter +import org.opendc.compute.simulator.scheduler.weights.CoreRamWeigher +import org.opendc.compute.simulator.scheduler.weights.InstanceCountWeigher +import org.opendc.compute.simulator.scheduler.weights.RamWeigher +import org.opendc.compute.simulator.scheduler.weights.VCpuWeigher +import java.util.SplittableRandom +import java.util.random.RandomGenerator + +public enum class ComputeSchedulerEnum { + Mem, + MemInv, + CoreMem, + CoreMemInv, + ActiveServers, + ActiveServersInv, + ProvisionedCores, + ProvisionedCoresInv, + Random, + Replay, +} + +public fun createComputeScheduler( + name: String, + seeder: RandomGenerator, + placements: Map = emptyMap(), +): ComputeScheduler { + return createComputeScheduler(ComputeSchedulerEnum.valueOf(name.uppercase()), seeder, placements) +} + +/** + * Create a [ComputeScheduler] for the experiment. + */ +public fun createComputeScheduler( + name: ComputeSchedulerEnum, + seeder: RandomGenerator, + placements: Map = emptyMap(), +): ComputeScheduler { + val cpuAllocationRatio = 1.0 + val ramAllocationRatio = 1.5 + return when (name) { + ComputeSchedulerEnum.Mem -> + FilterScheduler( + filters = listOf(ComputeFilter(), VCpuFilter(cpuAllocationRatio), RamFilter(ramAllocationRatio)), + weighers = listOf(RamWeigher(multiplier = 1.0)), + ) + ComputeSchedulerEnum.MemInv -> + FilterScheduler( + filters = listOf(ComputeFilter(), VCpuFilter(cpuAllocationRatio), RamFilter(ramAllocationRatio)), + weighers = listOf(RamWeigher(multiplier = -1.0)), + ) + ComputeSchedulerEnum.CoreMem -> + FilterScheduler( + filters = listOf(ComputeFilter(), VCpuFilter(cpuAllocationRatio), RamFilter(ramAllocationRatio)), + weighers = listOf(CoreRamWeigher(multiplier = 1.0)), + ) + ComputeSchedulerEnum.CoreMemInv -> + FilterScheduler( + filters = listOf(ComputeFilter(), VCpuFilter(cpuAllocationRatio), RamFilter(ramAllocationRatio)), + weighers = listOf(CoreRamWeigher(multiplier = -1.0)), + ) + ComputeSchedulerEnum.ActiveServers -> + FilterScheduler( + filters = listOf(ComputeFilter(), VCpuFilter(cpuAllocationRatio), RamFilter(ramAllocationRatio)), + weighers = listOf(InstanceCountWeigher(multiplier = -1.0)), + ) + ComputeSchedulerEnum.ActiveServersInv -> + FilterScheduler( + filters = listOf(ComputeFilter(), VCpuFilter(cpuAllocationRatio), RamFilter(ramAllocationRatio)), + weighers = listOf(InstanceCountWeigher(multiplier = 1.0)), + ) + ComputeSchedulerEnum.ProvisionedCores -> + FilterScheduler( + filters = listOf(ComputeFilter(), VCpuFilter(cpuAllocationRatio), RamFilter(ramAllocationRatio)), + weighers = listOf(VCpuWeigher(cpuAllocationRatio, multiplier = 1.0)), + ) + ComputeSchedulerEnum.ProvisionedCoresInv -> + FilterScheduler( + filters = listOf(ComputeFilter(), VCpuFilter(cpuAllocationRatio), RamFilter(ramAllocationRatio)), + weighers = listOf(VCpuWeigher(cpuAllocationRatio, multiplier = -1.0)), + ) + ComputeSchedulerEnum.Random -> + FilterScheduler( + filters = listOf(ComputeFilter(), VCpuFilter(cpuAllocationRatio), RamFilter(ramAllocationRatio)), + weighers = emptyList(), + subsetSize = Int.MAX_VALUE, + random = SplittableRandom(seeder.nextLong()), + ) + ComputeSchedulerEnum.Replay -> ReplayScheduler(placements) + } +} diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/FilterScheduler.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/FilterScheduler.kt new file mode 100644 index 00000000..9fd3a862 --- /dev/null +++ b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/FilterScheduler.kt @@ -0,0 +1,111 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.compute.simulator.scheduler + +import org.opendc.compute.simulator.scheduler.filters.HostFilter +import org.opendc.compute.simulator.scheduler.weights.HostWeigher +import org.opendc.compute.simulator.service.HostView +import org.opendc.compute.simulator.service.ServiceTask +import java.util.SplittableRandom +import java.util.random.RandomGenerator +import kotlin.math.min + +/** + * A [ComputeScheduler] implementation that uses filtering and weighing passes to select + * the host to schedule a [ServiceTask] on. + * + * This implementation is based on the filter scheduler from OpenStack Nova. + * See: https://docs.openstack.org/nova/latest/user/filter-scheduler.html + * + * @param filters The list of filters to apply when searching for an appropriate host. + * @param weighers The list of weighers to apply when searching for an appropriate host. + * @param subsetSize The size of the subset of best hosts from which a target is randomly chosen. + * @param random A [RandomGenerator] instance for selecting + */ +public class FilterScheduler( + private val filters: List, + private val weighers: List, + private val subsetSize: Int = 1, + private val random: RandomGenerator = SplittableRandom(0), +) : ComputeScheduler { + /** + * The pool of hosts available to the scheduler. + */ + private val hosts = mutableListOf() + + init { + require(subsetSize >= 1) { "Subset size must be one or greater" } + } + + override fun addHost(host: HostView) { + hosts.add(host) + } + + override fun removeHost(host: HostView) { + hosts.remove(host) + } + + override fun select(task: ServiceTask): HostView? { + val hosts = hosts + val filteredHosts = hosts.filter { host -> filters.all { filter -> filter.test(host, task) } } + + val subset = + if (weighers.isNotEmpty()) { + val results = weighers.map { it.getWeights(filteredHosts, task) } + val weights = DoubleArray(filteredHosts.size) + + for (result in results) { + val min = result.min + val range = (result.max - min) + + // Skip result if all weights are the same + if (range == 0.0) { + continue + } + + val multiplier = result.multiplier + val factor = multiplier / range + + for ((i, weight) in result.weights.withIndex()) { + weights[i] += factor * (weight - min) + } + } + + weights.indices + .asSequence() + .sortedByDescending { weights[it] } + .map { filteredHosts[it] } + .take(subsetSize) + .toList() + } else { + filteredHosts + } + + // fixme: currently finding no matching hosts can result in an error + return when (val maxSize = min(subsetSize, subset.size)) { + 0 -> null + 1 -> subset[0] + else -> subset[random.nextInt(maxSize)] + } + } +} diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/ReplayScheduler.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/ReplayScheduler.kt new file mode 100644 index 00000000..43e366d9 --- /dev/null +++ b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/ReplayScheduler.kt @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2020 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.compute.simulator.scheduler + +import mu.KotlinLogging +import org.opendc.compute.simulator.service.HostView +import org.opendc.compute.simulator.service.ServiceTask + +/** + * Policy replaying VM-cluster assignment. + * + * Within each cluster, the active servers on each node determine which node gets + * assigned the VM image. + */ +public class ReplayScheduler(private val vmPlacements: Map) : ComputeScheduler { + private val logger = KotlinLogging.logger {} + + /** + * The pool of hosts available to the scheduler. + */ + private val hosts = mutableListOf() + + override fun addHost(host: HostView) { + hosts.add(host) + } + + override fun removeHost(host: HostView) { + hosts.remove(host) + } + + override fun select(task: ServiceTask): HostView? { + val clusterName = + vmPlacements[task.name] + ?: throw IllegalStateException("Could not find placement data in VM placement file for VM ${task.name}") + val machinesInCluster = hosts.filter { it.host.getName().contains(clusterName) } + + if (machinesInCluster.isEmpty()) { + logger.info { "Could not find any machines belonging to cluster $clusterName for image ${task.name}, assigning randomly." } + return hosts.maxByOrNull { it.availableMemory } + } + + return machinesInCluster.maxByOrNull { it.availableMemory } + ?: throw IllegalStateException("Cloud not find any machine and could not randomly assign") + } +} diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/filters/ComputeFilter.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/filters/ComputeFilter.kt new file mode 100644 index 00000000..99a9390e --- /dev/null +++ b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/filters/ComputeFilter.kt @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.compute.simulator.scheduler.filters + +import org.opendc.compute.simulator.host.HostState +import org.opendc.compute.simulator.service.HostView +import org.opendc.compute.simulator.service.ServiceTask + +/** + * A [HostFilter] that filters on active hosts. + */ +public class ComputeFilter : HostFilter { + override fun test( + host: HostView, + task: ServiceTask, + ): Boolean { + val result = host.host.getState() == HostState.UP + return result + } + + override fun toString(): String = "ComputeFilter" +} diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/filters/DifferentHostFilter.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/filters/DifferentHostFilter.kt new file mode 100644 index 00000000..279a2717 --- /dev/null +++ b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/filters/DifferentHostFilter.kt @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2022 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.compute.simulator.scheduler.filters + +import org.opendc.compute.simulator.service.HostView +import org.opendc.compute.simulator.service.ServiceTask +import java.util.UUID + +/** + * A [HostFilter] that ensures an instance is scheduled on a different host from a set of instances. + */ +public class DifferentHostFilter : HostFilter { + override fun test( + host: HostView, + task: ServiceTask, + ): Boolean { + @Suppress("UNCHECKED_CAST") + val affinityUUIDs = task.meta["scheduler_hint:different_host"] as? Set ?: return true + return host.host.getInstances().none { it.uid in affinityUUIDs } + } +} diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/filters/HostFilter.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/filters/HostFilter.kt new file mode 100644 index 00000000..bb9c1cbf --- /dev/null +++ b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/filters/HostFilter.kt @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.compute.simulator.scheduler.filters + +import org.opendc.compute.simulator.service.HostView +import org.opendc.compute.simulator.service.ServiceTask + +/** + * A filter used by the [FilterScheduler] to filter hosts. + */ +public fun interface HostFilter { + /** + * Test whether the specified [host] should be included in the selection + * for scheduling the specified [task]. + */ + public fun test( + host: HostView, + task: ServiceTask, + ): Boolean +} diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/filters/InstanceCountFilter.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/filters/InstanceCountFilter.kt new file mode 100644 index 00000000..53d68acf --- /dev/null +++ b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/filters/InstanceCountFilter.kt @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.compute.simulator.scheduler.filters + +import org.opendc.compute.simulator.service.HostView +import org.opendc.compute.simulator.service.ServiceTask + +/** + * A [HostFilter] that filters hosts based on the number of instances on the host. + * + * @param limit The maximum number of instances on the host. + */ +public class InstanceCountFilter(private val limit: Int) : HostFilter { + override fun test( + host: HostView, + task: ServiceTask, + ): Boolean { + return host.instanceCount < limit + } + + override fun toString(): String = "InstanceCountFilter[limit=$limit]" +} diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/filters/RamFilter.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/filters/RamFilter.kt new file mode 100644 index 00000000..0b570d52 --- /dev/null +++ b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/filters/RamFilter.kt @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.compute.simulator.scheduler.filters + +import org.opendc.compute.simulator.service.HostView +import org.opendc.compute.simulator.service.ServiceTask + +/** + * A [HostFilter] that filters hosts based on the memory requirements of a [ServiceTask] and the RAM available on the host. + * + * @param allocationRatio Virtual RAM to physical RAM allocation ratio. + */ +public class RamFilter(private val allocationRatio: Double) : HostFilter { + override fun test( + host: HostView, + task: ServiceTask, + ): Boolean { + val requestedMemory = task.flavor.memorySize + val availableMemory = host.availableMemory + val memoryCapacity = host.host.getModel().memoryCapacity + + // Do not allow an instance to overcommit against itself, only against + // other instances. + if (requestedMemory > memoryCapacity) { + return false + } + + val limit = memoryCapacity * allocationRatio + val used = memoryCapacity - availableMemory + val usable = limit - used + + val result = usable >= requestedMemory + return result + } +} diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/filters/SameHostFilter.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/filters/SameHostFilter.kt new file mode 100644 index 00000000..761b125d --- /dev/null +++ b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/filters/SameHostFilter.kt @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2022 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.compute.simulator.scheduler.filters + +import org.opendc.compute.simulator.service.HostView +import org.opendc.compute.simulator.service.ServiceTask +import java.util.UUID + +/** + * A [HostFilter] that ensures an instance is scheduled on the same host as all other instances in a set of instances. + */ +public class SameHostFilter : HostFilter { + override fun test( + host: HostView, + task: ServiceTask, + ): Boolean { + @Suppress("UNCHECKED_CAST") + val affinityUUIDs = task.meta["scheduler_hint:same_host"] as? Set ?: return true + return host.host.getInstances().any { it.uid in affinityUUIDs } + } +} diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/filters/VCpuCapacityFilter.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/filters/VCpuCapacityFilter.kt new file mode 100644 index 00000000..256caa94 --- /dev/null +++ b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/filters/VCpuCapacityFilter.kt @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.compute.simulator.scheduler.filters + +import org.opendc.compute.simulator.service.HostView +import org.opendc.compute.simulator.service.ServiceTask + +/** + * A [HostFilter] that filters hosts based on the vCPU speed requirements of a [ServiceTask] and the available + * capacity on the host. + */ +public class VCpuCapacityFilter : HostFilter { + override fun test( + host: HostView, + task: ServiceTask, + ): Boolean { + val requiredCapacity = task.flavor.meta["cpu-capacity"] as? Double + val availableCapacity = host.host.getModel().cpuCapacity + + return requiredCapacity == null || availableCapacity >= (requiredCapacity / task.flavor.coreCount) + } +} diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/filters/VCpuFilter.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/filters/VCpuFilter.kt new file mode 100644 index 00000000..c179a7bf --- /dev/null +++ b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/filters/VCpuFilter.kt @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.compute.simulator.scheduler.filters + +import org.opendc.compute.simulator.service.HostView +import org.opendc.compute.simulator.service.ServiceTask + +/** + * A [HostFilter] that filters hosts based on the vCPU requirements of a [ServiceTask] and the available vCPUs on the host. + * + * @param allocationRatio Virtual CPU to physical CPU allocation ratio. + */ +public class VCpuFilter(private val allocationRatio: Double) : HostFilter { + override fun test( + host: HostView, + task: ServiceTask, + ): Boolean { + val requested = task.flavor.coreCount + val totalCores = host.host.getModel().coreCount + val limit = totalCores * allocationRatio + + // Do not allow an instance to overcommit against itself, only against other instances + if (requested > totalCores) { + return false + } + + val availableCores = limit - host.provisionedCores + return availableCores >= requested + } +} diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/weights/CoreRamWeigher.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/weights/CoreRamWeigher.kt new file mode 100644 index 00000000..b6c43c10 --- /dev/null +++ b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/weights/CoreRamWeigher.kt @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.compute.simulator.scheduler.weights + +import org.opendc.compute.simulator.service.HostView +import org.opendc.compute.simulator.service.ServiceTask + +/** + * A [HostWeigher] that weighs the hosts based on the available memory per core on the host. + * + * @param multiplier Weight multiplier ratio. A positive value will result in the scheduler preferring hosts with more + * available core memory, and a negative number will result in the scheduler preferring hosts with less available core + * memory. + */ +public class CoreRamWeigher(override val multiplier: Double = 1.0) : HostWeigher { + override fun getWeight( + host: HostView, + task: ServiceTask, + ): Double { + return host.availableMemory.toDouble() + } + + override fun toString(): String = "CoreRamWeigher" +} diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/weights/HostWeigher.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/weights/HostWeigher.kt new file mode 100644 index 00000000..c1e0c590 --- /dev/null +++ b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/weights/HostWeigher.kt @@ -0,0 +1,80 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.compute.simulator.scheduler.weights + +import org.opendc.compute.simulator.service.HostView +import org.opendc.compute.simulator.service.ServiceTask + +/** + * An interface used by the [FilterScheduler] to weigh the pool of host for a scheduling request. + */ +public interface HostWeigher { + /** + * The multiplier for the weigher. + */ + public val multiplier: Double + + /** + * Obtain the weight of the specified [host] when scheduling the specified [ServiceTask]. + */ + public fun getWeight( + host: HostView, + task: ServiceTask, + ): Double + + /** + * Obtain the weights for [hosts] when scheduling the specified [task]. + */ + public fun getWeights( + hosts: List, + task: ServiceTask, + ): Result { + val weights = DoubleArray(hosts.size) + var min = Double.MAX_VALUE + var max = Double.MIN_VALUE + + for ((i, host) in hosts.withIndex()) { + val weight = getWeight(host, task) + weights[i] = weight + min = kotlin.math.min(min, weight) + max = kotlin.math.max(max, weight) + } + + return Result(weights, min, max, multiplier) + } + + /** + * A result returned by the weigher. + * + * @param weights The weights returned by the weigher. + * @param min The minimum weight returned. + * @param max The maximum weight returned. + * @param multiplier The weight multiplier to use. + */ + public class Result( + public val weights: DoubleArray, + public val min: Double, + public val max: Double, + public val multiplier: Double, + ) +} diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/weights/InstanceCountWeigher.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/weights/InstanceCountWeigher.kt new file mode 100644 index 00000000..9277c1ae --- /dev/null +++ b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/weights/InstanceCountWeigher.kt @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.compute.simulator.scheduler.weights + +import org.opendc.compute.simulator.service.HostView +import org.opendc.compute.simulator.service.ServiceTask + +/** + * A [HostWeigher] that weighs the hosts based on the number of instances on the host. + */ +public class InstanceCountWeigher(override val multiplier: Double = 1.0) : HostWeigher { + override fun getWeight( + host: HostView, + task: ServiceTask, + ): Double { + return host.instanceCount.toDouble() + } + + override fun toString(): String = "InstanceCountWeigher" +} diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/weights/RamWeigher.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/weights/RamWeigher.kt new file mode 100644 index 00000000..1cbfea59 --- /dev/null +++ b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/weights/RamWeigher.kt @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.compute.simulator.scheduler.weights + +import org.opendc.compute.simulator.service.HostView +import org.opendc.compute.simulator.service.ServiceTask + +/** + * A [HostWeigher] that weighs the hosts based on the available RAM (memory) on the host. + * + * @param multiplier Weight multiplier ratio. A positive value will result in the scheduler preferring hosts with more + * available memory, and a negative number will result in the scheduler preferring hosts with less memory. + */ +public class RamWeigher(override val multiplier: Double = 1.0) : HostWeigher { + override fun getWeight( + host: HostView, + task: ServiceTask, + ): Double { + return host.availableMemory.toDouble() + } + + override fun toString(): String = "RamWeigher" +} diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/weights/VCpuCapacityWeigher.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/weights/VCpuCapacityWeigher.kt new file mode 100644 index 00000000..4f52e11a --- /dev/null +++ b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/weights/VCpuCapacityWeigher.kt @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.compute.simulator.scheduler.weights + +import org.opendc.compute.simulator.service.HostView +import org.opendc.compute.simulator.service.ServiceTask + +/** + * A [HostWeigher] that weighs the hosts based on the difference required vCPU capacity and the available CPU capacity. + */ +public class VCpuCapacityWeigher(override val multiplier: Double = 1.0) : HostWeigher { + override fun getWeight( + host: HostView, + task: ServiceTask, + ): Double { + val model = host.host.getModel() + val requiredCapacity = task.flavor.meta["cpu-capacity"] as? Double ?: 0.0 + return model.cpuCapacity - requiredCapacity / task.flavor.coreCount + } + + override fun toString(): String = "VCpuWeigher" +} diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/weights/VCpuWeigher.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/weights/VCpuWeigher.kt new file mode 100644 index 00000000..3f9a7f03 --- /dev/null +++ b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/weights/VCpuWeigher.kt @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.compute.simulator.scheduler.weights + +import org.opendc.compute.simulator.service.HostView +import org.opendc.compute.simulator.service.ServiceTask + +/** + * A [HostWeigher] that weighs the hosts based on the remaining number of vCPUs available. + * + * @param allocationRatio Virtual CPU to physical CPU allocation ratio. + */ +public class VCpuWeigher(private val allocationRatio: Double, override val multiplier: Double = 1.0) : HostWeigher { + init { + require(allocationRatio > 0.0) { "Allocation ratio must be greater than zero" } + } + + override fun getWeight( + host: HostView, + task: ServiceTask, + ): Double { + return allocationRatio - host.provisionedCores + } + + override fun toString(): String = "VCpuWeigher" +} diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/telemetry/ComputeMetricReader.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/telemetry/ComputeMetricReader.kt new file mode 100644 index 00000000..d5fb991d --- /dev/null +++ b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/telemetry/ComputeMetricReader.kt @@ -0,0 +1,664 @@ +/* + * Copyright (c) 2022 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.compute.simulator.telemetry + +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.delay +import kotlinx.coroutines.isActive +import kotlinx.coroutines.launch +import mu.KotlinLogging +import org.opendc.common.Dispatcher +import org.opendc.common.asCoroutineDispatcher +import org.opendc.compute.api.TaskState +import org.opendc.compute.carbon.CarbonTrace +import org.opendc.compute.simulator.host.SimHost +import org.opendc.compute.simulator.service.ComputeService +import org.opendc.compute.simulator.service.ServiceTask +import org.opendc.compute.simulator.telemetry.table.HostInfo +import org.opendc.compute.simulator.telemetry.table.HostTableReader +import org.opendc.compute.simulator.telemetry.table.ServiceTableReader +import org.opendc.compute.simulator.telemetry.table.TaskInfo +import org.opendc.compute.simulator.telemetry.table.TaskTableReader +import java.time.Duration +import java.time.Instant + +/** + * A helper class to collect metrics from a [ComputeService] instance and automatically export the metrics every + * export interval. + * + * @param dispatcher A [Dispatcher] for scheduling the future events. + * @param service The [ComputeService] to monitor. + * @param monitor The monitor to export the metrics to. + * @param exportInterval The export interval. + */ +public class ComputeMetricReader( + dispatcher: Dispatcher, + private val service: ComputeService, + private val monitor: ComputeMonitor, + private val exportInterval: Duration = Duration.ofMinutes(5), + private val startTime: Duration = Duration.ofMillis(0), + private val carbonTrace: CarbonTrace = CarbonTrace(null), +) : AutoCloseable { + private val logger = KotlinLogging.logger {} + private val scope = CoroutineScope(dispatcher.asCoroutineDispatcher()) + private val clock = dispatcher.timeSource + + /** + * Aggregator for service metrics. + */ + private val serviceTableReader = + ServiceTableReaderImpl( + service, + startTime, + ) + + private var loggCounter = 0 + + /** + * Mapping from [Host] instances to [HostTableReaderImpl] + */ + private val hostTableReaders = mutableMapOf() + + /** + * Mapping from [Task] instances to [TaskTableReaderImpl] + */ + private val taskTableReaders = mutableMapOf() + + /** + * The background job that is responsible for collecting the metrics every cycle. + */ + private val job = + scope.launch { + val intervalMs = exportInterval.toMillis() + try { + while (isActive) { + delay(intervalMs) + + loggState() + } + } finally { + loggState() + + if (monitor is AutoCloseable) { + monitor.close() + } + } + } + + private fun loggState() { + loggCounter++ + try { + val now = this.clock.instant() + + for (host in this.service.hosts) { + val reader = + this.hostTableReaders.computeIfAbsent(host) { + HostTableReaderImpl( + it, + startTime, + carbonTrace, + ) + } + reader.record(now) + this.monitor.record(reader.copy()) + reader.reset() + } + + for (task in this.service.tasks) { + val reader = + this.taskTableReaders.computeIfAbsent(task) { + TaskTableReaderImpl( + service, + it, + startTime, + ) + } + reader.record(now) + this.monitor.record(reader.copy()) + reader.reset() + } + + for (task in this.service.tasksToRemove) { + task.delete() + } + this.service.clearTasksToRemove() + + this.serviceTableReader.record(now) + monitor.record(this.serviceTableReader.copy()) + + if (loggCounter >= 100) { + var loggString = "\n\t\t\t\t\tMetrics after ${now.toEpochMilli() / 1000 / 60 / 60} hours:\n" + loggString += "\t\t\t\t\t\tTasks Total: ${this.serviceTableReader.tasksTotal}\n" + loggString += "\t\t\t\t\t\tTasks Active: ${this.serviceTableReader.tasksActive}\n" + loggString += "\t\t\t\t\t\tTasks Pending: ${this.serviceTableReader.tasksPending}\n" + loggString += "\t\t\t\t\t\tTasks Completed: ${this.serviceTableReader.tasksCompleted}\n" + loggString += "\t\t\t\t\t\tTasks Terminated: ${this.serviceTableReader.tasksTerminated}\n" + + this.logger.warn { loggString } + loggCounter = 0 + } + } catch (cause: Throwable) { + this.logger.warn(cause) { "Exporter threw an Exception" } + } + } + + override fun close() { + job.cancel() + } + + /** + * An aggregator for service metrics before they are reported. + */ + private class ServiceTableReaderImpl( + private val service: ComputeService, + private val startTime: Duration = Duration.ofMillis(0), + ) : ServiceTableReader { + override fun copy(): ServiceTableReader { + val newServiceTable = + ServiceTableReaderImpl( + service, + ) + newServiceTable.setValues(this) + + return newServiceTable + } + + override fun setValues(table: ServiceTableReader) { + _timestamp = table.timestamp + _timestampAbsolute = table.timestampAbsolute + + _hostsUp = table.hostsUp + _hostsDown = table.hostsDown + _tasksTotal = table.tasksTotal + _tasksPending = table.tasksPending + _tasksActive = table.tasksActive + _tasksCompleted = table.tasksCompleted + _tasksTerminated = table.tasksTerminated + _attemptsSuccess = table.attemptsSuccess + _attemptsFailure = table.attemptsFailure + } + + private var _timestamp: Instant = Instant.MIN + override val timestamp: Instant + get() = _timestamp + + private var _timestampAbsolute: Instant = Instant.MIN + override val timestampAbsolute: Instant + get() = _timestampAbsolute + + override val hostsUp: Int + get() = _hostsUp + private var _hostsUp = 0 + + override val hostsDown: Int + get() = _hostsDown + private var _hostsDown = 0 + + override val tasksTotal: Int + get() = _tasksTotal + private var _tasksTotal = 0 + + override val tasksPending: Int + get() = _tasksPending + private var _tasksPending = 0 + + override val tasksCompleted: Int + get() = _tasksCompleted + private var _tasksCompleted = 0 + + override val tasksActive: Int + get() = _tasksActive + private var _tasksActive = 0 + + override val tasksTerminated: Int + get() = _tasksTerminated + private var _tasksTerminated = 0 + + override val attemptsSuccess: Int + get() = _attemptsSuccess + private var _attemptsSuccess = 0 + + override val attemptsFailure: Int + get() = _attemptsFailure + private var _attemptsFailure = 0 + + /** + * Record the next cycle. + */ + fun record(now: Instant) { + _timestamp = now + _timestampAbsolute = now + startTime + + val stats = service.getSchedulerStats() + _hostsUp = stats.hostsAvailable + _hostsDown = stats.hostsUnavailable + _tasksTotal = stats.tasksTotal + _tasksPending = stats.tasksPending + _tasksCompleted = stats.tasksCompleted + _tasksActive = stats.tasksActive + _tasksTerminated = stats.tasksTerminated + _attemptsSuccess = stats.attemptsSuccess.toInt() + _attemptsFailure = stats.attemptsFailure.toInt() + } + } + + /** + * An aggregator for host metrics before they are reported. + */ + private class HostTableReaderImpl( + host: SimHost, + private val startTime: Duration = Duration.ofMillis(0), + private val carbonTrace: CarbonTrace = CarbonTrace(null), + ) : HostTableReader { + override fun copy(): HostTableReader { + val newHostTable = + HostTableReaderImpl(_host) + newHostTable.setValues(this) + + return newHostTable + } + + override fun setValues(table: HostTableReader) { + _timestamp = table.timestamp + _timestampAbsolute = table.timestampAbsolute + + _guestsTerminated = table.guestsTerminated + _guestsRunning = table.guestsRunning + _guestsError = table.guestsError + _guestsInvalid = table.guestsInvalid + _cpuLimit = table.cpuLimit + _cpuDemand = table.cpuDemand + _cpuUsage = table.cpuUsage + _cpuUtilization = table.cpuUtilization + _cpuActiveTime = table.cpuActiveTime + _cpuIdleTime = table.cpuIdleTime + _cpuStealTime = table.cpuStealTime + _cpuLostTime = table.cpuLostTime + _powerDraw = table.powerDraw + _energyUsage = table.energyUsage + _carbonIntensity = table.carbonIntensity + _carbonEmission = table.carbonEmission + _uptime = table.uptime + _downtime = table.downtime + _bootTime = table.bootTime + _bootTimeAbsolute = table.bootTimeAbsolute + } + + private val _host = host + + override val host: HostInfo = + HostInfo( + host.getUid().toString(), + host.getName(), + "x86", + host.getModel().coreCount, + host.getModel().cpuCapacity, + host.getModel().memoryCapacity, + ) + + override val timestamp: Instant + get() = _timestamp + private var _timestamp = Instant.MIN + + override val timestampAbsolute: Instant + get() = _timestampAbsolute + private var _timestampAbsolute = Instant.MIN + + override val guestsTerminated: Int + get() = _guestsTerminated + private var _guestsTerminated = 0 + + override val guestsRunning: Int + get() = _guestsRunning + private var _guestsRunning = 0 + + override val guestsError: Int + get() = _guestsError + private var _guestsError = 0 + + override val guestsInvalid: Int + get() = _guestsInvalid + private var _guestsInvalid = 0 + + override val cpuLimit: Float + get() = _cpuLimit + private var _cpuLimit = 0.0f + + override val cpuUsage: Float + get() = _cpuUsage + private var _cpuUsage = 0.0f + + override val cpuDemand: Float + get() = _cpuDemand + private var _cpuDemand = 0.0f + + override val cpuUtilization: Float + get() = _cpuUtilization + private var _cpuUtilization = 0.0f + + override val cpuActiveTime: Long + get() = _cpuActiveTime - previousCpuActiveTime + private var _cpuActiveTime = 0L + private var previousCpuActiveTime = 0L + + override val cpuIdleTime: Long + get() = _cpuIdleTime - previousCpuIdleTime + private var _cpuIdleTime = 0L + private var previousCpuIdleTime = 0L + + override val cpuStealTime: Long + get() = _cpuStealTime - previousCpuStealTime + private var _cpuStealTime = 0L + private var previousCpuStealTime = 0L + + override val cpuLostTime: Long + get() = _cpuLostTime - previousCpuLostTime + private var _cpuLostTime = 0L + private var previousCpuLostTime = 0L + + override val powerDraw: Float + get() = _powerDraw + private var _powerDraw = 0.0f + + override val energyUsage: Float + get() = _energyUsage - previousEnergyUsage + private var _energyUsage = 0.0f + private var previousEnergyUsage = 0.0f + + override val carbonIntensity: Float + get() = _carbonIntensity + private var _carbonIntensity = 0.0f + + override val carbonEmission: Float + get() = _carbonEmission + private var _carbonEmission = 0.0f + + override val uptime: Long + get() = _uptime - previousUptime + private var _uptime = 0L + private var previousUptime = 0L + + override val downtime: Long + get() = _downtime - previousDowntime + private var _downtime = 0L + private var previousDowntime = 0L + + override val bootTime: Instant? + get() = _bootTime + private var _bootTime: Instant? = null + + override val bootTimeAbsolute: Instant? + get() = _bootTimeAbsolute + private var _bootTimeAbsolute: Instant? = null + + /** + * Record the next cycle. + */ + fun record(now: Instant) { + val hostCpuStats = _host.getCpuStats() + val hostSysStats = _host.getSystemStats() + + _timestamp = now + _timestampAbsolute = now + startTime + + _guestsTerminated = hostSysStats.guestsTerminated + _guestsRunning = hostSysStats.guestsRunning + _guestsError = hostSysStats.guestsError + _guestsInvalid = hostSysStats.guestsInvalid + _cpuLimit = hostCpuStats.capacity + _cpuDemand = hostCpuStats.demand + _cpuUsage = hostCpuStats.usage + _cpuUtilization = hostCpuStats.utilization + _cpuActiveTime = hostCpuStats.activeTime + _cpuIdleTime = hostCpuStats.idleTime + _cpuStealTime = hostCpuStats.stealTime + _cpuLostTime = hostCpuStats.lostTime + _powerDraw = hostSysStats.powerDraw + _energyUsage = hostSysStats.energyUsage + _carbonIntensity = carbonTrace.getCarbonIntensity(timestampAbsolute) + + _carbonEmission = carbonIntensity * (energyUsage / 3600000.0f) // convert energy usage from J to kWh + _uptime = hostSysStats.uptime.toMillis() + _downtime = hostSysStats.downtime.toMillis() + _bootTime = hostSysStats.bootTime + _bootTime = hostSysStats.bootTime + startTime + } + + /** + * Finish the aggregation for this cycle. + */ + fun reset() { + // Reset intermediate state for next aggregation + previousCpuActiveTime = _cpuActiveTime + previousCpuIdleTime = _cpuIdleTime + previousCpuStealTime = _cpuStealTime + previousCpuLostTime = _cpuLostTime + previousEnergyUsage = _energyUsage + previousUptime = _uptime + previousDowntime = _downtime + + _guestsTerminated = 0 + _guestsRunning = 0 + _guestsError = 0 + _guestsInvalid = 0 + + _cpuLimit = 0.0f + _cpuUsage = 0.0f + _cpuDemand = 0.0f + _cpuUtilization = 0.0f + + _powerDraw = 0.0f + _energyUsage = 0.0f + _carbonIntensity = 0.0f + _carbonEmission = 0.0f + } + } + + /** + * An aggregator for task metrics before they are reported. + */ + private class TaskTableReaderImpl( + private val service: ComputeService, + private val task: ServiceTask, + private val startTime: Duration = Duration.ofMillis(0), + ) : TaskTableReader { + override fun copy(): TaskTableReader { + val newTaskTable = + TaskTableReaderImpl( + service, + task, + ) + newTaskTable.setValues(this) + + return newTaskTable + } + + override fun setValues(table: TaskTableReader) { + host = table.host + + _timestamp = table.timestamp + _timestampAbsolute = table.timestampAbsolute + + _cpuLimit = table.cpuLimit + _cpuActiveTime = table.cpuActiveTime + _cpuIdleTime = table.cpuIdleTime + _cpuStealTime = table.cpuStealTime + _cpuLostTime = table.cpuLostTime + _uptime = table.uptime + _downtime = table.downtime + _provisionTime = table.provisionTime + _bootTime = table.bootTime + _bootTimeAbsolute = table.bootTimeAbsolute + + _creationTime = table.creationTime + _finishTime = table.finishTime + + _taskState = table.taskState + } + + /** + * The static information about this task. + */ + override val taskInfo = + TaskInfo( + task.uid.toString(), + task.name, + "vm", + "x86", + task.flavor.coreCount, + task.flavor.memorySize, + ) + + /** + * The [HostInfo] of the host on which the task is hosted. + */ + override var host: HostInfo? = null + private var _host: SimHost? = null + + private var _timestamp = Instant.MIN + override val timestamp: Instant + get() = _timestamp + + private var _timestampAbsolute = Instant.MIN + override val timestampAbsolute: Instant + get() = _timestampAbsolute + + override val uptime: Long + get() = _uptime - previousUptime + private var _uptime: Long = 0 + private var previousUptime = 0L + + override val downtime: Long + get() = _downtime - previousDowntime + private var _downtime: Long = 0 + private var previousDowntime = 0L + + override val provisionTime: Instant? + get() = _provisionTime + private var _provisionTime: Instant? = null + + override val bootTime: Instant? + get() = _bootTime + private var _bootTime: Instant? = null + + override val creationTime: Instant? + get() = _creationTime + private var _creationTime: Instant? = null + + override val finishTime: Instant? + get() = _finishTime + private var _finishTime: Instant? = null + + override val cpuLimit: Float + get() = _cpuLimit + private var _cpuLimit = 0.0f + + override val cpuActiveTime: Long + get() = _cpuActiveTime - previousCpuActiveTime + private var _cpuActiveTime = 0L + private var previousCpuActiveTime = 0L + + override val cpuIdleTime: Long + get() = _cpuIdleTime - previousCpuIdleTime + private var _cpuIdleTime = 0L + private var previousCpuIdleTime = 0L + + override val cpuStealTime: Long + get() = _cpuStealTime - previousCpuStealTime + private var _cpuStealTime = 0L + private var previousCpuStealTime = 0L + + override val cpuLostTime: Long + get() = _cpuLostTime - previousCpuLostTime + private var _cpuLostTime = 0L + private var previousCpuLostTime = 0L + + override val bootTimeAbsolute: Instant? + get() = _bootTimeAbsolute + private var _bootTimeAbsolute: Instant? = null + + override val taskState: TaskState? + get() = _taskState + private var _taskState: TaskState? = null + + /** + * Record the next cycle. + */ + fun record(now: Instant) { + val newHost = service.lookupHost(task) + if (newHost != null && newHost.getUid() != _host?.getUid()) { + _host = newHost + host = + HostInfo( + newHost.getUid().toString(), + newHost.getName(), + "x86", + newHost.getModel().coreCount, + newHost.getModel().cpuCapacity, + newHost.getModel().memoryCapacity, + ) + } + + val cpuStats = _host?.getCpuStats(task) + val sysStats = _host?.getSystemStats(task) + + _timestamp = now + _timestampAbsolute = now + startTime + + _cpuLimit = cpuStats?.capacity ?: 0.0f + _cpuActiveTime = cpuStats?.activeTime ?: 0 + _cpuIdleTime = cpuStats?.idleTime ?: 0 + _cpuStealTime = cpuStats?.stealTime ?: 0 + _cpuLostTime = cpuStats?.lostTime ?: 0 + _uptime = sysStats?.uptime?.toMillis() ?: 0 + _downtime = sysStats?.downtime?.toMillis() ?: 0 + _provisionTime = task.launchedAt + _bootTime = sysStats?.bootTime + _creationTime = task.createdAt + _finishTime = task.finishedAt + + _taskState = task.state + + if (sysStats != null) { + _bootTimeAbsolute = sysStats.bootTime + startTime + } else { + _bootTimeAbsolute = null + } + } + + /** + * Finish the aggregation for this cycle. + */ + fun reset() { + previousUptime = _uptime + previousDowntime = _downtime + previousCpuActiveTime = _cpuActiveTime + previousCpuIdleTime = _cpuIdleTime + previousCpuStealTime = _cpuStealTime + previousCpuLostTime = _cpuLostTime + + _host = null + _cpuLimit = 0.0f + } + } +} diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/telemetry/ComputeMonitor.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/telemetry/ComputeMonitor.kt new file mode 100644 index 00000000..534bcc09 --- /dev/null +++ b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/telemetry/ComputeMonitor.kt @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2022 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.compute.simulator.telemetry + +import org.opendc.compute.simulator.telemetry.table.HostTableReader +import org.opendc.compute.simulator.telemetry.table.ServiceTableReader +import org.opendc.compute.simulator.telemetry.table.TaskTableReader + +/** + * A monitor that tracks the metrics and events of the OpenDC Compute service. + */ +public interface ComputeMonitor { + /** + * Record an entry with the specified [reader]. + */ + public fun record(reader: TaskTableReader) {} + + /** + * Record an entry with the specified [reader]. + */ + public fun record(reader: HostTableReader) {} + + /** + * Record an entry with the specified [reader]. + */ + public fun record(reader: ServiceTableReader) {} +} diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/telemetry/parquet/ComputeExportConfig.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/telemetry/parquet/ComputeExportConfig.kt new file mode 100644 index 00000000..3f220ad1 --- /dev/null +++ b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/telemetry/parquet/ComputeExportConfig.kt @@ -0,0 +1,192 @@ +/* + * Copyright (c) 2024 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.compute.simulator.telemetry.parquet + +import kotlinx.serialization.KSerializer +import kotlinx.serialization.Serializable +import kotlinx.serialization.builtins.ListSerializer +import kotlinx.serialization.descriptors.SerialDescriptor +import kotlinx.serialization.descriptors.buildClassSerialDescriptor +import kotlinx.serialization.encoding.Decoder +import kotlinx.serialization.encoding.Encoder +import kotlinx.serialization.encoding.encodeStructure +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.JsonDecoder +import kotlinx.serialization.json.JsonElement +import kotlinx.serialization.json.jsonObject +import org.opendc.common.logger.logger +import org.opendc.compute.simulator.telemetry.table.HostTableReader +import org.opendc.compute.simulator.telemetry.table.ServiceTableReader +import org.opendc.compute.simulator.telemetry.table.TaskTableReader +import org.opendc.trace.util.parquet.exporter.ColListSerializer +import org.opendc.trace.util.parquet.exporter.ExportColumn +import org.opendc.trace.util.parquet.exporter.Exportable +import org.opendc.trace.util.parquet.exporter.columnSerializer + +/** + * Aggregates the necessary settings to personalize the output + * parquet files for compute workloads. + * + * @param[hostExportColumns] the columns that will be included in the `host.parquet` raw output file. + * @param[taskExportColumns] the columns that will be included in the `task.parquet` raw output file. + * @param[serviceExportColumns] the columns that will be included in the `service.parquet` raw output file. + */ +@Serializable(with = ComputeExportConfig.Companion.ComputeExportConfigSerializer::class) +public data class ComputeExportConfig( + public val hostExportColumns: Set>, + public val taskExportColumns: Set>, + public val serviceExportColumns: Set>, +) { + public constructor( + hostExportColumns: Collection>, + taskExportColumns: Collection>, + serviceExportColumns: Collection>, + ) : this( + hostExportColumns.toSet() + DfltHostExportColumns.BASE_EXPORT_COLUMNS, + taskExportColumns.toSet() + DfltTaskExportColumns.BASE_EXPORT_COLUMNS, + serviceExportColumns.toSet() + DfltServiceExportColumns.BASE_EXPORT_COLUMNS, + ) + + /** + * @return formatted string representing the export config. + */ + public fun fmt(): String = + """ + | === Compute Export Config === + | Host columns : ${hostExportColumns.map { it.name }.toString().trim('[', ']')} + | Task columns : ${taskExportColumns.map { it.name }.toString().trim('[', ']')} + | Service columns : ${serviceExportColumns.map { it.name }.toString().trim('[', ']')} + """.trimIndent() + + public companion object { + internal val LOG by logger() + + /** + * Force the jvm to load the default [ExportColumn]s relevant to compute export, + * so that they are available for deserialization. + */ + public fun loadDfltColumns() { + DfltHostExportColumns + DfltTaskExportColumns + DfltServiceExportColumns + } + + /** + * Config that includes all columns defined in [DfltHostExportColumns], + * [DfltTaskExportColumns], [DfltServiceExportColumns] among all other loaded + * columns for [HostTableReader], [TaskTableReader] and [ServiceTableReader]. + */ + public val ALL_COLUMNS: ComputeExportConfig by lazy { + ComputeExportConfig.Companion.loadDfltColumns() + ComputeExportConfig( + hostExportColumns = ExportColumn.getAllLoadedColumns(), + taskExportColumns = ExportColumn.getAllLoadedColumns(), + serviceExportColumns = ExportColumn.getAllLoadedColumns(), + ) + } + + /** + * A runtime [KSerializer] is needed for reasons explained in [columnSerializer] docs. + * + * This serializer makes use of reified column serializers for the 2 properties. + */ + internal object ComputeExportConfigSerializer : KSerializer { + override val descriptor: SerialDescriptor = + buildClassSerialDescriptor("org.opendc.compute.telemetry.export.parquet.ComputeExportConfig") { + element( + "hostExportColumns", + ListSerializer(columnSerializer()).descriptor, + ) + element( + "taskExportColumns", + ListSerializer(columnSerializer()).descriptor, + ) + element( + "serviceExportColumns", + ListSerializer(columnSerializer()).descriptor, + ) + } + + override fun deserialize(decoder: Decoder): ComputeExportConfig { + val jsonDec = + (decoder as? JsonDecoder) ?: let { + // Basically a recursive call with a JsonDecoder. + return json.decodeFromString(decoder.decodeString().trim('"')) + } + + // Loads the default columns so that they are available for deserialization. + ComputeExportConfig.Companion.loadDfltColumns() + val elem = jsonDec.decodeJsonElement().jsonObject + + val hostFields: List> = elem["hostExportColumns"].toFieldList() + val taskFields: List> = elem["taskExportColumns"].toFieldList() + val serviceFields: List> = elem["serviceExportColumns"].toFieldList() + + return ComputeExportConfig( + hostExportColumns = hostFields, + taskExportColumns = taskFields, + serviceExportColumns = serviceFields, + ) + } + + override fun serialize( + encoder: Encoder, + value: ComputeExportConfig, + ) { + encoder.encodeStructure(ComputeExportConfig.Companion.ComputeExportConfigSerializer.descriptor) { + encodeSerializableElement( + ComputeExportConfig.Companion.ComputeExportConfigSerializer.descriptor, + 0, + ColListSerializer(columnSerializer()), + value.hostExportColumns.toList(), + ) + encodeSerializableElement( + ComputeExportConfig.Companion.ComputeExportConfigSerializer.descriptor, + 1, + ColListSerializer(columnSerializer()), + value.taskExportColumns.toList(), + ) + encodeSerializableElement( + ComputeExportConfig.Companion.ComputeExportConfigSerializer.descriptor, + 2, + ColListSerializer(columnSerializer()), + value.serviceExportColumns.toList(), + ) + } + } + } + } +} + +private val json = Json { ignoreUnknownKeys = true } + +private inline fun JsonElement?.toFieldList(): List> = + this?.let { + json.decodeFromJsonElement(ColListSerializer(columnSerializer()), it) + }?.ifEmpty { + ComputeExportConfig.Companion.LOG.warn( + "deserialized list of export columns for exportable ${T::class.simpleName} " + + "produced empty list, falling back to all loaded columns", + ) + ExportColumn.getAllLoadedColumns() + } ?: ExportColumn.getAllLoadedColumns() diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/telemetry/parquet/DfltHostExportColumns.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/telemetry/parquet/DfltHostExportColumns.kt new file mode 100644 index 00000000..1b76da6b --- /dev/null +++ b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/telemetry/parquet/DfltHostExportColumns.kt @@ -0,0 +1,195 @@ +/* + * Copyright (c) 2024 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.compute.simulator.telemetry.parquet + +import org.apache.parquet.io.api.Binary +import org.apache.parquet.schema.LogicalTypeAnnotation +import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.BINARY +import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.FLOAT +import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.INT32 +import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.INT64 +import org.apache.parquet.schema.Types +import org.opendc.compute.simulator.telemetry.table.HostTableReader +import org.opendc.trace.util.parquet.exporter.ExportColumn + +/** + * This object wraps the [ExportColumn]s to solves ambiguity for field + * names that are included in more than 1 exportable. + * + * Additionally, it allows to load all the fields at once by just its symbol, + * so that these columns can be deserialized. Additional fields can be added + * from anywhere, and they are deserializable as long as they are loaded by the jvm. + * + * ```kotlin + * ... + * // Loads the column + * DfltHostExportColumns + * ... + * ``` + */ +public object DfltHostExportColumns { + public val TIMESTAMP: ExportColumn = + ExportColumn( + field = Types.required(INT64).named("timestamp"), + ) { it.timestamp.toEpochMilli() } + + public val TIMESTAMP_ABS: ExportColumn = + ExportColumn( + field = Types.required(INT64).named("timestamp_absolute"), + ) { it.timestampAbsolute.toEpochMilli() } + + public val HOST_ID: ExportColumn = + ExportColumn( + field = + Types.required(BINARY) + .`as`(LogicalTypeAnnotation.stringType()) + .named("host_id"), + ) { Binary.fromString(it.host.id) } + + public val HOST_NAME: ExportColumn = + ExportColumn( + field = + Types.required(BINARY) + .`as`(LogicalTypeAnnotation.stringType()) + .named("host_name"), + ) { Binary.fromString(it.host.name) } + + public val CPU_COUNT: ExportColumn = + ExportColumn( + field = Types.required(INT32).named("core_count"), + ) { it.host.coreCount } + + public val MEM_CAPACITY: ExportColumn = + ExportColumn( + field = Types.required(INT64).named("mem_capacity"), + ) { it.host.memCapacity } + + public val GUESTS_TERMINATED: ExportColumn = + ExportColumn( + field = Types.required(INT32).named("guests_terminated"), + ) { it.guestsTerminated } + + public val GUESTS_RUNNING: ExportColumn = + ExportColumn( + field = Types.required(INT32).named("guests_running"), + ) { it.guestsRunning } + + public val GUESTS_ERROR: ExportColumn = + ExportColumn( + field = Types.required(INT32).named("guests_error"), + ) { it.guestsError } + + public val GUESTS_INVALID: ExportColumn = + ExportColumn( + field = Types.required(INT32).named("guests_invalid"), + ) { it.guestsInvalid } + + public val CPU_LIMIT: ExportColumn = + ExportColumn( + field = Types.required(FLOAT).named("cpu_limit"), + ) { it.cpuLimit } + + public val CPU_USAGE: ExportColumn = + ExportColumn( + field = Types.required(FLOAT).named("cpu_usage"), + ) { it.cpuUsage } + + public val CPU_DEMAND: ExportColumn = + ExportColumn( + field = Types.required(FLOAT).named("cpu_demand"), + ) { it.cpuDemand } + + public val CPU_UTILIZATION: ExportColumn = + ExportColumn( + field = Types.required(FLOAT).named("cpu_utilization"), + ) { it.cpuUtilization } + + public val CPU_TIME_ACTIVE: ExportColumn = + ExportColumn( + field = Types.required(INT64).named("cpu_time_active"), + ) { it.cpuActiveTime } + + public val CPU_TIME_IDLE: ExportColumn = + ExportColumn( + field = Types.required(INT64).named("cpu_time_idle"), + ) { it.cpuIdleTime } + + public val CPU_TIME_STEAL: ExportColumn = + ExportColumn( + field = Types.required(INT64).named("cpu_time_steal"), + ) { it.cpuStealTime } + + public val CPU_TIME_LOST: ExportColumn = + ExportColumn( + field = Types.required(INT64).named("cpu_time_lost"), + ) { it.cpuLostTime } + + public val POWER_DRAW: ExportColumn = + ExportColumn( + field = Types.required(FLOAT).named("power_draw"), + ) { it.powerDraw } + + public val ENERGY_USAGE: ExportColumn = + ExportColumn( + field = Types.required(FLOAT).named("energy_usage"), + ) { it.energyUsage } + + public val CARBON_INTENSITY: ExportColumn = + ExportColumn( + field = Types.required(FLOAT).named("carbon_intensity"), + ) { it.carbonIntensity } + + public val CARBON_EMISSION: ExportColumn = + ExportColumn( + field = Types.required(FLOAT).named("carbon_emission"), + ) { it.carbonEmission } + + public val UP_TIME: ExportColumn = + ExportColumn( + field = Types.required(INT64).named("uptime"), + ) { it.uptime } + + public val DOWN_TIME: ExportColumn = + ExportColumn( + field = Types.required(INT64).named("downtime"), + ) { it.downtime } + + public val BOOT_TIME: ExportColumn = + ExportColumn( + field = Types.optional(INT64).named("boot_time"), + ) { it.bootTime?.toEpochMilli() } + + public val BOOT_TIME_ABS: ExportColumn = + ExportColumn( + field = Types.optional(INT64).named("boot_time_absolute"), + ) { it.bootTimeAbsolute?.toEpochMilli() } + + /** + * The columns that are always included in the output file. + */ + internal val BASE_EXPORT_COLUMNS = + setOf( + TIMESTAMP_ABS, + TIMESTAMP, + ) +} diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/telemetry/parquet/DfltServiceExportColumns.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/telemetry/parquet/DfltServiceExportColumns.kt new file mode 100644 index 00000000..aa08e8ff --- /dev/null +++ b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/telemetry/parquet/DfltServiceExportColumns.kt @@ -0,0 +1,105 @@ +/* + * Copyright (c) 2024 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.compute.simulator.telemetry.parquet + +import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.INT32 +import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.INT64 +import org.apache.parquet.schema.Types +import org.opendc.compute.simulator.telemetry.table.ServiceTableReader +import org.opendc.trace.util.parquet.exporter.ExportColumn + +/** + * This object wraps the [ExportColumn]s to solves ambiguity for field + * names that are included in more than 1 exportable. + * + * Additionally, it allows to load all the fields at once by just its symbol, + * so that these columns can be deserialized. Additional fields can be added + * from anywhere, and they are deserializable as long as they are loaded by the jvm. + * + * ```kotlin + * ... + * // Loads the column + * DfltServiceExportColumns + * ... + * ``` + */ +public object DfltServiceExportColumns { + public val TIMESTAMP: ExportColumn = + ExportColumn( + field = Types.required(INT64).named("timestamp"), + ) { it.timestamp.toEpochMilli() } + + public val TIMESTAMP_ABS: ExportColumn = + ExportColumn( + field = Types.required(INT64).named("timestamp_absolute"), + ) { it.timestampAbsolute.toEpochMilli() } + + public val HOSTS_UP: ExportColumn = + ExportColumn( + field = Types.required(INT32).named("hosts_up"), + ) { it.hostsUp } + + public val TASKS_PENDING: ExportColumn = + ExportColumn( + field = Types.required(INT32).named("tasks_pending"), + ) { it.tasksPending } + + public val TASKS_TOTAL: ExportColumn = + ExportColumn( + field = Types.required(INT32).named("tasks_total"), + ) { it.tasksTotal } + + public val TASKS_ACTIVE: ExportColumn = + ExportColumn( + field = Types.required(INT32).named("tasks_active"), + ) { it.tasksActive } + + public val TASKS_COMPLETED: ExportColumn = + ExportColumn( + field = Types.required(INT32).named("tasks_completed"), + ) { it.tasksCompleted } + + public val TASKS_FAILED: ExportColumn = + ExportColumn( + field = Types.required(INT32).named("tasks_terminated"), + ) { it.tasksTerminated } + + public val ATTEMPTS_SUCCESS: ExportColumn = + ExportColumn( + field = Types.required(INT32).named("attempts_success"), + ) { it.attemptsSuccess } + + public val ATTEMPTS_FAILURE: ExportColumn = + ExportColumn( + field = Types.required(INT32).named("attempts_failure"), + ) { it.attemptsFailure } + + /** + * The columns that are always included in the output file. + */ + internal val BASE_EXPORT_COLUMNS = + setOf( + TIMESTAMP_ABS, + TIMESTAMP, + ) +} diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/telemetry/parquet/DfltTaskExportColumns.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/telemetry/parquet/DfltTaskExportColumns.kt new file mode 100644 index 00000000..6658e444 --- /dev/null +++ b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/telemetry/parquet/DfltTaskExportColumns.kt @@ -0,0 +1,171 @@ +/* + * Copyright (c) 2024 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.compute.simulator.telemetry.parquet + +import org.apache.parquet.io.api.Binary +import org.apache.parquet.schema.LogicalTypeAnnotation +import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.BINARY +import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.FLOAT +import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.INT32 +import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.INT64 +import org.apache.parquet.schema.Types +import org.opendc.compute.simulator.telemetry.table.TaskTableReader +import org.opendc.trace.util.parquet.exporter.ExportColumn + +/** + * This object wraps the [ExportColumn]s to solves ambiguity for field + * names that are included in more than 1 exportable + * + * Additionally, it allows to load all the fields at once by just its symbol, + * so that these columns can be deserialized. Additional fields can be added + * from anywhere, and they are deserializable as long as they are loaded by the jvm. + * + * ```kotlin + * ... + * // Loads the column + * DfltTaskExportColumns + * ... + * ``` + */ +public object DfltTaskExportColumns { + public val TIMESTAMP: ExportColumn = + ExportColumn( + field = Types.required(INT64).named("timestamp"), + ) { it.timestamp.toEpochMilli() } + + public val TIMESTAMP_ABS: ExportColumn = + ExportColumn( + field = Types.required(INT64).named("timestamp_absolute"), + ) { it.timestampAbsolute.toEpochMilli() } + + public val TASK_ID: ExportColumn = + ExportColumn( + field = + Types.required(BINARY) + .`as`(LogicalTypeAnnotation.stringType()) + .named("task_id"), + ) { Binary.fromString(it.taskInfo.id) } + + public val HOST_ID: ExportColumn = + ExportColumn( + field = + Types.optional(BINARY) + .`as`(LogicalTypeAnnotation.stringType()) + .named("host_id"), + ) { it.host?.id?.let { Binary.fromString(it) } } + + public val TASK_NAME: ExportColumn = + ExportColumn( + field = + Types.required(BINARY) + .`as`(LogicalTypeAnnotation.stringType()) + .named("task_name"), + ) { Binary.fromString(it.taskInfo.name) } + + public val CPU_COUNT: ExportColumn = + ExportColumn( + field = Types.required(INT32).named("cpu_count"), + ) { it.taskInfo.cpuCount } + + public val MEM_CAPACITY: ExportColumn = + ExportColumn( + field = Types.required(INT64).named("mem_capacity"), + ) { it.taskInfo.memCapacity } + + public val CPU_LIMIT: ExportColumn = + ExportColumn( + field = Types.required(FLOAT).named("cpu_limit"), + ) { it.cpuLimit } + + public val CPU_TIME_ACTIVE: ExportColumn = + ExportColumn( + field = Types.required(INT64).named("cpu_time_active"), + ) { it.cpuActiveTime } + + public val CPU_TIME_IDLE: ExportColumn = + ExportColumn( + field = Types.required(INT64).named("cpu_time_idle"), + ) { it.cpuIdleTime } + + public val CPU_TIME_STEAL: ExportColumn = + ExportColumn( + field = Types.required(INT64).named("cpu_time_steal"), + ) { it.cpuStealTime } + + public val CPU_TIME_LOST: ExportColumn = + ExportColumn( + field = Types.required(INT64).named("cpu_time_lost"), + ) { it.cpuLostTime } + + public val UP_TIME: ExportColumn = + ExportColumn( + field = Types.required(INT64).named("uptime"), + ) { it.uptime } + + public val DOWN_TIME: ExportColumn = + ExportColumn( + field = Types.required(INT64).named("downtime"), + ) { it.downtime } + + public val PROVISION_TIME: ExportColumn = + ExportColumn( + field = Types.optional(INT64).named("provision_time"), + ) { it.provisionTime?.toEpochMilli() } + + public val BOOT_TIME: ExportColumn = + ExportColumn( + field = Types.optional(INT64).named("boot_time"), + ) { it.bootTime?.toEpochMilli() } + + public val CREATION_TIME: ExportColumn = + ExportColumn( + field = Types.optional(INT64).named("creation_time"), + ) { it.creationTime?.toEpochMilli() } + + public val FINISH_TIME: ExportColumn = + ExportColumn( + field = Types.optional(INT64).named("finish_time"), + ) { it.finishTime?.toEpochMilli() } + + public val BOOT_TIME_ABS: ExportColumn = + ExportColumn( + field = Types.optional(INT64).named("boot_time_absolute"), + ) { it.bootTimeAbsolute?.toEpochMilli() } + + public val TASK_STATE: ExportColumn = + ExportColumn( + field = + Types.optional(BINARY) + .`as`(LogicalTypeAnnotation.stringType()) + .named("task_state"), + ) { Binary.fromString(it.taskState?.name) } + + /** + * The columns that are always included in the output file. + */ + internal val BASE_EXPORT_COLUMNS = + setOf( + TIMESTAMP_ABS, + TIMESTAMP, + ) +} diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/telemetry/parquet/ParquetComputeMonitor.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/telemetry/parquet/ParquetComputeMonitor.kt new file mode 100644 index 00000000..4cd920c4 --- /dev/null +++ b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/telemetry/parquet/ParquetComputeMonitor.kt @@ -0,0 +1,125 @@ +/* + * Copyright (c) 2022 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.compute.simulator.telemetry.parquet + +import org.opendc.compute.simulator.telemetry.ComputeMonitor +import org.opendc.compute.simulator.telemetry.table.HostTableReader +import org.opendc.compute.simulator.telemetry.table.ServiceTableReader +import org.opendc.compute.simulator.telemetry.table.TaskTableReader +import org.opendc.trace.util.parquet.exporter.ExportColumn +import org.opendc.trace.util.parquet.exporter.Exportable +import org.opendc.trace.util.parquet.exporter.Exporter +import java.io.File + +/** + * A [ComputeMonitor] that logs the events to a Parquet file. + */ +public class ParquetComputeMonitor( + private val hostExporter: Exporter, + private val taskExporter: Exporter, + private val serviceExporter: Exporter, +) : ComputeMonitor, AutoCloseable { + override fun record(reader: HostTableReader) { + hostExporter.write(reader) + } + + override fun record(reader: TaskTableReader) { + taskExporter.write(reader) + } + + override fun record(reader: ServiceTableReader) { + serviceExporter.write(reader) + } + + override fun close() { + hostExporter.close() + taskExporter.close() + serviceExporter.close() + } + + public companion object { + /** + * Overloaded constructor with [ComputeExportConfig] as parameter. + * + * @param[base] parent pathname for output file. + * @param[partition] child pathname for output file. + * @param[bufferSize] size of the buffer used by the writer thread. + */ + public operator fun invoke( + base: File, + partition: String, + bufferSize: Int, + computeExportConfig: ComputeExportConfig, + ): ParquetComputeMonitor = + invoke( + base = base, + partition = partition, + bufferSize = bufferSize, + hostExportColumns = computeExportConfig.hostExportColumns, + taskExportColumns = computeExportConfig.taskExportColumns, + serviceExportColumns = computeExportConfig.serviceExportColumns, + ) + + /** + * Constructor that loads default [ExportColumn]s defined in + * [DfltHostExportColumns], [DfltTaskExportColumns], [DfltServiceExportColumns] + * in case optional parameters are omitted and all fields need to be retrieved. + * + * @param[base] parent pathname for output file. + * @param[partition] child pathname for output file. + * @param[bufferSize] size of the buffer used by the writer thread. + */ + public operator fun invoke( + base: File, + partition: String, + bufferSize: Int, + hostExportColumns: Collection>? = null, + taskExportColumns: Collection>? = null, + serviceExportColumns: Collection>? = null, + ): ParquetComputeMonitor { + // Loads the fields in case they need to be retrieved if optional params are omitted. + ComputeExportConfig.loadDfltColumns() + + return ParquetComputeMonitor( + hostExporter = + Exporter( + outputFile = File(base, "$partition/host.parquet").also { it.parentFile.mkdirs() }, + columns = hostExportColumns ?: Exportable.getAllLoadedColumns(), + bufferSize = bufferSize, + ), + taskExporter = + Exporter( + outputFile = File(base, "$partition/task.parquet").also { it.parentFile.mkdirs() }, + columns = taskExportColumns ?: Exportable.getAllLoadedColumns(), + bufferSize = bufferSize, + ), + serviceExporter = + Exporter( + outputFile = File(base, "$partition/service.parquet").also { it.parentFile.mkdirs() }, + columns = serviceExportColumns ?: Exportable.getAllLoadedColumns(), + bufferSize = bufferSize, + ), + ) + } + } +} diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/telemetry/parquet/README.md b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/telemetry/parquet/README.md new file mode 100644 index 00000000..3baafed4 --- /dev/null +++ b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/telemetry/parquet/README.md @@ -0,0 +1,70 @@ +### Summary +Added output configuration, that can be defined in the scenario `.json` file, that allows to select which columns are to be included in the raw oputput files `host.parquet`, `task.parquet` and `service.parquet`. + +### Columns +The 'default' columns are defined in `DfltHostExportcolumns`, `DfltTaskExportColumns` and `DfltServiceExportColumns`. Any number of additional columns can be definied anywhere (`ExportColumn`) and it is going to be deserializable as long as it is loaded by the jvm. + +### Deserialization +Each `ExportColumn` has a `Regex`, used for deserialization. If no custom regex is provided, the default one is used. The default regex matches the column name in case-insensitive manner, either with `_` as in the name or with ` ` (blank space). + +###### E.g.: +***column name*** = "cpuModel\_count" +***default column regex*** = "\\s*(?:cpu_count|cpuModel count)\\s*" (case insensitive) +***matches*** = "cpuModel\_count", "cpuModel count", "CpU/_cOuNt" etc. + +### JSON Schema +```json +// scenario.json +{ + ... + "computeExportConfig": { + "type": "object", + "properties": { + "hostExportColumns": { "type": "array" }, + "taskExportColumns": { "type": "array" } , + "serviceExportColumns": { "type": "array" } , + "required": [ /* NONE REQUIRED */ ] + } + }, + ... + "required": [ + ... + // NOT REQUIRED + ] +} +``` + +  +###### Bad Formatting Cases +- If a column name (and type) does not match any deserializable column, the entry is ignored and error message is logged. +- If an empty list of columns is provided or those that are provided were not deserializable, then all loaded columns for that `Exportable` are used, and a warning message is logged. +- If no list is provided, then all loaded columns for that `Exportable` are used. + + +### Example + +```json +// scenario.json +{ + ... + "computeExportConfig": { + "hostExportColumns": ["timestamp", "timestamp_absolute", "invalid-entry1", "guests_invalid"], + "taskExportColumns": ["invalid-entry2"], + "serviceExportColumns": ["timestamp", "tasks_active", "tasks_pending"] + }, + ... +``` + +```json +// console output +10:51:56.561 [ERROR] ColListSerializer - no match found for column "invalid-entry1", ignoring... +10:51:56.563 [ERROR] ColListSerializer - no match found for column "invalid-entry2", ignoring... +10:51:56.564 [WARN] ComputeExportConfig - deserialized list of export columns for exportable TaskTableReader produced empty list, falling back to all loaded columns +10:51:56.584 [INFO] ScenariosSpec - +| === Compute Export Config === +| Host columns : timestamp, timestamp_absolute, guests_invalid +| Task columns : timestamp, timestamp_absolute, task_id, task_name, cpu_count, mem_capacity, cpu_limit, cpu_time_active, cpu_time_idle, cpu_time_steal, cpu_time_lost, uptime, downtime, provision_time, boot_time, boot_time_absolute +| Service columns : timestamp, tasks_active, tasks_pending + +``` + diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/telemetry/table/HostInfo.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/telemetry/table/HostInfo.kt new file mode 100644 index 00000000..1f1b9522 --- /dev/null +++ b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/telemetry/table/HostInfo.kt @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.compute.simulator.telemetry.table + +/** + * Information about a host exposed to the telemetry service. + */ +public data class HostInfo( + val id: String, + val name: String, + val arch: String, + val coreCount: Int, + val coreSpeed: Float, + val memCapacity: Long, +) diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/telemetry/table/HostTableReader.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/telemetry/table/HostTableReader.kt new file mode 100644 index 00000000..5f09e7f5 --- /dev/null +++ b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/telemetry/table/HostTableReader.kt @@ -0,0 +1,150 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.compute.simulator.telemetry.table + +import org.opendc.trace.util.parquet.exporter.Exportable +import java.time.Instant + +/** + * An interface that is used to read a row of a host trace entry. + */ +public interface HostTableReader : Exportable { + public fun copy(): HostTableReader + + public fun setValues(table: HostTableReader) + + /** + * The [HostInfo] of the host to which the row belongs to. + */ + public val host: HostInfo + + /** + * The timestamp of the current entry of the reader relative to the start of the workload. + */ + public val timestamp: Instant + + /** + * The timestamp of the current entry of the reader. + */ + public val timestampAbsolute: Instant + + /** + * The number of guests that are in a terminated state. + */ + public val guestsTerminated: Int + + /** + * The number of guests that are in a running state. + */ + public val guestsRunning: Int + + /** + * The number of guests that are in an error state. + */ + public val guestsError: Int + + /** + * The number of guests that are in an unknown state. + */ + public val guestsInvalid: Int + + /** + * The capacity of the CPUs in the host (in MHz). + */ + public val cpuLimit: Float + + /** + * The usage of all CPUs in the host (in MHz). + */ + public val cpuUsage: Float + + /** + * The demand of all vCPUs of the guests (in MHz) + */ + public val cpuDemand: Float + + /** + * The CPU utilization of the host. + */ + public val cpuUtilization: Float + + /** + * The duration (in ms) that a CPU was active in the host. + */ + public val cpuActiveTime: Long + + /** + * The duration (in ms) that a CPU was idle in the host. + */ + public val cpuIdleTime: Long + + /** + * The duration (in ms) that a vCPU wanted to run, but no capacity was available. + */ + public val cpuStealTime: Long + + /** + * The duration (in ms) of CPU time that was lost due to interference. + */ + public val cpuLostTime: Long + + /** + * The current power draw of the host in W. + */ + public val powerDraw: Float + + /** + * The total energy consumption of the host since last sample in J. + */ + public val energyUsage: Float + + /** + * The current carbon intensity of the host in gCO2 / kW. + */ + public val carbonIntensity: Float + + /** + * The current carbon emission since the last deadline in g. + */ + public val carbonEmission: Float + + /** + * The uptime of the host since last time in ms. + */ + public val uptime: Long + + /** + * The downtime of the host since last time in ms. + */ + public val downtime: Long + + /** + * The [Instant] at which the host booted relative to the start of the workload. + */ + public val bootTime: Instant? + + /** + * The [Instant] at which the host booted. + */ + public val bootTimeAbsolute: Instant? +} diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/telemetry/table/ServiceData.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/telemetry/table/ServiceData.kt new file mode 100644 index 00000000..16c38297 --- /dev/null +++ b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/telemetry/table/ServiceData.kt @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.compute.simulator.telemetry.table + +import java.time.Instant + +/** + * A trace entry for the compute service. + */ +public data class ServiceData( + val timestamp: Instant, + val hostsUp: Int, + val hostsDown: Int, + val tasksTotal: Int, + val tasksPending: Int, + val tasksActive: Int, + val attemptsSuccess: Int, + val attemptsTerminated: Int, +) + +/** + * Convert a [ServiceTableReader] into a persistent object. + */ +public fun ServiceTableReader.toServiceData(): ServiceData { + return ServiceData( + timestamp, + hostsUp, + hostsDown, + tasksTotal, + tasksPending, + tasksActive, + attemptsSuccess, + attemptsFailure, + ) +} diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/telemetry/table/ServiceTableReader.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/telemetry/table/ServiceTableReader.kt new file mode 100644 index 00000000..690dfe0a --- /dev/null +++ b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/telemetry/table/ServiceTableReader.kt @@ -0,0 +1,90 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.compute.simulator.telemetry.table + +import org.opendc.trace.util.parquet.exporter.Exportable +import java.time.Instant + +/** + * An interface that is used to read a row of a service trace entry. + */ +public interface ServiceTableReader : Exportable { + public fun copy(): ServiceTableReader + + public fun setValues(table: ServiceTableReader) + + /** + * The timestamp of the current entry of the reader. + */ + public val timestamp: Instant + + /** + * The timestamp of the current entry of the reader. + */ + public val timestampAbsolute: Instant + + /** + * The number of hosts that are up at this instant. + */ + public val hostsUp: Int + + /** + * The number of hosts that are down at this instant. + */ + public val hostsDown: Int + + /** + * The number of tasks that are registered with the compute service. + */ + public val tasksTotal: Int + + /** + * The number of tasks that are pending to be scheduled. + */ + public val tasksPending: Int + + /** + * The number of tasks that are currently active. + */ + public val tasksActive: Int + + /** + * The number of tasks that completed the tasks successfully + */ + public val tasksCompleted: Int + + /** + * The number of tasks that failed more times than allowed and are thus terminated + */ + public val tasksTerminated: Int + + /** + * The scheduling attempts that were successful. + */ + public val attemptsSuccess: Int + + /** + * The scheduling attempts that were unsuccessful due to client error. + */ + public val attemptsFailure: Int +} diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/telemetry/table/TaskInfo.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/telemetry/table/TaskInfo.kt new file mode 100644 index 00000000..6ff56541 --- /dev/null +++ b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/telemetry/table/TaskInfo.kt @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.compute.simulator.telemetry.table + +/** + * Static information about a task exposed to the telemetry service. + */ +public data class TaskInfo( + val id: String, + val name: String, + val type: String, + val arch: String, + val cpuCount: Int, + val memCapacity: Long, +) diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/telemetry/table/TaskTableReader.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/telemetry/table/TaskTableReader.kt new file mode 100644 index 00000000..bc6a4edd --- /dev/null +++ b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/telemetry/table/TaskTableReader.kt @@ -0,0 +1,125 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.compute.simulator.telemetry.table + +import org.opendc.compute.api.TaskState +import org.opendc.compute.simulator.telemetry.parquet.DfltTaskExportColumns +import org.opendc.trace.util.parquet.exporter.Exportable +import java.time.Instant + +/** + * An interface that is used to read a row of a task trace entry. + */ +public interface TaskTableReader : Exportable { + public fun copy(): TaskTableReader + + public fun setValues(table: TaskTableReader) + + /** + * The timestamp of the current entry of the reader relative to the start of the workload. + */ + public val timestamp: Instant + + /** + * The timestamp of the current entry of the reader. + */ + public val timestampAbsolute: Instant + + /** + * The [TaskInfo] of the task to which the row belongs to. + */ + public val taskInfo: TaskInfo + + /** + * The [HostInfo] of the host on which the task is hosted or `null` if it has no host. + */ + public val host: HostInfo? + + /** + * The uptime of the host since last time in ms. + */ + public val uptime: Long + + /** + * The downtime of the host since last time in ms. + */ + public val downtime: Long + + /** + * The [Instant] at which the task was enqueued for the scheduler. + */ + public val provisionTime: Instant? + + /** + * The [Instant] at which the task booted relative to the start of the workload. + */ + public val bootTime: Instant? + + /** + * The [Instant] at which the task booted. + */ + public val bootTimeAbsolute: Instant? + + /** + * The [Instant] at which the task booted relative to the start of the workload. + */ + public val creationTime: Instant? + + /** + * The [Instant] at which the task booted relative to the start of the workload. + */ + public val finishTime: Instant? + + /** + * The capacity of the CPUs of Host on which the task is running (in MHz). + */ + public val cpuLimit: Float + + /** + * The duration (in seconds) that a CPU was active in the task. + */ + public val cpuActiveTime: Long + + /** + * The duration (in seconds) that a CPU was idle in the task. + */ + public val cpuIdleTime: Long + + /** + * The duration (in seconds) that a vCPU wanted to run, but no capacity was available. + */ + public val cpuStealTime: Long + + /** + * The duration (in seconds) of CPU time that was lost due to interference. + */ + public val cpuLostTime: Long + + /** + * The state of the task + */ + public val taskState: TaskState? +} + +// Loads the default export fields for deserialization whenever this file is loaded. +private val _ignore = DfltTaskExportColumns diff --git a/opendc-compute/opendc-compute-simulator/src/test/kotlin/org/opendc/compute/simulator/SimHostTest.kt b/opendc-compute/opendc-compute-simulator/src/test/kotlin/org/opendc/compute/simulator/SimHostTest.kt deleted file mode 100644 index b5bc66a9..00000000 --- a/opendc-compute/opendc-compute-simulator/src/test/kotlin/org/opendc/compute/simulator/SimHostTest.kt +++ /dev/null @@ -1,393 +0,0 @@ -/* - * Copyright (c) 2020 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.compute.simulator - -import kotlinx.coroutines.coroutineScope -import kotlinx.coroutines.delay -import kotlinx.coroutines.suspendCancellableCoroutine -import org.junit.jupiter.api.Assertions.assertEquals -import org.junit.jupiter.api.BeforeEach -import org.junit.jupiter.api.Test -import org.junit.jupiter.api.assertAll -import org.opendc.compute.api.Flavor -import org.opendc.compute.api.Image -import org.opendc.compute.api.Task -import org.opendc.compute.api.TaskState -import org.opendc.compute.api.TaskWatcher -import org.opendc.compute.service.driver.Host -import org.opendc.compute.service.driver.HostListener -import org.opendc.simulator.compute.SimBareMetalMachine -import org.opendc.simulator.compute.kernel.SimHypervisor -import org.opendc.simulator.compute.model.Cpu -import org.opendc.simulator.compute.model.MachineModel -import org.opendc.simulator.compute.model.MemoryUnit -import org.opendc.simulator.compute.workload.SimTrace -import org.opendc.simulator.compute.workload.SimTraceFragment -import org.opendc.simulator.flow2.FlowEngine -import org.opendc.simulator.flow2.mux.FlowMultiplexerFactory -import org.opendc.simulator.kotlin.runSimulation -import java.time.Instant -import java.util.SplittableRandom -import java.util.UUID -import kotlin.coroutines.resume - -/** - * Basic test-suite for the hypervisor. - */ -internal class SimHostTest { - private lateinit var machineModel: MachineModel - - @BeforeEach - fun setUp() { - machineModel = - MachineModel( - Cpu( - 0, - 2, - 3200.0, - "Intel", - "Xeon", - "amd64", - ), - // memory - MemoryUnit("Crucial", "MTA18ASF4G72AZ-3G2B1", 3200.0, 32_000 * 4), - ) - } - - /** - * Test a single virtual machine hosted by the hypervisor. - */ - @Test - fun testSingle() = - runSimulation { - val duration = 5 * 60L - - val engine = FlowEngine.create(dispatcher) - val graph = engine.newGraph() - - val machine = SimBareMetalMachine.create(graph, machineModel) - val hypervisor = SimHypervisor.create(FlowMultiplexerFactory.maxMinMultiplexer(), SplittableRandom(1)) - - val host = - SimHost( - uid = UUID.randomUUID(), - name = "test", - meta = emptyMap(), - timeSource, - machine, - hypervisor, - ) - val vmImage = - MockImage( - UUID.randomUUID(), - "", - emptyMap(), - mapOf( - "workload" to - SimTrace.ofFragments( - SimTraceFragment(0, duration * 1000, 0.0, 2), - SimTraceFragment(duration * 1000, duration * 1000, 3200.0, 2), - SimTraceFragment(duration * 2000, duration * 1000, 0.0, 2), - SimTraceFragment(duration * 3000, duration * 1000, 6500.0, 2), - ).createWorkload(0), - ), - ) - - val flavor = MockFlavor(2, 0) - - suspendCancellableCoroutine { cont -> - host.addListener( - object : HostListener { - private var finished = 0 - - override fun onStateChanged( - host: Host, - task: Task, - newState: TaskState, - ) { - if (newState == TaskState.TERMINATED && ++finished == 1) { - cont.resume(Unit) - } - } - }, - ) - val server = MockTask(UUID.randomUUID(), "a", flavor, vmImage) - host.spawn(server) - host.start(server) - } - - // Ensure last cycle is collected -// delay(1000L * duration) - host.close() - - val cpuStats = host.getCpuStats() - - assertAll( - { assertEquals(450000, cpuStats.activeTime, "Active time does not match") }, - { assertEquals(750000, cpuStats.idleTime, "Idle time does not match") }, - { assertEquals(4688, cpuStats.stealTime, "Steal time does not match") }, - { assertEquals(1200000, timeSource.millis()) }, - ) - } - - /** - * Test overcommitting of resources by the hypervisor. - */ - @Test - fun testOvercommitted() = - runSimulation { - val duration = 5 * 60L - - val engine = FlowEngine.create(dispatcher) - val graph = engine.newGraph() - - val machine = SimBareMetalMachine.create(graph, machineModel) - val hypervisor = SimHypervisor.create(FlowMultiplexerFactory.maxMinMultiplexer(), SplittableRandom(1)) - - val host = - SimHost( - uid = UUID.randomUUID(), - name = "test", - meta = emptyMap(), - timeSource, - machine, - hypervisor, - ) - val vmImageA = - MockImage( - UUID.randomUUID(), - "", - emptyMap(), - mapOf( - "workload" to - SimTrace.ofFragments( - SimTraceFragment(0, duration * 1000, 0.0, 2), - SimTraceFragment(duration * 1000, duration * 1000, 3200.0, 2), - SimTraceFragment(duration * 2000, duration * 1000, 0.0, 2), - SimTraceFragment(duration * 3000, duration * 1000, 6500.0, 2), - ).createWorkload(0), - ), - ) - val vmImageB = - MockImage( - UUID.randomUUID(), - "", - emptyMap(), - mapOf( - "workload" to - SimTrace.ofFragments( - SimTraceFragment(0, duration * 1000, 0.0, 2), - SimTraceFragment(duration * 1000, duration * 1000, 3200.0, 2), - SimTraceFragment(duration * 2000, duration * 1000, 0.0, 2), - SimTraceFragment(duration * 3000, duration * 1000, 6500.0, 2), - ).createWorkload(0), - ), - ) - - val flavor = MockFlavor(2, 0) - - coroutineScope { - suspendCancellableCoroutine { cont -> - host.addListener( - object : HostListener { - private var finished = 0 - - override fun onStateChanged( - host: Host, - task: Task, - newState: TaskState, - ) { - if (newState == TaskState.TERMINATED && ++finished == 2) { - cont.resume(Unit) - } - } - }, - ) - val serverA = MockTask(UUID.randomUUID(), "a", flavor, vmImageA) - host.spawn(serverA) - val serverB = MockTask(UUID.randomUUID(), "b", flavor, vmImageB) - host.spawn(serverB) - - host.start(serverA) - host.start(serverB) - } - } - - // Ensure last cycle is collected - delay(1000L * duration) - host.close() - - val cpuStats = host.getCpuStats() - - assertAll( - { assertEquals(600000, cpuStats.activeTime, "Active time does not match") }, - { assertEquals(900000, cpuStats.idleTime, "Idle time does not match") }, - { assertEquals(309375, cpuStats.stealTime, "Steal time does not match") }, - { assertEquals(1500000, timeSource.millis()) }, - ) - } - - /** - * Test failure of the host. - */ - @Test - fun testFailure() = - runSimulation { - val duration = 5 * 60L - - val engine = FlowEngine.create(dispatcher) - val graph = engine.newGraph() - - val machine = SimBareMetalMachine.create(graph, machineModel) - val hypervisor = SimHypervisor.create(FlowMultiplexerFactory.maxMinMultiplexer(), SplittableRandom(1)) - val host = - SimHost( - uid = UUID.randomUUID(), - name = "test", - meta = emptyMap(), - timeSource, - machine, - hypervisor, - ) - val image = - MockImage( - UUID.randomUUID(), - "", - emptyMap(), - mapOf( - "workload" to - SimTrace.ofFragments( - SimTraceFragment(0, duration * 1000, 0.0, 2), - SimTraceFragment(duration * 1000, duration * 1000, 3200.0, 2), - SimTraceFragment(duration * 2000, duration * 1000, 0.0, 2), - SimTraceFragment(duration * 3000, duration * 1000, 6500.0, 2), - ).createWorkload(0), - ), - ) - val flavor = MockFlavor(2, 0) - val server = MockTask(UUID.randomUUID(), "a", flavor, image) - - coroutineScope { - host.spawn(server) - host.start(server) - delay(5000L) - host.fail() - delay(duration * 1000) - host.recover() - - suspendCancellableCoroutine { cont -> - host.addListener( - object : HostListener { - override fun onStateChanged( - host: Host, - task: Task, - newState: TaskState, - ) { - if (newState == TaskState.TERMINATED) { - cont.resume(Unit) - } - } - }, - ) - } - } - - host.close() - // Ensure last cycle is collected - delay(1000L * duration) - - val cpuStats = host.getCpuStats() - val sysStats = host.getSystemStats() - val guestSysStats = host.getSystemStats(server) - - assertAll( - { assertEquals(755000, cpuStats.idleTime, "Idle time does not match") }, - { assertEquals(450000, cpuStats.activeTime, "Active time does not match") }, - { assertEquals(1205000, sysStats.uptime.toMillis(), "Uptime does not match") }, - { assertEquals(300000, sysStats.downtime.toMillis(), "Downtime does not match") }, - { assertEquals(1205000, guestSysStats.uptime.toMillis(), "Guest uptime does not match") }, - { assertEquals(300000, guestSysStats.downtime.toMillis(), "Guest downtime does not match") }, - ) - } - - private class MockFlavor( - override val coreCount: Int, - override val memorySize: Long, - ) : Flavor { - override val uid: UUID = UUID.randomUUID() - override val name: String = "test" - override val labels: Map = emptyMap() - override val meta: Map = emptyMap() - - override fun delete() { - throw NotImplementedError() - } - - override fun reload() { - throw NotImplementedError() - } - } - - private class MockImage( - override val uid: UUID, - override val name: String, - override val labels: Map, - override val meta: Map, - ) : Image { - override fun delete() { - throw NotImplementedError() - } - - override fun reload() { - throw NotImplementedError() - } - } - - private class MockTask( - override val uid: UUID, - override val name: String, - override val flavor: Flavor, - override val image: Image, - override val numFailures: Int = 10, - ) : Task { - override val labels: Map = emptyMap() - - override val meta: Map = emptyMap() - - override val state: TaskState = TaskState.TERMINATED - - override val launchedAt: Instant? = null - - override fun start() {} - - override fun stop() {} - - override fun delete() {} - - override fun watch(watcher: TaskWatcher) {} - - override fun unwatch(watcher: TaskWatcher) {} - - override fun reload() {} - } -} diff --git a/opendc-compute/opendc-compute-telemetry/build.gradle.kts b/opendc-compute/opendc-compute-telemetry/build.gradle.kts deleted file mode 100644 index e8692449..00000000 --- a/opendc-compute/opendc-compute-telemetry/build.gradle.kts +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright (c) 2021 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -description = "OpenDC Compute Service implementation" - -// Build configuration -plugins { - `kotlin-library-conventions` - kotlin("plugin.serialization") version "1.9.22" -} - -dependencies { - api(projects.opendcCompute.opendcComputeApi) - api(projects.opendcTrace.opendcTraceParquet) - implementation(projects.opendcCommon) - implementation(libs.kotlin.logging) - implementation("org.jetbrains.kotlinx:kotlinx-serialization-json:1.6.0") - implementation(project(mapOf("path" to ":opendc-trace:opendc-trace-parquet"))) - implementation(project(mapOf("path" to ":opendc-compute:opendc-compute-service"))) - implementation(project(mapOf("path" to ":opendc-compute:opendc-compute-carbon"))) - - testImplementation(projects.opendcSimulator.opendcSimulatorCore) - testRuntimeOnly(libs.log4j.core) - testRuntimeOnly(libs.log4j.slf4j) -} diff --git a/opendc-compute/opendc-compute-telemetry/src/main/kotlin/org/opendc/compute/telemetry/ComputeMetricReader.kt b/opendc-compute/opendc-compute-telemetry/src/main/kotlin/org/opendc/compute/telemetry/ComputeMetricReader.kt deleted file mode 100644 index 56cda31c..00000000 --- a/opendc-compute/opendc-compute-telemetry/src/main/kotlin/org/opendc/compute/telemetry/ComputeMetricReader.kt +++ /dev/null @@ -1,594 +0,0 @@ -/* - * Copyright (c) 2022 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.compute.telemetry - -import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.delay -import kotlinx.coroutines.isActive -import kotlinx.coroutines.launch -import mu.KotlinLogging -import org.opendc.common.Dispatcher -import org.opendc.common.asCoroutineDispatcher -import org.opendc.compute.api.Task -import org.opendc.compute.api.TaskState -import org.opendc.compute.carbon.CarbonTrace -import org.opendc.compute.service.ComputeService -import org.opendc.compute.service.driver.Host -import org.opendc.compute.telemetry.table.HostInfo -import org.opendc.compute.telemetry.table.HostTableReader -import org.opendc.compute.telemetry.table.ServiceTableReader -import org.opendc.compute.telemetry.table.TaskInfo -import org.opendc.compute.telemetry.table.TaskTableReader -import java.time.Duration -import java.time.Instant - -/** - * A helper class to collect metrics from a [ComputeService] instance and automatically export the metrics every - * export interval. - * - * @param dispatcher A [Dispatcher] for scheduling the future events. - * @param service The [ComputeService] to monitor. - * @param monitor The monitor to export the metrics to. - * @param exportInterval The export interval. - */ -public class ComputeMetricReader( - dispatcher: Dispatcher, - private val service: ComputeService, - private val monitor: ComputeMonitor, - private val exportInterval: Duration = Duration.ofMinutes(5), - private val startTime: Duration = Duration.ofMillis(0), - private val carbonTrace: CarbonTrace = CarbonTrace(null), -) : AutoCloseable { - private val logger = KotlinLogging.logger {} - private val scope = CoroutineScope(dispatcher.asCoroutineDispatcher()) - private val clock = dispatcher.timeSource - - /** - * Aggregator for service metrics. - */ - private val serviceTableReader = ServiceTableReaderImpl(service, startTime) - - /** - * Mapping from [Host] instances to [HostTableReaderImpl] - */ - private val hostTableReaders = mutableMapOf() - - /** - * Mapping from [Task] instances to [TaskTableReaderImpl] - */ - private val taskTableReaders = mutableMapOf() - - /** - * The background job that is responsible for collecting the metrics every cycle. - */ - private val job = - scope.launch { - val intervalMs = exportInterval.toMillis() - try { - while (isActive) { - delay(intervalMs) - - loggState() - } - } finally { - loggState() - - if (monitor is AutoCloseable) { - monitor.close() - } - } - } - - private fun loggState() { - try { - val now = this.clock.instant() - - for (host in this.service.hosts) { - val reader = this.hostTableReaders.computeIfAbsent(host) { HostTableReaderImpl(it, startTime, carbonTrace) } - reader.record(now) - this.monitor.record(reader.copy()) - reader.reset() - } - - for (task in this.service.tasks) { - val reader = this.taskTableReaders.computeIfAbsent(task) { TaskTableReaderImpl(service, it, startTime) } - reader.record(now) - this.monitor.record(reader.copy()) - reader.reset() - } - - this.serviceTableReader.record(now) - monitor.record(this.serviceTableReader.copy()) - } catch (cause: Throwable) { - this.logger.warn(cause) { "Exporter threw an Exception" } - } - } - - override fun close() { - job.cancel() - } - - /** - * An aggregator for service metrics before they are reported. - */ - private class ServiceTableReaderImpl( - private val service: ComputeService, - private val startTime: Duration = Duration.ofMillis(0), - ) : ServiceTableReader { - override fun copy(): ServiceTableReader { - val newServiceTable = ServiceTableReaderImpl(service) - newServiceTable.setValues(this) - - return newServiceTable - } - - override fun setValues(table: ServiceTableReader) { - _timestamp = table.timestamp - _timestampAbsolute = table.timestampAbsolute - - _hostsUp = table.hostsUp - _hostsDown = table.hostsDown - _tasksTotal = table.tasksTotal - _tasksPending = table.tasksPending - _tasksActive = table.tasksActive - _attemptsSuccess = table.attemptsSuccess - _attemptsFailure = table.attemptsFailure - _attemptsError = table.attemptsError - } - - private var _timestamp: Instant = Instant.MIN - override val timestamp: Instant - get() = _timestamp - - private var _timestampAbsolute: Instant = Instant.MIN - override val timestampAbsolute: Instant - get() = _timestampAbsolute - - override val hostsUp: Int - get() = _hostsUp - private var _hostsUp = 0 - - override val hostsDown: Int - get() = _hostsDown - private var _hostsDown = 0 - - override val tasksTotal: Int - get() = _tasksTotal - private var _tasksTotal = 0 - - override val tasksPending: Int - get() = _tasksPending - private var _tasksPending = 0 - - override val tasksActive: Int - get() = _tasksActive - private var _tasksActive = 0 - - override val attemptsSuccess: Int - get() = _attemptsSuccess - private var _attemptsSuccess = 0 - - override val attemptsFailure: Int - get() = _attemptsFailure - private var _attemptsFailure = 0 - - override val attemptsError: Int - get() = _attemptsError - private var _attemptsError = 0 - - /** - * Record the next cycle. - */ - fun record(now: Instant) { - _timestamp = now - _timestampAbsolute = now + startTime - - val stats = service.getSchedulerStats() - _hostsUp = stats.hostsAvailable - _hostsDown = stats.hostsUnavailable - _tasksTotal = stats.tasksTotal - _tasksPending = stats.tasksPending - _tasksActive = stats.tasksActive - _attemptsSuccess = stats.attemptsSuccess.toInt() - _attemptsFailure = stats.attemptsFailure.toInt() - _attemptsError = stats.attemptsError.toInt() - } - } - - /** - * An aggregator for host metrics before they are reported. - */ - private class HostTableReaderImpl( - host: Host, - private val startTime: Duration = Duration.ofMillis(0), - private val carbonTrace: CarbonTrace = CarbonTrace(null), - ) : HostTableReader { - override fun copy(): HostTableReader { - val newHostTable = HostTableReaderImpl(_host) - newHostTable.setValues(this) - - return newHostTable - } - - override fun setValues(table: HostTableReader) { - _timestamp = table.timestamp - _timestampAbsolute = table.timestampAbsolute - - _guestsTerminated = table.guestsTerminated - _guestsRunning = table.guestsRunning - _guestsError = table.guestsError - _guestsInvalid = table.guestsInvalid - _cpuLimit = table.cpuLimit - _cpuDemand = table.cpuDemand - _cpuUsage = table.cpuUsage - _cpuUtilization = table.cpuUtilization - _cpuActiveTime = table.cpuActiveTime - _cpuIdleTime = table.cpuIdleTime - _cpuStealTime = table.cpuStealTime - _cpuLostTime = table.cpuLostTime - _powerDraw = table.powerDraw - _energyUsage = table.energyUsage - _carbonIntensity = table.carbonIntensity - _carbonEmission = table.carbonEmission - _uptime = table.uptime - _downtime = table.downtime - _bootTime = table.bootTime - _bootTimeAbsolute = table.bootTimeAbsolute - } - - private val _host = host - - override val host: HostInfo = - HostInfo(host.uid.toString(), host.name, "x86", host.model.coreCount, host.model.cpuCapacity, host.model.memoryCapacity) - - override val timestamp: Instant - get() = _timestamp - private var _timestamp = Instant.MIN - - override val timestampAbsolute: Instant - get() = _timestampAbsolute - private var _timestampAbsolute = Instant.MIN - - override val guestsTerminated: Int - get() = _guestsTerminated - private var _guestsTerminated = 0 - - override val guestsRunning: Int - get() = _guestsRunning - private var _guestsRunning = 0 - - override val guestsError: Int - get() = _guestsError - private var _guestsError = 0 - - override val guestsInvalid: Int - get() = _guestsInvalid - private var _guestsInvalid = 0 - - override val cpuLimit: Double - get() = _cpuLimit - private var _cpuLimit = 0.0 - - override val cpuUsage: Double - get() = _cpuUsage - private var _cpuUsage = 0.0 - - override val cpuDemand: Double - get() = _cpuDemand - private var _cpuDemand = 0.0 - - override val cpuUtilization: Double - get() = _cpuUtilization - private var _cpuUtilization = 0.0 - - override val cpuActiveTime: Long - get() = _cpuActiveTime - previousCpuActiveTime - private var _cpuActiveTime = 0L - private var previousCpuActiveTime = 0L - - override val cpuIdleTime: Long - get() = _cpuIdleTime - previousCpuIdleTime - private var _cpuIdleTime = 0L - private var previousCpuIdleTime = 0L - - override val cpuStealTime: Long - get() = _cpuStealTime - previousCpuStealTime - private var _cpuStealTime = 0L - private var previousCpuStealTime = 0L - - override val cpuLostTime: Long - get() = _cpuLostTime - previousCpuLostTime - private var _cpuLostTime = 0L - private var previousCpuLostTime = 0L - - override val powerDraw: Double - get() = _powerDraw - private var _powerDraw = 0.0 - - override val energyUsage: Double - get() = _energyUsage - previousPowerTotal - private var _energyUsage = 0.0 - private var previousPowerTotal = 0.0 - - override val carbonIntensity: Double - get() = _carbonIntensity - private var _carbonIntensity = 0.0 - - override val carbonEmission: Double - get() = _carbonEmission - private var _carbonEmission = 0.0 - - override val uptime: Long - get() = _uptime - previousUptime - private var _uptime = 0L - private var previousUptime = 0L - - override val downtime: Long - get() = _downtime - previousDowntime - private var _downtime = 0L - private var previousDowntime = 0L - - override val bootTime: Instant? - get() = _bootTime - private var _bootTime: Instant? = null - - override val bootTimeAbsolute: Instant? - get() = _bootTimeAbsolute - private var _bootTimeAbsolute: Instant? = null - - /** - * Record the next cycle. - */ - fun record(now: Instant) { - val hostCpuStats = _host.getCpuStats() - val hostSysStats = _host.getSystemStats() - - _timestamp = now - _timestampAbsolute = now + startTime - - _guestsTerminated = hostSysStats.guestsTerminated - _guestsRunning = hostSysStats.guestsRunning - _guestsError = hostSysStats.guestsError - _guestsInvalid = hostSysStats.guestsInvalid - _cpuLimit = hostCpuStats.capacity - _cpuDemand = hostCpuStats.demand - _cpuUsage = hostCpuStats.usage - _cpuUtilization = hostCpuStats.utilization - _cpuActiveTime = hostCpuStats.activeTime - _cpuIdleTime = hostCpuStats.idleTime - _cpuStealTime = hostCpuStats.stealTime - _cpuLostTime = hostCpuStats.lostTime - _powerDraw = hostSysStats.powerDraw - _energyUsage = hostSysStats.energyUsage - _carbonIntensity = carbonTrace.getCarbonIntensity(timestampAbsolute) - - _carbonEmission = carbonIntensity * (energyUsage / 3600000.0) // convert energy usage from J to kWh - _uptime = hostSysStats.uptime.toMillis() - _downtime = hostSysStats.downtime.toMillis() - _bootTime = hostSysStats.bootTime - _bootTime = hostSysStats.bootTime + startTime - } - - /** - * Finish the aggregation for this cycle. - */ - fun reset() { - // Reset intermediate state for next aggregation - previousCpuActiveTime = _cpuActiveTime - previousCpuIdleTime = _cpuIdleTime - previousCpuStealTime = _cpuStealTime - previousCpuLostTime = _cpuLostTime - previousPowerTotal = _energyUsage - previousUptime = _uptime - previousDowntime = _downtime - - _guestsTerminated = 0 - _guestsRunning = 0 - _guestsError = 0 - _guestsInvalid = 0 - - _cpuLimit = 0.0 - _cpuUsage = 0.0 - _cpuDemand = 0.0 - _cpuUtilization = 0.0 - - _powerDraw = 0.0 - _energyUsage = 0.0 - _carbonIntensity = 0.0 - _carbonEmission = 0.0 - } - } - - /** - * An aggregator for task metrics before they are reported. - */ - private class TaskTableReaderImpl( - private val service: ComputeService, - private val task: Task, - private val startTime: Duration = Duration.ofMillis(0), - ) : TaskTableReader { - override fun copy(): TaskTableReader { - val newTaskTable = TaskTableReaderImpl(service, task) - newTaskTable.setValues(this) - - return newTaskTable - } - - override fun setValues(table: TaskTableReader) { - host = table.host - - _timestamp = table.timestamp - _timestampAbsolute = table.timestampAbsolute - - _cpuLimit = table.cpuLimit - _cpuActiveTime = table.cpuActiveTime - _cpuIdleTime = table.cpuIdleTime - _cpuStealTime = table.cpuStealTime - _cpuLostTime = table.cpuLostTime - _uptime = table.uptime - _downtime = table.downtime - _provisionTime = table.provisionTime - _bootTime = table.bootTime - _bootTimeAbsolute = table.bootTimeAbsolute - - _taskState = table.taskState - } - - /** - * The static information about this task. - */ - override val taskInfo = - TaskInfo( - task.uid.toString(), - task.name, - "vm", - "x86", - task.image.uid.toString(), - task.image.name, - task.flavor.coreCount, - task.flavor.memorySize, - ) - - /** - * The [HostInfo] of the host on which the task is hosted. - */ - override var host: HostInfo? = null - private var _host: Host? = null - - private var _timestamp = Instant.MIN - override val timestamp: Instant - get() = _timestamp - - private var _timestampAbsolute = Instant.MIN - override val timestampAbsolute: Instant - get() = _timestampAbsolute - - override val uptime: Long - get() = _uptime - previousUptime - private var _uptime: Long = 0 - private var previousUptime = 0L - - override val downtime: Long - get() = _downtime - previousDowntime - private var _downtime: Long = 0 - private var previousDowntime = 0L - - override val provisionTime: Instant? - get() = _provisionTime - private var _provisionTime: Instant? = null - - override val bootTime: Instant? - get() = _bootTime - private var _bootTime: Instant? = null - - override val cpuLimit: Double - get() = _cpuLimit - private var _cpuLimit = 0.0 - - override val cpuActiveTime: Long - get() = _cpuActiveTime - previousCpuActiveTime - private var _cpuActiveTime = 0L - private var previousCpuActiveTime = 0L - - override val cpuIdleTime: Long - get() = _cpuIdleTime - previousCpuIdleTime - private var _cpuIdleTime = 0L - private var previousCpuIdleTime = 0L - - override val cpuStealTime: Long - get() = _cpuStealTime - previousCpuStealTime - private var _cpuStealTime = 0L - private var previousCpuStealTime = 0L - - override val cpuLostTime: Long - get() = _cpuLostTime - previousCpuLostTime - private var _cpuLostTime = 0L - private var previousCpuLostTime = 0L - - override val bootTimeAbsolute: Instant? - get() = _bootTimeAbsolute - private var _bootTimeAbsolute: Instant? = null - - override val taskState: TaskState? - get() = _taskState - private var _taskState: TaskState? = null - - /** - * Record the next cycle. - */ - fun record(now: Instant) { - val newHost = service.lookupHost(task) - if (newHost != null && newHost.uid != _host?.uid) { - _host = newHost - host = - HostInfo( - newHost.uid.toString(), - newHost.name, - "x86", - newHost.model.coreCount, - newHost.model.cpuCapacity, - newHost.model.memoryCapacity, - ) - } - - val cpuStats = _host?.getCpuStats(task) - val sysStats = _host?.getSystemStats(task) - - _timestamp = now - _timestampAbsolute = now + startTime - - _cpuLimit = cpuStats?.capacity ?: 0.0 - _cpuActiveTime = cpuStats?.activeTime ?: 0 - _cpuIdleTime = cpuStats?.idleTime ?: 0 - _cpuStealTime = cpuStats?.stealTime ?: 0 - _cpuLostTime = cpuStats?.lostTime ?: 0 - _uptime = sysStats?.uptime?.toMillis() ?: 0 - _downtime = sysStats?.downtime?.toMillis() ?: 0 - _provisionTime = task.launchedAt - _bootTime = sysStats?.bootTime - - _taskState = task.state - - if (sysStats != null) { - _bootTimeAbsolute = sysStats.bootTime + startTime - } else { - _bootTimeAbsolute = null - } - } - - /** - * Finish the aggregation for this cycle. - */ - fun reset() { - previousUptime = _uptime - previousDowntime = _downtime - previousCpuActiveTime = _cpuActiveTime - previousCpuIdleTime = _cpuIdleTime - previousCpuStealTime = _cpuStealTime - previousCpuLostTime = _cpuLostTime - - _host = null - _cpuLimit = 0.0 - } - } -} diff --git a/opendc-compute/opendc-compute-telemetry/src/main/kotlin/org/opendc/compute/telemetry/ComputeMonitor.kt b/opendc-compute/opendc-compute-telemetry/src/main/kotlin/org/opendc/compute/telemetry/ComputeMonitor.kt deleted file mode 100644 index 1df058fb..00000000 --- a/opendc-compute/opendc-compute-telemetry/src/main/kotlin/org/opendc/compute/telemetry/ComputeMonitor.kt +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright (c) 2022 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.compute.telemetry - -import org.opendc.compute.telemetry.table.HostTableReader -import org.opendc.compute.telemetry.table.ServiceTableReader -import org.opendc.compute.telemetry.table.TaskTableReader - -/** - * A monitor that tracks the metrics and events of the OpenDC Compute service. - */ -public interface ComputeMonitor { - /** - * Record an entry with the specified [reader]. - */ - public fun record(reader: TaskTableReader) {} - - /** - * Record an entry with the specified [reader]. - */ - public fun record(reader: HostTableReader) {} - - /** - * Record an entry with the specified [reader]. - */ - public fun record(reader: ServiceTableReader) {} -} diff --git a/opendc-compute/opendc-compute-telemetry/src/main/kotlin/org/opendc/compute/telemetry/export/parquet/ComputeExportConfig.kt b/opendc-compute/opendc-compute-telemetry/src/main/kotlin/org/opendc/compute/telemetry/export/parquet/ComputeExportConfig.kt deleted file mode 100644 index 161c0936..00000000 --- a/opendc-compute/opendc-compute-telemetry/src/main/kotlin/org/opendc/compute/telemetry/export/parquet/ComputeExportConfig.kt +++ /dev/null @@ -1,192 +0,0 @@ -/* - * Copyright (c) 2024 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.compute.telemetry.export.parquet - -import kotlinx.serialization.KSerializer -import kotlinx.serialization.Serializable -import kotlinx.serialization.builtins.ListSerializer -import kotlinx.serialization.descriptors.SerialDescriptor -import kotlinx.serialization.descriptors.buildClassSerialDescriptor -import kotlinx.serialization.encoding.Decoder -import kotlinx.serialization.encoding.Encoder -import kotlinx.serialization.encoding.encodeStructure -import kotlinx.serialization.json.Json -import kotlinx.serialization.json.JsonDecoder -import kotlinx.serialization.json.JsonElement -import kotlinx.serialization.json.jsonObject -import org.opendc.common.logger.logger -import org.opendc.compute.telemetry.table.HostTableReader -import org.opendc.compute.telemetry.table.ServiceTableReader -import org.opendc.compute.telemetry.table.TaskTableReader -import org.opendc.trace.util.parquet.exporter.ColListSerializer -import org.opendc.trace.util.parquet.exporter.ExportColumn -import org.opendc.trace.util.parquet.exporter.Exportable -import org.opendc.trace.util.parquet.exporter.columnSerializer - -/** - * Aggregates the necessary settings to personalize the output - * parquet files for compute workloads. - * - * @param[hostExportColumns] the columns that will be included in the `host.parquet` raw output file. - * @param[taskExportColumns] the columns that will be included in the `task.parquet` raw output file. - * @param[serviceExportColumns] the columns that will be included in the `service.parquet` raw output file. - */ -@Serializable(with = ComputeExportConfig.Companion.ComputeExportConfigSerializer::class) -public data class ComputeExportConfig( - public val hostExportColumns: Set>, - public val taskExportColumns: Set>, - public val serviceExportColumns: Set>, -) { - public constructor( - hostExportColumns: Collection>, - taskExportColumns: Collection>, - serviceExportColumns: Collection>, - ) : this( - hostExportColumns.toSet() + DfltHostExportColumns.BASE_EXPORT_COLUMNS, - taskExportColumns.toSet() + DfltTaskExportColumns.BASE_EXPORT_COLUMNS, - serviceExportColumns.toSet() + DfltServiceExportColumns.BASE_EXPORT_COLUMNS, - ) - - /** - * @return formatted string representing the export config. - */ - public fun fmt(): String = - """ - | === Compute Export Config === - | Host columns : ${hostExportColumns.map { it.name }.toString().trim('[', ']')} - | Task columns : ${taskExportColumns.map { it.name }.toString().trim('[', ']')} - | Service columns : ${serviceExportColumns.map { it.name }.toString().trim('[', ']')} - """.trimIndent() - - public companion object { - internal val LOG by logger() - - /** - * Force the jvm to load the default [ExportColumn]s relevant to compute export, - * so that they are available for deserialization. - */ - public fun loadDfltColumns() { - DfltHostExportColumns - DfltTaskExportColumns - DfltServiceExportColumns - } - - /** - * Config that includes all columns defined in [DfltHostExportColumns], - * [DfltTaskExportColumns], [DfltServiceExportColumns] among all other loaded - * columns for [HostTableReader], [TaskTableReader] and [ServiceTableReader]. - */ - public val ALL_COLUMNS: ComputeExportConfig by lazy { - loadDfltColumns() - ComputeExportConfig( - hostExportColumns = ExportColumn.getAllLoadedColumns(), - taskExportColumns = ExportColumn.getAllLoadedColumns(), - serviceExportColumns = ExportColumn.getAllLoadedColumns(), - ) - } - - /** - * A runtime [KSerializer] is needed for reasons explained in [columnSerializer] docs. - * - * This serializer makes use of reified column serializers for the 2 properties. - */ - internal object ComputeExportConfigSerializer : KSerializer { - override val descriptor: SerialDescriptor = - buildClassSerialDescriptor("org.opendc.compute.telemetry.export.parquet.ComputeExportConfig") { - element( - "hostExportColumns", - ListSerializer(columnSerializer()).descriptor, - ) - element( - "taskExportColumns", - ListSerializer(columnSerializer()).descriptor, - ) - element( - "serviceExportColumns", - ListSerializer(columnSerializer()).descriptor, - ) - } - - override fun deserialize(decoder: Decoder): ComputeExportConfig { - val jsonDec = - (decoder as? JsonDecoder) ?: let { - // Basically a recursive call with a JsonDecoder. - return json.decodeFromString(decoder.decodeString().trim('"')) - } - - // Loads the default columns so that they are available for deserialization. - loadDfltColumns() - val elem = jsonDec.decodeJsonElement().jsonObject - - val hostFields: List> = elem["hostExportColumns"].toFieldList() - val taskFields: List> = elem["taskExportColumns"].toFieldList() - val serviceFields: List> = elem["serviceExportColumns"].toFieldList() - - return ComputeExportConfig( - hostExportColumns = hostFields, - taskExportColumns = taskFields, - serviceExportColumns = serviceFields, - ) - } - - override fun serialize( - encoder: Encoder, - value: ComputeExportConfig, - ) { - encoder.encodeStructure(descriptor) { - encodeSerializableElement( - descriptor, - 0, - ColListSerializer(columnSerializer()), - value.hostExportColumns.toList(), - ) - encodeSerializableElement( - descriptor, - 1, - ColListSerializer(columnSerializer()), - value.taskExportColumns.toList(), - ) - encodeSerializableElement( - descriptor, - 2, - ColListSerializer(columnSerializer()), - value.serviceExportColumns.toList(), - ) - } - } - } - } -} - -private val json = Json { ignoreUnknownKeys = true } - -private inline fun JsonElement?.toFieldList(): List> = - this?.let { - json.decodeFromJsonElement(ColListSerializer(columnSerializer()), it) - }?.ifEmpty { - ComputeExportConfig.LOG.warn( - "deserialized list of export columns for exportable ${T::class.simpleName} " + - "produced empty list, falling back to all loaded columns", - ) - ExportColumn.getAllLoadedColumns() - } ?: ExportColumn.getAllLoadedColumns() diff --git a/opendc-compute/opendc-compute-telemetry/src/main/kotlin/org/opendc/compute/telemetry/export/parquet/DfltHostExportColumns.kt b/opendc-compute/opendc-compute-telemetry/src/main/kotlin/org/opendc/compute/telemetry/export/parquet/DfltHostExportColumns.kt deleted file mode 100644 index 261c5462..00000000 --- a/opendc-compute/opendc-compute-telemetry/src/main/kotlin/org/opendc/compute/telemetry/export/parquet/DfltHostExportColumns.kt +++ /dev/null @@ -1,195 +0,0 @@ -/* - * Copyright (c) 2024 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.compute.telemetry.export.parquet - -import org.apache.parquet.io.api.Binary -import org.apache.parquet.schema.LogicalTypeAnnotation -import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.BINARY -import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.DOUBLE -import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.INT32 -import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.INT64 -import org.apache.parquet.schema.Types -import org.opendc.compute.telemetry.table.HostTableReader -import org.opendc.trace.util.parquet.exporter.ExportColumn - -/** - * This object wraps the [ExportColumn]s to solves ambiguity for field - * names that are included in more than 1 exportable. - * - * Additionally, it allows to load all the fields at once by just its symbol, - * so that these columns can be deserialized. Additional fields can be added - * from anywhere, and they are deserializable as long as they are loaded by the jvm. - * - * ```kotlin - * ... - * // Loads the column - * DfltHostExportColumns - * ... - * ``` - */ -public object DfltHostExportColumns { - public val TIMESTAMP: ExportColumn = - ExportColumn( - field = Types.required(INT64).named("timestamp"), - ) { it.timestamp.toEpochMilli() } - - public val TIMESTAMP_ABS: ExportColumn = - ExportColumn( - field = Types.required(INT64).named("timestamp_absolute"), - ) { it.timestampAbsolute.toEpochMilli() } - - public val HOST_ID: ExportColumn = - ExportColumn( - field = - Types.required(BINARY) - .`as`(LogicalTypeAnnotation.stringType()) - .named("host_id"), - ) { Binary.fromString(it.host.id) } - - public val HOST_NAME: ExportColumn = - ExportColumn( - field = - Types.required(BINARY) - .`as`(LogicalTypeAnnotation.stringType()) - .named("host_name"), - ) { Binary.fromString(it.host.name) } - - public val CPU_COUNT: ExportColumn = - ExportColumn( - field = Types.required(INT32).named("core_count"), - ) { it.host.coreCount } - - public val MEM_CAPACITY: ExportColumn = - ExportColumn( - field = Types.required(INT64).named("mem_capacity"), - ) { it.host.memCapacity } - - public val GUESTS_TERMINATED: ExportColumn = - ExportColumn( - field = Types.required(INT32).named("guests_terminated"), - ) { it.guestsTerminated } - - public val GUESTS_RUNNING: ExportColumn = - ExportColumn( - field = Types.required(INT32).named("guests_running"), - ) { it.guestsRunning } - - public val GUESTS_ERROR: ExportColumn = - ExportColumn( - field = Types.required(INT32).named("guests_error"), - ) { it.guestsError } - - public val GUESTS_INVALID: ExportColumn = - ExportColumn( - field = Types.required(INT32).named("guests_invalid"), - ) { it.guestsInvalid } - - public val CPU_LIMIT: ExportColumn = - ExportColumn( - field = Types.required(DOUBLE).named("cpu_limit"), - ) { it.cpuLimit } - - public val CPU_USAGE: ExportColumn = - ExportColumn( - field = Types.required(DOUBLE).named("cpu_usage"), - ) { it.cpuUsage } - - public val CPU_DEMAND: ExportColumn = - ExportColumn( - field = Types.required(DOUBLE).named("cpu_demand"), - ) { it.cpuDemand } - - public val CPU_UTILIZATION: ExportColumn = - ExportColumn( - field = Types.required(DOUBLE).named("cpu_utilization"), - ) { it.cpuUtilization } - - public val CPU_TIME_ACTIVE: ExportColumn = - ExportColumn( - field = Types.required(INT64).named("cpu_time_active"), - ) { it.cpuActiveTime } - - public val CPU_TIME_IDLE: ExportColumn = - ExportColumn( - field = Types.required(INT64).named("cpu_time_idle"), - ) { it.cpuIdleTime } - - public val CPU_TIME_STEAL: ExportColumn = - ExportColumn( - field = Types.required(INT64).named("cpu_time_steal"), - ) { it.cpuStealTime } - - public val CPU_TIME_LOST: ExportColumn = - ExportColumn( - field = Types.required(INT64).named("cpu_time_lost"), - ) { it.cpuLostTime } - - public val POWER_DRAW: ExportColumn = - ExportColumn( - field = Types.required(DOUBLE).named("power_draw"), - ) { it.powerDraw } - - public val ENERGY_USAGE: ExportColumn = - ExportColumn( - field = Types.required(DOUBLE).named("energy_usage"), - ) { it.energyUsage } - - public val CARBON_INTENSITY: ExportColumn = - ExportColumn( - field = Types.required(DOUBLE).named("carbon_intensity"), - ) { it.carbonIntensity } - - public val CARBON_EMISSION: ExportColumn = - ExportColumn( - field = Types.required(DOUBLE).named("carbon_emission"), - ) { it.carbonEmission } - - public val UP_TIME: ExportColumn = - ExportColumn( - field = Types.required(INT64).named("uptime"), - ) { it.uptime } - - public val DOWN_TIME: ExportColumn = - ExportColumn( - field = Types.required(INT64).named("downtime"), - ) { it.downtime } - - public val BOOT_TIME: ExportColumn = - ExportColumn( - field = Types.optional(INT64).named("boot_time"), - ) { it.bootTime?.toEpochMilli() } - - public val BOOT_TIME_ABS: ExportColumn = - ExportColumn( - field = Types.optional(INT64).named("boot_time_absolute"), - ) { it.bootTimeAbsolute?.toEpochMilli() } - - /** - * The columns that are always included in the output file. - */ - internal val BASE_EXPORT_COLUMNS = - setOf( - TIMESTAMP_ABS, - TIMESTAMP, - ) -} diff --git a/opendc-compute/opendc-compute-telemetry/src/main/kotlin/org/opendc/compute/telemetry/export/parquet/DfltServiceExportColumns.kt b/opendc-compute/opendc-compute-telemetry/src/main/kotlin/org/opendc/compute/telemetry/export/parquet/DfltServiceExportColumns.kt deleted file mode 100644 index 8038060d..00000000 --- a/opendc-compute/opendc-compute-telemetry/src/main/kotlin/org/opendc/compute/telemetry/export/parquet/DfltServiceExportColumns.kt +++ /dev/null @@ -1,95 +0,0 @@ -/* - * Copyright (c) 2024 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.compute.telemetry.export.parquet - -import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.INT32 -import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.INT64 -import org.apache.parquet.schema.Types -import org.opendc.compute.telemetry.table.ServiceTableReader -import org.opendc.trace.util.parquet.exporter.ExportColumn - -/** - * This object wraps the [ExportColumn]s to solves ambiguity for field - * names that are included in more than 1 exportable. - * - * Additionally, it allows to load all the fields at once by just its symbol, - * so that these columns can be deserialized. Additional fields can be added - * from anywhere, and they are deserializable as long as they are loaded by the jvm. - * - * ```kotlin - * ... - * // Loads the column - * DfltServiceExportColumns - * ... - * ``` - */ -public object DfltServiceExportColumns { - public val TIMESTAMP: ExportColumn = - ExportColumn( - field = Types.required(INT64).named("timestamp"), - ) { it.timestamp.toEpochMilli() } - - public val TIMESTAMP_ABS: ExportColumn = - ExportColumn( - field = Types.required(INT64).named("timestamp_absolute"), - ) { it.timestampAbsolute.toEpochMilli() } - - public val HOSTS_UP: ExportColumn = - ExportColumn( - field = Types.required(INT32).named("hosts_up"), - ) { it.hostsUp } - - public val TASKS_PENDING: ExportColumn = - ExportColumn( - field = Types.required(INT32).named("tasks_pending"), - ) { it.tasksPending } - - public val TASKS_ACTIVE: ExportColumn = - ExportColumn( - field = Types.required(INT32).named("tasks_active"), - ) { it.tasksActive } - - public val ATTEMPTS_SUCCESS: ExportColumn = - ExportColumn( - field = Types.required(INT32).named("attempts_success"), - ) { it.attemptsSuccess } - - public val AT3yyTEMPTS_FAILURE: ExportColumn = - ExportColumn( - field = Types.required(INT32).named("attempts_failure"), - ) { it.attemptsFailure } - - public val ATTEMPTS_ERROR: ExportColumn = - ExportColumn( - field = Types.required(INT32).named("attempts_error"), - ) { it.attemptsError } - - /** - * The columns that are always included in the output file. - */ - internal val BASE_EXPORT_COLUMNS = - setOf( - TIMESTAMP_ABS, - TIMESTAMP, - ) -} diff --git a/opendc-compute/opendc-compute-telemetry/src/main/kotlin/org/opendc/compute/telemetry/export/parquet/DfltTaskExportColumns.kt b/opendc-compute/opendc-compute-telemetry/src/main/kotlin/org/opendc/compute/telemetry/export/parquet/DfltTaskExportColumns.kt deleted file mode 100644 index 9e86e1a3..00000000 --- a/opendc-compute/opendc-compute-telemetry/src/main/kotlin/org/opendc/compute/telemetry/export/parquet/DfltTaskExportColumns.kt +++ /dev/null @@ -1,161 +0,0 @@ -/* - * Copyright (c) 2024 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.compute.telemetry.export.parquet - -import org.apache.parquet.io.api.Binary -import org.apache.parquet.schema.LogicalTypeAnnotation -import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.BINARY -import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.DOUBLE -import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.INT32 -import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.INT64 -import org.apache.parquet.schema.Types -import org.opendc.compute.telemetry.table.TaskTableReader -import org.opendc.trace.util.parquet.exporter.ExportColumn - -/** - * This object wraps the [ExportColumn]s to solves ambiguity for field - * names that are included in more than 1 exportable - * - * Additionally, it allows to load all the fields at once by just its symbol, - * so that these columns can be deserialized. Additional fields can be added - * from anywhere, and they are deserializable as long as they are loaded by the jvm. - * - * ```kotlin - * ... - * // Loads the column - * DfltTaskExportColumns - * ... - * ``` - */ -public object DfltTaskExportColumns { - public val TIMESTAMP: ExportColumn = - ExportColumn( - field = Types.required(INT64).named("timestamp"), - ) { it.timestamp.toEpochMilli() } - - public val TIMESTAMP_ABS: ExportColumn = - ExportColumn( - field = Types.required(INT64).named("timestamp_absolute"), - ) { it.timestampAbsolute.toEpochMilli() } - - public val TASK_ID: ExportColumn = - ExportColumn( - field = - Types.required(BINARY) - .`as`(LogicalTypeAnnotation.stringType()) - .named("task_id"), - ) { Binary.fromString(it.taskInfo.id) } - - public val HOST_ID: ExportColumn = - ExportColumn( - field = - Types.optional(BINARY) - .`as`(LogicalTypeAnnotation.stringType()) - .named("host_id"), - ) { it.host?.id?.let { Binary.fromString(it) } } - - public val TASK_NAME: ExportColumn = - ExportColumn( - field = - Types.required(BINARY) - .`as`(LogicalTypeAnnotation.stringType()) - .named("task_name"), - ) { Binary.fromString(it.taskInfo.name) } - - public val CPU_COUNT: ExportColumn = - ExportColumn( - field = Types.required(INT32).named("cpu_count"), - ) { it.taskInfo.cpuCount } - - public val MEM_CAPACITY: ExportColumn = - ExportColumn( - field = Types.required(INT64).named("mem_capacity"), - ) { it.taskInfo.memCapacity } - - public val CPU_LIMIT: ExportColumn = - ExportColumn( - field = Types.required(DOUBLE).named("cpu_limit"), - ) { it.cpuLimit } - - public val CPU_TIME_ACTIVE: ExportColumn = - ExportColumn( - field = Types.required(INT64).named("cpu_time_active"), - ) { it.cpuActiveTime } - - public val CPU_TIME_IDLE: ExportColumn = - ExportColumn( - field = Types.required(INT64).named("cpu_time_idle"), - ) { it.cpuIdleTime } - - public val CPU_TIME_STEAL: ExportColumn = - ExportColumn( - field = Types.required(INT64).named("cpu_time_steal"), - ) { it.cpuStealTime } - - public val CPU_TIME_LOST: ExportColumn = - ExportColumn( - field = Types.required(INT64).named("cpu_time_lost"), - ) { it.cpuLostTime } - - public val UP_TIME: ExportColumn = - ExportColumn( - field = Types.required(INT64).named("uptime"), - ) { it.uptime } - - public val DOWN_TIME: ExportColumn = - ExportColumn( - field = Types.required(INT64).named("downtime"), - ) { it.downtime } - - public val PROVISION_TIME: ExportColumn = - ExportColumn( - field = Types.optional(INT64).named("provision_time"), - ) { it.provisionTime?.toEpochMilli() } - - public val BOOT_TIME: ExportColumn = - ExportColumn( - field = Types.optional(INT64).named("boot_time"), - ) { it.bootTime?.toEpochMilli() } - - public val BOOT_TIME_ABS: ExportColumn = - ExportColumn( - field = Types.optional(INT64).named("boot_time_absolute"), - ) { it.bootTimeAbsolute?.toEpochMilli() } - - public val TASK_STATE: ExportColumn = - ExportColumn( - field = - Types.optional(BINARY) - .`as`(LogicalTypeAnnotation.stringType()) - .named("task_state"), - ) { Binary.fromString(it.taskState?.name) } - - /** - * The columns that are always included in the output file. - */ - internal val BASE_EXPORT_COLUMNS = - setOf( - TIMESTAMP_ABS, - TIMESTAMP, - ) -} diff --git a/opendc-compute/opendc-compute-telemetry/src/main/kotlin/org/opendc/compute/telemetry/export/parquet/ParquetComputeMonitor.kt b/opendc-compute/opendc-compute-telemetry/src/main/kotlin/org/opendc/compute/telemetry/export/parquet/ParquetComputeMonitor.kt deleted file mode 100644 index 3b7a7c0c..00000000 --- a/opendc-compute/opendc-compute-telemetry/src/main/kotlin/org/opendc/compute/telemetry/export/parquet/ParquetComputeMonitor.kt +++ /dev/null @@ -1,125 +0,0 @@ -/* - * Copyright (c) 2022 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.compute.telemetry.export.parquet - -import org.opendc.compute.telemetry.ComputeMonitor -import org.opendc.compute.telemetry.table.HostTableReader -import org.opendc.compute.telemetry.table.ServiceTableReader -import org.opendc.compute.telemetry.table.TaskTableReader -import org.opendc.trace.util.parquet.exporter.ExportColumn -import org.opendc.trace.util.parquet.exporter.Exportable -import org.opendc.trace.util.parquet.exporter.Exporter -import java.io.File - -/** - * A [ComputeMonitor] that logs the events to a Parquet file. - */ -public class ParquetComputeMonitor( - private val hostExporter: Exporter, - private val taskExporter: Exporter, - private val serviceExporter: Exporter, -) : ComputeMonitor, AutoCloseable { - override fun record(reader: HostTableReader) { - hostExporter.write(reader) - } - - override fun record(reader: TaskTableReader) { - taskExporter.write(reader) - } - - override fun record(reader: ServiceTableReader) { - serviceExporter.write(reader) - } - - override fun close() { - hostExporter.close() - taskExporter.close() - serviceExporter.close() - } - - public companion object { - /** - * Overloaded constructor with [ComputeExportConfig] as parameter. - * - * @param[base] parent pathname for output file. - * @param[partition] child pathname for output file. - * @param[bufferSize] size of the buffer used by the writer thread. - */ - public operator fun invoke( - base: File, - partition: String, - bufferSize: Int, - computeExportConfig: ComputeExportConfig, - ): ParquetComputeMonitor = - invoke( - base = base, - partition = partition, - bufferSize = bufferSize, - hostExportColumns = computeExportConfig.hostExportColumns, - taskExportColumns = computeExportConfig.taskExportColumns, - serviceExportColumns = computeExportConfig.serviceExportColumns, - ) - - /** - * Constructor that loads default [ExportColumn]s defined in - * [DfltHostExportColumns], [DfltTaskExportColumns], [DfltServiceExportColumns] - * in case optional parameters are omitted and all fields need to be retrieved. - * - * @param[base] parent pathname for output file. - * @param[partition] child pathname for output file. - * @param[bufferSize] size of the buffer used by the writer thread. - */ - public operator fun invoke( - base: File, - partition: String, - bufferSize: Int, - hostExportColumns: Collection>? = null, - taskExportColumns: Collection>? = null, - serviceExportColumns: Collection>? = null, - ): ParquetComputeMonitor { - // Loads the fields in case they need to be retrieved if optional params are omitted. - ComputeExportConfig.loadDfltColumns() - - return ParquetComputeMonitor( - hostExporter = - Exporter( - outputFile = File(base, "$partition/host.parquet").also { it.parentFile.mkdirs() }, - columns = hostExportColumns ?: Exportable.getAllLoadedColumns(), - bufferSize = bufferSize, - ), - taskExporter = - Exporter( - outputFile = File(base, "$partition/task.parquet").also { it.parentFile.mkdirs() }, - columns = taskExportColumns ?: Exportable.getAllLoadedColumns(), - bufferSize = bufferSize, - ), - serviceExporter = - Exporter( - outputFile = File(base, "$partition/service.parquet").also { it.parentFile.mkdirs() }, - columns = serviceExportColumns ?: Exportable.getAllLoadedColumns(), - bufferSize = bufferSize, - ), - ) - } - } -} diff --git a/opendc-compute/opendc-compute-telemetry/src/main/kotlin/org/opendc/compute/telemetry/export/parquet/README.md b/opendc-compute/opendc-compute-telemetry/src/main/kotlin/org/opendc/compute/telemetry/export/parquet/README.md deleted file mode 100644 index aee63fc9..00000000 --- a/opendc-compute/opendc-compute-telemetry/src/main/kotlin/org/opendc/compute/telemetry/export/parquet/README.md +++ /dev/null @@ -1,70 +0,0 @@ -### Summary -Added output configuration, that can be defined in the scenario `.json` file, that allows to select which columns are to be included in the raw oputput files `host.parquet`, `task.parquet` and `service.parquet`. - -### Columns -The 'default' columns are defined in `DfltHostExportcolumns`, `DfltTaskExportColumns` and `DfltServiceExportColumns`. Any number of additional columns can be definied anywhere (`ExportColumn`) and it is going to be deserializable as long as it is loaded by the jvm. - -### Deserialization -Each `ExportColumn` has a `Regex`, used for deserialization. If no custom regex is provided, the default one is used. The default regex matches the column name in case-insensitive manner, either with `_` as in the name or with ` ` (blank space). - -###### E.g.: -***column name*** = "cpu\_count" -***default column regex*** = "\\s*(?:cpu_count|cpu count)\\s*" (case insensitive) -***matches*** = "cpu\_count", "cpu count", "CpU/_cOuNt" etc. - -### JSON Schema -```json -// scenario.json -{ - ... - "computeExportConfig": { - "type": "object", - "properties": { - "hostExportColumns": { "type": "array" }, - "taskExportColumns": { "type": "array" } , - "serviceExportColumns": { "type": "array" } , - "required": [ /* NONE REQUIRED */ ] - } - }, - ... - "required": [ - ... - // NOT REQUIRED - ] -} -``` - -  -###### Bad Formatting Cases -- If a column name (and type) does not match any deserializable column, the entry is ignored and error message is logged. -- If an empty list of columns is provided or those that are provided were not deserializable, then all loaded columns for that `Exportable` are used, and a warning message is logged. -- If no list is provided, then all loaded columns for that `Exportable` are used. - - -### Example - -```json -// scenario.json -{ - ... - "computeExportConfig": { - "hostExportColumns": ["timestamp", "timestamp_absolute", "invalid-entry1", "guests_invalid"], - "taskExportColumns": ["invalid-entry2"], - "serviceExportColumns": ["timestamp", "tasks_active", "tasks_pending"] - }, - ... -``` - -```json -// console output -10:51:56.561 [ERROR] ColListSerializer - no match found for column "invalid-entry1", ignoring... -10:51:56.563 [ERROR] ColListSerializer - no match found for column "invalid-entry2", ignoring... -10:51:56.564 [WARN] ComputeExportConfig - deserialized list of export columns for exportable TaskTableReader produced empty list, falling back to all loaded columns -10:51:56.584 [INFO] ScenariosSpec - -| === Compute Export Config === -| Host columns : timestamp, timestamp_absolute, guests_invalid -| Task columns : timestamp, timestamp_absolute, task_id, task_name, cpu_count, mem_capacity, cpu_limit, cpu_time_active, cpu_time_idle, cpu_time_steal, cpu_time_lost, uptime, downtime, provision_time, boot_time, boot_time_absolute -| Service columns : timestamp, tasks_active, tasks_pending - -``` - diff --git a/opendc-compute/opendc-compute-telemetry/src/main/kotlin/org/opendc/compute/telemetry/table/HostInfo.kt b/opendc-compute/opendc-compute-telemetry/src/main/kotlin/org/opendc/compute/telemetry/table/HostInfo.kt deleted file mode 100644 index 62b7ef0d..00000000 --- a/opendc-compute/opendc-compute-telemetry/src/main/kotlin/org/opendc/compute/telemetry/table/HostInfo.kt +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright (c) 2021 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.compute.telemetry.table - -/** - * Information about a host exposed to the telemetry service. - */ -public data class HostInfo( - val id: String, - val name: String, - val arch: String, - val coreCount: Int, - val coreSpeed: Double, - val memCapacity: Long, -) diff --git a/opendc-compute/opendc-compute-telemetry/src/main/kotlin/org/opendc/compute/telemetry/table/HostTableReader.kt b/opendc-compute/opendc-compute-telemetry/src/main/kotlin/org/opendc/compute/telemetry/table/HostTableReader.kt deleted file mode 100644 index a7b8bedb..00000000 --- a/opendc-compute/opendc-compute-telemetry/src/main/kotlin/org/opendc/compute/telemetry/table/HostTableReader.kt +++ /dev/null @@ -1,150 +0,0 @@ -/* - * Copyright (c) 2021 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.compute.telemetry.table - -import org.opendc.trace.util.parquet.exporter.Exportable -import java.time.Instant - -/** - * An interface that is used to read a row of a host trace entry. - */ -public interface HostTableReader : Exportable { - public fun copy(): HostTableReader - - public fun setValues(table: HostTableReader) - - /** - * The [HostInfo] of the host to which the row belongs to. - */ - public val host: HostInfo - - /** - * The timestamp of the current entry of the reader relative to the start of the workload. - */ - public val timestamp: Instant - - /** - * The timestamp of the current entry of the reader. - */ - public val timestampAbsolute: Instant - - /** - * The number of guests that are in a terminated state. - */ - public val guestsTerminated: Int - - /** - * The number of guests that are in a running state. - */ - public val guestsRunning: Int - - /** - * The number of guests that are in an error state. - */ - public val guestsError: Int - - /** - * The number of guests that are in an unknown state. - */ - public val guestsInvalid: Int - - /** - * The capacity of the CPUs in the host (in MHz). - */ - public val cpuLimit: Double - - /** - * The usage of all CPUs in the host (in MHz). - */ - public val cpuUsage: Double - - /** - * The demand of all vCPUs of the guests (in MHz) - */ - public val cpuDemand: Double - - /** - * The CPU utilization of the host. - */ - public val cpuUtilization: Double - - /** - * The duration (in ms) that a CPU was active in the host. - */ - public val cpuActiveTime: Long - - /** - * The duration (in ms) that a CPU was idle in the host. - */ - public val cpuIdleTime: Long - - /** - * The duration (in ms) that a vCPU wanted to run, but no capacity was available. - */ - public val cpuStealTime: Long - - /** - * The duration (in ms) of CPU time that was lost due to interference. - */ - public val cpuLostTime: Long - - /** - * The current power draw of the host in W. - */ - public val powerDraw: Double - - /** - * The total energy consumption of the host since last sample in J. - */ - public val energyUsage: Double - - /** - * The current carbon intensity of the host in gCO2 / kW. - */ - public val carbonIntensity: Double - - /** - * The current carbon emission since the last deadline in g. - */ - public val carbonEmission: Double - - /** - * The uptime of the host since last time in ms. - */ - public val uptime: Long - - /** - * The downtime of the host since last time in ms. - */ - public val downtime: Long - - /** - * The [Instant] at which the host booted relative to the start of the workload. - */ - public val bootTime: Instant? - - /** - * The [Instant] at which the host booted. - */ - public val bootTimeAbsolute: Instant? -} diff --git a/opendc-compute/opendc-compute-telemetry/src/main/kotlin/org/opendc/compute/telemetry/table/ServiceData.kt b/opendc-compute/opendc-compute-telemetry/src/main/kotlin/org/opendc/compute/telemetry/table/ServiceData.kt deleted file mode 100644 index 7a8ba6a7..00000000 --- a/opendc-compute/opendc-compute-telemetry/src/main/kotlin/org/opendc/compute/telemetry/table/ServiceData.kt +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Copyright (c) 2021 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.compute.telemetry.table - -import java.time.Instant - -/** - * A trace entry for the compute service. - */ -public data class ServiceData( - val timestamp: Instant, - val hostsUp: Int, - val hostsDown: Int, - val tasksTotal: Int, - val tasksPending: Int, - val tasksActive: Int, - val attemptsSuccess: Int, - val attemptsFailure: Int, - val attemptsError: Int, -) - -/** - * Convert a [ServiceTableReader] into a persistent object. - */ -public fun ServiceTableReader.toServiceData(): ServiceData { - return ServiceData( - timestamp, - hostsUp, - hostsDown, - tasksTotal, - tasksPending, - tasksActive, - attemptsSuccess, - attemptsFailure, - attemptsError, - ) -} diff --git a/opendc-compute/opendc-compute-telemetry/src/main/kotlin/org/opendc/compute/telemetry/table/ServiceTableReader.kt b/opendc-compute/opendc-compute-telemetry/src/main/kotlin/org/opendc/compute/telemetry/table/ServiceTableReader.kt deleted file mode 100644 index 23630fb4..00000000 --- a/opendc-compute/opendc-compute-telemetry/src/main/kotlin/org/opendc/compute/telemetry/table/ServiceTableReader.kt +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Copyright (c) 2021 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.compute.telemetry.table - -import org.opendc.trace.util.parquet.exporter.Exportable -import java.time.Instant - -/** - * An interface that is used to read a row of a service trace entry. - */ -public interface ServiceTableReader : Exportable { - public fun copy(): ServiceTableReader - - public fun setValues(table: ServiceTableReader) - - /** - * The timestamp of the current entry of the reader. - */ - public val timestamp: Instant - - /** - * The timestamp of the current entry of the reader. - */ - public val timestampAbsolute: Instant - - /** - * The number of hosts that are up at this instant. - */ - public val hostsUp: Int - - /** - * The number of hosts that are down at this instant. - */ - public val hostsDown: Int - - /** - * The number of tasks that are registered with the compute service. - */ - public val tasksTotal: Int - - /** - * The number of tasks that are pending to be scheduled. - */ - public val tasksPending: Int - - /** - * The number of tasks that are currently active. - */ - public val tasksActive: Int - - /** - * The scheduling attempts that were successful. - */ - public val attemptsSuccess: Int - - /** - * The scheduling attempts that were unsuccessful due to client error. - */ - public val attemptsFailure: Int - - /** - * The scheduling attempts that were unsuccessful due to scheduler error. - */ - public val attemptsError: Int -} diff --git a/opendc-compute/opendc-compute-telemetry/src/main/kotlin/org/opendc/compute/telemetry/table/TaskInfo.kt b/opendc-compute/opendc-compute-telemetry/src/main/kotlin/org/opendc/compute/telemetry/table/TaskInfo.kt deleted file mode 100644 index 2d1ae91a..00000000 --- a/opendc-compute/opendc-compute-telemetry/src/main/kotlin/org/opendc/compute/telemetry/table/TaskInfo.kt +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2021 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.compute.telemetry.table - -/** - * Static information about a task exposed to the telemetry service. - */ -public data class TaskInfo( - val id: String, - val name: String, - val type: String, - val arch: String, - val imageId: String, - val imageName: String, - val cpuCount: Int, - val memCapacity: Long, -) diff --git a/opendc-compute/opendc-compute-telemetry/src/main/kotlin/org/opendc/compute/telemetry/table/TaskTableReader.kt b/opendc-compute/opendc-compute-telemetry/src/main/kotlin/org/opendc/compute/telemetry/table/TaskTableReader.kt deleted file mode 100644 index ae7f7a49..00000000 --- a/opendc-compute/opendc-compute-telemetry/src/main/kotlin/org/opendc/compute/telemetry/table/TaskTableReader.kt +++ /dev/null @@ -1,115 +0,0 @@ -/* - * Copyright (c) 2021 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.compute.telemetry.table - -import org.opendc.compute.api.TaskState -import org.opendc.compute.telemetry.export.parquet.DfltTaskExportColumns -import org.opendc.trace.util.parquet.exporter.Exportable -import java.time.Instant - -/** - * An interface that is used to read a row of a task trace entry. - */ -public interface TaskTableReader : Exportable { - public fun copy(): TaskTableReader - - public fun setValues(table: TaskTableReader) - - /** - * The timestamp of the current entry of the reader relative to the start of the workload. - */ - public val timestamp: Instant - - /** - * The timestamp of the current entry of the reader. - */ - public val timestampAbsolute: Instant - - /** - * The [TaskInfo] of the task to which the row belongs to. - */ - public val taskInfo: TaskInfo - - /** - * The [HostInfo] of the host on which the task is hosted or `null` if it has no host. - */ - public val host: HostInfo? - - /** - * The uptime of the host since last time in ms. - */ - public val uptime: Long - - /** - * The downtime of the host since last time in ms. - */ - public val downtime: Long - - /** - * The [Instant] at which the task was enqueued for the scheduler. - */ - public val provisionTime: Instant? - - /** - * The [Instant] at which the task booted relative to the start of the workload. - */ - public val bootTime: Instant? - - /** - * The [Instant] at which the task booted. - */ - public val bootTimeAbsolute: Instant? - - /** - * The capacity of the CPUs of Host on which the task is running (in MHz). - */ - public val cpuLimit: Double - - /** - * The duration (in seconds) that a CPU was active in the task. - */ - public val cpuActiveTime: Long - - /** - * The duration (in seconds) that a CPU was idle in the task. - */ - public val cpuIdleTime: Long - - /** - * The duration (in seconds) that a vCPU wanted to run, but no capacity was available. - */ - public val cpuStealTime: Long - - /** - * The duration (in seconds) of CPU time that was lost due to interference. - */ - public val cpuLostTime: Long - - /** - * The state of the task - */ - public val taskState: TaskState? -} - -// Loads the default export fields for deserialization whenever this file is loaded. -private val _ignore = DfltTaskExportColumns diff --git a/opendc-compute/opendc-compute-telemetry/src/test/resources/log4j2.xml b/opendc-compute/opendc-compute-telemetry/src/test/resources/log4j2.xml deleted file mode 100644 index 0dfb75f2..00000000 --- a/opendc-compute/opendc-compute-telemetry/src/test/resources/log4j2.xml +++ /dev/null @@ -1,38 +0,0 @@ - - - - - - - - - - - - - - - - - - diff --git a/opendc-compute/opendc-compute-topology/src/main/kotlin/org/opendc/compute/topology/TopologyFactories.kt b/opendc-compute/opendc-compute-topology/src/main/kotlin/org/opendc/compute/topology/TopologyFactories.kt index e0fba34f..9e637b1b 100644 --- a/opendc-compute/opendc-compute-topology/src/main/kotlin/org/opendc/compute/topology/TopologyFactories.kt +++ b/opendc-compute/opendc-compute-topology/src/main/kotlin/org/opendc/compute/topology/TopologyFactories.kt @@ -28,11 +28,10 @@ import org.opendc.compute.topology.specs.ClusterSpec import org.opendc.compute.topology.specs.HostJSONSpec import org.opendc.compute.topology.specs.HostSpec import org.opendc.compute.topology.specs.TopologySpec -import org.opendc.simulator.compute.SimPsuFactories -import org.opendc.simulator.compute.model.Cpu -import org.opendc.simulator.compute.model.MachineModel -import org.opendc.simulator.compute.model.MemoryUnit -import org.opendc.simulator.compute.power.getPowerModel +import org.opendc.simulator.compute.cpu.getPowerModel +import org.opendc.simulator.compute.models.CpuModel +import org.opendc.simulator.compute.models.MachineModel +import org.opendc.simulator.compute.models.MemoryUnit import java.io.File import java.io.InputStream import java.util.SplittableRandom @@ -120,10 +119,10 @@ private fun HostJSONSpec.toHostSpecs( ): HostSpec { val units = List(cpu.count) { - Cpu( + CpuModel( globalCoreId++, cpu.coreCount, - cpu.coreSpeed.toMHz(), + cpu.coreSpeed.toMHz().toFloat(), ) } @@ -150,7 +149,7 @@ private fun HostJSONSpec.toHostSpecs( hostName, mapOf("cluster" to clusterId), machineModel, - SimPsuFactories.simple(powerModel), + powerModel, ) hostId++ diff --git a/opendc-compute/opendc-compute-topology/src/main/kotlin/org/opendc/compute/topology/specs/HostSpec.kt b/opendc-compute/opendc-compute-topology/src/main/kotlin/org/opendc/compute/topology/specs/HostSpec.kt index 9857f70a..1956ffde 100644 --- a/opendc-compute/opendc-compute-topology/src/main/kotlin/org/opendc/compute/topology/specs/HostSpec.kt +++ b/opendc-compute/opendc-compute-topology/src/main/kotlin/org/opendc/compute/topology/specs/HostSpec.kt @@ -22,10 +22,8 @@ package org.opendc.compute.topology.specs -import org.opendc.simulator.compute.SimPsuFactories -import org.opendc.simulator.compute.SimPsuFactory -import org.opendc.simulator.compute.model.MachineModel -import org.opendc.simulator.flow2.mux.FlowMultiplexerFactory +import org.opendc.simulator.compute.cpu.CpuPowerModel +import org.opendc.simulator.compute.models.MachineModel import java.util.UUID /** @@ -35,7 +33,7 @@ import java.util.UUID * @param name The name of the host. * @param meta The metadata of the host. * @param model The physical model of the machine. - * @param psuFactory The [SimPsuFactory] to construct the PSU that models the power consumption of the machine. + * @param cpuPowerModel The [SimPsuFactory] to construct the PSU that models the power consumption of the machine. * @param multiplexerFactory The [FlowMultiplexerFactory] that is used to multiplex the virtual machines over the host. */ public data class HostSpec( @@ -43,6 +41,5 @@ public data class HostSpec( val name: String, val meta: Map, val model: MachineModel, - val psuFactory: SimPsuFactory = SimPsuFactories.noop(), - val multiplexerFactory: FlowMultiplexerFactory = FlowMultiplexerFactory.maxMinMultiplexer(), + val cpuPowerModel: CpuPowerModel, ) diff --git a/opendc-compute/opendc-compute-workload/src/main/kotlin/org/opendc/compute/workload/ComputeWorkload.kt b/opendc-compute/opendc-compute-workload/src/main/kotlin/org/opendc/compute/workload/ComputeWorkload.kt index c9f784ff..9516c56e 100644 --- a/opendc-compute/opendc-compute-workload/src/main/kotlin/org/opendc/compute/workload/ComputeWorkload.kt +++ b/opendc-compute/opendc-compute-workload/src/main/kotlin/org/opendc/compute/workload/ComputeWorkload.kt @@ -29,10 +29,10 @@ import java.util.random.RandomGenerator */ public interface ComputeWorkload { /** - * Resolve the workload into a list of [VirtualMachine]s to simulate. + * Resolve the workload into a list of [Task]s to simulate. */ public fun resolve( loader: ComputeWorkloadLoader, random: RandomGenerator, - ): List + ): List } diff --git a/opendc-compute/opendc-compute-workload/src/main/kotlin/org/opendc/compute/workload/ComputeWorkloadLoader.kt b/opendc-compute/opendc-compute-workload/src/main/kotlin/org/opendc/compute/workload/ComputeWorkloadLoader.kt index 99863af8..f22bc1d1 100644 --- a/opendc-compute/opendc-compute-workload/src/main/kotlin/org/opendc/compute/workload/ComputeWorkloadLoader.kt +++ b/opendc-compute/opendc-compute-workload/src/main/kotlin/org/opendc/compute/workload/ComputeWorkloadLoader.kt @@ -23,13 +23,8 @@ package org.opendc.compute.workload import mu.KotlinLogging -import org.opendc.simulator.compute.kernel.interference.VmInterferenceModel -import org.opendc.simulator.compute.workload.SimTrace +import org.opendc.simulator.compute.workload.TraceWorkload import org.opendc.trace.Trace -import org.opendc.trace.conv.INTERFERENCE_GROUP_MEMBERS -import org.opendc.trace.conv.INTERFERENCE_GROUP_SCORE -import org.opendc.trace.conv.INTERFERENCE_GROUP_TARGET -import org.opendc.trace.conv.TABLE_INTERFERENCE_GROUPS import org.opendc.trace.conv.TABLE_RESOURCES import org.opendc.trace.conv.TABLE_RESOURCE_STATES import org.opendc.trace.conv.resourceCpuCapacity @@ -52,7 +47,12 @@ import kotlin.math.roundToLong * * @param baseDir The directory containing the traces. */ -public class ComputeWorkloadLoader(private val baseDir: File) { +public class ComputeWorkloadLoader( + private val baseDir: File, + private val checkpointInterval: Long, + private val checkpointDuration: Long, + private val checkpointIntervalScaling: Double, +) { /** * The logger for this instance. */ @@ -61,7 +61,7 @@ public class ComputeWorkloadLoader(private val baseDir: File) { /** * The cache of workloads. */ - private val cache = ConcurrentHashMap>>() + private val cache = ConcurrentHashMap>>() /** * Read the fragments into memory. @@ -83,7 +83,7 @@ public class ComputeWorkloadLoader(private val baseDir: File) { val cores = reader.getInt(coresCol) val cpuUsage = reader.getDouble(usageCol) - val builder = fragments.computeIfAbsent(id) { Builder() } + val builder = fragments.computeIfAbsent(id) { Builder(checkpointInterval, checkpointDuration, checkpointIntervalScaling) } builder.add(durationMs, cpuUsage, cores) } @@ -99,8 +99,7 @@ public class ComputeWorkloadLoader(private val baseDir: File) { private fun parseMeta( trace: Trace, fragments: Map, - interferenceModel: VmInterferenceModel, - ): List { + ): List { val reader = checkNotNull(trace.getTable(TABLE_RESOURCES)).newReader() val idCol = reader.resolve(resourceID) @@ -111,7 +110,7 @@ public class ComputeWorkloadLoader(private val baseDir: File) { val memCol = reader.resolve(resourceMemCapacity) var counter = 0 - val entries = mutableListOf() + val entries = mutableListOf() return try { while (reader.nextRow()) { @@ -131,7 +130,7 @@ public class ComputeWorkloadLoader(private val baseDir: File) { val totalLoad = builder.totalLoad entries.add( - VirtualMachine( + Task( uid, id, cpuCount, @@ -141,13 +140,12 @@ public class ComputeWorkloadLoader(private val baseDir: File) { submissionTime, duration, builder.build(), - interferenceModel.getProfile(id), ), ) } // Make sure the virtual machines are ordered by start time - entries.sortBy { it.startTime } + entries.sortBy { it.submissionTime } entries } catch (e: Exception) { @@ -158,41 +156,13 @@ public class ComputeWorkloadLoader(private val baseDir: File) { } } - /** - * Read the interference model associated with the specified [trace]. - */ - private fun parseInterferenceModel(trace: Trace): VmInterferenceModel { - val reader = checkNotNull(trace.getTable(TABLE_INTERFERENCE_GROUPS)).newReader() - - return try { - val membersCol = reader.resolve(INTERFERENCE_GROUP_MEMBERS) - val targetCol = reader.resolve(INTERFERENCE_GROUP_TARGET) - val scoreCol = reader.resolve(INTERFERENCE_GROUP_SCORE) - - val modelBuilder = VmInterferenceModel.builder() - - while (reader.nextRow()) { - val members = reader.getSet(membersCol, String::class.java)!! - val target = reader.getDouble(targetCol) - val score = reader.getDouble(scoreCol) - - modelBuilder - .addGroup(members, target, score) - } - - modelBuilder.build() - } finally { - reader.close() - } - } - /** * Load the trace with the specified [name] and [format]. */ public fun get( name: String, format: String, - ): List { + ): List { val ref = cache.compute(name) { key, oldVal -> val inst = oldVal?.get() @@ -203,8 +173,7 @@ public class ComputeWorkloadLoader(private val baseDir: File) { val trace = Trace.open(path, format) val fragments = parseFragments(trace) - val interferenceModel = parseInterferenceModel(trace) - val vms = parseMeta(trace, fragments, interferenceModel) + val vms = parseMeta(trace, fragments) SoftReference(vms) } else { @@ -225,7 +194,7 @@ public class ComputeWorkloadLoader(private val baseDir: File) { /** * A builder for a VM trace. */ - private class Builder { + private class Builder(checkpointInterval: Long, checkpointDuration: Long, checkpointIntervalScaling: Double) { /** * The total load of the trace. */ @@ -234,13 +203,12 @@ public class ComputeWorkloadLoader(private val baseDir: File) { /** * The internal builder for the trace. */ - private val builder = SimTrace.builder() + private val builder = TraceWorkload.builder(checkpointInterval, checkpointDuration, checkpointIntervalScaling) /** * Add a fragment to the trace. * - * @param timestamp Timestamp at which the fragment starts (in epoch millis). - * @param deadline Timestamp at which the fragment ends (in epoch millis). + * @param duration The duration of the fragment (in epoch millis). * @param usage CPU usage of this fragment. * @param cores Number of cores used. */ @@ -257,6 +225,6 @@ public class ComputeWorkloadLoader(private val baseDir: File) { /** * Build the trace. */ - fun build(): SimTrace = builder.build() + fun build(): TraceWorkload = builder.build() } } diff --git a/opendc-compute/opendc-compute-workload/src/main/kotlin/org/opendc/compute/workload/Task.kt b/opendc-compute/opendc-compute-workload/src/main/kotlin/org/opendc/compute/workload/Task.kt new file mode 100644 index 00000000..d121b381 --- /dev/null +++ b/opendc-compute/opendc-compute-workload/src/main/kotlin/org/opendc/compute/workload/Task.kt @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.compute.workload + +import org.opendc.simulator.compute.workload.TraceWorkload +import java.time.Instant +import java.util.UUID + +/** + * A virtual machine workload. + * + * @param uid The unique identifier of the virtual machine. + * @param name The name of the virtual machine. + * @param cpuCapacity The required CPU capacity for the VM in MHz. + * @param cpuCount The number of vCPUs in the VM. + * @param memCapacity The provisioned memory for the VM in MB. + * @param submissionTime The start time of the VM. + * @param trace The trace that belong to this VM. + * @param interferenceProfile The interference profile of this virtual machine. + */ +public data class Task( + val uid: UUID, + val name: String, + val cpuCount: Int, + val cpuCapacity: Double, + val memCapacity: Long, + val totalLoad: Double, + val submissionTime: Instant, + val duration: Long, + val trace: TraceWorkload, +) diff --git a/opendc-compute/opendc-compute-workload/src/main/kotlin/org/opendc/compute/workload/VirtualMachine.kt b/opendc-compute/opendc-compute-workload/src/main/kotlin/org/opendc/compute/workload/VirtualMachine.kt deleted file mode 100644 index 66d51127..00000000 --- a/opendc-compute/opendc-compute-workload/src/main/kotlin/org/opendc/compute/workload/VirtualMachine.kt +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright (c) 2021 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.compute.workload - -import org.opendc.simulator.compute.kernel.interference.VmInterferenceProfile -import org.opendc.simulator.compute.workload.SimTrace -import java.time.Instant -import java.util.UUID - -/** - * A virtual machine workload. - * - * @param uid The unique identifier of the virtual machine. - * @param name The name of the virtual machine. - * @param cpuCapacity The required CPU capacity for the VM in MHz. - * @param cpuCount The number of vCPUs in the VM. - * @param memCapacity The provisioned memory for the VM in MB. - * @param startTime The start time of the VM. - * @param stopTime The stop time of the VM. - * @param trace The trace that belong to this VM. - * @param interferenceProfile The interference profile of this virtual machine. - */ -public data class VirtualMachine( - val uid: UUID, - val name: String, - val cpuCount: Int, - val cpuCapacity: Double, - val memCapacity: Long, - val totalLoad: Double, - val startTime: Instant, - val duration: Long, - val trace: SimTrace, - val interferenceProfile: VmInterferenceProfile?, -) diff --git a/opendc-compute/opendc-compute-workload/src/main/kotlin/org/opendc/compute/workload/internal/CompositeComputeWorkload.kt b/opendc-compute/opendc-compute-workload/src/main/kotlin/org/opendc/compute/workload/internal/CompositeComputeWorkload.kt index aba493b6..998dbb34 100644 --- a/opendc-compute/opendc-compute-workload/src/main/kotlin/org/opendc/compute/workload/internal/CompositeComputeWorkload.kt +++ b/opendc-compute/opendc-compute-workload/src/main/kotlin/org/opendc/compute/workload/internal/CompositeComputeWorkload.kt @@ -25,7 +25,7 @@ package org.opendc.compute.workload.internal import mu.KotlinLogging import org.opendc.compute.workload.ComputeWorkload import org.opendc.compute.workload.ComputeWorkloadLoader -import org.opendc.compute.workload.VirtualMachine +import org.opendc.compute.workload.Task import java.util.random.RandomGenerator /** @@ -40,12 +40,12 @@ internal class CompositeComputeWorkload(val sources: Map { + ): List { val traces = sources.map { (source, fraction) -> fraction to source.resolve(loader, random) } val totalLoad = traces.sumOf { (_, vms) -> vms.sumOf { it.totalLoad } } - val res = mutableListOf() + val res = mutableListOf() for ((fraction, vms) in traces) { var currentLoad = 0.0 diff --git a/opendc-compute/opendc-compute-workload/src/main/kotlin/org/opendc/compute/workload/internal/HpcSampledComputeWorkload.kt b/opendc-compute/opendc-compute-workload/src/main/kotlin/org/opendc/compute/workload/internal/HpcSampledComputeWorkload.kt index 4207b2be..d3bdde31 100644 --- a/opendc-compute/opendc-compute-workload/src/main/kotlin/org/opendc/compute/workload/internal/HpcSampledComputeWorkload.kt +++ b/opendc-compute/opendc-compute-workload/src/main/kotlin/org/opendc/compute/workload/internal/HpcSampledComputeWorkload.kt @@ -25,7 +25,7 @@ package org.opendc.compute.workload.internal import mu.KotlinLogging import org.opendc.compute.workload.ComputeWorkload import org.opendc.compute.workload.ComputeWorkloadLoader -import org.opendc.compute.workload.VirtualMachine +import org.opendc.compute.workload.Task import java.util.UUID import java.util.random.RandomGenerator @@ -53,7 +53,7 @@ internal class HpcSampledComputeWorkload( override fun resolve( loader: ComputeWorkloadLoader, random: RandomGenerator, - ): List { + ): List { val vms = source.resolve(loader, random) val (hpc, nonHpc) = @@ -65,7 +65,7 @@ internal class HpcSampledComputeWorkload( val hpcSequence = generateSequence(0) { it + 1 } .map { index -> - val res = mutableListOf() + val res = mutableListOf() hpc.mapTo(res) { sample(it, index) } res } @@ -74,7 +74,7 @@ internal class HpcSampledComputeWorkload( val nonHpcSequence = generateSequence(0) { it + 1 } .map { index -> - val res = mutableListOf() + val res = mutableListOf() nonHpc.mapTo(res) { sample(it, index) } res } @@ -90,7 +90,7 @@ internal class HpcSampledComputeWorkload( var nonHpcCount = 0 var nonHpcLoad = 0.0 - val res = mutableListOf() + val res = mutableListOf() if (sampleLoad) { var currentLoad = 0.0 @@ -146,9 +146,9 @@ internal class HpcSampledComputeWorkload( * Sample a random trace entry. */ private fun sample( - entry: VirtualMachine, + entry: Task, i: Int, - ): VirtualMachine { + ): Task { val uid = UUID.nameUUIDFromBytes("${entry.uid}-$i".toByteArray()) return entry.copy(uid = uid) } diff --git a/opendc-compute/opendc-compute-workload/src/main/kotlin/org/opendc/compute/workload/internal/LoadSampledComputeWorkload.kt b/opendc-compute/opendc-compute-workload/src/main/kotlin/org/opendc/compute/workload/internal/LoadSampledComputeWorkload.kt index 51ddb27c..534ac6a0 100644 --- a/opendc-compute/opendc-compute-workload/src/main/kotlin/org/opendc/compute/workload/internal/LoadSampledComputeWorkload.kt +++ b/opendc-compute/opendc-compute-workload/src/main/kotlin/org/opendc/compute/workload/internal/LoadSampledComputeWorkload.kt @@ -25,7 +25,7 @@ package org.opendc.compute.workload.internal import mu.KotlinLogging import org.opendc.compute.workload.ComputeWorkload import org.opendc.compute.workload.ComputeWorkloadLoader -import org.opendc.compute.workload.VirtualMachine +import org.opendc.compute.workload.Task import java.util.random.RandomGenerator /** @@ -40,9 +40,9 @@ internal class LoadSampledComputeWorkload(val source: ComputeWorkload, val fract override fun resolve( loader: ComputeWorkloadLoader, random: RandomGenerator, - ): List { + ): List { val vms = source.resolve(loader, random) // fixme: Should be shuffled, otherwise the first fraction is always chosen - val res = mutableListOf() + val res = mutableListOf() val totalLoad = vms.sumOf { it.totalLoad } var currentLoad = 0.0 diff --git a/opendc-compute/opendc-compute-workload/src/main/kotlin/org/opendc/compute/workload/internal/TraceComputeWorkload.kt b/opendc-compute/opendc-compute-workload/src/main/kotlin/org/opendc/compute/workload/internal/TraceComputeWorkload.kt index 39255c59..d796341b 100644 --- a/opendc-compute/opendc-compute-workload/src/main/kotlin/org/opendc/compute/workload/internal/TraceComputeWorkload.kt +++ b/opendc-compute/opendc-compute-workload/src/main/kotlin/org/opendc/compute/workload/internal/TraceComputeWorkload.kt @@ -24,7 +24,7 @@ package org.opendc.compute.workload.internal import org.opendc.compute.workload.ComputeWorkload import org.opendc.compute.workload.ComputeWorkloadLoader -import org.opendc.compute.workload.VirtualMachine +import org.opendc.compute.workload.Task import java.util.random.RandomGenerator /** @@ -34,7 +34,7 @@ internal class TraceComputeWorkload(val name: String, val format: String) : Comp override fun resolve( loader: ComputeWorkloadLoader, random: RandomGenerator, - ): List { + ): List { return loader.get(name, format) } } -- cgit v1.2.3