summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorFabian Mastenbroek <mail.fabianm@gmail.com>2021-10-25 17:40:04 +0200
committerGitHub <noreply@github.com>2021-10-25 17:40:04 +0200
commitb4bf7268cbb6d22d3966f469a6b7721b04d91907 (patch)
tree13756bd08650c0e41e24132b8dab0fad327ddf31
parenta41cd2504f15f3e3e49eb533faca390911cc5110 (diff)
parentc4ce5ebb7de6494cb8d90076cba8596aa0cbabeb (diff)
merge: Support conversion from Azure traces to OpenDC traces (#36)
This pull request adds support for converting Azure traces to the OpenDC trace format. * Support GZIP files in Azure trace * Fix timestamp retrieval for Azure trace * Add column for CPU capacity in OpenDC format * Support conversion from Azure trace format
-rw-r--r--opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/util/CompositeTableReader.kt19
-rw-r--r--opendc-trace/opendc-trace-azure/src/main/kotlin/org/opendc/trace/azure/AzureResourceStateTableReader.kt2
-rw-r--r--opendc-trace/opendc-trace-azure/src/main/kotlin/org/opendc/trace/azure/AzureResourceTableReader.kt4
-rw-r--r--opendc-trace/opendc-trace-azure/src/main/kotlin/org/opendc/trace/azure/AzureTraceFormat.kt17
-rw-r--r--opendc-trace/opendc-trace-azure/src/test/kotlin/org/opendc/trace/azure/AzureTraceFormatTest.kt2
-rw-r--r--opendc-trace/opendc-trace-azure/src/test/resources/trace/vm_cpu_readings/vm_cpu_readings-file-1-of-125.csv100
-rw-r--r--opendc-trace/opendc-trace-azure/src/test/resources/trace/vm_cpu_readings/vm_cpu_readings-file-1-of-125.csv.gzbin0 -> 6905 bytes
-rw-r--r--opendc-trace/opendc-trace-azure/src/test/resources/trace/vmtable/vmtable.csv10
-rw-r--r--opendc-trace/opendc-trace-azure/src/test/resources/trace/vmtable/vmtable.csv.gzbin0 -> 1423 bytes
-rw-r--r--opendc-trace/opendc-trace-opendc/src/main/kotlin/org/opendc/trace/opendc/OdcVmResourceTableReader.kt8
-rw-r--r--opendc-trace/opendc-trace-opendc/src/main/kotlin/org/opendc/trace/opendc/OdcVmResourceTableWriter.kt1
-rw-r--r--opendc-trace/opendc-trace-opendc/src/main/kotlin/org/opendc/trace/opendc/OdcVmTraceFormat.kt4
-rw-r--r--opendc-trace/opendc-trace-tools/src/main/kotlin/org/opendc/trace/tools/TraceConverter.kt400
-rw-r--r--opendc-trace/opendc-trace-tools/src/main/resources/log4j2.xml38
14 files changed, 373 insertions, 232 deletions
diff --git a/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/util/CompositeTableReader.kt b/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/util/CompositeTableReader.kt
index dafc0798..11e032c7 100644
--- a/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/util/CompositeTableReader.kt
+++ b/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/util/CompositeTableReader.kt
@@ -46,11 +46,7 @@ public abstract class CompositeTableReader : TableReader {
protected abstract fun nextReader(): TableReader?
override fun nextRow(): Boolean {
- if (!hasStarted) {
- assert(delegate == null) { "Duplicate initialization" }
- delegate = nextReader()
- hasStarted = true
- }
+ tryStart()
var delegate = delegate
@@ -68,6 +64,8 @@ public abstract class CompositeTableReader : TableReader {
}
override fun resolve(column: TableColumn<*>): Int {
+ tryStart()
+
val delegate = delegate
return delegate?.resolve(column) ?: -1
}
@@ -107,4 +105,15 @@ public abstract class CompositeTableReader : TableReader {
}
override fun toString(): String = "CompositeTableReader"
+
+ /**
+ * Try to obtain the initial delegate.
+ */
+ private fun tryStart() {
+ if (!hasStarted) {
+ assert(delegate == null) { "Duplicate initialization" }
+ delegate = nextReader()
+ hasStarted = true
+ }
+ }
}
diff --git a/opendc-trace/opendc-trace-azure/src/main/kotlin/org/opendc/trace/azure/AzureResourceStateTableReader.kt b/opendc-trace/opendc-trace-azure/src/main/kotlin/org/opendc/trace/azure/AzureResourceStateTableReader.kt
index da8181fe..94a91999 100644
--- a/opendc-trace/opendc-trace-azure/src/main/kotlin/org/opendc/trace/azure/AzureResourceStateTableReader.kt
+++ b/opendc-trace/opendc-trace-azure/src/main/kotlin/org/opendc/trace/azure/AzureResourceStateTableReader.kt
@@ -53,7 +53,7 @@ internal class AzureResourceStateTableReader(private val parser: CsvParser) : Ta
when (parser.currentName) {
"timestamp" -> timestamp = Instant.ofEpochSecond(parser.longValue)
"vm id" -> id = parser.text
- "CPU avg cpu" -> cpuUsagePct = parser.doubleValue
+ "CPU avg cpu" -> cpuUsagePct = (parser.doubleValue / 100.0) // Convert from % to [0, 1]
}
}
diff --git a/opendc-trace/opendc-trace-azure/src/main/kotlin/org/opendc/trace/azure/AzureResourceTableReader.kt b/opendc-trace/opendc-trace-azure/src/main/kotlin/org/opendc/trace/azure/AzureResourceTableReader.kt
index a6352613..6246dc35 100644
--- a/opendc-trace/opendc-trace-azure/src/main/kotlin/org/opendc/trace/azure/AzureResourceTableReader.kt
+++ b/opendc-trace/opendc-trace-azure/src/main/kotlin/org/opendc/trace/azure/AzureResourceTableReader.kt
@@ -52,8 +52,8 @@ internal class AzureResourceTableReader(private val parser: CsvParser) : TableRe
when (parser.currentName) {
"vm id" -> id = parser.text
- "vm created" -> startTime = Instant.ofEpochSecond(parser.longValue)
- "vm deleted" -> stopTime = Instant.ofEpochSecond(parser.longValue)
+ "timestamp vm created" -> startTime = Instant.ofEpochSecond(parser.longValue)
+ "timestamp vm deleted" -> stopTime = Instant.ofEpochSecond(parser.longValue)
"vm virtual core count" -> cpuCores = parser.intValue
"vm memory" -> memCapacity = parser.doubleValue * 1e6 // GB to KB
}
diff --git a/opendc-trace/opendc-trace-azure/src/main/kotlin/org/opendc/trace/azure/AzureTraceFormat.kt b/opendc-trace/opendc-trace-azure/src/main/kotlin/org/opendc/trace/azure/AzureTraceFormat.kt
index 253c7057..c9982877 100644
--- a/opendc-trace/opendc-trace-azure/src/main/kotlin/org/opendc/trace/azure/AzureTraceFormat.kt
+++ b/opendc-trace/opendc-trace-azure/src/main/kotlin/org/opendc/trace/azure/AzureTraceFormat.kt
@@ -31,8 +31,9 @@ import org.opendc.trace.util.CompositeTableReader
import java.nio.file.Files
import java.nio.file.Path
import java.util.stream.Collectors
-import kotlin.io.path.extension
-import kotlin.io.path.nameWithoutExtension
+import java.util.zip.GZIPInputStream
+import kotlin.io.path.inputStream
+import kotlin.io.path.name
/**
* A format implementation for the Azure v1 format.
@@ -81,7 +82,10 @@ public class AzureTraceFormat : TraceFormat {
override fun newReader(path: Path, table: String): TableReader {
return when (table) {
- TABLE_RESOURCES -> AzureResourceTableReader(factory.createParser(path.resolve("vmtable/vmtable.csv").toFile()))
+ TABLE_RESOURCES -> {
+ val stream = GZIPInputStream(path.resolve("vmtable/vmtable.csv.gz").inputStream())
+ AzureResourceTableReader(factory.createParser(stream))
+ }
TABLE_RESOURCE_STATES -> newResourceStateReader(path)
else -> throw IllegalArgumentException("Table $table not supported")
}
@@ -96,8 +100,8 @@ public class AzureTraceFormat : TraceFormat {
*/
private fun newResourceStateReader(path: Path): TableReader {
val partitions = Files.walk(path.resolve("vm_cpu_readings"), 1)
- .filter { !Files.isDirectory(it) && it.extension == "csv" }
- .collect(Collectors.toMap({ it.nameWithoutExtension }, { it }))
+ .filter { !Files.isDirectory(it) && it.name.endsWith(".csv.gz") }
+ .collect(Collectors.toMap({ it.name.removeSuffix(".csv.gz") }, { it }))
.toSortedMap()
val it = partitions.iterator()
@@ -105,7 +109,8 @@ public class AzureTraceFormat : TraceFormat {
override fun nextReader(): TableReader? {
return if (it.hasNext()) {
val (_, partPath) = it.next()
- return AzureResourceStateTableReader(factory.createParser(partPath.toFile()))
+ val stream = GZIPInputStream(partPath.inputStream())
+ return AzureResourceStateTableReader(factory.createParser(stream))
} else {
null
}
diff --git a/opendc-trace/opendc-trace-azure/src/test/kotlin/org/opendc/trace/azure/AzureTraceFormatTest.kt b/opendc-trace/opendc-trace-azure/src/test/kotlin/org/opendc/trace/azure/AzureTraceFormatTest.kt
index b73bb728..eda0b214 100644
--- a/opendc-trace/opendc-trace-azure/src/test/kotlin/org/opendc/trace/azure/AzureTraceFormatTest.kt
+++ b/opendc-trace/opendc-trace-azure/src/test/kotlin/org/opendc/trace/azure/AzureTraceFormatTest.kt
@@ -77,7 +77,7 @@ class AzureTraceFormatTest {
{ assertTrue(reader.nextRow()) },
{ assertEquals("+ZcrOp5/c/fJ6mVgP5qMZlOAGDwyjaaDNM0WoWOt2IDb47gT0UwK9lFwkPQv3C7Q", reader.get(RESOURCE_ID)) },
{ assertEquals(0, reader.get(RESOURCE_STATE_TIMESTAMP).epochSecond) },
- { assertEquals(2.86979, reader.getDouble(RESOURCE_STATE_CPU_USAGE_PCT), 0.01) }
+ { assertEquals(0.0286979, reader.getDouble(RESOURCE_STATE_CPU_USAGE_PCT), 0.01) }
)
reader.close()
diff --git a/opendc-trace/opendc-trace-azure/src/test/resources/trace/vm_cpu_readings/vm_cpu_readings-file-1-of-125.csv b/opendc-trace/opendc-trace-azure/src/test/resources/trace/vm_cpu_readings/vm_cpu_readings-file-1-of-125.csv
deleted file mode 100644
index db6ddf8a..00000000
--- a/opendc-trace/opendc-trace-azure/src/test/resources/trace/vm_cpu_readings/vm_cpu_readings-file-1-of-125.csv
+++ /dev/null
@@ -1,100 +0,0 @@
-0,+ZcrOp5/c/fJ6mVgP5qMZlOAGDwyjaaDNM0WoWOt2IDb47gT0UwK9lFwkPQv3C7Q,2.052803,3.911587,2.86979
-0,2zrgeOqUDy+l0GVi5NXudU+3sqZH+nLowfcz+D/JsCymTXbKrRf1Hr3OjAtxjnKm,1.64695,8.794403,3.254472
-0,/34Wh1Kq/qkNkW0tQrMiQ1eZ8hg9hHopydCzsXriefhgrn+0Rg1j22k1IHcV6PIQ,2.440088,6.941048,4.33624
-0,2lzdXk1Rqn1ibH2kZhGamYTMvVcRP6+x8b5zGiD/8t++5BQhzU18hGaL5sfR01Lo,0.302992,2.046712,0.970692
-0,0GrUQuLhCER5bWWcoJAgblPJWkaU4v3nf+NUrZnFTlXWEK99qgTRBTkjjUjJVAqA,1.515922,4.471657,2.438805
-0,2I8OpI6bMkdzL3HYLz4KlBcDhy2VTEm3skQbOvEo9rLoxryB0iB9iVh3rGd5DW2j,0.148552,0.315007,0.264341
-0,2IuuDcRMd97gln/+CrgPqI/fwffx67s87T1odKrA0wLYf8YuzGooHdKihitv2Q+s,0.169838,2.277277,0.859669
-0,2KaB1faO0ZB2KqB8MGwasWkqRLJHIE6+2wPuhzlzLNEUyeGzo0dU7brFa/cll/VJ,0.539162,1.371926,0.782212
-0,2BMVXt472mr/Y8m1vaaGoyGTSXcLvXk968PCHixwCDjPSgCm7yYSimGuBw7VPIiS,3.625195,7.211996,4.807884
-0,/3+EY60PnzKwod6nAUGBFSDDpBBOVEVUi90JWWWjPAlNyTUrGwlfQcSDoSkRumD7,0.180582,1.313473,0.43792
-0,+hulsuci78MKSG60G/gHJLqmz5/TFEB3WpS6HI1G1mm052le8oeemF3kz3eoPsnS,2.653344,9.983403,4.262461
-0,0O4otykohyRcsqqsg68kqo6ZCY6sL6eQLHUMYZxGVRhwQmTXRUN89izib3pOucrC,0.72983,4.516831,1.846142
-0,239KfRqrlUdyYuU0ubcASPKztu3q7hernahrolO5AczjUFI/QgoU+OoKzPuivFHQ,1.42953,11.553488,4.271241
-0,/SVzWHvPhr7KAIOUFr10EK8WdKXbrJojgcc4IGvutJ2S6HpRMD0zTfv/h0720+Q6,1.676634,6.703915,3.102252
-0,2a7bYEHqZvcgOeos5Q3J5qxpY4lXinv8M9mORfel5DlWRut0JynZtobNGNlBWn41,0.54178,8.239316,2.234877
-0,1NwFYwEAgv8qnCaWzzWv9hHj0TIJAZ2HT+iH+dsZKeSAPGoJGyVSDB+Zj4EuqWRC,2.933124,4.630701,3.975568
-0,3rg4SRyS/p6eMuGCJpjkz4oHzXSeeF16a7jJ9GAAYPiAQAsQNOEjHOe07on5RbjK,0.719536,3.383279,1.506528
-0,0DVV+uR/jr4XbwYQhVf2Yg0Kg7DfIDa7qJNzqvjVgEqGRJAUisrnYFv7AWr1k7by,0.949333,22.157649,3.751856
-0,3bHtb6EIFo9yXByIhpVDOJ7bzbIQvnGGb+jm8eOsEf0eKbrKMJvUiOYc6Wq3DXbR,3.504534,15.581505,6.388424
-0,0O5yc/ZVSHWxf4UHf/1b8Nut9raakorgqDwGV9k7TJdq55alNeMDB7CREuxZystP,7.587743,20.323464,16.540802
-0,0ZbYi+cMH7hCzT+8ICYVp5ZgcRUFNKsODuH09bbPdPioUPCPkBK2PM2oHhE3y4I6,2.694185,6.361789,4.55337
-0,1jw70sEl89jY2iRpd38PuYSBiOcuwe6tF4Q+YuGBJg20+gRIW3A7H3WZ+uL0EVmb,3.570395,5.707428,4.233997
-0,24MvpVXzcNO2qxwF4hMwCToFTBfoAE5xUQ4L6fwfWuBZ1GW06hHh5jWwWCu+8lPm,5.102273,8.678012,6.369649
-0,2gHzFAqM+fL7f1wtNETuzSoM7I6xlEWk2BJmj1SNXly/7z1RQFmwYRXU49DiYciJ,0.27374,54.447146,5.445003
-0,/hCom+lGMIkeE1wQi+VTFh+zzgbikbO0jQDzchDMCUNSgo6cEJfD1sIT2Ok4NlD6,0.170892,1.843549,0.737087
-0,2UwesOu8HXTdHyj0jd1agckz1KH5+Z4KOFe+wKFo9uvRI4GalozAPaxsMrBmx7Wo,3.349887,6.272554,4.425039
-0,1V8Fr/ZhjQcxql5s9p3hA1b0Wx6Sx9e+np1OImlp3GKyleH87bYjmQLZJouKYJR2,2.022219,4.724097,2.616506
-0,23E/SPMZKCUWz8nBmuCdbNBWf9ou6IQuZjmh0x2/icPrbLLvUk5SvbTjwqoLQxBX,0,0.46365,0.178483
-0,0Mj8nT0fnkeMIbcTBf27pOtUuTtMZH8uAZqAViSaye+9mBIjsNPmU6Z5hLK6f2I0,15.023186,23.297875,18.965327
-0,2xM0uOcqSowNzsbFbzhy5J1Ms2vv0jVQ5aM+J2E/LCBzTVKPrCCeWQ/r/cKmS1Tm,8.272075,9.415241,8.797159
-0,0MYQXyW75q9UURkn+O/V6iww0JaBl2qRG0Mh2bqRcuU5/Ws+7HJMPKSzVKlUEgcU,3.798828,8.915124,4.856879
-0,/HQfnMjgclpCxPod9jmGVQxfTnsjyNWA4KNkLMn4IKRlqheUo9AhhWv4vAumZNqg,4.788548,7.269977,6.640435
-0,0Q2PP+9O7LcnNI7AJQQR7pwM4ISG4024Z+INOw+TWgf2DCl8/prdGC7QJRGjc+Aa,0.10703,0.183798,0.136907
-0,/zLQxB1DGXC7iK7JeyYrUSguf6DjNA1MVTJzieRWmcobm0M+xgd28r842y3p5u5J,1.306953,3.22913,2.226509
-0,42cXpXkVqdXH/ok/tD46zKKCToy0k6HXoH3x7eeo4+zIva3IJKle5xfSEW3R45ON,1.018462,3.240817,2.196357
-0,+9HYwMx1Ckj15bJswEycBgiBSfrBw5NJE3p86IeFpFYKKxdw3NzMPTFKpg67XhsF,1.859664,7.255261,3.501303
-0,10KKTL95cApo6Pf24KZqgrM67v4M6rgZBoX+w/I3j4KS66FNhKomGnap9H8SVAvy,0.041225,2.593651,0.25894
-0,+LyaeKb1faiLEjAzynXF3xO/ZAho1R/Zyh1H4d45+NGsIJR6ryUTDmhyNvMh1wQ9,4.614357,11.692623,6.05005
-0,1SS5EeD9rxdWRFYBkR36PAd96w+Q7V2V4fDcc/2IJ1L07In7RGpQk/HVcOTKd78w,0.020435,0.515471,0.135453
-0,+HFoxb6Eu9kwzVkxs+A+9Q7zXa4aSIcOFm3AnYDCTQQMYyf6EST9nSHslGhUkgAD,8.53904,48.459572,16.166212
-0,+N+B5FPJIUVyH9v1Zcc+kjSTNvULkosDBM48N2JkDjhuVhQtWSfYQMQTQkGeVjLi,3.139119,99.036916,51.090982
-0,1ey9c7Hc1FyxLVbESoty7AkXbuENFSDXRAZiizFifRmJNM6IEx9eNu3bkUR+qCUJ,2.466582,5.842213,3.765056
-0,35F/52yPsKPGondM8xnzX68EKiKiKiZMDqsVnvc9ZOAc/rS3zvQ6YYj3QkLAHFhN,1.963258,43.494868,16.459037
-0,2KX+BTc0TPZOtCgbzKtKvP1yrM+Cc3WQU9DPkZDFD/5aNN/aPV40aQCKwW/HeTzh,1.040522,5.961609,3.305858
-0,+8X+qRHRLwwgj70uuXrkus7lrNtjMeTHfy5yQgymNJI+yFd5pbhRfStfS7lkVOhP,0.436353,15.995153,1.431229
-0,/g5MAtFnYaMO5MpJg40BsFmhS22s0tfwHiivGhPbcZ+KgEAtNxKkFdZYDtrDUUFO,6.905489,8.196952,7.527238
-0,/ke0seVq80UFQeXSTUh5hTrjghtn5qqWf38lQVTis+/ZR6Pdv5vdAotz4dvZcKDp,6.444482,23.136676,15.470455
-0,+tQeKqKqbAui7YXK0Efk3GUnvbzM+0pOpmOJ6OhkMSozjRyl5tHl7+mZwFznU3Mk,17.90259,20.095464,18.937014
-0,/hiC5yD45GhNtMpJTVwVF5ZnNNWfEHttESv/+KH6go9FBoncns+CuQ1M92c0xzFA,2.290396,2.609893,2.523336
-0,0i9+1LVd2t4m1KScDuoJnAAEL0bz9UGXh2iLAGV/8Eq5hTsAliyraV7j6wsf2MZX,4.266491,16.607137,6.929279
-0,2PVcv0/vy8mIjzH7CiB9cJU737jRi6kAO7PhqkxEWA4GrxvaCsK3ZDckhD8YR04U,1.048596,2.309172,1.447266
-0,/kbT+MIfY7jEW2Nn+TKf5BKkLAmBslDqKuZ8HI2Ire6eMKinGP7aTt6SY77vt8PK,2.409783,7.79851,5.552826
-0,2cCRKSXs9v9tPskjJn8UmV15qynI3I3GLPTor/i81nxh5Ocwb7Fq1zwEN5zmtXyx,0.356014,1.468193,0.781642
-0,2qsVNbcvPD0H3cs/p/6MTpuvUBtr5QN3iavAmkCQBCtrHcEpgskYVJf/6WQkEhOF,2.688901,85.501739,37.676562
-0,30FpxnoytvMKoGeJYqwnuL2mPbvKlxpjPIfVT8LKqqFl9smEksQjEzG3lgxhT4U7,2.499018,6.534664,3.508567
-0,/f1C+4xtoPaBxD+FoFdM52MiaWXZEqPqSnBxz4q4XMzoXabJvdddHchLrxc6SlYc,1.894231,15.683948000000001,3.199591
-0,30tz9NOV1bIKUB6uIOy4qZT8BVk3escZ0bWXBD9oedOQN1Qi06pplm7WM9iMvvvL,0.959278,63.599827,14.983399
-0,2q0sA6/4VZfksnucqVASzYgruD9T0219afuGrf3O/u8jpGHpn0k3oWvY35I7x8F/,2.694575,11.900751,5.254742
-0,/Qq/SKTnRJ4RZPWKIdCyPmYQUf+csOcFYS+rVD+kc1OkLboeKHK7CLV88wVVLlm9,55.553347,99.204744,93.215797
-0,2PJIXiy3/m1MNf4SQAQ9xU+LDqsHvyyCIWA2X0nB9kgLyVNh3g9xxpAeUpkXgvK6,0.591771,0.676084,0.628958
-0,/VIH23Tzi+711eCdsc7apDAoSBY6hcNqCu8oaZcPrUQmUXUyH8HJS7Z1DyhR6j/I,3.136726,5.477124,4.036594
-0,3/bNFRCZog1M2qwSCcwMYYos07f/9kRsfeFyaOmT0mNx3ldbNvRRbMBhoseq0DIg,2.993954,5.787727,4.272684
-0,3F+42xbLAiVPTJeHpyDwx6ZXcxArLFiMGGZTa9jmsLIpxxkBqC1QwN8mAwzDqWsU,3.488578,6.178318,4.692753
-0,08iqvtN8ilXeJdfiL86fde5JRTrjuLTp8guNabblV7QqkkAL23TwtLdwuFtg4P9G,3.64316,22.992153,10.256498
-0,0ZiQ/5P4mgnYud0uaI1lZCIJaCzrlEJdnAz8bcFMLDFryCrUJJDecbWQbLo6K69J,2.924592,4.261972,3.543138
-0,28JHlDFu72v9lIhjKLF+h9g1pyPq9+ruVET8NnBGKksclnvwx0WlQ066nh6doanS,1.2833,1.589682,1.353967
-0,3ClcWgHBEw8WzFSqnMYKUib9Abx6RDf3ITN8ivUilopa4t+UTJU0Y/U25sT/1okS,1.387814,2.764987,2.116221
-0,/qj8bL8dARqa83U6HwU/bUF5kLq12PKaebM0/2WrM2a3oH+BCC/IxFf1PjIWBNC5,23.139855,97.95723,75.918613
-0,17KWFIkHqLQpslptyD70Qof2iISdFN4IzZBc/WffQeds/tDjuZ/1O4KY68u10srE,2.374392,4.461708,3.201956
-0,3fNyZ1Bf9hUvTVDbHwh8Fh3E2i0BgPPL3QkkS9T0cjanDQA0u0z/Y5TSdXldEJM8,1.199056,3.188352,2.14033
-0,3DYNNYBvhBlVPHsg1uoo7ZVjKX5k1c0gZsfc8W0o0cJ1WJAI8f049TnSu/yIfp/m,1.305688,4.700476,2.216015
-0,2e9qO7smv0DTuXeR3VEzG2jztbM9wntJ3bMt6/LlN3RZBQzIY9vP7FFsphJC9bsW,0.087859,22.556549,10.203507
-0,3EeP6Vgbh292ahLWQJrInzehyR4Nuj2vNtdWuEbvFjKcmCc2i6VZVN4dQTRfIVxR,7.663198,22.199953,15.461753
-0,/Vi7oNg70eAzJHXwsCM9nzwBMg4l7cMyZhUT14V48AWjIAQzVYsbdI0KwNlBAXhK,0.61977,2.24158,1.181003
-0,4/c7nkT3SrtRRrRCsZxUJXxJjUr61iivwZxdihwPAtpCDUawKfPUzaq/05zFYBAk,2.667104,7.383679,4.050989
-0,1HYzfmk+s4SedWtOeHk4j5Zj52ateGX5bRFK5K3rwTVdB2A2m+3iwbL1IEzx8ir8,5.366892,12.404488,6.877072
-0,3vPq2HsXQ9SQT+URugEaQ3ezvstcGd5Bt9FIiFx1SrUfUrvvi/Gj8Nyw5DZhvyAR,3.014601,13.363316,4.535414
-0,2YbmUab2MqBMpvMaoaMP3zVxOhgqkNytraWdt/GG261oZ/tmgEB239WsbKJh1bE3,3.121409,98.73306,51.009852
-0,1IYQhDD8NGuAFnVPnffmt1yk20B9JHQI5DMC4Ny09pe6Sedik6YCIIVeBHIEo34W,1.512222,3.53396,2.379989
-0,1SSMSUcJ7qKM7q2yka80+ZP0yYWiYxGQxcJ8KBi4+TsDpv5FLUS6i2DHLMtXB3An,3.9704,4.345802,4.126586
-0,17CA6zpUCxW+Pdh2g5W0kTdlPlgWbBKz4YrHvbGP/Hmf13nZQBc/VZO7EL6nM75C,8.052588,16.023168,13.600106
-0,04rwScmEvRr0aU/mAE7aKtKFwowolGaTAPyQHuaVKEFmEVMAKxo+7UBCk3vRRRBd,2.221999,5.809178,3.021269
-0,1H9K/TW4c28Aob/H1O53cyQT7pHRww0L1ocyn19z1+MxC+k+5M/PgEx9B3zT/CNf,2.985884,7.584636,3.995057
-0,2fgXOaNZld/i7o20ULRNhCeL+o+vgZYzDOIhQ2n28TcGxXR047+F1b7QiD+l1Ypf,0.068074,0.884132,0.239792
-0,+0bAvqEMTl/RGyFmuz4zJH3DLMI6Q+iHapYn5BpbZI+0PNNfM7PXm/mojw+e8Xpn,3.238927,4.259525,3.611511
-0,3OdFPkhA5Q99wyfxmgyxPAhWyDLkV++XFtPL8pD3w5f8mBWbokeBwgk4gmNIxCOL,0.461767,10.466777,4.985617
-0,0UE8gxQAdCGY+WGN9yd9CL2ZGGqoyGQ2PzQGndwecce24GyTUnuvREbnMWBZZ7bG,0.730279,6.785359,3.363408
-0,/Uk/U5u4d+KNQVPD63pklfxeWc2zDAkUnrVmvxgRTuqNFbn90h8TuU5GZ+OamGQ5,0.105853,1.739301,0.262678
-0,2im96EJfLyxm7TPrtOR9m6Inq4E4/qR+AvP0TbnSdvzXI+N9gHh7C2fzppzcR0i8,0.325895,2.012216,0.802437
-0,+CrXBNhT3ch1hYU2e9IGs7wfjSLRkKYgidJYc42LlsH39cYtwdAX3wKm1OGlf+Kl,18.815771,40.850218,22.470045
-0,/hXRrrjPrAw8xDSsJnEwLdkRN1e42zJLE/HO5DXk5gbGLRmRx5H9n4T0UmraZ8uW,0.361838,0.831517,0.423214
-0,/sTadDDv8poFeLWS7lD/SEtEgWCBHXB1IaiitjCru4AcK8Z32hNXlccdY8hlFzTp,3.203254,5.682829,3.859569
-0,333YaK054AGlUYuw0XWxYn5K8NwzhfzJ3mm4YNwB1YXKjgnO64ZItBNaBRQoOgXn,0.124811,0.384592,0.257066
-0,+ZkQz7QrPZIODz45A+60ZFnG18jnyYlSY/IgEe1Yj8c4cU8h+L8WDIKMv2uB7EwD,1.022656,6.508863,3.368929
-0,+X4DW7zA6whRfOWSHHONJ1u3f0DyBvC9PqDmXGFfbxT4aUGCC6kVm6fuGu9IsQyL,3.428286,15.183059,5.743137
-0,2KXdN0Pb4iyu0jVPocTTf3dwk2Z1LjIlAcydV3HURGIUn1dTycCDDCHg5G6l6i9t,0.282044,0.40582,0.311669
-0,2lGxRtUbBrRZmIYagONMp6vj0zHk4EGhu0aSH5Ws/CAXwBNZpCavBFDNCEcPsOkt,3.662958,8.660027,5.281077
-0,+IR6CKA4zeO742dCx1l2hR0plhTanlaxPWAbckkZNo6UAti83TpYPRXrrfdmm9Ar,0.086237,2.450893,0.969819
-0,2/hWJ+i+1FSHiD44Rr3S4xWMUHC6hIgoVBX2XGZ7cOFyLn9FWQ3Kevsocw7CGaxJ,1.499537,2.832775,1.900258
-0,1WnALZnCvRlfqnuRyrIf0wxQOGLhGuvxInHelnMBM6cw9G9hydTBxqV60JSL/48p,0.717535,5.066802,1.448937
diff --git a/opendc-trace/opendc-trace-azure/src/test/resources/trace/vm_cpu_readings/vm_cpu_readings-file-1-of-125.csv.gz b/opendc-trace/opendc-trace-azure/src/test/resources/trace/vm_cpu_readings/vm_cpu_readings-file-1-of-125.csv.gz
new file mode 100644
index 00000000..592c7316
--- /dev/null
+++ b/opendc-trace/opendc-trace-azure/src/test/resources/trace/vm_cpu_readings/vm_cpu_readings-file-1-of-125.csv.gz
Binary files differ
diff --git a/opendc-trace/opendc-trace-azure/src/test/resources/trace/vmtable/vmtable.csv b/opendc-trace/opendc-trace-azure/src/test/resources/trace/vmtable/vmtable.csv
deleted file mode 100644
index 299c518c..00000000
--- a/opendc-trace/opendc-trace-azure/src/test/resources/trace/vmtable/vmtable.csv
+++ /dev/null
@@ -1,10 +0,0 @@
-x/XsOfHO4ocsV99i4NluqKDuxctW2MMVmwqOPAlg4wp8mqbBOe3wxBlQo0+Qx+uf,VDU4C8cqdr+ORcqquwMRcsBA2l0SC6lCPys0wdghKROuxPYysA2XYii9Y5ZkaYaq,Pc2VLB8aDxK2DCC96itq4vW/zVDp4wioAUiB3HoGSFYQ0o6/ZCegTpb9vEH4LeMTEWVObHTPRYEY81TYivZCMQ==,0,2591700,99.369869,3.4240942342446719,10.194309,Delay-insensitive,1,1.75
-H5CxmMoVcZSpjgGbohnVA3R+7uCTe/hM2ht2uIYi3t7KwXB4tkBxmZHBrt2A4x+n,BSXOcywx8pUU0DueDo6UMol1YzR6tn47KLEKaoXp0a1bf2PpzJ7n7lLlmhQ0OJf9,3J17LcV4gXjFat62qhVFRfoiWArHnY763HVqqI6orJCfV8h5j9yeotRMnCLlX1ooGkMyQ2MDOuY1oz111AGN9Q==,0,1539300,100,6.18178366757598,33.98136,Interactive,1,0.75
-wR/G1YUjpMP4zUbxGM/XJNhYS8cAK3SGKM2tqhF7VdeTUYHGktQiKQNoDTtYvnAc,VDU4C8cqdr+ORcqquwMRcsBA2l0SC6lCPys0wdghKROuxPYysA2XYii9Y5ZkaYaq,Pc2VLB8aDxK2DCC96itq4vW/zVDp4wioAUiB3HoGSFYQ0o6/ZCegTpb9vEH4LeMT+hzuAPZnYJMu61JNhTDF/Q==,2188800,2591700,99.569027,3.5736346071428589,7.92425,Delay-insensitive,1,1.75
-1XiU+KpvIa3T1XP8kk3ZY71Of03+ogFL5Pag9Mc2jBuh0YqeW0Zcb9lepKLdPEDg,8u+M3WcFp8pq183WoMB79PhK7xUzbaviOBv0qWN6Xn4mbuNVM1GYJlIjswgit+k1,DHbeI+pYTYFjH8JAF8SewM0z/4SqQctvxcBRGIRglBmeLW5VjISVEw7/IpY345kHwHtk7+SKlEwc1upnT3PigA==,0,2591700,99.405085,16.2876105408034,95.69789,Delay-insensitive,8,56
-z5i2HiSaz6ZdLR6PXdnDjGva3jIlkMPXx23VtfXx9q3dXFRBQrxCOj7sHUsrmFLa,VDU4C8cqdr+ORcqquwMRcsBA2l0SC6lCPys0wdghKROuxPYysA2XYii9Y5ZkaYaq,Pc2VLB8aDxK2DCC96itq4vW/zVDp4wioAUiB3HoGSFYQ0o6/ZCegTpb9vEH4LeMTEWVObHTPRYEY81TYivZCMQ==,0,2188500,98.967961,3.036037969572376,9.445484,Delay-insensitive,1,1.75
-n77nP00/UpJmT+Yx1ZkDphvAqPoHU8yUpDCwyUtPNlRENqvNp6Inya1eiy7VP1+x,8u+M3WcFp8pq183WoMB79PhK7xUzbaviOBv0qWN6Xn4mbuNVM1GYJlIjswgit+k1,DHbeI+pYTYFjH8JAF8SewM0z/4SqQctvxcBRGIRglBmeLW5VjISVEw7/IpY345kHwHtk7+SKlEwc1upnT3PigA==,0,2591700,99.448473,34.17401179027781,98.553018,Delay-insensitive,8,56
-aTSXW3N1KepxKYwKumd7T1+f7DkGolSKV8EArYAdctjD26YqSMKezCVSdvmSgqIQ,dBub/K+8I6jD9t2ExqUdRNlVxPPvDWqICA9Sr+yzcBZ/nNuC0W2swapPoBNIRoF+,C9GnRqFF2lzW/elUsLEwhyAQj9D/d5JIOOgvwfPL1aINf+m1f29G7nXhr6mRPGbiofmjfP9GkepcWz9LX5tp7Q==,2290500,2292300,94.113335,32.461745857142866,94.113335,Unkown,1,1.75
-uSkGH3DS6BVo3RFnw3GZb6WCFSmGgvgKi4HIj08yxO4f5ladUQc3pqDOtqRN0W9+,8u+M3WcFp8pq183WoMB79PhK7xUzbaviOBv0qWN6Xn4mbuNVM1GYJlIjswgit+k1,DHbeI+pYTYFjH8JAF8SewM0z/4SqQctvxcBRGIRglBmeLW5VjISVEw7/IpY345kHwHtk7+SKlEwc1upnT3PigA==,0,2591700,99.276369,1.3500837561060346,23.450372,Delay-insensitive,8,56
-ztRY/Sk5mrSFFcpy2usZ0YZZ7Eumq130/5BB8WVXfWaYvFkU+EhXUQ2kOFkCXuCw,dBub/K+8I6jD9t2ExqUdRNlVxPPvDWqICA9Sr+yzcBZ/nNuC0W2swapPoBNIRoF+,C9GnRqFF2lzW/elUsLEwhyAQj9D/d5JIOOgvwfPL1aINf+m1f29G7nXhr6mRPGbiofmjfP9GkepcWz9LX5tp7Q==,2281200,2300100,98.671595,43.724999781249991,98.13707,Unkown,1,1.75
-bJoIb8ras2ZNNSdAz3CAu4HYRd6k9MOqij/+6/+/5XaYw4+EoGdUEr74DCi974gJ,8u+M3WcFp8pq183WoMB79PhK7xUzbaviOBv0qWN6Xn4mbuNVM1GYJlIjswgit+k1,DHbeI+pYTYFjH8JAF8SewM0z/4SqQctvxcBRGIRglBmeLW5VjISVEw7/IpY345kHwHtk7+SKlEwc1upnT3PigA==,0,2591700,99.498748,18.989459534151351,94.751666,Interactive,8,56
diff --git a/opendc-trace/opendc-trace-azure/src/test/resources/trace/vmtable/vmtable.csv.gz b/opendc-trace/opendc-trace-azure/src/test/resources/trace/vmtable/vmtable.csv.gz
new file mode 100644
index 00000000..0adc6b7e
--- /dev/null
+++ b/opendc-trace/opendc-trace-azure/src/test/resources/trace/vmtable/vmtable.csv.gz
Binary files differ
diff --git a/opendc-trace/opendc-trace-opendc/src/main/kotlin/org/opendc/trace/opendc/OdcVmResourceTableReader.kt b/opendc-trace/opendc-trace-opendc/src/main/kotlin/org/opendc/trace/opendc/OdcVmResourceTableReader.kt
index d93929aa..ffbdc440 100644
--- a/opendc-trace/opendc-trace-opendc/src/main/kotlin/org/opendc/trace/opendc/OdcVmResourceTableReader.kt
+++ b/opendc-trace/opendc-trace-opendc/src/main/kotlin/org/opendc/trace/opendc/OdcVmResourceTableReader.kt
@@ -69,6 +69,7 @@ internal class OdcVmResourceTableReader(private val reader: LocalParquetReader<G
COL_START_TIME -> Instant.ofEpochMilli(record[AVRO_COL_START_TIME] as Long)
COL_STOP_TIME -> Instant.ofEpochMilli(record[AVRO_COL_STOP_TIME] as Long)
COL_CPU_COUNT -> getInt(index)
+ COL_CPU_CAPACITY -> getDouble(index)
COL_MEM_CAPACITY -> getDouble(index)
else -> throw IllegalArgumentException("Invalid column")
}
@@ -95,6 +96,7 @@ internal class OdcVmResourceTableReader(private val reader: LocalParquetReader<G
val record = checkNotNull(record) { "Reader in invalid state" }
return when (index) {
+ COL_CPU_CAPACITY -> if (AVRO_COL_CPU_CAPACITY >= 0) (record[AVRO_COL_CPU_CAPACITY] as Number).toDouble() else 0.0
COL_MEM_CAPACITY -> (record[AVRO_COL_MEM_CAPACITY] as Number).toDouble()
else -> throw IllegalArgumentException("Invalid column")
}
@@ -115,6 +117,7 @@ internal class OdcVmResourceTableReader(private val reader: LocalParquetReader<G
AVRO_COL_START_TIME = (schema.getField("start_time") ?: schema.getField("submissionTime")).pos()
AVRO_COL_STOP_TIME = (schema.getField("stop_time") ?: schema.getField("endTime")).pos()
AVRO_COL_CPU_COUNT = (schema.getField("cpu_count") ?: schema.getField("maxCores")).pos()
+ AVRO_COL_CPU_CAPACITY = schema.getField("cpu_capacity")?.pos() ?: -1
AVRO_COL_MEM_CAPACITY = (schema.getField("mem_capacity") ?: schema.getField("requiredMemory")).pos()
} catch (e: NullPointerException) {
// This happens when the field we are trying to access does not exist
@@ -126,19 +129,22 @@ internal class OdcVmResourceTableReader(private val reader: LocalParquetReader<G
private var AVRO_COL_START_TIME = -1
private var AVRO_COL_STOP_TIME = -1
private var AVRO_COL_CPU_COUNT = -1
+ private var AVRO_COL_CPU_CAPACITY = -1
private var AVRO_COL_MEM_CAPACITY = -1
private val COL_ID = 0
private val COL_START_TIME = 1
private val COL_STOP_TIME = 2
private val COL_CPU_COUNT = 3
- private val COL_MEM_CAPACITY = 4
+ private val COL_CPU_CAPACITY = 4
+ private val COL_MEM_CAPACITY = 5
private val columns = mapOf(
RESOURCE_ID to COL_ID,
RESOURCE_START_TIME to COL_START_TIME,
RESOURCE_STOP_TIME to COL_STOP_TIME,
RESOURCE_CPU_COUNT to COL_CPU_COUNT,
+ RESOURCE_CPU_CAPACITY to COL_CPU_CAPACITY,
RESOURCE_MEM_CAPACITY to COL_MEM_CAPACITY,
)
}
diff --git a/opendc-trace/opendc-trace-opendc/src/main/kotlin/org/opendc/trace/opendc/OdcVmResourceTableWriter.kt b/opendc-trace/opendc-trace-opendc/src/main/kotlin/org/opendc/trace/opendc/OdcVmResourceTableWriter.kt
index 9cc6ca7d..4b66a86f 100644
--- a/opendc-trace/opendc-trace-opendc/src/main/kotlin/org/opendc/trace/opendc/OdcVmResourceTableWriter.kt
+++ b/opendc-trace/opendc-trace-opendc/src/main/kotlin/org/opendc/trace/opendc/OdcVmResourceTableWriter.kt
@@ -64,6 +64,7 @@ internal class OdcVmResourceTableWriter(
RESOURCE_START_TIME -> (schema.getField("start_time") ?: schema.getField("submissionTime")).pos()
RESOURCE_STOP_TIME -> (schema.getField("stop_time") ?: schema.getField("endTime")).pos()
RESOURCE_CPU_COUNT -> (schema.getField("cpu_count") ?: schema.getField("maxCores")).pos()
+ RESOURCE_CPU_CAPACITY -> schema.getField("cpu_capacity").pos()
RESOURCE_MEM_CAPACITY -> (schema.getField("mem_capacity") ?: schema.getField("requiredMemory")).pos()
else -> -1
}
diff --git a/opendc-trace/opendc-trace-opendc/src/main/kotlin/org/opendc/trace/opendc/OdcVmTraceFormat.kt b/opendc-trace/opendc-trace-opendc/src/main/kotlin/org/opendc/trace/opendc/OdcVmTraceFormat.kt
index 9b32f8fd..886f3d54 100644
--- a/opendc-trace/opendc-trace-opendc/src/main/kotlin/org/opendc/trace/opendc/OdcVmTraceFormat.kt
+++ b/opendc-trace/opendc-trace-opendc/src/main/kotlin/org/opendc/trace/opendc/OdcVmTraceFormat.kt
@@ -48,7 +48,7 @@ public class OdcVmTraceFormat : TraceFormat {
override fun create(path: Path) {
// Construct directory containing the trace files
- Files.createDirectory(path)
+ Files.createDirectories(path)
val tables = getTables(path)
@@ -68,6 +68,7 @@ public class OdcVmTraceFormat : TraceFormat {
RESOURCE_START_TIME,
RESOURCE_STOP_TIME,
RESOURCE_CPU_COUNT,
+ RESOURCE_CPU_CAPACITY,
RESOURCE_MEM_CAPACITY,
)
)
@@ -138,6 +139,7 @@ public class OdcVmTraceFormat : TraceFormat {
.name("start_time").type(TIMESTAMP_SCHEMA).noDefault()
.name("stop_time").type(TIMESTAMP_SCHEMA).noDefault()
.requiredInt("cpu_count")
+ .requiredDouble("cpu_capacity")
.requiredLong("mem_capacity")
.endRecord()
diff --git a/opendc-trace/opendc-trace-tools/src/main/kotlin/org/opendc/trace/tools/TraceConverter.kt b/opendc-trace/opendc-trace-tools/src/main/kotlin/org/opendc/trace/tools/TraceConverter.kt
index 6fad43be..a9bc9480 100644
--- a/opendc-trace/opendc-trace-tools/src/main/kotlin/org/opendc/trace/tools/TraceConverter.kt
+++ b/opendc-trace/opendc-trace-tools/src/main/kotlin/org/opendc/trace/tools/TraceConverter.kt
@@ -27,6 +27,8 @@ import com.github.ajalt.clikt.core.CliktCommand
import com.github.ajalt.clikt.parameters.arguments.argument
import com.github.ajalt.clikt.parameters.groups.OptionGroup
import com.github.ajalt.clikt.parameters.groups.cooccurring
+import com.github.ajalt.clikt.parameters.groups.defaultByName
+import com.github.ajalt.clikt.parameters.groups.groupChoice
import com.github.ajalt.clikt.parameters.options.*
import com.github.ajalt.clikt.parameters.types.*
import mu.KotlinLogging
@@ -35,6 +37,7 @@ import java.io.File
import java.time.Duration
import java.time.Instant
import java.util.*
+import kotlin.collections.HashMap
import kotlin.math.abs
import kotlin.math.max
import kotlin.math.min
@@ -83,6 +86,14 @@ internal class TraceConverterCli : CliktCommand(name = "trace-converter") {
*/
private val samplingOptions by SamplingOptions().cooccurring()
+ /**
+ * The converter strategy to use.
+ */
+ private val converter by option("-c", "--converter", help = "converter strategy to use").groupChoice(
+ "default" to DefaultTraceConverter(),
+ "azure" to AzureTraceConverter(),
+ ).defaultByName("default")
+
override fun run() {
val metaParquet = File(output, "meta.parquet")
val traceParquet = File(output, "trace.parquet")
@@ -101,7 +112,7 @@ internal class TraceConverterCli : CliktCommand(name = "trace-converter") {
val metaWriter = outputTrace.getTable(TABLE_RESOURCES)!!.newWriter()
- val selectedVms = metaWriter.use { convertResources(inputTrace, it) }
+ val selectedVms = metaWriter.use { converter.convertResources(inputTrace, it, samplingOptions) }
if (selectedVms.isEmpty()) {
logger.warn { "No VMs selected" }
@@ -113,146 +124,325 @@ internal class TraceConverterCli : CliktCommand(name = "trace-converter") {
val writer = outputTrace.getTable(TABLE_RESOURCE_STATES)!!.newWriter()
- val statesCount = writer.use { convertResourceStates(inputTrace, it, selectedVms) }
+ val statesCount = writer.use { converter.convertResourceStates(inputTrace, it, selectedVms) }
logger.info { "Wrote $statesCount rows" }
}
/**
- * Convert the resources table for the trace.
+ * Options for sampling the workload trace.
*/
- private fun convertResources(trace: Trace, writer: TableWriter): Set<String> {
- val random = samplingOptions?.let { Random(it.seed) }
- val samplingFraction = samplingOptions?.fraction ?: 1.0
- val reader = checkNotNull(trace.getTable(TABLE_RESOURCE_STATES)).newReader()
-
- var hasNextRow = reader.nextRow()
- val selectedVms = mutableSetOf<String>()
-
- while (hasNextRow) {
- var id: String
- var numCpus = Int.MIN_VALUE
- var memCapacity = Double.MIN_VALUE
- var memUsage = Double.MIN_VALUE
- var startTime = Long.MAX_VALUE
- var stopTime = Long.MIN_VALUE
-
- do {
- id = reader.get(RESOURCE_ID)
-
- val timestamp = reader.get(RESOURCE_STATE_TIMESTAMP).toEpochMilli()
- startTime = min(startTime, timestamp)
- stopTime = max(stopTime, timestamp)
-
- numCpus = max(numCpus, reader.getInt(RESOURCE_CPU_COUNT))
-
- memCapacity = max(memCapacity, reader.getDouble(RESOURCE_MEM_CAPACITY))
- if (reader.hasColumn(RESOURCE_STATE_MEM_USAGE)) {
- memUsage = max(memUsage, reader.getDouble(RESOURCE_STATE_MEM_USAGE))
- }
-
- hasNextRow = reader.nextRow()
- } while (hasNextRow && id == reader.get(RESOURCE_ID))
+ private class SamplingOptions : OptionGroup() {
+ /**
+ * The fraction of VMs to sample
+ */
+ val fraction by option("--sampling-fraction", help = "fraction of the workload to sample")
+ .double()
+ .restrictTo(0.0001, 1.0)
+ .required()
- // Sample only a fraction of the VMs
- if (random != null && random.nextDouble() > samplingFraction) {
- continue
- }
+ /**
+ * The seed for sampling the trace.
+ */
+ val seed by option("--sampling-seed", help = "seed for sampling the workload")
+ .long()
+ .default(0)
+ }
- logger.info { "Selecting VM $id" }
- selectedVms.add(id)
+ /**
+ * A trace conversion strategy.
+ */
+ private sealed class TraceConverter(name: String) : OptionGroup(name) {
+ /**
+ * Convert the resources table for the trace.
+ *
+ * @param trace The trace to convert.
+ * @param writer The table writer for the target format.
+ * @param samplingOptions The sampling options to use.
+ * @return The map of resources that have been selected.
+ */
+ abstract fun convertResources(trace: Trace, writer: TableWriter, samplingOptions: SamplingOptions?): Map<String, Resource>
- writer.startRow()
- writer.set(RESOURCE_ID, id)
- writer.set(RESOURCE_START_TIME, Instant.ofEpochMilli(startTime))
- writer.set(RESOURCE_STOP_TIME, Instant.ofEpochMilli(stopTime))
- writer.setInt(RESOURCE_CPU_COUNT, numCpus)
- writer.setDouble(RESOURCE_MEM_CAPACITY, max(memCapacity, memUsage))
- writer.endRow()
- }
+ /**
+ * Convert the resource states table for the trace.
+ *
+ * @param trace The trace to convert.
+ * @param writer The table writer for the target format.
+ * @param selected The set of virtual machines that have been selected.
+ * @return The number of rows written.
+ */
+ abstract fun convertResourceStates(trace: Trace, writer: TableWriter, selected: Map<String, Resource>): Int
- return selectedVms
+ /**
+ * A resource in the resource table.
+ */
+ data class Resource(
+ val id: String,
+ val startTime: Instant,
+ val stopTime: Instant,
+ val cpuCount: Int,
+ val cpuCapacity: Double,
+ val memCapacity: Double
+ )
}
/**
- * Convert the resource states table for the trace.
+ * Default implementation of [TraceConverter].
*/
- private fun convertResourceStates(trace: Trace, writer: TableWriter, selectedVms: Set<String>): Int {
- val reader = checkNotNull(trace.getTable(TABLE_RESOURCE_STATES)).newReader()
+ private class DefaultTraceConverter : TraceConverter("default") {
+ /**
+ * The logger instance for the converter.
+ */
+ private val logger = KotlinLogging.logger {}
+
+ override fun convertResources(trace: Trace, writer: TableWriter, samplingOptions: SamplingOptions?): Map<String, Resource> {
+ val random = samplingOptions?.let { Random(it.seed) }
+ val samplingFraction = samplingOptions?.fraction ?: 1.0
+ val reader = checkNotNull(trace.getTable(TABLE_RESOURCE_STATES)).newReader()
+
+ var hasNextRow = reader.nextRow()
+ val selectedVms = mutableMapOf<String, Resource>()
+
+ val idCol = reader.resolve(RESOURCE_ID)
+ val timestampCol = reader.resolve(RESOURCE_STATE_TIMESTAMP)
+ val cpuCountCol = reader.resolve(RESOURCE_CPU_COUNT)
+ val cpuCapacityCol = reader.resolve(RESOURCE_CPU_CAPACITY)
+ val memCapacityCol = reader.resolve(RESOURCE_MEM_CAPACITY)
+ val memUsageCol = reader.resolve(RESOURCE_STATE_MEM_USAGE)
+
+ while (hasNextRow) {
+ var id: String
+ var cpuCount = 0
+ var cpuCapacity = 0.0
+ var memCapacity = 0.0
+ var memUsage = 0.0
+ var startTime = Long.MAX_VALUE
+ var stopTime = Long.MIN_VALUE
+
+ do {
+ id = reader.get(idCol) as String
+
+ val timestamp = (reader.get(timestampCol) as Instant).toEpochMilli()
+ startTime = min(startTime, timestamp)
+ stopTime = max(stopTime, timestamp)
+
+ cpuCount = max(cpuCount, reader.getInt(cpuCountCol))
+ cpuCapacity = max(cpuCapacity, reader.getDouble(cpuCapacityCol))
+ memCapacity = max(memCapacity, reader.getDouble(memCapacityCol))
+ if (memUsageCol > 0) {
+ memUsage = max(memUsage, reader.getDouble(memUsageCol))
+ }
+
+ hasNextRow = reader.nextRow()
+ } while (hasNextRow && id == reader.get(RESOURCE_ID))
+
+ // Sample only a fraction of the VMs
+ if (random != null && random.nextDouble() > samplingFraction) {
+ continue
+ }
- var hasNextRow = reader.nextRow()
- var count = 0
- var lastId: String? = null
- var lastTimestamp = 0L
+ logger.info { "Selecting VM $id" }
- while (hasNextRow) {
- val id = reader.get(RESOURCE_ID)
+ val startInstant = Instant.ofEpochMilli(startTime)
+ val stopInstant = Instant.ofEpochMilli(stopTime)
- if (id !in selectedVms) {
- hasNextRow = reader.nextRow()
- continue
- }
+ selectedVms.computeIfAbsent(id) {
+ Resource(it, startInstant, stopInstant, cpuCount, cpuCapacity, max(memCapacity, memUsage))
+ }
- val cpuCount = reader.getInt(RESOURCE_CPU_COUNT)
- val cpuUsage = reader.getDouble(RESOURCE_STATE_CPU_USAGE)
+ writer.startRow()
+ writer.set(RESOURCE_ID, id)
+ writer.set(RESOURCE_START_TIME, startInstant)
+ writer.set(RESOURCE_STOP_TIME, stopInstant)
+ writer.setInt(RESOURCE_CPU_COUNT, cpuCount)
+ writer.setDouble(RESOURCE_CPU_CAPACITY, cpuCapacity)
+ writer.setDouble(RESOURCE_MEM_CAPACITY, max(memCapacity, memUsage))
+ writer.endRow()
+ }
- val startTimestamp = reader.get(RESOURCE_STATE_TIMESTAMP).toEpochMilli()
- var timestamp = startTimestamp
- var duration: Long
+ return selectedVms
+ }
- // Check whether the previous entry is from a different VM
- if (id != lastId) {
- lastTimestamp = timestamp - 5 * 60 * 1000L
- }
+ override fun convertResourceStates(trace: Trace, writer: TableWriter, selected: Map<String, Resource>): Int {
+ val reader = checkNotNull(trace.getTable(TABLE_RESOURCE_STATES)).newReader()
+
+ val idCol = reader.resolve(RESOURCE_ID)
+ val timestampCol = reader.resolve(RESOURCE_STATE_TIMESTAMP)
+ val cpuCountCol = reader.resolve(RESOURCE_CPU_COUNT)
+ val cpuUsageCol = reader.resolve(RESOURCE_STATE_CPU_USAGE)
+
+ var hasNextRow = reader.nextRow()
+ var count = 0
+ var lastId: String? = null
+ var lastTimestamp = 0L
+
+ while (hasNextRow) {
+ val id = reader.get(idCol) as String
+ val resource = selected[id]
+ if (resource == null) {
+ hasNextRow = reader.nextRow()
+ continue
+ }
- do {
- timestamp = reader.get(RESOURCE_STATE_TIMESTAMP).toEpochMilli()
+ val cpuCount = reader.getInt(cpuCountCol)
+ val cpuUsage = reader.getDouble(cpuUsageCol)
- duration = timestamp - lastTimestamp
- hasNextRow = reader.nextRow()
+ val startTimestamp = (reader.get(timestampCol) as Instant).toEpochMilli()
+ var timestamp = startTimestamp
+ var duration: Long
- if (!hasNextRow) {
- break
+ // Check whether the previous entry is from a different VM
+ if (id != lastId) {
+ lastTimestamp = timestamp - 5 * 60 * 1000L
}
- val shouldContinue = id == reader.get(RESOURCE_ID) &&
- abs(cpuUsage - reader.getDouble(RESOURCE_STATE_CPU_USAGE)) < 0.01 &&
- cpuCount == reader.getInt(RESOURCE_CPU_COUNT)
- } while (shouldContinue)
+ do {
+ timestamp = (reader.get(timestampCol) as Instant).toEpochMilli()
- writer.startRow()
- writer.set(RESOURCE_ID, id)
- writer.set(RESOURCE_STATE_TIMESTAMP, Instant.ofEpochMilli(startTimestamp))
- writer.set(RESOURCE_STATE_DURATION, Duration.ofMillis(duration))
- writer.setInt(RESOURCE_CPU_COUNT, cpuCount)
- writer.setDouble(RESOURCE_STATE_CPU_USAGE, cpuUsage)
- writer.endRow()
+ duration = timestamp - lastTimestamp
+ hasNextRow = reader.nextRow()
- count++
+ if (!hasNextRow) {
+ break
+ }
- lastId = id
- lastTimestamp = timestamp
- }
+ val shouldContinue = id == reader.get(idCol) &&
+ abs(cpuUsage - reader.getDouble(cpuUsageCol)) < 0.01 &&
+ cpuCount == reader.getInt(cpuCountCol)
+ } while (shouldContinue)
+
+ writer.startRow()
+ writer.set(RESOURCE_ID, id)
+ writer.set(RESOURCE_STATE_TIMESTAMP, Instant.ofEpochMilli(startTimestamp))
+ writer.set(RESOURCE_STATE_DURATION, Duration.ofMillis(duration))
+ writer.setInt(RESOURCE_CPU_COUNT, cpuCount)
+ writer.setDouble(RESOURCE_STATE_CPU_USAGE, cpuUsage)
+ writer.endRow()
+
+ count++
+
+ lastId = id
+ lastTimestamp = timestamp
+ }
- return count
+ return count
+ }
}
/**
- * Options for sampling the workload trace.
+ * Implementation of [TraceConverter] for the Azure trace format.
*/
- private class SamplingOptions : OptionGroup() {
+ private class AzureTraceConverter : TraceConverter("default") {
/**
- * The fraction of VMs to sample
+ * The logger instance for the converter.
*/
- val fraction by option("--sampling-fraction", help = "fraction of the workload to sample")
- .double()
- .restrictTo(0.0001, 1.0)
- .required()
+ private val logger = KotlinLogging.logger {}
/**
- * The seed for sampling the trace.
+ * CPU capacity of the machines used by Azure.
*/
- val seed by option("--sampling-seed", help = "seed for sampling the workload")
- .long()
- .default(0)
+ private val CPU_CAPACITY = 2500.0
+
+ override fun convertResources(trace: Trace, writer: TableWriter, samplingOptions: SamplingOptions?): Map<String, Resource> {
+ val random = samplingOptions?.let { Random(it.seed) }
+ val samplingFraction = samplingOptions?.fraction ?: 1.0
+ val reader = checkNotNull(trace.getTable(TABLE_RESOURCES)).newReader()
+
+ val idCol = reader.resolve(RESOURCE_ID)
+ val startTimeCol = reader.resolve(RESOURCE_START_TIME)
+ val stopTimeCol = reader.resolve(RESOURCE_STOP_TIME)
+ val cpuCountCol = reader.resolve(RESOURCE_CPU_COUNT)
+ val memCapacityCol = reader.resolve(RESOURCE_MEM_CAPACITY)
+
+ val selectedVms = mutableMapOf<String, Resource>()
+
+ while (reader.nextRow()) {
+ // Sample only a fraction of the VMs
+ if (random != null && random.nextDouble() > samplingFraction) {
+ continue
+ }
+
+ val id = reader.get(idCol) as String
+ val startTime = (reader.get(startTimeCol) as Instant).toEpochMilli()
+ val stopTime = (reader.get(stopTimeCol) as Instant).toEpochMilli()
+ val cpuCount = reader.getInt(cpuCountCol)
+ val memCapacity = reader.getDouble(memCapacityCol)
+
+ logger.info { "Selecting VM $id" }
+
+ val startInstant = Instant.ofEpochMilli(startTime)
+ val stopInstant = Instant.ofEpochMilli(stopTime)
+ val cpuCapacity = cpuCount * CPU_CAPACITY
+
+ selectedVms.computeIfAbsent(id) {
+ Resource(it, startInstant, stopInstant, cpuCount, cpuCapacity, memCapacity)
+ }
+
+ writer.startRow()
+ writer.set(RESOURCE_ID, id)
+ writer.set(RESOURCE_START_TIME, Instant.ofEpochMilli(startTime))
+ writer.set(RESOURCE_STOP_TIME, Instant.ofEpochMilli(stopTime))
+ writer.setInt(RESOURCE_CPU_COUNT, cpuCount)
+ writer.setDouble(RESOURCE_CPU_CAPACITY, cpuCount * CPU_CAPACITY)
+ writer.setDouble(RESOURCE_MEM_CAPACITY, memCapacity)
+ writer.endRow()
+ }
+
+ return selectedVms
+ }
+
+ override fun convertResourceStates(trace: Trace, writer: TableWriter, selected: Map<String, Resource>): Int {
+ val reader = checkNotNull(trace.getTable(TABLE_RESOURCE_STATES)).newReader()
+ val states = HashMap<String, State>()
+
+ val idCol = reader.resolve(RESOURCE_ID)
+ val timestampCol = reader.resolve(RESOURCE_STATE_TIMESTAMP)
+ val cpuUsageCol = reader.resolve(RESOURCE_STATE_CPU_USAGE_PCT)
+
+ var count = 0
+
+ while (reader.nextRow()) {
+ val id = reader.get(idCol) as String
+ val resource = selected[id] ?: continue
+
+ val cpuUsage = reader.getDouble(cpuUsageCol) * CPU_CAPACITY // MHz
+ val state = states.computeIfAbsent(id) { State(resource, cpuUsage) }
+ val timestamp = (reader.get(timestampCol) as Instant).toEpochMilli()
+ val duration = (timestamp - state.startTime)
+
+ state.duration = duration
+
+ if (abs(cpuUsage - state.cpuUsage) > 0.01) {
+ state.write(writer)
+
+ state.startTime = timestamp
+ state.duration = 0
+ state.cpuUsage = cpuUsage
+ }
+
+ count++
+ }
+
+ for ((_, state) in states) {
+ state.duration += state.startTime - state.resource.stopTime.toEpochMilli()
+ state.write(writer)
+ }
+
+ return count
+ }
+
+ private class State(@JvmField val resource: Resource, @JvmField var cpuUsage: Double) {
+ @JvmField var startTime: Long = resource.startTime.toEpochMilli()
+ @JvmField var duration: Long = 30000L
+
+ fun write(writer: TableWriter) {
+ writer.startRow()
+ writer.set(RESOURCE_ID, resource.id)
+ writer.set(RESOURCE_STATE_TIMESTAMP, Instant.ofEpochMilli(startTime))
+ writer.set(RESOURCE_STATE_DURATION, Duration.ofMillis(duration))
+ writer.setDouble(RESOURCE_STATE_CPU_USAGE, cpuUsage)
+ writer.setInt(RESOURCE_CPU_COUNT, resource.cpuCount)
+ writer.endRow()
+ }
+ }
}
}
diff --git a/opendc-trace/opendc-trace-tools/src/main/resources/log4j2.xml b/opendc-trace/opendc-trace-tools/src/main/resources/log4j2.xml
new file mode 100644
index 00000000..32d81416
--- /dev/null
+++ b/opendc-trace/opendc-trace-tools/src/main/resources/log4j2.xml
@@ -0,0 +1,38 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Copyright (c) 2021 AtLarge Research
+ ~
+ ~ Permission is hereby granted, free of charge, to any person obtaining a copy
+ ~ of this software and associated documentation files (the "Software"), to deal
+ ~ in the Software without restriction, including without limitation the rights
+ ~ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ ~ copies of the Software, and to permit persons to whom the Software is
+ ~ furnished to do so, subject to the following conditions:
+ ~
+ ~ The above copyright notice and this permission notice shall be included in all
+ ~ copies or substantial portions of the Software.
+ ~
+ ~ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ ~ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ ~ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ ~ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ ~ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ ~ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ ~ SOFTWARE.
+ -->
+
+<Configuration status="WARN">
+ <Appenders>
+ <Console name="Console" target="SYSTEM_OUT">
+ <PatternLayout pattern="%d{HH:mm:ss.SSS} [%highlight{%-5level}] %logger{36} - %msg%n" disableAnsi="false"/>
+ </Console>
+ </Appenders>
+ <Loggers>
+ <Logger name="org.opendc" level="info" additivity="false">
+ <AppenderRef ref="Console"/>
+ </Logger>
+ <Root level="error">
+ <AppenderRef ref="Console"/>
+ </Root>
+ </Loggers>
+</Configuration>