diff --git a/.gitignore b/.gitignore index 3edd8e5..0d0da1d 100644 --- a/.gitignore +++ b/.gitignore @@ -110,5 +110,83 @@ performanceData/ logs/ setups/ benchmarkSetting/ +simulation/*.csv + +# User-specific stuff +.idea/**/workspace.xml +.idea/**/tasks.xml +.idea/**/usage.statistics.xml +.idea/**/dictionaries +.idea/**/shelf + +# Generated files +.idea/**/contentModel.xml + +# Sensitive or high-churn files +.idea/**/dataSources/ +.idea/**/dataSources.ids +.idea/**/dataSources.local.xml +.idea/**/sqlDataSources.xml +.idea/**/dynamic.xml +.idea/**/uiDesigner.xml +.idea/**/dbnavigator.xml + +# Gradle +.idea/**/gradle.xml +.idea/**/libraries + +# Gradle and Maven with auto-import +# When using Gradle or Maven with auto-import, you should exclude module files, +# since they will be recreated, and may cause churn. Uncomment if using +# auto-import. +# .idea/modules.xml +# .idea/*.iml +# .idea/modules + +# CMake +cmake-build-*/ + +# Mongo Explorer plugin +.idea/**/mongoSettings.xml + +# File-based project format +*.iws + +# IntelliJ +out/ + +# mpeltonen/sbt-idea plugin +.idea_modules/ + +# JIRA plugin +atlassian-ide-plugin.xml + +# Cursive Clojure plugin +.idea/replstate.xml + +# Crashlytics plugin (for Android Studio and IntelliJ) +com_crashlytics_export_strings.xml +crashlytics.properties +crashlytics-build.properties +fabric.properties + +# Editor-based Rest Client +.idea/httpRequests + +# Android studio 3.1+ serialized cache file +.idea/caches/build_file_checksums.ser + +### Intellij Patch ### +# Comment Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-215987721 + +# *.iml +# modules.xml +.idea/*.xml +# *.ipr + +# Sonarlint plugin +.idea/sonarlint + +# End of https://www.gitignore.io/api/intellij # End of https://www.gitignore.io/api/git,java,gradle,eclipse \ No newline at end of file diff --git a/src/main/java/de/uniba/dsg/serverless/cli/UtilityFactory.java b/src/main/java/de/uniba/dsg/serverless/cli/UtilityFactory.java index 62f83b3..9de3d10 100644 --- a/src/main/java/de/uniba/dsg/serverless/cli/UtilityFactory.java +++ b/src/main/java/de/uniba/dsg/serverless/cli/UtilityFactory.java @@ -8,6 +8,7 @@ import de.uniba.dsg.serverless.cli.performance.AzurePerformanceDataUtility; import de.uniba.dsg.serverless.cli.performance.GooglePerformanceDataUtility; import de.uniba.dsg.serverless.cli.performance.IBMOpenWhiskPerformanceDataUtility; +import de.uniba.dsg.serverless.simulation.load.SimulationUtility; public class UtilityFactory { @@ -25,7 +26,10 @@ public class UtilityFactory { new DeploymentSizeUtility("deploymentSize"), // automated test generation - new SeMoDeUtility("awsSeMoDe") + new SeMoDeUtility("awsSeMoDe"), + + // simulation + new SimulationUtility("loadSimulation") ); public static Optional getUtilityClass(String name) { diff --git a/src/main/java/de/uniba/dsg/serverless/simulation/load/LoadPatternInterpreter.java b/src/main/java/de/uniba/dsg/serverless/simulation/load/LoadPatternInterpreter.java new file mode 100644 index 0000000..14ef539 --- /dev/null +++ b/src/main/java/de/uniba/dsg/serverless/simulation/load/LoadPatternInterpreter.java @@ -0,0 +1,70 @@ +package de.uniba.dsg.serverless.simulation.load; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import de.uniba.dsg.serverless.model.SeMoDeException; + +public class LoadPatternInterpreter { + + private final Path file; + + public LoadPatternInterpreter(Path file) { + super(); + this.file = file; + } + + // TODO configure the interval for the load interpretation phase, currently + // based on seconds + public Map interpretLoadPattern() throws SeMoDeException { + + List lines = this.readAllLines(file); + Map numberOfRequestPerSecond = this.generateDistribution(lines); + + return numberOfRequestPerSecond; + } + + public List getDoubleValues() throws SeMoDeException { + List lines = this.readAllLines(file); + List values = new ArrayList<>(); + + for (String line : lines) { + try { + values.add(Double.parseDouble(line)); + } catch (NumberFormatException e) { + throw new SeMoDeException("Load pattern file was corrupted. Line: " + line, e); + } + } + + return values; + } + + private Map generateDistribution(List lines) throws SeMoDeException { + Map distributionMap = new HashMap<>(); + for (Double doubleValue : this.getDoubleValues()) { + + // converts the double to the second before the comma + int intValue = (int) doubleValue.doubleValue(); + if (!distributionMap.containsKey(intValue)) { + distributionMap.put(intValue, 0); + } + distributionMap.put(intValue, distributionMap.get(intValue) + 1); + } + + return distributionMap; + } + + private List readAllLines(Path file) throws SeMoDeException { + try { + List lines = Files.readAllLines(file); + return lines; + } catch (IOException e) { + throw new SeMoDeException(e.getMessage(), e); + } + } +} diff --git a/src/main/java/de/uniba/dsg/serverless/simulation/load/LoadPatternSimulator.java b/src/main/java/de/uniba/dsg/serverless/simulation/load/LoadPatternSimulator.java new file mode 100644 index 0000000..11cf3b5 --- /dev/null +++ b/src/main/java/de/uniba/dsg/serverless/simulation/load/LoadPatternSimulator.java @@ -0,0 +1,174 @@ +package de.uniba.dsg.serverless.simulation.load; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import de.uniba.dsg.serverless.simulation.load.model.ContainerInstance; +import de.uniba.dsg.serverless.simulation.load.model.SimulationInput; + +public class LoadPatternSimulator { + + private static final Logger logger = LogManager.getLogger(LoadPatternSimulator.class); + + private final List inputValues; + + public LoadPatternSimulator(List inputValues) { + this.inputValues = inputValues; + } + + public Map simulate(SimulationInput simulationValues) { + + SimulationStep simulation = new SimulationStep(simulationValues); + + for (Double timestamp : inputValues) { + + // checks all running container, if they are finished, moving them to idle containers + simulation.checkFinishedContainers(timestamp); + + // checks all idle container, if they are longer idle than the assumed shutdown period + simulation.shutdownIdleContainer(timestamp); + + // check, if a container is idle + ContainerInstance container; + if (simulation.idleContainerAvailable()) { + // pick the first idle container and serve the request + container = simulation.getArbitraryIdleContainer(); + container.executeRequest(timestamp, simulationValues); + } else { + // create a new container, add them to the simulation + container = simulation.addNewContainer(timestamp); + container.executeRequestWithColdStart(timestamp, simulationValues); + } + simulation.addToExecuting(container); + + logger.info(container); + } + + // shutdown all containers (idle and running) + simulation.shutdownAllContainer(); + + // compute the distribution map + return simulation.getContainerDistribution(); + } +} + +class SimulationStep { + + private final List containerList; + private final List executingContainers; + private final List idleContainers; + private final SimulationInput simulationValues; + + public SimulationStep(SimulationInput simulationValues) { + this.containerList = new ArrayList<>(); + this.executingContainers = new ArrayList<>(); + this.idleContainers = new ArrayList<>(); + this.simulationValues = simulationValues; + } + + public boolean idleContainerAvailable() { + return !idleContainers.isEmpty(); + } + + public ContainerInstance getArbitraryIdleContainer() { + ContainerInstance temp = idleContainers.get(0); + this.idleContainers.remove(temp); + return temp; + } + + public ContainerInstance addNewContainer(double ongoingTime) { + ContainerInstance temp = new ContainerInstance(ongoingTime); + this.containerList.add(temp); + return temp; + } + + public void addToExecuting(ContainerInstance container) { + this.executingContainers.add(container); + } + + /** + * Checks if some executing containers are finished and move them to the idle + * list. + */ + public void checkFinishedContainers(double ongoingTime) { + + List finishedContainers = this.executingContainers.stream() + .filter((c) -> c.getBussyUntil() < ongoingTime).collect(Collectors.toList()); + + for (ContainerInstance container : finishedContainers) { + + this.executingContainers.remove(container); + this.idleContainers.add(container); + + } + } + + /** + * @param timestamp + */ + public void shutdownIdleContainer(double timestamp) { + + double shutdownTime = this.simulationValues.getShutdownAfter(); + + List containerForShutdown = this.idleContainers.stream() + .filter((c) -> c.getBussyUntil() < (timestamp - shutdownTime)).collect(Collectors.toList()); + + for (ContainerInstance container : containerForShutdown) { + container.setShutdownTime(container.getBussyUntil() + shutdownTime); + this.idleContainers.remove(container); + } + } + + public void shutdownAllContainer() { + + double shutdownTime = this.simulationValues.getShutdownAfter(); + + for (ContainerInstance container : this.executingContainers) { + container.setShutdownTime(container.getBussyUntil() + shutdownTime); + } + + for (ContainerInstance container : this.idleContainers) { + container.setShutdownTime(container.getBussyUntil() + shutdownTime); + } + + } + + public Map getContainerDistribution() { + + Map containerDistribution = new HashMap<>(); + + // max value for bussy until to get the size of the array (performance reasons) + double maxBussyUntil = 0.0; + for (ContainerInstance container : this.containerList) { + if (maxBussyUntil < container.getBussyUntil()) { + maxBussyUntil = container.getBussyUntil(); + } + } + + // 0 to 5.2 (maxBussyUntil) means 6 values in the array + int size = (int) maxBussyUntil + 1; + int[] containerDistributionInt = new int[size]; + + for (int i = 0; i < size; i++) { + containerDistributionInt[i] = 0; + } + + for (ContainerInstance container : this.containerList) { + for (int i = (int) container.getStartTime(); i <= (int) container.getBussyUntil(); i++) { + containerDistributionInt[i]++; + } + } + + for (int i = 0; i < size; i++) { + containerDistribution.put(i, containerDistributionInt[i]); + } + + return containerDistribution; + } +} diff --git a/src/main/java/de/uniba/dsg/serverless/simulation/load/SimulationUtility.java b/src/main/java/de/uniba/dsg/serverless/simulation/load/SimulationUtility.java new file mode 100644 index 0000000..a43da0a --- /dev/null +++ b/src/main/java/de/uniba/dsg/serverless/simulation/load/SimulationUtility.java @@ -0,0 +1,146 @@ +package de.uniba.dsg.serverless.simulation.load; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; + +import de.uniba.dsg.serverless.cli.CustomUtility; +import de.uniba.dsg.serverless.model.SeMoDeException; +import de.uniba.dsg.serverless.simulation.load.model.SimulationInput; + +public class SimulationUtility extends CustomUtility { + + private static final Logger logger = LogManager.getLogger(SimulationUtility.class); + + private Path file; + + public SimulationUtility(String name) { + super(name); + } + + @Override + public void start(List args) { + + if (args == null || args.size() != 2) { + logger.fatal("The provided element size is not correct! " + args); + return; + } + + try { + this.file = Paths.get(args.get(0)); + List simulationInput = new ObjectMapper().readValue(args.get(1), + new TypeReference>() { + }); + this.simulate(simulationInput); + + } catch (IOException e) { + e.printStackTrace(); + logger.fatal("Error reading simulation input json! " + args.get(1)); + return; + } catch (SeMoDeException e) { + logger.fatal(e.getMessage()); + return; + } + } + + public void simulate(List simInputs) throws SeMoDeException { + + // necessary for computing the output + List values = new ArrayList<>(); + + // interpret the load pattern and supply a distribution on a second basis + LoadPatternInterpreter interpreter = new LoadPatternInterpreter(file); + Map loadDistributionPerSecond = interpreter.interpretLoadPattern(); + + // adding the initial request distribution + values.add(new SimulationInputAndDistributionMap("Initial distribution", loadDistributionPerSecond)); + + for (SimulationInput simInput : simInputs) { + + LoadPatternSimulator sim = new LoadPatternSimulator(interpreter.getDoubleValues()); + Map containerDistribution = sim.simulate(simInput); + + values.add(new SimulationInputAndDistributionMap(simInput.toString(), containerDistribution)); + } + + this.createSimulationDirectory(); + this.writeDistributionToFile("distribution.csv", values); + + } + + private void createSimulationDirectory() throws SeMoDeException { + if (!Files.exists(Paths.get("simulation"))) { + try { + Files.createDirectory(Paths.get("simulation")); + } catch (IOException e) { + throw new SeMoDeException("Could not create simulation directory", e); + } + } + + } + + private void writeDistributionToFile(String fileName, List values) throws SeMoDeException { + List lines = new ArrayList<>(); + + // find the maximum time in all the simulation steps + Optional max = values.stream() + .map(s -> s.values.keySet().stream().max(Integer::compareTo)) + .filter(op -> op.isPresent()) + .map(op -> op.get()) + .max(Integer::compareTo); + + if(max.isPresent() == false) { + throw new SeMoDeException("There is no value present in none of the provided maps"); + } + + int maxTime = max.get(); + String headline = "Timestamp;"; + for (SimulationInputAndDistributionMap simMap : values ) { + headline += simMap.name + ";"; + } + lines.add(headline); + + for ( int i = 0 ; i < maxTime ; i ++ ) { + String line = "" + i + ";"; + + for (SimulationInputAndDistributionMap simMap : values ) { + if ( simMap.values.containsKey(i)) { + line += simMap.values.get(i) + ";"; + } else { + line += "0;"; + } + } + + lines.add(line); + } + + try { + Files.write(Paths.get("simulation/" + fileName), lines); + } catch (IOException e) { + throw new SeMoDeException("Write distribution to file failed", e); + } + } + + private class SimulationInputAndDistributionMap { + + private final String name; + private final Map values; + + public SimulationInputAndDistributionMap(String name, Map values) { + super(); + this.name = name; + this.values = values; + } + } +} diff --git a/src/main/java/de/uniba/dsg/serverless/simulation/load/model/ContainerInstance.java b/src/main/java/de/uniba/dsg/serverless/simulation/load/model/ContainerInstance.java new file mode 100644 index 0000000..3c62a4d --- /dev/null +++ b/src/main/java/de/uniba/dsg/serverless/simulation/load/model/ContainerInstance.java @@ -0,0 +1,58 @@ +package de.uniba.dsg.serverless.simulation.load.model; + +public class ContainerInstance { + + private static int internalIdCounter = 0; + + private final double startTime; + private double shutdownTime; + private final String id; + + private int servedRequests; + private double bussyUntil; + + public ContainerInstance(double startTime) { + super(); + this.id = "C-" + internalIdCounter++; + this.startTime = startTime; + this.servedRequests = 0; + this.bussyUntil = 0.0; + } + + public void executeRequest(double ongoingTime, SimulationInput simulationValues) { + this.servedRequests++; + this.bussyUntil = ongoingTime + simulationValues.getAverageExecutionTime(); + } + + public void executeRequestWithColdStart(double ongoingTime, SimulationInput simulationValues) { + this.servedRequests++; + this.bussyUntil = ongoingTime + simulationValues.getAverageExecutionTime() + + simulationValues.getAverageColdStartTime(); + } + + public double getStartTime() { + return startTime; + } + + public int getServedRequests() { + return servedRequests; + } + + public double getBussyUntil() { + return bussyUntil; + } + + public double getShutdownTime() { + return shutdownTime; + } + + public void setShutdownTime(double shutdownTime) { + this.shutdownTime = shutdownTime; + } + + @Override + public String toString() { + return "ContainerInstance [startTime=" + startTime + ", shutdownTime=" + shutdownTime + ", id=" + id + + ", servedRequests=" + servedRequests + ", bussyUntil=" + bussyUntil + "]"; + } +} diff --git a/src/main/java/de/uniba/dsg/serverless/simulation/load/model/SimulationInput.java b/src/main/java/de/uniba/dsg/serverless/simulation/load/model/SimulationInput.java new file mode 100644 index 0000000..a25931d --- /dev/null +++ b/src/main/java/de/uniba/dsg/serverless/simulation/load/model/SimulationInput.java @@ -0,0 +1,39 @@ +package de.uniba.dsg.serverless.simulation.load.model; + +public class SimulationInput { + + private double averageExecutionTime; + private double averageColdStartTime; + private double shutdownAfter; + + public SimulationInput() { + + } + + public SimulationInput(double averageExecutionTime, double averageColdStartTime, double shutdownAfter) { + super(); + this.averageExecutionTime = averageExecutionTime; + this.averageColdStartTime = averageColdStartTime; + this.shutdownAfter = shutdownAfter; + } + + public double getAverageExecutionTime() { + return averageExecutionTime; + } + + public double getAverageColdStartTime() { + return averageColdStartTime; + } + + public double getShutdownAfter() { + return shutdownAfter; + } + + @Override + public String toString() { + return "SimulationInput [" + averageExecutionTime + "," + + averageColdStartTime + "," + shutdownAfter + "]"; + } + + +} diff --git a/src/main/resources/simulation/load/timestamps.csv b/src/main/resources/simulation/load/timestamps.csv new file mode 100644 index 0000000..1c49fe4 --- /dev/null +++ b/src/main/resources/simulation/load/timestamps.csv @@ -0,0 +1 @@ +0.0 0.4 0.8 1.1 1.4 1.7 2.0 2.3 2.6 2.9 3.0 3.2 3.3 3.5 3.6 3.8 3.9 4.1 4.2 4.4 4.5 4.7 4.8 5.0 5.1 5.6 5.9 6.2 6.5 6.8 7.2 7.5 7.8 8.1 8.5 8.8 9.1 9.4 9.8 10.1 10.3 10.4 10.6 10.7 10.9 11.0 11.2 11.4 11.5 11.7 11.8 12.0 12.1 12.3 12.5 12.6 12.8 13.1 13.3 13.5 13.8 14.0 14.3 14.5 14.7 15.0 15.2 15.4 15.7 \ No newline at end of file