Standardized benchmarks

usethesource · Jan 29, 2025 · dfe9b41 · dfe9b41
1 parent 3e4b675
commit dfe9b41
Show file tree

Hide file tree

Showing 6 changed files with 89 additions and 35 deletions.
diff --git a/src/org/rascalmpl/library/lang/rascal/grammar/tests/CGrammar.rsc b/src/org/rascalmpl/library/lang/rascal/grammar/tests/CGrammar.rsc
@@ -69,7 +69,13 @@ void generateAndWriteCParser(){
 	writeFile(CParserLoc, generateCParser());
 }
 
+void warmup(){
+	for(_ <- [0..10]) 
+		generateCParser();
+}
+
 int generateAndTimeCParser() { 
+	warmup();
 	println("GenerateAndTimeCParser");
 	t = cpuTime(); 
 	generateCParser();

diff --git a/src/org/rascalmpl/library/lang/rascal/grammar/tests/PicoGrammar.rsc b/src/org/rascalmpl/library/lang/rascal/grammar/tests/PicoGrammar.rsc
@@ -40,7 +40,13 @@ void generateAndWritePicoParser(){
 	writeFile(PicoParserLoc, generatePicoParser());
 }
 
+void warmup(){
+	for(_ <- [0..10]){
+		generatePicoParser();
+	}
+}
 int generateAndTimePicoParser() { 
+	warmup();
 	t = cpuTime(); 
 	generatePicoParser();
 	return (cpuTime() - t)/1000000;

diff --git a/src/org/rascalmpl/library/lang/rascal/grammar/tests/RascalGrammar.rsc b/src/org/rascalmpl/library/lang/rascal/grammar/tests/RascalGrammar.rsc
@@ -158,7 +158,14 @@ void generateAndWriteRascalParser(){
 	writeFile(RascalParserLoc, generateRascalParser());
 }
 
+void warmup(){
+	for(int _ <- [1 .. 10]){
+		generateRascalParser();
+	}
+}
+
 int generateAndTimeRascalParser() { 
+	warmup();
     println("GenerateAndTimeRascalParser");
 	t = cpuTime();
 	for(int _ <- [1 .. 50]){

diff --git a/test/org/rascalmpl/benchmark/CompareShortestPath.rsc b/test/org/rascalmpl/benchmark/CompareShortestPath.rsc
@@ -116,18 +116,20 @@ public void measure1(rel[int,int] Graph1){
 
     jtime = 0.0; jmin = 10000.0; jmax = 0.0;
     rtime = 0.0; rmin = 10000.0; rmax = 0.0;
-    for(int _ <- [1 .. 20]){
+    for(int _ <- [1 .. 100]){
  		time1 = getMilliTime(); P1 = shortestPathPair(G, 1, 0); time2 = getMilliTime();
-                                P2 = shortestPathPair1(G, 1, 0); time3 = getMilliTime();
+                              P2 = shortestPathPair1(G, 1, 0); time3 = getMilliTime();
 
  		d1 = time2 - time1; jtime = jtime + d1; jmin = min(d1, jmin); jmax = max(d1, jmax);
  		d2 = time3 - time2; rtime = rtime + d2; rmin = min(d2, rmin); rmax = max(d2, rmax);
  		println("Java version:   <P1> in <d1> millis");
- 		println("Rascal version: <P1> in <d2> millis");
+ 		println("Rascal version: <P2> in <d2> millis");
  	}
  	println("Java average: <jtime/20> [<jmin> .. <jmax>]");
  	println("Rascal average: <rtime/20> [<rmin> .. <rmax>]");
- 	
+ 	println("Rascal/java ratio: <rtime/jtime>");
+   println("Total Java time: <jtime>");
+   println("Total Rascal time: <rtime>");   
 }
 
 public void measure2(rel[int,int] Graph2)
@@ -141,3 +143,5 @@ public void measure(){
 	Graph2 = randomGraph(10000, [0 .. 50]);
 	println("Graph2 -------"); measure1(Graph2);
 }
+
+void main() = measure();
diff --git a/test/org/rascalmpl/benchmark/RSF/RSFCalls.rsc b/test/org/rascalmpl/benchmark/RSF/RSFCalls.rsc
@@ -11,33 +11,44 @@
 module RSF::RSFCalls
 
 import Relation;
-import util::Math;
 import Set;
 import analysis::graphs::Graph;
 import lang::rsf::IO;
 import IO;
 import util::Benchmark;
 
-value main() = measure();
+void warmup(){
+  for(_ <- [0..5]){
+	measureOne();
+  }
+}
+void main() {
+	warmup();
+	used = 0;
+	for(_ <- [0..5]){
+		used += measure();
+	}
+	println("Total time (ms): <used>");
+}
 
-public bool measure(){
+public int measure(){
   return measure(["JHotDraw52.rsf", "JDK140AWT.rsf", "JWAM16FullAndreas.rsf", "jdk14v2.rsf", "Eclipse202a.rsf"]);
 }
 
-public bool measureOne(){
+public int measureOne(){
    return measure(["JHotDraw52.rsf"]);
 }
 
-public bool measure(list[str] names){
-	loc p = {loc x} := findResources("RSF") ? x : |not-found:///|;
-
+public int measure(list[str] names){
+	p = |file:///Users/paulklint/git/rascal//test/org/rascalmpl/benchmark/RSF/|;
+	begin = cpuTime();
 	for(str name <- names){
 		map[str, rel[str,str]] values = readRSF(p + name);
 		rel[str,str] CALL = values["CALL"];
 		n = size(CALL);
-		println("<name>: CALL contains <n> tuples");
+		//println("<name>: CALL contains <n> tuples");
 		nTop = size(top(CALL));
-		println("<name>: size top <nTop>");
+		//println("<name>: size top <nTop>");
 
 		time0 = realTime();
 		res1 = trans(CALL);          time1 = realTime();
@@ -48,16 +59,17 @@ public bool measure(list[str] names){
 		d2 = time2 - time1;
 		d3 = time3 - time2;
 
-		println("Time (msec): trans <d1>, reachFromTop1 <d2>, reachFromTop2 <d3>");
+		//println("Time (msec): trans <d1>, reachFromTop1 <d2>, reachFromTop2 <d3>");
 
 		size1 = size(res1); size2= size(res2); size3 = size(res3);
-		println("Size (elms): trans <size1>, reachFromTop1 <size2>, reachFromTop2 <size3>");
+		//println("Size (elms): trans <size1>, reachFromTop1 <size2>, reachFromTop2 <size3>");
 		if(res2 != res3){
 			println("***\> res2 != res3");
 		}
 
-    }
-    return true;
+	}
+	println("Total time (ms): <(cpuTime() - begin)/1000000>");
+	return (cpuTime() - begin)/1000000;
 }
 
 public rel[str,str] trans(rel[str,str] CALL){

diff --git a/test/org/rascalmpl/benchmark/Visiting/visiting.rsc b/test/org/rascalmpl/benchmark/Visiting/visiting.rsc
@@ -51,29 +51,48 @@ public ColoredTree swapAll(ColoredTree t){
    };
 }
 
-public void m(){
+int work(){
+   T = genTree(15, 10, 14);
+   //println("countAll for tree T = <countAll(T)>");
+
    N = 10;
    T = genTree(15, 10, 14);
-   println("countAll for tree T = <countAll(T)>");
-   begin1 = realTime();
-   for(int j <- [1 .. N]){
-       countAll(T);
+   //println("countAll for tree T = <countAll(T)>");
+   begin1 = cpuTime();
+   for(int _ <- [1 .. N]){
+      countAll(T);
    }
-   used1 = (realTime() - begin1);
-   println("countAll: <used1> millisec.");
+   used1 = (cpuTime() - begin1);
+   //println("countAll: <used1> millisec.");
 
-   begin2 = realTime();
-   for(int j <- [1 .. N]){
-       allBlack(T);
+   begin2 = cpuTime();
+   for(int _ <- [1 .. N]){
+      allBlack(T);
    }
-   used2 = (realTime() - begin2);
-   println("allBlack: <used2> millisec.");
-
+   used2 = (cpuTime() - begin2);
+   //println("allBlack: <used2> millisec.");
 
-   begin3 = realTime();
-   for(int j <- [1 .. N]){
-       swapAll(T);
+   begin3 = cpuTime();
+   for(int _ <- [1 .. N]){
+      swapAll(T);
+   }
+   used3 = (cpuTime() - begin3);
+   //println("swapAll: <used3> millisec.");
+   return used1 + used2 + used3;
+}
+
+void warmup(){
+   for(_ <- [0..10]){
+      work();
    }
-   used3 = (realTime() - begin3);
-   println("swapAll: <used3> millisec.");
+}
+
+void main(){
+   warmup();
+   used = 0;
+   for(_ <- [0..100]){
+      used += work();
+   }
+
+   println("Total time: <used/1000000> millisec.");
 }