Fix travis

kanterov · Feb 23, 2016 · 631c2ca · 631c2ca
1 parent 7a98455
commit 631c2ca
Show file tree

Hide file tree

Showing 2 changed files with 10 additions and 3 deletions.
diff --git a/dataset/src/main/scala/frameless/TypedDataset.scala b/dataset/src/main/scala/frameless/TypedDataset.scala
@@ -16,6 +16,10 @@ class TypedDataset[T](
     new TypedDataset(dataset.as[U](TypedExpressionEncoder[U]))
   }
 
+  def coalesce(numPartitions: Int): TypedDataset[T] = {
+    new TypedDataset(dataset.coalesce(numPartitions))
+  }
+
   /**
     * Returns `TypedColumn` of type `A` given it's name.
     *

diff --git a/dataset/src/test/scala/frameless/GroupByTests.scala b/dataset/src/test/scala/frameless/GroupByTests.scala
@@ -6,6 +6,9 @@ import org.scalacheck.Prop
 import org.scalacheck.Prop._
 
 class GroupByTests extends TypedDatasetSuite {
+
+  // datasets are coalesced due to https://issues.apache.org/jira/browse/SPARK-12675
+
   test("groupBy('a).agg(sum('b))") {
     def prop[A, B](data: List[X2[A, B]])(
       implicit
@@ -17,7 +20,7 @@ class GroupByTests extends TypedDatasetSuite {
       s: Summable[B],
       o: Ordering[A] // to compare ordered vectors
     ): Prop = {
-      val dataset = TypedDataset.create(data)
+      val dataset = TypedDataset.create(data).coalesce(2)
       val A = dataset.col[A]('a)
       val B = dataset.col[B]('b)
 
@@ -44,7 +47,7 @@ class GroupByTests extends TypedDatasetSuite {
       sc: Summable[C],
       o: Ordering[A] // to compare ordered vectors
     ): Prop = {
-      val dataset = TypedDataset.create(data)
+      val dataset = TypedDataset.create(data).coalesce(2)
       val A = dataset.col[A]('a)
       val B = dataset.col[B]('b)
       val C = dataset.col[C]('c)
@@ -81,7 +84,7 @@ class GroupByTests extends TypedDatasetSuite {
       sd: Summable[D],
       o: Ordering[(A, B)] // to compare ordered vectors
     ): Prop = {
-      val dataset = TypedDataset.create(data)
+      val dataset = TypedDataset.create(data).coalesce(2)
       val A = dataset.col[A]('a)
       val B = dataset.col[B]('b)
       val C = dataset.col[C]('c)