Skip to content

Commit

Permalink
Add test when saving dataframe in json format
Browse files Browse the repository at this point in the history
  • Loading branch information
andersonkmi committed Apr 5, 2024
1 parent e0d8c72 commit e99286c
Show file tree
Hide file tree
Showing 2 changed files with 9 additions and 2 deletions.
3 changes: 2 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -14,4 +14,5 @@ target/
*.jar
*.json
dataframe.csv/
dataframe.json/
dataframe.json/
dataframe.parquet/
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import org.apache.spark.SparkConf
import org.apache.spark.sql.types.{IntegerType, StringType, StructField, StructType}
import org.apache.spark.sql.functions.asc
import org.apache.spark.sql.{DataFrame, Row, SparkSession}
import org.codecraftlabs.spark.utils.DataFrameUtil.{extractDistinctValues, saveDataFrameToCsv}
import org.codecraftlabs.spark.utils.DataFrameUtil.{extractDistinctValues, saveDataFrameToCsv, saveDataFrameToJson}
import org.scalatest.BeforeAndAfterAll
import org.scalatest.flatspec.AnyFlatSpec
import org.scalatest.matchers.should.Matchers
Expand All @@ -15,6 +15,7 @@ class DataFrameUtilSpec extends AnyFlatSpec with Matchers with BeforeAndAfterAll
@transient var sparkSession: Option[SparkSession] = None
private val CurrentDirectory = Path("src/test/scala/org/codecraftlabs/spark/utils").toAbsolute
private val CsvFile = CurrentDirectory.toString + "/dataframe.csv"
private val JsonFile = CurrentDirectory.toString + "/dataframe.json"

override def beforeAll(): Unit = {
val sparkConfig = new SparkConf()
Expand Down Expand Up @@ -49,6 +50,11 @@ class DataFrameUtilSpec extends AnyFlatSpec with Matchers with BeforeAndAfterAll
saveDataFrameToCsv(df = dataFrame, destination = CsvFile)
}

"When saving the data frame as json" should "create a json file" in {
val dataFrame = createDataFrame()
saveDataFrameToJson(df = dataFrame, destination = JsonFile)
}

"When extracting distinct values" should "return a new dataframe with unique values" in {
val dataFrame = createDataFrame()
val df = extractDistinctValues(dataFrame, "userName").orderBy(asc("userName"))
Expand Down

0 comments on commit e99286c

Please sign in to comment.