diff --git a/Beirut/2016/2016-10-19-huffman-encoding/assignment.sbt b/Beirut/2016/2016-10-19-huffman-encoding/assignment.sbt new file mode 100644 index 0000000..c81bd43 --- /dev/null +++ b/Beirut/2016/2016-10-19-huffman-encoding/assignment.sbt @@ -0,0 +1,2 @@ +course := "progfun1" +assignment := "patmat" diff --git a/Beirut/2016/2016-10-19-huffman-encoding/build.sbt b/Beirut/2016/2016-10-19-huffman-encoding/build.sbt new file mode 100644 index 0000000..c89fdd5 --- /dev/null +++ b/Beirut/2016/2016-10-19-huffman-encoding/build.sbt @@ -0,0 +1,73 @@ +name := course.value + "-" + assignment.value + +scalaVersion := "2.11.7" + +scalacOptions ++= Seq("-deprecation") + +// grading libraries +libraryDependencies += "junit" % "junit" % "4.10" % "test" + +// for funsets +libraryDependencies += "org.scala-lang.modules" %% "scala-parser-combinators" % "1.0.4" + +// include the common dir +commonSourcePackages += "common" + +courseId := "bRPXgjY9EeW6RApRXdjJPw" + +// See documentation in ProgFunBuild.scala +assignmentsMap := { + val styleSheetPath = (baseDirectory.value / ".." / ".." / "project" / "scalastyle_config.xml").getPath + Map( + "example" -> Assignment( + packageName = "example", + key = "g4unnjZBEeWj7SIAC5PFxA", + itemId = "xIz9O", + partId = "d5jxI", + maxScore = 10d, + styleScoreRatio = 0.2, + styleSheet = styleSheetPath), + "recfun" -> Assignment( + packageName = "recfun", + key = "SNYuDzZEEeWNVyIAC92BaQ", + itemId = "Ey6Jf", + partId = "PzVVY", + maxScore = 10d, + styleScoreRatio = 0.2, + styleSheet = styleSheetPath), + "funsets" -> Assignment( + packageName = "funsets", + key = "FNHHMDfsEeWAGiIAC46PTg", + itemId = "BVa6a", + partId = "IljBE", + maxScore = 10d, + styleScoreRatio = 0.2, + styleSheet = styleSheetPath), + "objsets" -> Assignment( + packageName = "objsets", + key = "6PTXvD99EeWAiCIAC7Pj9w", + itemId = "Ogg05", + partId = "7hlkb", + maxScore = 10d, + styleScoreRatio = 0.2, + styleSheet = styleSheetPath, + options = Map("grader-timeout" -> "1800")), + "patmat" -> Assignment( + packageName = "patmat", + key = "BwkTtD9_EeWFZSIACtiVgg", + itemId = "uctOq", + partId = "2KYZc", + maxScore = 10d, + styleScoreRatio = 0.2, + styleSheet = styleSheetPath), + "forcomp" -> Assignment( + packageName = "forcomp", + key = "CPJe397VEeWLGArWOseZkw", + itemId = "nVRPb", + partId = "v2XIe", + maxScore = 10d, + styleScoreRatio = 0.2, + styleSheet = styleSheetPath, + options = Map("grader-timeout" -> "1800")) + ) +} diff --git a/Beirut/2016/2016-10-19-huffman-encoding/project/ScalaTestRunner.scala b/Beirut/2016/2016-10-19-huffman-encoding/project/ScalaTestRunner.scala new file mode 100644 index 0000000..2080bdc --- /dev/null +++ b/Beirut/2016/2016-10-19-huffman-encoding/project/ScalaTestRunner.scala @@ -0,0 +1,222 @@ +import sbt._ +import Keys._ +import sys.process.{Process => SysProc, ProcessLogger} +import java.util.concurrent._ +import collection.mutable.ListBuffer +import scala.pickling.Defaults._ +import scala.pickling.json._ + +final case class GradingSummary(score: Int, maxScore: Int, feedback: String) + +object ScalaTestRunner { + + class LimitedStringBuffer { + val buf = new ListBuffer[String]() + private var lines = 0 + private var lengthCropped = false + + override def toString = buf.mkString("\n").trim + + def append(s: String) = + if (lines < Settings.maxOutputLines) { + val shortS = + if (s.length > Settings.maxOutputLineLength) { + if (!lengthCropped) { + val msg = + """WARNING: OUTPUT LINES CROPPED + |Your program generates very long lines on the standard (or error) output. Some of + |the lines have been cropped. + |This should not have an impact on your grade or the grading process; however it is + |bad style to leave `print` statements in production code, so consider removing and + |replacing them by proper tests. + | """.stripMargin + buf.prepend(msg) + lengthCropped = true + } + s.substring(0, Settings.maxOutputLineLength) + } else s + buf.append(shortS) + lines += 1 + } else if (lines == Settings.maxOutputLines) { + val msg = + """WARNING: PROGRAM OUTPUT TOO LONG + |Your program generates massive amounts of data on the standard (or error) output. + |You are probably using `print` statements to debug your code. + |This should not have an impact on your grade or the grading process; however it is + |bad style to leave `print` statements in production code, so consider removing and + |replacing them by proper tests. + | """.stripMargin + buf.prepend(msg) + lines += 1 + } + } + + private def forkProcess(proc: SysProc, timeout: Int): Unit = { + val executor = Executors.newSingleThreadExecutor() + val future: Future[Unit] = executor.submit(new Callable[Unit] { + def call { + proc.exitValue() + } + }) + try { + future.get(timeout, TimeUnit.SECONDS) + } catch { + case to: TimeoutException => + future.cancel(true) + throw to + } finally { + executor.shutdown() + } + } + + private def runPathString(file: File) = file.getAbsolutePath.replace(" ", "\\ ") + + private def invokeScalaTestInSeparateProcess(scalaTestCommand: List[String], logError: String => Unit, timeout: Int): String = { + val out = new LimitedStringBuffer() + var proc: SysProc = null + try { + proc = SysProc(scalaTestCommand).run(ProcessLogger(out.append, out.append)) + forkProcess(proc, timeout) + } catch { + case e: TimeoutException => + val msg = "Timeout when running ScalaTest\n" + out.toString() + logError(msg) + proc.destroy() + case e: Throwable => + val msg = "Error occurred while running the ScalaTest command\n" + e.toString + "\n" + out.toString() + logError(msg) + proc.destroy() + throw e + } finally { + println(out.toString) + if (proc != null) { + println("Exit process: " + proc.exitValue()) + } + } + + out.toString + } + + private def computeSummary(outFilePath: String, classpathString: String, logError: String => Unit): String = { + val summaryFilePath = outFilePath + ".summary" + val summaryCmd = "java" :: + "-cp" :: classpathString :: + "ch.epfl.lamp.grading.GradingSummaryRunner" :: + outFilePath :: summaryFilePath :: Nil + var summaryProc: SysProc = null + try { + summaryProc = SysProc(summaryCmd).run() + summaryProc.exitValue + } catch { + case e: Throwable => + val msg = "Error occurred while running the test ScalaTest summary command\n" + e.toString + logError(msg) + summaryProc.destroy() + throw e + } /* finally { // Useful for debugging when Coursera kills our grader + println(scala.io.Source.fromFile(outFilePath).getLines().mkString("\n")) + println(scala.io.Source.fromFile(summaryFilePath).getLines().mkString("\n")) + }*/ + // Example output: + // { + // "$type": "ch.epfl.lamp.grading.Entry.SuiteStart", + // "suiteId": "ParallelCountChangeSuite::50" + // } + summaryFilePath + } + + def runScalaTest(classpath: Classpath, testClasses: File, outfile: File, + resourceFiles: List[File], gradeOptions: Map[String, String], + logError: String => Unit, instragentPath: String) = { + + // invoke scalatest in the separate process + val classpathString = classpath map { case Attributed(file) => file.getAbsolutePath } mkString ":" + val cmd = scalaTestCommand(testClasses, outfile, resourceFiles, gradeOptions, classpathString, instragentPath) + + val timeout = gradeOptions.getOrElse("totalTimeout", Settings.scalaTestTimeout.toString).toInt + val runLog = invokeScalaTestInSeparateProcess(cmd, logError, timeout) + + // compute the summary + val summaryFilePath = computeSummary(outfile.getAbsolutePath, classpathString, logError) + val summary = unpickleSummary(logError, runLog, summaryFilePath) + + // cleanup all the files + IO.delete(new File(summaryFilePath) :: outfile :: Nil) + + (summary.score, summary.maxScore, summary.feedback, runLog) + } + + private def unpickleSummary(logError: (String) => Unit, runLog: String, summaryFileStr: String): GradingSummary = { + try { + io.Source.fromFile(summaryFileStr).getLines.mkString("\n").unpickle[GradingSummary] + } catch { + case e: Throwable => + val msg = "Error occured while reading ScalaTest summary file\n" + e.toString + "\n" + runLog + logError(msg) + throw e + } + } + + private def scalaTestCommand(testClasses: File, outfile: File, resourceFiles: List[File], gradeOptions: Map[String, String], classpathString: String, + instragentPath: String): List[String] = { + val testRunPath = runPathString(testClasses) + val resourceFilesString = resourceFiles.map(_.getAbsolutePath).mkString(":") + // Deleting the file is helpful: it makes reading the file below crash in case ScalaTest doesn't + // run as expected. Problem is, it's hard to detect if ScalaTest ran successfully or not: it + // exits with non-zero if there are failed tests, and also if it crashes... + outfile.delete() + + def prop(name: String, value: String) = "-D" + name + "=" + value + + // grade options + val xmx = gradeOptions.get("Xmx").map("-Xmx" + _).getOrElse("-Xmx256m") + val xms = gradeOptions.get("Xms").map("-Xms" + _).getOrElse("-Xms10m") + val timeoutPerTest = gradeOptions.getOrElse("individualTimeout", Settings.individualTestTimeout.toString) + + // we don't specify "-w packageToTest" - the build file only compiles the tests + // for the current project. so we don't need to do it again here. + + // NOTICE: DON'T start test in parallel, it would break profiling. Check the + // implementation of @InstrumentedSuite for more details. + "java" :: + xmx :: xms :: + s"-javaagent:$instragentPath" :: + prop(Settings.scalaTestReportFileProperty, outfile.getAbsolutePath) :: + prop(Settings.scalaTestIndividualTestTimeoutProperty, timeoutPerTest) :: + prop(Settings.scalaTestReadableFilesProperty, resourceFilesString) :: + prop(Settings.scalaTestDefaultWeightProperty, Settings.scalaTestDefaultWeight.toString) :: + "-cp" :: classpathString :: + "org.scalatest.tools.Runner" :: + "-R" :: testRunPath :: + "-C" :: Settings.scalaTestReporter :: + Nil + } + + private def testEnv(options: Map[String, String]): String = { + val memory = options.get("Xmx").getOrElse("256m") + val timeout = options.get("totalTimeout").map(_.toInt).getOrElse(Settings.scalaTestTimeout) + val timeoutPerTest = options.get("individualTimeout").map(_.toInt).getOrElse(Settings.individualTestTimeout) + + "======== TESTING ENVIRONMENT ========\n" + + s"Limits: memory: $memory, total time: ${timeout}s, per test case time: ${timeoutPerTest}s\n" + } + + def scalaTestGrade(gradingReporter: GradingFeedback, classpath: Classpath, testClasses: File, outfile: File, + resourceFiles: List[File], gradeOptions: Map[String, String], instragentPath: String): Unit = { + + val (score, maxScore, feedback, runLog) = + runScalaTest(classpath, testClasses, outfile, resourceFiles, gradeOptions, gradingReporter.testExecutionFailed, instragentPath) + + if (score == maxScore) { + gradingReporter.allTestsPassed() + } else { + val scaledScore = gradingReporter.maxTestScore * score / maxScore + gradingReporter.testsFailed(feedback + testEnv(gradeOptions), scaledScore) + } + + if (!runLog.isEmpty) { + gradingReporter.testExecutionDebugLog(runLog) + } + } +} + diff --git a/Beirut/2016/2016-10-19-huffman-encoding/project/Settings.scala b/Beirut/2016/2016-10-19-huffman-encoding/project/Settings.scala new file mode 100644 index 0000000..78bfc1c --- /dev/null +++ b/Beirut/2016/2016-10-19-huffman-encoding/project/Settings.scala @@ -0,0 +1,27 @@ +object Settings { + val maxSubmitFileSize = { + val mb = 1024 * 1024 + 10 * mb + } + + val testResultsFileName = "grading-results-log" + + // time in seconds that we give scalatest for running + val scalaTestTimeout = 850 // coursera has a 15 minute timeout anyhow + val individualTestTimeout = 240 + + // default weight of each test in a GradingSuite, in case no weight is given + val scalaTestDefaultWeight = 10 + + // when students leave print statements in their code, they end up in the output of the + // system process running ScalaTest (ScalaTestRunner.scala); we need some limits. + val maxOutputLines = 10 * 1000 + val maxOutputLineLength = 1000 + + val scalaTestReportFileProperty = "scalatest.reportFile" + val scalaTestIndividualTestTimeoutProperty = "scalatest.individualTestTimeout" + val scalaTestReadableFilesProperty = "scalatest.readableFiles" + val scalaTestDefaultWeightProperty = "scalatest.defaultWeight" + val scalaTestReporter = "ch.epfl.lamp.grading.GradingReporter" + +} diff --git a/Beirut/2016/2016-10-19-huffman-encoding/project/buildSettings.sbt b/Beirut/2016/2016-10-19-huffman-encoding/project/buildSettings.sbt new file mode 100644 index 0000000..1f7ee33 --- /dev/null +++ b/Beirut/2016/2016-10-19-huffman-encoding/project/buildSettings.sbt @@ -0,0 +1,11 @@ +// used for style-checking submissions +libraryDependencies += "org.scalastyle" %% "scalastyle" % "0.8.0" + +// used for submitting the assignments to Coursera +libraryDependencies += "org.scalaj" %% "scalaj-http" % "2.2.1" + +// used for base64 encoding +libraryDependencies += "commons-codec" % "commons-codec" % "1.10" + +// used to escape json for the submission +libraryDependencies += "org.apache.commons" % "commons-lang3" % "3.4" diff --git a/Beirut/2016/2016-10-19-huffman-encoding/project/plugins.sbt b/Beirut/2016/2016-10-19-huffman-encoding/project/plugins.sbt new file mode 100644 index 0000000..737941d --- /dev/null +++ b/Beirut/2016/2016-10-19-huffman-encoding/project/plugins.sbt @@ -0,0 +1 @@ +addSbtPlugin("com.typesafe.sbteclipse" % "sbteclipse-plugin" % "4.0.0") diff --git a/Beirut/2016/2016-10-19-huffman-encoding/readme.md b/Beirut/2016/2016-10-19-huffman-encoding/readme.md new file mode 100644 index 0000000..407d11a --- /dev/null +++ b/Beirut/2016/2016-10-19-huffman-encoding/readme.md @@ -0,0 +1,14 @@ +###Date: 2016/10/19 +###Source: http://chara.epfl.ch/~dockermoocs/progfun1/patmat.zip +###Problem Description: Huffman encoding in Scala + +Workshop Description: +This session will start by a quick introduction to useful Scala functions and paradigm. +We will then explain how does Huffman encoding/decoding works and write its algorithm in Scala. +To save time, the tests for this session were already written. + +#Retrospective +1. The Scala presentation and explaining the exercise were helpful and clear +2. It is nice to discover new languages features +3. It was nice to see the how complicated and dirty it gets if we try to use non functional paradigm to implement functional code +4. I liked practicing Scala and it was nice to write clean code \ No newline at end of file diff --git a/Beirut/2016/2016-10-19-huffman-encoding/src/main/scala/common/package.scala b/Beirut/2016/2016-10-19-huffman-encoding/src/main/scala/common/package.scala new file mode 100644 index 0000000..f94e846 --- /dev/null +++ b/Beirut/2016/2016-10-19-huffman-encoding/src/main/scala/common/package.scala @@ -0,0 +1,41 @@ + +import java.io.File + +package object common { + + /** An alias for the `Nothing` type. + * Denotes that the type should be filled in. + */ + type ??? = Nothing + + /** An alias for the `Any` type. + * Denotes that the type should be filled in. + */ + type *** = Any + + + /** + * Get a child of a file. For example, + * + * subFile(homeDir, "b", "c") + * + * corresponds to ~/b/c + */ + def subFile(file: File, children: String*) = { + children.foldLeft(file)((file, child) => new File(file, child)) + } + + /** + * Get a resource from the `src/main/resources` directory. Eclipse does not copy + * resources to the output directory, then the class loader cannot find them. + */ + def resourceAsStreamFromSrc(resourcePath: List[String]): Option[java.io.InputStream] = { + val classesDir = new File(getClass.getResource(".").toURI) + val projectDir = classesDir.getParentFile.getParentFile.getParentFile.getParentFile + val resourceFile = subFile(projectDir, ("src" :: "main" :: "resources" :: resourcePath): _*) + if (resourceFile.exists) + Some(new java.io.FileInputStream(resourceFile)) + else + None + } +} diff --git a/Beirut/2016/2016-10-19-huffman-encoding/src/main/scala/patmat/Huffman.scala b/Beirut/2016/2016-10-19-huffman-encoding/src/main/scala/patmat/Huffman.scala new file mode 100644 index 0000000..a6e05ce --- /dev/null +++ b/Beirut/2016/2016-10-19-huffman-encoding/src/main/scala/patmat/Huffman.scala @@ -0,0 +1,221 @@ +package patmat + +import common._ + +/** + * Assignment 4: Huffman coding + * + */ +object Huffman { + + /** + * A huffman code is represented by a binary tree. + * + * Every `Leaf` node of the tree represents one character of the alphabet that the tree can encode. + * The weight of a `Leaf` is the frequency of appearance of the character. + * + * The branches of the huffman tree, the `Fork` nodes, represent a set containing all the characters + * present in the leaves below it. The weight of a `Fork` node is the sum of the weights of these + * leaves. + */ + abstract class CodeTree + case class Fork(left: CodeTree, right: CodeTree, chars: List[Char], weight: Int) extends CodeTree + case class Leaf(char: Char, weight: Int) extends CodeTree + + + // Part 1: Basics + def weight(tree: CodeTree): Int = tree match { + case Leaf(_,weight) => weight + case Fork(_,_,_,weight) => weight + } + + def chars(tree: CodeTree): List[Char] = tree match { + case Leaf(char,_) => List(char) + case Fork(_,_,chars,_) => chars + } + + def makeCodeTree(left: CodeTree, right: CodeTree) = + Fork(left, right, chars(left) ::: chars(right), weight(left) + weight(right)) + + + + // Part 2: Generating Huffman trees + + /** + * In this assignment, we are working with lists of characters. This function allows + * you to easily create a character list from a given string. + */ + def string2Chars(str: String): List[Char] = str.toList + + /** + * This function computes for each unique character in the list `chars` the number of + * times it occurs. For example, the invocation + * + * times(List('a', 'b', 'a')) + * + * should return the following (the order of the resulting list is not important): + * + * List(('a', 2), ('b', 1)) + * + * The type `List[(Char, Int)]` denotes a list of pairs, where each pair consists of a + * character and an integer. Pairs can be constructed easily using parentheses: + * + * val pair: (Char, Int) = ('c', 1) + * + * In order to access the two elements of a pair, you can use the accessors `_1` and `_2`: + * + * val theChar = pair._1 + * val theInt = pair._2 + * + * Another way to deconstruct a pair is using pattern matching: + * + * pair match { + * case (theChar, theInt) => + * println("character is: "+ theChar) + * println("integer is : "+ theInt) + * } + */ + def times(origChars: List[Char]): List[(Char, Int)] = { + def occurence(distinctChars: List[Char]): List[(Char, Int)] ={ + distinctChars match{ + case x::xs => (x,origChars.count(_==x)) :: occurence(xs) + case Nil => Nil + } + } + occurence(origChars.distinct) + } + + /** + * Returns a list of `Leaf` nodes for a given frequency table `freqs`. + * + * The returned list should be ordered by ascending weights (i.e. the + * head of the list should have the smallest weight), where the weight + * of a leaf is the frequency of the character. + */ + def makeOrderedLeafList(freqs: List[(Char, Int)]): List[Leaf] = { + def makeLeafList(freqs: List[(Char, Int)]): List[Leaf] = freqs match { + case (char, freq) :: xs => Leaf(char, freq) :: makeLeafList(xs) + case Nil => Nil + } + makeLeafList(freqs).sortWith((leafA,LeafB)=>leafA.weight