Spark Enrich: bump scala-common-enrich to 0.32.0 (closes snowplow#3674)

3-enrich/spark-enrich/project/Dependencies.scala

@@ -34,7 +34,7 @@ object Dependencies {
     val spark            = "2.2.0"
     val scalaz7          = "7.0.9"
     val scopt            = "3.5.0"
-    val commonEnrich     = "0.30.0"
+    val commonEnrich     = "0.32.0"
     val igluClient       = "0.5.0"
     // Scala (test only)
     val specs2           = "2.3.13"
@@ -45,19 +45,19 @@ object Dependencies {
 
   object Libraries {
     // Java
-    val hadoopLZO        = "com.hadoop.gplcompression" %  "hadoop-lzo"                   % V.hadoopLZO
-    val elephantBird     = "com.twitter.elephantbird"  %  "elephant-bird-core"           % V.elephantBird
+    val hadoopLZO        = "com.hadoop.gplcompression"    %  "hadoop-lzo"                   % V.hadoopLZO
+    val elephantBird     = "com.twitter.elephantbird"     %  "elephant-bird-core"           % V.elephantBird
     // Scala
-    val sparkCore        = "org.apache.spark"          %% "spark-core"                   % V.spark        % "provided"
-    val sparkSQL         = "org.apache.spark"          %% "spark-sql"                    % V.spark        % "provided"
-    val scalaz7          = "org.scalaz"                %% "scalaz-core"                  % V.scalaz7
-    val scopt            = "com.github.scopt"          %% "scopt"                        % V.scopt
-    val commonEnrich     = "com.snowplowanalytics"     %% "snowplow-common-enrich"       % V.commonEnrich
-    val igluClient       = "com.snowplowanalytics"     %% "iglu-scala-client"            % V.igluClient
+    val sparkCore        = "org.apache.spark"             %% "spark-core"                   % V.spark        % "provided"
+    val sparkSQL         = "org.apache.spark"             %% "spark-sql"                    % V.spark        % "provided"
+    val scalaz7          = "org.scalaz"                   %% "scalaz-core"                  % V.scalaz7
+    val scopt            = "com.github.scopt"             %% "scopt"                        % V.scopt
+    val commonEnrich     = "com.snowplowanalytics"        %% "snowplow-common-enrich"       % V.commonEnrich
+    val igluClient       = "com.snowplowanalytics"        %% "iglu-scala-client"            % V.igluClient
     // Thrift
-    val snowplowRawEvent = "com.snowplowanalytics"     % "snowplow-thrift-raw-event"     % V.snowplowRawEvent
-    val collectorPayload = "com.snowplowanalytics"     % "collector-payload-1"           % V.collectorPayload
+    val snowplowRawEvent = "com.snowplowanalytics"        % "snowplow-thrift-raw-event"     % V.snowplowRawEvent
+    val collectorPayload = "com.snowplowanalytics"        % "collector-payload-1"           % V.collectorPayload
     // Scala (test only)
-    val specs2           = "org.specs2"                %% "specs2-core"                  % V.specs2           % "test"
+    val specs2           = "org.specs2"                   %% "specs2-core"                  % V.specs2           % "test"
   }
 }

3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/EnrichJobSpec.scala

@@ -282,11 +282,10 @@ object EnrichJobSpec {
             |"database": "%s",
             |"uri": "http://snowplow-hosted-assets.s3.amazonaws.com/third-party/maxmind"
           |}""".format(lookup, lookup match {
-        case "geo"          => "GeoIPCity.dat"
-        case "isp"          => "GeoIPISP.dat"
-        case "organization" => "GeoIPOrg.dat"
-        case "domain"       => "GeoIPDomain.dat"
-        case "netspeed"     => "GeoIPNetSpeedCell.dat"
+        case "geo"            => "GeoIP2-City.mmdb"
+        case "isp"            => "GeoIP2-ISP.mmdb"
+        case "domain"         => "GeoIP2-Domain.mmdb"
+        case "connectionType" => "GeoIP2-Connection-Type.mmdb"
         })
     }
 
@@ -430,7 +429,7 @@ object EnrichJobSpec {
                 |}
               |},
               |{
-                |"schema": "iglu:com.snowplowanalytics.snowplow/ip_lookups/jsonschema/1-0-0",
+                |"schema": "iglu:com.snowplowanalytics.snowplow/ip_lookups/jsonschema/2-0-0",
                 |"data": {
                   |"vendor": "com.snowplowanalytics.snowplow",
                   |"name": "ip_lookups",

3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/bad/BadTrackerCfLinesSpec.scala

@@ -26,7 +26,7 @@ object BadTrackerCfLinesSpec {
   val expected = List(
     """{"line":"2012-05-24  00:08:40  LHR5  3397  74.125.17.210 GET d3gs014xn8p70.cloudfront.net  /ice.png  200 http://www.psychicbazaar.com/oracles/119-psycards-book-and-deck-starter-pack.html Mozilla/5.0%20(Linux;%20U;%20Android%202.3.4;%20generic)%20AppleWebKit/535.1%20(KHTML,%20like%20Gecko;%20Google%20Web%20Preview)%20Version/4.0%20Mobile%20Safari/535.1  e=pv&p=mobile&page=Psycards%2520book%2520and%2520deck%2520starter%2520pack%2520-%2520Psychic%2520Bazaar&tid=721410&uid=3798cdce0493133e&vid=1&lang=en&refr=http%253A%252F%252Fwww.google.com%252Fm%252Fsearch&res=640x960&cookie=1","errors":[{"level":"error","message":"Field [p]: [mobile] is not a supported tracking platform"}]}""",
     """{"line":"2012-05-24  00:06:42  LHR5  3402  213.52.50.8  GET d3gs014xn8p70.cloudfront.net  /ice.png  200 http://www.psychicbazaar.com/oracles/119-psycards-book-and-deck-starter-pack.html Mozilla/5.0%20(iPhone;%20CPU%20iPhone%20OS%205_1_1%20like%20Mac%20OS%20X)%20AppleWebKit/534.46%20(KHTML,%20like%20Gecko)%20Version/5.1%20Mobile/9B206%20Safari/7534.48.3  e=lol&page=Psycards%2520book%2520and%2520deck%2520starter%2520pack%2520-%2520Psychic%2520Bazaar&tid=019539&uid=e7bccbb647296c98&vid=a&p=web&aid=CFe23a&fp=1906624389&tz=Europe%2FLondon&cd=24&lang=en-us&refr=http%253A%252F%252Fwww.google.com%252Fsearch%253Fhl%253Den%2526q%253Dthe%252Bpsycard%252Bstory%2526oq%253Dthe%252Bpsycard%252Bstory%2526aq%253Df%2526aqi%253D%2526aql%253D%2526gs_l%253Dmobile-gws-serp.12...0.0.0.6358.0.0.0.0.0.0.0.0..0.0...0.0.JrNbKlRgHbQ%2526mvs%253D0&f_pdf=0&f_qt=1&f_realp=0&f_wma=1&f_dir=0&f_fla=1&f_java=0&f_gears=1&f_ag=0&res=320x480&cookie=1","errors":[{"level":"error","message":"Field [e]: [lol] is not a recognised event code"},{"level":"error","message":"Field [vid]: cannot convert [a] to Int"},{"level":"error","message":"Unrecognized event [null]"}]}""",
-    """{"line":"2012-05-24  00:06:42  LHR5  3402  90.194.12.51  GET d3gs014xn8p70.cloudfront.net  /ice.png  200 http://www.psychicbazaar.com/oracles/119-psycards-book-and-deck-starter-pack.html Mozilla/5.0%20(iPhone;%20CPU%20iPhone%20OS%205_1_1%20like%20Mac%20OS%20X)%20AppleWebKit/534.46%20(KHTML,%20like%20Gecko)%20Version/5.1%20Mobile/9B206%20Safari/7534.48.3  e=lol&ue_px=am9obitzbWl0aA&page=Psycards%2520book%2520and%2520deck%2520starter%2520pack%2520-%2520Psychic%2520Bazaar&tid=019539&uid=e7bccbb647296c98&vid=a&p=web&aid=CFe23a&fp=1906624389&tz=Europe%2FLondon&cd=24&lang=en-us&refr=http%253A%252F%252Fwww.google.com%252Fsearch%253Fhl%253Den%2526q%253Dthe%252Bpsycard%252Bstory%2526oq%253Dthe%252Bpsycard%252Bstory%2526aq%253Df%2526aqi%253D%2526aql%253D%2526gs_l%253Dmobile-gws-serp.12...0.0.0.6358.0.0.0.0.0.0.0.0..0.0...0.0.JrNbKlRgHbQ%2526mvs%253D0&f_pdf=0&f_qt=1&f_realp=0&f_wma=1&f_dir=0&f_fla=1&f_java=0&f_gears=1&f_ag=0&res=320x480&cookie=1","errors":[{"level":"error","message":"Field [e]: [lol] is not a recognised event code"},{"level":"error","message":"Field [vid]: cannot convert [a] to Int"},{"level":"error","message":"Field [ue_px]: invalid JSON [john+smith] with parsing error: Unrecognized token 'john': was expecting ('true', 'false' or 'null') at [Source: john+smith; line: 1, column: 5]"},{"level":"error","message":"Unrecognized event [null]"}]}"""
+    """{"line":"2012-05-24  00:06:42  LHR5  3402  90.194.12.51  GET d3gs014xn8p70.cloudfront.net  /ice.png  200 http://www.psychicbazaar.com/oracles/119-psycards-book-and-deck-starter-pack.html Mozilla/5.0%20(iPhone;%20CPU%20iPhone%20OS%205_1_1%20like%20Mac%20OS%20X)%20AppleWebKit/534.46%20(KHTML,%20like%20Gecko)%20Version/5.1%20Mobile/9B206%20Safari/7534.48.3  e=lol&ue_px=am9obitzbWl0aA&page=Psycards%2520book%2520and%2520deck%2520starter%2520pack%2520-%2520Psychic%2520Bazaar&tid=019539&uid=e7bccbb647296c98&vid=a&p=web&aid=CFe23a&fp=1906624389&tz=Europe%2FLondon&cd=24&lang=en-us&refr=http%253A%252F%252Fwww.google.com%252Fsearch%253Fhl%253Den%2526q%253Dthe%252Bpsycard%252Bstory%2526oq%253Dthe%252Bpsycard%252Bstory%2526aq%253Df%2526aqi%253D%2526aql%253D%2526gs_l%253Dmobile-gws-serp.12...0.0.0.6358.0.0.0.0.0.0.0.0..0.0...0.0.JrNbKlRgHbQ%2526mvs%253D0&f_pdf=0&f_qt=1&f_realp=0&f_wma=1&f_dir=0&f_fla=1&f_java=0&f_gears=1&f_ag=0&res=320x480&cookie=1","errors":[{"level":"error","message":"Field [e]: [lol] is not a recognised event code"},{"level":"error","message":"Field [vid]: cannot convert [a] to Int"},{"level":"error","message":"Field [ue_px]: invalid JSON [john+smith] with parsing error: Unrecognized token 'john': was expecting ('true', 'false' or 'null') at [Source: (String)\"john+smith\"; line: 1, column: 5]"},{"level":"error","message":"Unrecognized event [null]"}]}"""
   )
 }
 

3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/bad/NullNumericFieldsSpec.scala

@@ -33,7 +33,7 @@ class NullNumericFieldsSpec extends Specification with EnrichJobSpec {
   override def appName = "null-numeric-fields"
   sequential
   "A job which processes a CF file containing 1 event with null int and double fields" should {
-    runEnrichJob(NullNumericFieldsSpec.lines, "clj-tomcat", "2", true, List("geo", "organization"))
+    runEnrichJob(NullNumericFieldsSpec.lines, "clj-tomcat", "2", true, List("geo"))
 
     "write a bad row JSON containing the input line and all errors" in {
       val Some(bads) = readPartFile(dirs.badRows)

3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/ApiRequestEnrichmentCfLineSpec.scala

@@ -21,7 +21,7 @@ object ApiRequestEnrichmentCfLineSpec {
   import EnrichJobSpec._
   val contexts = """eyJkYXRhIjpbeyJkYXRhIjp7Im9zVHlwZSI6Ik9TWCIsImFwcGxlSWRmdiI6InNvbWVfYXBwbGVJZGZ2Iiwib3BlbklkZmEiOiJzb21lX0lkZmEiLCJjYXJyaWVyIjoic29tZV9jYXJyaWVyIiwiZGV2aWNlTW9kZWwiOiJsYXJnZSIsIm9zVmVyc2lvbiI6IjMuMC4wIiwiYXBwbGVJZGZhIjoic29tZV9hcHBsZUlkZmEiLCJhbmRyb2lkSWRmYSI6InNvbWVfYW5kcm9pZElkZmEiLCJkZXZpY2VNYW51ZmFjdHVyZXIiOiJBbXN0cmFkIn0sInNjaGVtYSI6ImlnbHU6Y29tLnNub3dwbG93YW5hbHl0aWNzLnNub3dwbG93L21vYmlsZV9jb250ZXh0L2pzb25zY2hlbWEvMS0wLTAifSx7ImRhdGEiOnsibG9uZ2l0dWRlIjoxMCwiYmVhcmluZyI6NTAsInNwZWVkIjoxNiwiYWx0aXR1ZGUiOjIwLCJhbHRpdHVkZUFjY3VyYWN5IjowLjMsImxhdGl0dWRlTG9uZ2l0dWRlQWNjdXJhY3kiOjAuNSwibGF0aXR1ZGUiOjd9LCJzY2hlbWEiOiJpZ2x1OmNvbS5zbm93cGxvd2FuYWx5dGljcy5zbm93cGxvdy9nZW9sb2NhdGlvbl9jb250ZXh0L2pzb25zY2hlbWEvMS0wLTAifV0sInNjaGVtYSI6ImlnbHU6Y29tLnNub3dwbG93YW5hbHl0aWNzLnNub3dwbG93L2NvbnRleHRzL2pzb25zY2hlbWEvMS0wLTAifQ=="""
   val lines = Lines(
-    s"2012-05-27  11:35:53  DFW3  3343  70.46.123.145 GET d3gs014xn8p70.cloudfront.net  /ice.png  200 http://www.psychicbazaar.com/oracles/119-psycards-book-and-deck-starter-pack.html?view=print#detail Mozilla/5.0%20(Windows%20NT%206.1;%20WOW64;%20rv:12.0)%20Gecko/20100101%20Firefox/12.0  &e=ue&cx=$contexts&ue_pr=%7B%22schema%22%3A%22iglu%3Acom.snowplowanalytics.snowplow%2Funstruct_event%2Fjsonschema%2F1-0-0%22%2C%22data%22%3A%7B%22schema%22%3A%22iglu%3Acom.snowplowanalytics.snowplow-website%2Fsignup_form_submitted%2Fjsonschema%2F1-0-0%22%2C%22data%22%3A%7B%22name%22%3A%22Bob%C2%AE%22%2C%22email%22%3A%22alex%2Btest%40snowplowanalytics.com%22%2C%22company%22%3A%22SP%22%2C%22eventsPerMonth%22%3A%22%3C%201%20million%22%2C%22serviceType%22%3A%22unsure%22%7D%7D%7D&dtm=1364230969450&evn=com.acme&tid=598951&vp=2560x934&ds=2543x1420&vid=43&duid=9795bd0203804cd1&p=web&tv=js-0.11.1&fp=2876815413&aid=pbzsite&lang=en-GB&cs=UTF-8&tz=Europe%2FLondon&refr=http%3A%2F%2Fwww.psychicbazaar.com%2F&f_pdf=1&f_qt=0&f_realp=0&f_wma=0&f_dir=0&f_fla=1&f_java=1&f_gears=0&f_ag=1&res=2560x1440&cd=32&cookie=1&url=http%3A%2F%2Fwww.psychicbazaar.com%2F2-tarot-cards"
+    s"2012-05-27  11:35:53  DFW3  3343  216.160.83.56 GET d3gs014xn8p70.cloudfront.net  /ice.png  200 http://www.psychicbazaar.com/oracles/119-psycards-book-and-deck-starter-pack.html?view=print#detail Mozilla/5.0%20(Windows%20NT%206.1;%20WOW64;%20rv:12.0)%20Gecko/20100101%20Firefox/12.0  &e=ue&cx=$contexts&ue_pr=%7B%22schema%22%3A%22iglu%3Acom.snowplowanalytics.snowplow%2Funstruct_event%2Fjsonschema%2F1-0-0%22%2C%22data%22%3A%7B%22schema%22%3A%22iglu%3Acom.snowplowanalytics.snowplow-website%2Fsignup_form_submitted%2Fjsonschema%2F1-0-0%22%2C%22data%22%3A%7B%22name%22%3A%22Bob%C2%AE%22%2C%22email%22%3A%22alex%2Btest%40snowplowanalytics.com%22%2C%22company%22%3A%22SP%22%2C%22eventsPerMonth%22%3A%22%3C%201%20million%22%2C%22serviceType%22%3A%22unsure%22%7D%7D%7D&dtm=1364230969450&evn=com.acme&tid=598951&vp=2560x934&ds=2543x1420&vid=43&duid=9795bd0203804cd1&p=web&tv=js-0.11.1&fp=2876815413&aid=pbzsite&lang=en-GB&cs=UTF-8&tz=Europe%2FLondon&refr=http%3A%2F%2Fwww.psychicbazaar.com%2F&f_pdf=1&f_qt=0&f_realp=0&f_wma=0&f_dir=0&f_fla=1&f_java=1&f_gears=0&f_ag=1&res=2560x1440&cd=32&cookie=1&url=http%3A%2F%2Fwww.psychicbazaar.com%2F2-tarot-cards"
   )
   val expected = List(
     "pbzsite",
@@ -37,21 +37,21 @@ object ApiRequestEnrichmentCfLineSpec {
     "cloudfront",
     etlVersion,
     null, // No user_id set
-    "70.46.123.145",
+    "216.160.83.56",
     "2876815413",
     "9795bd0203804cd1",
     "43",
     null, // No network_userid set
     "US", // US geolocation
-    "FL",
-    "Delray Beach",
+    "WA",
+    "Milton",
+    "98354",
+    "47.2513",
+    "-122.3149",
+    "Washington",
     null,
-    "26.461502",
-    "-80.0728",
-    "Florida",
     null,
-    null,
-    "nuvox.net",  // Using the MaxMind domain lookup service
+    null,  // Using the MaxMind domain lookup service
     null,
     "http://www.psychicbazaar.com/2-tarot-cards",
     null, // No page title for events
@@ -139,7 +139,7 @@ object ApiRequestEnrichmentCfLineSpec {
     null,
     null,
     null,
-    "America/New_York",
+    "America/Los_Angeles",
     null,
     null,
     null,