Merge branch 'release/r108-val-camonica'

2-collectors/clojure-collector/java-servlet/project.clj

@@ -13,25 +13,20 @@
 ;;;; Copyright: Copyright (c) 2012-2013 Snowplow Analytics Ltd
 ;;;; License:   Apache License Version 2.0
 
-(require 'cemerick.pomegranate.aether)
-(cemerick.pomegranate.aether/register-wagon-factory!
-  "http" #(org.apache.maven.wagon.providers.http.HttpWagon.))
-
-(defproject snowplow/clojure-collector "2.0.0" ;; MUST also bump version in server.xml
+(defproject snowplow/clojure-collector "2.1.0" ;; MUST also bump version in server.xml
   :license {:name "Apache Version 2.0"
   :url "http://www.apache.org/licenses/LICENSE-2.0"}
   :description "A SnowPlow event collector written in Clojure. AWS Elastic Beanstalk compatible."
-  :dependencies     [[org.clojure/clojure "1.4.0"]
-                     [ring/ring-core "1.1.8"]
-                     [ring/ring-devel "1.1.8"]
-                     [compojure "1.1.3"]
-                     [metrics-clojure "0.9.2"]
-                     [metrics-clojure-ring "0.9.2"]
-                     [commons-codec/commons-codec "1.7"]]
+  :dependencies     [[org.clojure/clojure "1.9.0"]
+                     [ring/ring-core "1.6.3"]
+                     [ring/ring-devel "1.6.3"]
+                     [compojure "1.6.1"]
+                     [metrics-clojure "2.10.0"]
+                     [metrics-clojure-ring "2.10.0"]
+                     [commons-codec/commons-codec "1.11"]]
   ;; The jetty adapter is only used during development
-  :profiles         {:dev {:dependencies [[ring/ring-jetty-adapter "1.1.8"]]}}
+  :profiles         {:dev {:dependencies [[ring/ring-jetty-adapter "1.6.3"]]}}
   :war-resources-path   "war-resources"
-  :plugins          [[lein-ring "0.8.3"]
-                     [lein-beanstalk "0.2.6"]]
+  :plugins          [[lein-ring "0.12.4"]]
   :ring {:handler snowplow.clojure-collector.beanstalk/app}) ; .beanstalk -> .core if you don't need Beanstalk support
 

2-collectors/clojure-collector/java-servlet/src/snowplow/clojure_collector/config.clj

@@ -24,6 +24,7 @@
 (def ^:const duration-varnames ["PARAM4", "SP_DURATION"])
 (def ^:const cross-domain-policy-domain-varnames ["PARAM5", "SP_CDP_DOMAIN"])
 (def ^:const cross-domain-policy-secure-varnames ["PARAM6", "SP_CDP_SECURE"])
+(def ^:const path-varnames ["PARAM7", "SP_PATH"])
 
 ;; Defaults
 (def ^:const default-p3p-header "policyref=\"/w3c/p3p.xml\", CP=\"NOI DSP COR NID PSA OUR IND COM NAV STA\"")
@@ -69,9 +70,14 @@
 (def domain
   "Get the domain the name cookies will be set on.
    Can be a wildcard e.g. '.foo.com'.
-   If undefined we'll just use the FQDN of the host"
+   If undefined we'll just use the FQDN of the host."
   (get-var domain-varnames))
 
+(def path
+  "Get the path the cookies will be set on.
+   If undefined we'll just use /"
+  (get-var path-varnames))
+
 (def cross-domain-policy-domain
   "Get the cross domain policy domain.
   See the specification for reference:

2-collectors/clojure-collector/java-servlet/src/snowplow/clojure_collector/core.clj

@@ -35,6 +35,7 @@
     cookies
     config/duration
     config/domain
+    config/path
     config/p3p-header
     pixel
     vendor

2-collectors/clojure-collector/java-servlet/src/snowplow/clojure_collector/responses.clj

@@ -38,13 +38,17 @@
 
 (defn- set-cookie
   "Sets a Snowplow cookie with visitor `id`,
-   to last `duration` seconds for `domain`.
+   to last `duration` seconds for `domain` at `path`.
    If domain is nil, leave out so the FQDN
-   of the host can be used instead"
-  [id duration domain]
+   of the host can be used instead.
+   If path is nil, path will be /."
+  [id duration domain path]
   (merge
     {:value    id
-     :expires (now-plus duration)}
+     :expires (now-plus duration)
+     :path (if (nil? path)
+       "/"
+       path)}
    (when-let [d domain]
     {:domain   d})))
 
@@ -95,11 +99,11 @@
 (defn send-cookie-pixel-or-200-or-redirect
   "Respond with the cookie and either a
    transparent pixel, a 200 or a redirect"
-  [cookies duration domain p3p-header pixel vendor params]
+  [cookies duration domain path p3p-header pixel vendor params]
   (let [id      (generate-id cookies)
         cookies (if (= duration 0)
                   {}
-                  {cookie-name (set-cookie id duration domain)})
+                  {cookie-name (set-cookie id duration domain path)})
         headers {"P3P" p3p-header}]
     (if (= vendor "r")
       (send-redirect cookies headers params)

2-collectors/clojure-collector/java-servlet/war-resources/.ebextensions/server.xml

@@ -135,7 +135,7 @@
         <Valve className="com.snowplowanalytics.snowplow.collectors.clojure.SnowplowAccessLogValve" directory="logs"
                prefix="localhost_access_log" suffix=".txt" rotatable="false" requestAttributesEnabled="true"
                encoding="UTF-8"
-               pattern="%{yyyy-MM-dd}t&#9;%{HH:mm:ss}t&#9;-&#9;%b&#9;%a&#9;%m&#9;%h&#9;%U&#9;%s&#9;%{Referer}i&#9;%{User-Agent}I&#9;%q&amp;cv=clj-2.0.0-%v&amp;nuid=%{sp}C&#9;-&#9;-&#9;-&#9;%~&#9;%w" />
+               pattern="%{yyyy-MM-dd}t&#9;%{HH:mm:ss}t&#9;-&#9;%b&#9;%a&#9;%m&#9;%h&#9;%U&#9;%s&#9;%{Referer}i&#9;%{User-Agent}I&#9;%q&amp;cv=clj-2.1.0-%v&amp;nuid=%{sp}C&#9;-&#9;-&#9;-&#9;%~&#9;%w" />
 
 
       </Host>

3-enrich/emr-etl-runner/Gemfile

@@ -25,7 +25,7 @@ gem "avro", "~> 1.8.1"
 gem "awrence", "~> 0.1.0"
 gem "snowplow-tracker", "~> 0.5.2"
 gem "iglu-ruby-client", ">= 0.1.0"
-gem "sluice", "~> 0.4.0"
+gem "aws-sdk-s3", "~> 1"
 gem "diplomat", "~> 2.0.1"
 
 group :development do

3-enrich/emr-etl-runner/Gemfile.lock

@@ -1,12 +1,26 @@
 GEM
   remote: https://rubygems.org/
   specs:
-    CFPropertyList (2.3.5)
     addressable (2.5.0)
       public_suffix (~> 2.0, >= 2.0.2)
     avro (1.8.1)
       multi_json
     awrence (0.1.0)
+    aws-eventstream (1.0.0)
+    aws-partitions (1.89.1)
+    aws-sdk-core (3.21.2)
+      aws-eventstream (~> 1.0)
+      aws-partitions (~> 1.0)
+      aws-sigv4 (~> 1.0)
+      jmespath (~> 1.0)
+    aws-sdk-kms (1.5.0)
+      aws-sdk-core (~> 3)
+      aws-sigv4 (~> 1.0)
+    aws-sdk-s3 (1.13.0)
+      aws-sdk-core (~> 3, >= 3.21.2)
+      aws-sdk-kms (~> 1)
+      aws-sigv4 (~> 1.0)
+    aws-sigv4 (1.0.2)
     builder (3.2.3)
     contracts (0.11.0)
     coveralls (0.8.19)
@@ -26,67 +40,24 @@ GEM
       fog-aws (~> 1.0)
       rest-client (~> 1.0)
       unf (~> 0.1)
-    excon (0.52.0)
+    excon (0.62.0)
     faraday (0.12.2)
       multipart-post (>= 1.2, < 3)
-    fission (0.5.0)
-      CFPropertyList (~> 2.2)
-    fog (1.25.0)
-      fog-brightbox (~> 0.4)
-      fog-core (~> 1.25)
-      fog-json
-      fog-profitbricks
-      fog-radosgw (>= 0.0.2)
-      fog-sakuracloud (>= 0.0.4)
-      fog-softlayer
-      fog-terremark
-      fog-vmfusion
-      fog-voxel
-      fog-xml (~> 0.1.1)
-      ipaddress (~> 0.5)
-      nokogiri (~> 1.5, >= 1.5.11)
-      opennebula
-    fog-aws (1.4.0)
+    fog-aws (1.4.1)
       fog-core (~> 1.38)
       fog-json (~> 1.0)
       fog-xml (~> 0.1)
       ipaddress (~> 0.8)
-    fog-brightbox (0.11.0)
-      fog-core (~> 1.22)
-      fog-json
-      inflecto (~> 0.0.2)
-    fog-core (1.42.0)
+    fog-core (1.45.0)
       builder
-      excon (~> 0.49)
+      excon (~> 0.58)
       formatador (~> 0.2)
     fog-json (1.0.2)
       fog-core (~> 1.0)
       multi_json (~> 1.10)
-    fog-profitbricks (3.0.0)
-      fog-core (~> 1.42)
-      fog-json (~> 1.0)
-    fog-radosgw (0.0.5)
-      fog-core (>= 1.21.0)
-      fog-json
-      fog-xml (>= 0.0.1)
-    fog-sakuracloud (1.7.5)
-      fog-core
-      fog-json
-    fog-softlayer (1.1.4)
-      fog-core
-      fog-json
-    fog-terremark (0.1.0)
-      fog-core
-      fog-xml
-    fog-vmfusion (0.1.0)
-      fission
+    fog-xml (0.1.3)
       fog-core
-    fog-voxel (0.1.0)
-      fog-core
-      fog-xml
-    fog-xml (0.1.2)
-      fog-core
-      nokogiri (~> 1.5, >= 1.5.11)
+      nokogiri (>= 1.5.11, < 2.0.0)
     formatador (0.2.5)
     http-cookie (1.0.3)
       domain_name (~> 0.5)
@@ -95,31 +66,21 @@ GEM
     iglu-ruby-client (0.1.0)
       httparty (<= 0.14.0)
       json-schema (~> 2.7.0, >= 2.7.0)
-    inflecto (0.0.2)
     ipaddress (0.8.3)
-    jruby-jars (9.1.7.0)
-    jruby-rack (1.1.20)
+    jmespath (1.4.0)
+    jruby-jars (9.2.0.0)
+    jruby-rack (1.1.21)
     json (2.0.3-java)
     json-schema (2.7.0)
       addressable (>= 2.4)
     mime-types (2.99.3)
-    multi_json (1.12.1)
+    multi_json (1.13.1)
     multi_xml (0.6.0)
     multipart-post (2.0.0)
-    net-ssh (2.9.4)
     netrc (0.11.0)
-    nokogiri (1.7.0.1-java)
-    opennebula (5.4.0)
-      json
-      nokogiri
-      rbvmomi
+    nokogiri (1.8.2-java)
     public_suffix (2.0.5)
-    rake (12.0.0)
-    rbvmomi (1.11.3)
-      builder (~> 3.0)
-      json (>= 1.8)
-      nokogiri (~> 1.5)
-      trollop (~> 2.1)
+    rake (12.3.1)
     rest-client (1.8.0)
       http-cookie (>= 1.0.2, < 2.0)
       mime-types (>= 1.16, < 3.0)
@@ -143,19 +104,14 @@ GEM
       json (>= 1.8, < 3)
       simplecov-html (~> 0.10.0)
     simplecov-html (0.10.0)
-    sluice (0.4.0)
-      contracts (~> 0.9)
-      fog (= 1.25)
-      net-ssh (~> 2.9.2)
     snowplow-tracker (0.5.2)
       contracts (~> 0.7, <= 0.11)
     term-ansicolor (1.4.0)
       tins (~> 1.0)
     thor (0.19.4)
     tins (1.13.2)
-    trollop (2.1.2)
     unf (0.1.4-java)
-    warbler (2.0.4)
+    warbler (2.0.5)
       jruby-jars (>= 9.0.0.0)
       jruby-rack (>= 1.1.1, < 1.3)
       rake (>= 10.1.0)
@@ -167,18 +123,18 @@ PLATFORMS
 DEPENDENCIES
   avro (~> 1.8.1)
   awrence (~> 0.1.0)
+  aws-sdk-s3 (~> 1)
   contracts (~> 0.9, <= 0.11)
   coveralls
   diplomat (~> 2.0.1)
   elasticity (~> 6.0.12)
   iglu-ruby-client (>= 0.1.0)
   rspec (~> 3.5.0)
-  sluice (~> 0.4.0)
   snowplow-tracker (~> 0.5.2)
   warbler
 
 RUBY VERSION
    ruby 2.3.1p0 (jruby 9.1.6.0)
 
 BUNDLED WITH
-   1.15.3
+   1.15.4

3-enrich/emr-etl-runner/config/config.yml.sample

@@ -8,6 +8,7 @@ aws:
       assets: s3://snowplow-hosted-assets # DO NOT CHANGE unless you are hosting the jarfiles etc yourself in your own bucket
       jsonpath_assets: # If you have defined your own JSON Schemas, add the s3:// path to your own JSON Path files in your own bucket here
       log: ADD HERE
+      encrypted: false # Whether the buckets below are enrcrypted using server side encryption (SSE-S3)
       raw:
         in:                  # This is a YAML array of one or more in buckets - you MUST use hyphens before each entry in the array, as below
           - ADD HERE         # e.g. s3://my-old-collector-bucket
@@ -32,6 +33,7 @@ aws:
     placement: ADD HERE     # Set this if not running in VPC. Leave blank otherwise
     ec2_subnet_id: ADD HERE # Set this if running in VPC. Leave blank otherwise
     ec2_key_name: ADD HERE
+    security_configuration: ADD HERE # Specify your EMR security configuration if needed. Leave blank otherwise
     bootstrap: []           # Set this to specify custom boostrap actions. Leave empty otherwise
     software:
       hbase:                # Optional. To launch on cluster, provide version, "0.92.0", keep quotes. Leave empty otherwise.
@@ -75,5 +77,7 @@ monitoring:
     level: DEBUG # You can optionally switch to INFO for production
   snowplow:
     method: get
+    protocol: http
+    port: 80
     app_id: ADD HERE # e.g. snowplow
     collector: ADD HERE # e.g. d3rkrsqld9gmqf.cloudfront.net

3-enrich/emr-etl-runner/config/stream_config.yml.sample

@@ -8,6 +8,7 @@ aws:
       assets: s3://snowplow-hosted-assets # DO NOT CHANGE unless you are hosting the jarfiles etc yourself in your own bucket
       jsonpath_assets: # If you have defined your own JSON Schemas, add the s3:// path to your own JSON Path files in your own bucket here
       log: ADD HERE
+      encrypted: false
       enriched:
         good: ADD HERE       # e.g. s3://my-out-bucket/enriched/good
         archive: ADD HERE    # Where to archive enriched events to, e.g. s3://my-archive-bucket/enriched
@@ -25,6 +26,7 @@ aws:
     placement: ADD HERE     # Set this if not running in VPC. Leave blank otherwise
     ec2_subnet_id: ADD HERE # Set this if running in VPC. Leave blank otherwise
     ec2_key_name: ADD HERE
+    security_configuration: ADD HERE # Specify your EMR security configuration if needed. Leave blank otherwise
     bootstrap: []           # Set this to specify custom boostrap actions. Leave empty otherwise
     software:
       hbase:                # Optional. To launch on cluster, provide version, "0.92.0", keep quotes. Leave empty otherwise.
@@ -67,3 +69,5 @@ monitoring:
     method: get
     app_id: ADD HERE # e.g. snowplow
     collector: ADD HERE # e.g. d3rkrsqld9gmqf.cloudfront.net
+    protocol: http
+    port: 80

3-enrich/emr-etl-runner/lib/snowplow-emr-etl-runner.rb

@@ -21,6 +21,7 @@
 require_relative 'snowplow-emr-etl-runner/monitoring/snowplow'
 require_relative 'snowplow-emr-etl-runner/cli'
 require_relative 'snowplow-emr-etl-runner/job_result'
+require_relative 'snowplow-emr-etl-runner/s3'
 require_relative 'snowplow-emr-etl-runner/emr_job'
 require_relative 'snowplow-emr-etl-runner/lock/lock'
 require_relative 'snowplow-emr-etl-runner/lock/file_lock'
@@ -35,6 +36,6 @@
 module Snowplow
   module EmrEtlRunner
     NAME    = "snowplow-emr-etl-runner"
-    VERSION = "0.32.0"
+    VERSION = "0.33.0"
   end
 end