Skip to content

Commit

Permalink
Merge branch 'release/0.5.0'
Browse files Browse the repository at this point in the history
  • Loading branch information
BenFradet authored Jul 7, 2017
2 parents bdb6ca5 + 57585cf commit bb57bca
Show file tree
Hide file tree
Showing 29 changed files with 428 additions and 587 deletions.
42 changes: 42 additions & 0 deletions .release.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
# Required: local settings
local:
root_dir : <%= ENV['TRAVIS_BUILD_DIR'] %>

# Required: deployment targets
targets:
- type : "bintray"
user : <%= ENV['BINTRAY_SNOWPLOW_GENERIC_USER'] %>
password : <%= ENV['BINTRAY_SNOWPLOW_GENERIC_API_KEY'] %>

# Required: packages to be deployed
packages:
- repo : "snowplow-generic"
name : "kinesis-s3"
user_org : "snowplow"
publish : true

# Will attempt to overwrite a published entity if one is found
override : false

# If the artifact already exists will determine whether or not to fail the release
continue_on_conflict : false

# The version of this package
version : <%= ENV['TRAVIS_TAG'] %>

# Required IF '--check-version' is passed: will assert that both versions are the same
build_version : <%= FUNC['sbt_version(.)'] %>

build_commands:
- sbt assembly

# Required: Artifact
artifacts:
# The artifact name is composed like so: {{prefix}}{{version}}{{suffix}}.zip
- prefix : "kinesis_s3_"
suffix : ""
type : "zip"

# The binaries to put in the zip
binary_paths:
- target/scala-2.11/snowplow-kinesis-s3-<%= ENV['TRAVIS_TAG'] %>
19 changes: 16 additions & 3 deletions .travis.yml
Original file line number Diff line number Diff line change
@@ -1,8 +1,21 @@
language: scala
scala:
- 2.10.1
- 2.11.11
jdk:
- oraclejdk7
- oraclejdk8

before_script:
- echo yes | sudo apt-get install lzop liblzo2-dev
- sudo apt-get install -y lzop liblzo2-dev

before_deploy: pip install --user release-manager==0.2.0
deploy:
- provider: script
script: release-manager --config .release.yml --make-artifact --upload-artifact --check-version
skip_cleanup: true
on:
tags: true

env:
global:
- secure: hJTWoOYgFMNaTZZ8gpIVl+SZQS5INschfkUUReuGXAzifaXTvT8J5a/aGryJqEOh/zMEOhKRZn136aHzMMnMDxoOkO4uSXnHnhi61DZHsbaZ51g0YvIjnJyS49ad+IKoIFJlcTKAMjvd+HK3TOHkkcWA/cc7EJWYbYwPEu3xfpkick6Y2+oQd3LYANS4wmTcVYMYKF08teQW3w0XqErHrxb7prIz7lEF8uC9BHHmUYfGEvdVLmphpsP/jJBPB+TB5IRwarEyEBaITEcVL/qeDRM3FxhErY6JI9omT6QRevO9dHx2mfpQFITCjeRf9IjJBiSr7fGVGUusxJSRqg1gMbGyImwUX4a13PaV0ACL1OUHCC3hQa3Rmoh0ObQBJ37rBvkyNKuEjJmt2bX7BMC5Ck7duJFNJhGqWiFpHyZV4MM8oYn+29OcJ7Ju03Qr6MXt04+iD3CjctHI9cbCe4mYuYJcyFYi847PaLicNqDg3Cj2Q2m8KzE9guBSY2PCUvxxFLR0LGFhXXtQYg2LxrAn8pnBE5QNVr6LQ9aoeQLYUscBCjMFyVNpciRLt/IPvzZRGL3vKiKwgJqxaV7CE1LXoTbIUTwCB8jnsqlSOPJbJQCciHA31GlFKz/m7kRRkI8G0Geie7leWjM6G7Pfr3aKXKpTw7HkN9F87AYJ47/YcFU=
- secure: LdhTiGwDK5BivJ+/j4CLCMNGSkWN4WiTQRGGrFf2L7wSI2fdOtZTQUcKVCZolvyd6V296BLbPYxgoXs1hdDCQb/jyoQCiWDC54XuJun4gGiKko04ZY7/UCUDQ2oo6995+UcKlvQKhqL9mvGT6stuBIUefB/r+/dQhwR+L9ehFlAiOXDFEzRh3BYgIZDnbJbMvjRptzqz/d88iC7un9e+wf+F8qyObjwJ+7RrL6rddvXg1HUyv0Kt8Yv16uSlFJPuY3fyTqDmiGbm2bYqM7cUVGRokTUOwqwtBhW/uJ071r6SlTxpTNzTJARRV2KvIT+cMvNJflvFZLOL7A55fKfpTDxWCsXcbKu3D842lppqlGH4O+GM5G3+fArSnjHmFhVRwhNMsHlGG/brXHerhCPGb2+hkIaIg/sopWP+OoEWZvlRUihv9nOLTCCZy4F45dz/B38XRM9vXZscw1nX5uu/f4LvRtOIsinOHpOCFWtxloh8qWGarcvj0Q6Ix4f/rb4L/ZIN5PthZJVEswchSK9Ee4b6Pt0EIsM2gEPjdQ5RXvYb0IWRVxbgJH7ljtk+WxHDfr67gvLAzrD2X2bKQCp+mJ3xGdiH+Ddu1r8Pb/Z6g4icFAxBWQYaSuw2etvNNGcH+Y77R7Q+Q5Hbp8omXXBj243uje/zZpNwlEeoaVbOms4=
29 changes: 29 additions & 0 deletions CHANGELOG
Original file line number Diff line number Diff line change
@@ -1,3 +1,32 @@
Version 0.5.0 (2017-06-30)
--------------------------
Add a newline at the end of a gzipped tsv file (#49)
Resolve environment variables in hocon (#67)
Add support for the Chinese Kinesis endpoint (#71)
Do not dismiss failures prior to serialization (#101)
Reduce INFO logging (#50)
Bump Scala version to 2.11 (#81)
Upgrade to Java 8 (#82)
Bump SBT to 0.13.15 (#83)
Bump Kinesis Client Library to 1.7.5 (#55)
Update amazon-kinesis-connectors to 1.3.0 (#17)
Bump hadoop-common to 2.7.3 (#94)
Bump elephant-bird to 4.15 (#95)
Bump hadoop-lzo to 0.4.20 (#93)
Bump scalaz to 7.0.9 (#86)
Bump sbt-assembly to 0.14.5 (#84)
Bump joda-time to 2.9.9 (#96)
Bump config to 1.3.1 (#97)
Bump snowplow-scala-tracker to 0.3.0 (#92)
Bump specs2-core to 3.9.1 (#89)
Replace argot by scopt (#85)
Remove scalazon dependency (#88)
Remove joda-convert dependency (#98)
Remove scalaz-specs2 dependency (#90)
Update README markdown in according with CommonMark (#76)
Add CI/CD to project (#79)
Add Bintray credentials to .travis.yml (#100)

Version 0.4.1 (2016-06-01)
--------------------------
Crash when unable to find stream instead of hanging (#58)
Expand Down
31 changes: 14 additions & 17 deletions README.md
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
# Kinesis S3

[ ![Build Status] [travis-image] ] [travis]
[ ![Release] [release-image] ] [releases]
[ ![License] [license-image] ] [license]
[![Build Status][travis-image]][travis]
[![Release][release-image]][releases]
[![License][license-image]][license]

## Overview

Expand All @@ -24,7 +24,7 @@ The records are treated as byte arrays containing UTF-8 encoded strings (whether

## Quickstart

Assuming git, **[Vagrant] [vagrant-install]** and **[VirtualBox] [virtualbox-install]** installed:
Assuming git, **[Vagrant][vagrant-install]** and **[VirtualBox][virtualbox-install]** installed:

```bash
host$ git clone https://github.com/snowplow/kinesis-s3.git
Expand All @@ -49,14 +49,11 @@ NOTE: These are already installed in the Vagrant quickstart environment.
The Kinesis S3 LZO Sink has the following command-line interface:

```
snowplow-kinesis-s3: Version 0.4.1. Copyright (c) 2014-2015, Snowplow Analytics
Ltd.
snowplow-kinesis-s3: Version 0.5.0
Usage: snowplow-lzo-s3-sink [OPTIONS]
Usage: snowplow-kinesis-s3 [options]
OPTIONS
--config filename
Configuration file.
--config <filename>
```

## Running
Expand All @@ -78,16 +75,16 @@ guest$ sbt "run --config my.conf"

## Find out more

| Technical Docs | Setup Guide | Roadmap & Contributing |
|-----------------------------|-----------------------|--------------------------------------|
| ![i1] [techdocs-image] | ![i2] [setup-image] | ![i3] [roadmap-image] |
| [Technical Docs] [techdocs] | [Setup Guide] [setup] | _coming soon_ |
| Technical Docs | Setup Guide | Roadmap & Contributing |
|----------------------------|----------------------|------------------------|
| ![i1][techdocs-image] | ![i2][setup-image] | ![i3][roadmap-image] |
| [Technical Docs][techdocs] | [Setup Guide][setup] | _coming soon_ |

## Copyright and license

Kinesis S3 is copyright 2014-2015 Snowplow Analytics Ltd.
Kinesis S3 is copyright 2014-2017 Snowplow Analytics Ltd.

Licensed under the [Apache License, Version 2.0] [license] (the "License");
Licensed under the [Apache License, Version 2.0][license] (the "License");
you may not use this software except in compliance with the License.

Unless required by applicable law or agreed to in writing, software
Expand All @@ -99,7 +96,7 @@ limitations under the License.
[travis-image]: https://travis-ci.org/snowplow/kinesis-s3.png?branch=master
[travis]: http://travis-ci.org/snowplow/kinesis-s3

[release-image]: http://img.shields.io/badge/release-0.4.1-blue.svg?style=flat
[release-image]: http://img.shields.io/badge/release-0.5.0-blue.svg?style=flat
[releases]: https://github.com/snowplow/kinesis-s3/releases

[license-image]: http://img.shields.io/badge/license-Apache--2-blue.svg?style=flat
Expand Down
46 changes: 46 additions & 0 deletions build.sbt
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
/*
* Copyright (c) 2014-2017 Snowplow Analytics Ltd. All rights reserved.
*
* This program is licensed to you under the Apache License Version 2.0,
* and you may not use this file except in compliance with the Apache License Version 2.0.
* You may obtain a copy of the Apache License Version 2.0 at
* http://www.apache.org/licenses/LICENSE-2.0.
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the Apache License Version 2.0 is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the Apache License Version 2.0 for the specific language governing permissions and
* limitations there under.
*/
lazy val root = project.in(file("."))
.settings(
name := "snowplow-kinesis-s3",
version := "0.5.0",
description := "Kinesis sink for S3"
)
.settings(BuildSettings.buildSettings)
.settings(BuildSettings.sbtAssemblySettings)
.settings(
libraryDependencies ++= Seq(
// Java
Dependencies.Libraries.kinesisClient,
Dependencies.Libraries.kinesisConnector,
Dependencies.Libraries.slf4j,
Dependencies.Libraries.hadoop,
Dependencies.Libraries.elephantbird,
Dependencies.Libraries.hadoopLZO,
Dependencies.Libraries.jodaTime,
// Scala
Dependencies.Libraries.scopt,
Dependencies.Libraries.config,
Dependencies.Libraries.scalaz7,
Dependencies.Libraries.json4sJackson,
Dependencies.Libraries.snowplowTracker,
// Scala (test only)
Dependencies.Libraries.specs2,
// Thrift (test only)
Dependencies.Libraries.collectorPayload
)
)

shellPrompt := { _ => "kinesis-s3> " }
Binary file removed lib/hadoop-lzo-0.4.20-SNAPSHOT.jar
Binary file not shown.
96 changes: 50 additions & 46 deletions project/BuildSettings.scala
Original file line number Diff line number Diff line change
Expand Up @@ -18,66 +18,70 @@ import Keys._
object BuildSettings {

// Basic settings for our app
lazy val basicSettings = Seq[Setting[_]](
lazy val basicSettings = Seq(
organization := "com.snowplowanalytics",
version := "0.4.1",
description := "Kinesis LZO sink for S3",
scalaVersion := "2.10.1",
scalacOptions := Seq("-deprecation", "-encoding", "utf8",
"-feature", "-target:jvm-1.7"),
scalacOptions in Test := Seq("-Yrangepos"),
resolvers ++= Dependencies.resolutionRepos
scalaVersion := "2.11.11",
scalacOptions := compilerOptions,
javacOptions := javaCompilerOptions,
resolvers ++= Dependencies.resolvers
)

lazy val compilerOptions = Seq(
"-deprecation",
"-encoding", "UTF-8",
"-feature",
"-language:existentials",
"-language:higherKinds",
"-language:implicitConversions",
"-unchecked",
"-Yno-adapted-args",
"-Ywarn-dead-code",
"-Ywarn-numeric-widen",
"-Xfuture",
"-Xlint"
)

lazy val javaCompilerOptions = Seq(
"-source", "1.8",
"-target", "1.8"
)

// Makes our SBT app settings available from within the app
lazy val scalifySettings = Seq(sourceGenerators in Compile <+= (sourceManaged in Compile, version, name, organization) map { (d, v, n, o) =>
val file = d / "settings.scala"
IO.write(file, """package com.snowplowanalytics.snowplow.storage.kinesis.s3.generated
|object Settings {
| val organization = "%s"
| val version = "%s"
| val name = "%s"
|}
|""".stripMargin.format(o, v, n))
Seq(file)
})
lazy val scalifySettings = Seq(
sourceGenerators in Compile += Def.task {
val file = (sourceManaged in Compile).value / "settings.scala"
IO.write(file, """package com.snowplowanalytics.snowplow.storage.kinesis.s3.generated
|object Settings {
| val organization = "%s"
| val version = "%s"
| val name = "%s"
|}
|""".stripMargin.format(organization.value, version.value, name.value))
Seq(file)
}.taskValue
)

lazy val buildSettings = basicSettings ++ scalifySettings

// sbt-assembly settings for building a fat jar
import sbtassembly.Plugin._
import AssemblyKeys._
lazy val sbtAssemblySettings = assemblySettings ++ Seq(
import sbtassembly.AssemblyPlugin.autoImport._
lazy val sbtAssemblySettings = Seq(
// Executable jarfile
assemblyOption in assembly ~= { _.copy(prependShellScript = Some(defaultShellScript)) },
assemblyOption in assembly :=
(assemblyOption in assembly).value.copy(prependShellScript = Some(
Seq("#!/usr/bin/env sh", """exec java -jar "$0" "$@"""" + "\n")
)),
// Name it as an executable
jarName in assembly := { s"${name.value}-${version.value}" },

excludedJars in assembly <<= (fullClasspath in assembly) map { cp =>
val excludes = Set(
"junit-4.8.2.jar",
"jsp-2.1-6.1.14.jar",
"jasper-compiler-5.5.12.jar",
"jsp-api-2.1-6.1.14.jar",
"servlet-api-2.5-6.1.14.jar",
"commons-beanutils-1.7.0.jar",
"hadoop-lzo-0.4.19.jar",
"stax-api-1.0.1.jar",
"commons-collections-3.2.1.jar"
)
cp filter { jar => excludes(jar.data.getName) }
},
assemblyJarName in assembly := { s"${name.value}-${version.value}" },

mergeStrategy in assembly := {
assemblyMergeStrategy in assembly := {
case PathList("javax", "servlet", xs @ _*) => MergeStrategy.first
case PathList("org", "objectweb", "asm", xs @ _*) => MergeStrategy.first
case PathList(ps @ _*) if ps.last endsWith ".html" => MergeStrategy.first
case "application.conf" => MergeStrategy.concat
case x =>
val oldStrategy = (mergeStrategy in assembly).value
val oldStrategy = (assemblyMergeStrategy in assembly).value
oldStrategy(x)
},

assemblyOption in assembly ~= { _.copy(cacheOutput = false) }
}
)

lazy val buildSettings = basicSettings ++ scalifySettings ++ sbtAssemblySettings
}
Loading

0 comments on commit bb57bca

Please sign in to comment.