Merge remote-tracking branch 'upstream/master'

.dockerignore

@@ -1 +1,2 @@
 .git
+target

.editorconfig

@@ -0,0 +1,5 @@
+[*.java]
+charset=utf-8
+end_of_line=lf
+insert_final_newline=true
+indent_style=tab

.travis.yml

@@ -25,3 +25,9 @@ env:
       MYSQL_VERSION=5.6
     - SHYKO_MODE=1
       MYSQL_VERSION=5.7
+    - SHYKO_MODE=1
+      GTID_MODE=1
+      MYSQL_VERSION=5.6
+    - SHYKO_MODE=1
+      GTID_MODE=1
+      MYSQL_VERSION=5.7

CHANGELOG.md

@@ -1,19 +1,55 @@
 # Maxwell changelog
 
+### [v1.9.0](https://github.com/zendesk/maxwell/releases/tag/v1.9.0): "now with added whimsy"
+
+Maxwell 1.9 adds one main feature: monitoring support, contributed by
+Scott Ferguson. Multiple backends can be configured, read the updated
+docs for full details.
+
+There's also some bugfixes:
+
+- filter DDL messages based on config
+- determine newest schema from binlog order, not creation order
+- add task manager to shutdown cleanly on error
+- minor logging improvements
+
+
+### [v1.8.2](https://github.com/zendesk/maxwell/releases/tag/v1.8.2): "just as the postcards wept"
+
+
+Bugfix release.
+
+- maxwell would crash on a quoted partition name
+- fixes for alters on non-string tables containing VARCHAR
+- use seconds instead of milliseconds for DDL messages
+
+
+### [v1.8.1](https://github.com/zendesk/maxwell/releases/tag/v1.8.1): "famous is faster, don't have to be talented"
+
+
+- performance improves in capturing and restoring schema, thx Joren
+  Minnaert
+- Allow for capturing from a separate mysql host (adds support for using
+  Maxscale as a replication proxy), thx Adam Szkoda
+
+### [v1.8.0](https://github.com/zendesk/maxwell/releases/tag/v1.8.0): "upbeat, honest, contradictory"
+
+
+In version 1.8.0 Maxwell gains alpha support for GTID-based positions!
+All praise due to Henry Cai.
+
 ### [v1.7.2](https://github.com/zendesk/maxwell/releases/tag/v1.7.2): "comparing self to better"
 
 
 - Fix a bug found where maxwell could cache the wrong TABLE_MAP_ID for a
-binlog event, leading to crashes or in some cases data mismatches.
-
+  binlog event, leading to crashes or in some cases data mismatches.
 
 ### [v1.7.1](https://github.com/zendesk/maxwell/releases/tag/v1.7.1): "blame it on your seratonin"
 
 
 - bootstrapping now can take a `--where` clause
 - performance improvements in the kafka producer
 
-
 ### [v1.7.0](https://github.com/zendesk/maxwell/releases/tag/v1.7.0): "lucky me, lucky mud"
 
 

Dockerfile

@@ -1,5 +1,5 @@
 FROM java:openjdk-7
-ENV MAXWELL_VERSION 1.7.2
+ENV MAXWELL_VERSION 1.8.2
 ENV KAFKA_VERSION 0.10.1.0
 
 RUN apt-get update && apt-get -y upgrade

Makefile

@@ -81,6 +81,9 @@ test%: compile-test
 
 test: compile-test
 	java -classpath $(JAVA_TEST_DEPENDS):target/test-classes:target/classes org.junit.runner.JUnitCore $(TEST_CLASSES)
+test-only: compile-test
+	java -classpath $(JAVA_TEST_DEPENDS):target/test-classes:target/classes org.junit.runner.JUnitCore ${ARGS}
+
 clean:
 	rm -f  target/.java target/.java-test
 	rm -rf target/classes

README.md

@@ -6,7 +6,8 @@ row updates to Kafka as JSON.  Maxwell has a low operational bar and produces a
 consistent, easy to ingest stream of updates.  It allows you to easily "bolt
 on" some of the benefits of stream processing systems without going through your
 entire code base to add (unreliable) instrumentation points.  Common use cases
-include ETL, cache building/expiring, metrics collection, and search indexing.
+include ETL, cache building/expiring, metrics collection, search indexing and
+inter-service communication.
 
 advanced features:
 
@@ -17,7 +18,7 @@ advanced features:
   parser for create/alter/drop statements (nope, there was no other way).
 
 → Download:
-[https://github.com/zendesk/maxwell/releases/download/v1.7.2/maxwell-1.7.2.tar.gz](https://github.com/zendesk/maxwell/releases/download/v1.7.2/maxwell-1.7.2.tar.gz)
+[https://github.com/zendesk/maxwell/releases/download/v1.8.2/maxwell-1.8.2.tar.gz](https://github.com/zendesk/maxwell/releases/download/v1.8.2/maxwell-1.8.2.tar.gz)
 <br/>
 &rarr; Source:
 [https://github.com/zendesk/maxwell](https://github.com/zendesk/maxwell)

build/Gemfile

@@ -1,4 +1,5 @@
 source 'https://rubygems.org'
 gem 'octokit'
 gem 'netrc'
+gem 'mime-types'
 

build/Gemfile.lock

@@ -4,6 +4,9 @@ GEM
     addressable (2.3.8)
     faraday (0.9.2)
       multipart-post (>= 1.2, < 3)
+    mime-types (3.1)
+      mime-types-data (~> 3.2015)
+    mime-types-data (3.2016.0521)
     multipart-post (2.0.0)
     netrc (0.11.0)
     octokit (4.2.0)
@@ -16,8 +19,9 @@ PLATFORMS
   ruby
 
 DEPENDENCIES
+  mime-types
   netrc
   octokit
 
 BUNDLED WITH
-   1.10.6
+   1.13.7

build/release

@@ -30,10 +30,11 @@ end
 
 def bump
   do_system(BINDIR + "/bump", $version)
+  do_system("/bin/bash -i")
 end
 
 def build
-  do_system("make package MAXWELL_VERSION=#{$version}")
+  do_system("make -C #{BINDIR}/.. package MAXWELL_VERSION=#{$version}")
 
   commit_template = Tempfile.new('maxwell_build')
   commit_template.write <<-EOL
@@ -48,7 +49,7 @@ def build
   commit_template.close
 
   do_system("git commit -av -t #{commit_template.path}")
-  do_system("git push")
+  do_system("git push origin HEAD:master")
 end
 
 def tag

config.properties.example

@@ -173,6 +173,42 @@ kinesis_stream=maxwell
 #blacklist_tables=table1,table_no
 #
 
+######## monitoring stuff ###########
+
+# Maxwell collects metrics via dropwizard. These can be exposed through the
+# base logging mechanism (slf4j), JMX, HTTP or pushed to Datadog.
+# Options: [jmx, slf4j, http, datadog]
+# Supplying multiple is allowed.
+#metrics_type=jmx,slf4j
+
+# The prefix maxwell will apply to all metrics
+#metrics_prefix=MaxwellMetrics # default MaxwellMetrics
+
+# When metrics_type includes slf4j this is the frequency metrics are emitted to the log, in seconds
+#metrics_slf4j_interval=60
+
+# When metrics_type includes http this is the port the server will bind to.
+#metrics_http_port=8080
+
+# ** The following are Datadog specific. **
+# When metrics_type includes datadog this is the way metrics will be reported.
+# Options: [udp, http]
+# Supplying multiple is not allowed.
+#metrics_datadog_type=udp
+
+# datadog tags that should be supplied
+#metrics_datadog_tags=tag1:value1,tag2:value2
+
+# The frequency metrics are pushed to datadog, in seconds
+#metrics_datadog_interval=60
+
+# required if metrics_datadog_type = http
+#metrics_datadog_apikey=API_KEY
+
+# required if metrics_datadog_type = udp
+#metrics_datadog_host=localhost # default localhost
+#metrics_datadog_port=8125 # default 8125
+
 
 ######### misc stuff ###############
 

docs/docs/bootstrapping.md

@@ -59,8 +59,13 @@ Corresponding replication stream output of table `fooDB.barTable`:
 {"database":"fooDB","table":"barTable","type":"bootstrap-complete","ts":1450557744,"data":{}}
 ```
 
+### Failure Scenarios
+If Maxwell crashes during bootstrapping the next time it runs it will rerun the bootstrap in its entirety - regardless of previous progress.
+If this behavior is not desired, manual updates to the `bootstrap` table are required.
+Specifically, marking the unfinished bootstrap row as 'complete' (`is_complete` = 1) or deleting the row.
+
 <script>
   jQuery(document).ready(function () {
     jQuery("table").addClass("table table-condensed table-bordered table-hover");
   });
-</script>
+</script>

docs/docs/compat.md

@@ -1,14 +1,9 @@
 ### Requirements:
 ***
 - JRE 7 or above
-- mysql 5.1, 5.5, 5.6, 5.7 (alpha)
+- mysql 5.1, 5.5, 5.6, 5.7
 - kafka 0.8.2 or greater
 
-### Mysql 5.7
-***
-- Mysql 5.7 support is considered alpha-quality at the moment.  JSON columns
-  are supported, but GTID replication is not as of yet.
-
 ### binlog_row_image=MINIMAL
 ***
 As of 0.16.2, Maxwell supports binlog_row_image=MINIMAL, but it may not be what you want.  It will differ
@@ -39,6 +34,6 @@ Notes:
   replication-hosts, due to the heartbeat mechanism.
 - this code should be considered alpha-quality.
 - on highly active servers, as much as 1 second of data may be duplicated.
-
+- master recovery is not available in GTID-mode.