Support: keep continue on error

README.md

@@ -218,8 +218,9 @@ Usage: es-import-bulk [options]
     -v, --version          output the version number
     -u, --url <url>        the elasticsearch url to connect to
     -f, --file <file>      the file to read data from
-    -m, --max <items>      the max number of lines to process per batch (default: 20,000) (default: 20000)
-    --requestTimeout <ms>  ES CLIENT OPTION: milliseconds before an HTTP request will be aborted and retried. This can also be set per request (default: 30000) (default: 30000)
+    -m, --max <items>      the max number of lines to process per batch (default: 20,000)
+    --requestTimeout <ms>  ES CLIENT OPTION: milliseconds before an HTTP request will be aborted and retried. This can also be set per request (default: 30,000)
+    -i, --ignore-error     ignore error when importing lines, print it to stderr. This allows us to manually handle failed documents after importing
     -h, --help             output usage information
 ```
 

lib/es-import-bulk.js

@@ -35,6 +35,7 @@ program
     helpers.integer,
     30000
   )
+  .option('-i, --ignore-error', 'Ignore error when importing lines, print it to stderr. This allows us to manually handle failed documents after importing')
   .parse(process.argv);
 
 // validate url and file
@@ -60,11 +61,23 @@ var client = new elasticsearch.Client({
 var bulkImport = function(cb) {
   client.bulk({ body: currentBatch }, function(err, response) {
     if (err) {
-      helpers.exit(err);
+      if (program.ignoreError) {
+        helpers.logError(err);
+      }
+      else {
+        helpers.exit(err);
+      }
     }
-    if (response.error) {
-      helpers.exit('When executing bulk query: ' + response.error.toString());
+    else if (response.error) {
+      const errorMessage = 'When executing bulk query: ' + response.error.toString();
+      if (program.ignoreError) {
+        helpers.logError(errorMessage);
+      }
+      else {
+        helpers.exit(errorMessage);
+      }
     }
+
     // reset global variables
     currentCount = 0;
     currentBatch = '';

lib/helpers.js

@@ -1,9 +1,13 @@
 // a few helper functions
 'use strict';
 
-exports.exit = function(err) {
+exports.logError = function(err) {
   var str = JSON.stringify(err).replace(/^error:/i, '');
   console.error('\n  error: ' + str);
+};
+
+exports.exit = function(err) {
+  exports.logError(err);
   process.exit(1);
 };