WIP: shaarli_api: add skip_existing: True/False option (don't overwri…

…te items with already existing URL)

hecat/importers/shaarli_api.py

@@ -11,6 +11,7 @@
     module_options:
       source_file: shaarli.json
       output_file: shaarli.yml
+      skip_existing: True # optional, default True, skip importing items whose 'url:' already exists in the output file
 
 Source directory structure:
 └── shaarli.json
@@ -19,17 +20,29 @@
 └── shaarli.yml
 """
 
+import os
 import logging
 import ruamel.yaml
 import json
+from ..utils import load_yaml_data
 
 yaml = ruamel.yaml.YAML()
 yaml.indent(sequence=2, offset=0)
 
 def import_shaarli_json(step):
     """Import data from the JSON output of Shaarli API"""
+    if 'skip_existing' not in step['module_options']:
+        step['module_options']['skip_existing'] = True
     with open(step['module_options']['source_file'], 'r', encoding="utf-8") as json_file:
         data = json.load(json_file)
-    with open(step['module_options']['output_file'], 'w+', encoding="utf-8") as yaml_file:
-        logging.debug('writing file %s', step['module_options']['output_file'])
-        yaml.dump(data, yaml_file)
+    if os.path.exists(step['module_options']['output_file']) and step['module_options']['skip_existing']:
+        logging.info('loading existing data from %s', step['module_options']['output_file'])
+        previous_data = load_yaml_data(step['module_options']['output_file'])
+        final_data = sorted({x["url"]: x for x in (data + previous_data)}.values(), key=lambda x: x["url"])
+        with open(step['module_options']['output_file'], 'w+', encoding="utf-8") as yaml_file:
+            logging.debug('writing file %s', step['module_options']['output_file'])
+            yaml.dump(final_data, yaml_file)
+    else:
+        with open(step['module_options']['output_file'], 'w+', encoding="utf-8") as yaml_file:
+            logging.debug('writing file %s', step['module_options']['output_file'])
+            yaml.dump(data, yaml_file)

tests/.hecat.import_shaarli.yml

@@ -4,3 +4,4 @@ steps:
     module_options:
       source_file: tests/shaarli.json
       output_file: tests/shaarli.yml
+      skip_existing: True # optional, default True