Remove C++ Event Handling (#22)

* Support --scaled; add saving capability; properly handle saving with errors, SIGINT * add stat_type to DistanceCalc message * Add pluggable Saturation, start on normalized distances * Completely remove c++ layer Event handling, migrate cdbg runner to curio async processor * Add streaming solid k-mer filter * Fix sourmash test * tweak cdbg metric output, have unitig fragmentation output n kmers per bin * misc
camillescott · Apr 7, 2020 · d38cc58 · d38cc58
1 parent a6361b8
commit d38cc58
Show file tree

Hide file tree

Showing 47 changed files with 1,372 additions and 1,615 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -75,7 +75,7 @@ if(NOT CMAKE_BUILD_TYPE)
         FORCE
     )
 endif(NOT CMAKE_BUILD_TYPE)
-set(CMAKE_CXX_FLAGS_RELEASE "-O2")
+set(CMAKE_CXX_FLAGS_RELEASE "-O3")
 set(CMAKE_CXX_FLAGS_DEBUG "-O0")
 
 #

diff --git a/generate_message_schemas.py b/generate_message_schemas.py
@@ -1,67 +1,85 @@
-import schemapi
+from goetia import schemapi
 
 msg_schema = {
     'definitions': {
         'Interval': {
-            "type": "object",
-            "required": ["msg_type", "t", "state", 'sample_name'],
+            'type': 'object',
+            'required': ['msg_type', 't', 'state', 'sample_name', 'file_names'],
             'properties': {
                 'msg_type': {'type': 'string',
                              'const': 'Interval',
                              'default': 'Interval'},
                 't': {'type': 'integer',
                        'minimum': 0},
-                'state': {'type': 'string',
-                          'enum': ['fine', 'medium', 'coarse']}
+                'state': {'type': 'array',
+                          'uniqueItems': True},
+                'file_names': {'type': 'array'}
             }
         },
         'SampleStarted': {
-            "type": "object",
-            "required": ["msg_type", "sample_name"],
-            "properties": {
-                "msg_type": {"type": "string",
-                             "enum": ['SampleStarted']},
-                "sample_name": {"type": "string"}
+            'type': 'object',
+            'required': ['msg_type', 'sample_name', 'file_names'],
+            'properties': {
+                'msg_type': {'type': 'string',
+                             'enum': ['SampleStarted']},
+                'sample_name': {'type': 'string'},
+                'file_names': {'type': 'array'}
             }
         },
-        "SampleFinished": {
-            "type": "object",
-            "required": ["msg_type", "sample_name", "t"],
-            "properties": {
+        'SampleFinished': {
+            'type': 'object',
+            'required': ['msg_type', 'sample_name', 't', 'file_names'],
+            'properties': {
                 'msg_type': {'type': 'string',
                              'enum': ['SampleFinished']},
                 't': {'type': 'integer',
                       'minimum': 0},
-                "sample_name": {"type": "string"}
+                'sample_name': {'type': 'string'},
+                'file_names': {'type': 'array'}
             }
         },
-        "Error": {
-            "type": 'object',
-            'required': ['msg_type', 't', 'sample_name', 'error'],
+        'SampleSaturated': {
+            'type': 'object',
+            'required': ['msg_type', 'sample_name', 't', 'file_names'],
+            'properties': {
+                'msg_type': {'type': 'string',
+                             'enum': ['SampleSaturated']},
+                't': {'type': 'integer',
+                      'minimum': 0},
+                'sample_name': {'type': 'string'},
+                'file_names': {'type': 'array'}
+            }
+        },
+        'Error': {
+            'type': 'object',
+            'required': ['msg_type', 't', 'sample_name', 'error', 'file_names'],
             'properties': {
                 'msg_type': {'type': 'string',
                               'enum': ['Error']},
                 't': {'type': 'integer',
                       'minimum': 0},
-                "sample_name": {"type": "string"},
-                "error": {"type": "string"}
+                'sample_name': {'type': 'string'},
+                'error': {'type': 'string'},
+                'file_names': {'type': 'array'}
             }
         },
-        "DistanceCalc": {
-            "type": 'object',
-            'required': ['msg_type', 't', 'sample_name', 'delta', 'distance', 'stdev'],
+        'DistanceCalc': {
+            'type': 'object',
+            'required': ['msg_type', 't', 'sample_name', 'delta', 'distance', 'stat', 'stat_type', 'file_names'],
             'properties': {
                 'msg_type': {'type': 'string',
                              'enum': ['DistanceCalc']},
                 't': {'type': 'integer',
                       'minimum': 0},
                 'sample_name': {'type': 'string'},
+                'stat_type': {'type': 'string'},
                 'delta': {'type': 'integer',
                           'minimum': 0},
                 'distance': {'type': 'number',
                              'minimum': 0.0,
                              'maximum': 1.0},
-                'stdev': {'type': 'number'}
+                'stat': {'type': 'number'},
+                'file_names': {'type': 'array'}
             }
         },
         'EndStream': {
@@ -74,13 +92,14 @@
         }
 
     },
-    "oneOf": [
-        {"$ref": "#/definitions/Interval"},
-        {"$ref": "#/definitions/SampleStarted"},
-        {"$ref": "#/definitions/SampleFinished"},
-        {"$ref": "#/definitions/Error"},
-        {"$ref": "#/definitions/DistanceCalc"},
-        {"$ref": "#/definitions/EndStream"}
+    'oneOf': [
+        {'$ref': '#/definitions/Interval'},
+        {'$ref': '#/definitions/SampleStarted'},
+        {'$ref': '#/definitions/SampleFinished'},
+        {'$ref': '#/definitions/SampleSaturated'},
+        {'$ref': '#/definitions/Error'},
+        {'$ref': '#/definitions/DistanceCalc'},
+        {'$ref': '#/definitions/EndStream'}
     ]
 }
 

diff --git a/goetia/__main__.py b/goetia/__main__.py
@@ -6,14 +6,16 @@
 # Author : Camille Scott <[email protected]>
 # Date   : 14.10.2019
 
-from goetia.cli import GoetiaRunner
-from goetia.cdbg import cDBGRunner
+from goetia.cli.runner import GoetiaRunner
+from goetia.cli.cdbg_stream import cDBGRunner
+from goetia.cli.solid_filter import SolidFilterRunner
 from goetia.signatures import SourmashRunner
 
 
 def main():
     runner = GoetiaRunner()
     runner.add_command('cdbg', cDBGRunner)
     runner.add_command('sourmash', SourmashRunner)
+    runner.add_command('solid-filter', SolidFilterRunner)
 
     return runner.run()
diff --git a/goetia/alphabets.py b/goetia/alphabets.py
@@ -1,6 +1,14 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# (c) Camille Scott, 2019
+# File   : alphabets.py
+# License: MIT
+# Author : Camille Scott <[email protected]>
+# Date   : 27.02.2020
+
 from goetia import libgoetia
 
 DNA_SIMPLE = libgoetia.DNA_SIMPLE
 DNAN_SIMPLE = libgoetia.DNAN_SIMPLE
 IUPAC_NUCL = libgoetia.IUPAC_NUCL
-ANY = libgoetia.ANY
+ANY = libgoetia.ANY