Merge branch 'develop'

musif/extract/extract.py

@@ -1,6 +1,7 @@
 import os
 import pickle
 import subprocess
+import types
 from pathlib import Path, PurePath
 from subprocess import DEVNULL
 from tempfile import mkstemp
@@ -9,39 +10,36 @@
 import ms3
 import pandas as pd
 from joblib import Parallel, delayed
-from music21.converter import parse
+from music21.converter import parse, toData
 from music21.stream import Measure, Part, Score
 from pandas import DataFrame
 from tqdm import tqdm
+from music21 import stream
 
 import musif.extract.constants as C
-from musif.cache import (
-    CACHE_FILE_EXTENSION,
-    FileCacheIntoRAM,
-    SmartModuleCache,
-    store_score_df,
-)
+from musif.cache import (CACHE_FILE_EXTENSION, FileCacheIntoRAM,
+                         SmartModuleCache, store_score_df)
 from musif.common._constants import GENERAL_FAMILY
 from musif.common.exceptions import FeatureError, ParseFileError
 from musif.config import ExtractConfiguration
 from musif.extract.common import _filter_parts_data
-from musif.extract.utils import (
-    cast_mixed_dtypes,
-    extract_global_time_signature,
-    process_musescore_file,
-)
+from musif.extract.utils import (cast_mixed_dtypes,
+                                 extract_global_time_signature,
+                                 process_musescore_file)
 from musif.logs import ldebug, lerr, linfo, lwarn, pdebug, perr, pinfo
 from musif.musescore import constants as mscore_c
 from musif.musicxml import constants as musicxml_c
-from musif.musicxml import extract_numeric_tempo, name_parts, split_layers, fix_repeats
-from musif.musicxml.scoring import (
-    _extract_abbreviated_part,
-    extract_sound,
-    to_abbreviation,
-)
+from musif.musicxml import (extract_numeric_tempo, fix_repeats, name_parts,
+                            split_layers)
+from musif.musicxml.scoring import (_extract_abbreviated_part, extract_sound,
+                                    to_abbreviation)
 
 _cache = FileCacheIntoRAM(10000)  # To cache scanned scores
 
+# attach a method to convert it into bytestring
+# the first argument of toData is the object to be translated, so that could be the `self` of a class method, perfectly ok
+# this must be done before of start the caching, because we are modifying the object!
+stream.Stream.toData = toData
 
 def parse_filename(
     file_path: str,
@@ -80,11 +78,14 @@ def parse_filename(
     if score is not None:
         return score
     try:
+
         score = parse(file_path).makeRests()
         if export_dfs_to is not None:
             dest_path = Path(export_dfs_to)
             dest_path /= Path(file_path).with_suffix(".pkl").name
             store_score_df(score, dest_path)
+
+
         # give a name to all parts in the score
         name_parts(score)
         if remove_unpitched_objects:
@@ -538,16 +539,29 @@ def _load_xml_data(self, filename: Union[str, PurePath]):
     def _get_score_data(
         self, filename: PurePath, load_cache: Optional[Path] = None
     ) -> dict:
-        data = None
+        from music21 import converter
+        import types
+        data = None 
         info_load_str = ""
+
         if load_cache is not None and load_cache.exists():
+            s = converter.parse(filename)
+            s.toData = types.MethodType(converter.toData, converter)
+            cached_object = SmartModuleCache(s)
             try:
                 data = pickle.load(open(load_cache, "rb"))
             except Exception as e:
                 info_load_str += f" Error while loading pickled object, continuing with extraction from scratch: {e}"
             else:
                 info_load_str += " File was loaded from cache."
-
+            # get bytes
+            bytes = cached_object.toData('midi')
+            # write to file
+            with open('output.mid', 'wb') as f:
+                f.write(bytes)
+            # save cached object
+            pickle.dump(cached_object, open(load_cache, 'wb'))
+
         if data is None:
             try:
                 score, filtered_parts, numeric_tempo = self._load_xml_data(filename)

musif/extract/features/density/__init__.py

@@ -1,2 +1,2 @@
 from .constants import *
-musif_dependencies = ['core']
+musif_dependencies = ['core', 'tempo']

musif/extract/features/jsymbolic/handler.py

@@ -23,9 +23,9 @@ def get_tmpdir():
 
 
 def write_midi(score, midi_path):
-    with open(os.devnull, "w") as devnull:
-        with contextlib.redirect_stdout(devnull):
-            score.write("MIDI", midi_path)
+    bytes = score.toData('midi') 
+    with open(midi_path, 'wb') as f:
+        f.write(bytes)
 
 
 def update_score_objects(
@@ -82,7 +82,7 @@ def update_score_objects(
                     out_path + "_def.xml",
                 ],
                 check=True,
-                stdout=subprocess.DEVNULL,
+                # stdout=subprocess.DEVNULL,
             )
         except Exception as e:
             filename = score_data[C.DATA_FILE]

musif/extract/features/music21/constants.py

@@ -1,4 +1,4 @@
-ERRORED_NAMES = {
+ERRORED_FEATURES_IDS = {
     "T1",
     "T2",
     "T3",

musif/extract/features/music21/handler.py

@@ -7,10 +7,10 @@
 from musif import cache
 from musif.config import ExtractConfiguration
 from musif.extract.constants import DATA_SCORE
-from .constants import ERRORED_NAMES
+from .constants import ERRORED_FEATURES_IDS
 
 
-def allFeaturesAsList(streamInput, includeJSymbolic=True):
+def allFeaturesAsList(streamInput, include_jSymbolic_from_m21=True):
     """
     only a little change around m21.features.base.allFeaturesAsList: no Parallel
     processing
@@ -19,19 +19,18 @@ def allFeaturesAsList(streamInput, includeJSymbolic=True):
     ds = m21.features.base.DataSet(classLabel="")
     ds.runParallel = False  # this is the only difference with the m21 original code
     all_features = list(native.featureExtractors)
-    if includeJSymbolic:
+    if include_jSymbolic_from_m21:
         all_features += list(jSymbolic.featureExtractors)
-    f = [feature for feature in all_features if feature.id not in
-         ERRORED_NAMES]
-    # f = list(native.featureExtractors)
-    ds.addFeatureExtractors(f)
+    final_features = [feature for feature in all_features if feature.id not in
+         ERRORED_FEATURES_IDS]
+    ds.addFeatureExtractors(final_features)
     ds.addData(streamInput)
     ds.process()
     allData = ds.getFeaturesAsList(
         includeClassLabel=False, includeId=False, concatenateLists=False
     )
 
-    return allData
+    return allData, [c.__name__ for c in final_features]
 
 
 def update_score_objects(
@@ -42,26 +41,22 @@ def update_score_objects(
     score_features: dict,
 ):
     score = score_data[DATA_SCORE]
-    # override the isinstance and hasattr definitions for the caching system
+    # Override the isinstance and hasattr definitions for the caching system
     m21.features.base.isinstance = cache.isinstance
     m21.features.base.hasattr = cache.hasattr
-    # avoid extracting jsymbolic features twice
-    includeJSymbolic = 'jsymbolic' in cfg.features
-    features = allFeaturesAsList(score, includeJSymbolic=includeJSymbolic)
-    if includeJSymbolic:
-        all_columns = [x.id for x in extractorsById("all")]
+    include_jSymbolic_from_m21 = 'jsymbolic' not in cfg.features
+    if include_jSymbolic_from_m21:
+        features, columns = allFeaturesAsList(score, include_jSymbolic_from_m21=include_jSymbolic_from_m21)
     else:
-        all_columns = [x.id for x in extractorsById("native")]
-    columns = [c for c in all_columns if c not in ERRORED_NAMES]
+        features, columns = allFeaturesAsList(score, include_jSymbolic_from_m21=include_jSymbolic_from_m21)
     score_features.update(
         {
             'm21_' + columns[outer] + f"_{i}": f
-            for outer in range(len(columns))
+            for outer in range(len(features))
             for i, f in enumerate(features[outer])
         }
     )
 
-
 def update_part_objects(
     score_data: dict, part_data: dict, cfg: ExtractConfiguration, part_features: dict
 ):

pyproject.toml

@@ -7,7 +7,7 @@ dependencies = [
     'roman>=3.3',
     'joblib>=1.0.0',
     'scipy>=1.6.0',
-    'music21>=8,<9',
+    'music21>=9.1',
     # music21 has a different version for each python 3.*
     "deepdiff>=6.2.1",
 ]