From b6c205aab2f3f744c477337996e812f946462540 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Merc=C3=A8=20Mart=C3=ADn=20Prats?= Date: Wed, 28 Feb 2024 11:51:46 +0100 Subject: [PATCH 1/3] Documenting and removing partially the need for Node.js in local Pipelines --- HISTORY.rst | 5 +++++ bigml/dataset.py | 12 ++++++++++++ bigml/version.py | 2 +- docs/index.rst | 5 +++++ docs/local_resources.rst | 5 +++-- 5 files changed, 26 insertions(+), 3 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 1a63a61c..2aa2591a 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -3,6 +3,11 @@ History ------- +9.8.0.dev1 (2024-02-28) +----------------------- + +- Documenting and removing partially the need for Node.js in Pipelines. + 9.8.0.dev (2024-02-19) ---------------------- diff --git a/bigml/dataset.py b/bigml/dataset.py index 1d3de5c2..280c285d 100644 --- a/bigml/dataset.py +++ b/bigml/dataset.py @@ -21,6 +21,7 @@ import os import logging import warnings +import subprocess from bigml.fields import Fields, sorted_headers, get_new_fields from bigml.api import get_api_connection, get_dataset_id, get_status @@ -30,6 +31,13 @@ from bigml.flatline import Flatline from bigml.featurizer import Featurizer +process = subprocess.Popen(['node -v'], stdout=subprocess.PIPE, shell=True) +out = process.stdout.read() +FLATLINE_READY = out.startswith(b"v") +if FLATLINE_READY: + from bigml.flatline import Flatline + + #pylint: disable=locally-disabled,bare-except,ungrouped-imports try: # avoiding tensorflow info logging @@ -211,6 +219,10 @@ def transform(self, input_data_list): rows = [self._input_array(input_data) for input_data in input_data_list] if self.transformations: + if not FLATLINE_READY: + raise ValueError("Nodejs should be installed to handle this" + " dataset's transformations. Please, check" + " the bindings documentation for details.") out_headers, out_arrays = self._transform(rows) rows = [dict(zip(out_headers, row)) for row in out_arrays] diff --git a/bigml/version.py b/bigml/version.py index 4e9967f4..9865fb2a 100644 --- a/bigml/version.py +++ b/bigml/version.py @@ -1 +1 @@ -__version__ = '9.8.0.dev' +__version__ = '9.8.0.dev1' diff --git a/docs/index.rst b/docs/index.rst index d80f7969..72f76660 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -77,6 +77,11 @@ The bindings will also use ``simplejson`` if you happen to have it installed, but that is optional: we fall back to Python's built-in JSON libraries is ``simplejson`` is not found. +`Node.js `_ is not installed by default, but will be +needed for `Local Pipelines Date: Wed, 28 Feb 2024 14:02:59 +0100 Subject: [PATCH 2/3] Changing waiting time dynamically in tests --- .github/workflows/tests_01.yml | 2 +- .github/workflows/tests_05.yml | 2 +- .github/workflows/tests_22.yml | 2 +- .github/workflows/tests_23.yml | 2 +- .github/workflows/tests_36.yml | 2 +- bigml/tests/test_35_d_compare_predictions.py | 6 +++--- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/tests_01.yml b/.github/workflows/tests_01.yml index a6709e17..c96759d8 100644 --- a/.github/workflows/tests_01.yml +++ b/.github/workflows/tests_01.yml @@ -22,7 +22,7 @@ jobs: BIGML_EXTERNAL_CONN_DB: ${{ secrets.BIGML_EXTERNAL_CONN_DB }} BIGML_EXTERNAL_CONN_USER: ${{ secrets.BIGML_EXTERNAL_CONN_USER }} BIGML_EXTERNAL_CONN_PWD: ${{ secrets.BIGML_EXTERNAL_CONN_PWD }} - BIGML_DELTA: 5 + BIGML_DELTA: ${{ env.BIGML_DELTA }} steps: - name: Install packages diff --git a/.github/workflows/tests_05.yml b/.github/workflows/tests_05.yml index dc01250a..14080f29 100644 --- a/.github/workflows/tests_05.yml +++ b/.github/workflows/tests_05.yml @@ -22,7 +22,7 @@ jobs: BIGML_EXTERNAL_CONN_DB: ${{ secrets.BIGML_EXTERNAL_CONN_DB }} BIGML_EXTERNAL_CONN_USER: ${{ secrets.BIGML_EXTERNAL_CONN_USER }} BIGML_EXTERNAL_CONN_PWD: ${{ secrets.BIGML_EXTERNAL_CONN_PWD }} - BIGML_DELTA: 5 + BIGML_DELTA: ${{ env.BIGML_DELTA }} steps: - name: Install packages uses: actions/checkout@v3 diff --git a/.github/workflows/tests_22.yml b/.github/workflows/tests_22.yml index 7d2d8c87..635c9abf 100644 --- a/.github/workflows/tests_22.yml +++ b/.github/workflows/tests_22.yml @@ -22,7 +22,7 @@ jobs: BIGML_EXTERNAL_CONN_DB: ${{ secrets.BIGML_EXTERNAL_CONN_DB }} BIGML_EXTERNAL_CONN_USER: ${{ secrets.BIGML_EXTERNAL_CONN_USER }} BIGML_EXTERNAL_CONN_PWD: ${{ secrets.BIGML_EXTERNAL_CONN_PWD }} - BIGML_DELTA: 10 + BIGML_DELTA: ${{ env.BIGML_DELTA }} steps: - name: Install packages diff --git a/.github/workflows/tests_23.yml b/.github/workflows/tests_23.yml index 859be798..c9290f5a 100644 --- a/.github/workflows/tests_23.yml +++ b/.github/workflows/tests_23.yml @@ -22,7 +22,7 @@ jobs: BIGML_EXTERNAL_CONN_DB: ${{ secrets.BIGML_EXTERNAL_CONN_DB }} BIGML_EXTERNAL_CONN_USER: ${{ secrets.BIGML_EXTERNAL_CONN_USER }} BIGML_EXTERNAL_CONN_PWD: ${{ secrets.BIGML_EXTERNAL_CONN_PWD }} - BIGML_DELTA: 5 + BIGML_DELTA: ${{ env.BIGML_DELTA }} steps: - name: Install packages diff --git a/.github/workflows/tests_36.yml b/.github/workflows/tests_36.yml index 934cfc83..79661c08 100644 --- a/.github/workflows/tests_36.yml +++ b/.github/workflows/tests_36.yml @@ -22,7 +22,7 @@ jobs: BIGML_EXTERNAL_CONN_DB: ${{ secrets.BIGML_EXTERNAL_CONN_DB }} BIGML_EXTERNAL_CONN_USER: ${{ secrets.BIGML_EXTERNAL_CONN_USER }} BIGML_EXTERNAL_CONN_PWD: ${{ secrets.BIGML_EXTERNAL_CONN_PWD }} - BIGML_DELTA: 5 + BIGML_DELTA: ${{ env.BIGML_DELTA }} steps: - name: Install packages diff --git a/bigml/tests/test_35_d_compare_predictions.py b/bigml/tests/test_35_d_compare_predictions.py index c99a90fe..023d3830 100644 --- a/bigml/tests/test_35_d_compare_predictions.py +++ b/bigml/tests/test_35_d_compare_predictions.py @@ -70,18 +70,18 @@ def test_scenario2(self): "input_data", "forecast", "model_conf"] examples = [ - ['data/grades.csv', '30', '30', '120', + ['data/grades.csv', '30', '30', '300', '{"000005": {"horizon": 5}}', '{"000005": [{"point_forecast": [73.96192, 74.04106, 74.12029, ' '74.1996, 74.27899], "model": "M,M,N"}]}', '{"objective_fields": ["000001", "000005"], "period": 12}'], - ['data/grades.csv', '30', '30', '120', + ['data/grades.csv', '30', '30', '300', '{"000005": {"horizon": 5, "ets_models": {"names": ["M,N,A"], ' '"criterion": "aic", "limit": 3}}}', '{"000005": [{"point_forecast": [67.43222, 68.24468, ' '64.14437, 67.5662, 67.79028], "model": "M,N,A"}]}', '{"objective_fields": ["000001", "000005"], "period": 12}'], - ['data/grades.csv', '30', '30', '120', + ['data/grades.csv', '30', '30', '300', '{"000005": {"horizon": 5, "ets_models": {"names": ["A,A,A"], ' '"criterion": "aic", "limit": 3}}}', '{"000005": [{"point_forecast": [74.73553, 71.6163, 71.90264, ' From a372c5c0265d587b4ebb047a2257a8b5b27478bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Merc=C3=A8=20Mart=C3=ADn=20Prats?= Date: Wed, 28 Feb 2024 14:07:24 +0100 Subject: [PATCH 3/3] Fixing vars reference in tests --- .github/workflows/tests_01.yml | 2 +- .github/workflows/tests_05.yml | 2 +- .github/workflows/tests_22.yml | 2 +- .github/workflows/tests_23.yml | 2 +- .github/workflows/tests_36.yml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/tests_01.yml b/.github/workflows/tests_01.yml index c96759d8..9f12863f 100644 --- a/.github/workflows/tests_01.yml +++ b/.github/workflows/tests_01.yml @@ -22,7 +22,7 @@ jobs: BIGML_EXTERNAL_CONN_DB: ${{ secrets.BIGML_EXTERNAL_CONN_DB }} BIGML_EXTERNAL_CONN_USER: ${{ secrets.BIGML_EXTERNAL_CONN_USER }} BIGML_EXTERNAL_CONN_PWD: ${{ secrets.BIGML_EXTERNAL_CONN_PWD }} - BIGML_DELTA: ${{ env.BIGML_DELTA }} + BIGML_DELTA: ${{ vars.BIGML_DELTA }} steps: - name: Install packages diff --git a/.github/workflows/tests_05.yml b/.github/workflows/tests_05.yml index 14080f29..8cc673ec 100644 --- a/.github/workflows/tests_05.yml +++ b/.github/workflows/tests_05.yml @@ -22,7 +22,7 @@ jobs: BIGML_EXTERNAL_CONN_DB: ${{ secrets.BIGML_EXTERNAL_CONN_DB }} BIGML_EXTERNAL_CONN_USER: ${{ secrets.BIGML_EXTERNAL_CONN_USER }} BIGML_EXTERNAL_CONN_PWD: ${{ secrets.BIGML_EXTERNAL_CONN_PWD }} - BIGML_DELTA: ${{ env.BIGML_DELTA }} + BIGML_DELTA: ${{ vars.BIGML_DELTA }} steps: - name: Install packages uses: actions/checkout@v3 diff --git a/.github/workflows/tests_22.yml b/.github/workflows/tests_22.yml index 635c9abf..c75adca1 100644 --- a/.github/workflows/tests_22.yml +++ b/.github/workflows/tests_22.yml @@ -22,7 +22,7 @@ jobs: BIGML_EXTERNAL_CONN_DB: ${{ secrets.BIGML_EXTERNAL_CONN_DB }} BIGML_EXTERNAL_CONN_USER: ${{ secrets.BIGML_EXTERNAL_CONN_USER }} BIGML_EXTERNAL_CONN_PWD: ${{ secrets.BIGML_EXTERNAL_CONN_PWD }} - BIGML_DELTA: ${{ env.BIGML_DELTA }} + BIGML_DELTA: ${{ vars.BIGML_DELTA }} steps: - name: Install packages diff --git a/.github/workflows/tests_23.yml b/.github/workflows/tests_23.yml index c9290f5a..042d57cc 100644 --- a/.github/workflows/tests_23.yml +++ b/.github/workflows/tests_23.yml @@ -22,7 +22,7 @@ jobs: BIGML_EXTERNAL_CONN_DB: ${{ secrets.BIGML_EXTERNAL_CONN_DB }} BIGML_EXTERNAL_CONN_USER: ${{ secrets.BIGML_EXTERNAL_CONN_USER }} BIGML_EXTERNAL_CONN_PWD: ${{ secrets.BIGML_EXTERNAL_CONN_PWD }} - BIGML_DELTA: ${{ env.BIGML_DELTA }} + BIGML_DELTA: ${{ vars.BIGML_DELTA }} steps: - name: Install packages diff --git a/.github/workflows/tests_36.yml b/.github/workflows/tests_36.yml index 79661c08..7b78c0a5 100644 --- a/.github/workflows/tests_36.yml +++ b/.github/workflows/tests_36.yml @@ -22,7 +22,7 @@ jobs: BIGML_EXTERNAL_CONN_DB: ${{ secrets.BIGML_EXTERNAL_CONN_DB }} BIGML_EXTERNAL_CONN_USER: ${{ secrets.BIGML_EXTERNAL_CONN_USER }} BIGML_EXTERNAL_CONN_PWD: ${{ secrets.BIGML_EXTERNAL_CONN_PWD }} - BIGML_DELTA: ${{ env.BIGML_DELTA }} + BIGML_DELTA: ${{ vars.BIGML_DELTA }} steps: - name: Install packages