Add ecl2csv and csv2ecl as ERT forward models (#177)

Python 3 only.

Compile ert and libres in CI for testing forward models.

.github/workflows/ecl2df.yml

@@ -14,6 +14,10 @@ on:
     # Run CI every night and check that tests are working with latest dependencies
     - cron:  '0 0 * * *'
 
+env:
+  INSTALL_DIR: ${{ github.workspace }}/install
+  ERT_SHOW_BACKTRACE: 1
+
 jobs:
 
   ecl2df:
@@ -31,9 +35,6 @@ jobs:
             pandas-version: '1.*'
 
 
-    env:
-      PYTHONPATH: ${{ github.workspace }}/install/lib/python${{ matrix.python-version }}/site-packages:${{ github.workspace }}/install/lib/python${{ matrix.python-version }}/dist-packages
-
     steps:
       - name: Checkout commit locally
         uses: actions/checkout@v2
@@ -48,6 +49,13 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
 
+      - name: Setup environment
+        run: |
+          echo "::add-path::${{ env.INSTALL_DIR}}/bin"
+          echo "::set-env name=LD_LIBRARY_PATH::${{ env.INSTALL_DIR }}/lib:${{ env.INSTALL_DIR }}/lib64"
+          echo "::set-env name=DYLD_LIBRARY_PATH::${{ env.INSTALL_DIR }}/lib:${{ env.INSTALL_DIR }}/lib64"
+          echo "::set-env name=PYTHONPATH::${{ env.INSTALL_DIR }}/lib/python${{ matrix.python-version }}/site-packages:${{ env.INSTALL_DIR }}/lib/python${{ matrix.python-version }}/dist-packages"
+
       - name: Check code style
         if: matrix.python-version != '2.7'
         run: |
@@ -57,7 +65,7 @@ jobs:
       - name: Compile and install opm-common
         if: matrix.python-version == '2.7'
         run: |
-          sudo apt-get install libboost-all-dev liblapack-dev
+          sudo apt-get install libboost-all-dev liblapack-dev libfmt-dev
           pushd ..
           git clone --recursive https://github.com/OPM/opm-common.git
           mkdir opm-common/build
@@ -72,10 +80,27 @@ jobs:
           popd
           popd
 
+
+      - name: Compile and install ERT and libres
+        if: matrix.python-version != '2.7'
+        run: |
+          git clone --branch master --depth 1 https://github.com/equinor/ert
+          source ert/.libres_version
+          git clone --branch $LIBRES_VERSION --depth 1 https://github.com/equinor/libres
+          source libres/.libecl_version
+          git clone --branch $LIBECL_VERSION --depth 1 https://github.com/equinor/libecl
+          bash ert/.build_install.sh libecl
+          bash ert/.build_install.sh libres
+          pip install -r ert/dev-requirements.txt
+          pip install ert/
+          pip list
+          ert --help
+
       - name: Install ecl2df with dependencies
         run: |
           pip install --upgrade pip
           pip install .
+          python -c "import ecl2df"
 
       - name: Install test dependencies
         run: pip install .[tests]

ecl2df/config_jobs/CSV2ECL

@@ -0,0 +1,5 @@
+EXECUTABLE csv2ecl
+
+ARGLIST <SUBCOMMAND> "--verbose" "--output" <OUTPUT> <CSVFILE>
+
+MIN_ARG 2

ecl2df/config_jobs/ECL2CSV

@@ -0,0 +1,5 @@
+EXECUTABLE ecl2csv
+
+ARGLIST <SUBCOMMAND> "--verbose" "--output" <OUTPUT> <ECLBASE>
+
+MIN_ARG 2

ecl2df/csv2ecl.py

@@ -17,6 +17,18 @@
 
 from ecl2df import __version__
 
+# String constants in use for generating ERT forward model documentation:
+DESCRIPTION = """Convert CSV files into Eclipse include files. Uses the command
+line utility ``csv2ecl``. Run ``csv2ecl --help`` to see which subcommands are supported.
+No options other than the output file is possible when
+used directly as a forward model."""
+CATEGORY = "utility.eclipse"
+EXAMPLES = (
+    "``FORWARD_MODEL "
+    "CSV2ECL(<SUBCOMMAND>=equil, <CSVFILE>=equil.csv, "
+    "<OUTPUT>=eclipse/include/equil.inc)``"
+)
+
 
 def get_parser():
     """Make parser"""

ecl2df/ecl2csv.py

@@ -32,6 +32,14 @@
 
 from ecl2df import __version__
 
+# String constants in use for generating ERT forward model documentation:
+DESCRIPTION = """Convert Eclipse input and output files into CSV files.
+Uses the command line utility ``ecl2csv``. Run ``ecl2csv --help`` to see
+which subcommands are supported. It is not possible to supply extra
+options to ecl2csv through this forward model."""
+CATEGORY = "utility.eclipse"
+EXAMPLES = "``FORWARD_MODEL ECL2CSV(<SUBCOMMAND>=equil, <OUTPUT>=equil.csv)``"
+
 
 def get_parser():
     """Make parser"""

ecl2df/hook_implementations/jobs.py

@@ -0,0 +1,58 @@
+import importlib
+import os
+from pkg_resources import resource_filename
+
+from ert_shared.plugins.plugin_manager import hook_implementation
+from ert_shared.plugins.plugin_response import plugin_response
+
+
+def _get_jobs_from_directory(directory):
+    resource_directory = resource_filename("ecl2df", directory)
+
+    all_files = [
+        os.path.join(resource_directory, f)
+        for f in os.listdir(resource_directory)
+        if os.path.isfile(os.path.join(resource_directory, f))
+    ]
+    return {os.path.basename(path): path for path in all_files}
+
+
+@hook_implementation
+@plugin_response(plugin_name="ecl2df")
+def installable_jobs():
+    return _get_jobs_from_directory("config_jobs")
+
+
+def _get_module_variable_if_exists(module_name, variable_name, default=""):
+    try:
+        script_module = importlib.import_module(module_name)
+    except ImportError:
+        return default
+
+    return getattr(script_module, variable_name, default)
+
+
+@hook_implementation
+@plugin_response(plugin_name="ecl2df")
+def job_documentation(job_name):
+    ecl2df_jobs = set(installable_jobs().data.keys())
+    if job_name not in ecl2df_jobs:
+        return None
+
+    module_name = "ecl2df.{job_name}".format(job_name=job_name.lower())
+
+    description = _get_module_variable_if_exists(
+        module_name=module_name, variable_name="DESCRIPTION"
+    )
+    examples = _get_module_variable_if_exists(
+        module_name=module_name, variable_name="EXAMPLES"
+    )
+    category = _get_module_variable_if_exists(
+        module_name=module_name, variable_name="CATEGORY", default="other"
+    )
+
+    return {
+        "description": description,
+        "examples": examples,
+        "category": category,
+    }

ecl2df/pillars.py

@@ -394,7 +394,7 @@ def fill_parser(parser):
         "--group",
         action="store_true",
         help=(
-            "If set, output will not be pr. pillar, but grouped over"
+            "If set, output will not be pr. pillar, but grouped over "
             "all pillars. If --region is set, data will be grouped over that vector. "
             "The aggregation operator is sum or mean, depending on datatype."
         ),

setup.py

@@ -2,7 +2,7 @@
 
 from os import path
 
-from setuptools import setup
+from setuptools import setup, find_packages
 
 try:
     from sphinx.setup_command import BuildDoc
@@ -48,9 +48,9 @@
     author="Håvard Berland",
     author_email="[email protected]",
     license="GPLv3",
-    packages=["ecl2df"],
+    packages=find_packages(include=["ecl2df*"]),
     package_dir={"ecl2df": "ecl2df"},
-    package_data={"ecl2df": ["opmkeywords/*"]},
+    package_data={"ecl2df": ["opmkeywords/*", "config_jobs/*"]},
     zip_safe=False,
     entry_points={
         "console_scripts": [
@@ -68,7 +68,8 @@
             "satfunc2csv=ecl2df.satfunc:main",
             "summary2csv=ecl2df.summary:main",
             "wcon2csv=ecl2df.wcon:main",
-        ]
+        ],
+        "ert": ["ecl2df_jobs = ecl2df.hook_implementations.jobs"],
     },
     test_suite="tests",
     install_requires=REQUIREMENTS,

tests/test_ert_hooks.py

@@ -0,0 +1,88 @@
+import os
+import sys
+import subprocess
+
+import pytest
+
+try:
+    import ert_shared  # noqa
+except ImportError:
+    pytest.skip(
+        "ERT is not installed, skipping hook implementation tests.",
+        allow_module_level=True,
+    )
+
+
+TESTDIR = os.path.dirname(os.path.abspath(__file__))
+DATADIR = os.path.join(TESTDIR, "data/reek/eclipse/model")
+
+
+@pytest.mark.skipif(sys.version_info < (3, 6), reason="requires python3.6 or higher")
+def test_ecl2csv_through_ert(tmpdir):
+    tmpdir.chdir()
+
+    # Symlink Eclipse output to our tmpdir:
+    eclbase = "2_R001_REEK-0"
+    ecl_extensions = [
+        "DATA",
+        "ECLEND",
+        "EGRID",
+        "INIT",
+        "RFT",
+        "SMSPEC",
+        "UNRST",
+        "UNSMRY",
+    ]
+
+    for ext in ecl_extensions:
+        f_name = eclbase + "." + ext
+        os.symlink(os.path.join(DATADIR, f_name), f_name)
+
+    ert_config = [
+        "ECLBASE " + eclbase + ".DATA",
+        "QUEUE_SYSTEM LOCAL",
+        "NUM_REALIZATIONS 1",
+        "RUNPATH .",
+    ]
+
+    ecl2csv_subcommands = [
+        "compdat",
+        "equil",
+        "grid",
+        "nnc",
+        "pillars",
+        "pvt",
+        "rft",
+        "satfunc",
+        "summary",
+    ]
+
+    csv2ecl_subcommands = ["equil", "pvt", "satfunc"]
+
+    for subcommand in ecl2csv_subcommands:
+        ert_config.append(
+            "FORWARD_MODEL ECL2CSV(<SUBCOMMAND>={}, <OUTPUT>={}.csv)".format(
+                subcommand, subcommand
+            )
+        )
+    for subcommand in csv2ecl_subcommands:
+        ert_config.append(
+            "FORWARD_MODEL CSV2ECL("
+            + "<SUBCOMMAND>={}, <CSVFILE>={}.csv, <OUTPUT>={}.inc".format(
+                subcommand, subcommand, subcommand
+            )
+            + ")"
+        )
+
+    ert_config_filename = "ecl2csv_test.ert"
+    with open(ert_config_filename, "w") as file_h:
+        file_h.write("\n".join(ert_config))
+
+    subprocess.call(["ert", "test_run", ert_config_filename])
+
+    assert os.path.exists("OK")
+
+    for subcommand in ecl2csv_subcommands:
+        assert os.path.exists(subcommand + ".csv")
+    for subcommand in csv2ecl_subcommands:
+        assert os.path.exists(subcommand + ".inc")

tests/test_hook_implementations.py

@@ -0,0 +1,82 @@
+import os
+import sys
+import shutil
+
+import pytest
+
+
+try:
+    import ert_shared  # noqa
+except ImportError:
+    pytest.skip(
+        "ERT is not installed, or Python2. Skipping hook implementations.",
+        allow_module_level=True,
+    )
+
+import ecl2df.hook_implementations.jobs
+from ert_shared.plugins.plugin_manager import ErtPluginManager
+
+EXPECTED_JOBS = {
+    "ECL2CSV": "ecl2df/config_jobs/ECL2CSV",
+    "CSV2ECL": "ecl2df/config_jobs/CSV2ECL",
+}
+
+
+@pytest.mark.skipif(sys.version_info < (3, 6), reason="requires python3.6 or higher")
+def test_hook_implementations():
+    pm = ErtPluginManager(plugins=[ecl2df.hook_implementations.jobs])
+
+    installable_jobs = pm.get_installable_jobs()
+    for wf_name, wf_location in EXPECTED_JOBS.items():
+        assert wf_name in installable_jobs
+        assert installable_jobs[wf_name].endswith(wf_location)
+        assert os.path.isfile(installable_jobs[wf_name])
+
+    assert set(installable_jobs.keys()) == set(EXPECTED_JOBS.keys())
+
+    expected_workflow_jobs = {}
+    installable_workflow_jobs = pm.get_installable_workflow_jobs()
+    for wf_name, wf_location in expected_workflow_jobs.items():
+        assert wf_name in installable_workflow_jobs
+        assert installable_workflow_jobs[wf_name].endswith(wf_location)
+
+    assert set(installable_workflow_jobs.keys()) == set(expected_workflow_jobs.keys())
+
+
+@pytest.mark.skipif(sys.version_info < (3, 6), reason="requires python3.6 or higher")
+def test_job_config_syntax():
+    """Check for syntax errors made in job configuration files"""
+    src_path = os.path.join(os.path.dirname(__file__), "../")
+    for _, job_config in EXPECTED_JOBS.items():
+        # Check (loosely) that double-dashes are enclosed in quotes:
+        with open(os.path.join(src_path, job_config)) as f_handle:
+            for line in f_handle.readlines():
+                if not line.strip().startswith("--") and "--" in line:
+                    assert '"--' in line and " --" not in line
+
+
+@pytest.mark.skipif(sys.version_info < (3, 6), reason="requires python3.6 or higher")
+@pytest.mark.integration
+def test_executables():
+    """Test executables listed in job configurations exist in $PATH"""
+    src_path = os.path.join(os.path.dirname(__file__), "../")
+    for _, job_config in EXPECTED_JOBS.items():
+        with open(os.path.join(src_path, job_config)) as f_handle:
+            executable = f_handle.readlines()[0].split()[1]
+            assert shutil.which(executable)
+
+
+@pytest.mark.skipif(sys.version_info < (3, 6), reason="requires python3.6 or higher")
+def test_hook_implementations_job_docs():
+    pm = ErtPluginManager(plugins=[ecl2df.hook_implementations.jobs])
+
+    installable_jobs = pm.get_installable_jobs()
+
+    docs = pm.get_documentation_for_jobs()
+
+    assert set(docs.keys()) == set(installable_jobs.keys())
+
+    for job_name in installable_jobs.keys():
+        print(job_name)
+        assert docs[job_name]["description"] != ""
+        assert docs[job_name]["category"] != "other"