Merge branch 'master' of github.com:ToucanToco/toucan-connectors

ToucanToco · Oct 16, 2020 · 750664b · 750664b
2 parents fc96df8 + 608672e
commit 750664b
Show file tree

Hide file tree

Showing 66 changed files with 1,774 additions and 389 deletions.
diff --git a/.coveragerc b/.coveragerc
@@ -8,4 +8,9 @@ exclude_lines =
     pragma: no cover
 
     # Don't cover NotImplemented methods (specially useful for abstract classes)
-    raise NotImplementedError
+    raise NotImplementedError
+
+# Files to exclude from consideration
+omit =
+    # Utilities for OAuth2 connectors testing
+    toucan_connectors/oauth2_connector/quickstart
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -15,7 +15,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: [3.6, 3.7, 3.8]
+        python-version: [3.8]
 
     steps:
     - uses: actions/checkout@v2
@@ -51,7 +51,7 @@ jobs:
 
     - name: Install psql-odbc dependencies
       run: sudo bash toucan_connectors/install_scripts/psql.sh
-      
+
     - name: install
       run: make install
 
@@ -69,7 +69,7 @@ jobs:
 
     - name: SonarCloud Scan
       # Only executed for one of the tested python version
-      if: ${{ always() && matrix['python-version'] == 3.6 }}
+      if: ${{ always() && matrix['python-version'] == 3.8 }}
       uses: sonarsource/sonarcloud-github-action@master
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

diff --git a/.ipynb_checkpoints/Method forwarding in connectors-checkpoint.ipynb b/.ipynb_checkpoints/Method forwarding in connectors-checkpoint.ipynb
@@ -0,0 +1,107 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 38,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from pydantic import BaseModel\n",
+    "from typing import Callable"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 39,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "class DummyDataSource(BaseModel):\n",
+    "    table: str\n",
+    "\n",
+    "class DummyConnector(BaseModel):\n",
+    "    host: str\n",
+    "    secrets_id: str\n",
+    "    \n",
+    "    get_secrets: Callable[[str], dict]\n",
+    "            \n",
+    "    def get_df(self, data_source: DummyDataSource):\n",
+    "        print('get_df for table', data_source.table)\n",
+    "        print('with secrets: ', self.get_secrets(self.secrets_id))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 40,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "get_df for table plop\n",
+      "with secrets:  {'token': '1234567890'}\n"
+     ]
+    }
+   ],
+   "source": [
+    "def get_dummy_secrets(secrets_id):\n",
+    "    if secrets_id == 'aaa':\n",
+    "        return {'token': '1234567890'}\n",
+    "    elif secrets_id == 'bbb':\n",
+    "        return {'token': 'azertyuiop'}\n",
+    "\n",
+    "conn_config = {'host': '0.2.3.4', 'secrets_id': 'aaa'}\n",
+    "conn = DummyConnector(get_secrets=get_dummy_secrets, **conn_config)\n",
+    "ds = DummyDataSource(table='plop')\n",
+    "\n",
+    "conn.get_df(ds)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 41,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'title': 'DummyConnector',\n",
+       " 'type': 'object',\n",
+       " 'properties': {'host': {'title': 'Host', 'type': 'string'},\n",
+       "  'secrets_id': {'title': 'Secrets Id', 'type': 'string'}},\n",
+       " 'required': ['host', 'secrets_id']}"
+      ]
+     },
+     "execution_count": 41,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "DummyConnector.schema()"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.7.4"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -0,0 +1,27 @@
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v3.2.0
+    hooks:
+      - id: check-yaml
+      - id: check-toml
+      - id: check-json
+      - id: end-of-file-fixer
+      - id: trailing-whitespace
+
+  - repo: http://gitlab.com/pycqa/flake8
+    rev: 3.8.3
+    hooks:
+      - id: flake8
+        name: Lint Python files using flake8
+
+  - repo: https://github.com/timothycrosley/isort
+    rev: 5.5.4
+    hooks:
+      - id: isort
+        name: Sort Python imports
+
+  - repo: https://github.com/psf/black
+    rev: 20.8b1
+    hooks:
+      - id: black
+        name: Reformat Python files
diff --git a/Makefile b/Makefile
@@ -1,6 +1,6 @@
 .DEFAULT_GOAL := all
 isort = isort toucan_connectors tests setup.py
-black = black -S -l 100 --target-version py36 toucan_connectors tests setup.py
+black = black toucan_connectors tests setup.py
 
 .PHONY: clean
 clean:

diff --git a/Method forwarding in connectors.ipynb b/Method forwarding in connectors.ipynb
@@ -0,0 +1,107 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 38,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from pydantic import BaseModel\n",
+    "from typing import Callable"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 39,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "class DummyDataSource(BaseModel):\n",
+    "    table: str\n",
+    "\n",
+    "class DummyConnector(BaseModel):\n",
+    "    host: str\n",
+    "    secrets_id: str\n",
+    "    \n",
+    "    get_secrets: Callable[[str], dict]\n",
+    "            \n",
+    "    def get_df(self, data_source: DummyDataSource):\n",
+    "        print('get_df for table', data_source.table)\n",
+    "        print('with secrets: ', self.get_secrets(self.secrets_id))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 42,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "get_df for table plop\n",
+      "with secrets:  {'token': '1234567890'}\n"
+     ]
+    }
+   ],
+   "source": [
+    "def get_dummy_secrets(secrets_id):\n",
+    "    if secrets_id == 'aaa':\n",
+    "        return {'token': '1234567890'}\n",
+    "    elif secrets_id == 'bbb':\n",
+    "        return {'token': 'azertyuiop'}\n",
+    "\n",
+    "conn_config = {'host': '0.2.3.4', 'secrets_id': 'aaa'}\n",
+    "conn = DummyConnector(get_secrets=get_dummy_secrets, **conn_config)\n",
+    "ds = DummyDataSource(table='plop')\n",
+    "\n",
+    "conn.get_df(ds)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 43,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'title': 'DummyConnector',\n",
+       " 'type': 'object',\n",
+       " 'properties': {'host': {'title': 'Host', 'type': 'string'},\n",
+       "  'secrets_id': {'title': 'Secrets Id', 'type': 'string'}},\n",
+       " 'required': ['host', 'secrets_id']}"
+      ]
+     },
+     "execution_count": 43,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "DummyConnector.schema()"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.7.4"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}
diff --git a/README.md b/README.md
@@ -11,14 +11,20 @@ configured with dictionaries (cf. `DataSource` class) and returning
 [Pandas DataFrames](https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.html).
 
 ## Setup
-In order to work you need `make` and `Python 3.6`, `3.7` or `3.8` (consider
+In order to work you need `make` and `Python 3.8` (consider
 running `pip install -U pip setuptools` if needed)
 You can then install:
 - the main dependencies by typing `pip install -e .`
 - the test requirements by typing `pip install -r requirements-testing.txt`
 
 You should be able to run basic tests `pytest tests/test_connector.py`
 
+Consider installing [pre-commit](https://pre-commit.com) to profit form linting hooks:
+```
+$ pip install pre-commit
+$ pre-commit install
+```
+
 :warning: To test and use `mssql` (and `azure_mssql`) you need to install the Microsoft ODBC driver for SQL Server for
 [Linux](https://docs.microsoft.com/en-us/sql/connect/odbc/linux-mac/installing-the-microsoft-odbc-driver-for-sql-server?view=sql-server-ver15)
 or [MacOS](https://docs.microsoft.com/en-us/sql/connect/odbc/linux-mac/install-microsoft-odbc-driver-sql-server-macos?view=sql-server-ver15)
@@ -27,22 +33,22 @@ or [MacOS](https://docs.microsoft.com/en-us/sql/connect/odbc/linux-mac/install-m
 You can then install the library with `env LDFLAGS='-L/usr/local/lib -L/usr/local/opt/openssl/lib -L/usr/local/opt/readline/lib' pip install psycopg2`
 
 ## Testing a connector
-If you want to run the tests for another connector, you can install the extra dependencies  
-(e.g to test MySQL just type `pip install -e ".[mysql]"`)  
+If you want to run the tests for another connector, you can install the extra dependencies
+(e.g to test MySQL just type `pip install -e ".[mysql]"`)
 Now `pytest tests/mysql` should run all the mysql tests properly.
 
-If you want to run the tests for all the connectors you can add all the dependencies by typing  
+If you want to run the tests for all the connectors you can add all the dependencies by typing
 `pip install -e ".[all]"` and `make test`.
 
 ## Adding a connector
 
-To generate the connector and test modules from boilerplate, run:  
+To generate the connector and test modules from boilerplate, run:
 
 ```
 $ make new_connector type=mytype
 ```
 
-`mytype` should be the name of a system we would like to build a connector for, 
+`mytype` should be the name of a system we would like to build a connector for,
 such as `MySQL` or `Hive` or `Magento`.
 
 #### Step 1 : Tests
@@ -75,7 +81,7 @@ from toucan_connectors.toucan_connector import ToucanConnector, ToucanDataSource
 class MyTypeDataSource(ToucanDataSource):
     """Model of my datasource"""
     query: str
-    
+
 
 class MyTypeConnector(ToucanConnector):
     """Model of my connector"""
@@ -84,13 +90,13 @@ class MyTypeConnector(ToucanConnector):
     host: str
     port: int
     database: str
-    
+
     def _retrieve_data(self, data_source: MyTypeDataSource) -> pd.DataFrame:
         """how to retrieve a dataframe"""
 ```
 
 Please add your connector in `toucan_connectors/__init__.py`.
-The key is what we call the `type` of the connector, which 
+The key is what we call the `type` of the connector, which
 is basically like an id used to retrieve it.
 ```python
 CONNECTORS_CATALOGUE = {