From de50431fb0623f9f650605d9ac9c9c005612b4cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Portela=20Afonso?= Date: Wed, 17 Jan 2024 10:02:23 +0000 Subject: [PATCH] feat: support to get resources from other projects --- src/ydata/sdk/connectors/connector.py | 43 ++++++++++++------- src/ydata/sdk/datasources/datasource.py | 55 +++++++++++++++++++------ 2 files changed, 71 insertions(+), 27 deletions(-) diff --git a/src/ydata/sdk/connectors/connector.py b/src/ydata/sdk/connectors/connector.py index bfa6de2b..85dc720e 100644 --- a/src/ydata/sdk/connectors/connector.py +++ b/src/ydata/sdk/connectors/connector.py @@ -8,7 +8,7 @@ from ydata.sdk.common.config import LOG_LEVEL from ydata.sdk.common.exceptions import CredentialTypeError, InvalidConnectorError from ydata.sdk.common.logger import create_logger -from ydata.sdk.common.types import UID +from ydata.sdk.common.types import UID, Project from ydata.sdk.connectors._models.connector import Connector as mConnector from ydata.sdk.connectors._models.connector_list import ConnectorsList from ydata.sdk.connectors._models.connector_type import ConnectorType @@ -25,6 +25,7 @@ class Connector(ModelFactoryMixin): connector_type (Union[ConnectorType, str]): Type of the connector to be created credentials (dict): Connector credentials name (Optional[str]): (optional) Connector name + project (Optional[Project]): (optional) Project name for this Connector client (Client): (optional) Client to connect to the backend Attributes: @@ -32,11 +33,15 @@ class Connector(ModelFactoryMixin): type (ConnectorType): Type of the connector """ - def __init__(self, connector_type: Union[ConnectorType, str] = None, credentials: Optional[Dict] = None, name: Optional[str] = None, client: Optional[Client] = None): + def __init__( + self, connector_type: Union[ConnectorType, str] = None, credentials: Optional[Dict] = None, + name: Optional[str] = None, project: Optional[Project] = None, client: Optional[Client] = None): self._init_common(client=client) self._model: Optional[mConnector] = self._create_model( connector_type, credentials, name, client=client) + self._project = project + @init_client def _init_common(self, client: Optional[Client] = None): self._client = client @@ -52,20 +57,22 @@ def type(self) -> ConnectorType: @staticmethod @init_client - def get(uid: UID, client: Optional[Client] = None) -> "Connector": + def get(uid: UID, project: Optional[Project] = None, client: Optional[Client] = None) -> "Connector": """Get an existing connector. Arguments: uid (UID): Connector identifier - client (Client): (optional) Client to connect to the backend + project (Optional[Project]): (optional) Project name from where to get the connector + client (Optional[Client]): (optional) Client to connect to the backend Returns: Connector """ - connectors: ConnectorsList = Connector.list(client=client) - data = connectors.get_by_uid(uid) - model = mConnector(**data) - connector = ModelFactoryMixin._init_from_model_data(Connector, model) + response = client.get(f'/connector/{uid}', project=project) + data = response.json() + connector = Connector._init_from_model_data(Connector, mConnector(**data)) + connector._project = project + return connector @staticmethod @@ -104,27 +111,34 @@ def _init_credentials(connector_type: ConnectorType, credentials: Union[str, Pat return _credentials @staticmethod - def create(connector_type: Union[ConnectorType, str], credentials: Union[str, Path, Dict, Credentials], name: Optional[str] = None, client: Optional[Client] = None) -> "Connector": + def create( + connector_type: Union[ConnectorType, str], credentials: Union[str, Path, Dict, Credentials], + name: Optional[str] = None, project: Optional[Project] = None, client: Optional[Client] = None + ) -> "Connector": """Create a new connector. Arguments: connector_type (Union[ConnectorType, str]): Type of the connector to be created credentials (dict): Connector credentials name (Optional[str]): (optional) Connector name + project (Optional[Project]): (optional) Project where to create the connector client (Client): (optional) Client to connect to the backend Returns: New connector """ model = Connector._create_model( - connector_type=connector_type, credentials=credentials, name=name, client=client) + connector_type=connector_type, credentials=credentials, name=name, project=project, client=client) connector = ModelFactoryMixin._init_from_model_data( Connector, model) + connector._project = project return connector @classmethod @init_client - def _create_model(cls, connector_type: Union[ConnectorType, str], credentials: Union[str, Path, Dict, Credentials], name: Optional[str] = None, client: Optional[Client] = None) -> mConnector: + def _create_model( + cls, connector_type: Union[ConnectorType, str], credentials: Union[str, Path, Dict, Credentials], + name: Optional[str] = None, project: Optional[Project] = None, client: Optional[Client] = None) -> mConnector: _name = name if name is not None else str(uuid4()) _connector_type = Connector._init_connector_type(connector_type) _credentials = Connector._init_credentials(_connector_type, credentials) @@ -133,23 +147,24 @@ def _create_model(cls, connector_type: Union[ConnectorType, str], credentials: U "credentials": _credentials.dict(by_alias=True), "name": _name } - response = client.post('/connector/', json=payload) + response = client.post('/connector/', project=project, json=payload) data: list = response.json() return mConnector(**data) @staticmethod @init_client - def list(client: Optional[Client] = None) -> ConnectorsList: + def list(project: Optional[Project] = None, client: Optional[Client] = None) -> ConnectorsList: """List the connectors instances. Arguments: + project (Optional[Project]): (optional) Project name from where to list the connectors client (Client): (optional) Client to connect to the backend Returns: List of connectors """ - response = client.get('/connector') + response = client.get('/connector', project=project) data: list = response.json() return ConnectorsList(data) diff --git a/src/ydata/sdk/datasources/datasource.py b/src/ydata/sdk/datasources/datasource.py index ec15a47a..80743c15 100644 --- a/src/ydata/sdk/datasources/datasource.py +++ b/src/ydata/sdk/datasources/datasource.py @@ -6,7 +6,7 @@ from ydata.sdk.common.client.utils import init_client from ydata.sdk.common.config import BACKOFF, LOG_LEVEL from ydata.sdk.common.logger import create_logger -from ydata.sdk.common.types import UID +from ydata.sdk.common.types import UID, Project from ydata.sdk.connectors.connector import Connector, ConnectorType from ydata.sdk.datasources._models.connector_to_datasource import CONNECTOR_TO_DATASOURCE from ydata.sdk.datasources._models.datasource import DataSource as mDataSource @@ -26,6 +26,7 @@ class DataSource(ModelFactoryMixin): connector (Connector): Connector from which the datasource is created datatype (Optional[Union[DataSourceType, str]]): (optional) DataSource type name (Optional[str]): (optional) DataSource name + project (Optional[Project]): (optional) Project name for this datasource wait_for_metadata (bool): If `True`, wait until the metadata is fully calculated client (Client): (optional) Client to connect to the backend **config: Datasource specific configuration @@ -37,15 +38,22 @@ class DataSource(ModelFactoryMixin): metadata (Metadata): Metadata associated to the datasource """ - def __init__(self, connector: Connector, datatype: Optional[Union[DataSourceType, str]] = DataSourceType.TABULAR, name: Optional[str] = None, wait_for_metadata: bool = True, client: Optional[Client] = None, **config): + def __init__( + self, connector: Connector, datatype: Optional[Union[DataSourceType, str]] = DataSourceType.TABULAR, + name: Optional[str] = None, project: Optional[Project] = None, wait_for_metadata: bool = True, + client: Optional[Client] = None, **config + ): datasource_type = CONNECTOR_TO_DATASOURCE.get(connector.type) self._init_common(client=client) self._model: Optional[mDataSource] = self._create_model( - connector=connector, datasource_type=datasource_type, datatype=datatype, config=config, name=name, client=self._client) + connector=connector, datasource_type=datasource_type, datatype=datatype, + config=config, name=name, client=self._client) if wait_for_metadata: self._model = DataSource._wait_for_metadata(self)._model + self._project = project + @init_client def _init_common(self, client: Optional[Client] = None): self._client = client @@ -73,11 +81,12 @@ def metadata(self) -> Metadata: @staticmethod @init_client - def list(client: Optional[Client] = None) -> DataSourceList: + def list(project: Optional[Project] = None, client: Optional[Client] = None) -> DataSourceList: """List the [`DataSource`][ydata.sdk.datasources.DataSource] instances. Arguments: + project (Optional[Project]): (optional) Project name from where to list the datasources client (Client): (optional) Client to connect to the backend Returns: @@ -90,7 +99,7 @@ def __process_data(data: list) -> list: e.pop(k, None) return data - response = client.get('/datasource') + response = client.get('/datasource', project=project) data: list = response.json() data = __process_data(data) @@ -98,32 +107,39 @@ def __process_data(data: list) -> list: @staticmethod @init_client - def get(uid: UID, client: Optional[Client] = None) -> "DataSource": + def get(uid: UID, project: Optional[Project] = None, client: Optional[Client] = None) -> "DataSource": """Get an existing [`DataSource`][ydata.sdk.datasources.DataSource]. Arguments: uid (UID): DataSource identifier + project (Optional[Project]): (optional) Project name from where to get the connector client (Client): (optional) Client to connect to the backend Returns: DataSource """ - response = client.get(f'/datasource/{uid}') + response = client.get(f'/datasource/{uid}', project=project) data: list = response.json() datasource_type = CONNECTOR_TO_DATASOURCE.get( ConnectorType(data['connector']['type'])) model = DataSource._model_from_api(data, datasource_type) datasource = ModelFactoryMixin._init_from_model_data(DataSource, model) + datasource._project = project return datasource @classmethod - def create(cls, connector: Connector, datatype: Optional[Union[DataSourceType, str]] = DataSourceType.TABULAR, name: Optional[str] = None, wait_for_metadata: bool = True, client: Optional[Client] = None, **config) -> "DataSource": + def create( + cls, connector: Connector, datatype: Optional[Union[DataSourceType, str]] = DataSourceType.TABULAR, + name: Optional[str] = None, project: Optional[Project] = None, wait_for_metadata: bool = True, + client: Optional[Client] = None, **config + ) -> "DataSource": """Create a new [`DataSource`][ydata.sdk.datasources.DataSource]. Arguments: connector (Connector): Connector from which the datasource is created datatype (Optional[Union[DataSourceType, str]]): (optional) DataSource type name (Optional[str]): (optional) DataSource name + project (Optional[Project]): (optional) Project name for this datasource wait_for_metadata (bool): If `True`, wait until the metadata is fully calculated client (Client): (optional) Client to connect to the backend **config: Datasource specific configuration @@ -132,22 +148,35 @@ def create(cls, connector: Connector, datatype: Optional[Union[DataSourceType, s DataSource """ datasource_type = CONNECTOR_TO_DATASOURCE.get(connector.type) - return cls._create(connector=connector, datasource_type=datasource_type, datatype=datatype, config=config, name=name, wait_for_metadata=wait_for_metadata, client=client) + return cls._create( + connector=connector, datasource_type=datasource_type, datatype=datatype, config=config, name=name, + project=project, wait_for_metadata=wait_for_metadata, client=client) @classmethod - def _create(cls, connector: Connector, datasource_type: Type[mDataSource], datatype: Optional[Union[DataSourceType, str]] = DataSourceType.TABULAR, config: Optional[Dict] = None, name: Optional[str] = None, wait_for_metadata: bool = True, client: Optional[Client] = None) -> "DataSource": + def _create( + cls, connector: Connector, datasource_type: Type[mDataSource], + datatype: Optional[Union[DataSourceType, str]] = DataSourceType.TABULAR, config: Optional[Dict] = None, + name: Optional[str] = None, project: Optional[Project] = None, wait_for_metadata: bool = True, + client: Optional[Client] = None + ) -> "DataSource": model = DataSource._create_model( - connector, datasource_type, datatype, config, name, client) + connector, datasource_type, datatype, config, name, project, client) datasource = ModelFactoryMixin._init_from_model_data(DataSource, model) if wait_for_metadata: datasource._model = DataSource._wait_for_metadata(datasource)._model + datasource._project = project + return datasource @classmethod @init_client - def _create_model(cls, connector: Connector, datasource_type: Type[mDataSource], datatype: Optional[Union[DataSourceType, str]] = DataSourceType.TABULAR, config: Optional[Dict] = None, name: Optional[str] = None, client: Optional[Client] = None) -> mDataSource: + def _create_model( + cls, connector: Connector, datasource_type: Type[mDataSource], + datatype: Optional[Union[DataSourceType, str]] = DataSourceType.TABULAR, config: Optional[Dict] = None, + name: Optional[str] = None, project: Optional[Project] = None, client: Optional[Client] = None + ) -> mDataSource: _name = name if name is not None else str(uuid4()) _config = config if config is not None else {} payload = { @@ -161,7 +190,7 @@ def _create_model(cls, connector: Connector, datasource_type: Type[mDataSource], if connector.type != ConnectorType.FILE: _config = datasource_type(**config).to_payload() payload.update(_config) - response = client.post('/datasource/', json=payload) + response = client.post('/datasource/', project=project, json=payload) data: list = response.json() return DataSource._model_from_api(data, datasource_type)