Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat: support to get resources from other projects #84

Merged
merged 1 commit into from
Jan 17, 2024
Merged

feat: support to get resources from other projects #84

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 29 additions & 14 deletions src/ydata/sdk/connectors/connector.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
from ydata.sdk.common.config import LOG_LEVEL
from ydata.sdk.common.exceptions import CredentialTypeError, InvalidConnectorError
from ydata.sdk.common.logger import create_logger
from ydata.sdk.common.types import UID
from ydata.sdk.common.types import UID, Project
from ydata.sdk.connectors._models.connector import Connector as mConnector
from ydata.sdk.connectors._models.connector_list import ConnectorsList
from ydata.sdk.connectors._models.connector_type import ConnectorType
Expand All @@ -25,18 +25,23 @@ class Connector(ModelFactoryMixin):
connector_type (Union[ConnectorType, str]): Type of the connector to be created
credentials (dict): Connector credentials
name (Optional[str]): (optional) Connector name
project (Optional[Project]): (optional) Project name for this Connector
client (Client): (optional) Client to connect to the backend

Attributes:
uid (UID): UID fo the connector instance (creating internally)
type (ConnectorType): Type of the connector
"""

def __init__(self, connector_type: Union[ConnectorType, str] = None, credentials: Optional[Dict] = None, name: Optional[str] = None, client: Optional[Client] = None):
def __init__(
self, connector_type: Union[ConnectorType, str] = None, credentials: Optional[Dict] = None,
name: Optional[str] = None, project: Optional[Project] = None, client: Optional[Client] = None):
self._init_common(client=client)
self._model: Optional[mConnector] = self._create_model(
connector_type, credentials, name, client=client)

self._project = project

@init_client
def _init_common(self, client: Optional[Client] = None):
self._client = client
Expand All @@ -52,20 +57,22 @@ def type(self) -> ConnectorType:

@staticmethod
@init_client
def get(uid: UID, client: Optional[Client] = None) -> "Connector":
def get(uid: UID, project: Optional[Project] = None, client: Optional[Client] = None) -> "Connector":
"""Get an existing connector.

Arguments:
uid (UID): Connector identifier
client (Client): (optional) Client to connect to the backend
project (Optional[Project]): (optional) Project name from where to get the connector
client (Optional[Client]): (optional) Client to connect to the backend

Returns:
Connector
"""
connectors: ConnectorsList = Connector.list(client=client)
data = connectors.get_by_uid(uid)
model = mConnector(**data)
connector = ModelFactoryMixin._init_from_model_data(Connector, model)
response = client.get(f'/connector/{uid}', project=project)
data = response.json()
connector = Connector._init_from_model_data(Connector, mConnector(**data))
connector._project = project

return connector

@staticmethod
Expand Down Expand Up @@ -104,27 +111,34 @@ def _init_credentials(connector_type: ConnectorType, credentials: Union[str, Pat
return _credentials

@staticmethod
def create(connector_type: Union[ConnectorType, str], credentials: Union[str, Path, Dict, Credentials], name: Optional[str] = None, client: Optional[Client] = None) -> "Connector":
def create(
connector_type: Union[ConnectorType, str], credentials: Union[str, Path, Dict, Credentials],
name: Optional[str] = None, project: Optional[Project] = None, client: Optional[Client] = None
) -> "Connector":
"""Create a new connector.

Arguments:
connector_type (Union[ConnectorType, str]): Type of the connector to be created
credentials (dict): Connector credentials
name (Optional[str]): (optional) Connector name
project (Optional[Project]): (optional) Project where to create the connector
client (Client): (optional) Client to connect to the backend

Returns:
New connector
"""
model = Connector._create_model(
connector_type=connector_type, credentials=credentials, name=name, client=client)
connector_type=connector_type, credentials=credentials, name=name, project=project, client=client)
connector = ModelFactoryMixin._init_from_model_data(
Connector, model)
connector._project = project
return connector

@classmethod
@init_client
def _create_model(cls, connector_type: Union[ConnectorType, str], credentials: Union[str, Path, Dict, Credentials], name: Optional[str] = None, client: Optional[Client] = None) -> mConnector:
def _create_model(
cls, connector_type: Union[ConnectorType, str], credentials: Union[str, Path, Dict, Credentials],
name: Optional[str] = None, project: Optional[Project] = None, client: Optional[Client] = None) -> mConnector:
_name = name if name is not None else str(uuid4())
_connector_type = Connector._init_connector_type(connector_type)
_credentials = Connector._init_credentials(_connector_type, credentials)
Expand All @@ -133,23 +147,24 @@ def _create_model(cls, connector_type: Union[ConnectorType, str], credentials: U
"credentials": _credentials.dict(by_alias=True),
"name": _name
}
response = client.post('/connector/', json=payload)
response = client.post('/connector/', project=project, json=payload)
data: list = response.json()

return mConnector(**data)

@staticmethod
@init_client
def list(client: Optional[Client] = None) -> ConnectorsList:
def list(project: Optional[Project] = None, client: Optional[Client] = None) -> ConnectorsList:
"""List the connectors instances.

Arguments:
project (Optional[Project]): (optional) Project name from where to list the connectors
client (Client): (optional) Client to connect to the backend

Returns:
List of connectors
"""
response = client.get('/connector')
response = client.get('/connector', project=project)
data: list = response.json()
return ConnectorsList(data)

Expand Down
55 changes: 42 additions & 13 deletions src/ydata/sdk/datasources/datasource.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
from ydata.sdk.common.client.utils import init_client
from ydata.sdk.common.config import BACKOFF, LOG_LEVEL
from ydata.sdk.common.logger import create_logger
from ydata.sdk.common.types import UID
from ydata.sdk.common.types import UID, Project
from ydata.sdk.connectors.connector import Connector, ConnectorType
from ydata.sdk.datasources._models.connector_to_datasource import CONNECTOR_TO_DATASOURCE
from ydata.sdk.datasources._models.datasource import DataSource as mDataSource
Expand All @@ -26,6 +26,7 @@ class DataSource(ModelFactoryMixin):
connector (Connector): Connector from which the datasource is created
datatype (Optional[Union[DataSourceType, str]]): (optional) DataSource type
name (Optional[str]): (optional) DataSource name
project (Optional[Project]): (optional) Project name for this datasource
wait_for_metadata (bool): If `True`, wait until the metadata is fully calculated
client (Client): (optional) Client to connect to the backend
**config: Datasource specific configuration
Expand All @@ -37,15 +38,22 @@ class DataSource(ModelFactoryMixin):
metadata (Metadata): Metadata associated to the datasource
"""

def __init__(self, connector: Connector, datatype: Optional[Union[DataSourceType, str]] = DataSourceType.TABULAR, name: Optional[str] = None, wait_for_metadata: bool = True, client: Optional[Client] = None, **config):
def __init__(
self, connector: Connector, datatype: Optional[Union[DataSourceType, str]] = DataSourceType.TABULAR,
name: Optional[str] = None, project: Optional[Project] = None, wait_for_metadata: bool = True,
client: Optional[Client] = None, **config
):
datasource_type = CONNECTOR_TO_DATASOURCE.get(connector.type)
self._init_common(client=client)
self._model: Optional[mDataSource] = self._create_model(
connector=connector, datasource_type=datasource_type, datatype=datatype, config=config, name=name, client=self._client)
connector=connector, datasource_type=datasource_type, datatype=datatype,
config=config, name=name, client=self._client)

if wait_for_metadata:
self._model = DataSource._wait_for_metadata(self)._model

self._project = project

@init_client
def _init_common(self, client: Optional[Client] = None):
self._client = client
Expand Down Expand Up @@ -73,11 +81,12 @@ def metadata(self) -> Metadata:

@staticmethod
@init_client
def list(client: Optional[Client] = None) -> DataSourceList:
def list(project: Optional[Project] = None, client: Optional[Client] = None) -> DataSourceList:
"""List the [`DataSource`][ydata.sdk.datasources.DataSource]
instances.

Arguments:
project (Optional[Project]): (optional) Project name from where to list the datasources
client (Client): (optional) Client to connect to the backend

Returns:
Expand All @@ -90,40 +99,47 @@ def __process_data(data: list) -> list:
e.pop(k, None)
return data

response = client.get('/datasource')
response = client.get('/datasource', project=project)
data: list = response.json()
data = __process_data(data)

return DataSourceList(data)

@staticmethod
@init_client
def get(uid: UID, client: Optional[Client] = None) -> "DataSource":
def get(uid: UID, project: Optional[Project] = None, client: Optional[Client] = None) -> "DataSource":
"""Get an existing [`DataSource`][ydata.sdk.datasources.DataSource].

Arguments:
uid (UID): DataSource identifier
project (Optional[Project]): (optional) Project name from where to get the connector
client (Client): (optional) Client to connect to the backend

Returns:
DataSource
"""
response = client.get(f'/datasource/{uid}')
response = client.get(f'/datasource/{uid}', project=project)
data: list = response.json()
datasource_type = CONNECTOR_TO_DATASOURCE.get(
ConnectorType(data['connector']['type']))
model = DataSource._model_from_api(data, datasource_type)
datasource = ModelFactoryMixin._init_from_model_data(DataSource, model)
datasource._project = project
return datasource

@classmethod
def create(cls, connector: Connector, datatype: Optional[Union[DataSourceType, str]] = DataSourceType.TABULAR, name: Optional[str] = None, wait_for_metadata: bool = True, client: Optional[Client] = None, **config) -> "DataSource":
def create(
cls, connector: Connector, datatype: Optional[Union[DataSourceType, str]] = DataSourceType.TABULAR,
name: Optional[str] = None, project: Optional[Project] = None, wait_for_metadata: bool = True,
client: Optional[Client] = None, **config
) -> "DataSource":
"""Create a new [`DataSource`][ydata.sdk.datasources.DataSource].

Arguments:
connector (Connector): Connector from which the datasource is created
datatype (Optional[Union[DataSourceType, str]]): (optional) DataSource type
name (Optional[str]): (optional) DataSource name
project (Optional[Project]): (optional) Project name for this datasource
wait_for_metadata (bool): If `True`, wait until the metadata is fully calculated
client (Client): (optional) Client to connect to the backend
**config: Datasource specific configuration
Expand All @@ -132,22 +148,35 @@ def create(cls, connector: Connector, datatype: Optional[Union[DataSourceType, s
DataSource
"""
datasource_type = CONNECTOR_TO_DATASOURCE.get(connector.type)
return cls._create(connector=connector, datasource_type=datasource_type, datatype=datatype, config=config, name=name, wait_for_metadata=wait_for_metadata, client=client)
return cls._create(
connector=connector, datasource_type=datasource_type, datatype=datatype, config=config, name=name,
project=project, wait_for_metadata=wait_for_metadata, client=client)

@classmethod
def _create(cls, connector: Connector, datasource_type: Type[mDataSource], datatype: Optional[Union[DataSourceType, str]] = DataSourceType.TABULAR, config: Optional[Dict] = None, name: Optional[str] = None, wait_for_metadata: bool = True, client: Optional[Client] = None) -> "DataSource":
def _create(
cls, connector: Connector, datasource_type: Type[mDataSource],
datatype: Optional[Union[DataSourceType, str]] = DataSourceType.TABULAR, config: Optional[Dict] = None,
name: Optional[str] = None, project: Optional[Project] = None, wait_for_metadata: bool = True,
client: Optional[Client] = None
) -> "DataSource":
model = DataSource._create_model(
connector, datasource_type, datatype, config, name, client)
connector, datasource_type, datatype, config, name, project, client)
datasource = ModelFactoryMixin._init_from_model_data(DataSource, model)

if wait_for_metadata:
datasource._model = DataSource._wait_for_metadata(datasource)._model

datasource._project = project

return datasource

@classmethod
@init_client
def _create_model(cls, connector: Connector, datasource_type: Type[mDataSource], datatype: Optional[Union[DataSourceType, str]] = DataSourceType.TABULAR, config: Optional[Dict] = None, name: Optional[str] = None, client: Optional[Client] = None) -> mDataSource:
def _create_model(
cls, connector: Connector, datasource_type: Type[mDataSource],
datatype: Optional[Union[DataSourceType, str]] = DataSourceType.TABULAR, config: Optional[Dict] = None,
name: Optional[str] = None, project: Optional[Project] = None, client: Optional[Client] = None
) -> mDataSource:
_name = name if name is not None else str(uuid4())
_config = config if config is not None else {}
payload = {
Expand All @@ -161,7 +190,7 @@ def _create_model(cls, connector: Connector, datasource_type: Type[mDataSource],
if connector.type != ConnectorType.FILE:
_config = datasource_type(**config).to_payload()
payload.update(_config)
response = client.post('/datasource/', json=payload)
response = client.post('/datasource/', project=project, json=payload)
data: list = response.json()
return DataSource._model_from_api(data, datasource_type)

Expand Down