-
Notifications
You must be signed in to change notification settings - Fork 20
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Adding Databases object #339
base: master
Are you sure you want to change the base?
Changes from 1 commit
a3e35d8
afd5ba5
f03f941
9ace785
1da8883
9b3c88a
a3e4d74
3e002cd
47f8b0c
5200b3b
a01af1e
8d7bea7
686aea4
573ae50
2520b7f
79b44bc
2cfabf3
a7da898
b837bc8
9433b1f
761e114
96c227c
ad6b5fd
539c75c
264ba9a
dd7d22d
5fd39bc
0f64c3d
d94e5a7
b26fefc
811ed11
b443b1c
3c6f952
29b481e
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
- Loading branch information
There are no files selected for viewing
Large diffs are not rendered by default.
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -15,14 +15,14 @@ | |
from abc import ABC, abstractmethod | ||
import json | ||
from pathlib import Path | ||
from typing import Any, Optional, Callable, Dict, List | ||
from typing import Any, Optional, Callable, Dict, List, Union | ||
|
||
from kgforge.core import Resource | ||
from kgforge.core.commons.context import Context | ||
from kgforge.core.archetypes import Mapping, Model | ||
from kgforge.core.commons.attributes import repr_class | ||
from kgforge.core.commons.exceptions import ConfigurationError | ||
from kgforge.core.commons.execution import not_supported | ||
from kgforge.core.commons.dictionaries import with_defaults | ||
from kgforge.core.commons.imports import import_class | ||
from kgforge.core.commons.dictionaries import with_defaults | ||
|
||
|
@@ -35,9 +35,10 @@ class Database(ABC): | |
# POLICY Implementations should not add methods but private functions in the file. | ||
# POLICY Implementations should pass tests/specializations/databases/test_databases.py. | ||
|
||
def __init__(self, source: str, **config) -> None: | ||
def __init__(self, forge : Optional["KnowledgeGraphForge"], source: str, **config) -> None: | ||
# POLICY Resolver data access should be lazy, unless it takes less than a second. | ||
# POLICY There could be data caching but it should be aware of changes made in the source. | ||
self._forge: Optional["KnowledgeGraphForge"] = forge | ||
# Model | ||
model_config = config.pop("model") | ||
if model_config.get('origin') == 'directory': | ||
|
@@ -51,9 +52,10 @@ def __init__(self, source: str, **config) -> None: | |
with open(context_path, 'r') as jfile: | ||
context_file = json.load(jfile) | ||
self.context = Context(context_file, iri) | ||
if 'model_context' not in config: | ||
config['model_context'] = self.context | ||
except Exception: | ||
self.context = None | ||
config['model_context'] = self.context | ||
elif model_config["origin"] == "store": | ||
with_defaults( | ||
model_config, | ||
|
@@ -65,7 +67,8 @@ def __init__(self, source: str, **config) -> None: | |
model_name = model_config.pop("name") | ||
model = import_class(model_name, "models") | ||
self._model: Model = model(**model_config) | ||
config['model_context'] = self._model.context() | ||
if 'model_context' not in config: | ||
config['model_context'] = self._model.context() | ||
else: | ||
raise NotImplementedError('DB Model not yet implemented.') | ||
self.source: str = source | ||
|
@@ -74,10 +77,6 @@ def __init__(self, source: str, **config) -> None: | |
def __repr__(self) -> str: | ||
return repr_class(self) | ||
|
||
def datatypes(self): | ||
# TODO: add other datatypes used, for instance, inside the mappings | ||
return self.mappings(pretty=False).keys() | ||
|
||
def _mappings(self) -> Dict[str, List[str]]: | ||
try: | ||
dirpath = Path(self._dirpath, "mappings") | ||
|
@@ -107,6 +106,42 @@ def mapping(self, entity: str, type: Callable) -> Mapping: | |
except AttributeError: | ||
raise ConfigurationError('No directory path was found from the configuration.') | ||
|
||
def map_resources(self, resources : Union[List[Resource], Resource], | ||
resource_type : Optional[str] = None) -> Optional[Union[Resource, List[Resource]]]: | ||
datatypes = self.types | ||
mappings = self.mappings() | ||
mapped_resources = [] | ||
for resource in resources: | ||
if resource_type is None: | ||
try: | ||
resource_type = resource.type | ||
except AttributeError: | ||
mapped_resources.append(resource) | ||
if resource_type in datatypes: | ||
mapping_class : Mapping = import_class(mappings[resource_type][0], "mappings") | ||
mapping = self.mapping(resource_type, mapping_class) | ||
mapped_resources.append(self._forge.map(self._forge.as_json(resource), mapping)) | ||
else: | ||
mapped_resources.append(resource) | ||
return mapped_resources | ||
|
||
def datatypes(self): | ||
# TODO: add other datatypes used, for instance, inside the mappings | ||
return list(self.mappings().keys()) | ||
|
||
@abstractmethod | ||
def search(self, resolvers, *filters, **params) -> Resource: | ||
pass | ||
|
||
@abstractmethod | ||
def sparql(self, query: str, debug: bool = False, limit: Optional[int] = None, | ||
offset: Optional[int] = None,**params) -> Resource: | ||
pass | ||
|
||
@abstractmethod | ||
def elastic(self, **params) -> Resource: | ||
pass | ||
|
||
@property | ||
@abstractmethod | ||
def health(self) -> Callable: | ||
|
@@ -127,6 +162,8 @@ def _initialize_service(self, source: str, **source_config) -> Any: | |
return self._service_from_web_service(source, **source_config) | ||
elif origin == "store": | ||
store = import_class(source, "stores") | ||
if source != 'DemoStore': | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Why do you need this check? |
||
source_config['store_context'] = self.context | ||
return self._service_from_store(store, **source_config) | ||
else: | ||
raise ConfigurationError(f"unrecognized DataBase origin '{origin}'") | ||
|
Original file line number | Diff line number | Diff line change | ||
---|---|---|---|---|
|
@@ -600,7 +600,7 @@ def replace(match: Match) -> str: | |||
return f"{pfx}\n{qr}" | ||||
|
||||
|
||||
def build_construct_query(data, context): | ||||
def resources_from_construct_query(data, context): | ||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. fun, we have the same one!
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. ha! I guess it was a "clear need" There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. for this and the next, what would be the cleanest solution? I would like to use your code as it looks more organized, and I agree it fits well in the SPARQLQueryBuilder There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Cleanest solution in what sense? (I've put the resource building methods in SparqlQueryBuilder following @MFSY 's comment, but I am of the opinion that there should be another Class for sparql response parsing, as it has nothing to do with query building. I guess the query builder can be treated as a set of helper functions for sparql, in which case maybe renaming it would be better) There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Yes, the latter is what it's happening. I would not mind also to have just a class for parsing. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Yes I'm not too sure about that, that's what I had meant on slack that i'd be reluctant to start an implementation for the read only store because it can benefit from work in our 2 branches There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I think it's a matter of which is merged first. But I don't think you should try to make the changes on this branch fit the ones on mine, that will happen when any of the branches needs to rebase to master |
||||
subject_triples = {} | ||||
for r in data["results"]["bindings"]: | ||||
subject = r["subject"]["value"] | ||||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -524,21 +524,23 @@ def db_sources(self, mappings: Optional[List[str]] = None, | |
:param pretty: a boolean | ||
:return: Optional[List[str]] | ||
""" | ||
if mappings is not None: | ||
if mappings is None: | ||
sources = self._db_sources | ||
else: | ||
sources = {} | ||
if isinstance(mappings, list): | ||
for type in mappings: | ||
for db in self._db_sources: | ||
if type in self._db_sources[db].datatypes(): | ||
types = self._db_sources[db].datatypes() | ||
if type in types: | ||
sources[db] = self._db_sources[db] | ||
else: | ||
for db in self._db_sources: | ||
if mappings in self._db_sources[db].datatypes(): | ||
types = self._db_sources[db].datatypes() | ||
if mappings in types: | ||
sources[db] = self._db_sources[db] | ||
if not sources: | ||
print("No Database sources were found for the given datatype(s)") | ||
else: | ||
sources = self._db_sources | ||
print("No Database sources were found for the given type(s)") | ||
if pretty: | ||
print(*["Available Database sources:", *sources], sep="\n") | ||
else: | ||
|
@@ -672,7 +674,10 @@ def search(self, *filters, **params) -> List[Resource]: | |
) | ||
db_source = params.pop('db_source', None) | ||
if db_source: | ||
return self._db_sources[db_source].search(resolvers, *filters, **params) | ||
if db_source in self.db_sources(): | ||
return self._db_sources[db_source].search(resolvers, *filters, **params) | ||
else: | ||
raise AttributeError('Selected database was not declared within forge.') | ||
else: | ||
return self._store.search(resolvers, *filters, **params) | ||
|
||
|
@@ -697,7 +702,10 @@ def sparql( | |
""" | ||
db_source = params.pop('db_source', None) | ||
if db_source: | ||
return self._db_sources[db_source].sparql(query, debug, limit, offset, **params) | ||
if db_source in self.db_sources(): | ||
return self._db_sources[db_source].sparql(query, debug, limit, offset, **params) | ||
else: | ||
raise AttributeError('Selected database was not declared within forge.') | ||
else: | ||
return self._store.sparql(query, debug, limit, offset, **params) | ||
|
||
|
@@ -720,7 +728,10 @@ def elastic( | |
:return: List[Resource] | ||
""" | ||
if db_source: | ||
return self._db_sources[db_source].elastic(query, debug, limit, offset) | ||
if db_source in self.db_sources(): | ||
return self._db_sources[db_source].elastic(query, debug, limit, offset) | ||
else: | ||
raise AttributeError('Selected database was not declared within forge.') | ||
else: | ||
return self._store.elastic(query, debug, limit, offset) | ||
|
||
|
@@ -992,7 +1003,7 @@ def get_model_context(self): | |
|
||
def create_db_sources(self, all_config: Optional[Dict[str, Dict[str, str]]], | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This doesn't seem to be called anywhere? |
||
store_config : Optional[Dict[str, Dict[str, str]]], | ||
model_config : Optional[Dict[str, Dict[str, str]]] | ||
model_context: Context | ||
) -> Union[Database, List[Database]]: | ||
crisely09 marked this conversation as resolved.
Show resolved
Hide resolved
|
||
names = all_config.keys() | ||
dbs = {} | ||
|
@@ -1001,17 +1012,15 @@ def create_db_sources(self, all_config: Optional[Dict[str, Dict[str, str]]], | |
origin = config.get('origin') | ||
if origin == 'store': | ||
source = config.get('source') | ||
# Provide store and model configuration to the database sources | ||
if "model" not in config: | ||
config.update(model=deepcopy(model_config)) | ||
# Complete configuration of the db store in case is the same | ||
if source == store_config['name']: | ||
# Reuse complete configuration of the store when Nexus is called | ||
if source == store_config['name'] == 'BlueBrainNexus': | ||
store_copy = deepcopy(store_config) | ||
with_defaults(config, store_copy, | ||
"source", "name", | ||
store_copy.keys()) | ||
crisely09 marked this conversation as resolved.
Show resolved
Hide resolved
|
||
config['model_context'] = model_context | ||
config.update(origin=origin) | ||
config.update(origin=origin) | ||
print('Configuration', config) | ||
config['name'] = name | ||
dbs[name] = StoreDatabase(self, **config) | ||
else: | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Why do you need this check?