From b8dadadfdc4641d66aaa1b4a0b318b0ed8faaa7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Such=C3=A1nek?= Date: Fri, 28 Jul 2023 15:32:13 +0200 Subject: [PATCH] WIP: Use dedicated repository for drafts --- pom.xml | 5 + .../metadata/GenericController.java | 105 +++++++-------- .../metadata/GenericMetaController.java | 2 +- .../api/dto/settings/SettingsDTO.java | 4 +- .../config/RepositoryConfig.java | 74 ++++++----- .../RepositoryConnectionProperties.java | 109 ++++++++++++++++ .../properties/RepositoryProperties.java | 73 +---------- .../metadata/RdfMetadataMigration.java | 3 +- .../production/Rdf_Migration_0001_Init.java | 5 +- .../Rdf_Migration_0002_Metadata_Draft.java | 5 +- .../production/Rdf_Migration_0003_FDPO.java | 11 +- .../Rdf_Migration_0004_Cleanup_Index.java | 8 +- .../rdf/repository/RepositoryMode.java | 7 + .../catalog/CatalogMetadataRepository.java | 3 +- .../CatalogMetadataRepositoryImpl.java | 18 ++- .../common/AbstractMetadataRepository.java | 109 ++++++++++------ .../repository/common/MetadataRepository.java | 25 ++-- .../GenericMetadataRepositoryImpl.java | 28 ++-- .../service/dashboard/DashboardService.java | 5 +- .../index/entry/IndexEntryService.java | 3 +- .../index/harvester/HarvesterService.java | 7 +- .../catalog/CatalogMetadataService.java | 7 +- .../common/AbstractMetadataService.java | 61 +++++---- .../metadata/common/MetadataService.java | 5 + .../metadata/state/MetadataStateService.java | 121 +++--------------- .../metadata/validator/MetadataValidator.java | 3 +- .../service/reset/ResetService.java | 9 +- .../service/search/SearchService.java | 24 +--- .../service/settings/SettingsMapper.java | 29 +++-- src/main/resources/application.yml | 47 ++++--- 30 files changed, 497 insertions(+), 418 deletions(-) create mode 100644 src/main/java/nl/dtls/fairdatapoint/config/properties/RepositoryConnectionProperties.java create mode 100644 src/main/java/nl/dtls/fairdatapoint/database/rdf/repository/RepositoryMode.java diff --git a/pom.xml b/pom.xml index 834d820d5..f84805d5d 100644 --- a/pom.xml +++ b/pom.xml @@ -225,6 +225,11 @@ rdf4j-sail-nativerdf ${rdf4j.version} + + org.eclipse.rdf4j + rdf4j-tools-federation + ${rdf4j.version} + io.jsonwebtoken jjwt-api diff --git a/src/main/java/nl/dtls/fairdatapoint/api/controller/metadata/GenericController.java b/src/main/java/nl/dtls/fairdatapoint/api/controller/metadata/GenericController.java index 5cb966964..4eae955da 100644 --- a/src/main/java/nl/dtls/fairdatapoint/api/controller/metadata/GenericController.java +++ b/src/main/java/nl/dtls/fairdatapoint/api/controller/metadata/GenericController.java @@ -24,6 +24,7 @@ import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.tags.Tag; +import nl.dtls.fairdatapoint.database.rdf.repository.RepositoryMode; import nl.dtls.fairdatapoint.database.rdf.repository.exception.MetadataRepositoryException; import nl.dtls.fairdatapoint.database.rdf.repository.generic.GenericMetadataRepository; import nl.dtls.fairdatapoint.entity.exception.ForbiddenException; @@ -89,9 +90,6 @@ public class GenericController { @Autowired private MetadataSchemaService metadataSchemaService; - @Autowired - private MetadataStateService metadataStateService; - @Autowired private MetadataEnhancer metadataEnhancer; @@ -101,6 +99,9 @@ public class GenericController { @Autowired private GenericMetadataRepository metadataRepository; + @Autowired + private MetadataStateService metadataStateService; + @Autowired private SearchFilterCache searchFilterCache; @@ -130,28 +131,23 @@ public Model getMetaDataExpanded( final MetadataService metadataService = metadataServiceFactory.getMetadataServiceByUrlPrefix(urlPrefix); final ResourceDefinition rd = resourceDefinitionService.getByUrlPrefix(urlPrefix); - // 2. Get entity + // 2. Get entity (from repository based on permissions) + final Optional oCurrentUser = currentUserService.getCurrentUser(); IRI entityUri = getMetadataIRI(persistentUrl, urlPrefix, recordId); - Model entity = metadataService.retrieve(entityUri); + RepositoryMode mode = oCurrentUser.isEmpty() ? RepositoryMode.MAIN : RepositoryMode.COMBINED; + Model entity = metadataService.retrieve(entityUri, mode); resultRdf.addAll(entity); - // 3. Check if it is DRAFT - final Metadata state = metadataStateService.get(entityUri); - final Optional oCurrentUser = currentUserService.getCurrentUser(); - if (state.getState().equals(MetadataState.DRAFT) && oCurrentUser.isEmpty()) { - throw new ForbiddenException(MSG_ERROR_DRAFT_FORBIDDEN); - } - - // 4. Enhance + // 3. Enhance metadataEnhancer.enhanceWithResourceDefinition(entityUri, rd, resultRdf); - // 5. Get parent + // 4. Get parent while (true) { final IRI parentUri = i(getStringObjectBy(entity, entityUri, DCTERMS.IS_PART_OF)); if (parentUri == null) { break; } - final Model parent = metadataService.retrieve(parentUri); + final Model parent = metadataService.retrieve(parentUri, mode); resultRdf.addAll(parent); entity = parent; entityUri = parentUri; @@ -179,34 +175,30 @@ public Model getMetaData( // 2. Get resource definition final ResourceDefinition rd = resourceDefinitionService.getByUrlPrefix(urlPrefix); - // 3. Get entity - final IRI entityUri = getMetadataIRI(persistentUrl, urlPrefix, recordId); - final Model entity = metadataService.retrieve(entityUri); - resultRdf.addAll(entity); - - // 4. Check if it is DRAFT - final Metadata state = metadataStateService.get(entityUri); + // 3. Get entity (from repository based on permissions) final Optional oCurrentUser = currentUserService.getCurrentUser(); - if (state.getState().equals(MetadataState.DRAFT) && oCurrentUser.isEmpty()) { - throw new ForbiddenException(MSG_ERROR_DRAFT_FORBIDDEN); - } - - // 5. Filter children - for (ResourceDefinitionChild rdChild : rd.getChildren()) { - final IRI relationUri = i(rdChild.getRelationUri()); - for (org.eclipse.rdf4j.model.Value childUri : getObjectsBy(entity, entityUri, relationUri)) { - final Metadata childState = metadataStateService.get(i(childUri.stringValue())); - if (!(childState.getState().equals(MetadataState.PUBLISHED) || oCurrentUser.isPresent())) { - resultRdf.remove(entityUri, relationUri, childUri); - } - } - } + IRI entityUri = getMetadataIRI(persistentUrl, urlPrefix, recordId); + RepositoryMode mode = oCurrentUser.isEmpty() ? RepositoryMode.MAIN : RepositoryMode.COMBINED; + Model entity = metadataService.retrieve(entityUri, mode); + resultRdf.addAll(entity); - // 6. Add links + // 4. Filter children + // TODO: get drafts only if permissions +// for (ResourceDefinitionChild rdChild : rd.getChildren()) { +// final IRI relationUri = i(rdChild.getRelationUri()); +// for (org.eclipse.rdf4j.model.Value childUri : getObjectsBy(entity, entityUri, relationUri)) { +// final Metadata childState = metadataStateService.get(i(childUri.stringValue())); +// if (!(childState.getState().equals(MetadataState.PUBLISHED) || oCurrentUser.isPresent())) { +// resultRdf.remove(entityUri, relationUri, childUri); +// } +// } +// } + + // 5. Add links metadataEnhancer.enhanceWithLinks(entityUri, entity, rd, persistentUrl, resultRdf); metadataEnhancer.enhanceWithResourceDefinition(entityUri, rd, resultRdf); - // 7. Create response + // 6. Create response return resultRdf; } @@ -225,7 +217,6 @@ public ResponseEntity storeMetaData( } // 2. Init - // String urlPrefix = getResourceNameForList(getRequestURL(request, persistentUrl)); final MetadataService metadataService = metadataServiceFactory.getMetadataServiceByUrlPrefix(urlPrefix); final ResourceDefinition rd = resourceDefinitionService.getByUrlPrefix(urlPrefix); @@ -283,6 +274,7 @@ public ResponseEntity updateMetaData( } // 4. Store metadata + // TODO: make sure that updating draft works as well final Model metadata = metadataService.update(reqDto, uri, rd, true); // 5. Invalidate search filters cache @@ -314,6 +306,7 @@ public ResponseEntity deleteMetadata( final IRI uri = getMetadataIRI(persistentUrl, urlPrefix, recordId); // 4. Store metadata + // TODO: make sure that updating draft works as well metadataService.delete(uri, rd); // 5. Invalidate search filters cache @@ -341,18 +334,14 @@ public ResponseEntity getMetaDataChildren( final String recordId = oRecordId.orElse(""); final MetadataService metadataService = metadataServiceFactory.getMetadataServiceByUrlPrefix(urlPrefix); - // 2. Get entity - final IRI entityUri = getMetadataIRI(persistentUrl, urlPrefix, recordId); - final Model entity = metadataService.retrieve(entityUri); - - // 3. Check if it is draft - final Metadata state = metadataStateService.get(entityUri); + // 2. Get entity (from repository based on permissions) final Optional oCurrentUser = currentUserService.getCurrentUser(); - if (state.getState().equals(MetadataState.DRAFT) && oCurrentUser.isEmpty()) { - throw new ForbiddenException(MSG_ERROR_DRAFT_FORBIDDEN); - } + IRI entityUri = getMetadataIRI(persistentUrl, urlPrefix, recordId); + RepositoryMode mode = oCurrentUser.isEmpty() ? RepositoryMode.MAIN : RepositoryMode.COMBINED; + Model entity = metadataService.retrieve(entityUri, mode); + resultRdf.addAll(entity); - // 4. Get Children + // 3. Get Children final ResourceDefinition rd = resourceDefinitionService.getByUrlPrefix(urlPrefix); final ResourceDefinition currentChildRd = resourceDefinitionService.getByUrlPrefix(childPrefix); final MetadataService childMetadataService = metadataServiceFactory.getMetadataServiceByUrlPrefix(childPrefix); @@ -361,19 +350,19 @@ public ResponseEntity getMetaDataChildren( if (rdChild.getResourceDefinitionUuid().equals(currentChildRd.getUuid())) { final IRI relationUri = i(rdChild.getRelationUri()); - // 4.1 Get all titles for sort - final Map titles = metadataRepository.findChildTitles(entityUri, relationUri); + // 3.1 Get all titles for sort + final Map titles = metadataRepository.findChildTitles(entityUri, relationUri, mode); - // 4.2 Get all children sorted + // 3.2 Get all children sorted final List children = getObjectsBy(entity, entityUri, relationUri) .stream() .filter(childUri -> getResourceNameForChild(childUri.toString()).equals(childPrefix)) .filter(childUri -> { - if (oCurrentUser.isPresent()) { - return true; + try { + return oCurrentUser.isPresent() || metadataStateService.isPublished(i(childUri.stringValue())); + } catch (MetadataServiceException e) { + return false; } - final Metadata childState = metadataStateService.get(i(childUri.stringValue())); - return childState.getState().equals(MetadataState.PUBLISHED); }) .sorted((value1, value2) -> { final String title1 = titles.get(value1.toString()); @@ -382,14 +371,14 @@ public ResponseEntity getMetaDataChildren( }) .toList(); - // 4.3 Retrieve children metadata only for requested page + // 3.3 Retrieve children metadata only for requested page final int childrenCount = children.size(); children.stream().skip((long) page * size).limit(size) .map(childUri -> retrieveChildModel(childMetadataService, childUri)) .flatMap(Optional::stream) .forEach(resultRdf::addAll); - // 4.4 Set Link headers and send response + // 3.4 Set Link headers and send response final HttpHeaders responseHeaders = new HttpHeaders(); responseHeaders.set( "Link", diff --git a/src/main/java/nl/dtls/fairdatapoint/api/controller/metadata/GenericMetaController.java b/src/main/java/nl/dtls/fairdatapoint/api/controller/metadata/GenericMetaController.java index 9aebcb985..8913ab23a 100644 --- a/src/main/java/nl/dtls/fairdatapoint/api/controller/metadata/GenericMetaController.java +++ b/src/main/java/nl/dtls/fairdatapoint/api/controller/metadata/GenericMetaController.java @@ -103,7 +103,7 @@ public MetaDTO getMeta( final MemberDTO member = oMember.orElse(new MemberDTO(null, null)); // 5. Get state - final MetaStateDTO state = metadataStateService.getState(entityUri, entity, definition); + final MetaStateDTO state = metadataStateService.getStateDTO(entityUri, entity, definition); // 6. Make path map final Map pathMap = new HashMap<>(); diff --git a/src/main/java/nl/dtls/fairdatapoint/api/dto/settings/SettingsDTO.java b/src/main/java/nl/dtls/fairdatapoint/api/dto/settings/SettingsDTO.java index c32691e27..cdba5d5d4 100644 --- a/src/main/java/nl/dtls/fairdatapoint/api/dto/settings/SettingsDTO.java +++ b/src/main/java/nl/dtls/fairdatapoint/api/dto/settings/SettingsDTO.java @@ -53,7 +53,9 @@ public class SettingsDTO { private SettingsPingDTO ping; - private SettingsRepositoryDTO repository; + private SettingsRepositoryDTO mainRepository; + + private SettingsRepositoryDTO draftsRepository; private SettingsSearchDTO search; diff --git a/src/main/java/nl/dtls/fairdatapoint/config/RepositoryConfig.java b/src/main/java/nl/dtls/fairdatapoint/config/RepositoryConfig.java index 09faef725..831643aa0 100644 --- a/src/main/java/nl/dtls/fairdatapoint/config/RepositoryConfig.java +++ b/src/main/java/nl/dtls/fairdatapoint/config/RepositoryConfig.java @@ -23,6 +23,7 @@ package nl.dtls.fairdatapoint.config; import lombok.extern.slf4j.Slf4j; +import nl.dtls.fairdatapoint.config.properties.RepositoryConnectionProperties; import nl.dtls.fairdatapoint.config.properties.RepositoryProperties; import org.eclipse.rdf4j.repository.Repository; import org.eclipse.rdf4j.repository.RepositoryException; @@ -51,16 +52,25 @@ public class RepositoryConfig { @Autowired private RepositoryProperties repositoryProperties; - @Bean(initMethod = "init", destroyMethod = "shutDown") - public Repository repository(ApplicationContext context) + @Bean(initMethod = "init", destroyMethod = "shutDown", name = "mainRepository") + public Repository mainRepository(ApplicationContext context) throws RepositoryException { + return prepareRepository(context, repositoryProperties.getMain()); + } + + @Bean(initMethod = "init", destroyMethod = "shutDown", name = "draftsRepository") + public Repository draftsRepository(ApplicationContext context) throws RepositoryException { + return prepareRepository(context, repositoryProperties.getMain()); + } + + public Repository prepareRepository(ApplicationContext context, RepositoryConnectionProperties properties) throws RepositoryException { - final Repository repository = switch (repositoryProperties.getType()) { - case RepositoryProperties.TYPE_IN_MEMORY -> getInMemoryStore(); - case RepositoryProperties.TYPE_NATIVE -> getNativeStore(); - case RepositoryProperties.TYPE_ALLEGRO -> getAgraphRepository(); - case RepositoryProperties.TYPE_GRAPHDB -> getGraphDBRepository(); - case RepositoryProperties.TYPE_BLAZEGRAPH -> getBlazeGraphRepository(); + final Repository repository = switch (properties.getType()) { + case RepositoryConnectionProperties.TYPE_IN_MEMORY -> getInMemoryStore(); + case RepositoryConnectionProperties.TYPE_NATIVE -> getNativeStore(properties); + case RepositoryConnectionProperties.TYPE_ALLEGRO -> getAgraphRepository(properties); + case RepositoryConnectionProperties.TYPE_GRAPHDB -> getGraphDBRepository(properties); + case RepositoryConnectionProperties.TYPE_BLAZEGRAPH -> getBlazeGraphRepository(properties); default -> null; }; @@ -81,26 +91,26 @@ private Repository getInMemoryStore() { return new SailRepository(store); } - private Repository getNativeStore() { + private Repository getNativeStore(RepositoryConnectionProperties properties) { log.info("Setting up Native Store"); - if (!repositoryProperties.getNativeRepo().getDir().isEmpty()) { - final File dataDir = new File(repositoryProperties.getNativeRepo().getDir()); + if (!properties.getNativeRepo().getDir().isEmpty()) { + final File dataDir = new File(properties.getNativeRepo().getDir()); return new SailRepository(new NativeStore(dataDir)); } log.warn("'repository.native.dir' is empty"); return null; } - private Repository getAgraphRepository() { + private Repository getAgraphRepository(RepositoryConnectionProperties properties) { log.info("Setting up Allegro Graph Store"); - if (!repositoryProperties.getAgraph().getUrl().isEmpty()) { + if (!properties.getAgraph().getUrl().isEmpty()) { final SPARQLRepository repository = - new SPARQLRepository(repositoryProperties.getAgraph().getUrl()); - if (!repositoryProperties.getAgraph().getUsername().isEmpty() - && !repositoryProperties.getAgraph().getPassword().isEmpty()) { + new SPARQLRepository(properties.getAgraph().getUrl()); + if (!properties.getAgraph().getUsername().isEmpty() + && !properties.getAgraph().getPassword().isEmpty()) { repository.setUsernameAndPassword( - repositoryProperties.getAgraph().getUsername(), - repositoryProperties.getAgraph().getPassword() + properties.getAgraph().getUsername(), + properties.getAgraph().getPassword() ); } return repository; @@ -109,17 +119,17 @@ private Repository getAgraphRepository() { return null; } - private Repository getBlazeGraphRepository() { + private Repository getBlazeGraphRepository(RepositoryConnectionProperties properties) { log.info("Setting up Blaze Graph Store"); - String blazegraphUrl = repositoryProperties.getBlazegraph().getUrl(); + String blazegraphUrl = properties.getBlazegraph().getUrl(); if (!blazegraphUrl.isEmpty()) { blazegraphUrl = removeLastSlash(blazegraphUrl); // Build url for blazegraph (Eg: http://localhost:8079/bigdata/namespace/test1/sparql) final StringBuilder urlBuilder = new StringBuilder(); urlBuilder.append(blazegraphUrl); urlBuilder.append("/namespace/"); - if (!repositoryProperties.getBlazegraph().getRepository().isEmpty()) { - urlBuilder.append(repositoryProperties.getBlazegraph().getRepository()); + if (!properties.getBlazegraph().getRepository().isEmpty()) { + urlBuilder.append(properties.getBlazegraph().getRepository()); } else { urlBuilder.append("kb"); @@ -131,28 +141,28 @@ private Repository getBlazeGraphRepository() { return null; } - private Repository getGraphDBRepository() { + private Repository getGraphDBRepository(RepositoryConnectionProperties properties) { log.info("Setting up GraphDB Store"); try { System.setProperty("org.eclipse.rdf4j.rio.binary.format_version", "1"); - if (!repositoryProperties.getGraphDb().getUrl().isEmpty() - && !repositoryProperties.getGraphDb().getRepository().isEmpty()) { + if (!properties.getGraphDb().getUrl().isEmpty() + && !properties.getGraphDb().getRepository().isEmpty()) { final RepositoryManager repositoryManager; - if (!repositoryProperties.getGraphDb().getUsername().isEmpty() - && !repositoryProperties.getGraphDb().getPassword().isEmpty()) { + if (!properties.getGraphDb().getUsername().isEmpty() + && !properties.getGraphDb().getPassword().isEmpty()) { repositoryManager = RemoteRepositoryManager.getInstance( - repositoryProperties.getGraphDb().getUrl(), - repositoryProperties.getGraphDb().getUsername(), - repositoryProperties.getGraphDb().getPassword() + properties.getGraphDb().getUrl(), + properties.getGraphDb().getUsername(), + properties.getGraphDb().getPassword() ); } else { repositoryManager = RemoteRepositoryManager.getInstance( - repositoryProperties.getGraphDb().getUrl() + properties.getGraphDb().getUrl() ); } return repositoryManager.getRepository( - repositoryProperties.getGraphDb().getRepository() + properties.getGraphDb().getRepository() ); } log.warn("'repository.graphDb.url' or 'repository.graphDb.repository' is empty"); diff --git a/src/main/java/nl/dtls/fairdatapoint/config/properties/RepositoryConnectionProperties.java b/src/main/java/nl/dtls/fairdatapoint/config/properties/RepositoryConnectionProperties.java new file mode 100644 index 000000000..a1d9d70ab --- /dev/null +++ b/src/main/java/nl/dtls/fairdatapoint/config/properties/RepositoryConnectionProperties.java @@ -0,0 +1,109 @@ +/** + * The MIT License + * Copyright © 2017 DTL + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package nl.dtls.fairdatapoint.config.properties; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import org.springframework.boot.context.properties.ConfigurationProperties; + +@NoArgsConstructor +@AllArgsConstructor +@Getter +@Setter +public class RepositoryConnectionProperties { + // TODO: use polymorphism for types of repository + + public static final int TYPE_IN_MEMORY = 1; + + public static final int TYPE_NATIVE = 2; + + public static final int TYPE_ALLEGRO = 3; + + public static final int TYPE_GRAPHDB = 4; + + public static final int TYPE_BLAZEGRAPH = 5; + + private int type; + private RepositoryNativeProperties nativeRepo; + private RepositoryBasicProperties agraph; + private RepositoryBasicProperties graphDb; + private RepositoryBasicProperties blazegraph; + + public void setNative(RepositoryNativeProperties repositoryNativeProperties) { + this.nativeRepo = repositoryNativeProperties; + } + + public String getStringType() { + return switch (type) { + case TYPE_IN_MEMORY -> "InMemory"; + case TYPE_NATIVE -> "Native"; + case TYPE_ALLEGRO -> "AllegroGraph"; + case TYPE_GRAPHDB -> "GraphDB"; + case TYPE_BLAZEGRAPH -> "Blazegraph"; + default -> "Invalid"; + }; + } + + public String getDir() { + if (type == TYPE_NATIVE) { + return nativeRepo.getDir(); + } + return null; + } + + public String getUrl() { + return switch (type) { + case TYPE_ALLEGRO -> agraph.getUrl(); + case TYPE_GRAPHDB -> graphDb.getUrl(); + case TYPE_BLAZEGRAPH -> blazegraph.getUrl(); + default -> null; + }; + } + + public String getRepository() { + return switch (type) { + case TYPE_ALLEGRO -> agraph.getRepository(); + case TYPE_GRAPHDB -> graphDb.getRepository(); + case TYPE_BLAZEGRAPH -> blazegraph.getRepository(); + default -> null; + }; + } + + public String getUsername() { + return switch (type) { + case TYPE_ALLEGRO -> agraph.getUsername(); + case TYPE_GRAPHDB -> graphDb.getUsername(); + default -> null; + }; + } + + public String getPassword() { + return switch (type) { + case TYPE_ALLEGRO -> agraph.getPassword(); + case TYPE_GRAPHDB -> graphDb.getPassword(); + default -> null; + }; + } +} diff --git a/src/main/java/nl/dtls/fairdatapoint/config/properties/RepositoryProperties.java b/src/main/java/nl/dtls/fairdatapoint/config/properties/RepositoryProperties.java index f6b610256..5256f0a6b 100644 --- a/src/main/java/nl/dtls/fairdatapoint/config/properties/RepositoryProperties.java +++ b/src/main/java/nl/dtls/fairdatapoint/config/properties/RepositoryProperties.java @@ -34,77 +34,8 @@ @Setter @ConfigurationProperties(prefix = "repository") public class RepositoryProperties { - // TODO: use polymorphism for types of repository - public static final int TYPE_IN_MEMORY = 1; + private RepositoryConnectionProperties main; - public static final int TYPE_NATIVE = 2; - - public static final int TYPE_ALLEGRO = 3; - - public static final int TYPE_GRAPHDB = 4; - - public static final int TYPE_BLAZEGRAPH = 5; - - private int type; - private RepositoryNativeProperties nativeRepo; - private RepositoryBasicProperties agraph; - private RepositoryBasicProperties graphDb; - private RepositoryBasicProperties blazegraph; - - public void setNative(RepositoryNativeProperties repositoryNativeProperties) { - this.nativeRepo = repositoryNativeProperties; - } - - public String getStringType() { - return switch (type) { - case TYPE_IN_MEMORY -> "InMemory"; - case TYPE_NATIVE -> "Native"; - case TYPE_ALLEGRO -> "AllegroGraph"; - case TYPE_GRAPHDB -> "GraphDB"; - case TYPE_BLAZEGRAPH -> "Blazegraph"; - default -> "Invalid"; - }; - } - - public String getDir() { - if (type == TYPE_NATIVE) { - return nativeRepo.getDir(); - } - return null; - } - - public String getUrl() { - return switch (type) { - case TYPE_ALLEGRO -> agraph.getUrl(); - case TYPE_GRAPHDB -> graphDb.getUrl(); - case TYPE_BLAZEGRAPH -> blazegraph.getUrl(); - default -> null; - }; - } - - public String getRepository() { - return switch (type) { - case TYPE_ALLEGRO -> agraph.getRepository(); - case TYPE_GRAPHDB -> graphDb.getRepository(); - case TYPE_BLAZEGRAPH -> blazegraph.getRepository(); - default -> null; - }; - } - - public String getUsername() { - return switch (type) { - case TYPE_ALLEGRO -> agraph.getUsername(); - case TYPE_GRAPHDB -> graphDb.getUsername(); - default -> null; - }; - } - - public String getPassword() { - return switch (type) { - case TYPE_ALLEGRO -> agraph.getPassword(); - case TYPE_GRAPHDB -> graphDb.getPassword(); - default -> null; - }; - } + private RepositoryConnectionProperties drafts; } diff --git a/src/main/java/nl/dtls/fairdatapoint/database/rdf/migration/development/metadata/RdfMetadataMigration.java b/src/main/java/nl/dtls/fairdatapoint/database/rdf/migration/development/metadata/RdfMetadataMigration.java index 67e3def0b..20479da3a 100644 --- a/src/main/java/nl/dtls/fairdatapoint/database/rdf/migration/development/metadata/RdfMetadataMigration.java +++ b/src/main/java/nl/dtls/fairdatapoint/database/rdf/migration/development/metadata/RdfMetadataMigration.java @@ -27,6 +27,7 @@ import nl.dtls.fairdatapoint.database.mongo.migration.development.resource.data.ResourceDefinitionFixtures; import nl.dtls.fairdatapoint.database.mongo.migration.development.user.data.UserFixtures; import nl.dtls.fairdatapoint.database.rdf.migration.development.metadata.data.RdfMetadataFixtures; +import nl.dtls.fairdatapoint.database.rdf.repository.RepositoryMode; import nl.dtls.fairdatapoint.database.rdf.repository.exception.MetadataRepositoryException; import nl.dtls.fairdatapoint.database.rdf.repository.generic.GenericMetadataRepository; import nl.dtls.fairdatapoint.entity.metadata.MetadataState; @@ -82,7 +83,7 @@ public class RdfMetadataMigration implements Migration { public void runMigration() { try { // 1. Remove all previous metadata - metadataRepository.removeAll(); + metadataRepository.removeAll(RepositoryMode.COMBINED); // 2. Auth user final String adminUuid = userFixtures.admin().getUuid(); diff --git a/src/main/java/nl/dtls/fairdatapoint/database/rdf/migration/production/Rdf_Migration_0001_Init.java b/src/main/java/nl/dtls/fairdatapoint/database/rdf/migration/production/Rdf_Migration_0001_Init.java index 38cfb083a..fbd985c74 100644 --- a/src/main/java/nl/dtls/fairdatapoint/database/rdf/migration/production/Rdf_Migration_0001_Init.java +++ b/src/main/java/nl/dtls/fairdatapoint/database/rdf/migration/production/Rdf_Migration_0001_Init.java @@ -48,9 +48,10 @@ @Slf4j @Service public class Rdf_Migration_0001_Init implements RdfProductionMigration { + // TODO: squash RDF migrations @Autowired - private Repository repository; + private Repository mainRepository; @Autowired @Qualifier("persistentUrl") @@ -74,7 +75,7 @@ public void runMigration() { } private void createRepositoryInTripleStore() { - try (RepositoryConnection conn = repository.getConnection()) { + try (RepositoryConnection conn = mainRepository.getConnection()) { final List statements = FactoryDefaults.repositoryStatements( persistentUrl, license, diff --git a/src/main/java/nl/dtls/fairdatapoint/database/rdf/migration/production/Rdf_Migration_0002_Metadata_Draft.java b/src/main/java/nl/dtls/fairdatapoint/database/rdf/migration/production/Rdf_Migration_0002_Metadata_Draft.java index 82c14cc6c..4c04d15ec 100644 --- a/src/main/java/nl/dtls/fairdatapoint/database/rdf/migration/production/Rdf_Migration_0002_Metadata_Draft.java +++ b/src/main/java/nl/dtls/fairdatapoint/database/rdf/migration/production/Rdf_Migration_0002_Metadata_Draft.java @@ -42,9 +42,10 @@ @Slf4j @Service public class Rdf_Migration_0002_Metadata_Draft implements RdfProductionMigration { + // TODO: squash RDF migrations @Autowired - private Repository repository; + private Repository mainRepository; @Autowired private MetadataRepository metadataRepository; @@ -54,7 +55,7 @@ public void runMigration() { } private void createRepositoryInTripleStore() { - try (RepositoryConnection conn = repository.getConnection()) { + try (RepositoryConnection conn = mainRepository.getConnection()) { conn.getContextIDs() .stream() .forEach(this::saveMetadataForResource); diff --git a/src/main/java/nl/dtls/fairdatapoint/database/rdf/migration/production/Rdf_Migration_0003_FDPO.java b/src/main/java/nl/dtls/fairdatapoint/database/rdf/migration/production/Rdf_Migration_0003_FDPO.java index 0dcbae49d..95f5aeacf 100644 --- a/src/main/java/nl/dtls/fairdatapoint/database/rdf/migration/production/Rdf_Migration_0003_FDPO.java +++ b/src/main/java/nl/dtls/fairdatapoint/database/rdf/migration/production/Rdf_Migration_0003_FDPO.java @@ -51,6 +51,7 @@ @Slf4j @Service public class Rdf_Migration_0003_FDPO implements RdfProductionMigration { + // TODO: squash RDF migrations private static final String LEGACY_CONFORMS_TO = "https://www.purl.org/fairtools/fdp/schema/0.1/fdpMetadata"; @@ -65,7 +66,7 @@ public class Rdf_Migration_0003_FDPO implements RdfProductionMigration { private static final String MSG_REMOVE = "Removing: {} {} {}"; @Autowired - private Repository repository; + private Repository mainRepository; public void runMigration() { removeOldConformsTo(); @@ -76,7 +77,7 @@ public void runMigration() { private void removeOldConformsTo() { // remove conformsTo for repository if present (https://www.purl.org/fairtools/fdp/schema/0.1/fdpMetadata) - try (RepositoryConnection conn = repository.getConnection()) { + try (RepositoryConnection conn = mainRepository.getConnection()) { final RepositoryResult queryResult = conn.getStatements(null, DCTERMS.CONFORMS_TO, i(LEGACY_CONFORMS_TO)); while (queryResult.hasNext()) { @@ -93,7 +94,7 @@ private void removeOldConformsTo() { private void updateRepositoryStatements() { // change r3d:Repository -> fdp-o:FAIRDataPoint (and dcat:DataService + fdp-o:MetadataService?) final List newTypes = List.of(DCAT.DATA_SERVICE, FDP.METADATASERVICE, FDP.FAIRDATAPOINT); - try (RepositoryConnection conn = repository.getConnection()) { + try (RepositoryConnection conn = mainRepository.getConnection()) { final RepositoryResult queryResult = conn.getStatements(null, RDF.TYPE, R3D.REPOSITORY); while (queryResult.hasNext()) { final Statement st = queryResult.next(); @@ -112,7 +113,7 @@ private void updateRepositoryStatements() { private void updateOldFdpoStatements() { // update old FDP-O generated metadata - try (RepositoryConnection conn = repository.getConnection()) { + try (RepositoryConnection conn = mainRepository.getConnection()) { RepositoryResult queryResult = conn.getStatements(null, OLD_METADATA_IDENTIFIER, null); while (queryResult.hasNext()) { final Statement st = queryResult.next(); @@ -145,7 +146,7 @@ private void updateOldFdpoStatements() { private void updateRepositoryCatalogLinks() { // change r3d:dataCatalog to fdp-o:metadataCatalog property (between Repository/FDP and Catalogs) - try (RepositoryConnection conn = repository.getConnection()) { + try (RepositoryConnection conn = mainRepository.getConnection()) { final RepositoryResult queryResult = conn.getStatements(null, R3D.DATACATALOG, null); while (queryResult.hasNext()) { final Statement st = queryResult.next(); diff --git a/src/main/java/nl/dtls/fairdatapoint/database/rdf/migration/production/Rdf_Migration_0004_Cleanup_Index.java b/src/main/java/nl/dtls/fairdatapoint/database/rdf/migration/production/Rdf_Migration_0004_Cleanup_Index.java index 4e81633d2..63314a6fd 100644 --- a/src/main/java/nl/dtls/fairdatapoint/database/rdf/migration/production/Rdf_Migration_0004_Cleanup_Index.java +++ b/src/main/java/nl/dtls/fairdatapoint/database/rdf/migration/production/Rdf_Migration_0004_Cleanup_Index.java @@ -25,6 +25,7 @@ import lombok.extern.slf4j.Slf4j; import nl.dtls.fairdatapoint.config.properties.InstanceProperties; import nl.dtls.fairdatapoint.database.mongo.repository.IndexEntryRepository; +import nl.dtls.fairdatapoint.database.rdf.repository.RepositoryMode; import nl.dtls.fairdatapoint.database.rdf.repository.exception.MetadataRepositoryException; import nl.dtls.fairdatapoint.database.rdf.repository.generic.GenericMetadataRepository; import nl.dtls.fairdatapoint.entity.index.entry.IndexEntry; @@ -46,9 +47,10 @@ @Slf4j @Service public class Rdf_Migration_0004_Cleanup_Index implements RdfProductionMigration { + // TODO: squash RDF migrations @Autowired - private Repository repository; + private Repository mainRepository; @Autowired private GenericMetadataRepository genericMetadataRepository; @@ -89,7 +91,7 @@ public void cleanupHarvestedRecordsFrom(IndexEntry entry) { return; } - try (RepositoryConnection conn = repository.getConnection()) { + try (RepositoryConnection conn = mainRepository.getConnection()) { conn.getContextIDs() .stream() .filter(Value::isIRI) @@ -98,7 +100,7 @@ public void cleanupHarvestedRecordsFrom(IndexEntry entry) { .forEach(contextId -> { log.info("Deleting harvested records for '{}': {}", entry.getClientUrl(), contextId); try { - genericMetadataRepository.remove(i(contextId)); + genericMetadataRepository.remove(i(contextId), RepositoryMode.COMBINED); } catch (MetadataRepositoryException exception) { throw new RuntimeException(exception); diff --git a/src/main/java/nl/dtls/fairdatapoint/database/rdf/repository/RepositoryMode.java b/src/main/java/nl/dtls/fairdatapoint/database/rdf/repository/RepositoryMode.java new file mode 100644 index 000000000..d75e9f457 --- /dev/null +++ b/src/main/java/nl/dtls/fairdatapoint/database/rdf/repository/RepositoryMode.java @@ -0,0 +1,7 @@ +package nl.dtls.fairdatapoint.database.rdf.repository; + +public enum RepositoryMode { + MAIN, + DRAFTS, + COMBINED +} diff --git a/src/main/java/nl/dtls/fairdatapoint/database/rdf/repository/catalog/CatalogMetadataRepository.java b/src/main/java/nl/dtls/fairdatapoint/database/rdf/repository/catalog/CatalogMetadataRepository.java index 6fbe9f957..5d0dfcf28 100644 --- a/src/main/java/nl/dtls/fairdatapoint/database/rdf/repository/catalog/CatalogMetadataRepository.java +++ b/src/main/java/nl/dtls/fairdatapoint/database/rdf/repository/catalog/CatalogMetadataRepository.java @@ -22,6 +22,7 @@ */ package nl.dtls.fairdatapoint.database.rdf.repository.catalog; +import nl.dtls.fairdatapoint.database.rdf.repository.RepositoryMode; import nl.dtls.fairdatapoint.database.rdf.repository.common.MetadataRepository; import nl.dtls.fairdatapoint.database.rdf.repository.exception.MetadataRepositoryException; import org.eclipse.rdf4j.model.IRI; @@ -30,6 +31,6 @@ public interface CatalogMetadataRepository extends MetadataRepository { - List getDatasetThemesForCatalog(IRI uri) throws MetadataRepositoryException; + List getDatasetThemesForCatalog(IRI uri, RepositoryMode mode) throws MetadataRepositoryException; } diff --git a/src/main/java/nl/dtls/fairdatapoint/database/rdf/repository/catalog/CatalogMetadataRepositoryImpl.java b/src/main/java/nl/dtls/fairdatapoint/database/rdf/repository/catalog/CatalogMetadataRepositoryImpl.java index 97c5391f6..804be4a6f 100644 --- a/src/main/java/nl/dtls/fairdatapoint/database/rdf/repository/catalog/CatalogMetadataRepositoryImpl.java +++ b/src/main/java/nl/dtls/fairdatapoint/database/rdf/repository/catalog/CatalogMetadataRepositoryImpl.java @@ -23,9 +23,11 @@ package nl.dtls.fairdatapoint.database.rdf.repository.catalog; import jakarta.annotation.PostConstruct; +import nl.dtls.fairdatapoint.database.rdf.repository.RepositoryMode; import nl.dtls.fairdatapoint.database.rdf.repository.common.AbstractMetadataRepository; import nl.dtls.fairdatapoint.database.rdf.repository.exception.MetadataRepositoryException; import org.eclipse.rdf4j.model.IRI; +import org.eclipse.rdf4j.repository.Repository; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.cache.Cache; import org.springframework.cache.concurrent.ConcurrentMapCacheManager; @@ -46,21 +48,29 @@ public class CatalogMetadataRepositoryImpl extends AbstractMetadataRepository im @Autowired private ConcurrentMapCacheManager cacheManager; + public CatalogMetadataRepositoryImpl(Repository mainRepository, Repository draftsRepository) { + super(mainRepository, draftsRepository); + } + @PostConstruct public void init() { cacheManager.setCacheNames(List.of(CATALOG_THEMES_CACHE)); } - public List getDatasetThemesForCatalog(IRI uri) throws MetadataRepositoryException { + public List getDatasetThemesForCatalog(IRI uri, RepositoryMode mode) throws MetadataRepositoryException { List result = cache().get(uri.toString(), List.class); if (result != null) { return result; } - result = runSparqlQuery(GET_DATASET_THEMES_FOR_CATALOG, CatalogMetadataRepository.class, Map.of( - "catalog", uri)) + result = runSparqlQuery( + GET_DATASET_THEMES_FOR_CATALOG, + CatalogMetadataRepository.class, + Map.of("catalog", uri), + mode + ) .stream() .map(item -> i(item.getValue("theme").stringValue())) - .collect(Collectors.toList()); + .toList(); cache().put(uri.toString(), result); return result; } diff --git a/src/main/java/nl/dtls/fairdatapoint/database/rdf/repository/common/AbstractMetadataRepository.java b/src/main/java/nl/dtls/fairdatapoint/database/rdf/repository/common/AbstractMetadataRepository.java index 457ef88ce..21bf70628 100644 --- a/src/main/java/nl/dtls/fairdatapoint/database/rdf/repository/common/AbstractMetadataRepository.java +++ b/src/main/java/nl/dtls/fairdatapoint/database/rdf/repository/common/AbstractMetadataRepository.java @@ -25,11 +25,14 @@ import com.google.common.base.Charsets; import com.google.common.io.Resources; import lombok.extern.slf4j.Slf4j; +import nl.dtls.fairdatapoint.database.rdf.repository.RepositoryMode; import nl.dtls.fairdatapoint.database.rdf.repository.exception.MetadataRepositoryException; import nl.dtls.fairdatapoint.entity.search.SearchFilterValue; import nl.dtls.fairdatapoint.entity.search.SearchResult; import nl.dtls.fairdatapoint.entity.search.SearchResultRelation; import org.eclipse.rdf4j.common.iteration.Iterations; +import org.eclipse.rdf4j.federated.FedXFactory; +import org.eclipse.rdf4j.federated.endpoint.EndpointFactory; import org.eclipse.rdf4j.model.*; import org.eclipse.rdf4j.query.BindingSet; import org.eclipse.rdf4j.query.QueryResults; @@ -41,6 +44,7 @@ import java.io.IOException; import java.net.URL; +import java.nio.charset.StandardCharsets; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -74,49 +78,77 @@ public abstract class AbstractMetadataRepository { private static final String FIELD_REL_PRED = "relationPredicate"; private static final String FIELD_REL_OBJ = "relationObject"; - @Autowired - private Repository repository; + private final Repository mainRepository; - protected Repository getRepository() { - return repository; + private final Repository draftsRepository; + + private final Repository combinedRepository; + + public AbstractMetadataRepository(Repository mainRepository, Repository draftsRepository) { + this.mainRepository = mainRepository; + this.draftsRepository = draftsRepository; + this.combinedRepository = FedXFactory.createFederation( + List.of( + EndpointFactory.loadEndpoint("main", mainRepository), + EndpointFactory.loadEndpoint("drafts", draftsRepository) + ) + ); + } + + protected Repository getMainRepository() { + return mainRepository; } - public List findResources() throws MetadataRepositoryException { - try (RepositoryConnection conn = repository.getConnection()) { + protected Repository getDraftsRepository() { + return draftsRepository; + } - return Iterations.asList( - conn.getContextIDs() - ); + protected Repository getCombinedRepository() { + return combinedRepository; + } + + protected Repository getRepository(RepositoryMode mode) { + if (mode == RepositoryMode.DRAFTS) { + return getDraftsRepository(); + } + if (mode == RepositoryMode.COMBINED) { + return getCombinedRepository(); + } + return getMainRepository(); + } + + public List findResources(RepositoryMode mode) throws MetadataRepositoryException { + try (RepositoryConnection conn = getRepository(mode).getConnection()) { + return conn.getContextIDs().stream().toList(); } catch (RepositoryException exception) { throw new MetadataRepositoryException(MSG_ERROR_RESOURCE + exception.getMessage()); } } - public List find(IRI context) throws MetadataRepositoryException { - try (RepositoryConnection conn = repository.getConnection()) { - return Iterations.asList( - conn.getStatements(null, null, null, context) - ); + public List find(IRI context, RepositoryMode mode) throws MetadataRepositoryException { + try (RepositoryConnection conn = getRepository(mode).getConnection()) { + return conn.getStatements(null, null, null, context).stream().toList(); } catch (RepositoryException exception) { throw new MetadataRepositoryException(MSG_ERROR_RESOURCE + exception.getMessage()); } } - public List findByLiteral(Literal query) throws MetadataRepositoryException { + public List findByLiteral(Literal query, RepositoryMode mode) throws MetadataRepositoryException { return runSparqlQuery( FIND_ENTITY_BY_LITERAL, AbstractMetadataRepository.class, - Map.of("query", query) + Map.of("query", query), + mode ) .stream() .map(item -> toSearchResult(item, true)) .toList(); } - public List findBySparqlQuery(String query) throws MetadataRepositoryException { - return runSparqlQuery(query) + public List findBySparqlQuery(String query, RepositoryMode mode) throws MetadataRepositoryException { + return runSparqlQuery(query, mode) .stream() .map(item -> toSearchResult(item, false)) .toList(); @@ -139,13 +171,14 @@ private SearchResult toSearchResult(BindingSet item, boolean withRelation) { ); } - public List findByFilterPredicate(IRI predicateUri) + public List findByFilterPredicate(IRI predicateUri, RepositoryMode mode) throws MetadataRepositoryException { final Map values = new HashMap<>(); runSparqlQuery( FIND_OBJECT_FOR_PREDICATE, AbstractMetadataRepository.class, - Map.of("predicate", predicateUri) + Map.of("predicate", predicateUri), + mode ).forEach(entry -> { values.put( entry.getValue(FIELD_VALUE).stringValue(), @@ -161,7 +194,7 @@ public List findByFilterPredicate(IRI predicateUri) .toList(); } - public Map findChildTitles(IRI parent, IRI relation) + public Map findChildTitles(IRI parent, IRI relation, RepositoryMode mode) throws MetadataRepositoryException { final Map titles = new HashMap<>(); @@ -171,7 +204,9 @@ public Map findChildTitles(IRI parent, IRI relation) Map.of( "parent", parent, "relation", relation - )); + ), + mode + ); for (var result : results) { final String childUri = result.getValue(FIELD_CHILD).stringValue(); @@ -182,9 +217,9 @@ public Map findChildTitles(IRI parent, IRI relation) return titles; } - public boolean checkExistence(Resource subject, IRI predicate, Value object) + public boolean checkExistence(Resource subject, IRI predicate, Value object, RepositoryMode mode) throws MetadataRepositoryException { - try (RepositoryConnection conn = repository.getConnection()) { + try (RepositoryConnection conn = getRepository(mode).getConnection()) { return conn.hasStatement(subject, predicate, object, false); } catch (RepositoryException exception) { @@ -192,8 +227,8 @@ public boolean checkExistence(Resource subject, IRI predicate, Value object) } } - public void save(List statements, IRI context) throws MetadataRepositoryException { - try (RepositoryConnection conn = repository.getConnection()) { + public void save(List statements, IRI context, RepositoryMode mode) throws MetadataRepositoryException { + try (RepositoryConnection conn = getRepository(mode).getConnection()) { conn.add(statements, context); } catch (RepositoryException exception) { @@ -201,8 +236,8 @@ public void save(List statements, IRI context) throws MetadataReposit } } - public void removeAll() throws MetadataRepositoryException { - try (RepositoryConnection conn = repository.getConnection()) { + public void removeAll(RepositoryMode mode) throws MetadataRepositoryException { + try (RepositoryConnection conn = getRepository(mode).getConnection()) { conn.clear(); } catch (RepositoryException exception) { @@ -210,13 +245,13 @@ public void removeAll() throws MetadataRepositoryException { } } - public void remove(IRI uri) throws MetadataRepositoryException { - removeStatement(null, null, null, uri); + public void remove(IRI uri, RepositoryMode mode) throws MetadataRepositoryException { + removeStatement(null, null, null, uri, mode); } - public void removeStatement(Resource subject, IRI predicate, Value object, IRI context) + public void removeStatement(Resource subject, IRI predicate, Value object, IRI context, RepositoryMode mode) throws MetadataRepositoryException { - try (RepositoryConnection conn = repository.getConnection()) { + try (RepositoryConnection conn = getRepository(mode).getConnection()) { conn.remove(subject, predicate, object, context); } catch (RepositoryException exception) { @@ -225,9 +260,9 @@ public void removeStatement(Resource subject, IRI predicate, Value object, IRI c } public List runSparqlQuery(String queryName, Class repositoryType, - Map bindings) + Map bindings, RepositoryMode mode) throws MetadataRepositoryException { - try (RepositoryConnection conn = repository.getConnection()) { + try (RepositoryConnection conn = getRepository(mode).getConnection()) { final String queryString = loadSparqlQuery(queryName, repositoryType); final TupleQuery query = conn.prepareTupleQuery(queryString); bindings.forEach(query::setBinding); @@ -242,8 +277,8 @@ public List runSparqlQuery(String queryName, Class repositoryType, } } - public List runSparqlQuery(String queryString) throws MetadataRepositoryException { - try (RepositoryConnection conn = repository.getConnection()) { + public List runSparqlQuery(String queryString, RepositoryMode mode) throws MetadataRepositoryException { + try (RepositoryConnection conn = getRepository(mode).getConnection()) { final TupleQuery query = conn.prepareTupleQuery(queryString); return query.evaluate().stream().toList(); } @@ -254,6 +289,6 @@ public List runSparqlQuery(String queryString) throws MetadataReposi protected String loadSparqlQuery(String queryName, Class repositoryType) throws IOException { final URL fileURL = repositoryType.getResource(queryName); - return Resources.toString(fileURL, Charsets.UTF_8); + return Resources.toString(fileURL, StandardCharsets.UTF_8); } } diff --git a/src/main/java/nl/dtls/fairdatapoint/database/rdf/repository/common/MetadataRepository.java b/src/main/java/nl/dtls/fairdatapoint/database/rdf/repository/common/MetadataRepository.java index 8af2a5ca0..a75de9d75 100644 --- a/src/main/java/nl/dtls/fairdatapoint/database/rdf/repository/common/MetadataRepository.java +++ b/src/main/java/nl/dtls/fairdatapoint/database/rdf/repository/common/MetadataRepository.java @@ -27,6 +27,7 @@ */ package nl.dtls.fairdatapoint.database.rdf.repository.common; +import nl.dtls.fairdatapoint.database.rdf.repository.RepositoryMode; import nl.dtls.fairdatapoint.database.rdf.repository.exception.MetadataRepositoryException; import nl.dtls.fairdatapoint.entity.search.SearchFilterValue; import nl.dtls.fairdatapoint.entity.search.SearchResult; @@ -38,33 +39,33 @@ public interface MetadataRepository { - List findResources() throws MetadataRepositoryException; + List findResources(RepositoryMode mode) throws MetadataRepositoryException; - List find(IRI context) throws MetadataRepositoryException; + List find(IRI context, RepositoryMode mode) throws MetadataRepositoryException; - List findByLiteral(Literal query) throws MetadataRepositoryException; + List findByLiteral(Literal query, RepositoryMode mode) throws MetadataRepositoryException; - List findBySparqlQuery(String query) throws MetadataRepositoryException; + List findBySparqlQuery(String query, RepositoryMode mode) throws MetadataRepositoryException; - List findByFilterPredicate(IRI predicateUri) + List findByFilterPredicate(IRI predicateUri, RepositoryMode mode) throws MetadataRepositoryException; - Map findChildTitles(IRI parent, IRI relation) + Map findChildTitles(IRI parent, IRI relation, RepositoryMode mode) throws MetadataRepositoryException; - boolean checkExistence(Resource subject, IRI predicate, Value object) + boolean checkExistence(Resource subject, IRI predicate, Value object, RepositoryMode mode) throws MetadataRepositoryException; - void save(List statements, IRI context) throws MetadataRepositoryException; + void save(List statements, IRI context, RepositoryMode mode) throws MetadataRepositoryException; - void removeAll() throws MetadataRepositoryException; + void removeAll(RepositoryMode mode) throws MetadataRepositoryException; - void remove(IRI uri) throws MetadataRepositoryException; + void remove(IRI uri, RepositoryMode mode) throws MetadataRepositoryException; - void removeStatement(Resource subject, IRI predicate, Value object, IRI context) + void removeStatement(Resource subject, IRI predicate, Value object, IRI context, RepositoryMode mode) throws MetadataRepositoryException; List runSparqlQuery(String queryName, Class repositoryType, - Map bindings) + Map bindings, RepositoryMode mode) throws MetadataRepositoryException; } diff --git a/src/main/java/nl/dtls/fairdatapoint/database/rdf/repository/generic/GenericMetadataRepositoryImpl.java b/src/main/java/nl/dtls/fairdatapoint/database/rdf/repository/generic/GenericMetadataRepositoryImpl.java index dee590abc..2a85ce3ee 100644 --- a/src/main/java/nl/dtls/fairdatapoint/database/rdf/repository/generic/GenericMetadataRepositoryImpl.java +++ b/src/main/java/nl/dtls/fairdatapoint/database/rdf/repository/generic/GenericMetadataRepositoryImpl.java @@ -27,10 +27,12 @@ */ package nl.dtls.fairdatapoint.database.rdf.repository.generic; +import nl.dtls.fairdatapoint.database.rdf.repository.RepositoryMode; import nl.dtls.fairdatapoint.database.rdf.repository.common.AbstractMetadataRepository; import nl.dtls.fairdatapoint.database.rdf.repository.exception.MetadataRepositoryException; import org.eclipse.rdf4j.model.*; import org.eclipse.rdf4j.model.impl.LinkedHashModel; +import org.eclipse.rdf4j.repository.Repository; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.cache.concurrent.ConcurrentMapCacheManager; import org.springframework.stereotype.Service; @@ -46,28 +48,32 @@ public class GenericMetadataRepositoryImpl extends AbstractMetadataRepository im @Autowired private ConcurrentMapCacheManager cacheManager; + public GenericMetadataRepositoryImpl(Repository mainRepository, Repository draftsRepository) { + super(mainRepository, draftsRepository); + } + @Override - public void save(List statements, IRI context) throws MetadataRepositoryException { - super.save(statements, context); - clearCatalogCache(context); + public void save(List statements, IRI context, RepositoryMode mode) throws MetadataRepositoryException { + super.save(statements, context, mode); + clearCatalogCache(context, mode); } @Override - public void remove(IRI uri) throws MetadataRepositoryException { - clearCatalogCache(uri); - super.remove(uri); + public void remove(IRI uri, RepositoryMode mode) throws MetadataRepositoryException { + clearCatalogCache(uri, mode); + super.remove(uri, mode); } @Override - public void removeStatement(Resource subject, IRI predicate, Value object, IRI context) + public void removeStatement(Resource subject, IRI predicate, Value object, IRI context, RepositoryMode mode) throws MetadataRepositoryException { - clearCatalogCache(context); - super.removeStatement(subject, predicate, object, context); + clearCatalogCache(context, mode); + super.removeStatement(subject, predicate, object, context, mode); } - private void clearCatalogCache(IRI uri) throws MetadataRepositoryException { + private void clearCatalogCache(IRI uri, RepositoryMode mode) throws MetadataRepositoryException { final Model metadata = new LinkedHashModel(); - metadata.addAll(find(uri)); + metadata.addAll(find(uri, mode)); final IRI parent = getParent(metadata); if (parent != null) { cacheManager.getCache(CATALOG_THEMES_CACHE).evict(parent.stringValue()); diff --git a/src/main/java/nl/dtls/fairdatapoint/service/dashboard/DashboardService.java b/src/main/java/nl/dtls/fairdatapoint/service/dashboard/DashboardService.java index a8cab6d0c..0d4a6a07b 100644 --- a/src/main/java/nl/dtls/fairdatapoint/service/dashboard/DashboardService.java +++ b/src/main/java/nl/dtls/fairdatapoint/service/dashboard/DashboardService.java @@ -26,6 +26,7 @@ import nl.dtls.fairdatapoint.api.dto.member.MemberDTO; import nl.dtls.fairdatapoint.api.dto.membership.MembershipDTO; import nl.dtls.fairdatapoint.entity.metadata.Metadata; +import nl.dtls.fairdatapoint.entity.metadata.MetadataState; import nl.dtls.fairdatapoint.entity.resource.ResourceDefinition; import nl.dtls.fairdatapoint.entity.resource.ResourceDefinitionChild; import nl.dtls.fairdatapoint.service.member.MemberService; @@ -94,7 +95,7 @@ private DashboardItemDTO getDashboardItem( final Optional member = memberService.getMemberForCurrentUser(metadataUri.stringValue(), Metadata.class); final Optional membership = member.map(MemberDTO::getMembership); - final Metadata state = metadataStateService.get(metadataUri); + final MetadataState state = metadataStateService.getState(metadataUri); return new DashboardItemDTO( metadataUri.toString(), getTitle(model).getLabel(), @@ -103,7 +104,7 @@ private DashboardItemDTO getDashboardItem( .filter(this::childOnDashboard) .toList(), membership, - state.getState() + state ); } diff --git a/src/main/java/nl/dtls/fairdatapoint/service/index/entry/IndexEntryService.java b/src/main/java/nl/dtls/fairdatapoint/service/index/entry/IndexEntryService.java index dc6787bdd..ed170b2b4 100644 --- a/src/main/java/nl/dtls/fairdatapoint/service/index/entry/IndexEntryService.java +++ b/src/main/java/nl/dtls/fairdatapoint/service/index/entry/IndexEntryService.java @@ -30,6 +30,7 @@ import nl.dtls.fairdatapoint.api.dto.index.entry.IndexEntryStateDTO; import nl.dtls.fairdatapoint.api.dto.index.ping.PingDTO; import nl.dtls.fairdatapoint.database.mongo.repository.IndexEntryRepository; +import nl.dtls.fairdatapoint.database.rdf.repository.RepositoryMode; import nl.dtls.fairdatapoint.database.rdf.repository.exception.MetadataRepositoryException; import nl.dtls.fairdatapoint.database.rdf.repository.generic.GenericMetadataRepository; import nl.dtls.fairdatapoint.entity.exception.ResourceNotFoundException; @@ -201,7 +202,7 @@ public Model getEntryHarvestedData(String uuid) throws MetadataRepositoryExcepti .findByUuid(uuid) .orElseThrow(() -> new ResourceNotFoundException(MSG_NOT_FOUND)); final Model model = new TreeModel(); - model.addAll(genericMetadataRepository.find(i(entry.getClientUrl()))); + model.addAll(genericMetadataRepository.find(i(entry.getClientUrl()), RepositoryMode.MAIN)); return model; } } diff --git a/src/main/java/nl/dtls/fairdatapoint/service/index/harvester/HarvesterService.java b/src/main/java/nl/dtls/fairdatapoint/service/index/harvester/HarvesterService.java index f30641b3f..6cd4d446d 100644 --- a/src/main/java/nl/dtls/fairdatapoint/service/index/harvester/HarvesterService.java +++ b/src/main/java/nl/dtls/fairdatapoint/service/index/harvester/HarvesterService.java @@ -23,6 +23,7 @@ package nl.dtls.fairdatapoint.service.index.harvester; import lombok.extern.slf4j.Slf4j; +import nl.dtls.fairdatapoint.database.rdf.repository.RepositoryMode; import nl.dtls.fairdatapoint.database.rdf.repository.exception.MetadataRepositoryException; import nl.dtls.fairdatapoint.database.rdf.repository.generic.GenericMetadataRepository; import org.eclipse.rdf4j.model.IRI; @@ -67,7 +68,7 @@ public class HarvesterService { private RestTemplate restTemplate; public void deleteHarvestedData(String clientUrl) throws MetadataRepositoryException { - genericMetadataRepository.remove(i(clientUrl)); + genericMetadataRepository.remove(i(clientUrl), RepositoryMode.MAIN); } @Async @@ -86,7 +87,7 @@ public void harvest(String clientUrl) throws MetadataRepositoryException { // 3. Store data for (Map.Entry item : result.entrySet()) { - genericMetadataRepository.save(new ArrayList<>(item.getValue()), i(clientUrl)); + genericMetadataRepository.save(new ArrayList<>(item.getValue()), i(clientUrl), RepositoryMode.MAIN); } log.info("Harvesting for '{}' completed", clientUrl); @@ -109,7 +110,7 @@ private void visitNode( nodes.put(uri, model); final List containers = getSubjectsBy(model, RDF.TYPE, LDP.DIRECT_CONTAINER); - if (containers.size() > 0) { + if (!containers.isEmpty()) { // Get children through LDP links for (Value container : containers) { for (Value child : getObjectsBy( diff --git a/src/main/java/nl/dtls/fairdatapoint/service/metadata/catalog/CatalogMetadataService.java b/src/main/java/nl/dtls/fairdatapoint/service/metadata/catalog/CatalogMetadataService.java index 2b22923f4..dbb562f52 100644 --- a/src/main/java/nl/dtls/fairdatapoint/service/metadata/catalog/CatalogMetadataService.java +++ b/src/main/java/nl/dtls/fairdatapoint/service/metadata/catalog/CatalogMetadataService.java @@ -23,6 +23,7 @@ package nl.dtls.fairdatapoint.service.metadata.catalog; import lombok.extern.slf4j.Slf4j; +import nl.dtls.fairdatapoint.database.rdf.repository.RepositoryMode; import nl.dtls.fairdatapoint.database.rdf.repository.catalog.CatalogMetadataRepository; import nl.dtls.fairdatapoint.database.rdf.repository.exception.MetadataRepositoryException; import nl.dtls.fairdatapoint.entity.resource.ResourceDefinition; @@ -48,10 +49,10 @@ public class CatalogMetadataService extends AbstractMetadataService { private CatalogMetadataRepository metadataRepository; @Override - public Model retrieve(@Nonnull IRI uri) throws MetadataServiceException { - final Model catalog = super.retrieve(uri); + public Model retrieve(@Nonnull IRI uri, RepositoryMode mode) throws MetadataServiceException { + final Model catalog = super.retrieve(uri, mode); try { - final List themes = metadataRepository.getDatasetThemesForCatalog(uri); + final List themes = metadataRepository.getDatasetThemesForCatalog(uri, mode); setThemeTaxonomies(catalog, uri, themes); } catch (MetadataRepositoryException exception) { diff --git a/src/main/java/nl/dtls/fairdatapoint/service/metadata/common/AbstractMetadataService.java b/src/main/java/nl/dtls/fairdatapoint/service/metadata/common/AbstractMetadataService.java index 8ea4b3bdf..132ba387e 100644 --- a/src/main/java/nl/dtls/fairdatapoint/service/metadata/common/AbstractMetadataService.java +++ b/src/main/java/nl/dtls/fairdatapoint/service/metadata/common/AbstractMetadataService.java @@ -23,6 +23,7 @@ package nl.dtls.fairdatapoint.service.metadata.common; import lombok.extern.slf4j.Slf4j; +import nl.dtls.fairdatapoint.database.rdf.repository.RepositoryMode; import nl.dtls.fairdatapoint.database.rdf.repository.common.MetadataRepository; import nl.dtls.fairdatapoint.database.rdf.repository.exception.MetadataRepositoryException; import nl.dtls.fairdatapoint.entity.exception.ResourceNotFoundException; @@ -34,7 +35,6 @@ import nl.dtls.fairdatapoint.service.member.MemberService; import nl.dtls.fairdatapoint.service.metadata.enhance.MetadataEnhancer; import nl.dtls.fairdatapoint.service.metadata.exception.MetadataServiceException; -import nl.dtls.fairdatapoint.service.metadata.state.MetadataStateService; import nl.dtls.fairdatapoint.service.metadata.validator.MetadataValidator; import nl.dtls.fairdatapoint.service.resource.ResourceDefinitionCache; import nl.dtls.fairdatapoint.service.resource.ResourceDefinitionService; @@ -50,7 +50,6 @@ import java.time.OffsetDateTime; import java.util.*; -import java.util.stream.Collectors; import static java.lang.String.format; import static nl.dtls.fairdatapoint.entity.metadata.MetadataGetter.getChildren; @@ -80,17 +79,19 @@ public abstract class AbstractMetadataService implements MetadataService { @Autowired private ResourceDefinitionCache resourceDefinitionCache; - @Autowired - private MetadataStateService metadataStateService; - @Autowired private ResourceDefinitionService resourceDefinitionService; @Override public Model retrieve(IRI uri) throws MetadataServiceException, ResourceNotFoundException { + return retrieve(uri, RepositoryMode.MAIN); + } + + @Override + public Model retrieve(IRI uri, RepositoryMode mode) throws MetadataServiceException, ResourceNotFoundException { try { // 1. Get metadata - final List statements = metadataRepository.find(uri); + final List statements = metadataRepository.find(uri, mode); if (statements.isEmpty()) { throw new ResourceNotFoundException( format("No metadata found for the uri '%s'", uri) @@ -109,11 +110,16 @@ public Model retrieve(IRI uri) throws MetadataServiceException, ResourceNotFound @Override public List retrieve(List uris) { + return retrieve(uris, RepositoryMode.MAIN); + } + + @Override + public List retrieve(List uris, RepositoryMode mode) { return uris .stream() - .map(suppress(this::retrieve)) + .map(suppress(uri -> retrieve(uri, mode))) .filter(Objects::nonNull) - .collect(Collectors.toList()); + .toList(); } @Override @@ -123,10 +129,9 @@ public Model store( try { metadataValidator.validate(metadata, uri, resourceDefinition); metadataEnhancer.enhance(metadata, uri, resourceDefinition); - metadataRepository.save(new ArrayList<>(metadata), uri); + metadataRepository.save(new ArrayList<>(metadata), uri, RepositoryMode.DRAFTS); updateParent(metadata, uri, resourceDefinition); addPermissions(uri); - addState(uri); return metadata; } catch (MetadataRepositoryException exception) { @@ -147,11 +152,17 @@ public Model update( if (validate) { metadataValidator.validate(metadata, uri, resourceDefinition); } - final Model oldMetadata = retrieve(uri); - metadataEnhancer.enhance(metadata, uri, resourceDefinition, oldMetadata); - metadataRepository.remove(uri); - metadataRepository.save(new ArrayList<>(metadata), uri); - updateParent(metadata, uri, resourceDefinition); + final Model oldMainMetadata = retrieve(uri, RepositoryMode.MAIN); + if (oldMainMetadata.isEmpty()) { + final Model oldDraftMetadata = retrieve(uri, RepositoryMode.DRAFTS); + metadataEnhancer.enhance(metadata, uri, resourceDefinition, oldDraftMetadata); + metadataRepository.remove(uri, RepositoryMode.DRAFTS); + metadataRepository.save(new ArrayList<>(metadata), uri, RepositoryMode.DRAFTS); + } else { + metadataEnhancer.enhance(metadata, uri, resourceDefinition, oldMainMetadata); + metadataRepository.remove(uri, RepositoryMode.MAIN); + metadataRepository.save(new ArrayList<>(metadata), uri, RepositoryMode.MAIN); + } return metadata; } catch (MetadataRepositoryException | MetadataServiceException exception) { @@ -193,7 +204,7 @@ public void delete(IRI uri, ResourceDefinition rd) throws MetadataServiceExcepti } // Delete itself - metadataRepository.remove(uri); + metadataRepository.remove(uri, RepositoryMode.COMBINED); } catch (MetadataRepositoryException | MetadataServiceException exception) { throw new MetadataServiceException(exception.getMessage()); @@ -203,6 +214,7 @@ public void delete(IRI uri, ResourceDefinition rd) throws MetadataServiceExcepti protected void updateParent( Model metadata, IRI uri, ResourceDefinition resourceDefinition ) throws MetadataServiceException { + // TODO REPO: first check if in DRAFTS or MAIN and then update final IRI parent = MetadataGetter.getParent(metadata); if (parent != null) { final ResourceDefinition rdParent = @@ -216,9 +228,16 @@ protected void updateParent( statements.add(s(parent, i(rdChild.getRelationUri()), uri)); } } - metadataRepository.removeStatement(parent, FDP.METADATAMODIFIED, null, parent); - statements.add(s(parent, FDP.METADATAMODIFIED, l(OffsetDateTime.now()))); - metadataRepository.save(statements, parent); + final Model parentMetadata = retrieve(parent, RepositoryMode.MAIN); + if (parentMetadata.isEmpty()) { + metadataRepository.removeStatement(parent, FDP.METADATAMODIFIED, null, parent, RepositoryMode.DRAFTS); + statements.add(s(parent, FDP.METADATAMODIFIED, l(OffsetDateTime.now()))); + metadataRepository.save(statements, parent, RepositoryMode.DRAFTS); + } else { + metadataRepository.removeStatement(parent, FDP.METADATAMODIFIED, null, parent, RepositoryMode.MAIN); + statements.add(s(parent, FDP.METADATAMODIFIED, l(OffsetDateTime.now()))); + metadataRepository.save(statements, parent, RepositoryMode.MAIN); + } } catch (MetadataRepositoryException exception) { throw new MetadataServiceException("Problem with updating parent timestamp"); @@ -237,10 +256,6 @@ private void addPermissions(IRI uri) { memberService.createOwner(uri.stringValue(), Metadata.class, user.get().getUuid()); } - private void addState(IRI uri) { - metadataStateService.initState(uri); - } - protected MemberService getMemberService() { return memberService; } diff --git a/src/main/java/nl/dtls/fairdatapoint/service/metadata/common/MetadataService.java b/src/main/java/nl/dtls/fairdatapoint/service/metadata/common/MetadataService.java index 7e106561a..3b79801c5 100644 --- a/src/main/java/nl/dtls/fairdatapoint/service/metadata/common/MetadataService.java +++ b/src/main/java/nl/dtls/fairdatapoint/service/metadata/common/MetadataService.java @@ -22,6 +22,7 @@ */ package nl.dtls.fairdatapoint.service.metadata.common; +import nl.dtls.fairdatapoint.database.rdf.repository.RepositoryMode; import nl.dtls.fairdatapoint.entity.exception.ResourceNotFoundException; import nl.dtls.fairdatapoint.entity.resource.ResourceDefinition; import nl.dtls.fairdatapoint.service.metadata.exception.MetadataServiceException; @@ -36,6 +37,10 @@ public interface MetadataService { List retrieve(List uri) throws MetadataServiceException, ResourceNotFoundException; + Model retrieve(IRI uri, RepositoryMode mode) throws MetadataServiceException, ResourceNotFoundException; + + List retrieve(List uri, RepositoryMode mode) throws MetadataServiceException, ResourceNotFoundException; + Model store( Model metadata, IRI uri, ResourceDefinition resourceDefinition ) throws MetadataServiceException; diff --git a/src/main/java/nl/dtls/fairdatapoint/service/metadata/state/MetadataStateService.java b/src/main/java/nl/dtls/fairdatapoint/service/metadata/state/MetadataStateService.java index aa3cf62f1..537fe5c7c 100644 --- a/src/main/java/nl/dtls/fairdatapoint/service/metadata/state/MetadataStateService.java +++ b/src/main/java/nl/dtls/fairdatapoint/service/metadata/state/MetadataStateService.java @@ -1,127 +1,46 @@ -/** - * The MIT License - * Copyright © 2017 DTL - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ package nl.dtls.fairdatapoint.service.metadata.state; import nl.dtls.fairdatapoint.api.dto.metadata.MetaStateChangeDTO; import nl.dtls.fairdatapoint.api.dto.metadata.MetaStateDTO; -import nl.dtls.fairdatapoint.database.mongo.repository.MetadataRepository; -import nl.dtls.fairdatapoint.entity.exception.ResourceNotFoundException; -import nl.dtls.fairdatapoint.entity.metadata.Metadata; +import nl.dtls.fairdatapoint.database.rdf.repository.RepositoryMode; import nl.dtls.fairdatapoint.entity.metadata.MetadataState; import nl.dtls.fairdatapoint.entity.resource.ResourceDefinition; -import nl.dtls.fairdatapoint.entity.resource.ResourceDefinitionChild; -import nl.dtls.fairdatapoint.service.metadata.validator.MetadataStateValidator; -import nl.dtls.fairdatapoint.service.user.CurrentUserService; +import nl.dtls.fairdatapoint.service.metadata.common.MetadataService; +import nl.dtls.fairdatapoint.service.metadata.exception.MetadataServiceException; import org.eclipse.rdf4j.model.IRI; import org.eclipse.rdf4j.model.Model; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Service; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.stream.Collectors; - -import static java.lang.String.format; -import static nl.dtls.fairdatapoint.entity.metadata.MetadataGetter.getUri; -import static nl.dtls.fairdatapoint.util.RdfUtil.getObjectsBy; -import static nl.dtls.fairdatapoint.util.ValueFactoryHelper.i; - @Service public class MetadataStateService { - private static final String MSG_NOT_FOUND = "Metadata info '%s' was not found"; - - @Autowired - private MetadataRepository metadataRepository; - @Autowired - private MetadataStateValidator metadataStateValidator; + @Qualifier("genericMetadataService") + MetadataService metadataService; - @Autowired - private CurrentUserService currentUserService; - - public Metadata get(IRI metadataUri) { - final Optional oMetadata = metadataRepository.findByUri(metadataUri.stringValue()); - if (oMetadata.isEmpty()) { - throw new ResourceNotFoundException(format(MSG_NOT_FOUND, metadataUri)); - } - return oMetadata.get(); + public boolean isDraft(IRI uri) throws MetadataServiceException { + return !metadataService.retrieve(uri, RepositoryMode.DRAFTS).isEmpty(); } - public MetaStateDTO getState(IRI metadataUri, Model model, ResourceDefinition definition) { - // 1. Return null if user is not log in - if (currentUserService.getCurrentUser().isEmpty()) { - return null; - } - - // 2. Get metadata info for current - final Optional oMetadata = metadataRepository.findByUri(metadataUri.stringValue()); - if (oMetadata.isEmpty()) { - throw new ResourceNotFoundException(format(MSG_NOT_FOUND, metadataUri)); - } - final Metadata metadata = oMetadata.get(); + public boolean isPublished(IRI uri) throws MetadataServiceException { + return !metadataService.retrieve(uri, RepositoryMode.MAIN).isEmpty(); + } - // 3. Get metadata info for children - final List childrenUris = new ArrayList<>(); - for (ResourceDefinitionChild rdChild : definition.getChildren()) { - final IRI relationUri = i(rdChild.getRelationUri()); - for (org.eclipse.rdf4j.model.Value childUri : getObjectsBy(model, metadataUri, relationUri)) { - childrenUris.add(childUri.stringValue()); - } + public MetadataState getState(IRI entityUri) throws MetadataServiceException { + if (isDraft(entityUri)) { + return MetadataState.DRAFT; } - final Map children = - metadataRepository.findByUriIn(childrenUris) - .stream() - .collect(Collectors.toMap(Metadata::getUri, Metadata::getState)); - - // 4. Build response - return new MetaStateDTO( - metadata.getState(), - children - ); + return MetadataState.PUBLISHED; } - public void initState(IRI metadataUri) { - final Metadata metadata = new Metadata(null, metadataUri.stringValue(), MetadataState.DRAFT); - metadataRepository.save(metadata); + public MetaStateDTO getStateDTO(IRI entityUri, Model entity, ResourceDefinition definition) { + // TODO + return null; } - public void modifyState(IRI metadataUri, MetaStateChangeDTO reqDto) { - // 1. Get metadata info for current - final Optional oMetadata = metadataRepository.findByUri(metadataUri.stringValue()); - if (oMetadata.isEmpty()) { - throw new ResourceNotFoundException(format(MSG_NOT_FOUND, metadataUri)); - } - final Metadata metadata = oMetadata.get(); - - // 2. Validate - metadataStateValidator.validate(reqDto, metadata); - - // 3. Update - metadata.setState(reqDto.getCurrent()); - metadataRepository.save(metadata); + public void modifyState(IRI entityUri, MetaStateChangeDTO reqDto) { + // TODO } - } diff --git a/src/main/java/nl/dtls/fairdatapoint/service/metadata/validator/MetadataValidator.java b/src/main/java/nl/dtls/fairdatapoint/service/metadata/validator/MetadataValidator.java index 6832d482c..5fbd9b854 100644 --- a/src/main/java/nl/dtls/fairdatapoint/service/metadata/validator/MetadataValidator.java +++ b/src/main/java/nl/dtls/fairdatapoint/service/metadata/validator/MetadataValidator.java @@ -22,6 +22,7 @@ */ package nl.dtls.fairdatapoint.service.metadata.validator; +import nl.dtls.fairdatapoint.database.rdf.repository.RepositoryMode; import nl.dtls.fairdatapoint.database.rdf.repository.common.MetadataRepository; import nl.dtls.fairdatapoint.database.rdf.repository.exception.MetadataRepositoryException; import nl.dtls.fairdatapoint.entity.exception.ValidationException; @@ -90,7 +91,7 @@ private void validateParent(Model metadata, ResourceDefinition definition) throw try { // select parent based on URI prefix for (String rdfType : resourceDefinitionService.getTargetClassUris(rdParent)) { - if (!metadataRepository.checkExistence(parent, RDF.TYPE, i(rdfType))) { + if (!metadataRepository.checkExistence(parent, RDF.TYPE, i(rdfType), RepositoryMode.COMBINED)) { throw new ValidationException(format("Parent is not of type (missing type: %s)", rdfType)); } } diff --git a/src/main/java/nl/dtls/fairdatapoint/service/reset/ResetService.java b/src/main/java/nl/dtls/fairdatapoint/service/reset/ResetService.java index 0879a829b..fe0ae7e46 100644 --- a/src/main/java/nl/dtls/fairdatapoint/service/reset/ResetService.java +++ b/src/main/java/nl/dtls/fairdatapoint/service/reset/ResetService.java @@ -70,7 +70,10 @@ public class ResetService { private IRI language; @Autowired - private Repository repository; + private Repository mainRepository; + + @Autowired + private Repository draftsRepository; @Autowired private ApiKeyRepository apiKeyRepository; @@ -169,6 +172,7 @@ private void clearResourceDefinitions() { } private void clearMetadata() throws MetadataServiceException { + // TODO REPO: clear DRAFTS repository log.debug("Clearing metadata"); final Optional resourceDefinition = resourceDefinitionRepository.findByUrlPrefix(""); @@ -195,8 +199,9 @@ private void restoreDefaultMemberships() { } private void restoreDefaultMetadata() { + // TODO REPO: saving to MAIN log.debug("Creating default metadata"); - try (RepositoryConnection conn = repository.getConnection()) { + try (RepositoryConnection conn = mainRepository.getConnection()) { final List statements = FactoryDefaults.fdpStatements( persistentUrl, license, diff --git a/src/main/java/nl/dtls/fairdatapoint/service/search/SearchService.java b/src/main/java/nl/dtls/fairdatapoint/service/search/SearchService.java index 3ecd0808b..dd27a8e3e 100644 --- a/src/main/java/nl/dtls/fairdatapoint/service/search/SearchService.java +++ b/src/main/java/nl/dtls/fairdatapoint/service/search/SearchService.java @@ -25,6 +25,7 @@ import com.google.common.base.Charsets; import com.google.common.io.Resources; import nl.dtls.fairdatapoint.api.dto.search.*; +import nl.dtls.fairdatapoint.database.rdf.repository.RepositoryMode; import nl.dtls.fairdatapoint.database.rdf.repository.exception.MetadataRepositoryException; import nl.dtls.fairdatapoint.database.rdf.repository.generic.GenericMetadataRepository; import nl.dtls.fairdatapoint.entity.exception.ResourceNotFoundException; @@ -46,6 +47,7 @@ import java.io.IOException; import java.net.URL; +import java.nio.charset.StandardCharsets; import java.util.*; import java.util.function.Function; import java.util.stream.Collectors; @@ -88,7 +90,7 @@ public List search( } public List search(SearchQueryDTO reqDto) throws MetadataRepositoryException { - final List results = metadataRepository.findByLiteral(l(reqDto.getQuery())); + final List results = metadataRepository.findByLiteral(l(reqDto.getQuery()), RepositoryMode.MAIN); return processSearchResults(results); } @@ -101,7 +103,7 @@ public List search( final SPARQLParser parser = new SPARQLParser(); parser.parseQuery(query, persistentUrl); // Get and process results for query - final List results = metadataRepository.findBySparqlQuery(query); + final List results = metadataRepository.findBySparqlQuery(query, RepositoryMode.MAIN); return processSearchResults(results); } @@ -150,7 +152,6 @@ private SearchFilterDTO enrichItems(SettingsSearchFilter filter) { } private List queryFilterItems(String predicate) { - // TODO: filter related to DRAFT records final SearchFilterCacheContainer cacheContainer = searchFilterCache.getFilter(predicate); if (cacheContainer != null) { @@ -158,7 +159,7 @@ private List queryFilterItems(String predicate) { } try { final List result = - metadataRepository.findByFilterPredicate(i(predicate)); + metadataRepository.findByFilterPredicate(i(predicate), RepositoryMode.MAIN); searchFilterCache.setFilter(predicate, new SearchFilterCacheContainer(result)); return result; } @@ -178,7 +179,6 @@ private List processSearchResults(List results) { ) .entrySet() .parallelStream() - .filter(entry -> isUsableForFilter(i(entry.getKey()))) .map(entry -> searchMapper.toResultDTO(entry.getKey(), entry.getValue())) .collect(Collectors.toMap(SearchResultDTO::getUri, Function.identity())); return results @@ -190,18 +190,6 @@ private List processSearchResults(List results) { .toList(); } - private boolean isUsableForFilter(IRI iri) { - try { - return !metadataStateService - .get(iri) - .getState() - .equals(MetadataState.DRAFT); - } - catch (ResourceNotFoundException exception) { - return true; - } - } - private String composeQuery(SearchQueryVariablesDTO reqDto) { final StrSubstitutor substitutor = new StrSubstitutor(searchMapper.toSubstitutions(reqDto), "{{", "}}"); @@ -211,7 +199,7 @@ private String composeQuery(SearchQueryVariablesDTO reqDto) { private static String loadSparqlQueryTemplate() { try { final URL fileURL = SearchService.class.getResource(QUERY_TEMPLATE_NAME); - return Resources.toString(fileURL, Charsets.UTF_8); + return Resources.toString(fileURL, StandardCharsets.UTF_8); } catch (IOException exception) { throw new RuntimeException( diff --git a/src/main/java/nl/dtls/fairdatapoint/service/settings/SettingsMapper.java b/src/main/java/nl/dtls/fairdatapoint/service/settings/SettingsMapper.java index 6ed192250..9ecd66afd 100644 --- a/src/main/java/nl/dtls/fairdatapoint/service/settings/SettingsMapper.java +++ b/src/main/java/nl/dtls/fairdatapoint/service/settings/SettingsMapper.java @@ -59,7 +59,8 @@ public SettingsDTO toDTO(Settings settings) { .persistentUrl(instanceProperties.getPersistentUrl()) .metadataMetrics(settings.getMetadataMetrics()) .ping(toDTO(settings.getPing())) - .repository(getRepositoryDTO()) + .mainRepository(getMainRepositoryDTO()) + .draftsRepository(getDraftsRepositoryDTO()) .search( SettingsSearchDTO .builder() @@ -129,15 +130,27 @@ public SettingsPingDTO toDTO(SettingsPing settingsPing) { .build(); } - public SettingsRepositoryDTO getRepositoryDTO() { + public SettingsRepositoryDTO getMainRepositoryDTO() { return SettingsRepositoryDTO .builder() - .type(repositoryProperties.getStringType()) - .dir(repositoryProperties.getDir()) - .url(repositoryProperties.getUrl()) - .repository(repositoryProperties.getRepository()) - .username(repositoryProperties.getUsername()) - .password(repositoryProperties.getPassword() != null ? "" : null) + .type(repositoryProperties.getMain().getStringType()) + .dir(repositoryProperties.getMain().getDir()) + .url(repositoryProperties.getMain().getUrl()) + .repository(repositoryProperties.getMain().getRepository()) + .username(repositoryProperties.getMain().getUsername()) + .password(repositoryProperties.getMain().getPassword() != null ? "" : null) + .build(); + } + + public SettingsRepositoryDTO getDraftsRepositoryDTO() { + return SettingsRepositoryDTO + .builder() + .type(repositoryProperties.getDrafts().getStringType()) + .dir(repositoryProperties.getDrafts().getDir()) + .url(repositoryProperties.getDrafts().getUrl()) + .repository(repositoryProperties.getDrafts().getRepository()) + .username(repositoryProperties.getDrafts().getUsername()) + .password(repositoryProperties.getDrafts().getPassword() != null ? "" : null) .build(); } diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 680fcbcf3..fcc63da1a 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -41,21 +41,38 @@ security: # valid repository type options {1 = inMemoryStore, 2 = NativeStore, 3 = AllegroGraph, 4 = graphDB, 5 = blazegraph} repository: - type: ${FDP_TRIPLE_STORE_TYPE:1} - native: - dir: ${FDP_TRIPLE_STORE_DIR:/tmp/fdp-store/} - agraph: - url: ${FDP_TRIPLE_STORE_URL:http://localhost:10035/repositories/fdp} - username: ${FDP_TRIPLE_STORE_USERNAME:user} - password: ${FDP_TRIPLE_STORE_PASSWORD:password} - graphDb: - url: ${FDP_TRIPLE_STORE_URL:http://localhost:7200} - repository: ${FDP_TRIPLE_STORE_REPOSITORY:test} - username: ${FDP_TRIPLE_STORE_USERNAME:user} - password: ${FDP_TRIPLE_STORE_PASSWORD:password} - blazegraph: - url: ${FDP_TRIPLE_STORE_URL:http://localhost:8888/blazegraph} - repository: ${FDP_TRIPLE_STORE_REPOSITORY:test} + main: + type: ${FDP_TRIPLE_STORE_TYPE:1} + native: + dir: ${FDP_TRIPLE_STORE_DIR:/tmp/fdp-store/} + agraph: + url: ${FDP_TRIPLE_STORE_URL:http://localhost:10035/repositories/fdp} + username: ${FDP_TRIPLE_STORE_USERNAME:user} + password: ${FDP_TRIPLE_STORE_PASSWORD:password} + graphDb: + url: ${FDP_TRIPLE_STORE_URL:http://localhost:7200} + repository: ${FDP_TRIPLE_STORE_REPOSITORY:test} + username: ${FDP_TRIPLE_STORE_USERNAME:user} + password: ${FDP_TRIPLE_STORE_PASSWORD:password} + blazegraph: + url: ${FDP_TRIPLE_STORE_URL:http://localhost:8888/blazegraph} + repository: ${FDP_TRIPLE_STORE_REPOSITORY:test} + drafts: + type: ${FDP_DRAFT_TRIPLE_STORE_TYPE:1} + native: + dir: ${FDP_DRAFT_TRIPLE_STORE_DIR:/tmp/fdp-store/} + agraph: + url: ${FDP_DRAFT_TRIPLE_STORE_URL:http://localhost:10035/repositories/fdp} + username: ${FDP_DRAFT_TRIPLE_STORE_USERNAME:user} + password: ${FDP_DRAFT_TRIPLE_STORE_PASSWORD:password} + graphDb: + url: ${FDP_DRAFT_TRIPLE_STORE_URL:http://localhost:7200} + repository: ${FDP_DRAFT_TRIPLE_STORE_REPOSITORY:test} + username: ${FDP_DRAFT_TRIPLE_STORE_USERNAME:user} + password: ${FDP_DRAFT_TRIPLE_STORE_PASSWORD:password} + blazegraph: + url: ${FDP_DRAFT_TRIPLE_STORE_URL:http://localhost:8888/blazegraph} + repository: ${FDP_DRAFT_TRIPLE_STORE_REPOSITORY:test} metadataProperties: language: ${FDP_METADATA_LANGUAGE:http://id.loc.gov/vocabulary/iso639-1/en}