From 296c981f24a92c95d8d1a86e83f4103877061780 Mon Sep 17 00:00:00 2001 From: wow-such-code Date: Thu, 15 Aug 2024 18:21:00 +0200 Subject: [PATCH 1/3] allow dataset connection to sample, improve error handling --- .../io/commandline/AuthenticationOptions.java | 30 +++++--- .../io/commandline/DownloadPetabCommand.java | 2 +- .../io/commandline/FindDatasetsCommand.java | 2 +- .../commandline/SampleHierarchyCommand.java | 2 +- .../commandline/SpaceStatisticsCommand.java | 2 +- .../TransferDataToSeekCommand.java | 73 +++++++++++++++++++ .../io/commandline/UploadDatasetCommand.java | 43 ++++++++--- .../commandline/UploadPetabResultCommand.java | 8 +- .../qbic/model/download/Authentication.java | 61 ---------------- .../qbic/model/download/OpenbisConnector.java | 51 ++++++++++--- 10 files changed, 176 insertions(+), 98 deletions(-) create mode 100644 src/main/java/life/qbic/io/commandline/TransferDataToSeekCommand.java delete mode 100644 src/main/java/life/qbic/model/download/Authentication.java diff --git a/src/main/java/life/qbic/io/commandline/AuthenticationOptions.java b/src/main/java/life/qbic/io/commandline/AuthenticationOptions.java index 948ef7e..6809a06 100644 --- a/src/main/java/life/qbic/io/commandline/AuthenticationOptions.java +++ b/src/main/java/life/qbic/io/commandline/AuthenticationOptions.java @@ -21,9 +21,9 @@ public class AuthenticationOptions { @Option( names = {"-u", "--user"}, description = "openBIS user name") - private String user; + private String openbisUser; @ArgGroup(multiplicity = "1") // ensures the password is provided once with at least one of the possible options. - PasswordOptions passwordOptions; + PasswordOptions openbisPasswordOptions; @Option( names = {"-as", "-as_url"}, @@ -43,11 +43,23 @@ public class AuthenticationOptions { scope = CommandLine.ScopeType.INHERIT) public String configPath; - public String getUser() { - if(user == null & configPath!=null && !configPath.isBlank()) { - user = ReadProperties.getProperties(configPath).get("user"); + @Option( + names = {"-seek-server", "-seek_url"}, + description = "SEEK API URL", + scope = CommandLine.ScopeType.INHERIT) + private String seek_url; + + public String getOpenbisUser() { + if(openbisUser == null & configPath!=null && !configPath.isBlank()) { + openbisUser = ReadProperties.getProperties(configPath).get("user"); } - return user; + return openbisUser; + } + + public String getSeekURL() { + log.error("No URL to the SEEK address provided."); + System.exit(2); + return seek_url; } public String getDSS() { @@ -64,8 +76,8 @@ public String getAS() { return as_url; } - public char[] getPassword() { - return passwordOptions.getPassword(); + public char[] getOpenbisPassword() { + return openbisPasswordOptions.getPassword(); } /** @@ -110,7 +122,7 @@ char[] getPassword() { @Override public String toString() { return new StringJoiner(", ", AuthenticationOptions.class.getSimpleName() + "[", "]") - .add("user='" + user + "'") + .add("user='" + openbisUser + "'") .toString(); //ATTENTION: do not expose the password here! } diff --git a/src/main/java/life/qbic/io/commandline/DownloadPetabCommand.java b/src/main/java/life/qbic/io/commandline/DownloadPetabCommand.java index f09d973..46efefe 100644 --- a/src/main/java/life/qbic/io/commandline/DownloadPetabCommand.java +++ b/src/main/java/life/qbic/io/commandline/DownloadPetabCommand.java @@ -27,7 +27,7 @@ public class DownloadPetabCommand implements Runnable { @Override public void run() { - OpenBIS authentication = App.loginToOpenBIS(auth.getPassword(), auth.getUser(), auth.getAS(), auth.getDSS()); + OpenBIS authentication = App.loginToOpenBIS(auth.getOpenbisPassword(), auth.getOpenbisUser(), auth.getAS(), auth.getDSS()); OpenbisConnector openbis = new OpenbisConnector(authentication); List datasets = openbis.findDataSets(Collections.singletonList(datasetCode)); diff --git a/src/main/java/life/qbic/io/commandline/FindDatasetsCommand.java b/src/main/java/life/qbic/io/commandline/FindDatasetsCommand.java index 2846fe7..2d340bd 100644 --- a/src/main/java/life/qbic/io/commandline/FindDatasetsCommand.java +++ b/src/main/java/life/qbic/io/commandline/FindDatasetsCommand.java @@ -39,7 +39,7 @@ public void run() { } else { System.out.println("Querying experiment in all available spaces..."); } - OpenBIS authentication = App.loginToOpenBIS(auth.getPassword(), auth.getUser(), auth.getAS()); + OpenBIS authentication = App.loginToOpenBIS(auth.getOpenbisPassword(), auth.getOpenbisUser(), auth.getAS()); OpenbisConnector openbis = new OpenbisConnector(authentication); List datasets = openbis.listDatasetsOfExperiment(spaces, experimentCode).stream() .sorted(Comparator.comparing( diff --git a/src/main/java/life/qbic/io/commandline/SampleHierarchyCommand.java b/src/main/java/life/qbic/io/commandline/SampleHierarchyCommand.java index 47353b6..53b2bf3 100644 --- a/src/main/java/life/qbic/io/commandline/SampleHierarchyCommand.java +++ b/src/main/java/life/qbic/io/commandline/SampleHierarchyCommand.java @@ -43,7 +43,7 @@ public void run() { } else { summary.add("Querying samples in all available spaces..."); } - OpenBIS authentication = App.loginToOpenBIS(auth.getPassword(), auth.getUser(), auth.getAS()); + OpenBIS authentication = App.loginToOpenBIS(auth.getOpenbisPassword(), auth.getOpenbisUser(), auth.getAS()); OpenbisConnector openbis = new OpenbisConnector(authentication); Map hierarchy = openbis.queryFullSampleHierarchy(spaces); diff --git a/src/main/java/life/qbic/io/commandline/SpaceStatisticsCommand.java b/src/main/java/life/qbic/io/commandline/SpaceStatisticsCommand.java index bce2144..ec2e2d0 100644 --- a/src/main/java/life/qbic/io/commandline/SpaceStatisticsCommand.java +++ b/src/main/java/life/qbic/io/commandline/SpaceStatisticsCommand.java @@ -50,7 +50,7 @@ public void run() { } else { summary.add("Querying samples in all available spaces..."); } - OpenBIS authentication = App.loginToOpenBIS(auth.getPassword(), auth.getUser(), auth.getAS()); + OpenBIS authentication = App.loginToOpenBIS(auth.getOpenbisPassword(), auth.getOpenbisUser(), auth.getAS()); OpenbisConnector openbis = new OpenbisConnector(authentication); if (spaces.isEmpty()) { diff --git a/src/main/java/life/qbic/io/commandline/TransferDataToSeekCommand.java b/src/main/java/life/qbic/io/commandline/TransferDataToSeekCommand.java new file mode 100644 index 0000000..5704b3d --- /dev/null +++ b/src/main/java/life/qbic/io/commandline/TransferDataToSeekCommand.java @@ -0,0 +1,73 @@ +package life.qbic.io.commandline; + +import ch.ethz.sis.openbis.generic.OpenBIS; +import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.DataSet; +import java.io.File; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import life.qbic.App; +import life.qbic.model.DatasetWithProperties; +import life.qbic.model.download.OpenbisConnector; +import picocli.CommandLine.Command; +import picocli.CommandLine.Mixin; +import picocli.CommandLine.Option; +import picocli.CommandLine.Parameters; + +@Command(name = "openbis-to-seek", + description = "Transfers data or metadata from openBIS to SEEK.") +public class TransferDataToSeekCommand implements Runnable { + + @Parameters(arity = "1", paramLabel = "dataset id", description = "The code of the dataset (or its metadata) to transfer. Can be found via list-data.") + private String datasetCode; + @Parameters(arity = "1", paramLabel = "seek node", description = "The node in SEEK to which to transfer the dataset.") + private String seekNode; + @Option(names = { "-d", "--data"}, usageHelp = true, description = "Transfers the data itself to SEEK along with the metadata") + private boolean transferData; + @Mixin + AuthenticationOptions auth = new AuthenticationOptions(); + + @Override + public void run() { + OpenBIS authentication = App.loginToOpenBIS(auth.getOpenbisPassword(), auth.getOpenbisUser(), auth.getAS(), auth.getDSS()); + OpenbisConnector openbis = new OpenbisConnector(authentication); + + List datasets = openbis.findDataSets(Collections.singletonList(datasetCode)); + + if(datasets.isEmpty()) { + System.out.println(datasetCode+" not found"); + return; + } + DatasetWithProperties result = new DatasetWithProperties(datasets.get(0)); + Optional patientID = openbis.findPropertyInSampleHierarchy("PATIENT_DKFZ_ID", + result.getExperiment().getIdentifier()); + patientID.ifPresent(s -> result.addProperty("patientID", s)); + + System.out.println("Found dataset, downloading."); + System.out.println(); + + final String tmpPath = "tmp/"; + + File downloadFolder = openbis.downloadDataset(tmpPath, datasetCode); + + + + cleanupTemp(new File(tmpPath)); + + System.out.println("Done"); + } + + private void cleanupTemp(File tmpFolder) { + File[] files = tmpFolder.listFiles(); + if (files != null) { //some JVMs return null for empty dirs + for (File f : files) { + if (f.isDirectory()) { + cleanupTemp(f); + } else { + f.delete(); + } + } + } + } + +} diff --git a/src/main/java/life/qbic/io/commandline/UploadDatasetCommand.java b/src/main/java/life/qbic/io/commandline/UploadDatasetCommand.java index 74fa98e..4e91a4a 100644 --- a/src/main/java/life/qbic/io/commandline/UploadDatasetCommand.java +++ b/src/main/java/life/qbic/io/commandline/UploadDatasetCommand.java @@ -4,10 +4,6 @@ import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.id.DataSetPermId; import java.io.File; import java.nio.file.Path; -import java.time.Instant; -import java.time.LocalDateTime; -import java.time.ZoneOffset; -import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.List; import life.qbic.App; @@ -23,9 +19,9 @@ public class UploadDatasetCommand implements Runnable { @Parameters(arity = "1", paramLabel = "file/folder", description = "The path to the file or folder to upload") private String dataPath; - @Parameters(arity = "1", paramLabel = "experiment ID", description = "The full identifier of the experiment the data should be attached to. " - + "The identifier must be of the format: /space/project/experiment") - private String experimentID; + @Parameters(arity = "1", paramLabel = "object ID", description = "The full identifier of the experiment or sample the data should be attached to. " + + "The identifier must be of the format: /space/project/experiment for experiments or /space/sample for samples") + private String objectID; @Option(arity = "1..*", paramLabel = "", description = "Optional list of dataset codes to act" + " as parents for the upload. E.g. when this dataset has been generated using these datasets as input.", names = {"-pa", "--parents"}) private List parents = new ArrayList<>(); @@ -36,15 +32,28 @@ public class UploadDatasetCommand implements Runnable { @Override public void run() { - OpenBIS authentication = App.loginToOpenBIS(auth.getPassword(), auth.getUser(), auth.getAS(), auth.getDSS()); + OpenBIS authentication = App.loginToOpenBIS(auth.getOpenbisPassword(), auth.getOpenbisUser(), auth.getAS(), auth.getDSS()); openbis = new OpenbisConnector(authentication); if(!pathValid(dataPath)) { System.out.printf("Path %s could not be found%n", dataPath); return; } - if(!experimentExists(experimentID)) { - System.out.printf("Experiment %s could not be found%n", experimentID); + boolean attachToSample = OpenbisConnector.sampleIdPattern.matcher(objectID).find(); + boolean attachToExperiment = false; + if(!attachToSample) { + attachToExperiment = OpenbisConnector.experimentIdPattern.matcher(objectID).find(); + } + if(!attachToExperiment && !attachToSample) { + System.out.printf("%s is neither a valid experiment nor sample identifier%n", objectID); + return; + } + if(attachToExperiment && !experimentExists(objectID)) { + System.out.printf("Experiment with identifier %s could not be found%n", objectID); + return; + } + if(attachToSample && !sampleExists(objectID)) { + System.out.printf("Sample object with identifier %s could not be found%n", objectID); return; } if(!datasetsExist(parents)) { @@ -54,10 +63,19 @@ public void run() { System.out.println(); System.out.println("Parameters verified, uploading dataset..."); System.out.println(); - DataSetPermId result = openbis.registerDataset(Path.of(dataPath), experimentID, parents); - System.out.printf("Dataset %s was successfully created%n", result.getPermId()); + if(attachToExperiment) { + DataSetPermId result = openbis.registerDatasetForExperiment(Path.of(dataPath), objectID, parents); + System.out.printf("Dataset %s was successfully created%n", result.getPermId()); + } else { + DataSetPermId result = openbis.registerDatasetForSample(Path.of(dataPath), objectID, parents); + System.out.printf("Dataset %s was successfully created%n", result.getPermId()); + } } + private boolean sampleExists(String objectID) { + return openbis.sampleExists(objectID); + } + private boolean datasetsExist(List datasetCodes) { return openbis.findDataSets(datasetCodes).size() == datasetCodes.size(); } @@ -70,4 +88,5 @@ private boolean pathValid(String dataPath) { return new File(dataPath).exists(); } + } diff --git a/src/main/java/life/qbic/io/commandline/UploadPetabResultCommand.java b/src/main/java/life/qbic/io/commandline/UploadPetabResultCommand.java index 3f29163..6ee72a9 100644 --- a/src/main/java/life/qbic/io/commandline/UploadPetabResultCommand.java +++ b/src/main/java/life/qbic/io/commandline/UploadPetabResultCommand.java @@ -33,13 +33,17 @@ public class UploadPetabResultCommand implements Runnable { @Override public void run() { - OpenBIS authentication = App.loginToOpenBIS(auth.getPassword(), auth.getUser(), auth.getAS(), auth.getDSS()); + OpenBIS authentication = App.loginToOpenBIS(auth.getOpenbisPassword(), auth.getOpenbisUser(), auth.getAS(), auth.getDSS()); openbis = new OpenbisConnector(authentication); if(!pathValid(dataPath)) { System.out.printf("Path %s could not be found%n", dataPath); return; } + if(!new File(dataPath).isDirectory()) { + System.out.printf("%s is not a directory. Please specify the PETab directory root%n", dataPath); + return; + } if(!experimentExists(experimentID)) { System.out.printf("Experiment %s could not be found%n", experimentID); return; @@ -60,7 +64,7 @@ public void run() { } System.out.println("Uploading dataset..."); //TODO copy and remove source references here - DataSetPermId result = openbis.registerDataset(Path.of(dataPath), experimentID, parents); + DataSetPermId result = openbis.registerDatasetForExperiment(Path.of(dataPath), experimentID, parents); System.out.printf("Dataset %s was successfully created%n", result.getPermId()); } diff --git a/src/main/java/life/qbic/model/download/Authentication.java b/src/main/java/life/qbic/model/download/Authentication.java deleted file mode 100644 index f151fd7..0000000 --- a/src/main/java/life/qbic/model/download/Authentication.java +++ /dev/null @@ -1,61 +0,0 @@ -package life.qbic.model.download; - -import ch.ethz.sis.openbis.generic.asapi.v3.IApplicationServerApi; -import ch.systemsx.cisd.common.spring.HttpInvokerUtils; - -public class Authentication { - - private static String sessionToken; - private String user; - private String password; - private final IApplicationServerApi applicationServer; - - - public Authentication( - String user, - String password, - String AppServerUri) { - this.user = user; - this.password = password; - if (!AppServerUri.isEmpty()) { - this.applicationServer = - HttpInvokerUtils.createServiceStub( - IApplicationServerApi.class, AppServerUri + IApplicationServerApi.SERVICE_URL, 10000); - } else { - this.applicationServer = null; - } - this.setCredentials(user, password); - } - - /** - * Login method for openBIS authentication - * - * @throws AuthenticationException in case the authentication failed - */ - public void login() throws ConnectionException, AuthenticationException { - try { - sessionToken = applicationServer.login(user, password); - } catch (Exception e) { - throw new ConnectionException("Connection to openBIS server failed.", e); - } - if (sessionToken == null || sessionToken.isEmpty()) { - throw new AuthenticationException("Authentication failed. Are you using the correct " - + "credentials?"); - } - } - - public String getSessionToken() { - return sessionToken; - } - - /** - * Setter for user and password credentials - * - * @param user The openBIS user - * @param password The openBIS user's password - */ - public void setCredentials(String user, String password) { - this.user = user; - this.password = password; - } -} diff --git a/src/main/java/life/qbic/model/download/OpenbisConnector.java b/src/main/java/life/qbic/model/download/OpenbisConnector.java index cf822a7..c73d796 100644 --- a/src/main/java/life/qbic/model/download/OpenbisConnector.java +++ b/src/main/java/life/qbic/model/download/OpenbisConnector.java @@ -15,6 +15,7 @@ import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.Sample; import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.SampleType; import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.fetchoptions.SampleFetchOptions; +import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.id.SampleIdentifier; import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.search.SampleSearchCriteria; import ch.ethz.sis.openbis.generic.asapi.v3.dto.space.Space; import ch.ethz.sis.openbis.generic.asapi.v3.dto.space.fetchoptions.SpaceFetchOptions; @@ -37,6 +38,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.regex.Pattern; import java.util.stream.Collectors; import life.qbic.model.SampleTypeConnection; import org.apache.logging.log4j.LogManager; @@ -45,7 +47,10 @@ public class OpenbisConnector { private static final Logger LOG = LogManager.getLogger(OpenbisConnector.class); - OpenBIS openBIS; + private final OpenBIS openBIS; + + public final static Pattern experimentIdPattern = Pattern.compile("\\/[A-Za-z0-9]+\\/[A-Za-z0-9]+\\/[A-Za-z0-9]+"); + public final static Pattern sampleIdPattern = Pattern.compile("\\/[A-Za-z0-9]+\\/[A-Za-z0-9]+"); public OpenbisConnector(OpenBIS authentication) { this.openBIS = authentication; @@ -58,16 +63,10 @@ public List getSpaces() { .stream().map(Space::getCode).collect(Collectors.toList()); } - public DataSetPermId registerDataset(Path uploadPath, String experimentID, + public DataSetPermId registerDatasetForExperiment(Path uploadPath, String experimentID, List parentCodes) { - final String uploadId = openBIS.uploadFileWorkspaceDSS(uploadPath); - - final UploadedDataSetCreation creation = new UploadedDataSetCreation(); - creation.setUploadId(uploadId); + UploadedDataSetCreation creation = prepareDataSetCreation(uploadPath, parentCodes); creation.setExperimentId(new ExperimentIdentifier(experimentID)); - creation.setParentIds(parentCodes.stream().map(DataSetPermId::new).collect( - Collectors.toList())); - creation.setTypeId(new EntityTypePermId("UNKNOWN", EntityKind.DATA_SET)); try { return openBIS.createUploadedDataSet(creation); @@ -77,6 +76,30 @@ public DataSetPermId registerDataset(Path uploadPath, String experimentID, return null; } + public DataSetPermId registerDatasetForSample(Path uploadPath, String sampleID, + List parentCodes) { + UploadedDataSetCreation creation = prepareDataSetCreation(uploadPath, parentCodes); + creation.setSampleId(new SampleIdentifier(sampleID)); + + try { + return openBIS.createUploadedDataSet(creation); + } catch (final Exception e) { + LOG.error(e.getMessage()); + } + return null; + } + + private UploadedDataSetCreation prepareDataSetCreation(Path uploadPath, List parentCodes) { + final String uploadId = openBIS.uploadFileWorkspaceDSS(uploadPath); + + final UploadedDataSetCreation creation = new UploadedDataSetCreation(); + creation.setUploadId(uploadId); + creation.setParentIds(parentCodes.stream().map(DataSetPermId::new).collect( + Collectors.toList())); + creation.setTypeId(new EntityTypePermId("UNKNOWN", EntityKind.DATA_SET)); + return creation; + } + private static void copyInputStreamToFile(InputStream inputStream, File file) throws IOException { System.err.println(file.getPath()); @@ -103,7 +126,7 @@ public List listDatasetsOfExperiment(List spaces, String experi return openBIS.searchDataSets(criteria, options).getObjects(); } - public void downloadDataset(String targetPath, String datasetID) { + public File downloadDataset(String targetPath, String datasetID) { DataSetFileDownloadOptions options = new DataSetFileDownloadOptions(); IDataSetFileId fileToDownload = new DataSetFilePermId(new DataSetPermId(datasetID), ""); @@ -135,6 +158,7 @@ public void downloadDataset(String targetPath, String datasetID) { } } } + return new File(targetPath); } public Map queryFullSampleHierarchy(List spaces) { @@ -357,4 +381,11 @@ public boolean experimentExists(String experimentID) { .isEmpty(); } + public boolean sampleExists(String objectID) { + SampleSearchCriteria criteria = new SampleSearchCriteria(); + criteria.withIdentifier().thatEquals(objectID); + + return !openBIS.searchSamples(criteria, new SampleFetchOptions()).getObjects() + .isEmpty(); + } } From eaf8d7e761d15fd55a9d9c4768b1c92a2667a6bf Mon Sep 17 00:00:00 2001 From: wow-such-code Date: Mon, 2 Sep 2024 15:30:52 +0200 Subject: [PATCH 2/3] Update src/main/java/life/qbic/io/commandline/UploadDatasetCommand.java --- .../java/life/qbic/io/commandline/UploadDatasetCommand.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/life/qbic/io/commandline/UploadDatasetCommand.java b/src/main/java/life/qbic/io/commandline/UploadDatasetCommand.java index 4e91a4a..bd05795 100644 --- a/src/main/java/life/qbic/io/commandline/UploadDatasetCommand.java +++ b/src/main/java/life/qbic/io/commandline/UploadDatasetCommand.java @@ -65,7 +65,7 @@ public void run() { System.out.println(); if(attachToExperiment) { DataSetPermId result = openbis.registerDatasetForExperiment(Path.of(dataPath), objectID, parents); - System.out.printf("Dataset %s was successfully created%n", result.getPermId()); + System.out.printf("Dataset %s was successfully attached to experiment%n", result.getPermId()); } else { DataSetPermId result = openbis.registerDatasetForSample(Path.of(dataPath), objectID, parents); System.out.printf("Dataset %s was successfully created%n", result.getPermId()); From cb3933dac84f12956e31c8d1a8b0e3cb0388d080 Mon Sep 17 00:00:00 2001 From: wow-such-code Date: Mon, 2 Sep 2024 15:31:35 +0200 Subject: [PATCH 3/3] Update src/main/java/life/qbic/io/commandline/UploadDatasetCommand.java --- .../java/life/qbic/io/commandline/UploadDatasetCommand.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/life/qbic/io/commandline/UploadDatasetCommand.java b/src/main/java/life/qbic/io/commandline/UploadDatasetCommand.java index bd05795..f165c9a 100644 --- a/src/main/java/life/qbic/io/commandline/UploadDatasetCommand.java +++ b/src/main/java/life/qbic/io/commandline/UploadDatasetCommand.java @@ -68,7 +68,7 @@ public void run() { System.out.printf("Dataset %s was successfully attached to experiment%n", result.getPermId()); } else { DataSetPermId result = openbis.registerDatasetForSample(Path.of(dataPath), objectID, parents); - System.out.printf("Dataset %s was successfully created%n", result.getPermId()); + System.out.printf("Dataset %s was successfully attached to sample%n", result.getPermId()); } }