Skip to content

Commit

Permalink
Merge pull request #27 from saalfeldlab/fix/getS3Key
Browse files Browse the repository at this point in the history 
Fix/get s3 key
  • Loading branch information
@bogovicj
bogovicj authored Mar 13, 2024
2 parents 727983d + 5a0ec9b commit ff959ee
Show file tree
Hide file tree
Showing 3 changed files with 115 additions and 30 deletions.
75 changes: 47 additions & 28 deletions src/main/java/org/janelia/saalfeldlab/n5/s3/AmazonS3KeyValueAccess.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@
import java.util.List;
import java.util.stream.Collectors;

import com.amazonaws.services.s3.AmazonS3URI;
import org.janelia.saalfeldlab.n5.KeyValueAccess;
import org.janelia.saalfeldlab.n5.LockedChannel;
import org.janelia.saalfeldlab.n5.N5Exception;
Expand Down Expand Up @@ -137,7 +138,17 @@ public AmazonS3KeyValueAccess(final AmazonS3 s3, final URI containerURI, final b
@Override
public String[] components(final String path) {

final String[] baseComponents = path.split("/");
/* If the path is a valid URI with a scheme then use it to get the key. Otherwise,
* use the path directly, assuming it's a path only */
String key = path;
try {
final URI uri = URI.create(path);
final String scheme = uri.getScheme();
if (scheme != null && !scheme.isEmpty())
key = AmazonS3Utils.getS3Key(uri);
} catch (Throwable ignore) {}

final String[] baseComponents = removeLeadingSlash(key).split("/");
if (baseComponents.length <= 1)
return baseComponents;
return Arrays.stream(baseComponents)
Expand Down Expand Up @@ -169,10 +180,11 @@ public String compose(final String... components) {
@Override
public String compose(final URI uri, final String... components) {

final String[] uriComponents = new String[components.length + 1];
System.arraycopy(components, 0, uriComponents, 1, components.length);
uriComponents[0] = AmazonS3Utils.getS3Key(uri);
return compose(uriComponents);
try {
return uriResolve(uri, compose(components)).toString();
} catch (URISyntaxException x) {
throw new IllegalArgumentException(x.getMessage(), x);
}
}

@Override
Expand All @@ -192,7 +204,10 @@ public String relativize(final String path, final String base) {
* It's not true that the inputs are always referencing absolute paths, but it doesn't matter in this
* case, since we only care about the relative portion of `path` to `base`, so the result always
* ignores the absolute prefix anyway. */
return AmazonS3Utils.getS3Key(normalize(uri("/" + base).relativize(uri("/" + path)).toString()));
final URI baseAsUri = uri("/" + base);
final URI pathAsUri = uri("/" + path);
final URI relativeUri = baseAsUri.relativize(pathAsUri);
return relativeUri.getPath();
} catch (final URISyntaxException e) {
throw new N5Exception("Cannot relativize path (" + path + ") with base (" + base + ")", e);
}
Expand Down Expand Up @@ -223,11 +238,16 @@ public String normalize(final String path) {
@Override
public URI uri(final String normalPath) throws URISyntaxException {

return uriResolve(containerURI, normalPath);
}

private URI uriResolve(URI uri, String normalPath) throws URISyntaxException {

if (normalize(normalPath).equals(normalize("/")))
return containerURI;
return uri;

final Path containerPath = Paths.get(containerURI.getPath());
final Path givenPath = Paths.get(URI.create(normalPath).getPath());
final Path containerPath = Paths.get(uri.getPath());
final Path givenPath = Paths.get(new URI(normalPath).getPath());

final Path resolvedPath = containerPath.resolve(givenPath);
final String[] pathParts = new String[resolvedPath.getNameCount() + 1];
Expand All @@ -237,7 +257,7 @@ public URI uri(final String normalPath) throws URISyntaxException {
}
final String normalResolvedPath = compose(pathParts);

return new URI(containerURI.getScheme(), containerURI.getAuthority(), normalResolvedPath, null, null);
return new URI(uri.getScheme(), uri.getAuthority(), normalResolvedPath, null, null);
}

/**
Expand Down Expand Up @@ -273,14 +293,9 @@ private ListObjectsV2Result queryPrefix(final String prefix) {
*/
private boolean keyExists(final String key) {

/* In very preliminary tests, it appears that the obj listing request with one key max
* returns the correct key, but I'm not confident we can count on that in general.
* HeadObjectFunction (found by Caleb) is probably preferable for that reason. -John
*/
try {
final ObjectMetadata objMeta = new HeadObjectFunction(s3).apply(new GetObjectMetadataRequest(bucketName, key));
return objMeta != null;
} catch (Exception e) {
return s3.doesObjectExist(bucketName, key);
} catch (Throwable e) {
return false;
}
}
Expand Down Expand Up @@ -334,7 +349,8 @@ private static String removeLeadingSlash(final String path) {
@Override
public boolean isDirectory(final String normalPath) {

final String key = removeLeadingSlash(addTrailingSlash(normalPath));
final String s3Key = AmazonS3Utils.getS3Key(normalPath);
final String key = removeLeadingSlash(addTrailingSlash(s3Key));
if (key.equals(normalize("/"))) {
return s3.doesBucketExistV2(bucketName);
}
Expand All @@ -354,19 +370,22 @@ public boolean isDirectory(final String normalPath) {
@Override
public boolean isFile(final String normalPath) {

return !normalPath.endsWith("/") && keyExists(removeLeadingSlash(normalPath));
final String key = AmazonS3Utils.getS3Key(normalPath);
return !key.endsWith("/") && keyExists(removeLeadingSlash(key));
}

@Override
public LockedChannel lockForReading(final String normalPath) {

return new S3ObjectChannel(removeLeadingSlash(normalPath), true);
final String key = AmazonS3Utils.getS3Key(normalPath);
return new S3ObjectChannel(removeLeadingSlash(key), true);
}

@Override
public LockedChannel lockForWriting(final String normalPath) {

return new S3ObjectChannel(removeLeadingSlash(normalPath), false);
final String key = AmazonS3Utils.getS3Key(normalPath);
return new S3ObjectChannel(removeLeadingSlash(key), false);
}

@Override
Expand All @@ -381,8 +400,9 @@ private String[] list(final String normalPath, final boolean onlyDirectories) {
throw new N5Exception.N5IOException(normalPath + " is not a valid group");
}

final String pathKey = AmazonS3Utils.getS3Key(normalPath);
final List<String> subGroups = new ArrayList<>();
final String prefix = removeLeadingSlash(addTrailingSlash(normalPath));
final String prefix = removeLeadingSlash(addTrailingSlash(pathKey));
final ListObjectsV2Request listObjectsRequest = new ListObjectsV2Request()
.withBucketName(bucketName)
.withPrefix(prefix)
Expand Down Expand Up @@ -434,7 +454,7 @@ public void delete(final String normalPath) {
return;

// remove bucket when deleting "/"
if (normalPath.equals(normalize("/"))) {
if (AmazonS3Utils.getS3Key(normalPath).equals(normalize("/"))) {

// need to delete all objects before deleting the bucket
// see: https://docs.aws.amazon.com/AmazonS3/latest/userguide/delete-bucket.html
Expand All @@ -460,14 +480,13 @@ public void delete(final String normalPath) {
return;
}

final String path = removeLeadingSlash(normalPath);

if (!path.endsWith("/")) {
final String key = removeLeadingSlash(AmazonS3Utils.getS3Key(normalPath));
if (!key.endsWith("/")) {
s3.deleteObjects(new DeleteObjectsRequest(bucketName)
.withKeys(path));
.withKeys(key));
}

final String prefix = addTrailingSlash(path);
final String prefix = addTrailingSlash(key);
final ListObjectsV2Request listObjectsRequest = new ListObjectsV2Request()
.withBucketName(bucketName)
.withPrefix(prefix);
Expand Down
11 changes: 9 additions & 2 deletions src/main/java/org/janelia/saalfeldlab/n5/s3/AmazonS3Utils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@

import java.net.URI;
import java.net.URISyntaxException;
import java.util.Arrays;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

Expand Down Expand Up @@ -67,8 +68,14 @@ public static String getS3Key(final URI uri) {
} catch (final IllegalArgumentException e) {
}
// parse key manually when AmazonS3URI can't
final String path = uri.getPath().replaceFirst("^/", "");
return path.substring(path.indexOf('/') + 1);
final StringBuilder keyBuilder = new StringBuilder();
final String[] parts = uri.getPath().replaceFirst("^/", "").split("/");
for (int i = 1; i < parts.length; i++) {
keyBuilder.append(parts[i]);
if (i != parts.length - 1 || uri.getPath().endsWith("/"))
keyBuilder.append("/");
}
return keyBuilder.toString();
}

public static boolean areAnonymous(final AWSCredentialsProvider credsProvider) {
Expand Down
59 changes: 59 additions & 0 deletions src/test/java/org/janelia/saalfeldlab/n5/s3/AmazonS3UtilsTest.java
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,14 @@
package org.janelia.saalfeldlab.n5.s3;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;

import java.net.URI;
import java.net.URISyntaxException;
import java.util.HashMap;
import java.util.Map;

import com.google.common.collect.Maps;
import org.junit.Test;

public class AmazonS3UtilsTest {
Expand Down Expand Up @@ -38,4 +42,59 @@ public void testUriParsing() throws URISyntaxException {

}

@Test
public void getS3InfoTest() {

final String bucketName = "my-s3-bucket";
final String[] noKeyTests = new String[]{
"https://test.com/" + bucketName,
"http://test.com/" + bucketName + "/",
"s3://" + bucketName,
"s3://" + bucketName + "/",

};


final String[] onePartKeyWithSlashTests = new String[]{
"https://test.com/" + bucketName + "/a/",
"s3://" + bucketName + "/a/",
};

final String[] multiPartKeyWithSlashTests = new String[]{
"https://test.com/" + bucketName + "/a/b/c/d/",
"s3://" + bucketName + "/a/b/c/d/",
};



final String[] onePartKeyNoSlashTests = new String[]{
"https://test.com/" + bucketName + "/a",
"s3://" + bucketName + "/a",
};

final String[] multiPartKeyNoSlashTests = new String[]{
"https://test.com/" + bucketName + "/a/b/c/d",
"s3://" + bucketName + "/a/b/c/d",
};

final HashMap<String, String[]> keyToTests = new HashMap<>();

keyToTests.put("", noKeyTests);
keyToTests.put("a/", onePartKeyWithSlashTests);
keyToTests.put("a/b/c/d/", multiPartKeyWithSlashTests);
keyToTests.put("a", onePartKeyNoSlashTests);
keyToTests.put("a/b/c/d", multiPartKeyNoSlashTests);

for (Map.Entry<String, String[]> tests : keyToTests.entrySet()) {
final String expectedKey = tests.getKey();
for (String uri : tests.getValue()) {
assertEquals(bucketName, AmazonS3Utils.getS3Bucket(uri));
assertEquals("Unexpected key for " + uri, expectedKey, AmazonS3Utils.getS3Key(uri));
}
}

assertNull("Invalid URI should return null for bucket", AmazonS3Utils.getS3Bucket("invalid uri \\ _ ~ 435: q2234[;5."));
assertEquals("Invalid URI returns empty string for key", "", AmazonS3Utils.getS3Key("invalid uri \\ _ ~ 435: q2234[;5."));
}

}

0 comments on commit ff959ee

Please sign in to comment.