Skip to content

Commit

Permalink
fix sirixdb#582: Replace FileReader with FileChannelReader
Browse files Browse the repository at this point in the history
  • Loading branch information
@sband
sband committed Jun 8, 2023
1 parent 37fa795 commit d7de235
Show file tree
Hide file tree
Showing 5 changed files with 293 additions and 309 deletions.
89 changes: 38 additions & 51 deletions bundles/sirix-core/src/main/java/org/sirix/io/cloud/amazon/AmazonS3Storage.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,57 +48,49 @@ public final class AmazonS3Storage implements ICloudStorage {
* Revisions file name.
*/
private static final String REVISIONS_FILENAME = "sirix.revisions";
/**
* Instance to local storage.
*/
private final Path file;

/**
* Instance to local storage.
*/
private final Path file;

private S3Client s3Client;

/** Logger. */
private static final LogWrapper LOGGER = new LogWrapper(LoggerFactory.getLogger(AmazonS3Storage.class));

/**
* Byte handler pipeline.
*/
private final ByteHandlerPipeline byteHandlerPipeline;
*/
private final ByteHandlerPipeline byteHandlerPipeline;

/**
* Revision file data cache.
*/
private final AsyncCache<Integer, RevisionFileData> cache;

private ResourceConfiguration.AWSStorageInformation awsStorageInfo;
*/
private final AsyncCache<Integer, RevisionFileData> cache;

private final AmazonS3StorageReader reader;
private ResourceConfiguration.AWSStorageInformation awsStorageInfo;

private final AmazonS3StorageReader reader;

/**
* Support AWS authentication only with .aws credentials file with the required
* profile name from the creds file
*/
public AmazonS3Storage(final ResourceConfiguration resourceConfig,
AsyncCache<Integer, RevisionFileData> cache) {
public AmazonS3Storage(final ResourceConfiguration resourceConfig, AsyncCache<Integer, RevisionFileData> cache) {
this.awsStorageInfo = resourceConfig.awsStoreInfo;
this.cache = cache;
this.byteHandlerPipeline = resourceConfig.byteHandlePipeline;
this.byteHandlerPipeline = resourceConfig.byteHandlePipeline;
this.file = resourceConfig.resourcePath;
this.s3Client = getS3Client(); //this client is needed for the below checks, so initialize it here only.
this.s3Client = getS3Client(); // this client is needed for the below checks, so initialize it here only.
String bucketName = awsStorageInfo.getBucketName();
boolean shouldCreateBucketIfNotExists = awsStorageInfo.shouldCreateBucketIfNotExists();
if(!isBucketExists(bucketName) && shouldCreateBucketIfNotExists) {
if (!isBucketExists(bucketName) && shouldCreateBucketIfNotExists) {
createBucket(bucketName);
}
this.reader = new AmazonS3StorageReader(bucketName,
s3Client,
getDataFilePath().toAbsolutePath().toString(),
getRevisionFilePath().toAbsolutePath().toString(),
new ByteHandlerPipeline(this.byteHandlerPipeline),
SerializationType.DATA,
new PagePersister(),
cache.synchronous(),
resourceConfig);
this.reader = new AmazonS3StorageReader(bucketName, s3Client, getDataFilePath().toAbsolutePath().toString(),
getRevisionFilePath().toAbsolutePath().toString(), new ByteHandlerPipeline(this.byteHandlerPipeline),
SerializationType.DATA, new PagePersister(), cache.synchronous(), resourceConfig);
}

void createBucket(String bucketName) {
Expand All @@ -121,38 +113,33 @@ void createBucket(String bucketName) {
}

boolean isBucketExists(String bucketName) {
HeadBucketRequest headBucketRequest = HeadBucketRequest.builder().bucket(bucketName).build();
HeadBucketRequest headBucketRequest = HeadBucketRequest.builder().bucket(bucketName).build();

try {
s3Client.headBucket(headBucketRequest);
return true;
} catch (NoSuchBucketException e) {
return false;
}
}
}

S3Client getS3Client() {
return this.s3Client==null ? S3Client.builder()
.region(Region.of(awsStorageInfo.getAwsRegion()))
.credentialsProvider(ProfileCredentialsProvider.create(awsStorageInfo.getAwsProfile()))
.build() : this.s3Client;
return this.s3Client == null
? S3Client.builder().region(Region.of(awsStorageInfo.getAwsRegion()))
.credentialsProvider(ProfileCredentialsProvider.create(awsStorageInfo.getAwsProfile())).build()
: this.s3Client;
}

S3AsyncClient getAsyncS3Client() {
return S3AsyncClient.builder()
.region(Region.of(awsStorageInfo.getAwsRegion()))
.credentialsProvider(ProfileCredentialsProvider.create(awsStorageInfo.getAwsProfile()))
.build();
return S3AsyncClient.builder().region(Region.of(awsStorageInfo.getAwsRegion()))
.credentialsProvider(ProfileCredentialsProvider.create(awsStorageInfo.getAwsProfile())).build();
}

@Override
public Writer createWriter() {
return new AmazonS3StorageWriter (getDataFilePath().toAbsolutePath().toString(),
getRevisionFilePath().toAbsolutePath().toString(),
awsStorageInfo.getBucketName(),
SerializationType.DATA,new PagePersister(),
cache,reader,
this.getAsyncS3Client());
return new AmazonS3StorageWriter(getDataFilePath().toAbsolutePath().toString(),
getRevisionFilePath().toAbsolutePath().toString(), awsStorageInfo.getBucketName(),
SerializationType.DATA, new PagePersister(), cache, reader, this.getAsyncS3Client());
}

@Override
Expand All @@ -169,10 +156,10 @@ public void close() {
public boolean exists() {
Path storage = this.reader.readObjectDataFromS3(getDataFilePath().toAbsolutePath().toString());
try {
return Files.exists(storage) && Files.size(storage) > 0;
} catch (final IOException e) {
throw new UncheckedIOException(e);
}
return Files.exists(storage) && Files.size(storage) > 0;
} catch (final IOException e) {
throw new UncheckedIOException(e);
}
}

@Override
Expand All @@ -181,12 +168,12 @@ public ByteHandler getByteHandler() {
}

/**
* Getting path for data file.
* This path would be used on the local storage
* Getting path for data file. This path would be used on the local storage
*
* @return the path for this data file
*/
private Path getDataFilePath() {
return file.resolve(ResourceConfiguration.ResourcePaths.DATA.getPath()).resolve(FILENAME);
return file.resolve(ResourceConfiguration.ResourcePaths.DATA.getPath()).resolve(FILENAME);
}

/**
Expand All @@ -195,6 +182,6 @@ private Path getDataFilePath() {
* @return the concrete storage for this database
*/
private Path getRevisionFilePath() {
return file.resolve(ResourceConfiguration.ResourcePaths.DATA.getPath()).resolve(REVISIONS_FILENAME);
return file.resolve(ResourceConfiguration.ResourcePaths.DATA.getPath()).resolve(REVISIONS_FILENAME);
}
}
91 changes: 40 additions & 51 deletions bundles/sirix-core/src/main/java/org/sirix/io/cloud/amazon/AmazonS3StorageReader.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@
import java.io.IOException;
import java.io.OutputStream;
import java.io.RandomAccessFile;
import java.nio.file.FileSystems;
import java.nio.file.Path;
import java.time.Instant;

Expand All @@ -16,6 +15,7 @@
import org.sirix.io.RevisionFileData;
import org.sirix.io.bytepipe.ByteHandler;
import org.sirix.io.file.FileReader;
import org.sirix.io.filechannel.FileChannelReader;
import org.sirix.page.PagePersister;
import org.sirix.page.PageReference;
import org.sirix.page.RevisionRootPage;
Expand All @@ -34,83 +34,72 @@
import software.amazon.awssdk.services.s3.model.S3Exception;

public class AmazonS3StorageReader implements Reader {

/**
* S3 storage bucket name
*
*/
*/
private final String bucketName;

private final S3Client s3Client;

private final ResourceConfiguration resourceConfig;
/** Logger. */
private static final LogWrapper LOGGER = new LogWrapper(LoggerFactory.getLogger(AmazonS3StorageReader.class));


private FileReader reader;

public AmazonS3StorageReader(String bucketName,
S3Client s3Client,
String dataFileKeyName,
String revisionsOffsetFileKeyName,
final ByteHandler byteHandler,
final SerializationType serializationType,
final PagePersister pagePersister,
final Cache<Integer, RevisionFileData> cache,
ResourceConfiguration resourceConfig) {

private FileChannelReader reader;

public AmazonS3StorageReader(String bucketName, S3Client s3Client, String dataFileKeyName,
String revisionsOffsetFileKeyName, final ByteHandler byteHandler, final SerializationType serializationType,
final PagePersister pagePersister, final Cache<Integer, RevisionFileData> cache,
ResourceConfiguration resourceConfig) {
this.bucketName = bucketName;
this.s3Client = s3Client;
this.resourceConfig = resourceConfig;
Path dataFilePath = readObjectDataFromS3(dataFileKeyName);
Path revisionOffsetFilePath = readObjectDataFromS3(revisionsOffsetFileKeyName);
try {
this.reader = new FileReader(new RandomAccessFile(dataFilePath.toFile(), "r"),
new RandomAccessFile(revisionOffsetFilePath.toFile(), "r"),
byteHandler,
serializationType,
pagePersister,
cache);
}catch(IOException io) {
this.reader = new FileChannelReader(new RandomAccessFile(dataFilePath.toFile(), "r").getChannel(),
new RandomAccessFile(revisionOffsetFilePath.toFile(), "r").getChannel(), byteHandler, serializationType,
pagePersister, cache);
} catch (IOException io) {
LOGGER.error(io.getMessage());
System.exit(1);
}

}

/**
* @param keyName - Key name of the object to be read from S3 storage
* @return path - The location of the local file that contains the data that is written to the file system storage
* in the system temp directory.
* @return path - The location of the local file that contains the data that is
* written to the file system storage in the system temp directory.
*/
protected Path readObjectDataFromS3(String keyName) {

try {
GetObjectRequest objectRequest = GetObjectRequest
.builder()
.key(keyName)
.bucket(bucketName)
.build();

ResponseBytes<GetObjectResponse> objectBytes = s3Client.getObjectAsBytes(objectRequest);
byte[] data = objectBytes.asByteArray();
/*As the bucketName has to be same as the database name, it makes sense to use/create file on the local filesystem
* instead of in the tmp partition*/
Path path = resourceConfig.resourcePath;
// Write the data to a local file.
File myFile = path.toFile();
try(OutputStream os = new FileOutputStream(myFile)){
os.write(data);
}
return path;
} catch (IOException ex) {
ex.printStackTrace();
} catch (S3Exception e) {
LOGGER.error(e.awsErrorDetails().errorMessage());
System.exit(1);
}
GetObjectRequest objectRequest = GetObjectRequest.builder().key(keyName).bucket(bucketName).build();

ResponseBytes<GetObjectResponse> objectBytes = s3Client.getObjectAsBytes(objectRequest);
byte[] data = objectBytes.asByteArray();
/*
* As the bucketName has to be same as the database name, it makes sense to
* use/create file on the local filesystem instead of in the tmp partition
*/
Path path = resourceConfig.resourcePath;
// Write the data to a local file.
File myFile = path.toFile();
try (OutputStream os = new FileOutputStream(myFile)) {
os.write(data);
}
return path;
} catch (IOException ex) {
ex.printStackTrace();
} catch (S3Exception e) {
LOGGER.error(e.awsErrorDetails().errorMessage());
System.exit(1);
}
return null;
}
}

ByteHandler getByteHandler() {
return this.reader.getByteHandler();
Expand Down
Loading

0 comments on commit d7de235

Please sign in to comment.