Skip to content

Commit

Permalink
rework for wasm support
Browse files Browse the repository at this point in the history
  • Loading branch information
luizirber committed Feb 13, 2025
1 parent bda11a5 commit a7561d9
Show file tree
Hide file tree
Showing 3 changed files with 154 additions and 133 deletions.
5 changes: 3 additions & 2 deletions src/core/src/collection.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ use camino::Utf8PathBuf as PathBuf;
use crate::encodings::Idx;
use crate::manifest::{Manifest, Record};
use crate::prelude::*;
use crate::storage::{FSStorage, InnerStorage, MemStorage, SigStore, ZipStorage};
use crate::storage::{FSStorage, InnerStorage, MemStorage, SigStore};
use crate::{Error, Result, ScaledType};

#[cfg(feature = "parallel")]
Expand Down Expand Up @@ -130,8 +130,9 @@ impl Collection {
.ok_or(Error::MismatchKSizes)
}

#[cfg(not(target_arch = "wasm32"))]
pub fn from_zipfile<P: AsRef<Path>>(zipfile: P) -> Result<Self> {
let storage = ZipStorage::from_file(zipfile)?;
let storage = crate::storage::ZipStorage::from_file(zipfile)?;
// Load manifest from standard location in zipstorage
let manifest = Manifest::from_reader(storage.load("SOURMASH-MANIFEST.csv")?.as_slice())?;
Ok(Self {
Expand Down
145 changes: 14 additions & 131 deletions src/core/src/storage/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,9 @@ use std::io::{BufReader, BufWriter, Read, Write};
use std::ops::Deref;
use std::sync::{Arc, RwLock};

use camino::Utf8Path as Path;
use camino::Utf8PathBuf as PathBuf;
use cfg_if::cfg_if;
use once_cell::sync::OnceCell;
use rc_zip_sync::{ArchiveHandle, ReadZip};
use serde::{Deserialize, Serialize};
use thiserror::Error;
use typed_builder::TypedBuilder;
Expand Down Expand Up @@ -125,26 +123,19 @@ pub struct FSStorage {
subdir: String,
}

/// Store files in a zip file.
#[ouroboros::self_referencing]
pub struct ZipStorage {
file: std::fs::File,

#[borrows(file)]
#[covariant]
archive: ArchiveHandle<'this, std::fs::File>,

subdir: Option<String>,
path: Option<PathBuf>,
}

/// Store data in memory (no permanent storage)
#[derive(TypedBuilder, Debug, Clone, Default)]
pub struct MemStorage {
//store: HashMap<String, Vec<u8>>,
sigs: Arc<RwLock<HashMap<String, SigStore>>>,
}

#[cfg(not(target_arch = "wasm32"))]
pub mod zip;

#[cfg(not(target_arch = "wasm32"))]
pub use self::zip::ZipStorage;

#[cfg(all(feature = "branchwater", not(target_arch = "wasm32")))]
pub mod rocksdb;

Expand Down Expand Up @@ -178,7 +169,14 @@ impl InnerStorage {
}
x if x.starts_with("zip") => {
let path = x.split("://").last().expect("not a valid path");
InnerStorage::new(ZipStorage::from_file(path)?)

cfg_if! {
if #[cfg(not(target_arch = "wasm32"))] {
InnerStorage::new(ZipStorage::from_file(path)?)
} else {
return Err(StorageError::MissingFeature("wasm".into(), path.into()).into())
}
}
}
_ => todo!("storage not supported, throw error"),
})
Expand Down Expand Up @@ -320,121 +318,6 @@ impl Storage for FSStorage {
}
}

impl Storage for ZipStorage {
fn save(&self, _path: &str, _content: &[u8]) -> Result<String> {
unimplemented!();
}

fn load(&self, path: &str) -> Result<Vec<u8>> {
let archive = self.borrow_archive();
if let Some(entry) = archive.by_name(path) {
return Ok(entry.bytes()?);
}

if let Some(subdir) = &self.borrow_subdir() {
if let Some(entry) = archive.by_name(subdir.to_owned() + path) {
return Ok(entry.bytes()?);
}
}

Err(StorageError::PathNotFoundError(path.into()).into())
}

fn args(&self) -> StorageArgs {
unimplemented!();
}

fn load_sig(&self, path: &str) -> Result<SigStore> {
let raw = self.load(path)?;
let mut vs = Signature::from_reader(&mut &raw[..])?;
if vs.len() > 1 {
unimplemented!("only one Signature currently allowed");
}
let sig = vs.swap_remove(0);

Ok(sig.into())
}

fn spec(&self) -> String {
format!("zip://{}", self.borrow_path().clone().unwrap_or("".into()))
}
}

impl ZipStorage {
pub fn from_file<P: AsRef<Path>>(location: P) -> Result<Self> {
let file = File::open(location.as_ref())?;

let mut storage = ZipStorageBuilder {
file,
archive_builder: |file: &std::fs::File| file.read_zip().expect("Error loading zipfile"),
subdir: None,
path: Some(location.as_ref().into()),
}
.build();

let subdir = {
let subdirs: Vec<_> = storage
.borrow_archive()
.entries()
.filter(|entry| matches!(entry.kind(), rc_zip::parse::EntryKind::Directory))
.collect();
if subdirs.len() == 1 {
Some(
subdirs[0]
.sanitized_name()
.expect("TODO throw right error")
.into(),
)
} else {
None
}
};

storage.with_mut(|fields| *fields.subdir = subdir);
Ok(storage)
}

pub fn path(&self) -> Option<PathBuf> {
self.borrow_path().clone()
}

pub fn subdir(&self) -> Option<String> {
self.borrow_subdir().clone()
}

pub fn set_subdir(&mut self, path: String) {
self.with_mut(|fields| *fields.subdir = Some(path))
}

pub fn list_sbts(&self) -> Result<Vec<String>> {
Ok(self
.borrow_archive()
.entries()
.filter_map(|entry| {
let path = entry.sanitized_name().expect("TODO throw right error");
if path.ends_with(".sbt.json") {
Some(path.into())
} else {
None
}
})
.collect())
}

pub fn filenames(&self) -> Result<Vec<String>> {
Ok(self
.borrow_archive()
.entries()
.map(|entry| {
entry
.sanitized_name()
.expect("TODO throw right error")
.into()
})
.collect())
}
}

impl SigStore {
pub fn new_with_storage(sig: Signature, storage: InnerStorage) -> Self {
let name = sig.name_str();
Expand Down
137 changes: 137 additions & 0 deletions src/core/src/storage/zip.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,137 @@
use std::fs::File;

use camino::Utf8Path as Path;
use camino::Utf8PathBuf as PathBuf;
use rc_zip_sync::{ArchiveHandle, ReadZip};

use crate::prelude::*;
use crate::storage::{StorageError,SigStore, StorageArgs};
use crate::Result;

/// Store files in a zip file.
#[ouroboros::self_referencing]
pub struct ZipStorage {
file: std::fs::File,

#[borrows(file)]
#[covariant]
archive: ArchiveHandle<'this, std::fs::File>,

subdir: Option<String>,
path: Option<PathBuf>,
}

impl Storage for ZipStorage {
fn save(&self, _path: &str, _content: &[u8]) -> Result<String> {
unimplemented!();

Check warning on line 26 in src/core/src/storage/zip.rs

View check run for this annotation

Codecov / codecov/patch

src/core/src/storage/zip.rs#L25-L26

Added lines #L25 - L26 were not covered by tests
}

fn load(&self, path: &str) -> Result<Vec<u8>> {
let archive = self.borrow_archive();
if let Some(entry) = archive.by_name(path) {
return Ok(entry.bytes()?);
}

if let Some(subdir) = &self.borrow_subdir() {
if let Some(entry) = archive.by_name(subdir.to_owned() + path) {
return Ok(entry.bytes()?);
}
}

Check warning on line 39 in src/core/src/storage/zip.rs

View check run for this annotation

Codecov / codecov/patch

src/core/src/storage/zip.rs#L38-L39

Added lines #L38 - L39 were not covered by tests

Err(StorageError::PathNotFoundError(path.into()).into())

Check warning on line 41 in src/core/src/storage/zip.rs

View check run for this annotation

Codecov / codecov/patch

src/core/src/storage/zip.rs#L41

Added line #L41 was not covered by tests
}

fn args(&self) -> StorageArgs {
unimplemented!();

Check warning on line 45 in src/core/src/storage/zip.rs

View check run for this annotation

Codecov / codecov/patch

src/core/src/storage/zip.rs#L44-L45

Added lines #L44 - L45 were not covered by tests
}

fn load_sig(&self, path: &str) -> Result<SigStore> {
let raw = self.load(path)?;
let mut vs = Signature::from_reader(&mut &raw[..])?;
if vs.len() > 1 {
unimplemented!("only one Signature currently allowed");

Check warning on line 52 in src/core/src/storage/zip.rs

View check run for this annotation

Codecov / codecov/patch

src/core/src/storage/zip.rs#L52

Added line #L52 was not covered by tests
}
let sig = vs.swap_remove(0);

Ok(sig.into())
}

fn spec(&self) -> String {
format!("zip://{}", self.borrow_path().clone().unwrap_or("".into()))
}
}

impl ZipStorage {
pub fn from_file<P: AsRef<Path>>(location: P) -> Result<Self> {
let file = File::open(location.as_ref())?;

let mut storage = ZipStorageBuilder {
file,
archive_builder: |file: &std::fs::File| file.read_zip().expect("Error loading zipfile"),
subdir: None,
path: Some(location.as_ref().into()),
}
.build();

let subdir = {
let subdirs: Vec<_> = storage
.borrow_archive()
.entries()
.filter(|entry| matches!(entry.kind(), rc_zip::parse::EntryKind::Directory))
.collect();
if subdirs.len() == 1 {
Some(
subdirs[0]
.sanitized_name()
.expect("TODO throw right error")
.into(),
)
} else {
None
}
};

storage.with_mut(|fields| *fields.subdir = subdir);
Ok(storage)
}

pub fn path(&self) -> Option<PathBuf> {
self.borrow_path().clone()
}

Check warning on line 100 in src/core/src/storage/zip.rs

View check run for this annotation

Codecov / codecov/patch

src/core/src/storage/zip.rs#L98-L100

Added lines #L98 - L100 were not covered by tests

pub fn subdir(&self) -> Option<String> {
self.borrow_subdir().clone()
}

Check warning on line 104 in src/core/src/storage/zip.rs

View check run for this annotation

Codecov / codecov/patch

src/core/src/storage/zip.rs#L102-L104

Added lines #L102 - L104 were not covered by tests

pub fn set_subdir(&mut self, path: String) {
self.with_mut(|fields| *fields.subdir = Some(path))
}

Check warning on line 108 in src/core/src/storage/zip.rs

View check run for this annotation

Codecov / codecov/patch

src/core/src/storage/zip.rs#L106-L108

Added lines #L106 - L108 were not covered by tests

pub fn list_sbts(&self) -> Result<Vec<String>> {
Ok(self
.borrow_archive()
.entries()
.filter_map(|entry| {
let path = entry.sanitized_name().expect("TODO throw right error");
if path.ends_with(".sbt.json") {
Some(path.into())
} else {
None
}
})
.collect())
}

pub fn filenames(&self) -> Result<Vec<String>> {
Ok(self
.borrow_archive()
.entries()
.map(|entry| {
entry
.sanitized_name()
.expect("TODO throw right error")
.into()
})
.collect())
}

Check warning on line 136 in src/core/src/storage/zip.rs

View check run for this annotation

Codecov / codecov/patch

src/core/src/storage/zip.rs#L125-L136

Added lines #L125 - L136 were not covered by tests
}

0 comments on commit a7561d9

Please sign in to comment.