-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat: support http range request to fetch subset of file (#3)
- Loading branch information
Showing
14 changed files
with
756 additions
and
180 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -22,9 +22,13 @@ SOFTWARE. | |
|
|
||
| --- | ||
|
|
||
| Portions of this software are derived from FlatGeobuf, which is licensed under the BSD 2-Clause License. | ||
| Portions of this software are derived from the FlatGeobuf project, | ||
| specifically the `packed_r_tree.rs` file, which is licensed under the | ||
| BSD 2-Clause License. The original code can be found at: | ||
| https://github.com/flatgeobuf/flatgeobuf/blob/master/src/rust/src/packed_r_tree.rs | ||
|
|
||
| Copyright (c) 2018, Björn Harrtell, Postnummer Stockholm AB | ||
| Copyright (c) 2024, Pirmin Kalberer <[email protected]> (Rust implementation) | ||
| All rights reserved. | ||
|
|
||
| Redistribution and use in source and binary forms, with or without | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,17 @@ | ||
| // Current version of FlatCityBuf | ||
| pub(crate) const VERSION: u8 = 1; | ||
|
|
||
| // Magic bytes for FlatCityBuf | ||
| pub(crate) const MAGIC_BYTES: [u8; 8] = [b'f', b'c', b'b', VERSION, b'f', b'c', b'b', 0]; | ||
|
|
||
| // Maximum buffer size for header | ||
| pub(crate) const HEADER_MAX_BUFFER_SIZE: usize = 1024 * 1024 * 512; // 512MB | ||
|
|
||
| // Size of magic bytes | ||
| pub(crate) const MAGIC_BYTES_SIZE: usize = 8; | ||
|
|
||
| // Size of header size | ||
| pub(crate) const HEADER_SIZE_SIZE: usize = 4; | ||
|
|
||
| // // Offset of header size | ||
| // pub(crate) const HEADER_SIZE_OFFSET: usize = MAGIC_BYTES_SIZE + HEADER_SIZE_SIZE; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,107 @@ | ||
| use crate::HttpFcbReader; | ||
| use anyhow::Result; | ||
| use bytes::Bytes; | ||
| use http_range_client::AsyncHttpRangeClient; | ||
| use std::fs::File; | ||
| use std::io::{BufReader, Read, Seek, SeekFrom}; | ||
| use std::ops::Range; | ||
| use std::path::PathBuf; | ||
| use std::sync::{Arc, RwLock}; | ||
| use tracing::trace; | ||
|
|
||
| impl HttpFcbReader<MockHttpRangeClient> { | ||
| /// NOTE: For debugging expediency, this test class often prefers panics over returning a result. | ||
| pub async fn mock_from_file( | ||
| path: &str, | ||
| ) -> Result<( | ||
| HttpFcbReader<MockHttpRangeClient>, | ||
| Arc<RwLock<RequestStats>>, | ||
| )> { | ||
| trace!("starting: opening http reader, reading header"); | ||
|
|
||
| let stats = Arc::new(RwLock::new(RequestStats::new())); | ||
| let http_client = MockHttpRangeClient::new(path, stats.clone()); | ||
| let client = http_range_client::AsyncBufferedHttpRangeClient::with(http_client, path); | ||
| Ok((Self::_open(client).await?, stats)) | ||
| } | ||
| } | ||
|
|
||
| /// NOTE: For debugging expediency, this test class often prefers panics over returning a result. | ||
| pub(crate) struct MockHttpRangeClient { | ||
| path: PathBuf, | ||
| stats: Arc<RwLock<RequestStats>>, | ||
| } | ||
|
|
||
| pub(crate) struct RequestStats { | ||
| pub request_count: u64, | ||
| pub bytes_requested: u64, | ||
| } | ||
|
|
||
| impl RequestStats { | ||
| fn new() -> Self { | ||
| Self { | ||
| request_count: 0, | ||
| bytes_requested: 0, | ||
| } | ||
| } | ||
| } | ||
|
|
||
| #[async_trait::async_trait] | ||
| impl AsyncHttpRangeClient for MockHttpRangeClient { | ||
| async fn get_range(&self, url: &str, range: &str) -> http_range_client::Result<Bytes> { | ||
| assert_eq!(url, self.path.to_str().unwrap()); | ||
|
|
||
| /// This is a hack, but we need the start and length of the range | ||
| /// since all we're given is the pre-formatted range string, we | ||
| /// need to parse it into its components | ||
| /// | ||
| /// For expediency, this test code panics rather than returns a result. | ||
| fn parse_range_header(range: &str) -> Range<u64> { | ||
| let bytes = range.strip_prefix("bytes=").unwrap(); | ||
| let parts: Vec<&str> = bytes.split('-').collect(); | ||
| assert!(parts.len() == 2); | ||
| let start = parts[0].parse().expect("should have valid start range"); | ||
| let end: u64 = parts[1].parse().expect("should have valid end range"); | ||
| // Range headers are *inclusive* | ||
| start..(end + 1) | ||
| } | ||
|
|
||
| let range = parse_range_header(range); | ||
| let request_length = range.end - range.start; | ||
|
|
||
| let mut stats = self | ||
| .stats | ||
| .write() | ||
| .expect("test code does not handle actual concurrency"); | ||
|
|
||
| stats.request_count += 1; | ||
| stats.bytes_requested += request_length; | ||
|
|
||
| let mut file_reader = BufReader::new(File::open(&self.path).unwrap()); | ||
| file_reader | ||
| .seek(SeekFrom::Start(range.start)) | ||
| .expect("unable to seek test reader"); | ||
| let mut output = vec![0; request_length as usize]; | ||
| file_reader | ||
| .read_exact(&mut output) | ||
| .expect("failed to read from test reader"); | ||
| Ok(Bytes::from(output)) | ||
| } | ||
|
|
||
| async fn head_response_header( | ||
| &self, | ||
| _url: &str, | ||
| _header: &str, | ||
| ) -> http_range_client::Result<Option<String>> { | ||
| unimplemented!() | ||
| } | ||
| } | ||
|
|
||
| impl MockHttpRangeClient { | ||
| fn new(path: &str, stats: Arc<RwLock<RequestStats>>) -> Self { | ||
| Self { | ||
| path: path.into(), | ||
| stats, | ||
| } | ||
| } | ||
| } |
Oops, something went wrong.