Skip to content

Commit

Permalink
Add array::bytes_to_ndarray() and make `array::elements_to_ndarray(…
Browse files Browse the repository at this point in the history
…)` public
  • Loading branch information
LDeakin committed Feb 21, 2024
1 parent 371b165 commit a904ccf
Show file tree
Hide file tree
Showing 3 changed files with 34 additions and 19 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- Add `--usage-log` argument to examples to use `UsageLog` storage transformer
- Add more tests for `Array`, codecs, store locks, and more
- Add `array_write_read_ndarray` example
- Add `array::bytes_to_ndarray()` and make `array::elements_to_ndarray()` public

### Changed
#### Arrays
Expand Down
29 changes: 11 additions & 18 deletions examples/sharded_array_write_read.rs
Original file line number Diff line number Diff line change
@@ -1,7 +1,10 @@
use itertools::Itertools;
use zarrs::storage::{
storage_transformer::{StorageTransformerExtension, UsageLogStorageTransformer},
ReadableWritableListableStorage,
use zarrs::{
array::bytes_to_ndarray,
storage::{
storage_transformer::{StorageTransformerExtension, UsageLogStorageTransformer},
ReadableWritableListableStorage,
},
};

fn sharded_array_write_read() -> Result<(), Box<dyn std::error::Error>> {
Expand Down Expand Up @@ -133,24 +136,14 @@ fn sharded_array_write_read() -> Result<(), Box<dyn std::error::Error>> {
ArraySubset::new_with_start_shape(vec![0, 0], inner_chunk_shape.clone())?,
ArraySubset::new_with_start_shape(vec![0, 4], inner_chunk_shape.clone())?,
];
let decoded_inner_chunks = partial_decoder.partial_decode(&inner_chunks_to_decode)?;
let decoded_inner_chunks = decoded_inner_chunks
let decoded_inner_chunks_bytes = partial_decoder.partial_decode(&inner_chunks_to_decode)?;
let decoded_inner_chunks_ndarray = decoded_inner_chunks_bytes
.into_iter()
.map(|bytes| {
let elements = zarrs::array::transmute_from_bytes_vec(bytes);
ndarray::ArrayD::<u16>::from_shape_vec(
inner_chunk_shape
.iter()
.map(|&u| u as usize)
.collect::<Vec<_>>(),
elements,
)
.unwrap()
})
.collect::<Vec<_>>();
.map(|bytes| bytes_to_ndarray::<u16>(&inner_chunk_shape, bytes))
.collect::<Result<Vec<_>, _>>()?;
println!("Decoded inner chunks:");
for (inner_chunk_subset, decoded_inner_chunk) in
std::iter::zip(inner_chunks_to_decode, decoded_inner_chunks)
std::iter::zip(inner_chunks_to_decode, decoded_inner_chunks_ndarray)
{
println!("{inner_chunk_subset}\n{decoded_inner_chunk}\n");
}
Expand Down
23 changes: 22 additions & 1 deletion src/array.rs
Original file line number Diff line number Diff line change
Expand Up @@ -733,7 +733,11 @@ fn validate_element_size<T>(data_type: &DataType) -> Result<(), ArrayError> {
}

#[cfg(feature = "ndarray")]
fn elements_to_ndarray<T>(
/// Convert a vector of elements to an [`ndarray::ArrayD`].
///
/// # Errors
/// Returns an error if the length of `elements` is not equal to the product of the components in `shape`.
pub fn elements_to_ndarray<T>(
shape: &[u64],
elements: Vec<T>,
) -> Result<ndarray::ArrayD<T>, ArrayError> {
Expand All @@ -746,6 +750,23 @@ fn elements_to_ndarray<T>(
})
}

#[cfg(feature = "ndarray")]
/// Convert a vector of bytes to an [`ndarray::ArrayD`].
///
/// # Errors
/// Returns an error if the length of `bytes` is not equal to the product of the components in `shape` and the size of `T`.
pub fn bytes_to_ndarray<T: bytemuck::Pod>(
shape: &[u64],
bytes: Vec<u8>,
) -> Result<ndarray::ArrayD<T>, ArrayError> {
let expected_len = shape.iter().product::<u64>() * core::mem::size_of::<T>() as u64;
if bytes.len() as u64 != expected_len {
return Err(ArrayError::InvalidBytesInputSize(bytes.len(), expected_len));
}
let elements = transmute_from_bytes_vec::<T>(bytes);
elements_to_ndarray(shape, elements)
}

#[cfg(test)]
mod tests {
use itertools::Itertools;
Expand Down

0 comments on commit a904ccf

Please sign in to comment.