wip

.vscode/settings.json

@@ -42,9 +42,16 @@
     },
     "rust-analyzer.diagnostics.enable": false, // False positives
     "rust-analyzer.showUnlinkedFileNotification": false,
-    // Uncomment the following option and restart rust-analyzer to get it to check code behind `cfg(target_arch=wasm32)`.
+
+    // Uncomment the following options and restart rust-analyzer to get it to check code behind `cfg(target_arch=wasm32)`.
     // Don't forget to put it in a comment again before committing.
-    // "rust-analyzer.cargo.target": "wasm32-unknown-unknown"
+    "rust-analyzer.cargo.target": "wasm32-unknown-unknown",
+    "rust-analyzer.cargo.cfgs": {
+        "web": null,
+        "webgl": null,
+        "webgpu": null,
+    },
+
     "C_Cpp.default.configurationProvider": "ms-vscode.cmake-tools", // Use cmake-tools to grab configs.
     "C_Cpp.autoAddFileAssociations": false,
     "cmake.buildDirectory": "${workspaceRoot}/build/debug",

ARCHITECTURE.md

@@ -214,6 +214,7 @@ Update instructions:
 | re_string_interner | Yet another string interning library                                                 |
 | re_tracing         | Helpers for tracing/spans/flamegraphs and such.                                      |
 | re_tuid            | 128-bit Time-based Unique Identifier                                                 |
+| re_video           | Video decoding library                                                               |
 
 
 

Cargo.lock

@@ -4776,6 +4776,7 @@ dependencies = [
  "gltf",
  "half 2.3.1",
  "itertools 0.13.0",
+ "js-sys",
  "never",
  "notify",
  "ordered-float",
@@ -4788,6 +4789,7 @@ dependencies = [
  "re_log",
  "re_math",
  "re_tracing",
+ "re_video",
  "serde",
  "slotmap",
  "smallvec",
@@ -4798,9 +4800,12 @@ dependencies = [
  "type-map",
  "unindent",
  "walkdir",
+ "wasm-bindgen",
  "wasm-bindgen-futures",
+ "web-sys",
  "wgpu",
  "wgpu-core",
+ "wgpu-types",
 ]
 
 [[package]]

Cargo.toml

@@ -287,6 +287,7 @@ wgpu = { version = "0.20.1", default-features = false, features = [
   "fragile-send-sync-non-atomic-wasm",
 ] }
 wgpu-core = "0.21.0"
+wgpu-types = "0.20.0"
 xshell = "0.2"
 zip = { version = "0.6", default-features = false }
 

crates/store/re_video/src/lib.rs

@@ -1,7 +1,8 @@
 mod mp4;
+pub use mp4::load_mp4;
 
 #[derive(Clone)]
-pub struct Video {
+pub struct VideoData {
     pub config: Config,
 
     /// How many time units per second there are.
@@ -62,6 +63,7 @@ pub enum VideoLoadError {
     NoVideoTrack,
     InvalidConfigFormat,
     InvalidSamples,
+    UnknownMediaType,
 }
 
 impl std::fmt::Display for VideoLoadError {
@@ -71,6 +73,7 @@ impl std::fmt::Display for VideoLoadError {
             Self::NoVideoTrack => write!(f, "video file has no video tracks"),
             Self::InvalidConfigFormat => write!(f, "video file track config is invalid"),
             Self::InvalidSamples => write!(f, "video file has invalid sample entries"),
+            Self::UnknownMediaType => write!(f, "unrecognized media type"),
         }
     }
 }
@@ -83,7 +86,7 @@ impl From<::mp4::Error> for VideoLoadError {
     }
 }
 
-impl std::fmt::Debug for Video {
+impl std::fmt::Debug for VideoData {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         f.debug_struct("Video")
             .field("config", &self.config)

crates/store/re_video/src/mp4.rs

@@ -1,96 +1,94 @@
 #![allow(clippy::map_err_ignore)]
 
-use super::{Config, Sample, Segment, Video, VideoLoadError};
+use super::{Config, Sample, Segment, VideoData, VideoLoadError};
 use ::mp4;
 
 use mp4::TrackType;
 use std::io::Cursor;
 use std::io::Write as _;
 
-impl Video {
-    pub fn load_mp4(bytes: &[u8]) -> Result<Self, VideoLoadError> {
-        let mut mp4 = ::mp4::Mp4Reader::read_header(Cursor::new(bytes), bytes.len() as u64)?;
-
-        let video_track = mp4
-            .tracks()
-            .values()
-            .find(|t| t.track_type().ok() == Some(TrackType::Video))
-            .ok_or_else(|| VideoLoadError::NoVideoTrack)?;
-        let track_id = video_track.track_id();
-        let num_samples = video_track.sample_count();
-
-        let (codec, description);
-        if let Some(::mp4::Av01Box { av1c, .. }) = &video_track.trak.mdia.minf.stbl.stsd.av01 {
-            let profile = av1c.profile;
-            let level = av1c.level;
-            let tier = if av1c.tier == 0 { "M" } else { "H" };
-            let bit_depth = av1c.bit_depth;
-
-            codec = format!("av01.{profile}.{level:02}{tier}.{bit_depth:02}");
-            description = write_box_without_header(av1c)?;
-        } else {
-            panic!("todo: support other codecs");
-        }
-
-        let timescale = video_track.trak.mdia.mdhd.timescale as u64;
-        let duration = video_track.trak.mdia.mdhd.duration;
-        let coded_height = video_track.height();
-        let coded_width = video_track.width();
-
-        let config = Config {
-            codec,
-            description,
-            coded_height,
-            coded_width,
-        };
-
-        let mut time_offset = None;
-        let mut samples = Vec::<Sample>::new();
-        let mut segments = Vec::<Segment>::new();
-        let mut data = Vec::<u8>::new();
-
-        for sample_idx in 0..num_samples {
-            let sample = mp4
-                .read_sample(track_id, sample_idx + 1)
-                .map_err(|_err| VideoLoadError::InvalidSamples)?
-                .ok_or_else(|| VideoLoadError::InvalidSamples)?;
-
-            if sample.is_sync && !samples.is_empty() {
-                segments.push(Segment {
-                    timestamp: samples[0].timestamp,
-                    samples,
-                });
-                samples = Vec::new();
-            }
-
-            let time_offset = *time_offset.get_or_insert(sample.start_time);
-            let timestamp = sample.start_time - time_offset;
-            let byte_offset = data.len() as u32;
-            let byte_length = sample.bytes.len() as u32;
-            data.write_all(&sample.bytes).expect("oom");
-
-            samples.push(Sample {
-                timestamp,
-                byte_offset,
-                byte_length,
-            });
-        }
+pub fn load_mp4(bytes: &[u8]) -> Result<VideoData, VideoLoadError> {
+    let mut mp4 = ::mp4::Mp4Reader::read_header(Cursor::new(bytes), bytes.len() as u64)?;
+
+    let video_track = mp4
+        .tracks()
+        .values()
+        .find(|t| t.track_type().ok() == Some(TrackType::Video))
+        .ok_or_else(|| VideoLoadError::NoVideoTrack)?;
+    let track_id = video_track.track_id();
+    let num_samples = video_track.sample_count();
+
+    let (codec, description);
+    if let Some(::mp4::Av01Box { av1c, .. }) = &video_track.trak.mdia.minf.stbl.stsd.av01 {
+        let profile = av1c.profile;
+        let level = av1c.level;
+        let tier = if av1c.tier == 0 { "M" } else { "H" };
+        let bit_depth = av1c.bit_depth;
+
+        codec = format!("av01.{profile}.{level:02}{tier}.{bit_depth:02}");
+        description = write_box_without_header(av1c)?;
+    } else {
+        panic!("todo: support other codecs");
+    }
 
-        if !samples.is_empty() {
+    let timescale = video_track.trak.mdia.mdhd.timescale as u64;
+    let duration = video_track.trak.mdia.mdhd.duration;
+    let coded_height = video_track.height();
+    let coded_width = video_track.width();
+
+    let config = Config {
+        codec,
+        description,
+        coded_height,
+        coded_width,
+    };
+
+    let mut time_offset = None;
+    let mut samples = Vec::<Sample>::new();
+    let mut segments = Vec::<Segment>::new();
+    let mut data = Vec::<u8>::new();
+
+    for sample_idx in 0..num_samples {
+        let sample = mp4
+            .read_sample(track_id, sample_idx + 1)
+            .map_err(|_err| VideoLoadError::InvalidSamples)?
+            .ok_or_else(|| VideoLoadError::InvalidSamples)?;
+
+        if sample.is_sync && !samples.is_empty() {
             segments.push(Segment {
                 timestamp: samples[0].timestamp,
                 samples,
             });
+            samples = Vec::new();
         }
 
-        Ok(Self {
-            config,
-            data,
-            timescale,
-            duration,
-            segments,
-        })
+        let time_offset = *time_offset.get_or_insert(sample.start_time);
+        let timestamp = sample.start_time - time_offset;
+        let byte_offset = data.len() as u32;
+        let byte_length = sample.bytes.len() as u32;
+        data.write_all(&sample.bytes).expect("oom");
+
+        samples.push(Sample {
+            timestamp,
+            byte_offset,
+            byte_length,
+        });
+    }
+
+    if !samples.is_empty() {
+        segments.push(Segment {
+            timestamp: samples[0].timestamp,
+            samples,
+        });
     }
+
+    Ok(VideoData {
+        config,
+        data,
+        timescale,
+        duration,
+        segments,
+    })
 }
 
 fn write_box_without_header<Box: for<'a> mp4::WriteBox<&'a mut Vec<u8>>>(

crates/store/re_video/tests/parse.rs

@@ -1,13 +1,13 @@
 use std::path::Path;
 
-use re_video::Video;
+use re_video::VideoData;
 
 const DATA_DIR: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/tests/data");
 
 #[test]
 fn parse_mp4() {
     let file = std::fs::read(Path::new(DATA_DIR).join("bbb_video_av1_frag.mp4")).unwrap();
-    let video = Video::load_mp4(&file).unwrap();
+    let video = VideoData::load_mp4(&file).unwrap();
 
     println!("{video:#?}");
 

crates/viewer/re_renderer/Cargo.toml

@@ -52,6 +52,7 @@ re_error.workspace = true
 re_log.workspace = true
 re_math.workspace = true
 re_tracing.workspace = true
+re_video.workspace = true
 
 ahash.workspace = true
 anyhow.workspace = true
@@ -78,6 +79,7 @@ thiserror.workspace = true
 type-map.workspace = true
 wgpu.workspace = true
 wgpu-core.workspace = true                                           # Needed for error handling when wgpu-core implemented backend is used.
+wgpu-types.workspace = true
 
 # optional
 arrow2 = { workspace = true, optional = true }
@@ -97,6 +99,17 @@ getrandom = { workspace = true, features = [
   "js",
 ] } # getrandom needs the `js` feature to be enabled. It is dragged in indirectly.
 wasm-bindgen-futures.workspace = true
+web-sys = { workspace = true, features = [
+  "EncodedVideoChunk",
+  "EncodedVideoChunkInit",
+  "EncodedVideoChunkType",
+  "VideoFrame",
+  "VideoDecoder",
+  "VideoDecoderConfig",
+  "VideoDecoderInit",
+] }
+js-sys = { workspace = true }
+wasm-bindgen = { workspace = true }
 
 
 [dev-dependencies]

crates/viewer/re_renderer/src/renderer/mod.rs

@@ -31,6 +31,9 @@ pub(crate) use compositor::CompositorDrawData;
 mod debug_overlay;
 pub use debug_overlay::{DebugOverlayDrawData, DebugOverlayError, DebugOverlayRenderer};
 
+mod video;
+pub use video::Video;
+
 pub mod gpu_data {
     pub use super::lines::gpu_data::{LineStripInfo, LineVertex};
     pub use super::point_cloud::gpu_data::PositionRadius;