Fix: re_sdk no longer depends on rustls (#6210)

Split out parts of `re_data_source` to a new crate `re_data_loader`.
This makes the dependency list of `re_sdk` much lighter.

* Closes #6090

`scripts/ci/count_dependencies.py -p re_sdk --all-features` goes from
204 -> 185 dependencies.

### Checklist
* [x] I have read and agree to [Contributor
Guide](https://github.com/rerun-io/rerun/blob/main/CONTRIBUTING.md) and
the [Code of
Conduct](https://github.com/rerun-io/rerun/blob/main/CODE_OF_CONDUCT.md)
* [x] I've included a screenshot or gif (if applicable)
* [x] I have tested the web demo (if applicable):
* Using examples from latest `main` build:
[rerun.io/viewer](https://rerun.io/viewer/pr/6210?manifest_url=https://app.rerun.io/version/main/examples_manifest.json)
* Using full set of examples from `nightly` build:
[rerun.io/viewer](https://rerun.io/viewer/pr/6210?manifest_url=https://app.rerun.io/version/nightly/examples_manifest.json)
* [x] The PR title and labels are set such as to maximize their
usefulness for the next release's CHANGELOG
* [x] If applicable, add a new check to the [release
checklist](https://github.com/rerun-io/rerun/blob/main/tests/python/release_checklist)!

- [PR Build Summary](https://build.rerun.io/pr/6210)
- [Recent benchmark results](https://build.rerun.io/graphs/crates.html)
- [Wasm size tracking](https://build.rerun.io/graphs/sizes.html)

To run all checks from `main`, comment on the PR with `@rerun-bot
full-check`.

ARCHITECTURE.md

@@ -88,11 +88,11 @@ Of course, this will only take us so far. In the future we plan on caching queri
 Here is an overview of the crates included in the project:
 
 <picture>
- <img src="https://static.rerun.io/crates/b5d47defe239a5e495c7633c0ed4a924727d5588/full.png" alt="">
- <source media="(max-width: 480px)" srcset="https://static.rerun.io/crates/b5d47defe239a5e495c7633c0ed4a924727d5588/480w.png">
- <source media="(max-width: 768px)" srcset="https://static.rerun.io/crates/b5d47defe239a5e495c7633c0ed4a924727d5588/768w.png">
- <source media="(max-width: 1024px)" srcset="https://static.rerun.io/crates/b5d47defe239a5e495c7633c0ed4a924727d5588/1024w.png">
- <source media="(max-width: 1200px)" srcset="https://static.rerun.io/crates/b5d47defe239a5e495c7633c0ed4a924727d5588/1200w.png">
+ <img src="https://static.rerun.io/crates/f0024a424aa35efab710ae88ba6a1ed741e7c248/full.png" alt="">
+ <source media="(max-width: 480px)" srcset="https://static.rerun.io/crates/f0024a424aa35efab710ae88ba6a1ed741e7c248/480w.png">
+ <source media="(max-width: 768px)" srcset="https://static.rerun.io/crates/f0024a424aa35efab710ae88ba6a1ed741e7c248/768w.png">
+ <source media="(max-width: 1024px)" srcset="https://static.rerun.io/crates/f0024a424aa35efab710ae88ba6a1ed741e7c248/1024w.png">
+ <source media="(max-width: 1200px)" srcset="https://static.rerun.io/crates/f0024a424aa35efab710ae88ba6a1ed741e7c248/1200w.png">
 </picture>
 
 <!-- !!! IMPORTANT!!!
@@ -171,7 +171,8 @@ Update instructions:
 | re_sdk_comms | TCP communication between Rerun SDK and Rerun Server |
 | re_web_viewer_server | Serves the Rerun web viewer (Wasm and HTML) over HTTP |
 | re_ws_comms | WebSocket communication library (encoding, decoding, client, server) between a Rerun server and viewer |
-| re_data_source | Handles loading of Rerun data |
+| re_data_loader | Handles loading of Rerun data from file using data loader plugins |
+| re_data_source | Handles loading of Rerun data from different sources |
 
 
 ### Build support

Cargo.toml

@@ -32,6 +32,7 @@ re_analytics = { path = "crates/re_analytics", version = "=0.16.0-alpha.2", defa
 re_build_info = { path = "crates/re_build_info", version = "=0.16.0-alpha.2", default-features = false }
 re_build_tools = { path = "crates/re_build_tools", version = "=0.16.0-alpha.2", default-features = false }
 re_crash_handler = { path = "crates/re_crash_handler", version = "=0.16.0-alpha.2", default-features = false }
+re_data_loader = { path = "crates/re_data_loader", version = "=0.16.0-alpha.2", default-features = false }
 re_data_source = { path = "crates/re_data_source", version = "=0.16.0-alpha.2", default-features = false }
 re_data_store = { path = "crates/re_data_store", version = "=0.16.0-alpha.2", default-features = false }
 re_data_ui = { path = "crates/re_data_ui", version = "=0.16.0-alpha.2", default-features = false }

crates/re_data_loader/Cargo.toml

@@ -0,0 +1,44 @@
+[package]
+name = "re_data_loader"
+authors.workspace = true
+description = "Handles loading of Rerun data from file using data loader plugins"
+edition.workspace = true
+homepage.workspace = true
+include.workspace = true
+license.workspace = true
+publish = true
+readme = "README.md"
+repository.workspace = true
+rust-version.workspace = true
+version.workspace = true
+
+[lints]
+workspace = true
+
+[package.metadata.docs.rs]
+all-features = true
+
+
+[features]
+default = []
+
+
+[dependencies]
+re_log_encoding = { workspace = true, features = ["decoder"] }
+re_log_types.workspace = true
+re_log.workspace = true
+re_smart_channel.workspace = true
+re_tracing.workspace = true
+re_types = { workspace = true, features = ["image"] }
+
+ahash.workspace = true
+anyhow.workspace = true
+image.workspace = true
+once_cell.workspace = true
+parking_lot.workspace = true
+rayon.workspace = true
+thiserror.workspace = true
+walkdir.workspace = true
+
+[build-dependencies]
+re_build_tools.workspace = true

crates/re_data_loader/README.md

@@ -0,0 +1,10 @@
+# re_data_loader
+
+Part of the [`rerun`](https://github.com/rerun-io/rerun) family of crates.
+
+[![Latest version](https://img.shields.io/crates/v/re_data_loader.svg)](https://crates.io/crates/re_data_loader)
+[![Documentation](https://docs.rs/re_data_loader/badge.svg)](https://docs.rs/re_data_loader)
+![MIT](https://img.shields.io/badge/license-MIT-blue.svg)
+![Apache](https://img.shields.io/badge/license-Apache-blue.svg)
+
+Handles loading of Rerun data from file using data loader plugins

crates/re_data_source/src/data_loader/mod.rs → crates/re_data_loader/src/lib.rs

@@ -1,10 +1,36 @@
+//! Handles loading of Rerun data from file using data loader plugins.
+
 use std::sync::Arc;
 
 use once_cell::sync::Lazy;
 
 use re_log_types::{ArrowMsg, DataRow, EntityPath, LogMsg, TimePoint};
 
-// ---
+// ----------------------------------------------------------------------------
+
+mod load_file;
+mod loader_archetype;
+mod loader_directory;
+mod loader_rrd;
+
+#[cfg(not(target_arch = "wasm32"))]
+mod loader_external;
+
+pub use self::{
+ load_file::load_from_file_contents, loader_archetype::ArchetypeLoader,
+ loader_directory::DirectoryLoader, loader_rrd::RrdLoader,
+};
+
+#[cfg(not(target_arch = "wasm32"))]
+pub use self::{
+ load_file::load_from_path,
+ loader_external::{
+ iter_external_loaders, ExternalLoader, EXTERNAL_DATA_LOADER_INCOMPATIBLE_EXIT_CODE,
+ EXTERNAL_DATA_LOADER_PREFIX,
+ },
+};
+
+// ----------------------------------------------------------------------------
 
 /// Recommended settings for the [`DataLoader`].
 ///
@@ -340,7 +366,7 @@ impl LoadedData {
  }
 }
 
-// ---
+// ----------------------------------------------------------------------------
 
 /// Keeps track of all builtin [`DataLoader`]s.
 ///
@@ -379,21 +405,52 @@ pub fn register_custom_data_loader(loader: impl DataLoader + 'static) {
  CUSTOM_LOADERS.write().push(Arc::new(loader));
 }
 
-// ---
+// ----------------------------------------------------------------------------
 
-mod loader_archetype;
-mod loader_directory;
-mod loader_rrd;
+/// Empty string if no extension.
+#[inline]
+pub(crate) fn extension(path: &std::path::Path) -> String {
+ path.extension()
+ .unwrap_or_default()
+ .to_ascii_lowercase()
+ .to_string_lossy()
+ .to_string()
+}
 
-#[cfg(not(target_arch = "wasm32"))]
-mod loader_external;
+// ----------------------------------------------------------------------------
 
-pub use self::loader_archetype::ArchetypeLoader;
-pub use self::loader_directory::DirectoryLoader;
-pub use self::loader_rrd::RrdLoader;
+// …given that all feature flags are turned on for the `image` crate.
+pub const SUPPORTED_IMAGE_EXTENSIONS: &[&str] = &[
+ "avif", "bmp", "dds", "exr", "farbfeld", "ff", "gif", "hdr", "ico", "jpeg", "jpg", "pam",
+ "pbm", "pgm", "png", "ppm", "tga", "tif", "tiff", "webp",
+];
 
-#[cfg(not(target_arch = "wasm32"))]
-pub use self::loader_external::{
- iter_external_loaders, ExternalLoader, EXTERNAL_DATA_LOADER_INCOMPATIBLE_EXIT_CODE,
- EXTERNAL_DATA_LOADER_PREFIX,
-};
+pub const SUPPORTED_MESH_EXTENSIONS: &[&str] = &["glb", "gltf", "obj", "stl"];
+
+// TODO(#4532): `.ply` data loader should support 2D point cloud & meshes
+pub const SUPPORTED_POINT_CLOUD_EXTENSIONS: &[&str] = &["ply"];
+
+pub const SUPPORTED_RERUN_EXTENSIONS: &[&str] = &["rbl", "rrd"];
+
+// TODO(#4555): Add catch-all builtin `DataLoader` for text files
+pub const SUPPORTED_TEXT_EXTENSIONS: &[&str] = &["txt", "md"];
+
+/// All file extension supported by our builtin [`DataLoader`]s.
+pub fn supported_extensions() -> impl Iterator<Item = &'static str> {
+ SUPPORTED_RERUN_EXTENSIONS
+ .iter()
+ .chain(SUPPORTED_IMAGE_EXTENSIONS)
+ .chain(SUPPORTED_MESH_EXTENSIONS)
+ .chain(SUPPORTED_POINT_CLOUD_EXTENSIONS)
+ .chain(SUPPORTED_TEXT_EXTENSIONS)
+ .copied()
+}
+
+/// Is this a supported file extension by any of our builtin [`DataLoader`]s?
+pub fn is_supported_file_extension(extension: &str) -> bool {
+ SUPPORTED_IMAGE_EXTENSIONS.contains(&extension)
+ || SUPPORTED_MESH_EXTENSIONS.contains(&extension)
+ || SUPPORTED_POINT_CLOUD_EXTENSIONS.contains(&extension)
+ || SUPPORTED_RERUN_EXTENSIONS.contains(&extension)
+ || SUPPORTED_TEXT_EXTENSIONS.contains(&extension)
+}

crates/re_data_source/src/load_file.rs → crates/re_data_loader/src/load_file.rs

@@ -3,7 +3,7 @@ use std::borrow::Cow;
 use re_log_types::{FileSource, LogMsg};
 use re_smart_channel::Sender;
 
-use crate::{DataLoaderError, LoadedData};
+use crate::{extension, DataLoaderError, LoadedData};
 
 // ---
 
@@ -88,16 +88,6 @@ pub fn load_from_file_contents(
 
 // ---
 
-/// Empty string if no extension.
-#[inline]
-pub fn extension(path: &std::path::Path) -> String {
- path.extension()
- .unwrap_or_default()
- .to_ascii_lowercase()
- .to_string_lossy()
- .to_string()
-}
-
 /// Prepares an adequate [`re_log_types::StoreInfo`] [`LogMsg`] given the input.
 pub(crate) fn prepare_store_info(
  store_id: &re_log_types::StoreId,

crates/re_data_source/Cargo.toml

@@ -24,6 +24,7 @@ default = []
 
 
 [dependencies]
+re_data_loader.workspace = true
 re_log_encoding = { workspace = true, features = [
  "decoder",
  "stream_from_http",
@@ -32,18 +33,11 @@ re_log_types.workspace = true
 re_log.workspace = true
 re_smart_channel.workspace = true
 re_tracing.workspace = true
-re_types = { workspace = true, features = ["image"] }
 re_ws_comms = { workspace = true, features = ["client"] }
 
-ahash.workspace = true
 anyhow.workspace = true
-image.workspace = true
 itertools.workspace = true
-once_cell.workspace = true
-parking_lot.workspace = true
 rayon.workspace = true
-thiserror.workspace = true
-walkdir.workspace = true
 
 [build-dependencies]
 re_build_tools.workspace = true