diff --git a/crates/web/Cargo.toml b/crates/web/Cargo.toml new file mode 100644 index 00000000..9da73e81 --- /dev/null +++ b/crates/web/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "web" +version = "0.1.0" +edition = "2024" + +[dependencies] +anyhow = "1.0.94" +mime = "0.3.17" +regex = "1.11.1" +reqwest = { version = "0.12.9", features = ["blocking", "json", "stream"] } +select = "0.6.0" +serde = { version = "1.0.215", features = ["derive"] } +tempfile = "3.14.0" +tokio = { version = "1.42.0", features = ["full"] } +url = "2.5.4" diff --git a/crates/web/src/lib.rs b/crates/web/src/lib.rs new file mode 100644 index 00000000..1e5c72ae --- /dev/null +++ b/crates/web/src/lib.rs @@ -0,0 +1,15 @@ +mod paginated; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_reverse_dependencies() -> Result<()> { + for dep in paginated::ReverseDependencies::of("serde")?.take(5) { + let dependency = dep?; + println!("{} depends on {}", dependency.id, dependency.crate_id); + } + Ok(()) + } +} diff --git a/crates/web/src/paginated.rs b/crates/web/src/paginated.rs new file mode 100644 index 00000000..a26410b2 --- /dev/null +++ b/crates/web/src/paginated.rs @@ -0,0 +1,79 @@ +use reqwest::Result; +use reqwest::header::USER_AGENT; +use serde::Deserialize; + +#[derive(Deserialize)] +struct ApiResponse { + dependencies: Vec, + meta: Meta, +} + +#[derive(Deserialize)] +pub struct Dependency { + pub crate_id: String, + pub id: u32, +} + +#[derive(Deserialize)] +struct Meta { + total: u32, +} + +pub struct ReverseDependencies { + crate_id: String, + dependencies: as IntoIterator>::IntoIter, + client: reqwest::blocking::Client, + page: u32, + per_page: u32, + total: u32, +} + +impl ReverseDependencies { + pub fn of(crate_id: &str) -> Result { + Ok(ReverseDependencies { + crate_id: crate_id.to_owned(), + dependencies: vec![].into_iter(), + client: reqwest::blocking::Client::new(), + page: 0, + per_page: 100, + total: 0, + }) + } + + fn try_next(&mut self) -> Result> { + if let Some(dep) = self.dependencies.next() { + return Ok(Some(dep)); + } + + if self.page > 0 && self.page * self.per_page >= self.total { + return Ok(None); + } + + self.page += 1; + let url = format!("https://crates.io/api/v1/crates/{}/reverse_dependencies?page={}&per_page={}", + self.crate_id, + self.page, + self.per_page); + println!("{}", url); + + let response = self.client.get(&url).header( + USER_AGENT, + "cookbook agent", + ).send()?.json::()?; + self.dependencies = response.dependencies.into_iter(); + self.total = response.meta.total; + Ok(self.dependencies.next()) + } +} + +impl Iterator for ReverseDependencies { + type Item = Result; + + fn next(&mut self) -> Option { + match self.try_next() { + Ok(Some(dep)) => Some(Ok(dep)), + Ok(None) => None, + Err(err) => Some(Err(err)), + } + } +} diff --git a/src/web/clients/api/paginated.md b/src/web/clients/api/paginated.md index 5780c6d8..a021a2ea 100644 --- a/src/web/clients/api/paginated.md +++ b/src/web/clients/api/paginated.md @@ -3,87 +3,22 @@ [![reqwest-badge]][reqwest] [![serde-badge]][serde] [![cat-net-badge]][cat-net] [![cat-encoding-badge]][cat-encoding] Wraps a paginated web API in a convenient Rust iterator. The iterator lazily -fetches the next page of results from the remote server as it arrives at the end -of each page. +fetches the next page of results from the remote server as it arrives at the end of each page. -```rust,edition2018,no_run -use reqwest::Result; -use serde::Deserialize; - -#[derive(Deserialize)] -struct ApiResponse { - dependencies: Vec, - meta: Meta, -} - -#[derive(Deserialize)] -struct Dependency { - crate_id: String, -} - -#[derive(Deserialize)] -struct Meta { - total: u32, -} - -struct ReverseDependencies { - crate_id: String, - dependencies: as IntoIterator>::IntoIter, - client: reqwest::blocking::Client, - page: u32, - per_page: u32, - total: u32, -} - -impl ReverseDependencies { - fn of(crate_id: &str) -> Result { - Ok(ReverseDependencies { - crate_id: crate_id.to_owned(), - dependencies: vec![].into_iter(), - client: reqwest::blocking::Client::new(), - page: 0, - per_page: 100, - total: 0, - }) - } - - fn try_next(&mut self) -> Result> { - if let Some(dep) = self.dependencies.next() { - return Ok(Some(dep)); - } - - if self.page > 0 && self.page * self.per_page >= self.total { - return Ok(None); - } - - self.page += 1; - let url = format!("https://crates.io/api/v1/crates/{}/reverse_dependencies?page={}&per_page={}", - self.crate_id, - self.page, - self.per_page); - - let response = self.client.get(&url).send()?.json::()?; - self.dependencies = response.dependencies.into_iter(); - self.total = response.meta.total; - Ok(self.dependencies.next()) - } -} +This file is named paginated.rs. +```rust,edition2024,no_run +{{#include ../../../../crates/web/src/paginated.rs}} +``` -impl Iterator for ReverseDependencies { - type Item = Result; +In this main we import the paginated file and use the `ReverseDependencies` iterator to fetch all the crates that depend on the `serde` crate. This file is named main.rs. The two files are in the same directory. - fn next(&mut self) -> Option { - match self.try_next() { - Ok(Some(dep)) => Some(Ok(dep)), - Ok(None) => None, - Err(err) => Some(Err(err)), - } - } -} +```rust,no_run +mod paginated; fn main() -> Result<()> { - for dep in ReverseDependencies::of("serde")? { - println!("reverse dependency: {}", dep?.crate_id); + for dep in paginated::ReverseDependencies::of("serde")? { + let dependency = dep?; + println!("{} depends on {}", dependency.id, dependency.crate_id); } Ok(()) }