diff --git a/Cargo.lock b/Cargo.lock index 58768895..b0c300f3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -401,7 +401,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "117725a109d387c937a1533ce01b450cbde6b88abceea8473c4d7a85853cda3c" dependencies = [ "lazy_static", - "windows-sys 0.48.0", + "windows-sys 0.59.0", ] [[package]] @@ -410,7 +410,7 @@ version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fde0e0ec90c9dfb3b4b1a0891a7dcd0e2bffde2f7efed5fe7c9bb00e5bfb915e" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.59.0", ] [[package]] @@ -592,6 +592,20 @@ dependencies = [ "syn", ] +[[package]] +name = "dashmap" +version = "6.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf" +dependencies = [ + "cfg-if", + "crossbeam-utils", + "hashbrown 0.14.5", + "lock_api", + "once_cell", + "parking_lot_core", +] + [[package]] name = "data-encoding" version = "2.9.0" @@ -1026,6 +1040,12 @@ version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" + [[package]] name = "hashbrown" version = "0.15.3" @@ -3607,7 +3627,7 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.59.0", ] [[package]] @@ -4101,6 +4121,7 @@ dependencies = [ "clipanion", "colored 3.0.0", "convert_case 0.8.0", + "dashmap", "dialoguer", "env_logger", "fancy-regex", diff --git a/packages/zpm/Cargo.toml b/packages/zpm/Cargo.toml index 723b5a5f..ca61d1ee 100644 --- a/packages/zpm/Cargo.toml +++ b/packages/zpm/Cargo.toml @@ -48,3 +48,4 @@ hickory-resolver = "0.25.2" indexmap = {version = "2.11.0", features = ["serde"]} sha2 = "0.10.8" hex = "0.4.3" +dashmap = "6.1.0" diff --git a/packages/zpm/src/install.rs b/packages/zpm/src/install.rs index 6d47b36b..1d011238 100644 --- a/packages/zpm/src/install.rs +++ b/packages/zpm/src/install.rs @@ -1,5 +1,6 @@ use std::{collections::{BTreeMap, BTreeSet}, hash::Hash, marker::PhantomData, sync::LazyLock}; +use dashmap::DashMap; use rayon::iter::{IntoParallelIterator, ParallelIterator}; use zpm_config::PackageExtension; use zpm_primitives::{Descriptor, Ident, Locator, PatchRange, PeerRange, Range, RegistrySemverRange, RegistryTagRange, SemverDescriptor, SemverPeerRange}; @@ -21,6 +22,7 @@ pub struct InstallContext<'a> { pub check_checksums: bool, pub check_resolutions: bool, pub enforced_resolutions: BTreeMap, + pub npm_metadata_cache: Option<&'a DashMap>, pub refresh_lockfile: bool, pub mode: Option, } @@ -34,6 +36,7 @@ impl<'a> Default for InstallContext<'a> { check_checksums: false, check_resolutions: false, enforced_resolutions: BTreeMap::new(), + npm_metadata_cache: None, refresh_lockfile: false, mode: None, } @@ -51,6 +54,11 @@ impl<'a> InstallContext<'a> { self } + pub fn with_npm_metadata_cache(mut self, npm_metadata_cache: Option<&'a DashMap>) -> Self { + self.npm_metadata_cache = npm_metadata_cache; + self + } + pub fn set_check_checksums(mut self, check_checksums: bool) -> Self { self.check_checksums = check_checksums; self diff --git a/packages/zpm/src/project.rs b/packages/zpm/src/project.rs index bab02b89..ee372242 100644 --- a/packages/zpm/src/project.rs +++ b/packages/zpm/src/project.rs @@ -1,5 +1,6 @@ use std::{collections::{BTreeMap, HashSet}, io::ErrorKind, sync::Arc, time::UNIX_EPOCH}; +use dashmap::DashMap; use globset::{GlobBuilder, GlobSetBuilder}; use zpm_config::{Configuration, ConfigurationContext}; use zpm_macro_enum::zpm_enum; @@ -625,9 +626,13 @@ impl Project { } } + let npm_metadata_cache + = DashMap::new(); + let install_context = InstallContext::default() .with_package_cache(Some(&package_cache)) .with_project(Some(self)) + .with_npm_metadata_cache(Some(&npm_metadata_cache)) .set_check_checksums(options.check_checksums) .set_enforced_resolutions(options.enforced_resolutions) .set_refresh_lockfile(options.refresh_lockfile) diff --git a/packages/zpm/src/resolvers/npm.rs b/packages/zpm/src/resolvers/npm.rs index 57a316f8..41353532 100644 --- a/packages/zpm/src/resolvers/npm.rs +++ b/packages/zpm/src/resolvers/npm.rs @@ -207,34 +207,52 @@ pub async fn resolve_semver_or_workspace_descriptor(context: &InstallContext<'_> resolve_semver_descriptor(context, descriptor, params).await } -pub async fn resolve_semver_descriptor(context: &InstallContext<'_>, descriptor: &Descriptor, params: &RegistrySemverRange) -> Result { +async fn get_package_metadata_text(context: &InstallContext<'_>, package_ident: &Ident, f: impl FnOnce(&String) -> Result) -> Result { + let cache_entry = context.npm_metadata_cache.as_ref() + .and_then(|cache| cache.get(package_ident)); + + if let Some(cache_entry) = cache_entry { + return Ok(f(&cache_entry)?); + } + let project = context.project .expect("The project is required for resolving a workspace package"); - let package_ident = params.ident.as_ref() - .unwrap_or(&descriptor.ident); - let registry_url = npm::registry_url_for_all_versions(&project.config.registry_base_for(package_ident), package_ident); - let response + let response = project.http_client.get(®istry_url)?.send().await?; let registry_text = response.text().await .map_err(|err| Error::RemoteRegistryError(Arc::new(err)))?; + let result + = f(®istry_text)?; - let mut deserializer - = sonic_rs::Deserializer::from_str(registry_text.as_str()); + context.npm_metadata_cache.as_ref() + .map(|cache| cache.insert(package_ident.clone(), registry_text)); - let (version, manifest) = deserializer.deserialize_map(FindFieldNested { - field: "versions", - nested: FindHighestCompatibleVersion { - range: params.range.clone(), - phantom: PhantomData::, - }, - })?.ok_or_else(|| { - Error::NoCandidatesFound(descriptor.range.clone()) - })?; + Ok(result) +} + +pub async fn resolve_semver_descriptor(context: &InstallContext<'_>, descriptor: &Descriptor, params: &RegistrySemverRange) -> Result { + let package_ident = params.ident.as_ref() + .unwrap_or(&descriptor.ident); + + let (version, manifest) = get_package_metadata_text(context, package_ident, |registry_text| { + let mut deserializer + = sonic_rs::Deserializer::from_str(registry_text); + + deserializer.deserialize_map(FindFieldNested { + field: "versions", + nested: FindHighestCompatibleVersion { + range: params.range.clone(), + phantom: PhantomData::, + }, + })?.ok_or_else(|| { + Error::NoCandidatesFound(descriptor.range.clone()) + }) + }).await?; Ok(build_resolution_result(context, descriptor, package_ident, version, manifest)) } @@ -253,44 +271,37 @@ pub async fn resolve_tag_or_workspace_descriptor(context: &InstallContext<'_>, d } pub async fn resolve_tag_descriptor(context: &InstallContext<'_>, descriptor: &Descriptor, params: &RegistryTagRange) -> Result { - let project = context.project - .expect("The project is required for resolving a workspace package"); - let package_ident = params.ident.as_ref() .unwrap_or(&descriptor.ident); - let registry_url - = npm::registry_url_for_all_versions(&project.config.registry_base_for(package_ident), package_ident); + let (version, manifest) = get_package_metadata_text(context, package_ident, |registry_text| { + #[derive(Deserialize)] + struct RegistryMetadata { + #[serde(rename(deserialize = "dist-tags"))] + dist_tags: sonic_rs::Value, + versions: sonic_rs::Value, + } - let response - = project.http_client.get(®istry_url)?.send().await?; + let registry_data: RegistryMetadata = sonic_rs::from_str(registry_text.as_str()) + .map_err(Arc::new)?; - let registry_text = response.text().await - .map_err(|err| Error::RemoteRegistryError(Arc::new(err)))?; + let version = registry_data.dist_tags.deserialize_map(FindField { + value: params.tag.as_str(), + phantom: PhantomData::, + })?.ok_or_else(|| { + Error::TagNotFound(params.tag.clone()) + })?; - #[derive(Deserialize)] - struct RegistryMetadata { - #[serde(rename(deserialize = "dist-tags"))] - dist_tags: sonic_rs::Value, - versions: sonic_rs::Value, - } + let manifest = registry_data.versions.deserialize_map(FindField { + value: &version.to_file_string(), + phantom: PhantomData::, + })?.ok_or_else(|| { + Error::NoCandidatesFound(descriptor.range.clone()) + })?; + + Ok((version, manifest)) + }).await?; - let registry_data: RegistryMetadata = sonic_rs::from_str(registry_text.as_str()) - .map_err(Arc::new)?; - - let version = registry_data.dist_tags.deserialize_map(FindField { - value: params.tag.as_str(), - phantom: PhantomData::, - })?.ok_or_else(|| { - Error::TagNotFound(params.tag.clone()) - })?; - - let manifest = registry_data.versions.deserialize_map(FindField { - value: &version.to_file_string(), - phantom: PhantomData::, - })?.ok_or_else(|| { - Error::NoCandidatesFound(descriptor.range.clone()) - })?; Ok(build_resolution_result(context, descriptor, package_ident, version, manifest)) }