From e973812aaf3f97d83ebddaf2e2d33f4bb1de4b3d Mon Sep 17 00:00:00 2001 From: LynithDev <61880709+LynithDev@users.noreply.github.com> Date: Sun, 8 Dec 2024 11:34:18 +0100 Subject: [PATCH] feat: SkyClient Part 2 --- Cargo.lock | 7 + Cargo.toml | 1 + packages/client/src/bindings.ts | 2 +- packages/core/Cargo.toml | 1 + .../src/api/package/content/curseforge.rs | 2 +- packages/core/src/api/package/content/mod.rs | 22 ++- .../core/src/api/package/content/modrinth.rs | 2 +- .../core/src/api/package/content/skyclient.rs | 165 +++++++++++++----- packages/core/src/store/package.rs | 78 +++++++-- packages/core/src/utils/crypto.rs | 17 ++ 10 files changed, 234 insertions(+), 63 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 654a9b3..49463d4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3212,6 +3212,12 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "md5" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771" + [[package]] name = "memchr" version = "2.7.4" @@ -3819,6 +3825,7 @@ dependencies = [ "interpulse", "iota_stronghold", "lazy_static", + "md5", "murmur2", "notify", "notify-debouncer-mini", diff --git a/Cargo.toml b/Cargo.toml index d4a2c25..9ac6c84 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -134,6 +134,7 @@ flate2 = { version = "1.0" } sha1_smol = { version = "1.0", features = [ "std" ] } sha2 = { version = "0.10" } murmur2 = { version = "0.1.0" } +md5 = { version = "0.7.0" } bytes = { version = "1.7" } # # ============= UTILITY ============= # # diff --git a/packages/client/src/bindings.ts b/packages/client/src/bindings.ts index 6ff12a3..30d97b7 100644 --- a/packages/client/src/bindings.ts +++ b/packages/client/src/bindings.ts @@ -865,7 +865,7 @@ export interface ManagedUser { id: string; username: string; url?: string | null /** * Universal managed package version of a package. */ -export interface ManagedVersion { id: string; package_id: string; author: string; name: string; featured: boolean; version_display: string; changelog: string; changelog_url: string | null; published: string; downloads: number; version_type: ManagedVersionReleaseType; files: ManagedVersionFile[]; is_available: boolean; deps: ManagedDependency[]; game_versions: string[]; loaders: Loader[] }; +export interface ManagedVersion { id: string; package_id: string; author: string; name: string; featured: boolean; version_display: string; changelog: string; changelog_url: string | null; published: string | null; downloads: number; version_type: ManagedVersionReleaseType; files: ManagedVersionFile[]; is_available: boolean; deps: ManagedDependency[]; game_versions: string[]; loaders: Loader[] }; /** * Universal interface for managed package files. */ diff --git a/packages/core/Cargo.toml b/packages/core/Cargo.toml index f9f577a..aa3177f 100644 --- a/packages/core/Cargo.toml +++ b/packages/core/Cargo.toml @@ -51,6 +51,7 @@ dirs = { workspace = true } sha1_smol = { workspace = true } sha2 = { workspace = true } murmur2 = { workspace = true } +md5 = { workspace = true } async_zip = { workspace = true } discord-rich-presence = { workspace = true } regex = { workspace = true } diff --git a/packages/core/src/api/package/content/curseforge.rs b/packages/core/src/api/package/content/curseforge.rs index cd55b21..247897f 100644 --- a/packages/core/src/api/package/content/curseforge.rs +++ b/packages/core/src/api/package/content/curseforge.rs @@ -352,7 +352,7 @@ impl Into for ModFile { is_available: self.is_available && files.len() > 0, files, game_versions: game_versions, - published: self.file_date, + published: Some(self.file_date), version_display: self.display_name, version_type: self.release_type.into(), } diff --git a/packages/core/src/api/package/content/mod.rs b/packages/core/src/api/package/content/mod.rs index 8b50d97..de68984 100644 --- a/packages/core/src/api/package/content/mod.rs +++ b/packages/core/src/api/package/content/mod.rs @@ -3,6 +3,7 @@ //! Utilities for searching and downloading content packages to `OneLauncher`. use std::collections::HashMap; +use std::path::PathBuf; use modrinth::{Facet, FacetOperation}; use reqwest::Method; @@ -11,6 +12,7 @@ use serde::{Deserialize, Serialize}; use crate::data::{Loader, ManagedPackage, ManagedUser, ManagedVersion, PackageType}; use crate::package::content::modrinth::FacetBuilder; use crate::store::{Author, PackageBody, ProviderSearchResults}; +use crate::utils::crypto; use crate::utils::http::fetch_json; use crate::utils::pagination::Pagination; use crate::{Result, State}; @@ -178,7 +180,10 @@ impl Providers { (data.0.into_iter().map(Into::into).collect(), data.1) } - Self::SkyClient => todo!(), + Self::SkyClient => { + let data = skyclient::get_all_versions(project_id, game_versions, loaders, page, page_size).await?; + (data.0.into_iter().map(Into::into).collect(), data.1) + }, }) } @@ -194,7 +199,7 @@ impl Providers { .into_iter() .map(Into::into) .collect(), - Self::SkyClient => todo!(), + Self::SkyClient => skyclient::get_versions(versions).await?, }) } @@ -230,9 +235,20 @@ impl Providers { .into_iter() .map(|(hash, version)| (hash, version.into())) .collect(), - Self::SkyClient => todo!(), + Self::SkyClient => skyclient::get_versions_by_hashes(hashes).await?, }) } + + pub fn hash_file(&self, path: &PathBuf) -> Result { + match self { + Providers::Modrinth => crypto::sha1_file(path), + Providers::SkyClient => crypto::md5_file(path), + Providers::Curseforge => { + let hash = crypto::murmur2_file(path)?; + Ok(hash.to_string()) + }, + } + } } #[cfg_attr(feature = "specta", derive(specta::Type))] diff --git a/packages/core/src/api/package/content/modrinth.rs b/packages/core/src/api/package/content/modrinth.rs index 19595e0..9ce5a23 100644 --- a/packages/core/src/api/package/content/modrinth.rs +++ b/packages/core/src/api/package/content/modrinth.rs @@ -177,7 +177,7 @@ impl From for ManagedVersion { changelog: value.changelog, changelog_url: value.changelog_url, - published: value.date_published, + published: Some(value.date_published), downloads: value.downloads, version_type: ManagedVersionReleaseType::from(value.version_type), diff --git a/packages/core/src/api/package/content/skyclient.rs b/packages/core/src/api/package/content/skyclient.rs index adb5422..f945a5f 100644 --- a/packages/core/src/api/package/content/skyclient.rs +++ b/packages/core/src/api/package/content/skyclient.rs @@ -1,9 +1,9 @@ -use std::sync::Arc; +use std::{collections::HashMap, sync::Arc}; use serde::{de::DeserializeOwned, Deserialize, Serialize}; use tokio::sync::{OnceCell, RwLock}; -use crate::{data::{Loader, ManagedPackage, ManagedUser, ManagedVersion}, store::{ProviderSearchResults, SearchResult}, utils::{http, pagination::Pagination}, Result, State}; +use crate::{data::{Loader, ManagedPackage, ManagedUser, ManagedVersion}, store::{ManagedVersionFile, ProviderSearchResults, SearchResult}, utils::{http, pagination::Pagination}, Result, State}; async fn fetch(url: &str) -> Result { let state = State::get().await?; @@ -143,6 +143,37 @@ pub struct SkyClientModVersion { pub mod_id: Option, } +fn get_managed_version(m: &SkyClientMod, v: &SkyClientModVersion) -> ManagedVersion { + let mut map = HashMap::new(); + map.insert(String::from("md5"), v.hash.clone()); + + ManagedVersion { + id: v.version.clone(), + package_id: m.id.clone(), + author: m.creator.clone(), + changelog_url: None, + changelog: String::new(), + deps: vec![], + downloads: 0, + featured: false, + files: vec![ManagedVersionFile { + file_name: v.file.clone(), + url: v.url.clone(), + file_type: None, + hashes: map, + primary: true, + size: 0, + }], + game_versions: v.game_versions.clone(), + is_available: true, + loaders: v.loaders.iter().map(|l| Loader::from_string(l.to_owned())).collect(), + name: format!("{} v{}", m.display, v.version), + published: None, + version_display: v.version.clone(), + version_type: crate::store::ManagedVersionReleaseType::Release, + } +} + pub async fn get(id: &str) -> Result { let store = SkyClientStore::get().await?; @@ -181,45 +212,97 @@ pub async fn get_multiple(slug_or_ids: &[String]) -> Result> { Ok(results) } -// pub async fn get_all_versions( -// project_id: &str, -// game_versions: Option>, -// loaders: Option>, -// page: Option, -// page_size: Option, -// ) -> Result<(Vec, Pagination)> { -// let store = SkyClientStore::get().await?; - -// let mods = match &store.mods { -// Some(mods) => mods, -// None => return Ok((vec![], Pagination::default())) -// }; - - -// let mut versions = vec![]; - -// for m in mods { -// if m.id == project_id { -// for v in m.versions { -// let mut can_add = true; - -// if let Some(game_versions) = &game_versions { -// can_add = v.game_versions.iter().any(|gv| game_versions.contains(gv)) -// } - -// if can_add { -// if let Some(loaders) = &loaders { -// can_add = v.loaders.iter().any(|l| loaders.contains(&Loader::from_string(l))) -// } -// } - -// if can_add { -// -// } -// } -// } -// } -// } +pub async fn get_all_versions( + project_id: &str, + game_versions: Option>, + loaders: Option>, + page: Option, + page_size: Option, +) -> Result<(Vec, Pagination)> { + let store = SkyClientStore::get().await?; + + let mods = match &store.mods { + Some(mods) => mods, + None => return Ok((vec![], Pagination::default())) + }; + + let mut pagination = Pagination::default(); + let mut versions = vec![]; + + for m in mods { + if m.id == project_id { + for v in &m.versions { + let mut can_add = true; + + if let Some(game_versions) = &game_versions { + if !game_versions.is_empty() { + can_add = v.game_versions.iter().any(|gv| game_versions.contains(gv)) + } + } + + if can_add { + if let Some(loaders) = &loaders { + if !loaders.is_empty() { + can_add = v.loaders.iter().any(|l| loaders.contains(&Loader::from_string(l.to_owned()))) + } + } + } + + if can_add { + if page_size.map(|ps| versions.len() < ps as usize).unwrap_or(true) && page.map(|p| p <= pagination.index).unwrap_or(true) { + versions.push(get_managed_version(m, v)); + } + + pagination.total_count += 1; + } + } + } + } + + Ok((versions, pagination)) +} + +pub async fn get_versions(versions: Vec) -> Result> { + let store = SkyClientStore::get().await?; + + let mods = match &store.mods { + Some(mods) => mods, + None => return Ok(vec![]) + }; + + let mut results = vec![]; + + for m in mods { + for v in &m.versions { + if versions.contains(&v.version) { + results.push(get_managed_version(m, v)); + } + } + } + + Ok(results) +} + +pub async fn get_versions_by_hashes(hashes: Vec) -> Result> { + let store = SkyClientStore::get().await?; + + let mods = match &store.mods { + Some(mods) => mods, + None => return Ok(HashMap::new()) + }; + + let mut results = HashMap::new(); + + for m in mods { + for v in &m.versions { + if hashes.contains(&v.hash) { + results.insert(v.hash.clone(), get_managed_version(m, v)); + } + } + } + + Ok(results) +} pub async fn search( query: Option, diff --git a/packages/core/src/store/package.rs b/packages/core/src/store/package.rs index 09f82f5..6857706 100644 --- a/packages/core/src/store/package.rs +++ b/packages/core/src/store/package.rs @@ -11,6 +11,7 @@ use onelauncher_utils::io; use serde::{Deserialize, Serialize}; use std::collections::{HashMap, HashSet}; use std::path::{Path, PathBuf}; +use std::vec; use tokio::fs::DirEntry; use super::{ClusterPath, Clusters, Directories, PackagePath}; @@ -471,21 +472,21 @@ impl PackageManager { // Infer any packages if packages_to_infer.len() > 0 { - let not_found = infer_provider(Providers::Modrinth, package_type, packages, packages_to_infer, crypto::sha1_file).await; - - if let Some(not_found) = not_found { - if let Some(not_found) = infer_provider(Providers::Curseforge, package_type, packages, not_found, |path| { - let hash = crypto::murmur2_file(path)?; - Ok(hash.to_string()) - }).await { - for path in not_found { - tracing::warn!("failed to infer package: {:?}", path); - - let package_path = PackagePath::new(&path); - let meta = PackageMetadata::Unknown; - let package = Package::new(&package_path, meta)?; - - packages.insert(package_path, package); + if let Some(not_found) = infer_provider(Providers::Modrinth, package_type, packages, packages_to_infer, crypto::sha1_file).await { + if let Some(not_found) = infer_provider(Providers::SkyClient, package_type, packages, not_found, crypto::md5_file).await { + if let Some(not_found) = infer_provider(Providers::Curseforge, package_type, packages, not_found, |path| { + let hash = crypto::murmur2_file(path)?; + Ok(hash.to_string()) + }).await { + for path in not_found { + tracing::warn!("failed to infer package: {:?}", path); + + let package_path = PackagePath::new(&path); + let meta = PackageMetadata::Unknown; + let package = Package::new(&package_path, meta)?; + + packages.insert(package_path, package); + } } } } @@ -529,6 +530,49 @@ impl PackageManager { } } +// #[tracing::instrument(skip(packages))] +// async fn infer_packages( +// packages: &mut PackagesMap, +// package_type: PackageType, +// to_infer: HashSet, +// ) { +// let infer_order = vec![ +// Providers::Modrinth, +// Providers::SkyClient, +// Providers::Curseforge, +// ]; + + +// // if let Some(not_found) = infer_provider(Providers::Modrinth, package_type, packages, packages_to_infer, crypto::sha1_file).await { +// // if let Some(not_found) = infer_provider(Providers::SkyClient, package_type, packages, not_found, crypto::md5_file).await { +// // if let Some(not_found) = infer_provider(Providers::Curseforge, package_type, packages, not_found, |path| { +// // let hash = crypto::murmur2_file(path)?; +// // Ok(hash.to_string()) +// // }).await { +// // } +// // } +// // } + +// let mut to_infer = to_infer; +// for provider in infer_order { +// if let Some(not_found) = infer_provider(provider, package_type, packages, to_infer, move|path| { +// Ok(provider.hash_file(path)) +// }).await { +// to_infer = not_found; +// } +// } + +// for path in to_infer { +// tracing::warn!("failed to infer package: {:?}", path); + +// let package_path = PackagePath::new(&path); +// let meta = PackageMetadata::Unknown; +// if let Ok(package) = Package::new(&package_path, meta) { +// packages.insert(package_path, package); +// } +// } +// } + #[tracing::instrument(skip(packages, to_infer))] async fn infer_provider( provider: Providers, @@ -537,6 +581,8 @@ async fn infer_provider( to_infer: HashSet, hash_fn: fn(&PathBuf) -> crate::Result, ) -> Option> { + // TODO: Add ingress + let mut hash_map = HashMap::::new(); to_infer.iter().for_each(|path| { if let Ok(hash) = hash_fn(path) { @@ -756,7 +802,7 @@ pub struct ManagedVersion { pub changelog: String, pub changelog_url: Option, - pub published: DateTime, + pub published: Option>, pub downloads: u32, pub version_type: ManagedVersionReleaseType, diff --git a/packages/core/src/utils/crypto.rs b/packages/core/src/utils/crypto.rs index a9f3a83..f898ca6 100644 --- a/packages/core/src/utils/crypto.rs +++ b/packages/core/src/utils/crypto.rs @@ -18,6 +18,23 @@ pub fn sha1_files(files: Vec<&PathBuf>) -> Vec> { hashes } +pub fn mda5(bytes: &[u8]) -> String { + let digest = md5::compute(bytes); + format!("{:x}", digest) +} + +pub fn md5_file(file: &PathBuf) -> Result { + Ok(mda5(&std::fs::read(file)?)) +} + +pub fn md5_files(files: Vec<&PathBuf>) -> Vec> { + let mut hashes = Vec::new(); + for file in files { + hashes.push(md5_file(file).ok()); + } + hashes +} + const CURSEFORGE_FINGERPRINT_SEED: u32 = 1; pub fn murmur2(bytes: &[u8]) -> u32 { let normalized = normalize_byte_array(bytes);