From 82de2ac84a1c083872193d530f84a4f897490037 Mon Sep 17 00:00:00 2001 From: Jai A Date: Sat, 7 Sep 2024 15:31:47 -0700 Subject: [PATCH] Optimize user-generated images for reduced bandwidth --- Cargo.lock | 46 ++- Cargo.toml | 1 + migrations/20240907192840_raw-images.sql | 22 ++ src/database/models/collection_item.rs | 10 +- src/database/models/image_item.rs | 12 +- src/database/models/oauth_client_item.rs | 14 +- src/database/models/organization_item.rs | 12 +- src/database/models/project_item.rs | 28 +- src/database/models/user_item.rs | 9 +- src/models/v2/projects.rs | 2 + src/models/v3/projects.rs | 2 + src/routes/internal/flows.rs | 51 ++- src/routes/v3/collections.rs | 137 ++++---- src/routes/v3/images.rs | 340 ++++++++++---------- src/routes/v3/oauth_clients.rs | 114 +++---- src/routes/v3/organizations.rs | 171 +++++----- src/routes/v3/project_creation.rs | 95 +++--- src/routes/v3/projects.rs | 392 +++++++++++------------ src/routes/v3/users.rs | 113 +++---- src/util/img.rs | 156 ++++++++- 20 files changed, 956 insertions(+), 771 deletions(-) create mode 100644 migrations/20240907192840_raw-images.sql diff --git a/Cargo.lock b/Cargo.lock index 91c07464..d49adb9b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -734,6 +734,12 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" +[[package]] +name = "byteorder-lite" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495" + [[package]] name = "bytes" version = "1.6.0" @@ -1800,6 +1806,12 @@ version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40ecd4077b5ae9fd2e9e169b102c6c330d0605168eb0e8bf79952b256dbefffd" +[[package]] +name = "glob" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" + [[package]] name = "governor" version = "0.6.3" @@ -2243,6 +2255,17 @@ dependencies = [ "tiff", ] +[[package]] +name = "image" +version = "0.25.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99314c8a2152b8ddb211f924cdae532d8c5e4c8bb54728e12fff1b0cd5963a10" +dependencies = [ + "bytemuck", + "byteorder-lite", + "num-traits", +] + [[package]] name = "indexmap" version = "1.9.3" @@ -2493,7 +2516,7 @@ dependencies = [ "hmac 0.11.0", "hyper 0.14.29", "hyper-tls 0.5.0", - "image", + "image 0.24.9", "itertools 0.12.1", "jemallocator", "json-patch", @@ -2530,6 +2553,7 @@ dependencies = [ "urlencoding", "uuid 1.9.1", "validator", + "webp", "woothee", "xml-rs", "yaserde", @@ -2627,6 +2651,16 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "libwebp-sys" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54cd30df7c7165ce74a456e4ca9732c603e8dc5e60784558c1c6dc047f876733" +dependencies = [ + "cc", + "glob", +] + [[package]] name = "libz-sys" version = "1.1.18" @@ -5631,6 +5665,16 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "webp" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f53152f51fb5af0c08484c33d16cca96175881d1f3dec068c23b31a158c2d99" +dependencies = [ + "image 0.25.2", + "libwebp-sys", +] + [[package]] name = "webpki-roots" version = "0.25.4" diff --git a/Cargo.toml b/Cargo.toml index 71d238ae..5cb762ac 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -102,6 +102,7 @@ sentry-actix = "0.32.1" image = "0.24.6" color-thief = "0.2.2" +webp = "0.3.0" woothee = "0.13.0" diff --git a/migrations/20240907192840_raw-images.sql b/migrations/20240907192840_raw-images.sql new file mode 100644 index 00000000..f929afa9 --- /dev/null +++ b/migrations/20240907192840_raw-images.sql @@ -0,0 +1,22 @@ +ALTER TABLE mods ADD COLUMN raw_icon_url TEXT NULL; +UPDATE mods SET raw_icon_url = icon_url; + +ALTER TABLE users ADD COLUMN raw_avatar_url TEXT NULL; +UPDATE users SET raw_avatar_url = avatar_url; + +ALTER TABLE oauth_clients ADD COLUMN raw_icon_url TEXT NULL; +UPDATE oauth_clients SET raw_icon_url = icon_url; + +ALTER TABLE organizations ADD COLUMN raw_icon_url TEXT NULL; +UPDATE organizations SET raw_icon_url = icon_url; + +ALTER TABLE collections ADD COLUMN raw_icon_url TEXT NULL; +UPDATE collections SET raw_icon_url = icon_url; + +ALTER TABLE mods_gallery ADD COLUMN raw_image_url TEXT NULL; +UPDATE mods_gallery SET raw_image_url = image_url; +ALTER TABLE mods_gallery ALTER COLUMN raw_image_url SET NOT NULL; + +ALTER TABLE uploaded_images ADD COLUMN raw_url TEXT NULL; +UPDATE uploaded_images SET raw_url = url; +ALTER TABLE uploaded_images ALTER COLUMN raw_url SET NOT NULL; \ No newline at end of file diff --git a/src/database/models/collection_item.rs b/src/database/models/collection_item.rs index 1f703950..9bb93761 100644 --- a/src/database/models/collection_item.rs +++ b/src/database/models/collection_item.rs @@ -33,6 +33,7 @@ impl CollectionBuilder { created: Utc::now(), updated: Utc::now(), icon_url: None, + raw_icon_url: None, color: None, status: self.status, projects: self.projects, @@ -51,6 +52,7 @@ pub struct Collection { pub created: DateTime, pub updated: DateTime, pub icon_url: Option, + pub raw_icon_url: Option, pub color: Option, pub status: CollectionStatus, pub projects: Vec, @@ -65,11 +67,11 @@ impl Collection { " INSERT INTO collections ( id, user_id, name, description, - created, icon_url, status + created, icon_url, raw_icon_url, status ) VALUES ( $1, $2, $3, $4, - $5, $6, $7 + $5, $6, $7, $8 ) ", self.id as CollectionId, @@ -78,6 +80,7 @@ impl Collection { self.description.as_ref(), self.created, self.icon_url.as_ref(), + self.raw_icon_url.as_ref(), self.status.to_string(), ) .execute(&mut **transaction) @@ -165,7 +168,7 @@ impl Collection { let collections = sqlx::query!( " SELECT c.id id, c.name name, c.description description, - c.icon_url icon_url, c.color color, c.created created, c.user_id user_id, + c.icon_url icon_url, c.raw_icon_url raw_icon_url, c.color color, c.created created, c.user_id user_id, c.updated updated, c.status status, ARRAY_AGG(DISTINCT cm.mod_id) filter (where cm.mod_id is not null) mods FROM collections c @@ -183,6 +186,7 @@ impl Collection { name: m.name.clone(), description: m.description.clone(), icon_url: m.icon_url.clone(), + raw_icon_url: m.raw_icon_url.clone(), color: m.color.map(|x| x as u32), created: m.created, updated: m.updated, diff --git a/src/database/models/image_item.rs b/src/database/models/image_item.rs index d9562630..1386429c 100644 --- a/src/database/models/image_item.rs +++ b/src/database/models/image_item.rs @@ -11,6 +11,7 @@ const IMAGES_NAMESPACE: &str = "images"; pub struct Image { pub id: ImageId, pub url: String, + pub raw_url: String, pub size: u64, pub created: DateTime, pub owner_id: UserId, @@ -32,14 +33,15 @@ impl Image { sqlx::query!( " INSERT INTO uploaded_images ( - id, url, size, created, owner_id, context, mod_id, version_id, thread_message_id, report_id + id, url, raw_url, size, created, owner_id, context, mod_id, version_id, thread_message_id, report_id ) VALUES ( - $1, $2, $3, $4, $5, $6, $7, $8, $9, $10 + $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11 ); ", self.id as ImageId, self.url, + self.raw_url, self.size as i64, self.created, self.owner_id as UserId, @@ -119,7 +121,7 @@ impl Image { use futures::stream::TryStreamExt; sqlx::query!( " - SELECT id, url, size, created, owner_id, context, mod_id, version_id, thread_message_id, report_id + SELECT id, url, raw_url, size, created, owner_id, context, mod_id, version_id, thread_message_id, report_id FROM uploaded_images WHERE context = $1 AND (mod_id = $2 OR ($2 IS NULL AND mod_id IS NULL)) @@ -142,6 +144,7 @@ impl Image { Image { id, url: row.url, + raw_url: row.raw_url, size: row.size as u64, created: row.created, owner_id: UserId(row.owner_id), @@ -185,7 +188,7 @@ impl Image { |image_ids| async move { let images = sqlx::query!( " - SELECT id, url, size, created, owner_id, context, mod_id, version_id, thread_message_id, report_id + SELECT id, url, raw_url, size, created, owner_id, context, mod_id, version_id, thread_message_id, report_id FROM uploaded_images WHERE id = ANY($1) GROUP BY id; @@ -197,6 +200,7 @@ impl Image { let img = Image { id: ImageId(i.id), url: i.url, + raw_url: i.raw_url, size: i.size as u64, created: i.created, owner_id: UserId(i.owner_id), diff --git a/src/database/models/oauth_client_item.rs b/src/database/models/oauth_client_item.rs index c2abbea7..4c34b356 100644 --- a/src/database/models/oauth_client_item.rs +++ b/src/database/models/oauth_client_item.rs @@ -18,6 +18,7 @@ pub struct OAuthClient { pub id: OAuthClientId, pub name: String, pub icon_url: Option, + pub raw_icon_url: Option, pub max_scopes: Scopes, pub secret_hash: String, pub redirect_uris: Vec, @@ -31,6 +32,7 @@ struct ClientQueryResult { id: i64, name: String, icon_url: Option, + raw_icon_url: Option, max_scopes: i64, secret_hash: String, created: DateTime, @@ -53,6 +55,7 @@ macro_rules! select_clients_with_predicate { clients.id as "id!", clients.name as "name!", clients.icon_url as "icon_url?", + clients.raw_icon_url as "raw_icon_url?", clients.max_scopes as "max_scopes!", clients.secret_hash as "secret_hash!", clients.created as "created!", @@ -133,15 +136,16 @@ impl OAuthClient { sqlx::query!( " INSERT INTO oauth_clients ( - id, name, icon_url, max_scopes, secret_hash, created_by + id, name, icon_url, raw_icon_url, max_scopes, secret_hash, created_by ) VALUES ( - $1, $2, $3, $4, $5, $6 + $1, $2, $3, $4, $5, $6, $7 ) ", self.id.0, self.name, self.icon_url, + self.raw_icon_url, self.max_scopes.to_postgres(), self.secret_hash, self.created_by.0 @@ -161,11 +165,12 @@ impl OAuthClient { sqlx::query!( " UPDATE oauth_clients - SET name = $1, icon_url = $2, max_scopes = $3, url = $4, description = $5 - WHERE (id = $6) + SET name = $1, icon_url = $2, raw_icon_url = $3, max_scopes = $4, url = $5, description = $6 + WHERE (id = $7) ", self.name, self.icon_url, + self.raw_icon_url, self.max_scopes.to_postgres(), self.url, self.description, @@ -243,6 +248,7 @@ impl From for OAuthClient { id: OAuthClientId(r.id), name: r.name, icon_url: r.icon_url, + raw_icon_url: r.raw_icon_url, max_scopes: Scopes::from_postgres(r.max_scopes), secret_hash: r.secret_hash, redirect_uris: redirects, diff --git a/src/database/models/organization_item.rs b/src/database/models/organization_item.rs index 7f9a9073..7651baac 100644 --- a/src/database/models/organization_item.rs +++ b/src/database/models/organization_item.rs @@ -30,6 +30,7 @@ pub struct Organization { /// The display icon for the organization pub icon_url: Option, + pub raw_icon_url: Option, pub color: Option, } @@ -40,8 +41,8 @@ impl Organization { ) -> Result<(), super::DatabaseError> { sqlx::query!( " - INSERT INTO organizations (id, slug, name, team_id, description, icon_url, color) - VALUES ($1, $2, $3, $4, $5, $6, $7) + INSERT INTO organizations (id, slug, name, team_id, description, icon_url, raw_icon_url, color) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8) ", self.id.0, self.slug, @@ -49,6 +50,7 @@ impl Organization { self.team_id as TeamId, self.description, self.icon_url, + self.raw_icon_url, self.color.map(|x| x as i32), ) .execute(&mut **transaction) @@ -125,7 +127,7 @@ impl Organization { let organizations = sqlx::query!( " - SELECT o.id, o.slug, o.name, o.team_id, o.description, o.icon_url, o.color + SELECT o.id, o.slug, o.name, o.team_id, o.description, o.icon_url, o.raw_icon_url, o.color FROM organizations o WHERE o.id = ANY($1) OR LOWER(o.slug) = ANY($2) GROUP BY o.id; @@ -142,6 +144,7 @@ impl Organization { team_id: TeamId(m.team_id), description: m.description, icon_url: m.icon_url, + raw_icon_url: m.raw_icon_url, color: m.color.map(|x| x as u32), }; @@ -168,7 +171,7 @@ impl Organization { { let result = sqlx::query!( " - SELECT o.id, o.slug, o.name, o.team_id, o.description, o.icon_url, o.color + SELECT o.id, o.slug, o.name, o.team_id, o.description, o.icon_url, o.raw_icon_url, o.color FROM organizations o LEFT JOIN mods m ON m.organization_id = o.id WHERE m.id = $1 @@ -187,6 +190,7 @@ impl Organization { team_id: TeamId(result.team_id), description: result.description, icon_url: result.icon_url, + raw_icon_url: result.raw_icon_url, color: result.color.map(|x| x as u32), })) } else { diff --git a/src/database/models/project_item.rs b/src/database/models/project_item.rs index 71484b49..453c22aa 100644 --- a/src/database/models/project_item.rs +++ b/src/database/models/project_item.rs @@ -58,6 +58,7 @@ impl LinkUrl { #[derive(Clone, Debug, Serialize, Deserialize)] pub struct GalleryItem { pub image_url: String, + pub raw_image_url: String, pub featured: bool, pub name: Option, pub description: Option, @@ -71,7 +72,8 @@ impl GalleryItem { project_id: ProjectId, transaction: &mut sqlx::Transaction<'_, sqlx::Postgres>, ) -> Result<(), sqlx::error::Error> { - let (project_ids, image_urls, featureds, names, descriptions, orderings): ( + let (project_ids, image_urls, raw_image_urls, featureds, names, descriptions, orderings): ( + Vec<_>, Vec<_>, Vec<_>, Vec<_>, @@ -84,6 +86,7 @@ impl GalleryItem { ( project_id.0, gi.image_url, + gi.raw_image_url, gi.featured, gi.name, gi.description, @@ -94,12 +97,13 @@ impl GalleryItem { sqlx::query!( " INSERT INTO mods_gallery ( - mod_id, image_url, featured, name, description, ordering + mod_id, image_url, raw_image_url, featured, name, description, ordering ) - SELECT * FROM UNNEST ($1::bigint[], $2::varchar[], $3::bool[], $4::varchar[], $5::varchar[], $6::bigint[]) + SELECT * FROM UNNEST ($1::bigint[], $2::varchar[], $3::varchar[], $4::bool[], $5::varchar[], $6::varchar[], $7::bigint[]) ", &project_ids[..], &image_urls[..], + &raw_image_urls[..], &featureds[..], &names[..] as &[Option], &descriptions[..] as &[Option], @@ -153,6 +157,7 @@ pub struct ProjectBuilder { pub summary: String, pub description: String, pub icon_url: Option, + pub raw_icon_url: Option, pub license_url: Option, pub categories: Vec, pub additional_categories: Vec, @@ -192,6 +197,7 @@ impl ProjectBuilder { downloads: 0, follows: 0, icon_url: self.icon_url, + raw_icon_url: self.raw_icon_url, license_url: self.license_url, license: self.license, slug: self.slug, @@ -253,6 +259,7 @@ pub struct Project { pub downloads: i32, pub follows: i32, pub icon_url: Option, + pub raw_icon_url: Option, pub license_url: Option, pub license: String, pub slug: Option, @@ -273,15 +280,15 @@ impl Project { " INSERT INTO mods ( id, team_id, name, summary, description, - published, downloads, icon_url, status, requested_status, + published, downloads, icon_url, raw_icon_url, status, requested_status, license_url, license, slug, color, monetization_status, organization_id ) VALUES ( $1, $2, $3, $4, $5, $6, - $7, $8, $9, $10, - $11, $12, - LOWER($13), $14, $15, $16 + $7, $8, $9, $10, $11, + $12, $13, + LOWER($14), $15, $16, $17 ) ", self.id as ProjectId, @@ -292,6 +299,7 @@ impl Project { self.published, self.downloads, self.icon_url.as_ref(), + self.raw_icon_url.as_ref(), self.status.as_str(), self.requested_status.map(|x| x.as_str()), self.license_url.as_ref(), @@ -620,7 +628,7 @@ impl Project { let mods_gallery: DashMap> = sqlx::query!( " - SELECT DISTINCT mod_id, mg.image_url, mg.featured, mg.name, mg.description, mg.created, mg.ordering + SELECT DISTINCT mod_id, mg.image_url, mg.raw_image_url, mg.featured, mg.name, mg.description, mg.created, mg.ordering FROM mods_gallery mg INNER JOIN mods m ON mg.mod_id = m.id WHERE m.id = ANY($1) OR m.slug = ANY($2) @@ -633,6 +641,7 @@ impl Project { .or_default() .push(GalleryItem { image_url: m.image_url, + raw_image_url: m.raw_image_url, featured: m.featured.unwrap_or(false), name: m.name, description: m.description, @@ -742,7 +751,7 @@ impl Project { let projects = sqlx::query!( " SELECT m.id id, m.name name, m.summary summary, m.downloads downloads, m.follows follows, - m.icon_url icon_url, m.description description, m.published published, + m.icon_url icon_url, m.raw_icon_url raw_icon_url, m.description description, m.published published, m.updated updated, m.approved approved, m.queued, m.status status, m.requested_status requested_status, m.license_url license_url, m.team_id team_id, m.organization_id organization_id, m.license license, m.slug slug, m.moderation_message moderation_message, m.moderation_message_body moderation_message_body, @@ -788,6 +797,7 @@ impl Project { summary: m.summary.clone(), downloads: m.downloads, icon_url: m.icon_url.clone(), + raw_icon_url: m.raw_icon_url.clone(), published: m.published, updated: m.updated, license_url: m.license_url.clone(), diff --git a/src/database/models/user_item.rs b/src/database/models/user_item.rs index 45efee74..a4f1cac7 100644 --- a/src/database/models/user_item.rs +++ b/src/database/models/user_item.rs @@ -40,6 +40,7 @@ pub struct User { pub email: Option, pub email_verified: bool, pub avatar_url: Option, + pub raw_avatar_url: Option, pub bio: Option, pub created: DateTime, pub role: String, @@ -57,7 +58,7 @@ impl User { " INSERT INTO users ( id, username, email, - avatar_url, bio, created, + avatar_url, raw_avatar_url, bio, created, github_id, discord_id, gitlab_id, google_id, steam_id, microsoft_id, email_verified, password, paypal_id, paypal_country, paypal_email, venmo_handle, stripe_customer_id @@ -66,13 +67,14 @@ impl User { $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, - $14, $15, $16, $17, $18, $19 + $14, $15, $16, $17, $18, $19, $20 ) ", self.id as UserId, &self.username, self.email.as_ref(), self.avatar_url.as_ref(), + self.raw_avatar_url.as_ref(), self.bio.as_ref(), self.created, self.github_id, @@ -165,7 +167,7 @@ impl User { let users = sqlx::query!( " SELECT id, email, - avatar_url, username, bio, + avatar_url, raw_avatar_url, username, bio, created, role, badges, balance, github_id, discord_id, gitlab_id, google_id, steam_id, microsoft_id, @@ -190,6 +192,7 @@ impl User { email: u.email, email_verified: u.email_verified, avatar_url: u.avatar_url, + raw_avatar_url: u.raw_avatar_url, username: u.username.clone(), bio: u.bio, created: u.created, diff --git a/src/models/v2/projects.rs b/src/models/v2/projects.rs index ef9f9acc..d87601cc 100644 --- a/src/models/v2/projects.rs +++ b/src/models/v2/projects.rs @@ -357,6 +357,7 @@ impl From for LegacyVersion { #[derive(Serialize, Deserialize, Clone, Debug)] pub struct LegacyGalleryItem { pub url: String, + pub raw_url: String, pub featured: bool, pub title: Option, pub description: Option, @@ -368,6 +369,7 @@ impl LegacyGalleryItem { fn from(data: crate::models::projects::GalleryItem) -> Self { Self { url: data.url, + raw_url: data.raw_url, featured: data.featured, title: data.name, description: data.description, diff --git a/src/models/v3/projects.rs b/src/models/v3/projects.rs index 8e75d079..6b16bf3a 100644 --- a/src/models/v3/projects.rs +++ b/src/models/v3/projects.rs @@ -215,6 +215,7 @@ impl From for Project { .into_iter() .map(|x| GalleryItem { url: x.image_url, + raw_url: x.raw_image_url, featured: x.featured, name: x.name, description: x.description, @@ -387,6 +388,7 @@ impl Project { #[derive(Serialize, Deserialize, Clone, Debug)] pub struct GalleryItem { pub url: String, + pub raw_url: String, pub featured: bool, pub name: Option, pub description: Option, diff --git a/src/routes/internal/flows.rs b/src/routes/internal/flows.rs index edf8f5b6..c0883e64 100644 --- a/src/routes/internal/flows.rs +++ b/src/routes/internal/flows.rs @@ -14,7 +14,8 @@ use crate::routes::internal::session::issue_session; use crate::routes::ApiError; use crate::util::captcha::check_turnstile_captcha; use crate::util::env::parse_strings_from_var; -use crate::util::ext::{get_image_content_type, get_image_ext}; +use crate::util::ext::get_image_ext; +use crate::util::img::upload_image_optimized; use crate::util::validate::{validation_errors_to_string, RE_URL_SAFE}; use actix_web::web::{scope, Data, Payload, Query, ServiceConfig}; use actix_web::{delete, get, patch, post, web, HttpRequest, HttpResponse}; @@ -112,9 +113,7 @@ impl TempUser { } } - let avatar_url = if let Some(avatar_url) = self.avatar_url { - let cdn_url = dotenvy::var("CDN_URL")?; - + let (avatar_url, raw_avatar_url) = if let Some(avatar_url) = self.avatar_url { let res = reqwest::get(&avatar_url).await?; let headers = res.headers().clone(); @@ -122,36 +121,34 @@ impl TempUser { .get(reqwest::header::CONTENT_TYPE) .and_then(|ct| ct.to_str().ok()) { - get_image_ext(content_type).map(|ext| (ext, content_type)) - } else if let Some(ext) = avatar_url.rsplit('.').next() { - get_image_content_type(ext).map(|content_type| (ext, content_type)) + get_image_ext(content_type) } else { - None + avatar_url.rsplit('.').next() }; - if let Some((ext, content_type)) = img_data { + if let Some(ext) = img_data { let bytes = res.bytes().await?; - let hash = sha1::Sha1::from(&bytes).hexdigest(); - - let upload_data = file_host - .upload_file( - content_type, - &format!( - "user/{}/{}.{}", - crate::models::users::UserId::from(user_id), - hash, - ext - ), - bytes, - ) - .await?; - Some(format!("{}/{}", cdn_url, upload_data.file_name)) + let upload_result = upload_image_optimized( + &format!("user/{}", crate::models::users::UserId::from(user_id)), + bytes, + ext, + Some(96), + Some(1.0), + &**file_host, + ) + .await; + + if let Ok(upload_result) = upload_result { + (Some(upload_result.url), Some(upload_result.raw_url)) + } else { + (None, None) + } } else { - None + (None, None) } } else { - None + (None, None) }; if let Some(username) = username { @@ -223,6 +220,7 @@ impl TempUser { email: self.email, email_verified: true, avatar_url, + raw_avatar_url, bio: self.bio, created: Utc::now(), role: Role::Developer.to_string(), @@ -1518,6 +1516,7 @@ pub async fn create_account_with_password( email: Some(new_account.email.clone()), email_verified: false, avatar_url: None, + raw_avatar_url: None, bio: None, created: Utc::now(), role: Role::Developer.to_string(), diff --git a/src/routes/v3/collections.rs b/src/routes/v3/collections.rs index 473e47df..739cabab 100644 --- a/src/routes/v3/collections.rs +++ b/src/routes/v3/collections.rs @@ -10,6 +10,7 @@ use crate::models::pats::Scopes; use crate::queue::session::AuthQueue; use crate::routes::v3::project_creation::CreateError; use crate::routes::ApiError; +use crate::util::img::delete_old_images; use crate::util::routes::read_from_payload; use crate::util::validate::validation_errors_to_string; use crate::{database, models}; @@ -371,78 +372,69 @@ pub async fn collection_icon_edit( mut payload: web::Payload, session_queue: web::Data, ) -> Result { - if let Some(content_type) = crate::util::ext::get_image_content_type(&ext.ext) { - let cdn_url = dotenvy::var("CDN_URL")?; - let user = get_user_from_headers( - &req, - &**pool, - &redis, - &session_queue, - Some(&[Scopes::COLLECTION_WRITE]), - ) - .await? - .1; - - let string = info.into_inner().0; - let id = database::models::CollectionId(parse_base62(&string)? as i64); - let collection_item = database::models::Collection::get(id, &**pool, &redis) - .await? - .ok_or_else(|| { - ApiError::InvalidInput("The specified collection does not exist!".to_string()) - })?; - - if !can_modify_collection(&collection_item, &user) { - return Ok(HttpResponse::Unauthorized().body("")); - } - - if let Some(icon) = collection_item.icon_url { - let name = icon.split(&format!("{cdn_url}/")).nth(1); + let user = get_user_from_headers( + &req, + &**pool, + &redis, + &session_queue, + Some(&[Scopes::COLLECTION_WRITE]), + ) + .await? + .1; - if let Some(icon_path) = name { - file_host.delete_file_version("", icon_path).await?; - } - } + let string = info.into_inner().0; + let id = database::models::CollectionId(parse_base62(&string)? as i64); + let collection_item = database::models::Collection::get(id, &**pool, &redis) + .await? + .ok_or_else(|| { + ApiError::InvalidInput("The specified collection does not exist!".to_string()) + })?; - let bytes = - read_from_payload(&mut payload, 262144, "Icons must be smaller than 256KiB").await?; + if !can_modify_collection(&collection_item, &user) { + return Ok(HttpResponse::Unauthorized().body("")); + } - let color = crate::util::img::get_color_from_img(&bytes)?; + delete_old_images( + collection_item.icon_url, + collection_item.raw_icon_url, + &***file_host, + ) + .await?; - let hash = sha1::Sha1::from(&bytes).hexdigest(); - let collection_id: CollectionId = collection_item.id.into(); - let upload_data = file_host - .upload_file( - content_type, - &format!("data/{}/{}.{}", collection_id, hash, ext.ext), - bytes.freeze(), - ) - .await?; + let bytes = + read_from_payload(&mut payload, 262144, "Icons must be smaller than 256KiB").await?; + + let collection_id: CollectionId = collection_item.id.into(); + let upload_result = crate::util::img::upload_image_optimized( + &format!("data/{}", collection_id), + bytes.freeze(), + &ext.ext, + Some(96), + Some(1.0), + &***file_host, + ) + .await?; - let mut transaction = pool.begin().await?; + let mut transaction = pool.begin().await?; - sqlx::query!( - " - UPDATE collections - SET icon_url = $1, color = $2 - WHERE (id = $3) - ", - format!("{}/{}", cdn_url, upload_data.file_name), - color.map(|x| x as i32), - collection_item.id as database::models::ids::CollectionId, - ) - .execute(&mut *transaction) - .await?; + sqlx::query!( + " + UPDATE collections + SET icon_url = $1, raw_icon_url = $2, color = $3 + WHERE (id = $4) + ", + upload_result.url, + upload_result.raw_url, + upload_result.color.map(|x| x as i32), + collection_item.id as database::models::ids::CollectionId, + ) + .execute(&mut *transaction) + .await?; - transaction.commit().await?; - database::models::Collection::clear_cache(collection_item.id, &redis).await?; + transaction.commit().await?; + database::models::Collection::clear_cache(collection_item.id, &redis).await?; - Ok(HttpResponse::NoContent().body("")) - } else { - Err(ApiError::InvalidInput(format!( - "Invalid format for collection icon: {}", - ext.ext - ))) - } + Ok(HttpResponse::NoContent().body("")) } pub async fn delete_collection_icon( @@ -474,21 +466,18 @@ pub async fn delete_collection_icon( return Ok(HttpResponse::Unauthorized().body("")); } - let cdn_url = dotenvy::var("CDN_URL")?; - if let Some(icon) = collection_item.icon_url { - let name = icon.split(&format!("{cdn_url}/")).nth(1); - - if let Some(icon_path) = name { - file_host.delete_file_version("", icon_path).await?; - } - } - + delete_old_images( + collection_item.icon_url, + collection_item.raw_icon_url, + &***file_host, + ) + .await?; let mut transaction = pool.begin().await?; sqlx::query!( " UPDATE collections - SET icon_url = NULL, color = NULL + SET icon_url = NULL, raw_icon_url = NULL, color = NULL WHERE (id = $1) ", collection_item.id as database::models::ids::CollectionId, diff --git a/src/routes/v3/images.rs b/src/routes/v3/images.rs index 86b202ef..0ec48aca 100644 --- a/src/routes/v3/images.rs +++ b/src/routes/v3/images.rs @@ -1,5 +1,6 @@ use std::sync::Arc; +use super::threads::is_authorized_thread; use crate::auth::checks::{is_team_member_project, is_team_member_version}; use crate::auth::get_user_from_headers; use crate::database; @@ -11,13 +12,12 @@ use crate::models::images::{Image, ImageContext}; use crate::models::reports::ReportId; use crate::queue::session::AuthQueue; use crate::routes::ApiError; +use crate::util::img::upload_image_optimized; use crate::util::routes::read_from_payload; use actix_web::{web, HttpRequest, HttpResponse}; use serde::{Deserialize, Serialize}; use sqlx::PgPool; -use super::threads::is_authorized_thread; - pub fn config(cfg: &mut web::ServiceConfig) { cfg.route("image", web::post().to(images_add)); } @@ -46,198 +46,182 @@ pub async fn images_add( redis: web::Data, session_queue: web::Data, ) -> Result { - if let Some(content_type) = crate::util::ext::get_image_content_type(&data.ext) { - let mut context = ImageContext::from_str(&data.context, None); - - let scopes = vec![context.relevant_scope()]; - - let cdn_url = dotenvy::var("CDN_URL")?; - let user = get_user_from_headers(&req, &**pool, &redis, &session_queue, Some(&scopes)) - .await? - .1; - - // Attempt to associated a supplied id with the context - // If the context cannot be found, or the user is not authorized to upload images for the context, return an error - match &mut context { - ImageContext::Project { project_id } => { - if let Some(id) = data.project_id { - let project = project_item::Project::get(&id, &**pool, &redis).await?; - if let Some(project) = project { - if is_team_member_project(&project.inner, &Some(user.clone()), &pool) - .await? - { - *project_id = Some(project.inner.id.into()); - } else { - return Err(ApiError::CustomAuthentication( - "You are not authorized to upload images for this project" - .to_string(), - )); - } + let mut context = ImageContext::from_str(&data.context, None); + + let scopes = vec![context.relevant_scope()]; + + let user = get_user_from_headers(&req, &**pool, &redis, &session_queue, Some(&scopes)) + .await? + .1; + + // Attempt to associated a supplied id with the context + // If the context cannot be found, or the user is not authorized to upload images for the context, return an error + match &mut context { + ImageContext::Project { project_id } => { + if let Some(id) = data.project_id { + let project = project_item::Project::get(&id, &**pool, &redis).await?; + if let Some(project) = project { + if is_team_member_project(&project.inner, &Some(user.clone()), &pool).await? { + *project_id = Some(project.inner.id.into()); } else { - return Err(ApiError::InvalidInput( - "The project could not be found.".to_string(), + return Err(ApiError::CustomAuthentication( + "You are not authorized to upload images for this project".to_string(), )); } + } else { + return Err(ApiError::InvalidInput( + "The project could not be found.".to_string(), + )); } } - ImageContext::Version { version_id } => { - if let Some(id) = data.version_id { - let version = version_item::Version::get(id.into(), &**pool, &redis).await?; - if let Some(version) = version { - if is_team_member_version( - &version.inner, - &Some(user.clone()), - &pool, - &redis, - ) + } + ImageContext::Version { version_id } => { + if let Some(id) = data.version_id { + let version = version_item::Version::get(id.into(), &**pool, &redis).await?; + if let Some(version) = version { + if is_team_member_version(&version.inner, &Some(user.clone()), &pool, &redis) .await? - { - *version_id = Some(version.inner.id.into()); - } else { - return Err(ApiError::CustomAuthentication( - "You are not authorized to upload images for this version" - .to_string(), - )); - } + { + *version_id = Some(version.inner.id.into()); } else { - return Err(ApiError::InvalidInput( - "The version could not be found.".to_string(), + return Err(ApiError::CustomAuthentication( + "You are not authorized to upload images for this version".to_string(), )); } + } else { + return Err(ApiError::InvalidInput( + "The version could not be found.".to_string(), + )); } } - ImageContext::ThreadMessage { thread_message_id } => { - if let Some(id) = data.thread_message_id { - let thread_message = thread_item::ThreadMessage::get(id.into(), &**pool) - .await? - .ok_or_else(|| { - ApiError::InvalidInput( - "The thread message could not found.".to_string(), - ) - })?; - let thread = thread_item::Thread::get(thread_message.thread_id, &**pool) - .await? - .ok_or_else(|| { - ApiError::InvalidInput( - "The thread associated with the thread message could not be found" - .to_string(), - ) - })?; - if is_authorized_thread(&thread, &user, &pool).await? { - *thread_message_id = Some(thread_message.id.into()); - } else { - return Err(ApiError::CustomAuthentication( - "You are not authorized to upload images for this thread message" + } + ImageContext::ThreadMessage { thread_message_id } => { + if let Some(id) = data.thread_message_id { + let thread_message = thread_item::ThreadMessage::get(id.into(), &**pool) + .await? + .ok_or_else(|| { + ApiError::InvalidInput("The thread message could not found.".to_string()) + })?; + let thread = thread_item::Thread::get(thread_message.thread_id, &**pool) + .await? + .ok_or_else(|| { + ApiError::InvalidInput( + "The thread associated with the thread message could not be found" .to_string(), - )); - } + ) + })?; + if is_authorized_thread(&thread, &user, &pool).await? { + *thread_message_id = Some(thread_message.id.into()); + } else { + return Err(ApiError::CustomAuthentication( + "You are not authorized to upload images for this thread message" + .to_string(), + )); } } - ImageContext::Report { report_id } => { - if let Some(id) = data.report_id { - let report = report_item::Report::get(id.into(), &**pool) - .await? - .ok_or_else(|| { - ApiError::InvalidInput("The report could not be found.".to_string()) - })?; - let thread = thread_item::Thread::get(report.thread_id, &**pool) - .await? - .ok_or_else(|| { - ApiError::InvalidInput( - "The thread associated with the report could not be found." - .to_string(), - ) - })?; - if is_authorized_thread(&thread, &user, &pool).await? { - *report_id = Some(report.id.into()); - } else { - return Err(ApiError::CustomAuthentication( - "You are not authorized to upload images for this report".to_string(), - )); - } + } + ImageContext::Report { report_id } => { + if let Some(id) = data.report_id { + let report = report_item::Report::get(id.into(), &**pool) + .await? + .ok_or_else(|| { + ApiError::InvalidInput("The report could not be found.".to_string()) + })?; + let thread = thread_item::Thread::get(report.thread_id, &**pool) + .await? + .ok_or_else(|| { + ApiError::InvalidInput( + "The thread associated with the report could not be found.".to_string(), + ) + })?; + if is_authorized_thread(&thread, &user, &pool).await? { + *report_id = Some(report.id.into()); + } else { + return Err(ApiError::CustomAuthentication( + "You are not authorized to upload images for this report".to_string(), + )); } } - ImageContext::Unknown => { - return Err(ApiError::InvalidInput( - "Context must be one of: project, version, thread_message, report".to_string(), - )); - } } - - // Upload the image to the file host - let bytes = - read_from_payload(&mut payload, 1_048_576, "Icons must be smaller than 1MiB").await?; - - let hash = sha1::Sha1::from(&bytes).hexdigest(); - let upload_data = file_host - .upload_file( - content_type, - &format!("data/cached_images/{}.{}", hash, data.ext), - bytes.freeze(), - ) - .await?; - - let mut transaction = pool.begin().await?; - - let db_image: database::models::Image = database::models::Image { - id: database::models::generate_image_id(&mut transaction).await?, - url: format!("{}/{}", cdn_url, upload_data.file_name), - size: upload_data.content_length as u64, - created: chrono::Utc::now(), - owner_id: database::models::UserId::from(user.id), - context: context.context_as_str().to_string(), - project_id: if let ImageContext::Project { - project_id: Some(id), - } = context - { - Some(database::models::ProjectId::from(id)) - } else { - None - }, - version_id: if let ImageContext::Version { - version_id: Some(id), - } = context - { - Some(database::models::VersionId::from(id)) - } else { - None - }, - thread_message_id: if let ImageContext::ThreadMessage { - thread_message_id: Some(id), - } = context - { - Some(database::models::ThreadMessageId::from(id)) - } else { - None - }, - report_id: if let ImageContext::Report { - report_id: Some(id), - } = context - { - Some(database::models::ReportId::from(id)) - } else { - None - }, - }; - - // Insert - db_image.insert(&mut transaction).await?; - - let image = Image { - id: db_image.id.into(), - url: db_image.url, - size: db_image.size, - created: db_image.created, - owner_id: db_image.owner_id.into(), - context, - }; - - transaction.commit().await?; - - Ok(HttpResponse::Ok().json(image)) - } else { - Err(ApiError::InvalidInput( - "The specified file is not an image!".to_string(), - )) + ImageContext::Unknown => { + return Err(ApiError::InvalidInput( + "Context must be one of: project, version, thread_message, report".to_string(), + )); + } } + + // Upload the image to the file host + let bytes = + read_from_payload(&mut payload, 1_048_576, "Icons must be smaller than 1MiB").await?; + + let content_length = bytes.len(); + let upload_result = upload_image_optimized( + "data/cached_images", + bytes.freeze(), + &data.ext, + None, + None, + &***file_host, + ) + .await?; + + let mut transaction = pool.begin().await?; + + let db_image: database::models::Image = database::models::Image { + id: database::models::generate_image_id(&mut transaction).await?, + url: upload_result.url, + raw_url: upload_result.raw_url, + size: content_length as u64, + created: chrono::Utc::now(), + owner_id: database::models::UserId::from(user.id), + context: context.context_as_str().to_string(), + project_id: if let ImageContext::Project { + project_id: Some(id), + } = context + { + Some(crate::database::models::ProjectId::from(id)) + } else { + None + }, + version_id: if let ImageContext::Version { + version_id: Some(id), + } = context + { + Some(database::models::VersionId::from(id)) + } else { + None + }, + thread_message_id: if let ImageContext::ThreadMessage { + thread_message_id: Some(id), + } = context + { + Some(database::models::ThreadMessageId::from(id)) + } else { + None + }, + report_id: if let ImageContext::Report { + report_id: Some(id), + } = context + { + Some(database::models::ReportId::from(id)) + } else { + None + }, + }; + + // Insert + db_image.insert(&mut transaction).await?; + + let image = Image { + id: db_image.id.into(), + url: db_image.url, + size: db_image.size, + created: db_image.created, + owner_id: db_image.owner_id.into(), + context, + }; + + transaction.commit().await?; + + Ok(HttpResponse::Ok().json(image)) } diff --git a/src/routes/v3/oauth_clients.rs b/src/routes/v3/oauth_clients.rs index 060de288..a7ea42bd 100644 --- a/src/routes/v3/oauth_clients.rs +++ b/src/routes/v3/oauth_clients.rs @@ -42,6 +42,7 @@ use crate::{ use crate::database::models::oauth_client_item::OAuthClient as DBOAuthClient; use crate::models::ids::OAuthClientId as ApiOAuthClientId; +use crate::util::img::{delete_old_images, upload_image_optimized}; pub fn config(cfg: &mut web::ServiceConfig) { cfg.service( @@ -135,12 +136,6 @@ pub struct NewOAuthApp { )] pub name: String, - #[validate( - custom(function = "crate::util::validate::validate_url"), - length(max = 255) - )] - pub icon_url: Option, - #[validate(custom(function = "crate::util::validate::validate_no_restricted_scopes"))] pub max_scopes: Scopes, @@ -190,7 +185,8 @@ pub async fn oauth_client_create<'a>( let client = OAuthClient { id: client_id, - icon_url: new_oauth_app.icon_url.clone(), + icon_url: None, + raw_icon_url: None, max_scopes: new_oauth_app.max_scopes, name: new_oauth_app.name.clone(), redirect_uris, @@ -349,63 +345,56 @@ pub async fn oauth_client_icon_edit( mut payload: web::Payload, session_queue: web::Data, ) -> Result { - if let Some(content_type) = crate::util::ext::get_image_content_type(&ext.ext) { - let cdn_url = dotenvy::var("CDN_URL")?; - let user = get_user_from_headers( - &req, - &**pool, - &redis, - &session_queue, - Some(&[Scopes::SESSION_ACCESS]), - ) - .await? - .1; - - let client = OAuthClient::get((*client_id).into(), &**pool) - .await? - .ok_or_else(|| { - ApiError::InvalidInput("The specified client does not exist!".to_string()) - })?; + let user = get_user_from_headers( + &req, + &**pool, + &redis, + &session_queue, + Some(&[Scopes::SESSION_ACCESS]), + ) + .await? + .1; - client.validate_authorized(Some(&user))?; + let client = OAuthClient::get((*client_id).into(), &**pool) + .await? + .ok_or_else(|| { + ApiError::InvalidInput("The specified client does not exist!".to_string()) + })?; - if let Some(ref icon) = client.icon_url { - let name = icon.split(&format!("{cdn_url}/")).nth(1); + client.validate_authorized(Some(&user))?; - if let Some(icon_path) = name { - file_host.delete_file_version("", icon_path).await?; - } - } + delete_old_images( + client.icon_url.clone(), + client.raw_icon_url.clone(), + &***file_host, + ) + .await?; - let bytes = - read_from_payload(&mut payload, 262144, "Icons must be smaller than 256KiB").await?; - let hash = sha1::Sha1::from(&bytes).hexdigest(); - let upload_data = file_host - .upload_file( - content_type, - &format!("data/{}/{}.{}", client_id, hash, ext.ext), - bytes.freeze(), - ) - .await?; + let bytes = + read_from_payload(&mut payload, 262144, "Icons must be smaller than 256KiB").await?; + let upload_result = upload_image_optimized( + &format!("data/{}", client_id), + bytes.freeze(), + &ext.ext, + Some(96), + Some(1.0), + &***file_host, + ) + .await?; - let mut transaction = pool.begin().await?; + let mut transaction = pool.begin().await?; - let mut editable_client = client.clone(); - editable_client.icon_url = Some(format!("{}/{}", cdn_url, upload_data.file_name)); + let mut editable_client = client.clone(); + editable_client.icon_url = Some(upload_result.url); + editable_client.raw_icon_url = Some(upload_result.raw_url); - editable_client - .update_editable_fields(&mut *transaction) - .await?; + editable_client + .update_editable_fields(&mut *transaction) + .await?; - transaction.commit().await?; + transaction.commit().await?; - Ok(HttpResponse::NoContent().body("")) - } else { - Err(ApiError::InvalidInput(format!( - "Invalid format for project icon: {}", - ext.ext - ))) - } + Ok(HttpResponse::NoContent().body("")) } #[delete("app/{id}/icon")] @@ -417,7 +406,6 @@ pub async fn oauth_client_icon_delete( file_host: web::Data>, session_queue: web::Data, ) -> Result { - let cdn_url = dotenvy::var("CDN_URL")?; let user = get_user_from_headers( &req, &**pool, @@ -435,18 +423,18 @@ pub async fn oauth_client_icon_delete( })?; client.validate_authorized(Some(&user))?; - if let Some(ref icon) = client.icon_url { - let name = icon.split(&format!("{cdn_url}/")).nth(1); - - if let Some(icon_path) = name { - file_host.delete_file_version("", icon_path).await?; - } - } + delete_old_images( + client.icon_url.clone(), + client.raw_icon_url.clone(), + &***file_host, + ) + .await?; let mut transaction = pool.begin().await?; let mut editable_client = client.clone(); editable_client.icon_url = None; + editable_client.raw_icon_url = None; editable_client .update_editable_fields(&mut *transaction) diff --git a/src/routes/v3/organizations.rs b/src/routes/v3/organizations.rs index cb361963..f7b44311 100644 --- a/src/routes/v3/organizations.rs +++ b/src/routes/v3/organizations.rs @@ -14,6 +14,7 @@ use crate::models::pats::Scopes; use crate::models::teams::{OrganizationPermissions, ProjectPermissions}; use crate::queue::session::AuthQueue; use crate::routes::v3::project_creation::CreateError; +use crate::util::img::delete_old_images; use crate::util::routes::read_from_payload; use crate::util::validate::validation_errors_to_string; use crate::{database, models}; @@ -164,6 +165,7 @@ pub async fn organization_create( description: new_organization.description.clone(), team_id, icon_url: None, + raw_icon_url: None, color: None, }; organization.clone().insert(&mut transaction).await?; @@ -926,98 +928,89 @@ pub async fn organization_icon_edit( mut payload: web::Payload, session_queue: web::Data, ) -> Result { - if let Some(content_type) = crate::util::ext::get_image_content_type(&ext.ext) { - let cdn_url = dotenvy::var("CDN_URL")?; - let user = get_user_from_headers( - &req, - &**pool, - &redis, - &session_queue, - Some(&[Scopes::ORGANIZATION_WRITE]), - ) - .await? - .1; - let string = info.into_inner().0; - - let organization_item = database::models::Organization::get(&string, &**pool, &redis) - .await? - .ok_or_else(|| { - ApiError::InvalidInput("The specified organization does not exist!".to_string()) - })?; - - if !user.role.is_mod() { - let team_member = database::models::TeamMember::get_from_user_id( - organization_item.team_id, - user.id.into(), - &**pool, - ) - .await - .map_err(ApiError::Database)?; + let user = get_user_from_headers( + &req, + &**pool, + &redis, + &session_queue, + Some(&[Scopes::ORGANIZATION_WRITE]), + ) + .await? + .1; + let string = info.into_inner().0; - let permissions = - OrganizationPermissions::get_permissions_by_role(&user.role, &team_member) - .unwrap_or_default(); + let organization_item = database::models::Organization::get(&string, &**pool, &redis) + .await? + .ok_or_else(|| { + ApiError::InvalidInput("The specified organization does not exist!".to_string()) + })?; - if !permissions.contains(OrganizationPermissions::EDIT_DETAILS) { - return Err(ApiError::CustomAuthentication( - "You don't have permission to edit this organization's icon.".to_string(), - )); - } - } + if !user.role.is_mod() { + let team_member = database::models::TeamMember::get_from_user_id( + organization_item.team_id, + user.id.into(), + &**pool, + ) + .await + .map_err(ApiError::Database)?; - if let Some(icon) = organization_item.icon_url { - let name = icon.split(&format!("{cdn_url}/")).nth(1); + let permissions = + OrganizationPermissions::get_permissions_by_role(&user.role, &team_member) + .unwrap_or_default(); - if let Some(icon_path) = name { - file_host.delete_file_version("", icon_path).await?; - } + if !permissions.contains(OrganizationPermissions::EDIT_DETAILS) { + return Err(ApiError::CustomAuthentication( + "You don't have permission to edit this organization's icon.".to_string(), + )); } + } - let bytes = - read_from_payload(&mut payload, 262144, "Icons must be smaller than 256KiB").await?; - - let color = crate::util::img::get_color_from_img(&bytes)?; + delete_old_images( + organization_item.icon_url, + organization_item.raw_icon_url, + &***file_host, + ) + .await?; - let hash = sha1::Sha1::from(&bytes).hexdigest(); - let organization_id: OrganizationId = organization_item.id.into(); - let upload_data = file_host - .upload_file( - content_type, - &format!("data/{}/{}.{}", organization_id, hash, ext.ext), - bytes.freeze(), - ) - .await?; + let bytes = + read_from_payload(&mut payload, 262144, "Icons must be smaller than 256KiB").await?; + + let organization_id: OrganizationId = organization_item.id.into(); + let upload_result = crate::util::img::upload_image_optimized( + &format!("data/{}", organization_id), + bytes.freeze(), + &ext.ext, + Some(96), + Some(1.0), + &***file_host, + ) + .await?; - let mut transaction = pool.begin().await?; + let mut transaction = pool.begin().await?; - sqlx::query!( - " - UPDATE organizations - SET icon_url = $1, color = $2 - WHERE (id = $3) - ", - format!("{}/{}", cdn_url, upload_data.file_name), - color.map(|x| x as i32), - organization_item.id as database::models::ids::OrganizationId, - ) - .execute(&mut *transaction) - .await?; + sqlx::query!( + " + UPDATE organizations + SET icon_url = $1, raw_icon_url = $2, color = $3 + WHERE (id = $4) + ", + upload_result.url, + upload_result.raw_url, + upload_result.color.map(|x| x as i32), + organization_item.id as database::models::ids::OrganizationId, + ) + .execute(&mut *transaction) + .await?; - transaction.commit().await?; - database::models::Organization::clear_cache( - organization_item.id, - Some(organization_item.slug), - &redis, - ) - .await?; + transaction.commit().await?; + database::models::Organization::clear_cache( + organization_item.id, + Some(organization_item.slug), + &redis, + ) + .await?; - Ok(HttpResponse::NoContent().body("")) - } else { - Err(ApiError::InvalidInput(format!( - "Invalid format for project icon: {}", - ext.ext - ))) - } + Ok(HttpResponse::NoContent().body("")) } pub async fn delete_organization_icon( @@ -1065,21 +1058,19 @@ pub async fn delete_organization_icon( } } - let cdn_url = dotenvy::var("CDN_URL")?; - if let Some(icon) = organization_item.icon_url { - let name = icon.split(&format!("{cdn_url}/")).nth(1); - - if let Some(icon_path) = name { - file_host.delete_file_version("", icon_path).await?; - } - } + delete_old_images( + organization_item.icon_url, + organization_item.raw_icon_url, + &***file_host, + ) + .await?; let mut transaction = pool.begin().await?; sqlx::query!( " UPDATE organizations - SET icon_url = NULL, color = NULL + SET icon_url = NULL, raw_icon_url = NULL, color = NULL WHERE (id = $1) ", organization_item.id as database::models::ids::OrganizationId, diff --git a/src/routes/v3/project_creation.rs b/src/routes/v3/project_creation.rs index da984f89..fc19c538 100644 --- a/src/routes/v3/project_creation.rs +++ b/src/routes/v3/project_creation.rs @@ -6,6 +6,7 @@ use crate::database::models::{self, image_item, User}; use crate::database::redis::RedisPool; use crate::file_hosting::{FileHost, FileHostingError}; use crate::models::error::ApiError; +use crate::models::ids::base62_impl::to_base62; use crate::models::ids::{ImageId, OrganizationId}; use crate::models::images::{Image, ImageContext}; use crate::models::pats::Scopes; @@ -17,6 +18,7 @@ use crate::models::threads::ThreadType; use crate::models::users::UserId; use crate::queue::session::AuthQueue; use crate::search::indexing::IndexingError; +use crate::util::img::upload_image_optimized; use crate::util::routes::read_from_field; use crate::util::validate::validation_errors_to_string; use actix_multipart::{Field, Multipart}; @@ -481,7 +483,6 @@ async fn project_create_inner( file_extension, file_host, field, - &cdn_url, ) .await?, ); @@ -496,33 +497,40 @@ async fn project_create_inner( if let Some(item) = gallery_items.iter().find(|x| x.item == name) { let data = read_from_field( &mut field, - 5 * (1 << 20), - "Gallery image exceeds the maximum of 5MiB.", + 2 * (1 << 20), + "Gallery image exceeds the maximum of 2MiB.", ) .await?; - let hash = sha1::Sha1::from(&data).hexdigest(); + let (_, file_extension) = super::version_creation::get_name_ext(&content_disposition)?; - let content_type = crate::util::ext::get_image_content_type(file_extension) - .ok_or_else(|| { - CreateError::InvalidIconFormat(file_extension.to_string()) - })?; - let url = format!("data/{project_id}/images/{hash}.{file_extension}"); - let upload_data = file_host - .upload_file(content_type, &url, data.freeze()) - .await?; + + let url = format!("data/{project_id}/images/"); + let upload_result = upload_image_optimized( + &url, + data.freeze(), + file_extension, + Some(350), + Some(1.0), + file_host, + ) + .await + .map_err(|e| CreateError::InvalidIconFormat(e.to_string()))?; + uploaded_files.push(UploadedFile { - file_id: upload_data.file_id, - file_name: upload_data.file_name, + file_id: upload_result.raw_url_path.clone(), + file_name: upload_result.raw_url_path, }); gallery_urls.push(crate::models::projects::GalleryItem { - url: format!("{cdn_url}/{url}"), + url: upload_result.url, + raw_url: upload_result.raw_url, featured: item.featured, name: item.name.clone(), description: item.description.clone(), created: Utc::now(), ordering: item.ordering, }); + return Ok(()); } } @@ -715,6 +723,7 @@ async fn project_create_inner( summary: project_create_data.summary, description: project_create_data.description, icon_url: icon_data.clone().map(|x| x.0), + raw_icon_url: icon_data.clone().map(|x| x.1), license_url: project_create_data.license_url, categories, @@ -729,6 +738,7 @@ async fn project_create_inner( .iter() .map(|x| models::project_item::GalleryItem { image_url: x.url.clone(), + raw_image_url: x.raw_url.clone(), featured: x.featured, name: x.name.clone(), description: x.description.clone(), @@ -736,7 +746,7 @@ async fn project_create_inner( ordering: x.ordering, }) .collect(), - color: icon_data.and_then(|x| x.1), + color: icon_data.and_then(|x| x.2), monetization_status: MonetizationStatus::Monetized, }; let project_builder = project_builder_actual.clone(); @@ -943,29 +953,32 @@ async fn process_icon_upload( file_extension: &str, file_host: &dyn FileHost, mut field: Field, - cdn_url: &str, -) -> Result<(String, Option), CreateError> { - if let Some(content_type) = crate::util::ext::get_image_content_type(file_extension) { - let data = read_from_field(&mut field, 262144, "Icons must be smaller than 256KiB").await?; - - let color = crate::util::img::get_color_from_img(&data)?; - - let hash = sha1::Sha1::from(&data).hexdigest(); - let upload_data = file_host - .upload_file( - content_type, - &format!("data/{id}/{hash}.{file_extension}"), - data.freeze(), - ) - .await?; - - uploaded_files.push(UploadedFile { - file_id: upload_data.file_id, - file_name: upload_data.file_name.clone(), - }); - - Ok((format!("{}/{}", cdn_url, upload_data.file_name), color)) - } else { - Err(CreateError::InvalidIconFormat(file_extension.to_string())) - } +) -> Result<(String, String, Option), CreateError> { + let data = read_from_field(&mut field, 262144, "Icons must be smaller than 256KiB").await?; + let upload_result = crate::util::img::upload_image_optimized( + &format!("data/{}", to_base62(id)), + data.freeze(), + file_extension, + Some(96), + Some(1.0), + file_host, + ) + .await + .map_err(|e| CreateError::InvalidIconFormat(e.to_string()))?; + + uploaded_files.push(UploadedFile { + file_id: upload_result.raw_url_path.clone(), + file_name: upload_result.raw_url_path, + }); + + uploaded_files.push(UploadedFile { + file_id: upload_result.url_path.clone(), + file_name: upload_result.url_path, + }); + + Ok(( + upload_result.url, + upload_result.raw_url, + upload_result.color, + )) } diff --git a/src/routes/v3/projects.rs b/src/routes/v3/projects.rs index 5bbef14b..33145724 100644 --- a/src/routes/v3/projects.rs +++ b/src/routes/v3/projects.rs @@ -26,6 +26,7 @@ use crate::routes::ApiError; use crate::search::indexing::remove_documents; use crate::search::{search_for_project, SearchConfig, SearchError}; use crate::util::img; +use crate::util::img::{delete_old_images, upload_image_optimized}; use crate::util::routes::read_from_payload; use crate::util::validate::validation_errors_to_string; use actix_web::{web, HttpRequest, HttpResponse}; @@ -1317,109 +1318,95 @@ pub async fn project_icon_edit( mut payload: web::Payload, session_queue: web::Data, ) -> Result { - if let Some(content_type) = crate::util::ext::get_image_content_type(&ext.ext) { - let cdn_url = dotenvy::var("CDN_URL")?; - let user = get_user_from_headers( - &req, - &**pool, - &redis, - &session_queue, - Some(&[Scopes::PROJECT_WRITE]), - ) - .await? - .1; - let string = info.into_inner().0; - - let project_item = db_models::Project::get(&string, &**pool, &redis) - .await? - .ok_or_else(|| { - ApiError::InvalidInput("The specified project does not exist!".to_string()) - })?; - - if !user.role.is_mod() { - let (team_member, organization_team_member) = - db_models::TeamMember::get_for_project_permissions( - &project_item.inner, - user.id.into(), - &**pool, - ) - .await?; + let user = get_user_from_headers( + &req, + &**pool, + &redis, + &session_queue, + Some(&[Scopes::PROJECT_WRITE]), + ) + .await? + .1; + let string = info.into_inner().0; - // Hide the project - if team_member.is_none() && organization_team_member.is_none() { - return Err(ApiError::CustomAuthentication( - "The specified project does not exist!".to_string(), - )); - } + let project_item = db_models::Project::get(&string, &**pool, &redis) + .await? + .ok_or_else(|| { + ApiError::InvalidInput("The specified project does not exist!".to_string()) + })?; - let permissions = ProjectPermissions::get_permissions_by_role( - &user.role, - &team_member, - &organization_team_member, + if !user.role.is_mod() { + let (team_member, organization_team_member) = + db_models::TeamMember::get_for_project_permissions( + &project_item.inner, + user.id.into(), + &**pool, ) - .unwrap_or_default(); + .await?; - if !permissions.contains(ProjectPermissions::EDIT_DETAILS) { - return Err(ApiError::CustomAuthentication( - "You don't have permission to edit this project's icon.".to_string(), - )); - } + // Hide the project + if team_member.is_none() && organization_team_member.is_none() { + return Err(ApiError::CustomAuthentication( + "The specified project does not exist!".to_string(), + )); } - if let Some(icon) = project_item.inner.icon_url { - let name = icon.split(&format!("{cdn_url}/")).nth(1); + let permissions = ProjectPermissions::get_permissions_by_role( + &user.role, + &team_member, + &organization_team_member, + ) + .unwrap_or_default(); - if let Some(icon_path) = name { - file_host.delete_file_version("", icon_path).await?; - } + if !permissions.contains(ProjectPermissions::EDIT_DETAILS) { + return Err(ApiError::CustomAuthentication( + "You don't have permission to edit this project's icon.".to_string(), + )); } + } - let bytes = - read_from_payload(&mut payload, 262144, "Icons must be smaller than 256KiB").await?; - - let color = crate::util::img::get_color_from_img(&bytes)?; + delete_old_images( + project_item.inner.icon_url, + project_item.inner.raw_icon_url, + &***file_host, + ) + .await?; - let hash = sha1::Sha1::from(&bytes).hexdigest(); - let project_id: ProjectId = project_item.inner.id.into(); - let upload_data = file_host - .upload_file( - content_type, - &format!("data/{}/{}.{}", project_id, hash, ext.ext), - bytes.freeze(), - ) - .await?; + let bytes = + read_from_payload(&mut payload, 262144, "Icons must be smaller than 256KiB").await?; + + let project_id: ProjectId = project_item.inner.id.into(); + let upload_result = upload_image_optimized( + &format!("data/{}", project_id), + bytes.freeze(), + &ext.ext, + Some(96), + Some(1.0), + &***file_host, + ) + .await?; - let mut transaction = pool.begin().await?; + let mut transaction = pool.begin().await?; - sqlx::query!( - " + sqlx::query!( + " UPDATE mods - SET icon_url = $1, color = $2 - WHERE (id = $3) + SET icon_url = $1, raw_icon_url = $2, color = $3 + WHERE (id = $4) ", - format!("{}/{}", cdn_url, upload_data.file_name), - color.map(|x| x as i32), - project_item.inner.id as db_ids::ProjectId, - ) - .execute(&mut *transaction) - .await?; + upload_result.url, + upload_result.raw_url, + upload_result.color.map(|x| x as i32), + project_item.inner.id as db_ids::ProjectId, + ) + .execute(&mut *transaction) + .await?; - transaction.commit().await?; - db_models::Project::clear_cache( - project_item.inner.id, - project_item.inner.slug, - None, - &redis, - ) + transaction.commit().await?; + db_models::Project::clear_cache(project_item.inner.id, project_item.inner.slug, None, &redis) .await?; - Ok(HttpResponse::NoContent().body("")) - } else { - Err(ApiError::InvalidInput(format!( - "Invalid format for project icon: {}", - ext.ext - ))) - } + Ok(HttpResponse::NoContent().body("")) } pub async fn delete_project_icon( @@ -1476,21 +1463,19 @@ pub async fn delete_project_icon( } } - let cdn_url = dotenvy::var("CDN_URL")?; - if let Some(icon) = project_item.inner.icon_url { - let name = icon.split(&format!("{cdn_url}/")).nth(1); - - if let Some(icon_path) = name { - file_host.delete_file_version("", icon_path).await?; - } - } + delete_old_images( + project_item.inner.icon_url, + project_item.inner.raw_icon_url, + &***file_host, + ) + .await?; let mut transaction = pool.begin().await?; sqlx::query!( " UPDATE mods - SET icon_url = NULL, color = NULL + SET icon_url = NULL, raw_icon_url = NULL, color = NULL WHERE (id = $1) ", project_item.inner.id as db_ids::ProjectId, @@ -1527,132 +1512,122 @@ pub async fn add_gallery_item( mut payload: web::Payload, session_queue: web::Data, ) -> Result { - if let Some(content_type) = crate::util::ext::get_image_content_type(&ext.ext) { - item.validate() - .map_err(|err| ApiError::Validation(validation_errors_to_string(err, None)))?; - - let cdn_url = dotenvy::var("CDN_URL")?; - let user = get_user_from_headers( - &req, - &**pool, - &redis, - &session_queue, - Some(&[Scopes::PROJECT_WRITE]), - ) - .await? - .1; - let string = info.into_inner().0; - - let project_item = db_models::Project::get(&string, &**pool, &redis) - .await? - .ok_or_else(|| { - ApiError::InvalidInput("The specified project does not exist!".to_string()) - })?; + item.validate() + .map_err(|err| ApiError::Validation(validation_errors_to_string(err, None)))?; - if project_item.gallery_items.len() > 64 { - return Err(ApiError::CustomAuthentication( - "You have reached the maximum of gallery images to upload.".to_string(), - )); - } + let user = get_user_from_headers( + &req, + &**pool, + &redis, + &session_queue, + Some(&[Scopes::PROJECT_WRITE]), + ) + .await? + .1; + let string = info.into_inner().0; - if !user.role.is_admin() { - let (team_member, organization_team_member) = - db_models::TeamMember::get_for_project_permissions( - &project_item.inner, - user.id.into(), - &**pool, - ) - .await?; + let project_item = db_models::Project::get(&string, &**pool, &redis) + .await? + .ok_or_else(|| { + ApiError::InvalidInput("The specified project does not exist!".to_string()) + })?; - // Hide the project - if team_member.is_none() && organization_team_member.is_none() { - return Err(ApiError::CustomAuthentication( - "The specified project does not exist!".to_string(), - )); - } + if project_item.gallery_items.len() > 64 { + return Err(ApiError::CustomAuthentication( + "You have reached the maximum of gallery images to upload.".to_string(), + )); + } - let permissions = ProjectPermissions::get_permissions_by_role( - &user.role, - &team_member, - &organization_team_member, + if !user.role.is_admin() { + let (team_member, organization_team_member) = + db_models::TeamMember::get_for_project_permissions( + &project_item.inner, + user.id.into(), + &**pool, ) - .unwrap_or_default(); + .await?; - if !permissions.contains(ProjectPermissions::EDIT_DETAILS) { - return Err(ApiError::CustomAuthentication( - "You don't have permission to edit this project's gallery.".to_string(), - )); - } + // Hide the project + if team_member.is_none() && organization_team_member.is_none() { + return Err(ApiError::CustomAuthentication( + "The specified project does not exist!".to_string(), + )); } - let bytes = read_from_payload( - &mut payload, - 5 * (1 << 20), - "Gallery image exceeds the maximum of 5MiB.", + let permissions = ProjectPermissions::get_permissions_by_role( + &user.role, + &team_member, + &organization_team_member, ) - .await?; - let hash = sha1::Sha1::from(&bytes).hexdigest(); - - let id: ProjectId = project_item.inner.id.into(); - let url = format!("data/{}/images/{}.{}", id, hash, &*ext.ext); + .unwrap_or_default(); - let file_url = format!("{cdn_url}/{url}"); - if project_item - .gallery_items - .iter() - .any(|x| x.image_url == file_url) - { - return Err(ApiError::InvalidInput( - "You may not upload duplicate gallery images!".to_string(), + if !permissions.contains(ProjectPermissions::EDIT_DETAILS) { + return Err(ApiError::CustomAuthentication( + "You don't have permission to edit this project's gallery.".to_string(), )); } + } - file_host - .upload_file(content_type, &url, bytes.freeze()) - .await?; + let bytes = read_from_payload( + &mut payload, + 2 * (1 << 20), + "Gallery image exceeds the maximum of 2MiB.", + ) + .await?; - let mut transaction = pool.begin().await?; + let id: ProjectId = project_item.inner.id.into(); + let upload_result = crate::util::img::upload_image_optimized( + &format!("data/{}/images/", id), + bytes.freeze(), + &ext.ext, + Some(350), + Some(1.0), + &***file_host, + ) + .await?; - if item.featured { - sqlx::query!( - " + if project_item + .gallery_items + .iter() + .any(|x| x.image_url == upload_result.url) + { + return Err(ApiError::InvalidInput( + "You may not upload duplicate gallery images!".to_string(), + )); + } + + let mut transaction = pool.begin().await?; + + if item.featured { + sqlx::query!( + " UPDATE mods_gallery SET featured = $2 WHERE mod_id = $1 ", - project_item.inner.id as db_ids::ProjectId, - false, - ) - .execute(&mut *transaction) - .await?; - } + project_item.inner.id as db_ids::ProjectId, + false, + ) + .execute(&mut *transaction) + .await?; + } - let gallery_item = vec![db_models::project_item::GalleryItem { - image_url: file_url, - featured: item.featured, - name: item.name, - description: item.description, - created: Utc::now(), - ordering: item.ordering.unwrap_or(0), - }]; - GalleryItem::insert_many(gallery_item, project_item.inner.id, &mut transaction).await?; + let gallery_item = vec![db_models::project_item::GalleryItem { + image_url: upload_result.url, + raw_image_url: upload_result.raw_url, + featured: item.featured, + name: item.name, + description: item.description, + created: Utc::now(), + ordering: item.ordering.unwrap_or(0), + }]; + GalleryItem::insert_many(gallery_item, project_item.inner.id, &mut transaction).await?; - transaction.commit().await?; - db_models::Project::clear_cache( - project_item.inner.id, - project_item.inner.slug, - None, - &redis, - ) + transaction.commit().await?; + db_models::Project::clear_cache(project_item.inner.id, project_item.inner.slug, None, &redis) .await?; - Ok(HttpResponse::NoContent().body("")) - } else { - Err(ApiError::InvalidInput(format!( - "Invalid format for gallery image: {}", - ext.ext - ))) - } + Ok(HttpResponse::NoContent().body("")) } #[derive(Serialize, Deserialize, Validate)] @@ -1891,9 +1866,9 @@ pub async fn delete_gallery_item( } let mut transaction = pool.begin().await?; - let id = sqlx::query!( + let item = sqlx::query!( " - SELECT id FROM mods_gallery + SELECT id, image_url, raw_image_url FROM mods_gallery WHERE image_url = $1 ", item.url @@ -1905,15 +1880,14 @@ pub async fn delete_gallery_item( "Gallery item at URL {} is not part of the project's gallery.", item.url )) - })? - .id; - - let cdn_url = dotenvy::var("CDN_URL")?; - let name = item.url.split(&format!("{cdn_url}/")).nth(1); + })?; - if let Some(icon_path) = name { - file_host.delete_file_version("", icon_path).await?; - } + delete_old_images( + Some(item.image_url), + Some(item.raw_image_url), + &***file_host, + ) + .await?; let mut transaction = pool.begin().await?; @@ -1922,7 +1896,7 @@ pub async fn delete_gallery_item( DELETE FROM mods_gallery WHERE id = $1 ", - id + item.id ) .execute(&mut *transaction) .await?; diff --git a/src/routes/v3/users.rs b/src/routes/v3/users.rs index a3f611f3..464b6cfa 100644 --- a/src/routes/v3/users.rs +++ b/src/routes/v3/users.rs @@ -7,6 +7,8 @@ use serde::{Deserialize, Serialize}; use sqlx::PgPool; use validator::Validate; +use super::{oauth_clients::get_user_clients, ApiError}; +use crate::util::img::delete_old_images; use crate::{ auth::{filter_visible_projects, get_user_from_headers}, database::{models::User, redis::RedisPool}, @@ -23,8 +25,6 @@ use crate::{ util::{routes::read_from_payload, validate::validation_errors_to_string}, }; -use super::{oauth_clients::get_user_clients, ApiError}; - pub fn config(cfg: &mut web::ServiceConfig) { cfg.route("user", web::get().to(user_auth_get)); cfg.route("users", web::get().to(users_get)); @@ -446,71 +446,62 @@ pub async fn user_icon_edit( mut payload: web::Payload, session_queue: web::Data, ) -> Result { - if let Some(content_type) = crate::util::ext::get_image_content_type(&ext.ext) { - let cdn_url = dotenvy::var("CDN_URL")?; - let user = get_user_from_headers( - &req, - &**pool, - &redis, - &session_queue, - Some(&[Scopes::USER_WRITE]), - ) - .await? - .1; - let id_option = User::get(&info.into_inner().0, &**pool, &redis).await?; - - if let Some(actual_user) = id_option { - if user.id != actual_user.id.into() && !user.role.is_mod() { - return Err(ApiError::CustomAuthentication( - "You don't have permission to edit this user's icon.".to_string(), - )); - } - - let icon_url = actual_user.avatar_url; - let user_id: UserId = actual_user.id.into(); - - if let Some(icon) = icon_url { - let name = icon.split(&format!("{cdn_url}/")).nth(1); + let user = get_user_from_headers( + &req, + &**pool, + &redis, + &session_queue, + Some(&[Scopes::USER_WRITE]), + ) + .await? + .1; + let id_option = User::get(&info.into_inner().0, &**pool, &redis).await?; - if let Some(icon_path) = name { - file_host.delete_file_version("", icon_path).await?; - } - } + if let Some(actual_user) = id_option { + if user.id != actual_user.id.into() && !user.role.is_mod() { + return Err(ApiError::CustomAuthentication( + "You don't have permission to edit this user's icon.".to_string(), + )); + } - let bytes = - read_from_payload(&mut payload, 2097152, "Icons must be smaller than 2MiB").await?; + delete_old_images( + actual_user.avatar_url, + actual_user.raw_avatar_url, + &***file_host, + ) + .await?; - let hash = sha1::Sha1::from(&bytes).hexdigest(); - let upload_data = file_host - .upload_file( - content_type, - &format!("user/{}/{}.{}", user_id, hash, ext.ext), - bytes.freeze(), - ) - .await?; + let bytes = + read_from_payload(&mut payload, 262144, "Icons must be smaller than 256KiB").await?; + + let user_id: UserId = actual_user.id.into(); + let upload_result = crate::util::img::upload_image_optimized( + &format!("data/{}", user_id), + bytes.freeze(), + &ext.ext, + Some(96), + Some(1.0), + &***file_host, + ) + .await?; - sqlx::query!( - " - UPDATE users - SET avatar_url = $1 - WHERE (id = $2) - ", - format!("{}/{}", cdn_url, upload_data.file_name), - actual_user.id as crate::database::models::ids::UserId, - ) - .execute(&**pool) - .await?; - User::clear_caches(&[(actual_user.id, None)], &redis).await?; + sqlx::query!( + " + UPDATE users + SET avatar_url = $1, raw_avatar_url = $2 + WHERE (id = $3) + ", + upload_result.url, + upload_result.raw_url, + actual_user.id as crate::database::models::ids::UserId, + ) + .execute(&**pool) + .await?; + User::clear_caches(&[(actual_user.id, None)], &redis).await?; - Ok(HttpResponse::NoContent().body("")) - } else { - Err(ApiError::NotFound) - } + Ok(HttpResponse::NoContent().body("")) } else { - Err(ApiError::InvalidInput(format!( - "Invalid format for user icon: {}", - ext.ext - ))) + Err(ApiError::NotFound) } } diff --git a/src/util/img.rs b/src/util/img.rs index a184a5de..e869b528 100644 --- a/src/util/img.rs +++ b/src/util/img.rs @@ -1,11 +1,14 @@ use crate::database; use crate::database::models::image_item; use crate::database::redis::RedisPool; +use crate::file_hosting::FileHost; use crate::models::images::ImageContext; use crate::routes::ApiError; use color_thief::ColorFormat; use image::imageops::FilterType; -use image::{EncodableLayout, ImageError}; +use image::{DynamicImage, EncodableLayout, GenericImageView, ImageError, ImageOutputFormat}; +use std::io::Cursor; +use webp::Encoder; pub fn get_color_from_img(data: &[u8]) -> Result, ImageError> { let image = image::load_from_memory(data)? @@ -19,6 +22,157 @@ pub fn get_color_from_img(data: &[u8]) -> Result, ImageError> { Ok(color) } +pub struct UploadImageResult { + pub url: String, + pub url_path: String, + + pub raw_url: String, + pub raw_url_path: String, + + pub color: Option, +} + +pub async fn upload_image_optimized( + upload_folder: &str, + bytes: bytes::Bytes, + file_extension: &str, + target_width: Option, + min_aspect_ratio: Option, + file_host: &dyn FileHost, +) -> Result { + let content_type = + crate::util::ext::get_image_content_type(file_extension).ok_or_else(|| { + ApiError::InvalidInput(format!( + "Invalid format for image: {}", + file_extension + )) + })?; + + let cdn_url = dotenvy::var("CDN_URL")?; + + let hash = sha1::Sha1::from(&bytes).hexdigest(); + let (processed_image, processed_image_ext) = + process_image(bytes.clone(), content_type, target_width, min_aspect_ratio)?; + let color = get_color_from_img(&bytes)?; + + // Only upload the processed image if it's smaller than the original + let processed_upload_data = if processed_image.len() < bytes.len() { + Some( + file_host + .upload_file( + content_type, + &format!( + "{}/{}_{}.{}", + upload_folder, + hash, + target_width.unwrap_or(0), + processed_image_ext + ), + processed_image, + ) + .await?, + ) + } else { + None + }; + + let upload_data = file_host + .upload_file( + content_type, + &format!("{}/{}.{}", upload_folder, hash, file_extension), + bytes, + ) + .await?; + + let url = format!("{}/{}", cdn_url, upload_data.file_name); + Ok(UploadImageResult { + raw_url: processed_upload_data + .clone() + .map(|x| format!("{}/{}", cdn_url, x.file_name)) + .unwrap_or_else(|| url.clone()), + raw_url_path: processed_upload_data + .map(|x| x.file_name) + .unwrap_or_else(|| upload_data.file_name.clone()), + + url, + url_path: upload_data.file_name, + color, + }) +} + +fn process_image( + image_bytes: bytes::Bytes, + content_type: &str, + target_width: Option, + min_aspect_ratio: Option, +) -> Result<(bytes::Bytes, String), ImageError> { + if content_type.to_lowercase() == "image/gif" { + return Ok((image_bytes.clone(), "gif".to_string())); + } + + let mut img = image::load_from_memory(&image_bytes)?; + + let webp_bytes = convert_to_webp(&img)?; + img = image::load_from_memory(&webp_bytes)?; + + // Resize the image + let (orig_width, orig_height) = img.dimensions(); + let aspect_ratio = orig_width as f32 / orig_height as f32; + + if let Some(target_width) = target_width { + let new_height = (target_width as f32 / aspect_ratio).round() as u32; + img = img.resize(target_width, new_height, FilterType::Lanczos3); + + if let Some(min_aspect_ratio) = min_aspect_ratio { + // Crop if necessary + if aspect_ratio < min_aspect_ratio { + let crop_height = (target_width as f32 / min_aspect_ratio).round() as u32; + let y_offset = (new_height - crop_height) / 2; + img = img.crop_imm(0, y_offset, target_width, crop_height); + } + } + } + + // Optimize and compress + let mut output = Vec::new(); + img.write_to(&mut Cursor::new(&mut output), ImageOutputFormat::WebP)?; + + Ok((bytes::Bytes::from(output), "webp".to_string())) +} + +fn convert_to_webp(img: &DynamicImage) -> Result, ImageError> { + let rgba = img.to_rgba8(); + let encoder = Encoder::from_rgba(&rgba, img.width(), img.height()); + let webp = encoder.encode(75.0); // Quality factor: 0-100, 75 is a good balance + Ok(webp.to_vec()) +} + +pub async fn delete_old_images( + image_url: Option, + raw_image_url: Option, + file_host: &dyn FileHost, +) -> Result<(), ApiError> { + let cdn_url = dotenvy::var("CDN_URL")?; + let cdn_url_start = format!("{cdn_url}/"); + if let Some(image_url) = image_url { + let name = image_url.split(&cdn_url_start).nth(1); + + if let Some(icon_path) = name { + file_host.delete_file_version("", icon_path).await?; + } + } + + if let Some(raw_image_url) = raw_image_url { + let name = raw_image_url.split(&cdn_url_start).nth(1); + + if let Some(icon_path) = name { + file_host.delete_file_version("", icon_path).await?; + } + } + + Ok(()) +} + // check changes to associated images // if they no longer exist in the String list, delete them // Eg: if description is modified and no longer contains a link to an iamge