From eb20f4d8475e249b0acbf70270caee366e62e659 Mon Sep 17 00:00:00 2001 From: CodeMyst Date: Tue, 4 Feb 2025 13:55:34 +0100 Subject: [PATCH] feat: created database migrator --- api/PasteMyst.Migrator/.gitignore | 274 ++++++++++++++ .../Assets/default_avatar.png | Bin 0 -> 4727 bytes .../PasteMyst.Migrator.csproj | 30 ++ api/PasteMyst.Migrator/Program.cs | 348 ++++++++++++++++++ api/PasteMyst.Migrator/preprocess.d | 68 ++++ api/PasteMyst.Web/Models/EncryptedPaste.cs | 2 + api/PasteMyst.Web/Models/V2/ApiKeyV2.cs | 12 + api/PasteMyst.Web/Models/V2/BasePasteV2.cs | 30 ++ api/PasteMyst.Web/Models/V2/EditV2.cs | 21 ++ .../Models/V2/EncryptedPasteV2.cs | 14 + api/PasteMyst.Web/Models/V2/PasteV2.cs | 10 + api/PasteMyst.Web/Models/V2/PastyV2.cs | 17 + api/PasteMyst.Web/Models/V2/UserV2.cs | 27 ++ api/PasteMyst.Web/Services/AuthService.cs | 2 +- api/pastemyst.sln | 7 + 15 files changed, 861 insertions(+), 1 deletion(-) create mode 100644 api/PasteMyst.Migrator/.gitignore create mode 100644 api/PasteMyst.Migrator/Assets/default_avatar.png create mode 100644 api/PasteMyst.Migrator/PasteMyst.Migrator.csproj create mode 100644 api/PasteMyst.Migrator/Program.cs create mode 100755 api/PasteMyst.Migrator/preprocess.d create mode 100644 api/PasteMyst.Web/Models/V2/ApiKeyV2.cs create mode 100644 api/PasteMyst.Web/Models/V2/BasePasteV2.cs create mode 100644 api/PasteMyst.Web/Models/V2/EditV2.cs create mode 100644 api/PasteMyst.Web/Models/V2/EncryptedPasteV2.cs create mode 100644 api/PasteMyst.Web/Models/V2/PasteV2.cs create mode 100644 api/PasteMyst.Web/Models/V2/PastyV2.cs create mode 100644 api/PasteMyst.Web/Models/V2/UserV2.cs diff --git a/api/PasteMyst.Migrator/.gitignore b/api/PasteMyst.Migrator/.gitignore new file mode 100644 index 00000000..8afe457e --- /dev/null +++ b/api/PasteMyst.Migrator/.gitignore @@ -0,0 +1,274 @@ +# Created by https://www.toptal.com/developers/gitignore/api/aspnetcore +# Edit at https://www.toptal.com/developers/gitignore?templates=aspnetcore + +### ASPNETCore ### +## Ignore Visual Studio temporary files, build results, and +## files generated by popular Visual Studio add-ons. + +# User-specific files +*.suo +*.user +*.userosscache +*.sln.docstates + +# User-specific files (MonoDevelop/Xamarin Studio) +*.userprefs + +# Build results +[Dd]ebug/ +[Dd]ebugPublic/ +[Rr]elease/ +[Rr]eleases/ +x64/ +x86/ +bld/ +[Bb]in/ +[Oo]bj/ +[Ll]og/ + +# Visual Studio 2015 cache/options directory +.vs/ +# Uncomment if you have tasks that create the project's static files in wwwroot +#wwwroot/ + +# MSTest test Results +[Tt]est[Rr]esult*/ +[Bb]uild[Ll]og.* + +# NUNIT +*.VisualState.xml +TestResult.xml + +# Build Results of an ATL Project +[Dd]ebugPS/ +[Rr]eleasePS/ +dlldata.c + +# DNX +project.lock.json +project.fragment.lock.json +artifacts/ + +*_i.c +*_p.c +*_i.h +*.ilk +*.meta +*.obj +*.pch +*.pdb +*.pgc +*.pgd +*.rsp +*.sbr +*.tlb +*.tli +*.tlh +*.tmp +*.tmp_proj +*.log +*.vspscc +*.vssscc +.builds +*.pidb +*.svclog +*.scc + +# Chutzpah Test files +_Chutzpah* + +# Visual C++ cache files +ipch/ +*.aps +*.ncb +*.opendb +*.opensdf +*.sdf +*.cachefile +*.VC.db +*.VC.VC.opendb + +# Visual Studio profiler +*.psess +*.vsp +*.vspx +*.sap + +# TFS 2012 Local Workspace +$tf/ + +# Guidance Automation Toolkit +*.gpState + +# ReSharper is a .NET coding add-in +_ReSharper*/ +*.[Rr]e[Ss]harper +*.DotSettings.user + +# JustCode is a .NET coding add-in +.JustCode + +# TeamCity is a build add-in +_TeamCity* + +# DotCover is a Code Coverage Tool +*.dotCover + +# Visual Studio code coverage results +*.coverage +*.coveragexml + +# NCrunch +_NCrunch_* +.*crunch*.local.xml +nCrunchTemp_* + +# MightyMoose +*.mm.* +AutoTest.Net/ + +# Web workbench (sass) +.sass-cache/ + +# Installshield output folder +[Ee]xpress/ + +# DocProject is a documentation generator add-in +DocProject/buildhelp/ +DocProject/Help/*.HxT +DocProject/Help/*.HxC +DocProject/Help/*.hhc +DocProject/Help/*.hhk +DocProject/Help/*.hhp +DocProject/Help/Html2 +DocProject/Help/html + +# Click-Once directory +publish/ + +# Publish Web Output +*.[Pp]ublish.xml +*.azurePubxml +# TODO: Comment the next line if you want to checkin your web deploy settings +# but database connection strings (with potential passwords) will be unencrypted +*.pubxml +*.publishproj + +# Microsoft Azure Web App publish settings. Comment the next line if you want to +# checkin your Azure Web App publish settings, but sensitive information contained +# in these scripts will be unencrypted +PublishScripts/ + +# NuGet Packages +*.nupkg +# The packages folder can be ignored because of Package Restore +**/packages/* +# except build/, which is used as an MSBuild target. +!**/packages/build/ +# Uncomment if necessary however generally it will be regenerated when needed +#!**/packages/repositories.config +# NuGet v3's project.json files produces more ignoreable files +*.nuget.props +*.nuget.targets + +# Microsoft Azure Build Output +csx/ +*.build.csdef + +# Microsoft Azure Emulator +ecf/ +rcf/ + +# Windows Store app package directories and files +AppPackages/ +BundleArtifacts/ +Package.StoreAssociation.xml +_pkginfo.txt + +# Visual Studio cache files +# files ending in .cache can be ignored +*.[Cc]ache +# but keep track of directories ending in .cache +!*.[Cc]ache/ + +# Others +ClientBin/ +~$* +*~ +*.dbmdl +*.dbproj.schemaview +*.jfm +*.pfx +*.publishsettings +node_modules/ +orleans.codegen.cs + +# Since there are multiple workflows, uncomment next line to ignore bower_components +# (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) +#bower_components/ + +# RIA/Silverlight projects +Generated_Code/ + +# Backup & report files from converting an old project file +# to a newer Visual Studio version. Backup files are not needed, +# because we have git ;-) +_UpgradeReport_Files/ +Backup*/ +UpgradeLog*.XML +UpgradeLog*.htm + +# SQL Server files +*.mdf +*.ldf + +# Business Intelligence projects +*.rdl.data +*.bim.layout +*.bim_*.settings + +# Microsoft Fakes +FakesAssemblies/ + +# GhostDoc plugin setting file +*.GhostDoc.xml + +# Node.js Tools for Visual Studio +.ntvs_analysis.dat + +# Visual Studio 6 build log +*.plg + +# Visual Studio 6 workspace options file +*.opt + +# Visual Studio LightSwitch build output +**/*.HTMLClient/GeneratedArtifacts +**/*.DesktopClient/GeneratedArtifacts +**/*.DesktopClient/ModelManifest.xml +**/*.Server/GeneratedArtifacts +**/*.Server/ModelManifest.xml +_Pvt_Extensions + +# Paket dependency manager +.paket/paket.exe +paket-files/ + +# FAKE - F# Make +.fake/ + +# JetBrains Rider +.idea/ +*.sln.iml + +# CodeRush +.cr/ + +# Python Tools for Visual Studio (PTVS) +__pycache__/ +*.pyc + +# Cake - Uncomment if you are using it +# tools/ + +# End of https://www.toptal.com/developers/gitignore/api/aspnetcore diff --git a/api/PasteMyst.Migrator/Assets/default_avatar.png b/api/PasteMyst.Migrator/Assets/default_avatar.png new file mode 100644 index 0000000000000000000000000000000000000000..581d383ad5544e48d3235a72bdf01f333c1be5fc GIT binary patch literal 4727 zcma)AcQ72l_uo6mDW?X}dkKP5qegG1_sEG7U5MVIoCuQWDMat096_AZqD8bQA)przb-43#1`pIT)Piq97%9YGj+})z_&0uU4c>6&L^B{98P*p`DTrY8SFo}C2 z1^B-JrOWk|p+gM*Ew*B>8ZGEw20=hRd5gN3bz!G}%a@&@gB)X871b5nZ75yV`)mFu z{n*vBrkd$1H=p^Fr3G4FV?dRu3t0=Vk;j?!)fqj&yAXL$!NeeRS!QJxLYAm9vm%RY z47;(L%q!%%9MMe$1qHPqr>(zZEvo2kueL1kJ9lpUW=MlkvokF;SyOWoI3OUCuQmV$ zQnihkN>(fbO1s7yM&VqcgdK>|CKijWk@{F|+9%RJNVmojJ^!W57}bobZo!Y9z|SX~ zPlW-LE25{FWa*}l7HRhY<8HA6Z@48ym@%r|inm1Nh8J1-2lGPm#qhs=c#k`4K zl#}Ur+ft0kMQ8!;9iqiE$>Kl+jmBg>!*P)FQjqpMuhT zi%m3{r%7HR%u9Mi=*yd72HkU6aH!v9VSZ-cadM;kFeG0=^^_zbUErKMmtdO?_T!$# zIxw|#n&_Ul2tz&qz9GlbKXHocThxcXTv~0gSQT>K`2*s~Zx%3jqD;6(P*vAy394)4 zg)q>gPx2DX$#vmguz;2Vk~3C~bGxEH8x_o4#55#5F(1Xq9k>t}dRvTO7_o`?$=hF` zSqM_9+|7Y}XGqKUJowrC;6b4)-L1mzcz%&H2P_7i$s_eZ0Srb%zB20{RB3-h49& z0Nuq&_8SatO;6(bH zb$65F^&Ao=cKMP-;C_R%ZdI8cJ+1|C1=KV9%1H>;)Mp7#a=PL23VBB56L)+=p3Qoj*1h{ED5?~N1f{{)?FQ5_hjPj%-O!p3qf8M#6G*$e;kYgOI!#+ zxn1%=BOvb^|3;@%jJ%KlO_JNC3NZq}Mo3{1uR4WD$z8Q=+ck(W2#Ps@aTyy3%(8T0 zU-r~J?oqTMGEsiTQ(p5ra=dG-o?Aj6d@d7VHoQq8I&f>e{=p7NihMlVN7GGAnZ_o5 zs~)oS_oJknkJ4D1{SZ7_#hYClt0XM z&rf=^9P~IulDO`NMuTN}j(9wUCtHH(`Q^RPc-Dhvo`PVau-*}t?LzJ8_}9VCVu)G~ zITB>JMh?N|g5JR%q3DMp(T?(s5a$I>W`;h)J=0XK^+CxHkSZ#${i66K9M`y`%2=fKi{sI14l@Y#kWR=$@08YX0uWvJ%5jZ5 z<*!(jl-y_B!h`Y3KOx3iikTrRGrNvNY4_E!dm(1IHkeyB*j z+F3EW{|>IU(YE)TyXY~%lE6UAq6x;A-V8x9s;XU)%o_QFEen&jH)oK~~ z)neF+OO95%YWlO)=QqvC1u6|Ak9r;qr;rOocNU$=T1`w%q{M#c%{GQ`eh7#7LJsq( zd9_CWLQh7MDH@u6cNrE6Rv@=x`np)>k87gh457sL{P;c(oKys76ivUndnVlN|9U-( zdShl(;O>HGwGs#C>NH_4xioyHvtJl8F~+vqkDK%sRCZd0+t|0KnZ*^r7^7iKc8Z2* zr-0wtjGT@=?2bJTDKo9!&VYXc-2-AtnWSTCh;am8j*4y&2gPLpJx-{lCLLw|wS%01^@(mYlNp-CU`dE+3zD=cm$h z3-)jOFodz0CC21#bvvcs*LS25U*t)a;xeE@JE__{sd|Wp4o~;EmSZQ|V<`xd>8L>j zQ(oHc{7*{C>(+yXL8UZ>KBhn489-GK`DVp8LGJGO`hue$goX#v8LuOl%LeFGk%(H8 zF*^nRL8}+`X?~>~fQtsKH8olI{(=Cp8nBj`u~ix>G0Z;#uklv8!>R>zRs zS3Elv&t1?se|^J zr*5V-2-D^dv`Y|OWgLDRoLH=q{yVQR(aI4N7ISjN9#u8R$@aClFvGp@hwni%MGxU0 z_Tt0)tSK=^mejI>Mk!ie1O6wYZ<)Irj;c=LWRvE_%iLyf1b<;l+IIMt0W1z;+YUVF zWE5uma_n=`qbf@{8^5r)^*Wm!HSDKAY`r8~?;YelarFOSoW6aYAD08?HuxsI&L(9zq@UzaBe=EB*q z^WAEqPasRbSo+9eHj^u{9@8^ndbSG0(@d zY&GUDym*wF^(Y05;!esX#j<0ye^cjfIkq)bG*juG=lHpRZWymC;5J0X6$$k2Bw>)lb6(rMVo6i(4Z; zql<$)LN9cSHcGSm!{FewJcky9q4$}TwP|I}gziy~9fx3gJE$+ua4ErWWtisugAVeG zuHFt0M#Z?=*X;i9gg#x1EH3WxK4}tJh>D%TZRup5n^un;Gv~@xo;pA8-RRq+E(ml_ zT9FC9yGFx0os;S%&Ns;ce3?egH0y89KJ1QyjI4ClBKKM!e9HeCx^A%;)FxAW-D{zF zT(+HHwKLn3+$$5NIN&^FunkvvcTv&Hus5Jvc+)zxrMZ|r&W zzNgD|B-4O1RN9Xp-kb84I`*t6icjYed557VRAtmkbFZBx=josPBbDiiUo`m|E$@#z4?Q23WamvIBl(0O_f{ zx1cqG%ohqOs0!!F1OL)3d6HCa?mgZA!~#sMGX=nLe?KC8hnC?~7Y-U-$DH;TbcZ5K zI_NS2lJ8!J?DHpvH+8P>_V~Xyi~u_)0e?!#R0pgaPT-Hc?A+QEgz;8OE!XEyw`Yyy z6R~tJJ6L@-a^M!Ov9V&1eDpjgv*|Qxlf=unhUNxfK}B}DlmFuIFE;ySP@i#iNKgV< zSN5j&?|DT@>}8xXNvb{rpMi53h#*6EHH*SeQmzYr#(d4R#BZqQ-qt*jXe;&M^9SL5-2>b_XbUm$>)RD=D#qcT;%+?KUkr*% zdrhHU${s?9#DP&o8aS9)WF96z%Y25633yI!3}RU+)Oz_{bArb7Sc2fp<$cUx83ncg z{%1FZ)@n3jiZ!)`s0+jYL|A3w1M(K}w}h#mvHz`AVKQ7mBh%D9<)mVjmqC#Td?6gi z6v0N4Snz2AqA4gE`9O+TN5sBzU~Xtmj2sHg-DB}W&{u#TH?9bZQUP}w*o|Mc4-a^k zXj-snKcITHo1|a#;>A*bpW)i20OogOS9{K?F2BxuB@;OOO6$T3Gq~U{Bs(R4hJbhd zPPphmXP2+V!u*fV zHJE_sZ{>v5d?}BJ%W}q}>%v^xCjd<$jA?A;)#lO$<&O1-Ued0uh*Tnb|Ce>Tw;^dx z@&{QLEY@T%fNEztP3c1s%g#)Uuw5T>Af1Q=B_f=*^r6swso^Hw1>N;gDF_*m)~4Xr zCDMQSTWEbF$re4Z+}+6s(jM+3W2D&U?Tdd4&gfu%6XKWL`Z z3GBr-2T!W4X}c{e_L&^>-NVFNtuRZv>;9sZR^W!x{F2(qDzD-G^c1}p z)qSp^e%dnMR>b|&oWm$m01AQ)W#doFcI=^AwTtaUL}y2oTQf4#?YKKIyh8*evz1aC zS1aY$xc$bY%A%tqilF?D2mlE`7oMLIEI`CHDEpT}v@_7y+`4vhD?1x~`>>{P zlFJW%pN{&!i?O(|xVU)1{hK$xwhn!lY@sfBj9bHmCv~g6kAVW(wy>crQ;yuxk?)hd zSGhqNi_XzmN@>;7PS)LyR0RQIt}|C)1I3s9Mwu?xmZ|3+7?1;{w<>8!V*T-Ngr9Sh zBK+^>?fgjqEUJ6NK^N1sV>P?S-$@aH7nKJ~E&lS|YF?*XA3uIf*xy$n6s|svy|k() z;+gWX-Cv$}yustOYfQ0N4h!W9x?Ls}!le&rJ>LFao_2N;8Wx5s+uhxz0|1HJLZi{y zIXO8&mmw3@drVfT8gt8jPxJ9i0EsG2-C1;z`NMz8QT%_B8W|TvPKmEkh1W?ME_c8m zsC6sQ*)h;X!O7p{k^o7hq>Lz1LR3=HLPA18T2?_)Mg)meKqAGiHSzo}fVYpchilk> q2G|Et=U)P@{f~q|4{w)%Ku2%i|BNAZ5L3Iv06i@u%{S@}G5-Zv=Z9nf literal 0 HcmV?d00001 diff --git a/api/PasteMyst.Migrator/PasteMyst.Migrator.csproj b/api/PasteMyst.Migrator/PasteMyst.Migrator.csproj new file mode 100644 index 00000000..7698474f --- /dev/null +++ b/api/PasteMyst.Migrator/PasteMyst.Migrator.csproj @@ -0,0 +1,30 @@ + + + + Exe + net8.0 + enable + enable + + + + + + + + + + + + + + PreserveNewest + + + + + + PreserveNewest + + + diff --git a/api/PasteMyst.Migrator/Program.cs b/api/PasteMyst.Migrator/Program.cs new file mode 100644 index 00000000..28db25f3 --- /dev/null +++ b/api/PasteMyst.Migrator/Program.cs @@ -0,0 +1,348 @@ +using System.Security.Cryptography; +using System.Text; +using MongoDB.Bson; +using MongoDB.Bson.Serialization; +using MongoDB.Bson.Serialization.Conventions; +using MongoDB.Driver; +using MongoDB.Driver.GridFS; +using PasteMyst.Web.Models; +using PasteMyst.Web.Models.Auth; +using PasteMyst.Web.Models.V2; +using PasteMyst.Web.Serializers; +using PasteMyst.Web.Services; +using ShellProgressBar; + +Console.WriteLine("Preprocessing the database..."); + +var process = new System.Diagnostics.Process +{ + StartInfo = new System.Diagnostics.ProcessStartInfo + { + FileName = "dub", + Arguments = "preprocess.d", + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true + } +}; + +process.OutputDataReceived += (sender, args) => {}; +process.ErrorDataReceived += (sender, args) => {}; + +process.Start(); +process.BeginOutputReadLine(); +process.BeginErrorReadLine(); +process.WaitForExit(); + +if (process.ExitCode != 0) +{ + Console.WriteLine("Preprocessing failed."); + return; +} + +Console.WriteLine("Migrating the database..."); + +var connectionString = "mongodb://127.0.0.1:27017"; + +BsonSerializer.TryRegisterSerializer(new CustomEnumStringSerializer()); + +var camelCaseConvention = new ConventionPack { new CamelCaseElementNameConvention() }; +ConventionRegistry.Register("CamelCase", camelCaseConvention, type => true); + +var mongoClient = new MongoClient(connectionString); + +var v2Db = mongoClient.GetDatabase("pastemyst-v2"); +var v3Db = mongoClient.GetDatabase("pastemyst"); + +var usersV2 = v2Db.GetCollection("users"); +var pastesV2 = v2Db.GetCollection("pastes").Find(Builders.Filter.Eq(p => p.Encrypted, false)).ToList(); +var encryptedPastesV2 = v2Db.GetCollection("pastes").Find(Builders.Filter.Eq(p => p.Encrypted, true)).ToList(); +var apiKeysV2 = v2Db.GetCollection("api-keys"); + +var usersV3 = v3Db.GetCollection("users"); +var basePastesV3 = v3Db.GetCollection("pastes"); +var pastesV3 = basePastesV3.OfType(); +var encryptedPastesV3 = basePastesV3.OfType(); +var accessTokensV3 = v3Db.GetCollection("accessTokens"); +var actionLogsV3 = v3Db.GetCollection("actionLogs"); + +var imagesV3 = new GridFSBucket(v3Db, new() +{ + BucketName = "images", + ChunkSizeBytes = 1_000_000 +}); + +var usernameIndex = Builders.IndexKeys.Ascending(u => u.Username); + +usersV3.Indexes.CreateOne(new CreateIndexModel(usernameIndex, new() +{ + Unique = true +})); + +var defaultAvatarId = await UploadDefaultAvatar(); +await MigrateUsers(defaultAvatarId); +await MigrateUnencryptedPastes(); +await MigrateEncryptedPastes(); +await MigrateApiKeys(); + +Console.WriteLine("Migration completed successfully."); + +async Task UploadDefaultAvatar() +{ + var fileBytes = await File.ReadAllBytesAsync("Assets/default_avatar.png"); + using var stream = new MemoryStream(fileBytes); + + var options = new GridFSUploadOptions + { + Metadata = new BsonDocument + { + { "Content-Type", "image/png" } + } + }; + + return await imagesV3.UploadFromStreamAsync("", stream, options); +} + +async Task MigrateUsers(ObjectId defaultAvatarId) +{ + var httpClient = new HttpClient(); + + var progressBarOptions = new ProgressBarOptions + { + ForegroundColor = ConsoleColor.Yellow, + ForegroundColorDone = ConsoleColor.DarkGreen, + BackgroundColor = ConsoleColor.DarkGray, + ProgressCharacter = '─' + }; + + var allUsersV2 = await usersV2.Find(new BsonDocument()).ToListAsync(); + + using var progressBar = new ProgressBar(allUsersV2.Count, "Migrating users", progressBarOptions); + + foreach (var userV2 in allUsersV2) + { + ObjectId avatarId = defaultAvatarId; + + try { + var avatarResponse = await httpClient.GetAsync(userV2.AvatarUrl); + if (avatarResponse.IsSuccessStatusCode) + { + var contentType = avatarResponse.Content.Headers.ContentType?.MediaType ?? "image/png"; + var buffer = await avatarResponse.Content.ReadAsByteArrayAsync(); + + using var stream = new MemoryStream(buffer); + var uploadOptions = new GridFSUploadOptions + { + Metadata = new BsonDocument + { + { "Content-Type", contentType } + } + }; + + avatarId = await imagesV3.UploadFromStreamAsync(userV2.Username, stream, uploadOptions); + } + } + catch {} + + var userV3 = new User + { + Id = userV2.Id, + Username = userV2.Username, + AvatarId = avatarId.ToString(), + IsContributor = userV2.Contributor, + IsSupporter = userV2.SupporterLength > 0, + IsAdmin = false, + ProviderName = userV2.ServiceIds.FirstOrDefault().Key, + ProviderId = userV2.ServiceIds.FirstOrDefault().Value, + UserSettings = new() { + ShowAllPastesOnProfile = userV2.PublicProfile + }, + Settings = new() {} + }; + + await usersV3.InsertOneAsync(userV3); + + var actionLog = new ActionLog + { + CreatedAt = DateTime.UtcNow, + Type = ActionLogType.UserCreated, + ObjectId = userV3.Id + }; + + await actionLogsV3.InsertOneAsync(actionLog); + + progressBar.Tick(); + + await Task.Delay(250); + } +} + +async Task MigrateUnencryptedPastes() +{ + var progressBarOptions = new ProgressBarOptions + { + ForegroundColor = ConsoleColor.Yellow, + ForegroundColorDone = ConsoleColor.DarkGreen, + BackgroundColor = ConsoleColor.DarkGray, + ProgressCharacter = '─' + }; + + using var progressBar = new ProgressBar(pastesV2.Count, "Migrating unencrypted pastes", progressBarOptions); + + foreach (var pasteV2 in pastesV2) + { + foreach (var pasty in pasteV2.Pasties) + { + pasty.Language = pasty.Language switch + { + "Vue.js Component" => "Vue", + "TypeScript-JSX" => "TSX", + "Asterisk" => "Text", + "GitHub Flavored Markdown" => "Markdown", + "JSON-LD" => "JSON", + "SQLite" => "SQL", + "Properties files" => "INI", + "Z80" => "Assembly", + "Solr" => "Text", + "Spreadsheet" => "Text", + "mscgen" => "Text", + "MS SQL" => "SQL", + _ => pasty.Language + }; + } + + var starsFilter = Builders.Filter.ElemMatch(u => u.Stars, p => p == pasteV2.Id); + var stars = (await usersV2.Find(starsFilter).ToListAsync()).Select(u => u.Id).ToList(); + + var pasties = pasteV2.Pasties.Select(p => new Pasty + { + Id = p.Id, + Title = p.Title == "" ? "untitled" : p.Title, + Language = p.Language, + Content = p.Code + }).ToList(); + + var paste = new Paste + { + Id = pasteV2.Id, + Title = pasteV2.Title, + CreatedAt = DateTimeOffset.FromUnixTimeSeconds(pasteV2.CreatedAt).UtcDateTime, + ExpiresIn = pasteV2.ExpiresIn, + DeletesAt = pasteV2.DeletesAt == 0 ? null : DateTimeOffset.FromUnixTimeSeconds(pasteV2.DeletesAt).UtcDateTime, + OwnerId = pasteV2.OwnerId == "" ? null : pasteV2.OwnerId, + Private = pasteV2.IsPrivate, + Pinned = pasteV2.IsPublic, + Tags = pasteV2.Tags, + Stars = stars, + Pasties = pasties + }; + + await pastesV3.InsertOneAsync(paste); + + var actionLog = new ActionLog + { + CreatedAt = paste.CreatedAt, + Type = ActionLogType.PasteCreated, + ObjectId = paste.Id + }; + + await actionLogsV3.InsertOneAsync(actionLog); + + progressBar.Tick(); + } +} + +async Task MigrateEncryptedPastes() +{ + var progressBarOptions = new ProgressBarOptions + { + ForegroundColor = ConsoleColor.Yellow, + ForegroundColorDone = ConsoleColor.DarkGreen, + BackgroundColor = ConsoleColor.DarkGray, + ProgressCharacter = '─' + }; + + using var progressBar = new ProgressBar(encryptedPastesV2.Count, "Migrating encrypted pastes", progressBarOptions); + + foreach (var pasteV2 in encryptedPastesV2) + { + var starsFilter = Builders.Filter.ElemMatch(u => u.Stars, p => p == pasteV2.Id); + var stars = (await usersV2.Find(starsFilter).ToListAsync()).Select(u => u.Id).ToList(); + + var paste = new EncryptedPaste + { + Id = pasteV2.Id, + Title = "untitled", + CreatedAt = DateTimeOffset.FromUnixTimeSeconds(pasteV2.CreatedAt).UtcDateTime, + ExpiresIn = pasteV2.ExpiresIn, + DeletesAt = pasteV2.DeletesAt == 0 ? null : DateTimeOffset.FromUnixTimeSeconds(pasteV2.DeletesAt).UtcDateTime, + OwnerId = pasteV2.OwnerId == "" ? null : pasteV2.OwnerId, + Private = pasteV2.IsPrivate, + Pinned = pasteV2.IsPublic, + Tags = pasteV2.Tags, + Stars = stars, + EncryptedData = pasteV2.EncryptedData, + Iv = pasteV2.EncryptedKey, + Salt = pasteV2.Salt, + EncryptionVersion = 2 + }; + + await encryptedPastesV3.InsertOneAsync(paste); + + var actionLog = new ActionLog + { + CreatedAt = paste.CreatedAt, + Type = ActionLogType.PasteCreated, + ObjectId = paste.Id + }; + + await actionLogsV3.InsertOneAsync(actionLog); + + progressBar.Tick(); + } +} + +async Task MigrateApiKeys() +{ + var progressBarOptions = new ProgressBarOptions + { + ForegroundColor = ConsoleColor.Yellow, + ForegroundColorDone = ConsoleColor.DarkGreen, + BackgroundColor = ConsoleColor.DarkGray, + ProgressCharacter = '─' + }; + + var allApiKeysV2 = await apiKeysV2.Find(new BsonDocument()).ToListAsync(); + + using var progressBar = new ProgressBar(allApiKeysV2.Count, "Migrating API keys", progressBarOptions); + + var idProvider = new IdProvider(); + + foreach (var apiKeyV2 in allApiKeysV2) + { + var hashedToken = SHA512.HashData(Encoding.UTF8.GetBytes(apiKeyV2.Key)); + + var hashStringBuilder = new StringBuilder(); + foreach (var b in hashedToken) + { + hashStringBuilder.Append(b.ToString("x2")); + } + + var apiKey = new AccessToken + { + Id = await idProvider.GenerateId(async id => await accessTokensV3.Find(a => a.Id == id).FirstOrDefaultAsync() is not null), + Description = "v2 api key", + Hidden = false, + CreatedAt = DateTime.UtcNow, + ExpiresAt = null, + Token = hashStringBuilder.ToString(), + OwnerId = apiKeyV2.Id, + Scopes = [Scope.Paste, Scope.User] + }; + + await accessTokensV3.InsertOneAsync(apiKey); + + progressBar.Tick(); + } +} diff --git a/api/PasteMyst.Migrator/preprocess.d b/api/PasteMyst.Migrator/preprocess.d new file mode 100755 index 00000000..2292af94 --- /dev/null +++ b/api/PasteMyst.Migrator/preprocess.d @@ -0,0 +1,68 @@ +#!/usr/bin/env dub +/+ dub.sdl: + name "pastemyst-db-preprocess" + dependency "vibe-d" version="~>0.10.1" + dependency "vibe-stream:tls" version="~>1.1.1" + subConfiguration "vibe-stream:tls" "notls" ++/ + +import std.stdio; +import std.array; +import std.base64; +import vibe.d; + +public enum ExpiresIn +{ + never = "never", + oneHour = "1h", + twoHours = "2h", + tenHours = "10h", + oneDay = "1d", + twoDays = "2d", + oneWeek = "1w", + oneMonth = "1m", + oneYear = "1y" +} + +public struct EncryptedPaste +{ + @name("_id") + public string id; + public ulong createdAt; + public ExpiresIn expiresIn; + public ulong deletesAt; + public string ownerId; + public bool isPrivate; + public bool isPublic; + public string[] tags; + public ulong stars; + public bool encrypted; + public string encryptedData; + public string encryptedKey; + public string salt; +} + +void main() +{ + MongoDatabase mongo = connectMongoDB("mongodb://localhost:27017").getDatabase("pastemyst-v2"); + MongoCollection collection = mongo["pastes"]; + auto pastes = collection.find!EncryptedPaste(["encrypted": true]); + + foreach (paste; pastes.array) + { + try + { + Base64.decode(paste.encryptedData); + Base64.decode(paste.encryptedKey); + Base64.decode(paste.salt); + } + catch (Exception) + { + paste.encryptedData = Base64.encode(cast(const(ubyte[])) paste.encryptedData); + paste.encryptedKey = Base64.encode(cast(const(ubyte[])) paste.encryptedKey); + paste.salt = Base64.encode(cast(const(ubyte[])) paste.salt); + } + + collection.replaceOne(["_id": paste.id], paste); + } +} \ No newline at end of file diff --git a/api/PasteMyst.Web/Models/EncryptedPaste.cs b/api/PasteMyst.Web/Models/EncryptedPaste.cs index 2017304a..4316022d 100644 --- a/api/PasteMyst.Web/Models/EncryptedPaste.cs +++ b/api/PasteMyst.Web/Models/EncryptedPaste.cs @@ -7,4 +7,6 @@ public class EncryptedPaste : BasePaste public string Iv { get; set; } public string Salt { get; set; } + + public int EncryptionVersion { get; set; } = 3; } diff --git a/api/PasteMyst.Web/Models/V2/ApiKeyV2.cs b/api/PasteMyst.Web/Models/V2/ApiKeyV2.cs new file mode 100644 index 00000000..a63b3973 --- /dev/null +++ b/api/PasteMyst.Web/Models/V2/ApiKeyV2.cs @@ -0,0 +1,12 @@ +using MongoDB.Bson; +using MongoDB.Bson.Serialization.Attributes; + +namespace PasteMyst.Web.Models.V2; + +public class ApiKeyV2 +{ + [BsonId] + [BsonRepresentation(BsonType.String)] + public required string Id { get; init; } + public required string Key { get; set; } +} diff --git a/api/PasteMyst.Web/Models/V2/BasePasteV2.cs b/api/PasteMyst.Web/Models/V2/BasePasteV2.cs new file mode 100644 index 00000000..73ed23f8 --- /dev/null +++ b/api/PasteMyst.Web/Models/V2/BasePasteV2.cs @@ -0,0 +1,30 @@ +using MongoDB.Bson; +using MongoDB.Bson.Serialization.Attributes; + +namespace PasteMyst.Web.Models.V2; + +[BsonKnownTypes(typeof(PasteV2), typeof(EncryptedPasteV2))] +public class BasePasteV2 +{ + [BsonId] + [BsonRepresentation(BsonType.String)] + public string Id { get; init; } + + public string OwnerId { get; set; } + + public long CreatedAt { get; set; } + + public ExpiresIn ExpiresIn { get; set; } + + public long DeletesAt { get; set; } + + public bool IsPrivate { get; set; } + + public bool IsPublic { get; set; } + + public List Tags { get; set; } + + public int Stars { get; set; } + + public bool Encrypted { get; set; } +} diff --git a/api/PasteMyst.Web/Models/V2/EditV2.cs b/api/PasteMyst.Web/Models/V2/EditV2.cs new file mode 100644 index 00000000..69b23d06 --- /dev/null +++ b/api/PasteMyst.Web/Models/V2/EditV2.cs @@ -0,0 +1,21 @@ +using MongoDB.Bson; +using MongoDB.Bson.Serialization.Attributes; + +namespace PasteMyst.Web.Models.V2; + +public class EditV2 +{ + [BsonId] + [BsonRepresentation(BsonType.String)] + public string Id { get; init; } + + public int EditId { get; set; } + + public int EditType { get; set; } + + public List Metadata { get; set; } + + public string Edit { get; set; } + + public long EditedAt { get; set; } +} diff --git a/api/PasteMyst.Web/Models/V2/EncryptedPasteV2.cs b/api/PasteMyst.Web/Models/V2/EncryptedPasteV2.cs new file mode 100644 index 00000000..84a0c11f --- /dev/null +++ b/api/PasteMyst.Web/Models/V2/EncryptedPasteV2.cs @@ -0,0 +1,14 @@ +using MongoDB.Bson; +using MongoDB.Bson.Serialization.Attributes; + +namespace PasteMyst.Web.Models.V2; + +[BsonIgnoreExtraElements] +public class EncryptedPasteV2 : BasePasteV2 +{ + public string EncryptedData { get; set; } + + public string EncryptedKey { get; set; } + + public string Salt { get; set; } +} diff --git a/api/PasteMyst.Web/Models/V2/PasteV2.cs b/api/PasteMyst.Web/Models/V2/PasteV2.cs new file mode 100644 index 00000000..9dff779f --- /dev/null +++ b/api/PasteMyst.Web/Models/V2/PasteV2.cs @@ -0,0 +1,10 @@ +namespace PasteMyst.Web.Models.V2; + +public class PasteV2 : BasePasteV2 +{ + public List Edits { get; set; } = []; + + public string Title { get; set; } = "untitled"; + + public List Pasties { get; set; } = []; +} diff --git a/api/PasteMyst.Web/Models/V2/PastyV2.cs b/api/PasteMyst.Web/Models/V2/PastyV2.cs new file mode 100644 index 00000000..37ef658c --- /dev/null +++ b/api/PasteMyst.Web/Models/V2/PastyV2.cs @@ -0,0 +1,17 @@ +using MongoDB.Bson; +using MongoDB.Bson.Serialization.Attributes; + +namespace PasteMyst.Web.Models.V2; + +public class PastyV2 +{ + [BsonId] + [BsonRepresentation(BsonType.String)] + public string Id { get; init; } + + public string Title { get; set; } + + public string Language { get; set; } + + public string Code { get; set; } +} diff --git a/api/PasteMyst.Web/Models/V2/UserV2.cs b/api/PasteMyst.Web/Models/V2/UserV2.cs new file mode 100644 index 00000000..66a5c88e --- /dev/null +++ b/api/PasteMyst.Web/Models/V2/UserV2.cs @@ -0,0 +1,27 @@ +using MongoDB.Bson; +using MongoDB.Bson.Serialization.Attributes; + +namespace PasteMyst.Web.Models.V2; + +public class UserV2 +{ + [BsonId] + [BsonRepresentation(BsonType.String)] + public string Id { get; init; } + + public string Username { get; set; } + + public Dictionary ServiceIds { get; set; } + + public bool Contributor { get; set; } + + public List Stars { get; set; } + + public string AvatarUrl { get; set; } + + public bool PublicProfile { get; set; } + + public string DefaultLang { get; set; } + + public int SupporterLength { get; set; } +} diff --git a/api/PasteMyst.Web/Services/AuthService.cs b/api/PasteMyst.Web/Services/AuthService.cs index 6c924d21..a08ce020 100644 --- a/api/PasteMyst.Web/Services/AuthService.cs +++ b/api/PasteMyst.Web/Services/AuthService.cs @@ -231,7 +231,7 @@ public async Task Logout(HttpContext httpContext, CancellationToken canc var accessToken = new AccessToken { - Id = await idProvider.GenerateId(async (id) => await AccessTokenExistsById(id)), + Id = await idProvider.GenerateId(AccessTokenExistsById), Token = hashStringBuilder.ToString(), Scopes = scopes, OwnerId = owner.Id, diff --git a/api/pastemyst.sln b/api/pastemyst.sln index b4c69aa2..a06cee24 100644 --- a/api/pastemyst.sln +++ b/api/pastemyst.sln @@ -1,5 +1,6 @@  Microsoft Visual Studio Solution File, Format Version 12.00 +# Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PasteMyst.Web", "PasteMyst.Web\PasteMyst.Web.csproj", "{213FA8A8-BB41-4E6C-94BD-31EE4EC37366}" EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PasteMyst.Web.Test.Unit", "PasteMyst.Web.Test.Unit\PasteMyst.Web.Test.Unit.csproj", "{933519F1-CADA-48CD-8CD4-744B81D50551}" @@ -8,6 +9,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Source", "Source", "{B97C05 EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Tests", "Tests", "{DB0EAFD1-8F6B-4F2D-98FD-49BB12D1EE1D}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PasteMyst.Migrator", "PasteMyst.Migrator\PasteMyst.Migrator.csproj", "{B0C53925-B3AC-4E4F-ACDD-CA4E0B9BAC7E}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -22,6 +25,10 @@ Global {933519F1-CADA-48CD-8CD4-744B81D50551}.Debug|Any CPU.Build.0 = Debug|Any CPU {933519F1-CADA-48CD-8CD4-744B81D50551}.Release|Any CPU.ActiveCfg = Release|Any CPU {933519F1-CADA-48CD-8CD4-744B81D50551}.Release|Any CPU.Build.0 = Release|Any CPU + {B0C53925-B3AC-4E4F-ACDD-CA4E0B9BAC7E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {B0C53925-B3AC-4E4F-ACDD-CA4E0B9BAC7E}.Debug|Any CPU.Build.0 = Debug|Any CPU + {B0C53925-B3AC-4E4F-ACDD-CA4E0B9BAC7E}.Release|Any CPU.ActiveCfg = Release|Any CPU + {B0C53925-B3AC-4E4F-ACDD-CA4E0B9BAC7E}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(NestedProjects) = preSolution {213FA8A8-BB41-4E6C-94BD-31EE4EC37366} = {B97C05A0-0F16-4EE4-B3D1-5329BFDF5592}