From 55d6c316e346d1841f4796188a351429110abb7c Mon Sep 17 00:00:00 2001 From: "jchande@panopto.com" Date: Thu, 9 Feb 2023 13:24:37 +0000 Subject: [PATCH] Added more verbose logging for during upload --- UpgradeLog.htm | Bin 0 -> 32342 bytes WatchFolderService/Common.cs | 70 +++++++++++-------- WatchFolderService/WatchFolderService.cs | 26 ++++--- WatchFolderService/WatchFolderService.csproj | 6 +- WatchFolderService/packages.config | 4 +- 5 files changed, 62 insertions(+), 44 deletions(-) create mode 100644 UpgradeLog.htm diff --git a/UpgradeLog.htm b/UpgradeLog.htm new file mode 100644 index 0000000000000000000000000000000000000000..fcb66de958547e98f16e941e5a17bbc9bc41c76d GIT binary patch literal 32342 zcmeI5`*YMtl7RK=UBvwly>L4)$M_A#Fw1QC3C6}?Y|Owiv0?ni#u&F>u)FuC-`)3B zxll`P%U1g_yK#h~TT(r;vNE$Wv$9Ik|NhUPvOi>F+4F2Po6p{4FS7Bhk*#NY*~e@v z`z8C2?B6p>ba6DYX116eWGmS+*Q?oX_K8xBY@e&G>``_*>*eUn?q)wy?mfBtz{zwnNHF_LvX4p+O>>ZR?(vj+8x$1`fIQEriv zv2G)4t*smtYsmSYPHkDA2`I>Fw%Wb_FvQ(0oP?9D5(k3Pu#7BJ~T*E z3L5<6_6GQRw@mI5R8}fXZvd;OR_n^T?bgBgZ?#INe!hpoOW9_iwc~4>b{~TA&1@nY zIR4$ue(ngiV?mkNJ%-qaE{MO04k1c+DcS`G7QpZe+2&3k)|mBPoRK(kPdycTO0-Iw4}S`ay}l!S6ofWHALM}uuhwGaPDjboa9m; z^xZ*^O6PdaQ_@vu5|PWV8u2UClbikYa|cRw2D9A=jy9`x?2Xe(`gPilJdU!^6|C_d zI7iFa2tANuH?d@W$G?8=wW3?}b(!AP)4)kXm9Gf{VIa@Vxyo`|=>E3FuPL9+Ycxcf!AAtEvb{ISXHJ`rNC;Fjfb4)wNm$;8s zxJgR=aom3m<2dI=>)7hchAuF+a+sIL;Uqon^v=C+dZ&@)*T0XHUxp+O1o^3!#3o+H z`(ta{DrZ`2^YqUf*b^mh2Z{WYeK?j#$C+M$7Qens4qU5DUcUU~vRRaO<)%|xa#@=3 zs%85;Hg6AEiJqL;=$Y#|)*DNmjsa(L(fTU6TO7{(;_S(}t-jG_kZAo`~FN8a#&bRDCaFH@Imp4zV6|#BCD+-p z(vZ(3wLljiiS4|IXx%0l;qg~@l-HNG7-^Y?!;$xDg6JPKIBVaGE9Vw3k3P2dD4e+& zHEo5X{8(=zXqksZBR$H|YCY~W@6*aguSq!02gO7V6n&Lg-l%=_&~Ks5^11IV(vxN~ zt{hcn=$-S=6gRH#4Rr^6BQ2+oDS7o4-kHM1qq9{2rX^)U!>x@N~{ipjA8`aCD{{|9LnU_0}lk)lnT| zMF~4KNgi`bk>I#ik~rI7jK|26nq>(H&Xc^(HJFkq^b}2YuHWQ{Yhux7Qd6i=EE_e9 zFpLM(n-g@0UbuSJ+9%`NS1*H!Ze_bKRT6dU$xHo_82LHR*i!?SDc#DS$j@?#+-r)K z!ZExLXN)JBw;AB>W6Se7v8%t)3vQj;#Tk-V)Q#$Nm++R8O4ZeK@DN#Erl0GZqohjPww3e4!+BBBZn&e`bZKQD2JlOr~+(&b#h{UhXV%w}LCM z&OwW5^<=QREq5@v=91P#9|k$OUO%lUpGFFz)W^L@xvXB*7`S?QjQ;2hwAMRTmydBB z*Vgqg9E-+G%`JL4AA=5ze#rY!>Jw7kO{qyrr>IIkKZ((iXb0Wx<9;vgu5hKKBQmam z8$Wgakt}a2rBBsL^NnDw-M7~Wb&Z?2t5JQ@U=>5m#VKB^J>t%>1TU_0E(aZyV{93F zY@{XbyQ!<+l3YjH{)I8(@hkAQ=0lQPmT@~INP=ACy884BF-fW6}YVKtlK0apX)hi@&od31s zw2^PQ!`Uoz)?s$rrt+)k?H>HX^=~PoaCrpIyypDuIMr2*SmQZ5ewa2FX!#I2zCTVM zL1M?@{5UNvfZJVaf5x+(M`B*_8|D8c`?yTIXvw{K8({T@^wH4v z5|AG-jwh&PPoGih9?*t?xkSHvXlI<hVTfjU7&Kv4H2ksyc*STJ! z#5#G?VDXglqtsmIZWm}`F$C@IQ|f?}Va|858GZ+WLI+$O0ZFWfz~(U+-=U9lK>0%H z5nvBz$q;{ARo3!s<*VkN6BXiS~Q5!8&ZkO86*r%{e?E~T? zwxQG#xV-=?TX!CCKdhuSq0L8X%z;fW(4_PM<+h>P3n=1@ZHlz7w0c0v8Cr5pc^Pai z{{oz*p_BADf)?^;nCn?EehclE`F{ock6a&-ZmYet@|b*k{}l|6_|4PGTiSR*{Ugde zA%B6~9VoTMxfh5J=;;(NzEGl&uMFp-hHI40ie9%JV?$gu&215rXMTd`xNdU@b3-~DR2iYhXdc9QfeMNr@%{J zW1PGL+B|^%YP2I@xL4#o7)?T}H(;x003N*k_s z_mHzr$~<-YXw^}$3Xbn7;~wW{^nRQBd(<2Q<~lt4zF>t{guKwr0^@>lMal6hKKC1o5My`XP<(4-gm=}_M+1bs9`e;jp6`=g*q>)8HI zz4_!-$6a*E)4CczC$&|yXmxrSBN)lLp7;FS!}(YDaFXN{rS!X}qX&1Ihjz7|&IlT` z?6EY_jnQNIgkGLS)4xQg-beEtq94Dap+BJ$KLqPIh4x)W(`}=@m-+3J^Ai2*lT1U@ zeuky!BV8}#4zLa=)g*0^{?5|&2wHlH`%&P(Mx%}ftE0A`1?Cbuc> z$pfH1Ag2eM4!BwaihH2-e#W?33N_cD@k?s!=d9A!2WpK_=LzupLXK4Z3_WM)!82gE z+hLJ>ZT=P%ouZA|U=b&=iQYHqlXJtzQ1t=5c@S)MFKxaB_9mr`u=arI5T#yne@LC# zK(ZhZKE} zSfuDcQ6AbwEQ<2;-H!i$c6=D!+eySiOQr5^jaclUE9cO?vuM>(&X$=#|K8<1jIJFa zZJL}T&b#RDp`fD|N%tvE=gp&N@;UVGOU_0i?sGSc&Yi{nXeHF@ugNpIHH$TPjkf+Z zXn(c-uUL&qaQZ^cdz?Sh$^fnJQP!F2b7~tgalURO<{l;A0$~8XKguyh{u`h;mS=;7 zy^DPurw5plF=y#|yO!^(5`0SZ8`vox38)-tb zO)QA9ffwL(0GvVk=4^Y1QetY1Wsow?o_oM(lRM!J(#9OUG5WC$oc++_d$jL*xE{n1 zwR zxbNXSKzSpWMs2pi$eaoK*#qPWD0P?PDftWJDO=JqS;rA_UQ**8RNVz9IqC7}8fU!u z%For<#vFc84}PELx`v)2@5=D^NPddJr?u2vcD~!h-_Ir*jV{k^KciLW(W$S{hT6QB z=r&`7hv@DXSdmxgxFNLiFk1SMf9Dka=*oS{8KE9VzdBcQhBS`mcAhhWjvPmus%M>b zO_1)YL@(_ukz$0{Xz@O^*U_)qQ`$jwQ z#vVFfoT5yV^Hf8m#0ia0NLGlNHF%V$xQ$t#fQQ{d?yiaRyxPJg%Ltr@!^tZrtwZT0>t|CtcOXDty zzQCJ9xh20THZCP@Ft-ABsw)4~yDK7xsS{uMl3BiN!f>D5cyt}5#D zNVPpmNcE5K`91Ko<%d{MciO)t{WZ{)1M~VwH=6&1vYS9M!mU>z{(a;arGE>CyObW~ zZj3(Y7wiGinduSz>*2RUsb;A67^-~$w*}f=B4v%7`_O!u8Z)##!XZ_*Xiqs5n+g4f#ux#DLG4AZ-VI<<&E4gQrlIh z7tqPRj?t&P^i03WU8N78;~ecPgPWwM<8tTDXUB(qlS#hgKfcMtx6&vzt_M3}^oM-U ziYv77oi88gf%_?41xRIYP}=xI>$_yE)xK=iTT-h0g!nYU9Jsjdx(5~A<#~1uSC)*u zxz=lqWQEh&Rr2`$33rhiz0y8Qfw*R#*C@Ypr3XH5P}cOruwy5#@~3Aj%Io~)`f=qy zu7CSq)fZ_VpOkOyzi~c&iG1ziQNX*HdYcuMM)o&;MaX^^aC=oYuze>=xYGrst9Cw*ucB2f0+0 zsp`spRS)C((`h4ag|>_i=yB@>o}`8QYI7>MQYb!hWd)wgyQ)jX`~ZPa_NJ3Cu+#K!%Waqo~;-bn0wevWRncD{2c?FV&- zkUGkmjuLfWuGfs38_`uheOf|2=x!XfK{eibLPhQ7PQQzNfyb4NuB~-t`_Mu2uUr@ej1~&Ps9&Z&7ZS@o<(@$FZO6#cing z2(QK+!5jSk#IJ~%Pb4|M^%mEU%iX~zs-{G2Rm*8s;dUVj1K z-`K?#DKt!J`>%8v!C66f`RrYkjPp_@ViT>vA9yR@Y1sUb{Qy4gbGyMoJF6G+JlKD!kJv~?3%~-O!Om%J*9w#+d^k{vjhO2U)!u-G; zH||+%<#*IBt=g+vN1kWArjc5tdX%j1`m4Q8s2X>dufGz9+Ks!@^=S3DvtHfte;s%?pKIXj0m`lBc0 z8intzI6s_f-E$u870Xm7&wayQu zcC>u)X&1}H=hE_>;Q8#d=%waw+D)@p^Q-VbsoPUc`|>47PK;+%%kz0Ow;W1iGN-`b zv%izx%TM3Lah~#uo?mtQk^BCi7n4+@EZwoU=a=p%J5@AJEj5TD`4qI#G(EKZ?Zfh2 z-uWhCITrctT+x&ww^LqGPSOm;H$UXFmi&zrMHuJ9HtF%W??1_Sv?V=AM#A{-->#bQ zGUEJW_730Z9sTw97kbo1%x{=~_fmHe*?Y(TJN)P0@a1n)wq2RucrM__?9TzlpDER@ zOoH=QN?G!!Q2HJJ?|}RRPaD3={*$K+-XUB6jn;aHguKI_`VGiFb>JFtwRXt)cRsqc zm#FU2$i5vk>K3uos5_K_Tg-oawsbv|T?OMC;Qj;uzl5`K$VIN5Pc+WJ`nq+Rv70sHMlAy)Co{5tWi z8^~lU-{Q=1lriG^NqkrT9bmSn*m+`3b+EbB7901!#?_BqR;7F{pd;N@DS zv7fW?s^(uc_p3O6a&*q&!8+c~iWU?l;QJ`Oo>7|nq%ndMXLzm8+}`>o?dA8-G=rSU z+ru-pLDG(OV@H=a7OVBU8tZ(TG*QgAAZf$n8+CjZ+BJdb+3Up}aMv39jC9J^N~`Ir zKj>UrPU!Rd@9L#rAlc&16^Go;rB#dSQEeYRE{6xXcD|^#5vOX6Q zO4Qcgxv#?A{a(j+>eT%@%0<47;`eak*HCO*9E@)tkfyHI$9IQYzP6pdXXAV0%XmHy zE5N%!wv6n|10#N4#aiXuNQeHmZI<6O6I*p{H?Bj8r}7ui>*4J8T;f$+6)R)+BIx<# zSLJJ3t(Mbvisi)eEV!`uiC;bPxoRcDsI7X<^*Qg$v)7yK-jOrz zUT_bz?;mre-nRloDUMHG`Gpr{PanY?rKGc(_S!kJ=Q4IuA>tj+owe|;L(IBEWX9*q zrPzmnVJkjmZlpNaC=`j>K_7lO(B|?eolCkj?+aKZ`dtLOUx@Rx^5P<}iV=~UfdcKJ z*OT+&TU#zJ>vKWEUH^?==ZHGh2ynFEWqn!J*X8}l5Otjq5x>BtKJ0YnqGzYB%OxT104UDFTlWI}9Avuv=*wl= zt7HKq&!Z>d^*XyyPx=dGJcjAxa>@Q`Q(C=+D0lh^2`2^>kY4s!hwiv(9jA^-x|S4Svx_3M5ZQUbgyDMN3?y zo)$pMXur!kq-ga|_mGO^;<(Pey5tks?^gcg)*<#G_xU@cN}@q0zQ^UsPu{CslB)IG zY5WkUqQ32nE|=v~JZw7;kG7)dsU@rQ3!~q;lB0ZwBkDhWwrIm#Pg)N;G3|qMR$GpE zIaiH)pyOJ&@&EX4DmkrY6^<$S8^hcK=AO9a@GdZS%cpl~_=1oN`&wJX#M6-_qp_Ul!NngvN*nq zv_Gk9(&|u|XPrw#Ydq+px+Y=w|5w)}3S6zOaU{0Rc6@t)@n}bP@wzo`o`1vqm+b!l DSsYZC literal 0 HcmV?d00001 diff --git a/WatchFolderService/Common.cs b/WatchFolderService/Common.cs index 73b8b73..5aca993 100644 --- a/WatchFolderService/Common.cs +++ b/WatchFolderService/Common.cs @@ -7,21 +7,19 @@ using Amazon.S3.Model; using System.Web; using System.Web.Script.Serialization; - +using System.ComponentModel; +using System.Diagnostics; namespace WatchFolderService { public class Common { // This is the target server url public static string server = "foo.bar.com"; - public static string uriStem = "https://" + server + "/Panopto/PublicAPI/REST"; public static readonly string AuthCookieName = ".ASPXAUTH"; - public static readonly string UploadBucketName = "Upload"; public static readonly string UploadTargetPathFragment_Panopto = "/Panopto/"; public static readonly string UploadTargetPathFragment_PanoptoUpload = "/Panopto/Upload/"; - /// /// Change the server that the program is directed towards /// @@ -165,7 +163,7 @@ public static HttpWebRequest CreateRequest( /// Text to parse. /// New instance of the desired object type or null if a null string was passed in. public static T ParseResponseBody( - string responseBody) where T: class, new() + string responseBody) where T : class, new() { if (responseBody == null) { @@ -214,7 +212,7 @@ public static string GetResponseBody( return reader.ReadToEnd(); } } - + /// /// Assert that a response body is parsable as the desired type. /// @@ -301,7 +299,7 @@ public static T GetResponse( /// A new S3 client. public static AmazonS3Client CreateS3Client( string uploadTarget, - string accessKeyId = "foo", + string accessKeyId = "foo", string secretAccessKey = "bar") { if (uploadTarget == null) @@ -359,7 +357,7 @@ public static InitiateMultipartUploadResponse OpenUpload( // AWS SDK for .NET 4.5 default content-length is 0. // Set this to zero so it's always valid. // - + initiateRequest.Headers.ContentLength = 0; return s3Client.InitiateMultipartUpload(initiateRequest); } @@ -375,11 +373,13 @@ public static InitiateMultipartUploadResponse OpenUpload( /// List of upload part responses from the server. public static List UploadParts( AmazonS3Client s3Client, - string uploadTarget, + string uploadTarget, string fileName, string uploadId, long partSize) { + + if (s3Client == null || uploadTarget == null || fileName == null || uploadId == null || partSize <= 0) throw new InvalidDataException(); @@ -390,21 +390,35 @@ public static List UploadParts( long filePosition = 0; for (int i = 1; filePosition < fileSize; i++) { - UploadPartRequest uploadRequest = new UploadPartRequest + try + { + UploadPartRequest uploadRequest = new UploadPartRequest + { + BucketName = Common.UploadBucketName, + Key = fileKey, + UploadId = uploadId, + PartNumber = i, + PartSize = partSize, + FilePosition = filePosition, + FilePath = fileName + }; + + // add the response to the list since it will be needed to complete the upload + uploadResponses.Add(s3Client.UploadPart(uploadRequest)); + + filePosition += partSize; + if (WatchFolderService.verbose) + { + WatchFolderService.eventLog.WriteEntry($"Part number {i} has uploaded successfully for {fileName}. {filePosition} of {fileSize} uploaded", EventLogEntryType.Information); + } + } + catch (Exception ex) { - BucketName = Common.UploadBucketName, - Key = fileKey, - UploadId = uploadId, - PartNumber = i, - PartSize = partSize, - FilePosition = filePosition, - FilePath = fileName - }; - - // add the response to the list since it will be needed to complete the upload - uploadResponses.Add(s3Client.UploadPart(uploadRequest)); - - filePosition += partSize; + if (WatchFolderService.verbose) + { + WatchFolderService.eventLog.WriteEntry($"Part number {i} failed to upload for {fileName}. {filePosition} of {fileSize} uploaded. {ex}", EventLogEntryType.Warning); + } + } } return uploadResponses; @@ -435,7 +449,7 @@ public static CompleteMultipartUploadResponse CloseUpload( BucketName = Common.UploadBucketName, Key = fileKey, UploadId = uploadId, - + }; completeRequest.AddPartETags(partResponses); @@ -489,10 +503,10 @@ private static string GetServiceUrlFromUploadTarget( int i = uploadTarget.IndexOf(Common.UploadTargetPathFragment_Panopto); if (i < 0) - throw new InvalidDataException(); + throw new InvalidDataException(); string result = uploadTarget.Substring( - 0, + 0, i + Common.UploadTargetPathFragment_Panopto.Length); return result; @@ -543,7 +557,7 @@ private static string GetFileKey(string uploadTarget, string fileName) throw new InvalidDataException(); return Path.Combine( - GetFileKeyPrefixFromUploadTarget(uploadTarget), + GetFileKeyPrefixFromUploadTarget(uploadTarget), Path.GetFileName(fileName)); } @@ -559,7 +573,7 @@ private static string GetFileKey(string uploadTarget, string fileName) public static T CreateRestObject( string authCookie, string noun, - T value) where T : BaseObject, new() + T value) where T : BaseObject, new() { HttpWebRequest request = Common.CreateRequest( "POST", diff --git a/WatchFolderService/WatchFolderService.cs b/WatchFolderService/WatchFolderService.cs index 9c41e24..c548db3 100644 --- a/WatchFolderService/WatchFolderService.cs +++ b/WatchFolderService/WatchFolderService.cs @@ -35,7 +35,9 @@ public partial class WatchFolderService : ServiceBase private bool inputValid = true; private string inputFailureMessage = ""; private int maxNumberOfAttempts = 3; - private bool verbose = false; + public static bool verbose = false; + public static EventLog eventLog = new EventLog(); + // This name must be something different from "PanoptoWatchFolderService". // That name was once used associated with custom log and the association seems @@ -48,16 +50,18 @@ public WatchFolderService() InitializeComponent(); // Setup event log - this.AutoLog = true; - ((ISupportInitialize)this.EventLog).BeginInit(); + AutoLog = true; + + ((ISupportInitialize)EventLog).BeginInit(); if (!EventLog.SourceExists(WatchFolderService.EventLogSourceName)) { EventLog.CreateEventSource(WatchFolderService.EventLogSourceName, "Application"); } - ((ISupportInitialize)this.EventLog).EndInit(); + ((ISupportInitialize)EventLog).EndInit(); + + eventLog.Source = WatchFolderService.EventLogSourceName; + eventLog.Log = "Application"; - this.EventLog.Source = WatchFolderService.EventLogSourceName; - this.EventLog.Log = "Application"; // Parse config file server = ConfigurationManager.AppSettings["Server"]; @@ -484,7 +488,7 @@ private Dictionary GetLastSyncInfo() folderInfo.Add(info[0], syncInfo); } - + return folderInfo; } @@ -500,7 +504,7 @@ private void SetSyncInfo(Dictionary info) { string line = fileName + ";" - + info[fileName].LastSyncWriteTime.ToString(DATETIME_FORMAT, CultureInfo.InvariantCulture) + ";" + + info[fileName].LastSyncWriteTime.ToString(DATETIME_FORMAT, CultureInfo.InvariantCulture) + ";" + info[fileName].NewSyncWriteTime.ToString(DATETIME_FORMAT, CultureInfo.InvariantCulture) + ";" + info[fileName].FileStableTime + ";" + info[fileName].NumberOfAttempts; @@ -509,7 +513,7 @@ private void SetSyncInfo(Dictionary info) { this.EventLog.WriteEntry("Writing to InfoFile: " + line, EventLogEntryType.Information); } - + infoFile.WriteLine(line); } } @@ -522,7 +526,7 @@ private void SetSyncInfo(Dictionary info) /// DateTime created from timeString private DateTime GetDateTime(string timeString) { - return DateTime.ParseExact(timeString,DATETIME_FORMAT,CultureInfo.InvariantCulture); + return DateTime.ParseExact(timeString, DATETIME_FORMAT, CultureInfo.InvariantCulture); } /// @@ -548,7 +552,7 @@ private FileInfo[] GetFileInfo(DirectoryInfo dirInfo) } FileInfo[] result = new FileInfo[resultArray.Count]; - int i = 0; + int i = 0; foreach (FileInfo fileInfo in resultArray) { result[i] = fileInfo; diff --git a/WatchFolderService/WatchFolderService.csproj b/WatchFolderService/WatchFolderService.csproj index 687b514..e76f066 100644 --- a/WatchFolderService/WatchFolderService.csproj +++ b/WatchFolderService/WatchFolderService.csproj @@ -53,10 +53,10 @@ - ..\packages\AWSSDK.Core.3.5.1.2\lib\net45\AWSSDK.Core.dll + ..\packages\AWSSDK.Core.3.7.104.5\lib\net45\AWSSDK.Core.dll - ..\packages\AWSSDK.S3.3.5.0.4\lib\net45\AWSSDK.S3.dll + ..\packages\AWSSDK.S3.3.7.0\lib\net45\AWSSDK.S3.dll @@ -121,7 +121,7 @@ - +