feat: allow asking for using xref streams, which opens the option to …

…build very large files (#291)

PDFWriter/DocumentContext.cpp

@@ -60,6 +60,7 @@ DocumentContext::DocumentContext()
 	mObjectsContext = NULL;
 	mParserExtender = NULL;
     mModifiedDocumentIDExists = false;
+	SetWriteXrefAsXrefStream(false);
 }
 
 DocumentContext::~DocumentContext(void)
@@ -80,12 +81,26 @@ void DocumentContext::SetObjectsContext(ObjectsContext* inObjectsContext)
 	mPNGImageHandler.SetOperationsContexts(this, mObjectsContext);
 #endif
 	mExtGStateRegistry.SetObjectsContext(mObjectsContext);
+	SetupXrefMaxWritePositionValidation();
+}
+
+void DocumentContext::SetupXrefMaxWritePositionValidation()
+{
+	// Validating Max Xref position to be 10 digits long is only relevant for regular xref writing.
+	// Cancel validation if xref stream is used instead.
+	if(mObjectsContext)
+		mObjectsContext->GetInDirectObjectsRegistry().SetShouldValidateMaxWritePositionForXref(!mWriteXrefAsXrefStream);
 }
 
 void DocumentContext::SetEmbedFonts(bool inEmbedFonts) {
 	mUsedFontsRepository.SetEmbedFonts(inEmbedFonts);
 }
 
+void DocumentContext::SetWriteXrefAsXrefStream(bool inWriteXrefAsXrefStream) {
+	mWriteXrefAsXrefStream = inWriteXrefAsXrefStream;
+	SetupXrefMaxWritePositionValidation();
+}
+
 void DocumentContext::SetOutputFileInformation(OutputFile* inOutputFile)
 {
 	// just save the output file path for the ID generation in the end
@@ -222,13 +237,20 @@ EStatusCode	DocumentContext::FinalizeNewPDF()
 		if (status != eSuccess)
 			break;
 
-		status = mObjectsContext->WriteXrefTable(xrefTablePosition);
-		if(status != eSuccess)
-			break;
+		if(mWriteXrefAsXrefStream) {
+			status = WriteXrefStream(xrefTablePosition);
+            if(status != eSuccess)
+                break;			
+		} else {
+			status = mObjectsContext->WriteXrefTable(xrefTablePosition);
+			if(status != eSuccess)
+				break;
 
-		status = WriteTrailerDictionary();
-		if(status != eSuccess)
-			break;
+			status = WriteTrailerDictionary();
+			if(status != eSuccess)
+				break;
+
+		}
 
 		WriteXrefReference(xrefTablePosition);
 		WriteFinalEOF();
@@ -2343,8 +2365,31 @@ void DocumentContext::UnRegisterCopyingContext(PDFDocumentCopyingContext* inCopy
 	mCopyingContexts.erase(inCopyingContext);
 }
 
+
+bool DocumentContext::RequiresXrefStream(PDFParser* inModifiedFileParser)
+{
+    // modification requires xref stream if the original document uses one...so just ask trailer
+    if(!inModifiedFileParser->GetTrailer())
+        return false;
+
+    PDFObjectCastPtr<PDFName> typeObject = inModifiedFileParser->GetTrailer()->QueryDirectObject("Type");
+
+    if(!typeObject)
+        return false;
+
+    return typeObject->GetValue() == "XRef";
+
+
+}
+
+
 EStatusCode DocumentContext::SetupModifiedFile(PDFParser* inModifiedFileParser)
 {
+	// determine if file requires xref stream, in which case set it up
+	if(RequiresXrefStream(inModifiedFileParser)) {
+		SetWriteXrefAsXrefStream(true); // it may already have been setup to be true earlier by the users request, but if not, and this file requires it, set it up now
+	}		
+
     // setup trailer and save original document ID
 
     if(!inModifiedFileParser->GetTrailer())
@@ -2514,9 +2559,11 @@ EStatusCode	DocumentContext::FinalizeModifiedPDF(PDFParser* inModifiedFileParser
 		status = CopyEncryptionDictionary(inModifiedFileParser);
         if(status != eSuccess)
             break;
-        if(RequiresXrefStream(inModifiedFileParser))
+        if(mWriteXrefAsXrefStream)
         {
             status = WriteXrefStream(xrefTablePosition);
+            if(status != eSuccess)
+                break;			
         }
         else
         {
@@ -2781,21 +2828,6 @@ EStatusCode DocumentContext::CopyEncryptionDictionary(PDFParser* inModifiedFileP
 	return eSuccess;
 }
 
-bool DocumentContext::RequiresXrefStream(PDFParser* inModifiedFileParser)
-{
-    // modification requires xref stream if the original document uses one...so just ask trailer
-    if(!inModifiedFileParser->GetTrailer())
-        return false;
-
-    PDFObjectCastPtr<PDFName> typeObject = inModifiedFileParser->GetTrailer()->QueryDirectObject("Type");
-
-    if(!typeObject)
-        return false;
-
-    return typeObject->GetValue() == "XRef";
-
-
-}
 
 EStatusCode DocumentContext::WriteXrefStream(LongFilePositionType& outXrefPosition)
 {

PDFWriter/DocumentContext.h

@@ -119,6 +119,7 @@ namespace PDFHummus
 		void SetObjectsContext(ObjectsContext* inObjectsContext);
 		void SetOutputFileInformation(OutputFile* inOutputFile);
 		void SetEmbedFonts(bool inEmbedFonts);
+		void SetWriteXrefAsXrefStream(bool inWriteXrefAsXrefStream);
 		PDFHummus::EStatusCode	WriteHeader(EPDFVersion inPDFVersion);
 		PDFHummus::EStatusCode	FinalizeNewPDF();
         PDFHummus::EStatusCode	FinalizeModifiedPDF(PDFParser* inModifiedFileParser,EPDFVersion inModifiedPDFVersion);
@@ -413,6 +414,7 @@ namespace PDFHummus
 	    StringAndULongPairToHummusImageInformationMap mImagesInformation;
 		EncryptionHelper mEncryptionHelper;
 		ExtGStateRegistry mExtGStateRegistry;
+		bool mWriteXrefAsXrefStream;
 
 		void WriteHeaderComment(EPDFVersion inPDFVersion);
 		void Write4BinaryBytes();
@@ -461,5 +463,6 @@ namespace PDFHummus
 		bool RequiresXrefStream(PDFParser* inModifiedFileParser);
         PDFHummus::EStatusCode WriteXrefStream(LongFilePositionType& outXrefPosition);
 		HummusImageInformation& GetImageInformationStructFor(const std::string& inImageFile,unsigned long inImageIndex);
+		void SetupXrefMaxWritePositionValidation();
 	};
 }

PDFWriter/IndirectObjectsReferenceRegistry.cpp

@@ -37,6 +37,7 @@ using namespace PDFHummus;
 IndirectObjectsReferenceRegistry::IndirectObjectsReferenceRegistry(void)
 {
     SetupInitialFreeObject();
+	SetShouldValidateMaxWritePositionForXref(true);
 }
 
 void IndirectObjectsReferenceRegistry::SetupInitialFreeObject()
@@ -51,6 +52,11 @@ void IndirectObjectsReferenceRegistry::SetupInitialFreeObject()
 	mObjectsWritesRegistry.push_back(singleFreeObjectInformation);
 }
 
+void IndirectObjectsReferenceRegistry::SetShouldValidateMaxWritePositionForXref(bool inShouldValidate)
+{
+	mShouldValidateMaxWritePositionForXref = inShouldValidate;
+}
+
 IndirectObjectsReferenceRegistry::~IndirectObjectsReferenceRegistry(void)
 {
 }
@@ -70,6 +76,20 @@ ObjectIDType IndirectObjectsReferenceRegistry::AllocateNewObjectID()
 	return newObjectID;
 }
 
+EStatusCode IndirectObjectsReferenceRegistry::MaybeValidateMaxWritePositionForXref(LongFilePositionType inWritePosition)
+{
+	if(!mShouldValidateMaxWritePositionForXref)
+		return PDFHummus::eSuccess;
+
+	if(inWritePosition > 9999999999LL) // if write position is larger than what can be represented by 10 digits, xref write will fail
+	{
+		TRACE_LOG1("IndirectObjectsReferenceRegistry::MaybeValidateMaxWritePositionForXref, Write position out of bounds. Trying to write an object at position that cannot be represented in Xref = %lld. probably means file got too long",inWritePosition);
+		return PDFHummus::eFailure;
+	}
+
+	return PDFHummus::eSuccess;	
+}
+
 
 EStatusCode IndirectObjectsReferenceRegistry::MarkObjectAsWritten(ObjectIDType inObjectID,LongFilePositionType inWritePosition)
 {
@@ -86,9 +106,8 @@ EStatusCode IndirectObjectsReferenceRegistry::MarkObjectAsWritten(ObjectIDType i
 		return PDFHummus::eFailure; // trying to mark as written an object that was already marked as such in the past. probably a mistake [till we have revisions]
 	}
 
-	if(inWritePosition > 9999999999LL) // if write position is larger than what can be represented by 10 digits, xref write will fail
+	if(MaybeValidateMaxWritePositionForXref(inWritePosition) != PDFHummus::eSuccess) // if write position is larger than what can be represented by 10 digits, xref write will fail
 	{
-		TRACE_LOG1("IndirectObjectsReferenceRegistry::MarkObjectAsWritten, Write position out of bounds. Trying to write an object at position that cannot be represented in Xref = %lld. probably means file got too long",inWritePosition);
 		return PDFHummus::eFailure;
 	}
 
@@ -155,12 +174,10 @@ PDFHummus::EStatusCode IndirectObjectsReferenceRegistry::MarkObjectAsUpdated(Obj
 		return PDFHummus::eFailure; 
 	}
 
-	if(inNewWritePosition > 9999999999LL) // if write position is larger than what can be represented by 10 digits, xref write will fail
+	if(MaybeValidateMaxWritePositionForXref(inNewWritePosition) != PDFHummus::eSuccess) // if write position is larger than what can be represented by 10 digits, xref write will fail
 	{
-		TRACE_LOG1("IndirectObjectsReferenceRegistry::MarkObjectAsUpdated, Write position out of bounds. Trying to write an object at position that cannot be represented in Xref = %lld. probably means file got too long",inNewWritePosition);
 		return PDFHummus::eFailure;
-	}
-
+	}	
 
     mObjectsWritesRegistry[inObjectID].mIsDirty = true;
     mObjectsWritesRegistry[inObjectID].mWritePosition = inNewWritePosition;

PDFWriter/IndirectObjectsReferenceRegistry.h

@@ -89,12 +89,16 @@ class IndirectObjectsReferenceRegistry
 	void Reset();
 
     void SetupXrefFromModifiedFile(PDFParser* inModifiedFileParser);
+
+	void SetShouldValidateMaxWritePositionForXref(bool inShouldValidateMaxWritePositionForXref);
 
 private:
 	ObjectWriteInformationVector mObjectsWritesRegistry;
+	bool mShouldValidateMaxWritePositionForXref;
 
     void SetupInitialFreeObject();
     void AppendExistingItem(ObjectWriteInformation::EObjectReferenceType inObjectReferenceType,
                             unsigned long inGenerationNumber,
                             LongFilePositionType inWritePosition);
+	PDFHummus::EStatusCode MaybeValidateMaxWritePositionForXref(LongFilePositionType inWritePosition);
 };

PDFWriter/PDFWriter.cpp

@@ -56,8 +56,9 @@ PDFWriter::~PDFWriter(void)
 {
 }
 
-EPDFVersion thisOrDefaultVersion(EPDFVersion inPDFVersion) {
-	return ePDFVersionUndefined == inPDFVersion ? ePDFVersion14 : inPDFVersion;
+EPDFVersion thisOrDefaultVersion(EPDFVersion inPDFVersion, bool inWriteXrefAsXrefStream) {
+	// if version is undefined, return 1.4 if xref stream is not used, 1.5 if it is (As this would be the lower version it's supported in)
+	return ePDFVersionUndefined == inPDFVersion ? (inWriteXrefAsXrefStream ? ePDFVersion15: ePDFVersion14) : inPDFVersion;
 }
 
 EStatusCode PDFWriter::StartPDF(
@@ -66,6 +67,7 @@ EStatusCode PDFWriter::StartPDF(
 							const LogConfiguration& inLogConfiguration,
 							const PDFCreationSettings& inPDFCreationSettings)
 {
+	EPDFVersion pdfVersion = thisOrDefaultVersion(inPDFVersion, inPDFCreationSettings.WriteXrefAsXrefStream);
 	SetupLog(inLogConfiguration);
 	SetupCreationSettings(inPDFCreationSettings);
 
@@ -77,7 +79,7 @@ EStatusCode PDFWriter::StartPDF(
 	mDocumentContext.SetOutputFileInformation(&mOutputFile);
 
 	if (inPDFCreationSettings.DocumentEncryptionOptions.ShouldEncrypt) {
-		mDocumentContext.SetupEncryption(inPDFCreationSettings.DocumentEncryptionOptions, thisOrDefaultVersion(inPDFVersion));
+		mDocumentContext.SetupEncryption(inPDFCreationSettings.DocumentEncryptionOptions, pdfVersion);
 		if (!mDocumentContext.SupportsEncryption()) {
 			mOutputFile.CloseFile(); // close the file, to keep things clean
 			return eFailure;
@@ -86,7 +88,7 @@ EStatusCode PDFWriter::StartPDF(
 
 	mIsModified = false;
 
-	return mDocumentContext.WriteHeader(thisOrDefaultVersion(inPDFVersion));
+	return mDocumentContext.WriteHeader(pdfVersion);
 }
 
 EStatusCode PDFWriter::EndPDF()
@@ -174,6 +176,7 @@ void PDFWriter::SetupCreationSettings(const PDFCreationSettings& inPDFCreationSe
 {
 	mObjectsContext.SetCompressStreams(inPDFCreationSettings.CompressStreams);
 	mDocumentContext.SetEmbedFonts(inPDFCreationSettings.EmbedFonts);
+	mDocumentContext.SetWriteXrefAsXrefStream(inPDFCreationSettings.WriteXrefAsXrefStream);
 }
 
 void PDFWriter::ReleaseLog()
@@ -555,18 +558,19 @@ EStatusCode PDFWriter::StartPDFForStream(IByteWriterWithPosition* inOutputStream
 										 const LogConfiguration& inLogConfiguration,
 										 const PDFCreationSettings& inPDFCreationSettings)
 {
+	EPDFVersion pdfVersion = thisOrDefaultVersion(inPDFVersion, inPDFCreationSettings.WriteXrefAsXrefStream);
 	SetupLog(inLogConfiguration);
 	SetupCreationSettings(inPDFCreationSettings);
 	if (inPDFCreationSettings.DocumentEncryptionOptions.ShouldEncrypt) {
-		mDocumentContext.SetupEncryption(inPDFCreationSettings.DocumentEncryptionOptions, thisOrDefaultVersion(inPDFVersion));
+		mDocumentContext.SetupEncryption(inPDFCreationSettings.DocumentEncryptionOptions, pdfVersion);
 		if (!mDocumentContext.SupportsEncryption())
 			return eFailure;
 	}
 
 	mObjectsContext.SetOutputStream(inOutputStream);
     mIsModified = false;
 
-	return mDocumentContext.WriteHeader(thisOrDefaultVersion(inPDFVersion));
+	return mDocumentContext.WriteHeader(pdfVersion);
 }
 EStatusCode PDFWriter::EndPDFForStream()
 {
@@ -691,7 +695,7 @@ EStatusCode PDFWriter::ModifyPDF(const std::string& inModifiedFile,
 
         // do setup for modification
         mIsModified = true;
-        status = SetupStateFromModifiedFile(inModifiedFile, thisOrDefaultVersion(inPDFVersion), inPDFCreationSettings);
+        status = SetupStateFromModifiedFile(inModifiedFile, inPDFVersion, inPDFCreationSettings);
     }
     while (false);
 
@@ -724,13 +728,14 @@ EStatusCode PDFWriter::ModifyPDFForStream(
 
     mIsModified = true;
 
-    return SetupStateFromModifiedStream(inModifiedSourceStream, thisOrDefaultVersion(inPDFVersion), inPDFCreationSettings);
+    return SetupStateFromModifiedStream(inModifiedSourceStream, inPDFVersion, inPDFCreationSettings);
 }
 
 EStatusCode PDFWriter::SetupStateFromModifiedStream(IByteReaderWithPosition* inModifiedSourceStream,
                                                     EPDFVersion inPDFVersion,
 													const PDFCreationSettings& inPDFCreationSettings)
 {
+	EPDFVersion pdfVersion = thisOrDefaultVersion(inPDFVersion, inPDFCreationSettings.WriteXrefAsXrefStream);
     EStatusCode status;
 	PDFParsingOptions parsingOptions;
 
@@ -761,7 +766,7 @@ EStatusCode PDFWriter::SetupStateFromModifiedStream(IByteReaderWithPosition* inM
 			}
 		}
 
-        mModifiedFileVersion = thisOrDefaultVersion(inPDFVersion);
+        mModifiedFileVersion = pdfVersion;
     }
     while (false);
 
@@ -778,7 +783,7 @@ EStatusCode PDFWriter::SetupStateFromModifiedFile(const std::string& inModifiedF
         if(status != eSuccess)
             break;
 
-        status = SetupStateFromModifiedStream(mModifiedFile.GetInputStream(), thisOrDefaultVersion(inPDFVersion), inPDFCreationSettings);
+        status = SetupStateFromModifiedStream(mModifiedFile.GetInputStream(), inPDFVersion, inPDFCreationSettings);
     }
     while(false);
 

PDFWriter/PDFWriter.h

@@ -59,10 +59,16 @@ struct PDFCreationSettings
 	bool CompressStreams;
 	bool EmbedFonts;
 	EncryptionOptions DocumentEncryptionOptions;
+	bool WriteXrefAsXrefStream;
 
-	PDFCreationSettings(bool inCompressStreams, bool inEmbedFonts,EncryptionOptions inDocumentEncryptionOptions = EncryptionOptions::DefaultEncryptionOptions()):DocumentEncryptionOptions(inDocumentEncryptionOptions){
+	PDFCreationSettings(
+		bool inCompressStreams, 
+		bool inEmbedFonts,
+		EncryptionOptions inDocumentEncryptionOptions = EncryptionOptions::DefaultEncryptionOptions(), 
+		bool inWriteXrefAsXrefStream = false):DocumentEncryptionOptions(inDocumentEncryptionOptions){
 		CompressStreams = inCompressStreams;
 		EmbedFonts = inEmbedFonts;
+		WriteXrefAsXrefStream = inWriteXrefAsXrefStream;
 	}
 
 };

PDFWriterTesting/CMakeLists.txt

@@ -76,6 +76,7 @@ create_test_sourcelist (Tests
   UppercaseSequanceTest.cpp
   WatermarkTest.cpp
   WatermarkWithContextOpacityTest.cpp
+  XrefStreamsTest.cpp
 )
 
 # add the testing executable