#391: Fix reading files with invalid encoding (#392)

Co-authored-by: kaklakariada <[email protected]>
itsallcode · Feb 26, 2024 · ea8baaf · ea8baaf
1 parent 966668b
commit ea8baaf
Show file tree

Hide file tree

Showing 9 changed files with 57 additions and 20 deletions.
diff --git a/README.md b/README.md
@@ -68,10 +68,10 @@ OpenFastTrace at it's core is a Java Archive (short "[JAR](https://docs.oracle.c
 
 ### Getting Pre-Built Packages
 
-Pre-Built JAR files (called `openfasttrace-3.7.1.jar`) are available from the following places:
+Pre-Built JAR files (called `openfasttrace-3.8.0.jar`) are available from the following places:
 
-* [Maven Central](https://repo1.maven.org/maven2/org/itsallcode/openfasttrace/openfasttrace/3.7.1/openfasttrace-3.7.1.jar)
-* [GitHub](https://github.com/itsallcode/openfasttrace/releases/download/3.7.1/openfasttrace-3.7.1.jar)
+* [Maven Central](https://repo1.maven.org/maven2/org/itsallcode/openfasttrace/openfasttrace/3.8.0/openfasttrace-3.8.0.jar)
+* [GitHub](https://github.com/itsallcode/openfasttrace/releases/download/3.8.0/openfasttrace-3.8.0.jar)
 
 Check our [developer guide](doc/developer_guide.md#getting-the-openfasttrace-library) to learn how to use the OFT JAR as dependency in your own code with popular build tools.
 
@@ -94,7 +94,7 @@ If you just want to run OFT:
 The most basic variant to run OpenFastTrace is directly from the JAR file via the command line:
 
 ```bash
-java -jar product/target/openfasttrace-3.7.1.jar trace /path/to/directory/being/traced
+java -jar product/target/openfasttrace-3.8.0.jar trace /path/to/directory/being/traced
 ```
 
 If you want to run OFT automatically as part of a continuous build, we recommend using our plugins for [Gradle](https://github.com/itsallcode/openfasttrace-gradle) and [Maven](https://github.com/itsallcode/openfasttrace-maven-plugin).

diff --git a/api/src/main/java/org/itsallcode/openfasttrace/api/importer/input/RealFileInput.java b/api/src/main/java/org/itsallcode/openfasttrace/api/importer/input/RealFileInput.java
@@ -1,7 +1,6 @@
 package org.itsallcode.openfasttrace.api.importer.input;
 
-import java.io.BufferedReader;
-import java.io.IOException;
+import java.io.*;
 import java.nio.charset.Charset;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
@@ -51,7 +50,12 @@ public static InputFile forPath(final Path path, final Charset charset)
     @Override
     public BufferedReader createReader() throws IOException
     {
-        return Files.newBufferedReader(this.path, this.charset);
+        // Don't fail when reading files with invalid encoding.
+        // Files.newBufferedReader(this.path, this.charset) won't work here.
+        // See https://stackoverflow.com/a/43446789 for details.
+        final InputStream stream = Files.newInputStream(this.path);
+        final InputStreamReader reader = new InputStreamReader(stream, this.charset);
+        return new BufferedReader(reader);
     }
 
     @Override

diff --git a/api/src/test/java/org/itsallcode/openfasttrace/api/importer/input/TestRealFileInput.java b/api/src/test/java/org/itsallcode/openfasttrace/api/importer/input/TestRealFileInput.java
@@ -8,9 +8,7 @@
 import java.io.IOException;
 import java.nio.charset.Charset;
 import java.nio.charset.StandardCharsets;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
+import java.nio.file.*;
 
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
@@ -100,6 +98,15 @@ void testReadWithUtf8Encoding() throws IOException
         assertThat(readContent(inputFile), equalTo(CONTENT));
     }
 
+    @Test
+    void testReadFileWithInvalidEncoding2() throws IOException
+    {
+        final Path path = tempDir.resolve("file");
+        Files.write(path, new byte[] { (byte) 0x9F, (byte) 0x88 });
+        final InputFile inputFile = RealFileInput.forPath(path, StandardCharsets.UTF_8);
+        assertThat(readContent(inputFile), equalTo("��"));
+    }
+
     private Path writeTempFile(final String content, final Charset charset) throws IOException
     {
         final Path path = this.tempDir.resolve("test");

diff --git a/doc/changes/changes_3.8.0.md b/doc/changes/changes_3.8.0.md
@@ -1,4 +1,4 @@
-# OpenFastTrace 3.8.0, released 2024-02-??
+# OpenFastTrace 3.8.0, released 2024-02-26
 
 Code name: RST Importer
 
@@ -29,6 +29,7 @@ Now you can also specify a revision for coverage tags instead of the default rev
 * #373: Ignore spaces after items in "Needs:" and "Tags:" lists (thanks to [@sambishop](https://github.com/sambishop) for his contribution!)
 * #378: Merged integration test coverage with unit test coverage for representative overall figure
 * #303: Escape special characters in HTML report to avoid broken HTML when a specification item contains text like `<section>`
+* #391: Fixed reading files with invalid encoding (thanks to [@ayankuma](https://github.com/ayankuma) for the bug report!)
 
 ## Refactoring
 

diff --git a/doc/developer_guide.md b/doc/developer_guide.md
@@ -15,7 +15,7 @@ To use OpenFastTrace as a dependency in your [Maven](https://maven.apache.org) p
     <dependency>
         <groupId>org.itsallcode.openfasttrace</groupId>
         <artifactId>openfasttrace</artifactId>
-        <version>3.7.1</version>
+        <version>3.8.0</version>
         <scope>compile</scope>
     </dependency>
 </dependencies>
@@ -27,7 +27,7 @@ To use OpenFastTrace as a dependency in your [Gradle](https://gradle.org/) proje
 
 ```groovy
 dependencies {
-    compile "org.itsallcode.openfasttrace:openfasttrace:3.7.1"
+    compile "org.itsallcode.openfasttrace:openfasttrace:3.8.0"
 }
 ```
 

diff --git a/importer/zip/src/test/java/org/itsallcode/openfasttrace/importer/zip/ITZipFileImporter.java b/importer/zip/src/test/java/org/itsallcode/openfasttrace/importer/zip/ITZipFileImporter.java
@@ -133,6 +133,16 @@ void testImportZipWithMultipleFilesAssertFileContent() throws IOException
         assertThat(this.actualFileContent.get(1), equalTo(FILE_CONTENT2_STRING));
     }
 
+    @Test
+    void testImportZipWithInvalidEncoding() throws IOException
+    {
+        initializeZipFile();
+        addEntryToZip("file1.c", new byte[] { (byte) 0x9F, (byte) 0x88 });
+        final List<InputFile> importedFiles = runImporter(1);
+        assertThat(importedFiles.get(0).getPath(), equalTo(this.zipFile.getPath() + "!file1.c"));
+        assertThat(this.actualFileContent.get(0), equalTo("��"));
+    }
+
     private void addZipEntryDirectory(final String name) throws IOException
     {
         assertThat(name, not(Matchers.endsWith("/")));

diff --git a/.../zip/src/test/java/org/itsallcode/openfasttrace/importer/zip/input/TestZipEntryInput.java b/.../zip/src/test/java/org/itsallcode/openfasttrace/importer/zip/input/TestZipEntryInput.java
@@ -5,14 +5,10 @@
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.junit.jupiter.api.Assertions.assertThrows;
 
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
+import java.io.*;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Path;
-import java.util.zip.ZipEntry;
-import java.util.zip.ZipFile;
-import java.util.zip.ZipOutputStream;
+import java.util.zip.*;
 
 import org.itsallcode.openfasttrace.api.importer.ImporterException;
 import org.itsallcode.openfasttrace.api.importer.input.InputFile;
@@ -118,6 +114,18 @@ void testReadContentIso() throws IOException
         }
     }
 
+    @Test
+    void testReadInvalidEncoding() throws IOException
+    {
+        addEntryToZip("file", new byte[] { (byte) 0x9F, (byte) 0x88 });
+        try (final ZipFile zip = getZipFile())
+        {
+            final InputFile inputFile = ZipEntryInput.forZipEntry(zip, new ZipEntry("file"),
+                    StandardCharsets.UTF_8);
+            assertThat(readContent(inputFile), equalTo("��"));
+        }
+    }
+
     private String readContent(final InputFile inputFile) throws IOException
     {
         return inputFile.createReader().lines().collect(joining("\n"));

diff --git a/parent/pom.xml b/parent/pom.xml
@@ -8,7 +8,7 @@
     <description>Free requirement tracking suite</description>
     <url>https://github.com/itsallcode/openfasttrace</url>
     <properties>
-        <revision>3.7.2</revision>
+        <revision>3.8.0</revision>
         <java.version>11</java.version>
         <junit.version>5.10.2</junit.version>
         <maven.surefire.version>3.2.5</maven.surefire.version>

diff --git a/product/src/test/resources/example/src/invalid_encoding.c b/product/src/test/resources/example/src/invalid_encoding.c
@@ -0,0 +1,7 @@
+Test case for https://github.com/itsallcode/openfasttrace/issues/391
+This is used in test case org.itsallcode.openfasttrace.ITestExampleProject
+This is based on https://xenbits.xen.org/gitweb/?p=xen.git;a=blob;f=xen/drivers/video/font_8x14.c;h=e80d178c3869a4b506313cdea19c76b18fe2b4c2;hb=HEAD
+
+Ensure to not modify the rest of this file when editing it!
+
+/* 128 0x80 '�� */