From ea8baaf28886270711220989174ee1f291543081 Mon Sep 17 00:00:00 2001 From: Christoph Pirkl Date: Mon, 26 Feb 2024 14:19:13 +0100 Subject: [PATCH] #391: Fix reading files with invalid encoding (#392) Co-authored-by: kaklakariada --- README.md | 8 ++++---- .../api/importer/input/RealFileInput.java | 10 +++++++--- .../api/importer/input/TestRealFileInput.java | 13 +++++++++--- doc/changes/changes_3.8.0.md | 3 ++- doc/developer_guide.md | 4 ++-- .../importer/zip/ITZipFileImporter.java | 10 ++++++++++ .../importer/zip/input/TestZipEntryInput.java | 20 +++++++++++++------ parent/pom.xml | 2 +- .../resources/example/src/invalid_encoding.c | 7 +++++++ 9 files changed, 57 insertions(+), 20 deletions(-) create mode 100644 product/src/test/resources/example/src/invalid_encoding.c diff --git a/README.md b/README.md index 5710ba33c..3fd8c4bf9 100644 --- a/README.md +++ b/README.md @@ -68,10 +68,10 @@ OpenFastTrace at it's core is a Java Archive (short "[JAR](https://docs.oracle.c ### Getting Pre-Built Packages -Pre-Built JAR files (called `openfasttrace-3.7.1.jar`) are available from the following places: +Pre-Built JAR files (called `openfasttrace-3.8.0.jar`) are available from the following places: -* [Maven Central](https://repo1.maven.org/maven2/org/itsallcode/openfasttrace/openfasttrace/3.7.1/openfasttrace-3.7.1.jar) -* [GitHub](https://github.com/itsallcode/openfasttrace/releases/download/3.7.1/openfasttrace-3.7.1.jar) +* [Maven Central](https://repo1.maven.org/maven2/org/itsallcode/openfasttrace/openfasttrace/3.8.0/openfasttrace-3.8.0.jar) +* [GitHub](https://github.com/itsallcode/openfasttrace/releases/download/3.8.0/openfasttrace-3.8.0.jar) Check our [developer guide](doc/developer_guide.md#getting-the-openfasttrace-library) to learn how to use the OFT JAR as dependency in your own code with popular build tools. @@ -94,7 +94,7 @@ If you just want to run OFT: The most basic variant to run OpenFastTrace is directly from the JAR file via the command line: ```bash -java -jar product/target/openfasttrace-3.7.1.jar trace /path/to/directory/being/traced +java -jar product/target/openfasttrace-3.8.0.jar trace /path/to/directory/being/traced ``` If you want to run OFT automatically as part of a continuous build, we recommend using our plugins for [Gradle](https://github.com/itsallcode/openfasttrace-gradle) and [Maven](https://github.com/itsallcode/openfasttrace-maven-plugin). diff --git a/api/src/main/java/org/itsallcode/openfasttrace/api/importer/input/RealFileInput.java b/api/src/main/java/org/itsallcode/openfasttrace/api/importer/input/RealFileInput.java index 99e4e98a1..f8c5d219e 100644 --- a/api/src/main/java/org/itsallcode/openfasttrace/api/importer/input/RealFileInput.java +++ b/api/src/main/java/org/itsallcode/openfasttrace/api/importer/input/RealFileInput.java @@ -1,7 +1,6 @@ package org.itsallcode.openfasttrace.api.importer.input; -import java.io.BufferedReader; -import java.io.IOException; +import java.io.*; import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; import java.nio.file.Files; @@ -51,7 +50,12 @@ public static InputFile forPath(final Path path, final Charset charset) @Override public BufferedReader createReader() throws IOException { - return Files.newBufferedReader(this.path, this.charset); + // Don't fail when reading files with invalid encoding. + // Files.newBufferedReader(this.path, this.charset) won't work here. + // See https://stackoverflow.com/a/43446789 for details. + final InputStream stream = Files.newInputStream(this.path); + final InputStreamReader reader = new InputStreamReader(stream, this.charset); + return new BufferedReader(reader); } @Override diff --git a/api/src/test/java/org/itsallcode/openfasttrace/api/importer/input/TestRealFileInput.java b/api/src/test/java/org/itsallcode/openfasttrace/api/importer/input/TestRealFileInput.java index 5a9b5bee3..0fd56d46d 100644 --- a/api/src/test/java/org/itsallcode/openfasttrace/api/importer/input/TestRealFileInput.java +++ b/api/src/test/java/org/itsallcode/openfasttrace/api/importer/input/TestRealFileInput.java @@ -8,9 +8,7 @@ import java.io.IOException; import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; +import java.nio.file.*; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -100,6 +98,15 @@ void testReadWithUtf8Encoding() throws IOException assertThat(readContent(inputFile), equalTo(CONTENT)); } + @Test + void testReadFileWithInvalidEncoding2() throws IOException + { + final Path path = tempDir.resolve("file"); + Files.write(path, new byte[] { (byte) 0x9F, (byte) 0x88 }); + final InputFile inputFile = RealFileInput.forPath(path, StandardCharsets.UTF_8); + assertThat(readContent(inputFile), equalTo("��")); + } + private Path writeTempFile(final String content, final Charset charset) throws IOException { final Path path = this.tempDir.resolve("test"); diff --git a/doc/changes/changes_3.8.0.md b/doc/changes/changes_3.8.0.md index 6dd60db59..5178aa31c 100644 --- a/doc/changes/changes_3.8.0.md +++ b/doc/changes/changes_3.8.0.md @@ -1,4 +1,4 @@ -# OpenFastTrace 3.8.0, released 2024-02-?? +# OpenFastTrace 3.8.0, released 2024-02-26 Code name: RST Importer @@ -29,6 +29,7 @@ Now you can also specify a revision for coverage tags instead of the default rev * #373: Ignore spaces after items in "Needs:" and "Tags:" lists (thanks to [@sambishop](https://github.com/sambishop) for his contribution!) * #378: Merged integration test coverage with unit test coverage for representative overall figure * #303: Escape special characters in HTML report to avoid broken HTML when a specification item contains text like `
` +* #391: Fixed reading files with invalid encoding (thanks to [@ayankuma](https://github.com/ayankuma) for the bug report!) ## Refactoring diff --git a/doc/developer_guide.md b/doc/developer_guide.md index dc637d684..0aca205fc 100644 --- a/doc/developer_guide.md +++ b/doc/developer_guide.md @@ -15,7 +15,7 @@ To use OpenFastTrace as a dependency in your [Maven](https://maven.apache.org) p org.itsallcode.openfasttrace openfasttrace - 3.7.1 + 3.8.0 compile @@ -27,7 +27,7 @@ To use OpenFastTrace as a dependency in your [Gradle](https://gradle.org/) proje ```groovy dependencies { - compile "org.itsallcode.openfasttrace:openfasttrace:3.7.1" + compile "org.itsallcode.openfasttrace:openfasttrace:3.8.0" } ``` diff --git a/importer/zip/src/test/java/org/itsallcode/openfasttrace/importer/zip/ITZipFileImporter.java b/importer/zip/src/test/java/org/itsallcode/openfasttrace/importer/zip/ITZipFileImporter.java index fd22493e0..2fe3dfebd 100644 --- a/importer/zip/src/test/java/org/itsallcode/openfasttrace/importer/zip/ITZipFileImporter.java +++ b/importer/zip/src/test/java/org/itsallcode/openfasttrace/importer/zip/ITZipFileImporter.java @@ -133,6 +133,16 @@ void testImportZipWithMultipleFilesAssertFileContent() throws IOException assertThat(this.actualFileContent.get(1), equalTo(FILE_CONTENT2_STRING)); } + @Test + void testImportZipWithInvalidEncoding() throws IOException + { + initializeZipFile(); + addEntryToZip("file1.c", new byte[] { (byte) 0x9F, (byte) 0x88 }); + final List importedFiles = runImporter(1); + assertThat(importedFiles.get(0).getPath(), equalTo(this.zipFile.getPath() + "!file1.c")); + assertThat(this.actualFileContent.get(0), equalTo("��")); + } + private void addZipEntryDirectory(final String name) throws IOException { assertThat(name, not(Matchers.endsWith("/"))); diff --git a/importer/zip/src/test/java/org/itsallcode/openfasttrace/importer/zip/input/TestZipEntryInput.java b/importer/zip/src/test/java/org/itsallcode/openfasttrace/importer/zip/input/TestZipEntryInput.java index e42b6d3d1..33fed0ec7 100644 --- a/importer/zip/src/test/java/org/itsallcode/openfasttrace/importer/zip/input/TestZipEntryInput.java +++ b/importer/zip/src/test/java/org/itsallcode/openfasttrace/importer/zip/input/TestZipEntryInput.java @@ -5,14 +5,10 @@ import static org.hamcrest.MatcherAssert.assertThat; import static org.junit.jupiter.api.Assertions.assertThrows; -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; +import java.io.*; import java.nio.charset.StandardCharsets; import java.nio.file.Path; -import java.util.zip.ZipEntry; -import java.util.zip.ZipFile; -import java.util.zip.ZipOutputStream; +import java.util.zip.*; import org.itsallcode.openfasttrace.api.importer.ImporterException; import org.itsallcode.openfasttrace.api.importer.input.InputFile; @@ -118,6 +114,18 @@ void testReadContentIso() throws IOException } } + @Test + void testReadInvalidEncoding() throws IOException + { + addEntryToZip("file", new byte[] { (byte) 0x9F, (byte) 0x88 }); + try (final ZipFile zip = getZipFile()) + { + final InputFile inputFile = ZipEntryInput.forZipEntry(zip, new ZipEntry("file"), + StandardCharsets.UTF_8); + assertThat(readContent(inputFile), equalTo("��")); + } + } + private String readContent(final InputFile inputFile) throws IOException { return inputFile.createReader().lines().collect(joining("\n")); diff --git a/parent/pom.xml b/parent/pom.xml index 088be175c..71f822a79 100644 --- a/parent/pom.xml +++ b/parent/pom.xml @@ -8,7 +8,7 @@ Free requirement tracking suite https://github.com/itsallcode/openfasttrace - 3.7.2 + 3.8.0 11 5.10.2 3.2.5 diff --git a/product/src/test/resources/example/src/invalid_encoding.c b/product/src/test/resources/example/src/invalid_encoding.c new file mode 100644 index 000000000..db371c682 --- /dev/null +++ b/product/src/test/resources/example/src/invalid_encoding.c @@ -0,0 +1,7 @@ +Test case for https://github.com/itsallcode/openfasttrace/issues/391 +This is used in test case org.itsallcode.openfasttrace.ITestExampleProject +This is based on https://xenbits.xen.org/gitweb/?p=xen.git;a=blob;f=xen/drivers/video/font_8x14.c;h=e80d178c3869a4b506313cdea19c76b18fe2b4c2;hb=HEAD + +Ensure to not modify the rest of this file when editing it! + +/* 128 0x80 '�� */