Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: 1840 invalid characters #1892

Merged
merged 9 commits into from
Oct 21, 2024
Next Next commit
added check for invalid characters in asString method
qcdyx committed Oct 16, 2024

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature. The key has expired.
commit cace781476c6f57b0eb163915a960551b3f70750
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
package org.mobilitydata.gtfsvalidator.notice;

import static org.mobilitydata.gtfsvalidator.notice.SeverityLevel.ERROR;

import org.mobilitydata.gtfsvalidator.annotation.GtfsValidationNotice;

/**
* Invalid characters.
*
* <p>This field contains invalid characters such as the replacement character (U+FFFD). Fields with
* customer-facing text should not contain invalid characters to ensure good readability and
* accessibility.
*/
@GtfsValidationNotice(severity = ERROR)
public class InvalidCharactersNotice extends ValidationNotice {
/** The name of the file containing the invalid characters. */
private final String filename;

/** The row number in the CSV file where the invalid characters were found. */
private final long csvRowNumber;

/** The name of the field containing the invalid characters. */
private final String fieldName;

/** The value of the field containing the invalid characters. */
private final String fieldValue;

public InvalidCharactersNotice(
String filename, long csvRowNumber, String fieldName, String fieldValue) {
this.filename = filename;
this.csvRowNumber = csvRowNumber;
this.fieldName = fieldName;
this.fieldValue = fieldValue;
}
}
Original file line number Diff line number Diff line change
@@ -23,23 +23,7 @@
import java.util.function.Function;
import javax.annotation.Nullable;
import org.mobilitydata.gtfsvalidator.annotation.FieldLevelEnum;
import org.mobilitydata.gtfsvalidator.notice.EmptyRowNotice;
import org.mobilitydata.gtfsvalidator.notice.InvalidColorNotice;
import org.mobilitydata.gtfsvalidator.notice.InvalidCurrencyNotice;
import org.mobilitydata.gtfsvalidator.notice.InvalidDateNotice;
import org.mobilitydata.gtfsvalidator.notice.InvalidFloatNotice;
import org.mobilitydata.gtfsvalidator.notice.InvalidIntegerNotice;
import org.mobilitydata.gtfsvalidator.notice.InvalidLanguageCodeNotice;
import org.mobilitydata.gtfsvalidator.notice.InvalidRowLengthNotice;
import org.mobilitydata.gtfsvalidator.notice.InvalidTimeNotice;
import org.mobilitydata.gtfsvalidator.notice.InvalidTimezoneNotice;
import org.mobilitydata.gtfsvalidator.notice.MissingRecommendedFieldNotice;
import org.mobilitydata.gtfsvalidator.notice.MissingRequiredFieldNotice;
import org.mobilitydata.gtfsvalidator.notice.NoticeContainer;
import org.mobilitydata.gtfsvalidator.notice.NumberOutOfRangeNotice;
import org.mobilitydata.gtfsvalidator.notice.TooManyRowsNotice;
import org.mobilitydata.gtfsvalidator.notice.UnexpectedEnumValueNotice;
import org.mobilitydata.gtfsvalidator.notice.ValidationNotice;
import org.mobilitydata.gtfsvalidator.notice.*;
import org.mobilitydata.gtfsvalidator.table.GtfsColumnDescriptor;
import org.mobilitydata.gtfsvalidator.table.GtfsEnum;
import org.mobilitydata.gtfsvalidator.type.GtfsColor;
@@ -137,6 +121,15 @@ public String asString(int columnIndex, GtfsColumnDescriptor columnDescriptor) {
fileName, getRowNumber(), columnDescriptor.columnName()));
}
if (s != null) {
// Validate if the string contains invalid characters
if (fileName.equals("stops.txt")) {
if (containsInvalidCharacters(s)) {
noticeContainer.addValidationNotice(
new InvalidCharactersNotice(
fileName, getRowNumber(), columnDescriptor.columnName(), s));
return null;
}
}
s =
fieldValidator.validateField(
s,
@@ -146,6 +139,10 @@ public String asString(int columnIndex, GtfsColumnDescriptor columnDescriptor) {
return s;
}

private boolean containsInvalidCharacters(String string) {
return string.contains("\uFFFD");
}

@Nullable
public String asText(int columnIndex, GtfsColumnDescriptor columnDescriptor) {
return asString(columnIndex, columnDescriptor);