feat: 1840 validator accepts replacement character in stop name field

build.gradle

@@ -186,6 +186,10 @@ reporting {
     }
 }
 
+tasks.named('verifyGoogleJavaFormat') {
+    dependsOn tasks.named('copyRulesToWebClient')
+}
+
 task copyRulesToWebClient (type: Copy) {
     from "./RULES.md"
     into "./web/client/static"

core/src/main/java/org/mobilitydata/gtfsvalidator/notice/NoInvalidCharactersNotice.java

@@ -0,0 +1,54 @@
+package org.mobilitydata.gtfsvalidator.notice;
+
+import static org.mobilitydata.gtfsvalidator.notice.SeverityLevel.WARNING;
+
+import org.mobilitydata.gtfsvalidator.annotation.GtfsValidationNotice;
+import org.mobilitydata.gtfsvalidator.annotation.GtfsValidationNotice.UrlRef;
+
+/**
+ * This field contains invalid characters such as the replacement character (\uFFFD).
+ *
+ * <p>Fields with customer-facing text should not contain invalid characters to ensure good
+ * readability and accessibility.
+ *
+ * <table style="table-layout:auto; width:auto;">
+ *   <caption>Examples:</caption>
+ *   <tr>
+ *     <th><code>Field Text</code></th>
+ *     <th><code>Dataset</code></th>
+ *   </tr>
+ *   <tr>
+ *     <td>"Invalid�Character"</td>
+ *     <td><a href="http://example.com">Example Dataset</a></td>
+ *   </tr>
+ * </table>
+ */
+@GtfsValidationNotice(
+    severity = WARNING,
+    urls = {
+      @UrlRef(
+          label = "Best Practices for All Files",
+          url = "https://gtfs.org/schedule/reference/#file-requirements")
+    })
+public class NoInvalidCharactersNotice extends ValidationNotice {
+
+  /** Name of the faulty file. */
+  private final String filename;
+
+  /** Name of the faulty field. */
+  private final String fieldName;
+
+  /** Faulty value. */
+  private final String fieldValue;
+
+  /** The row number of the faulty record. */
+  private final int csvRowNumber;
+
+  public NoInvalidCharactersNotice(
+      String filename, String fieldName, String fieldValue, int csvRowNumber) {
+    this.filename = filename;
+    this.fieldName = fieldName;
+    this.fieldValue = fieldValue;
+    this.csvRowNumber = csvRowNumber;
+  }
+}

main/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsAgencySchema.java

@@ -30,6 +30,7 @@ public interface GtfsAgencySchema extends GtfsEntity {
 
   @Required
   @MixedCase
+  @NoInvalidCharacters
   String agencyName();
 
   @FieldType(FieldTypeEnum.URL)

main/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsAttributionSchema.java

@@ -16,12 +16,7 @@
 
 package org.mobilitydata.gtfsvalidator.table;
 
-import org.mobilitydata.gtfsvalidator.annotation.FieldType;
-import org.mobilitydata.gtfsvalidator.annotation.FieldTypeEnum;
-import org.mobilitydata.gtfsvalidator.annotation.ForeignKey;
-import org.mobilitydata.gtfsvalidator.annotation.GtfsTable;
-import org.mobilitydata.gtfsvalidator.annotation.PrimaryKey;
-import org.mobilitydata.gtfsvalidator.annotation.Required;
+import org.mobilitydata.gtfsvalidator.annotation.*;
 
 @GtfsTable("attributions.txt")
 public interface GtfsAttributionSchema extends GtfsEntity {
@@ -42,6 +37,7 @@ public interface GtfsAttributionSchema extends GtfsEntity {
   String tripId();
 
   @Required
+  @NoInvalidCharacters
   String organizationName();
 
   GtfsAttributionRole isProducer();

main/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsLevelSchema.java

@@ -29,5 +29,6 @@ public interface GtfsLevelSchema extends GtfsEntity {
   double levelIndex();
 
   @MixedCase
+  @NoInvalidCharacters
   String levelName();
 }

main/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsRouteSchema.java

@@ -34,10 +34,12 @@ public interface GtfsRouteSchema extends GtfsEntity {
 
   @MixedCase
   @ConditionallyRequired
+  @NoInvalidCharacters
   String routeShortName();
 
   @MixedCase
   @ConditionallyRequired
+  @NoInvalidCharacters
   String routeLongName();
 
   @MixedCase

main/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsStopSchema.java

@@ -31,6 +31,7 @@ public interface GtfsStopSchema extends GtfsEntity {
 
   @MixedCase
   @ConditionallyRequired
+  @NoInvalidCharacters
   String stopName();
 
   String ttsStopName();

model/src/main/java/org/mobilitydata/gtfsvalidator/annotation/NoInvalidCharacters.java

@@ -0,0 +1,24 @@
+package org.mobilitydata.gtfsvalidator.annotation;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * Specifies that a field should not contain the replacement character (\uFFFD). A validator will be
+ * automatically generated.
+ *
+ * <p>Example.
+ *
+ * <pre>
+ *   {@literal @}GtfsTable("stops.txt")
+ *   public interface GtfsStopSchema extends GtfsEntity {
+ *       {@literal @}NoReplacementChar
+ *       String stopName();
+ *   }
+ * </pre>
+ */
+@Target({ElementType.METHOD, ElementType.TYPE})
+@Retention(RetentionPolicy.SOURCE)
+public @interface NoInvalidCharacters {}

processor/src/main/java/org/mobilitydata/gtfsvalidator/processor/Analyser.java

@@ -73,6 +73,7 @@ public GtfsFileDescriptor analyzeGtfsFileType(TypeElement type) {
       fieldBuilder.setValueRequired(method.getAnnotation(Required.class) != null);
       fieldBuilder.setColumnRecommended(method.getAnnotation(RecommendedColumn.class) != null);
       fieldBuilder.setMixedCase(method.getAnnotation(MixedCase.class) != null);
+      fieldBuilder.setNoInvalidCharacters(method.getAnnotation(NoInvalidCharacters.class) != null);
       PrimaryKey primaryKey = method.getAnnotation(PrimaryKey.class);
       if (primaryKey != null) {
         fieldBuilder.setPrimaryKey(primaryKey);

processor/src/main/java/org/mobilitydata/gtfsvalidator/processor/GtfsAnnotationProcessor.java

@@ -118,6 +118,8 @@ public boolean process(Set<? extends TypeElement> annotations, RoundEnvironment
         new CurrencyAmountValidatorGenerator().generateValidator(fileDescriptors));
     generatedValidators.addAll(
         new MixedCaseValidatorGenerator().generateValidator(fileDescriptors));
+    generatedValidators.addAll(
+        new NoInvalidCharactersValidatorGenerator().generateValidator(fileDescriptors));
 
     for (TypeSpec typeSpec : generatedValidators) {
       writeJavaFile(JavaFile.builder(VALIDATOR_PACKAGE_NAME, typeSpec).build());

processor/src/main/java/org/mobilitydata/gtfsvalidator/processor/GtfsFieldDescriptor.java

@@ -67,6 +67,8 @@ public boolean isHeaderRequired() {
 
   public abstract boolean mixedCase();
 
+  public abstract boolean noInvalidCharacters();
+
   public abstract Optional<RowParser.NumberBounds> numberBounds();
 
   @AutoValue.Builder
@@ -87,6 +89,8 @@ public abstract static class Builder {
 
     public abstract Builder setMixedCase(boolean value);
 
+    public abstract Builder setNoInvalidCharacters(boolean value);
+
     public abstract Builder setPrimaryKey(PrimaryKey annotation);
 
     public abstract Builder setIndex(boolean value);

processor/src/main/java/org/mobilitydata/gtfsvalidator/processor/NoInvalidCharactersValidatorGenerator.java

@@ -0,0 +1,79 @@
+package org.mobilitydata.gtfsvalidator.processor;
+
+import com.google.common.collect.ImmutableList;
+import com.squareup.javapoet.ClassName;
+import com.squareup.javapoet.MethodSpec;
+import com.squareup.javapoet.ParameterizedTypeName;
+import com.squareup.javapoet.TypeSpec;
+import java.util.List;
+import javax.lang.model.element.Modifier;
+import org.mobilitydata.gtfsvalidator.annotation.Generated;
+import org.mobilitydata.gtfsvalidator.annotation.GtfsValidator;
+import org.mobilitydata.gtfsvalidator.annotation.NoInvalidCharacters;
+import org.mobilitydata.gtfsvalidator.notice.NoInvalidCharactersNotice;
+import org.mobilitydata.gtfsvalidator.notice.NoticeContainer;
+import org.mobilitydata.gtfsvalidator.validator.SingleEntityValidator;
+
+/**
+ * Generates validator classes to check that a string field does not contain invalid characters such
+ * as the replacement character (\uFFFD).
+ *
+ * @see NoInvalidCharacters
+ * @see NoInvalidCharactersNotice
+ */
+public class NoInvalidCharactersValidatorGenerator {
+  static final String REPLACEMENT_CHAR = "\uFFFD";
+
+  public ImmutableList<TypeSpec> generateValidator(List<GtfsFileDescriptor> fileDescriptors) {
+    ImmutableList.Builder<TypeSpec> validators = ImmutableList.builder();
+    for (GtfsFileDescriptor fileDescriptor : fileDescriptors) {
+      if (fileDescriptor.fields().stream().anyMatch(f -> f.noInvalidCharacters())) {
+        validators.add(generateValidator(fileDescriptor));
+      }
+    }
+    return validators.build();
+  }
+
+  private static TypeSpec generateValidator(GtfsFileDescriptor fileDescriptor) {
+    GtfsEntityClasses entityClasses = new GtfsEntityClasses(fileDescriptor);
+    TypeSpec.Builder typeSpec =
+        TypeSpec.classBuilder(fileDescriptor.className() + "NoInvalidCharactersValidator")
+            .addModifiers(Modifier.PUBLIC, Modifier.FINAL)
+            .addAnnotation(Generated.class)
+            .addAnnotation(GtfsValidator.class)
+            .superclass(
+                ParameterizedTypeName.get(
+                    ClassName.get(SingleEntityValidator.class),
+                    entityClasses.entityImplementationTypeName()));
+
+    MethodSpec.Builder validateMethod =
+        MethodSpec.methodBuilder("validate")
+            .addModifiers(Modifier.PUBLIC)
+            .addAnnotation(Override.class)
+            .returns(void.class)
+            .addParameter(entityClasses.entityImplementationTypeName(), "entity")
+            .addParameter(NoticeContainer.class, "noticeContainer");
+
+    for (GtfsFieldDescriptor noInvalidCharactersField : fileDescriptor.fields()) {
+      if (!noInvalidCharactersField.noInvalidCharacters()) {
+        continue;
+      }
+      validateMethod
+          .beginControlFlow(
+              "if (entity.$L())", FieldNameConverter.hasMethodName(noInvalidCharactersField.name()))
+          .addStatement("$T value = entity.$L()", String.class, noInvalidCharactersField.name())
+          .beginControlFlow("if (value.contains(\"$L\"))", REPLACEMENT_CHAR)
+          .addStatement(
+              "noticeContainer.addValidationNotice(new $T(\"$L\", \"$L\", value, entity.csvRowNumber()))",
+              NoInvalidCharactersNotice.class,
+              fileDescriptor.filename(),
+              FieldNameConverter.gtfsColumnName(noInvalidCharactersField.name()))
+          .endControlFlow()
+          .endControlFlow();
+    }
+
+    typeSpec.addMethod(validateMethod.build());
+
+    return typeSpec.build();
+  }
+}