Skip to content

Commit

Permalink
Cleanup interface
Browse files Browse the repository at this point in the history
  • Loading branch information
@RustedBones
RustedBones committed Sep 12, 2024
1 parent c497656 commit ee6fa40
Show file tree
Hide file tree
Showing 12 changed files with 122 additions and 175 deletions.
18 changes: 4 additions & 14 deletions api/src/main/java/com/github/sbt/avro/AvroCompiler.java
View file
Original file line number Diff line number Diff line change
@@ -1,20 +1,10 @@
package com.github.sbt.avro;

import java.io.File;
import java.io.IOException;

public interface AvroCompiler {
void compileAvroSchema(
Class<?>[] records,
File[] avdls,
AvroFileRef[] avscs,
File[] avprs,
File target,
String stringType,
String fieldVisibility,
Boolean enableDecimalLogicalType,
Boolean useNamespace,
Boolean optionalGetters,
Boolean createSetters
) throws IOException;
void recompile(Class<?>[] records, File target) throws Exception;
void compileIdls(File[] idls, File target) throws Exception;
void compileAvscs(AvroFileRef[] avscs, File target) throws Exception;
void compileAvprs(File[] avprs, File target) throws Exception;
}
131 changes: 56 additions & 75 deletions bridge/src/main/java/com/github/sbt/avro/AvroCompilerBridge.java
View file
Original file line number Diff line number Diff line change
@@ -1,30 +1,53 @@
package com.github.sbt.avro;

import org.apache.avro.Schema;
import org.apache.avro.specific.SpecificRecord;
import xsbti.Logger;

import org.apache.avro.Protocol;
import org.apache.avro.compiler.idl.Idl;
import org.apache.avro.compiler.idl.ParseException;
import org.apache.avro.compiler.specific.SpecificCompiler;
import org.apache.avro.compiler.specific.SpecificCompiler.FieldVisibility;
import org.apache.avro.generic.GenericData.StringType;
import org.apache.avro.specific.SpecificRecord;

import java.io.File;
import java.io.IOException;
import java.util.HashSet;
import java.util.Set;

public class AvroCompilerBridge implements AvroCompiler {

void recompile(
Class<? extends SpecificRecord>[] records,
File target,
StringType stringType,
FieldVisibility fieldVisibility,
Boolean enableDecimalLogicalType,
Boolean useNamespace,
Boolean optionalGetters,
Boolean createSetters
// builder: SchemaParserBuilder
private final Logger logger;
private final StringType stringType;
private final FieldVisibility fieldVisibility;
private final boolean useNamespace;
private final boolean enableDecimalLogicalType;
private final boolean createSetters;
private final boolean optionalGetters;

public AvroCompilerBridge(
Logger logger,
String stringType,
String fieldVisibility,
boolean useNamespace,
boolean enableDecimalLogicalType,
boolean createSetters,
boolean optionalGetters
) {
this.logger = logger;
this.stringType = StringType.valueOf(stringType);
this.fieldVisibility = FieldVisibility.valueOf(fieldVisibility);
this.useNamespace = useNamespace;
this.enableDecimalLogicalType = enableDecimalLogicalType;
this.createSetters = createSetters;
this.optionalGetters = optionalGetters;
}

protected Schema.Parser createParser() {
return new Schema.Parser();
}

@Override
public void recompile(Class<?>[] records, File target) throws Exception {
AvscFilesCompiler compiler = new AvscFilesCompiler();
compiler.setStringType(stringType);
compiler.setFieldVisibility(fieldVisibility);
Expand All @@ -37,22 +60,20 @@ void recompile(
if (AvroVersion.getMinor() > 9) {
compiler.setOptionalGettersForNullableFieldsOnly(optionalGetters);
}
compiler.setLogCompileExceptions(true);
compiler.setTemplateDirectory("/org/apache/avro/compiler/specific/templates/java/classic/");
compiler.compileClasses(Set.of(records), target);

Set<Class<? extends SpecificRecord>> classes = new HashSet<>();
for (Class<?> record : records) {
logger.info(() -> "Recompiling Avro record: " + record.getName());
classes.add((Class<? extends SpecificRecord>) record);
}
compiler.compileClasses(classes, target);
}

void compileIdls(
File[] idls,
File target,
StringType stringType,
FieldVisibility fieldVisibility,
Boolean enableDecimalLogicalType,
Boolean optionalGetters,
Boolean createSetters
) throws IOException, ParseException {
@Override
public void compileIdls(File[] idls, File target) throws Exception {
for (File idl : idls) {
// log.info(s"Compiling Avro IDL $idl")
logger.info(() -> "Compiling Avro IDL " + idl);
Idl parser = new Idl(idl);
Protocol protocol = parser.CompilationUnit();
SpecificCompiler compiler = new SpecificCompiler(protocol);
Expand All @@ -70,17 +91,8 @@ void compileIdls(
}
}

void compileAvscs(
AvroFileRef[] avscs,
File target,
StringType stringType,
FieldVisibility fieldVisibility,
Boolean enableDecimalLogicalType,
Boolean useNamespace,
Boolean optionalGetters,
Boolean createSetters
// builder: SchemaParserBuilder
) {
@Override
public void compileAvscs(AvroFileRef[] avscs, File target) throws Exception {
AvscFilesCompiler compiler = new AvscFilesCompiler();
compiler.setStringType(stringType);
compiler.setFieldVisibility(fieldVisibility);
Expand All @@ -93,23 +105,20 @@ void compileAvscs(
if (AvroVersion.getMinor() > 9) {
compiler.setOptionalGettersForNullableFieldsOnly(optionalGetters);
}
compiler.setLogCompileExceptions(true);
compiler.setTemplateDirectory("/org/apache/avro/compiler/specific/templates/java/classic/");

Set<AvroFileRef> files = new HashSet<>();
for (AvroFileRef ref: avscs) {
logger.info(() -> "Compiling Avro schema: " + ref.getFile());
files.add(ref);
}
compiler.compileFiles(Set.of(avscs), target);
}

void compileAvprs(
File[] avprs,
File target,
StringType stringType,
FieldVisibility fieldVisibility,
Boolean enableDecimalLogicalType,
Boolean optionalGetters,
Boolean createSetters
) throws IOException {
@Override
public void compileAvprs(File[] avprs, File target) throws Exception {
for (File avpr : avprs) {
// log.info(s"Compiling Avro protocol $avpr")
logger.info(() -> "Compiling Avro protocol " + avpr);
Protocol protocol = Protocol.parse(avpr);
SpecificCompiler compiler = new SpecificCompiler(protocol);
compiler.setStringType(stringType);
Expand All @@ -125,32 +134,4 @@ void compileAvprs(
compiler.compileToDestination(null, target);
}
}


@Override
public void compileAvroSchema(
Class<?>[] records,
File[] avdls,
AvroFileRef[] avscs,
File[] avprs,
File target,
String stringType,
String fieldVisibility,
Boolean enableDecimalLogicalType,
Boolean useNamespace,
Boolean optionalGetters,
Boolean createSetters) {
StringType stringTypeEnum = StringType.valueOf(stringType);
FieldVisibility fieldVisibilityEnum = FieldVisibility.valueOf(fieldVisibility);

try {
recompile((Class<? extends SpecificRecord>[]) records, target, stringTypeEnum, fieldVisibilityEnum, enableDecimalLogicalType, useNamespace, optionalGetters, createSetters);

compileIdls(avdls, target, stringTypeEnum, fieldVisibilityEnum, enableDecimalLogicalType, optionalGetters, createSetters);
compileAvscs(avscs, target,stringTypeEnum, fieldVisibilityEnum, enableDecimalLogicalType, useNamespace, optionalGetters, createSetters);
compileAvprs(avprs, target, stringTypeEnum, fieldVisibilityEnum, enableDecimalLogicalType, optionalGetters, createSetters);
} catch (Exception e) {
throw new RuntimeException("Avro schema compilation failed: " + e.getMessage(), e);
}
}
}
17 changes: 1 addition & 16 deletions bridge/src/main/java/com/github/sbt/avro/AvscFilesCompiler.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,19 +6,15 @@
import org.apache.avro.generic.GenericData;
import org.apache.avro.specific.SpecificData;
import org.apache.avro.specific.SpecificRecord;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.File;
import java.io.IOException;
import java.util.*;
import java.util.function.Supplier;
import java.util.stream.Collectors;

public class AvscFilesCompiler {

private static final Logger LOG = LoggerFactory.getLogger(AvscFilesCompiler.class);

// private final SchemaParserBuilder builder;
private Schema.Parser schemaParser;
private String templateDirectory;
private GenericData.StringType stringType;
Expand All @@ -29,7 +25,6 @@ public class AvscFilesCompiler {
private Boolean gettersReturnOptional;
private Boolean optionalGettersForNullableFieldsOnly;
private Map<AvroFileRef, Exception> compileExceptions;
private boolean logCompileExceptions;

public AvscFilesCompiler() {
// this.builder = builder;
Expand Down Expand Up @@ -66,9 +61,6 @@ public void compileFiles(Set<AvroFileRef> files, File outputDirectory) {
for (AvroFileRef file : uncompiledFiles) {
Exception e = compileExceptions.get(file);
if (e != null) {
if (logCompileExceptions) {
LOG.error(file.toString(), e);
}
ex.addSuppressed(e);
}
}
Expand Down Expand Up @@ -107,9 +99,6 @@ public void compileClasses(Set<Class<? extends SpecificRecord>> classes, File ou
for (Class<?> clazz : uncompiledClasses) {
Exception e = compileExceptions.get(clazz);
if (e != null) {
if (logCompileExceptions) {
LOG.error(clazz.toString(), e);
}
ex.addSuppressed(e);
}
}
Expand Down Expand Up @@ -214,10 +203,6 @@ public void setCreateSetters(boolean createSetters) {
this.createSetters = createSetters;
}

public void setLogCompileExceptions(final boolean logCompileExceptions) {
this.logCompileExceptions = logCompileExceptions;
}

public void setGettersReturnOptional(final boolean gettersReturnOptional) {
this.gettersReturnOptional = gettersReturnOptional;
}
Expand Down
42 changes: 18 additions & 24 deletions ...hub/sbt/avro/AvroCompilerBridgeSpec.scala → ...thub/sbt/avro/AvscFilesCompilerSpec.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,18 +3,29 @@ package com.github.sbt.avro
import com.github.sbt.avro.test.{TestSpecificRecord, TestSpecificRecordParent}
import org.apache.avro.compiler.specific.SpecificCompiler.FieldVisibility
import org.apache.avro.generic.GenericData.StringType
import org.apache.avro.specific.SpecificRecord
import org.specs2.mutable.Specification

import java.io.File
import java.nio.file.Files
import scala.collection.JavaConverters._

class AvroCompilerBridgeSpec extends Specification {
class AvscFilesCompilerSpec extends Specification {
val sourceDir = new File(getClass.getClassLoader.getResource("avro").toURI)


val targetDir = Files.createTempDirectory("sbt-avro-compiler-bridge").toFile
val packageDir = new File(targetDir, "com/github/sbt/avro/test")

val compiler = new AvscFilesCompiler()
compiler.setUseNamespace(false)
compiler.setStringType(StringType.CharSequence)
compiler.setFieldVisibility(FieldVisibility.PRIVATE)
compiler.setEnableDecimalLogicalType(true)
compiler.setGettersReturnOptional(true)
compiler.setOptionalGettersForNullableFieldsOnly(true)
compiler.setCreateSetters(true)
compiler.setTemplateDirectory("/org/apache/avro/compiler/specific/templates/java/classic/")

"It should be possible to compile types depending on others if source files are provided in right order" >> {
val fullyQualifiedNames = Seq(
new File(sourceDir, "a.avsc"),
Expand Down Expand Up @@ -57,17 +68,7 @@ class AvroCompilerBridgeSpec extends Specification {
_eJavaFile.delete()

val refs = sourceFiles.map(s => new AvroFileRef(sourceDir, s.getName))
val compiler = new AvroCompilerBridge
compiler.compileAvscs(
refs.toArray,
targetDir,
StringType.CharSequence,
FieldVisibility.PRIVATE,
true,
false,
false,
true,
)
compiler.compileFiles(refs.toSet.asJava, targetDir)

aJavaFile.isFile must beTrue
bJavaFile.isFile must beTrue
Expand All @@ -84,20 +85,13 @@ class AvroCompilerBridgeSpec extends Specification {

"It should be possible to compile types depending on others if classes are provided in right order" >> {
// TestSpecificRecordParent and TestSpecificRecord were previously generated from test_records.avsc
val compiler = new AvroCompilerBridge
compiler.recompile(
Array(
compiler.compileClasses(
Set[Class[_ <: SpecificRecord]](
// put parent 1st
classOf[TestSpecificRecordParent],
classOf[TestSpecificRecord]
),
targetDir,
StringType.CharSequence,
FieldVisibility.PRIVATE,
true,
false,
false,
true
).asJava,
targetDir
)

val record = new File(packageDir, "TestSpecificRecord.java")
Expand Down
6 changes: 2 additions & 4 deletions build.sbt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,10 +61,7 @@ lazy val `sbt-avro-compiler-api`: Project = project
.in(file("api"))
.settings(
crossPaths := false,
autoScalaLibrary := false,
libraryDependencies ++= Seq(
Dependencies.Provided.AvroCompiler
)
autoScalaLibrary := false
)

lazy val `sbt-avro-compiler-bridge`: Project = project
Expand All @@ -75,6 +72,7 @@ lazy val `sbt-avro-compiler-bridge`: Project = project
autoScalaLibrary := false,
libraryDependencies ++= Seq(
Dependencies.Provided.AvroCompiler,
Dependencies.Provided.SbtUtilInterface,
Dependencies.Test.Specs2Core,
)
)
Expand Down
Loading

0 comments on commit ee6fa40

Please sign in to comment.