From bb418538278f51d1b08468f687fcd10e9c59f717 Mon Sep 17 00:00:00 2001 From: Michel Davit Date: Thu, 4 May 2023 13:52:55 +0200 Subject: [PATCH 1/9] Add joda-time and ByteBuffer support in avro --- .../main/scala/magnolify/avro/AvroType.scala | 8 + .../magnolify/avro/logical/package.scala | 166 +++++++++++++++--- .../scala/magnolify/avro/AvroTypeSuite.scala | 103 +++++++++-- build.sbt | 4 +- .../test/scala/magnolify/cats/TestEq.scala | 23 ++- 5 files changed, 243 insertions(+), 61 deletions(-) diff --git a/avro/src/main/scala/magnolify/avro/AvroType.scala b/avro/src/main/scala/magnolify/avro/AvroType.scala index 954876323..bd6b31ade 100644 --- a/avro/src/main/scala/magnolify/avro/AvroType.scala +++ b/avro/src/main/scala/magnolify/avro/AvroType.scala @@ -196,6 +196,7 @@ object AvroField { implicit val afLong: AvroField[Long] = id[Long](Schema.Type.LONG) implicit val afFloat: AvroField[Float] = id[Float](Schema.Type.FLOAT) implicit val afDouble: AvroField[Double] = id[Double](Schema.Type.DOUBLE) + implicit val afByteBuffer: AvroField[ByteBuffer] = id[ByteBuffer](Schema.Type.BYTES) implicit val afBytes: AvroField[Array[Byte]] = new Aux[Array[Byte], ByteBuffer, ByteBuffer] { override protected def buildSchema(cm: CaseMapper): Schema = Schema.create(Schema.Type.BYTES) // `JacksonUtils.toJson` expects `Array[Byte]` for `BYTES` defaults @@ -318,6 +319,13 @@ object AvroField { ) implicit val afDate: AvroField[LocalDate] = logicalType[Int](LogicalTypes.date())(x => LocalDate.ofEpochDay(x.toLong))(_.toEpochDay.toInt) + private val epochJodaDate = new org.joda.time.LocalDate(1970, 1, 1); + implicit val afJodaDate: AvroField[org.joda.time.LocalDate] = + logicalType[Int](LogicalTypes.date()) { daysFromEpoch => + epochJodaDate.plusDays(daysFromEpoch) + } { date => + org.joda.time.Days.daysBetween(epochJodaDate, date).getDays + } def fixed[T: ClassTag]( size: Int diff --git a/avro/src/main/scala/magnolify/avro/logical/package.scala b/avro/src/main/scala/magnolify/avro/logical/package.scala index 7f2f66574..03b8f8b02 100644 --- a/avro/src/main/scala/magnolify/avro/logical/package.scala +++ b/avro/src/main/scala/magnolify/avro/logical/package.scala @@ -22,48 +22,116 @@ import java.time.{Instant, LocalDateTime, LocalTime, ZoneOffset} import java.time.format.{DateTimeFormatter, DateTimeFormatterBuilder} import org.apache.avro.{LogicalType, LogicalTypes, Schema} +import java.util.concurrent.TimeUnit + package object logical { + // Duplicate implementation from org.apache.avro.data.TimeConversions + // to support both 1.8 (joda-time based) and 1.9+ (java-time based) object micros { + private def toTimestampMicros(microsFromEpoch: Long): Instant = { + val epochSeconds = microsFromEpoch / 1000000L + val nanoAdjustment = (microsFromEpoch % 1000000L) * 1000L; + Instant.ofEpochSecond(epochSeconds, nanoAdjustment) + } + + private def fromTimestampMicros(instant: Instant): Long = { + val seconds = instant.getEpochSecond + val nanos = instant.getNano + if (seconds < 0 && nanos > 0) { + val micros = Math.multiplyExact(seconds + 1, 1000000L) + val adjustment = (nanos / 1000L) - 1000000 + Math.addExact(micros, adjustment) + } else { + val micros = Math.multiplyExact(seconds, 1000000L) + Math.addExact(micros, nanos / 1000L) + } + } + implicit val afTimestampMicros: AvroField[Instant] = - AvroField.logicalType[Long](LogicalTypes.timestampMicros())(us => - Instant.ofEpochMilli(us / 1000) - )(_.toEpochMilli * 1000) + AvroField.logicalType[Long](LogicalTypes.timestampMicros())(toTimestampMicros)( + fromTimestampMicros + ) implicit val afTimeMicros: AvroField[LocalTime] = - AvroField.logicalType[Long](LogicalTypes.timeMicros())(us => - LocalTime.ofNanoOfDay(us * 1000) - )(_.toNanoOfDay / 1000) + AvroField.logicalType[Long](LogicalTypes.timeMicros()) { us => + LocalTime.ofNanoOfDay(TimeUnit.MICROSECONDS.toNanos(us)) + } { time => + TimeUnit.NANOSECONDS.toMicros(time.toNanoOfDay) + } - // `LogicalTypes.localTimestampMicros` is Avro 1.10.0+ + // `LogicalTypes.localTimestampMicros()` is Avro 1.10 implicit val afLocalTimestampMicros: AvroField[LocalDateTime] = - AvroField.logicalType[Long](new LogicalType("local-timestamp-micros"))(us => - LocalDateTime.ofInstant(Instant.ofEpochMilli(us / 1000), ZoneOffset.UTC) - )(_.toInstant(ZoneOffset.UTC).toEpochMilli * 1000) + AvroField.logicalType[Long](new LogicalType("local-timestamp-micros")) { microsFromEpoch => + val instant = toTimestampMicros(microsFromEpoch) + LocalDateTime.ofInstant(instant, ZoneOffset.UTC) + } { timestamp => + val instant = timestamp.toInstant(ZoneOffset.UTC) + fromTimestampMicros(instant) + } + + // avro 1.8 uses joda-time + implicit val afJodaTimestampMicros: AvroField[org.joda.time.DateTime] = + AvroField.logicalType[Long](LogicalTypes.timestampMicros()) { microsFromEpoch => + new org.joda.time.DateTime(microsFromEpoch / 1000, org.joda.time.DateTimeZone.UTC) + } { timestamp => + 1000 * timestamp.getMillis + } + + implicit val afJodaTimeMicros: AvroField[org.joda.time.LocalTime] = + AvroField.logicalType[Long](LogicalTypes.timeMicros()) { microsFromMidnight => + org.joda.time.LocalTime.fromMillisOfDay(microsFromMidnight / 1000) + } { time => + // from LossyTimeMicrosConversion + 1000L * time.millisOfDay().get() + } } object millis { implicit val afTimestampMillis: AvroField[Instant] = - AvroField.logicalType[Long](LogicalTypes.timestampMillis())(Instant.ofEpochMilli)( - _.toEpochMilli - ) + AvroField.logicalType[Long](LogicalTypes.timestampMillis()) { millisFromEpoch => + Instant.ofEpochMilli(millisFromEpoch) + } { timestamp => + timestamp.toEpochMilli + } implicit val afTimeMillis: AvroField[LocalTime] = - AvroField.logicalType[Int](LogicalTypes.timeMillis())(ms => - LocalTime.ofNanoOfDay(ms * 1000000L) - )(t => (t.toNanoOfDay / 1000000).toInt) + AvroField.logicalType[Int](LogicalTypes.timeMillis()) { millisFromMidnight => + LocalTime.ofNanoOfDay(TimeUnit.MILLISECONDS.toNanos(millisFromMidnight.toLong)) + } { time => + TimeUnit.NANOSECONDS.toMillis(time.toNanoOfDay).toInt + } // `LogicalTypes.localTimestampMillis` is Avro 1.10.0+ implicit val afLocalTimestampMillis: AvroField[LocalDateTime] = - AvroField.logicalType[Long](new LogicalType("local-timestamp-millis"))(ms => - LocalDateTime.ofInstant(Instant.ofEpochMilli(ms), ZoneOffset.UTC) - )(_.toInstant(ZoneOffset.UTC).toEpochMilli) + AvroField.logicalType[Long](new LogicalType("local-timestamp-millis")) { millisFromEpoch => + val instant = Instant.ofEpochMilli(millisFromEpoch) + LocalDateTime.ofInstant(instant, ZoneOffset.UTC) + } { timestamp => + val instant = timestamp.toInstant(ZoneOffset.UTC) + instant.toEpochMilli + } + + // avro 1.8 uses joda-time + implicit val afJodaTimestampMillis: AvroField[org.joda.time.DateTime] = + AvroField.logicalType[Long](LogicalTypes.timestampMillis()) { millisFromEpoch => + new org.joda.time.DateTime(millisFromEpoch, org.joda.time.DateTimeZone.UTC) + } { timestamp => + timestamp.getMillis + } + + implicit val afJodaTimeMillis: AvroField[org.joda.time.LocalTime] = + AvroField.logicalType[Int](LogicalTypes.timeMillis()) { millisFromMidnight => + org.joda.time.LocalTime.fromMillisOfDay(millisFromMidnight.toLong) + } { time => + time.millisOfDay().get() + } } object bigquery { // datetime is a custom logical type and must be registered private final val DateTimeTypeName = "datetime" private final val DateTimeLogicalTypeFactory: LogicalTypeFactory = (_: Schema) => - new org.apache.avro.LogicalType(DateTimeTypeName) + new LogicalType(DateTimeTypeName) /** * Register custom logical types with avro, which is necessary to correctly parse a custom @@ -72,38 +140,80 @@ package object logical { * on the type name. */ def registerLogicalTypes(): Unit = - org.apache.avro.LogicalTypes.register(DateTimeTypeName, DateTimeLogicalTypeFactory) + LogicalTypes.register(DateTimeTypeName, DateTimeLogicalTypeFactory) // DATETIME // YYYY-[M]M-[D]D[ [H]H:[M]M:[S]S[.DDDDDD]] - private val DatetimePrinter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSSSSS") + private val DatePattern = "yyyy-MM-dd" + private val TimePattern = "HH:mm:ss" + private val DecimalPattern = "SSSSSS" + private val DatetimePattern = s"$DatePattern $TimePattern.$DecimalPattern" + private val DatetimePrinter = DateTimeFormatter.ofPattern(DatetimePattern) private val DatetimeParser = new DateTimeFormatterBuilder() - .append(DateTimeFormatter.ofPattern("yyyy-MM-dd")) + .appendPattern(DatePattern) .appendOptional( new DateTimeFormatterBuilder() - .append(DateTimeFormatter.ofPattern(" HH:mm:ss")) - .appendOptional(DateTimeFormatter.ofPattern(".SSSSSS")) + .appendLiteral(' ') + .append(new DateTimeFormatterBuilder().appendPattern(TimePattern).toFormatter) + .appendOptional( + new DateTimeFormatterBuilder() + .appendLiteral('.') + .appendPattern(DecimalPattern) + .toFormatter + ) .toFormatter ) .toFormatter .withZone(ZoneOffset.UTC) + private val JodaDatetimePrinter = new org.joda.time.format.DateTimeFormatterBuilder() + .appendPattern(DatetimePattern) + .toFormatter + + private val JodaDatetimeParser = new org.joda.time.format.DateTimeFormatterBuilder() + .appendPattern(DatePattern) + .appendOptional( + new org.joda.time.format.DateTimeFormatterBuilder() + .appendLiteral(' ') + .appendPattern(TimePattern) + .appendOptional( + new org.joda.time.format.DateTimeFormatterBuilder() + .appendLiteral('.') + .appendPattern(DecimalPattern) + .toParser + ) + .toParser + ) + .toFormatter + .withZone(org.joda.time.DateTimeZone.UTC) + // NUMERIC // https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types#numeric-type implicit val afBigQueryNumeric: AvroField[BigDecimal] = AvroField.bigDecimal(38, 9) // TIMESTAMP implicit val afBigQueryTimestamp: AvroField[Instant] = micros.afTimestampMicros + implicit val afBigQueryJodaTimestamp: AvroField[org.joda.time.DateTime] = + micros.afJodaTimestampMicros // DATE: `AvroField.afDate` // TIME implicit val afBigQueryTime: AvroField[LocalTime] = micros.afTimeMicros + implicit val afBigQueryJodaTime: AvroField[org.joda.time.LocalTime] = micros.afJodaTimeMicros // DATETIME -> sqlType: DATETIME implicit val afBigQueryDatetime: AvroField[LocalDateTime] = - AvroField.logicalType[CharSequence](new org.apache.avro.LogicalType(DateTimeTypeName))(cs => - LocalDateTime.from(DatetimeParser.parse(cs)) - )(DatetimePrinter.format) + AvroField.logicalType[CharSequence](new org.apache.avro.LogicalType(DateTimeTypeName)) { cs => + LocalDateTime.parse(cs.toString, DatetimeParser) + } { datetime => + DatetimePrinter.format(datetime) + } + implicit val afBigQueryJodaDatetime: AvroField[org.joda.time.LocalDateTime] = + AvroField.logicalType[CharSequence](new org.apache.avro.LogicalType(DateTimeTypeName)) { cs => + org.joda.time.LocalDateTime.parse(cs.toString, JodaDatetimeParser) + } { datetime => + JodaDatetimePrinter.print(datetime) + } } } diff --git a/avro/src/test/scala/magnolify/avro/AvroTypeSuite.scala b/avro/src/test/scala/magnolify/avro/AvroTypeSuite.scala index 7b957aa93..c0cf63ece 100644 --- a/avro/src/test/scala/magnolify/avro/AvroTypeSuite.scala +++ b/avro/src/test/scala/magnolify/avro/AvroTypeSuite.scala @@ -20,7 +20,6 @@ import cats._ import com.fasterxml.jackson.core.JsonFactory import com.fasterxml.jackson.databind.JsonNode import com.fasterxml.jackson.databind.ObjectMapper -import magnolify.avro._ import magnolify.avro.unsafe._ import magnolify.cats.auto._ import magnolify.cats.TestEq._ @@ -140,11 +139,44 @@ class AvroTypeSuite extends MagnolifySuite { } } - implicit val arbBigDecimal: Arbitrary[BigDecimal] = - Arbitrary(Gen.chooseNum(0L, Long.MaxValue).map(BigDecimal(_, 0))) + implicit val arbJodaDate: Arbitrary[org.joda.time.LocalDate] = Arbitrary { + Arbitrary.arbitrary[LocalDate].map { ld => + new org.joda.time.LocalDate(ld.getYear, ld.getMonthValue, ld.getDayOfMonth) + } + } + implicit val arbJodaDateTime: Arbitrary[org.joda.time.DateTime] = Arbitrary { + Arbitrary.arbitrary[Instant].map { i => + new org.joda.time.DateTime(i.toEpochMilli, org.joda.time.DateTimeZone.UTC) + } + } + implicit val arbJodaLocalTime: Arbitrary[org.joda.time.LocalTime] = Arbitrary { + Arbitrary.arbitrary[LocalTime].map { lt => + org.joda.time.LocalTime.fromMillisOfDay(lt.toNanoOfDay / 1000) + } + } + implicit val arbJodaLocalDateTime: Arbitrary[org.joda.time.LocalDateTime] = Arbitrary { + Arbitrary.arbitrary[LocalDateTime].map { ldt => + org.joda.time.LocalDateTime.parse(ldt.toString) + } + } + implicit val arbByteBuffer: Arbitrary[ByteBuffer] = Arbitrary { + Arbitrary.arbitrary[Array[Byte]].map(ByteBuffer.wrap) + } + implicit val arbBigDecimal: Arbitrary[BigDecimal] = Arbitrary { + // bq logical type has precision of 38 and scale of 9 + val max = BigInt(10).pow(38) - 1 + Gen.choose(-max, max).map(BigDecimal(_, 9)) + } implicit val arbCountryCode: Arbitrary[CountryCode] = Arbitrary( - Gen.oneOf("US", "UK", "CA", "MX").map(CountryCode(_)) + Gen.oneOf("US", "UK", "CA", "MX").map(CountryCode.apply) ) + + implicit val eqJodaDate: Eq[org.joda.time.LocalDate] = Eq.fromUniversalEquals + implicit val eqJodaDateTime: Eq[org.joda.time.DateTime] = Eq.fromUniversalEquals + implicit val eqJodaLocalTime: Eq[org.joda.time.LocalTime] = Eq.fromUniversalEquals + implicit val eqJodaLocalDateTime: Eq[org.joda.time.LocalDateTime] = Eq.fromUniversalEquals + implicit val eqByteBuffer: Eq[ByteBuffer] = Eq.by(_.array()) + implicit val afUri: AvroField[URI] = AvroField.from[String](URI.create)(_.toString) implicit val afDuration: AvroField[Duration] = AvroField.from[Long](Duration.ofMillis)(_.toMillis) @@ -169,7 +201,13 @@ class AvroTypeSuite extends MagnolifySuite { val at: AvroType[AvroTypes] = AvroType[AvroTypes] val copier = new Copier(at.schema) // original uses String as CharSequence implementation - val original = AvroTypes("String", "CharSequence", Array.emptyByteArray, null) + val original = AvroTypes( + "String", + "CharSequence", + Array.emptyByteArray, + ByteBuffer.allocate(0), + null + ) val copy = at.from(copier.apply(at.to(original))) // copy uses avro Utf8 as CharSequence implementation assert(original != copy) @@ -199,8 +237,10 @@ class AvroTypeSuite extends MagnolifySuite { test("MicrosLogicalTypes") { val schema = AvroType[LogicalMicros].schema assertLogicalType(schema, "i", "timestamp-micros") - assertLogicalType(schema, "dt", "local-timestamp-micros", false) - assertLogicalType(schema, "t", "time-micros") + assertLogicalType(schema, "ldt", "local-timestamp-micros", false) + assertLogicalType(schema, "lt", "time-micros") + assertLogicalType(schema, "jdt", "timestamp-micros") + assertLogicalType(schema, "jlt", "time-micros") } } @@ -211,8 +251,10 @@ class AvroTypeSuite extends MagnolifySuite { test("MilliLogicalTypes") { val schema = AvroType[LogicalMillis].schema assertLogicalType(schema, "i", "timestamp-millis") - assertLogicalType(schema, "dt", "local-timestamp-millis", false) - assertLogicalType(schema, "t", "time-millis") + assertLogicalType(schema, "ldt", "local-timestamp-millis", false) + assertLogicalType(schema, "lt", "time-millis") + assertLogicalType(schema, "jdt", "timestamp-millis") + assertLogicalType(schema, "jlt", "time-millis") } } @@ -229,8 +271,11 @@ class AvroTypeSuite extends MagnolifySuite { val schema = AvroType[LogicalBigQuery].schema assertLogicalType(schema, "bd", "decimal") assertLogicalType(schema, "i", "timestamp-micros") - assertLogicalType(schema, "dt", "datetime") - assertLogicalType(schema, "t", "time-micros") + assertLogicalType(schema, "lt", "time-micros") + assertLogicalType(schema, "ldt", "datetime") + assertLogicalType(schema, "jdt", "timestamp-micros") + assertLogicalType(schema, "jlt", "time-micros") + assertLogicalType(schema, "jldt", "datetime") } } @@ -367,14 +412,34 @@ class AvroTypeSuite extends MagnolifySuite { } case class Unsafe(b: Byte, c: Char, s: Short) -case class AvroTypes(str: String, cs: CharSequence, ba: Array[Byte], n: Null) +case class AvroTypes(str: String, cs: CharSequence, ba: Array[Byte], bb: ByteBuffer, n: Null) case class MapPrimitive(strMap: Map[String, Int], charSeqMap: Map[CharSequence, Int]) case class MapNested(m: Map[String, Nested], charSeqMap: Map[CharSequence, Nested]) -case class Logical(u: UUID, d: LocalDate) -case class LogicalMicros(i: Instant, t: LocalTime, dt: LocalDateTime) -case class LogicalMillis(i: Instant, t: LocalTime, dt: LocalDateTime) -case class LogicalBigQuery(bd: BigDecimal, i: Instant, t: LocalTime, dt: LocalDateTime) +case class Logical(u: UUID, ld: LocalDate, jld: org.joda.time.LocalDate) +case class LogicalMicros( + i: Instant, + lt: LocalTime, + ldt: LocalDateTime, + jdt: org.joda.time.DateTime, + jlt: org.joda.time.LocalTime +) +case class LogicalMillis( + i: Instant, + lt: LocalTime, + ldt: LocalDateTime, + jdt: org.joda.time.DateTime, + jlt: org.joda.time.LocalTime +) +case class LogicalBigQuery( + bd: BigDecimal, + i: Instant, + lt: LocalTime, + ldt: LocalDateTime, + jdt: org.joda.time.DateTime, + jlt: org.joda.time.LocalTime, + jldt: org.joda.time.LocalDateTime +) case class BigDec(bd: BigDecimal) @doc("Fixed with doc") @@ -412,9 +477,9 @@ case class DefaultInner( bd: BigDecimal = BigDecimal(111.111), u: UUID = UUID.fromString("11112222-abcd-abcd-abcd-111122223333"), ts: Instant = Instant.ofEpochSecond(11223344), - d: LocalDate = LocalDate.ofEpochDay(1122), - t: LocalTime = LocalTime.of(1, 2, 3), - dt: LocalDateTime = LocalDateTime.of(2001, 2, 3, 4, 5, 6) + ld: LocalDate = LocalDate.ofEpochDay(1122), + lt: LocalTime = LocalTime.of(1, 2, 3), + ldt: LocalDateTime = LocalDateTime.of(2001, 2, 3, 4, 5, 6) ) case class DefaultOuter( i: DefaultInner = DefaultInner( diff --git a/build.sbt b/build.sbt index af998c048..8ea111386 100644 --- a/build.sbt +++ b/build.sbt @@ -20,7 +20,7 @@ val magnoliaScala2Version = "1.1.3" val magnoliaScala3Version = "1.1.4" val algebirdVersion = "0.13.9" -val avroVersion = Option(sys.props("avro.version")).getOrElse("1.11.0") +val avroVersion = Option(sys.props("avro.version")).getOrElse("1.11.1") val bigqueryVersion = "v2-rev20230422-2.0.0" val bigtableVersion = "2.22.0" val catsVersion = "2.9.0" @@ -28,6 +28,7 @@ val datastoreVersion = "2.14.5" val guavaVersion = "31.1-jre" val hadoopVersion = "3.3.5" val jacksonVersion = "2.15.0" +val jodaTimeVersion = "2.12.5" val munitVersion = "0.7.29" val neo4jDriverVersion = "4.4.9" val paigesVersion = "0.4.2" @@ -375,6 +376,7 @@ lazy val avro = project description := "Magnolia add-on for Apache Avro", libraryDependencies ++= Seq( "org.apache.avro" % "avro" % avroVersion % Provided, + "joda-time" % "joda-time" % jodaTimeVersion % Provided, "com.fasterxml.jackson.core" % "jackson-databind" % jacksonVersion % Test ) ) diff --git a/cats/src/test/scala/magnolify/cats/TestEq.scala b/cats/src/test/scala/magnolify/cats/TestEq.scala index b7fb7d396..fdf962728 100644 --- a/cats/src/test/scala/magnolify/cats/TestEq.scala +++ b/cats/src/test/scala/magnolify/cats/TestEq.scala @@ -31,13 +31,10 @@ object TestEq { // other implicit lazy val eqNull: Eq[Null] = Eq.allEqual implicit lazy val eqUri: Eq[URI] = Eq.fromUniversalEquals - implicit lazy val eqArray: Eq[Array[Int]] = Eq.by(_.toList) - implicit def eqIterable[T, C[_]](implicit eq: Eq[T], ti: C[T] => Iterable[T]): Eq[C[T]] = - Eq.instance { (x, y) => - val xs = ti(x) - val ys = ti(y) - xs.size == ys.size && (xs zip ys).forall((eq.eqv _).tupled) - } + implicit def eqArray[T: Eq]: Eq[Array[T]] = + Eq.by(_.toList) + implicit def eqIterable[T: Eq, C[_]](implicit ti: C[T] => Iterable[T]): Eq[C[T]] = + Eq.by[C[T], List[T]](ti(_).toList)(Eq.catsKernelEqForList[T]) // java implicit lazy val eqCharSequence: Eq[CharSequence] = Eq.by(_.toString) @@ -48,12 +45,12 @@ object TestEq { } // time - implicit lazy val eqInstant: Eq[Instant] = Eq.by(_.toEpochMilli) - implicit lazy val eqLocalDate: Eq[LocalDate] = Eq.by(_.toEpochDay) - implicit lazy val eqLocalTime: Eq[LocalTime] = Eq.by(_.toNanoOfDay) - implicit lazy val eqLocalDateTime: Eq[LocalDateTime] = Eq.by(_.toEpochSecond(ZoneOffset.UTC)) - implicit lazy val eqOffsetTime: Eq[OffsetTime] = Eq.by(_.toLocalTime.toNanoOfDay) - implicit lazy val eqDuration: Eq[Duration] = Eq.by(_.toMillis) + implicit lazy val eqInstant: Eq[Instant] = Eq.fromUniversalEquals + implicit lazy val eqLocalDate: Eq[LocalDate] = Eq.fromUniversalEquals + implicit lazy val eqLocalTime: Eq[LocalTime] = Eq.fromUniversalEquals + implicit lazy val eqLocalDateTime: Eq[LocalDateTime] = Eq.fromUniversalEquals + implicit lazy val eqOffsetTime: Eq[OffsetTime] = Eq.fromUniversalEquals + implicit lazy val eqDuration: Eq[Duration] = Eq.fromUniversalEquals // enum implicit lazy val eqJavaEnum: Eq[JavaEnums.Color] = Eq.fromUniversalEquals From 9ed90653e2129df9cd887f4e468fb025d6efcc63 Mon Sep 17 00:00:00 2001 From: Michel Davit Date: Thu, 4 May 2023 15:56:44 +0200 Subject: [PATCH 2/9] Move joda-time to common --- .../main/scala/magnolify/avro/AvroType.scala | 4 +- .../scala/magnolify/avro/AvroTypeSuite.scala | 38 ++----------------- build.sbt | 36 +++++++++--------- .../test/scala/magnolify/cats/TestEq.scala | 10 ++++- .../magnolify/scalacheck/TestArbitrary.scala | 28 +++++++++++++- 5 files changed, 59 insertions(+), 57 deletions(-) diff --git a/avro/src/main/scala/magnolify/avro/AvroType.scala b/avro/src/main/scala/magnolify/avro/AvroType.scala index bd6b31ade..58d035071 100644 --- a/avro/src/main/scala/magnolify/avro/AvroType.scala +++ b/avro/src/main/scala/magnolify/avro/AvroType.scala @@ -317,9 +317,11 @@ object AvroField { logicalType[CharSequence](LogicalTypes.uuid())(cs => ju.UUID.fromString(cs.toString))( _.toString ) + + // date implicit val afDate: AvroField[LocalDate] = logicalType[Int](LogicalTypes.date())(x => LocalDate.ofEpochDay(x.toLong))(_.toEpochDay.toInt) - private val epochJodaDate = new org.joda.time.LocalDate(1970, 1, 1); + private val epochJodaDate = new org.joda.time.LocalDate(1970, 1, 1) implicit val afJodaDate: AvroField[org.joda.time.LocalDate] = logicalType[Int](LogicalTypes.date()) { daysFromEpoch => epochJodaDate.plusDays(daysFromEpoch) diff --git a/avro/src/test/scala/magnolify/avro/AvroTypeSuite.scala b/avro/src/test/scala/magnolify/avro/AvroTypeSuite.scala index c0cf63ece..4f08357a7 100644 --- a/avro/src/test/scala/magnolify/avro/AvroTypeSuite.scala +++ b/avro/src/test/scala/magnolify/avro/AvroTypeSuite.scala @@ -40,11 +40,7 @@ import java.io.ByteArrayInputStream import java.io.ByteArrayOutputStream import java.net.URI import java.nio.ByteBuffer -import java.time.Duration -import java.time.Instant -import java.time.LocalDate -import java.time.LocalDateTime -import java.time.LocalTime +import java.time.{Duration, Instant, LocalDate, LocalDateTime, LocalTime} import java.time.format.DateTimeFormatter import java.util.UUID import scala.jdk.CollectionConverters._ @@ -139,29 +135,6 @@ class AvroTypeSuite extends MagnolifySuite { } } - implicit val arbJodaDate: Arbitrary[org.joda.time.LocalDate] = Arbitrary { - Arbitrary.arbitrary[LocalDate].map { ld => - new org.joda.time.LocalDate(ld.getYear, ld.getMonthValue, ld.getDayOfMonth) - } - } - implicit val arbJodaDateTime: Arbitrary[org.joda.time.DateTime] = Arbitrary { - Arbitrary.arbitrary[Instant].map { i => - new org.joda.time.DateTime(i.toEpochMilli, org.joda.time.DateTimeZone.UTC) - } - } - implicit val arbJodaLocalTime: Arbitrary[org.joda.time.LocalTime] = Arbitrary { - Arbitrary.arbitrary[LocalTime].map { lt => - org.joda.time.LocalTime.fromMillisOfDay(lt.toNanoOfDay / 1000) - } - } - implicit val arbJodaLocalDateTime: Arbitrary[org.joda.time.LocalDateTime] = Arbitrary { - Arbitrary.arbitrary[LocalDateTime].map { ldt => - org.joda.time.LocalDateTime.parse(ldt.toString) - } - } - implicit val arbByteBuffer: Arbitrary[ByteBuffer] = Arbitrary { - Arbitrary.arbitrary[Array[Byte]].map(ByteBuffer.wrap) - } implicit val arbBigDecimal: Arbitrary[BigDecimal] = Arbitrary { // bq logical type has precision of 38 and scale of 9 val max = BigInt(10).pow(38) - 1 @@ -171,13 +144,8 @@ class AvroTypeSuite extends MagnolifySuite { Gen.oneOf("US", "UK", "CA", "MX").map(CountryCode.apply) ) - implicit val eqJodaDate: Eq[org.joda.time.LocalDate] = Eq.fromUniversalEquals - implicit val eqJodaDateTime: Eq[org.joda.time.DateTime] = Eq.fromUniversalEquals - implicit val eqJodaLocalTime: Eq[org.joda.time.LocalTime] = Eq.fromUniversalEquals - implicit val eqJodaLocalDateTime: Eq[org.joda.time.LocalDateTime] = Eq.fromUniversalEquals - implicit val eqByteBuffer: Eq[ByteBuffer] = Eq.by(_.array()) - - implicit val afUri: AvroField[URI] = AvroField.from[String](URI.create)(_.toString) + implicit val afUri: AvroField[URI] = + AvroField.from[String](URI.create)(_.toString) implicit val afDuration: AvroField[Duration] = AvroField.from[Long](Duration.ofMillis)(_.toMillis) implicit val afCountryCode: AvroField[CountryCode] = diff --git a/build.sbt b/build.sbt index 8ea111386..a72b843e1 100644 --- a/build.sbt +++ b/build.sbt @@ -217,24 +217,23 @@ val commonSettings = Seq( HeaderFileType.scala -> keepExistingHeader, HeaderFileType.java -> keepExistingHeader ), - libraryDependencies ++= { - CrossVersion.partialVersion(scalaVersion.value) match { - case Some((3, _)) => - Seq( - "com.softwaremill.magnolia1_3" %% "magnolia" % magnoliaScala3Version, - "org.scala-lang.modules" %% "scala-collection-compat" % scalaCollectionCompatVersion - ) - case Some((2, _)) => - Seq( - "com.softwaremill.magnolia1_2" %% "magnolia" % magnoliaScala2Version, - "com.chuusai" %% "shapeless" % shapelessVersion, - "org.scala-lang.modules" %% "scala-collection-compat" % scalaCollectionCompatVersion, - "org.scala-lang" % "scala-reflect" % scalaVersion.value % Provided - ) - case _ => - throw new Exception("Unsupported scala version") - } - }, + libraryDependencies ++= Seq( + "org.scala-lang.modules" %% "scala-collection-compat" % scalaCollectionCompatVersion, + "joda-time" % "joda-time" % jodaTimeVersion % Provided + ) ++ (CrossVersion.partialVersion(scalaVersion.value) match { + case Some((3, _)) => + Seq( + "com.softwaremill.magnolia1_3" %% "magnolia" % magnoliaScala3Version + ) + case Some((2, _)) => + Seq( + "com.softwaremill.magnolia1_2" %% "magnolia" % magnoliaScala2Version, + "com.chuusai" %% "shapeless" % shapelessVersion, + "org.scala-lang" % "scala-reflect" % scalaVersion.value % Provided + ) + case _ => + throw new Exception("Unsupported scala version") + }), // https://github.com/typelevel/scalacheck/pull/427#issuecomment-424330310 // FIXME: workaround for Java serialization issues Test / classLoaderLayeringStrategy := ClassLoaderLayeringStrategy.Flat @@ -376,7 +375,6 @@ lazy val avro = project description := "Magnolia add-on for Apache Avro", libraryDependencies ++= Seq( "org.apache.avro" % "avro" % avroVersion % Provided, - "joda-time" % "joda-time" % jodaTimeVersion % Provided, "com.fasterxml.jackson.core" % "jackson-databind" % jacksonVersion % Test ) ) diff --git a/cats/src/test/scala/magnolify/cats/TestEq.scala b/cats/src/test/scala/magnolify/cats/TestEq.scala index fdf962728..6a930cbdc 100644 --- a/cats/src/test/scala/magnolify/cats/TestEq.scala +++ b/cats/src/test/scala/magnolify/cats/TestEq.scala @@ -24,6 +24,7 @@ import magnolify.test.Simple._ import magnolify.shared.UnsafeEnum import java.net.URI +import java.nio.ByteBuffer import java.time._ object TestEq { @@ -43,8 +44,9 @@ object TestEq { // Can only be used as a key value list m.map { case (k, v) => k.toString -> v } } + implicit val eqByteBuffer: Eq[ByteBuffer] = Eq.by(_.array()) - // time + // java-time implicit lazy val eqInstant: Eq[Instant] = Eq.fromUniversalEquals implicit lazy val eqLocalDate: Eq[LocalDate] = Eq.fromUniversalEquals implicit lazy val eqLocalTime: Eq[LocalTime] = Eq.fromUniversalEquals @@ -52,6 +54,12 @@ object TestEq { implicit lazy val eqOffsetTime: Eq[OffsetTime] = Eq.fromUniversalEquals implicit lazy val eqDuration: Eq[Duration] = Eq.fromUniversalEquals + // joda-time + implicit val eqJodaDate: Eq[org.joda.time.LocalDate] = Eq.fromUniversalEquals + implicit val eqJodaDateTime: Eq[org.joda.time.DateTime] = Eq.fromUniversalEquals + implicit val eqJodaLocalTime: Eq[org.joda.time.LocalTime] = Eq.fromUniversalEquals + implicit val eqJodaLocalDateTime: Eq[org.joda.time.LocalDateTime] = Eq.fromUniversalEquals + // enum implicit lazy val eqJavaEnum: Eq[JavaEnums.Color] = Eq.fromUniversalEquals implicit lazy val eqScalaEnum: Eq[ScalaEnums.Color.Type] = Eq.fromUniversalEquals diff --git a/scalacheck/src/test/scala/magnolify/scalacheck/TestArbitrary.scala b/scalacheck/src/test/scala/magnolify/scalacheck/TestArbitrary.scala index 901070e09..7bbaf10c1 100644 --- a/scalacheck/src/test/scala/magnolify/scalacheck/TestArbitrary.scala +++ b/scalacheck/src/test/scala/magnolify/scalacheck/TestArbitrary.scala @@ -25,6 +25,7 @@ import org.scalacheck._ import java.time._ import java.net.URI +import java.nio.ByteBuffer object TestArbitrary { // null @@ -38,8 +39,11 @@ object TestArbitrary { sb } } + implicit val arbByteBuffer: Arbitrary[ByteBuffer] = Arbitrary { + Arbitrary.arbitrary[Array[Byte]].map(ByteBuffer.wrap) + } - // time + // java-time implicit lazy val arbInstant: Arbitrary[Instant] = Arbitrary(Gen.posNum[Long].map(Instant.ofEpochMilli)) implicit lazy val arbLocalDate: Arbitrary[LocalDate] = @@ -53,6 +57,28 @@ object TestArbitrary { implicit lazy val arbDuration: Arbitrary[Duration] = Arbitrary(Gen.posNum[Long].map(Duration.ofMillis)) + // joda-time + implicit val arbJodaDate: Arbitrary[org.joda.time.LocalDate] = Arbitrary { + Arbitrary.arbitrary[LocalDate].map { ld => + new org.joda.time.LocalDate(ld.getYear, ld.getMonthValue, ld.getDayOfMonth) + } + } + implicit val arbJodaDateTime: Arbitrary[org.joda.time.DateTime] = Arbitrary { + Arbitrary.arbitrary[Instant].map { i => + new org.joda.time.DateTime(i.toEpochMilli, org.joda.time.DateTimeZone.UTC) + } + } + implicit val arbJodaLocalTime: Arbitrary[org.joda.time.LocalTime] = Arbitrary { + Arbitrary.arbitrary[LocalTime].map { lt => + org.joda.time.LocalTime.fromMillisOfDay(lt.toNanoOfDay / 1000) + } + } + implicit val arbJodaLocalDateTime: Arbitrary[org.joda.time.LocalDateTime] = Arbitrary { + Arbitrary.arbitrary[LocalDateTime].map { ldt => + org.joda.time.LocalDateTime.parse(ldt.toString) + } + } + // enum implicit lazy val arbJavaEnum: Arbitrary[JavaEnums.Color] = Arbitrary(Gen.oneOf(JavaEnums.Color.values.toSeq)) From 71abf9239b4d6fb7c8434211fa2dc0a1353b6518 Mon Sep 17 00:00:00 2001 From: Michel Davit Date: Thu, 4 May 2023 15:57:04 +0200 Subject: [PATCH 3/9] Add avro duration logical-type --- .../main/scala/magnolify/avro/AvroType.scala | 18 ++++++++++++++++++ .../scala/magnolify/avro/AvroTypeSuite.scala | 12 ++++++++++-- 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/avro/src/main/scala/magnolify/avro/AvroType.scala b/avro/src/main/scala/magnolify/avro/AvroType.scala index 58d035071..d6d503222 100644 --- a/avro/src/main/scala/magnolify/avro/AvroType.scala +++ b/avro/src/main/scala/magnolify/avro/AvroType.scala @@ -329,6 +329,24 @@ object AvroField { org.joda.time.Days.daysBetween(epochJodaDate, date).getDays } + // duration, as in the avro spec. do not make implicit as there is not a specific type for it + // A duration logical type annotates Avro fixed type of size 12, which stores three little-endian unsigned integers + // that represent durations at different granularities of time. + // The first stores a number in months, the second stores a number in days, and the third stores a number in milliseconds. + val afDuration: AvroField[(Int, Int, Int)] = + logicalType[ByteBuffer](new LogicalType("duration")) { bs => + val months = bs.getInt + val days = bs.getInt + val millis = bs.getInt + (months, days, millis) + } { case (months, days, millis) => + val bs = ByteBuffer.allocate(12) + bs.putInt(months) + bs.putInt(days) + bs.putInt(millis) + bs + }(AvroField.fixed(12)(ByteBuffer.wrap)(_.array())) + def fixed[T: ClassTag]( size: Int )(f: Array[Byte] => T)(g: T => Array[Byte])(implicit an: AnnotationType[T]): AvroField[T] = diff --git a/avro/src/test/scala/magnolify/avro/AvroTypeSuite.scala b/avro/src/test/scala/magnolify/avro/AvroTypeSuite.scala index 4f08357a7..96d05e456 100644 --- a/avro/src/test/scala/magnolify/avro/AvroTypeSuite.scala +++ b/avro/src/test/scala/magnolify/avro/AvroTypeSuite.scala @@ -196,7 +196,15 @@ class AvroTypeSuite extends MagnolifySuite { test[MapPrimitive] test[MapNested] - test[Logical] + { + implicit val afDuration: AvroField[(Int, Int, Int)] = AvroField.afDuration + test[Logical] + + test("Duration") { + val schema = AvroType[Logical].schema + assertLogicalType(schema, "d", "duration") + } + } { import magnolify.avro.logical.micros._ @@ -384,7 +392,7 @@ case class AvroTypes(str: String, cs: CharSequence, ba: Array[Byte], bb: ByteBuf case class MapPrimitive(strMap: Map[String, Int], charSeqMap: Map[CharSequence, Int]) case class MapNested(m: Map[String, Nested], charSeqMap: Map[CharSequence, Nested]) -case class Logical(u: UUID, ld: LocalDate, jld: org.joda.time.LocalDate) +case class Logical(u: UUID, ld: LocalDate, jld: org.joda.time.LocalDate, d: (Int, Int, Int)) case class LogicalMicros( i: Instant, lt: LocalTime, From 5f23eb730d5a3863c4870efc233bc12cb9017a55 Mon Sep 17 00:00:00 2001 From: Michel Davit Date: Thu, 4 May 2023 16:34:40 +0200 Subject: [PATCH 4/9] handle unsigned integers values properly --- .../main/scala/magnolify/avro/AvroType.scala | 22 ++++++++++--------- .../scala/magnolify/avro/AvroTypeSuite.scala | 17 ++++++++++++-- 2 files changed, 27 insertions(+), 12 deletions(-) diff --git a/avro/src/main/scala/magnolify/avro/AvroType.scala b/avro/src/main/scala/magnolify/avro/AvroType.scala index d6d503222..08bba45f3 100644 --- a/avro/src/main/scala/magnolify/avro/AvroType.scala +++ b/avro/src/main/scala/magnolify/avro/AvroType.scala @@ -16,7 +16,7 @@ package magnolify.avro -import java.nio.ByteBuffer +import java.nio.{ByteBuffer, ByteOrder} import java.time._ import java.{util => ju} import magnolia1._ @@ -333,18 +333,20 @@ object AvroField { // A duration logical type annotates Avro fixed type of size 12, which stores three little-endian unsigned integers // that represent durations at different granularities of time. // The first stores a number in months, the second stores a number in days, and the third stores a number in milliseconds. - val afDuration: AvroField[(Int, Int, Int)] = + val afDuration: AvroField[(Long, Long, Long)] = logicalType[ByteBuffer](new LogicalType("duration")) { bs => - val months = bs.getInt - val days = bs.getInt - val millis = bs.getInt + bs.order(ByteOrder.LITTLE_ENDIAN) + val months = java.lang.Integer.toUnsignedLong(bs.getInt) + val days = java.lang.Integer.toUnsignedLong(bs.getInt) + val millis = java.lang.Integer.toUnsignedLong(bs.getInt) (months, days, millis) } { case (months, days, millis) => - val bs = ByteBuffer.allocate(12) - bs.putInt(months) - bs.putInt(days) - bs.putInt(millis) - bs + ByteBuffer + .allocate(12) + .order(ByteOrder.LITTLE_ENDIAN) + .putInt(months.toInt) + .putInt(days.toInt) + .putInt(millis.toInt) }(AvroField.fixed(12)(ByteBuffer.wrap)(_.array())) def fixed[T: ClassTag]( diff --git a/avro/src/test/scala/magnolify/avro/AvroTypeSuite.scala b/avro/src/test/scala/magnolify/avro/AvroTypeSuite.scala index 96d05e456..6dfe31c9a 100644 --- a/avro/src/test/scala/magnolify/avro/AvroTypeSuite.scala +++ b/avro/src/test/scala/magnolify/avro/AvroTypeSuite.scala @@ -197,7 +197,19 @@ class AvroTypeSuite extends MagnolifySuite { test[MapNested] { - implicit val afDuration: AvroField[(Int, Int, Int)] = AvroField.afDuration + // generate unsigned-int duration values + implicit val arbAvroDuration: Arbitrary[AvroDuration] = Arbitrary { + for { + months <- Gen.chooseNum(0L, 0xffffffffL) + days <- Gen.chooseNum(0L, 0xffffffffL) + millis <- Gen.chooseNum(0L, 0xffffffffL) + } yield AvroDuration(months, days, millis) + } + + implicit val afAvroDuration: AvroField[AvroDuration] = AvroField.from[(Long, Long, Long)] { + case (months, days, millis) => AvroDuration(months, days, millis) + }(d => (d.months, d.days, d.millis))(AvroField.afDuration) + test[Logical] test("Duration") { @@ -392,7 +404,8 @@ case class AvroTypes(str: String, cs: CharSequence, ba: Array[Byte], bb: ByteBuf case class MapPrimitive(strMap: Map[String, Int], charSeqMap: Map[CharSequence, Int]) case class MapNested(m: Map[String, Nested], charSeqMap: Map[CharSequence, Nested]) -case class Logical(u: UUID, ld: LocalDate, jld: org.joda.time.LocalDate, d: (Int, Int, Int)) +case class AvroDuration(months: Long, days: Long, millis: Long) +case class Logical(u: UUID, ld: LocalDate, jld: org.joda.time.LocalDate, d: AvroDuration) case class LogicalMicros( i: Instant, lt: LocalTime, From ae2d53ccd75538724404d00c9fd7b3cadb349567 Mon Sep 17 00:00:00 2001 From: Michel Davit Date: Thu, 4 May 2023 16:41:32 +0200 Subject: [PATCH 5/9] Add all logical types tests --- avro/src/test/scala/magnolify/avro/AvroTypeSuite.scala | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/avro/src/test/scala/magnolify/avro/AvroTypeSuite.scala b/avro/src/test/scala/magnolify/avro/AvroTypeSuite.scala index 6dfe31c9a..dee6ce512 100644 --- a/avro/src/test/scala/magnolify/avro/AvroTypeSuite.scala +++ b/avro/src/test/scala/magnolify/avro/AvroTypeSuite.scala @@ -212,8 +212,10 @@ class AvroTypeSuite extends MagnolifySuite { test[Logical] - test("Duration") { + test("LogicalTypes") { val schema = AvroType[Logical].schema + assertLogicalType(schema, "ld", "date") + assertLogicalType(schema, "jld", "date") assertLogicalType(schema, "d", "duration") } } From 4ba749cb4b21216bd0c9d8f16127127c90ec9160 Mon Sep 17 00:00:00 2001 From: Michel Davit Date: Thu, 4 May 2023 16:43:33 +0200 Subject: [PATCH 6/9] FIx constant naming --- avro/src/main/scala/magnolify/avro/AvroType.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/avro/src/main/scala/magnolify/avro/AvroType.scala b/avro/src/main/scala/magnolify/avro/AvroType.scala index 08bba45f3..db6ee8a8f 100644 --- a/avro/src/main/scala/magnolify/avro/AvroType.scala +++ b/avro/src/main/scala/magnolify/avro/AvroType.scala @@ -321,12 +321,12 @@ object AvroField { // date implicit val afDate: AvroField[LocalDate] = logicalType[Int](LogicalTypes.date())(x => LocalDate.ofEpochDay(x.toLong))(_.toEpochDay.toInt) - private val epochJodaDate = new org.joda.time.LocalDate(1970, 1, 1) + private val EpochJodaDate = new org.joda.time.LocalDate(1970, 1, 1) implicit val afJodaDate: AvroField[org.joda.time.LocalDate] = logicalType[Int](LogicalTypes.date()) { daysFromEpoch => - epochJodaDate.plusDays(daysFromEpoch) + EpochJodaDate.plusDays(daysFromEpoch) } { date => - org.joda.time.Days.daysBetween(epochJodaDate, date).getDays + org.joda.time.Days.daysBetween(EpochJodaDate, date).getDays } // duration, as in the avro spec. do not make implicit as there is not a specific type for it From fdc9adc80a21e38b98177b9df83e96523d172c5c Mon Sep 17 00:00:00 2001 From: Michel Davit Date: Thu, 4 May 2023 18:11:34 +0200 Subject: [PATCH 7/9] Fix default bytes behavior --- .../main/scala/magnolify/avro/AvroType.scala | 12 +++++----- .../scala/magnolify/avro/AvroTypeSuite.scala | 24 ++++++++++++++----- 2 files changed, 24 insertions(+), 12 deletions(-) diff --git a/avro/src/main/scala/magnolify/avro/AvroType.scala b/avro/src/main/scala/magnolify/avro/AvroType.scala index db6ee8a8f..113de4582 100644 --- a/avro/src/main/scala/magnolify/avro/AvroType.scala +++ b/avro/src/main/scala/magnolify/avro/AvroType.scala @@ -31,6 +31,7 @@ import scala.collection.concurrent import scala.reflect.ClassTag import scala.jdk.CollectionConverters._ import scala.collection.compat._ +import scala.util.chaining._ sealed trait AvroType[T] extends Converter[T, GenericRecord, GenericRecord] { val schema: Schema @@ -196,15 +197,14 @@ object AvroField { implicit val afLong: AvroField[Long] = id[Long](Schema.Type.LONG) implicit val afFloat: AvroField[Float] = id[Float](Schema.Type.FLOAT) implicit val afDouble: AvroField[Double] = id[Double](Schema.Type.DOUBLE) - implicit val afByteBuffer: AvroField[ByteBuffer] = id[ByteBuffer](Schema.Type.BYTES) - implicit val afBytes: AvroField[Array[Byte]] = new Aux[Array[Byte], ByteBuffer, ByteBuffer] { + implicit val afByteBuffer: AvroField[ByteBuffer] = new Aux[ByteBuffer, ByteBuffer, ByteBuffer] { override protected def buildSchema(cm: CaseMapper): Schema = Schema.create(Schema.Type.BYTES) // `JacksonUtils.toJson` expects `Array[Byte]` for `BYTES` defaults - override def makeDefault(d: Array[Byte])(cm: CaseMapper): Array[Byte] = d - override def from(v: ByteBuffer)(cm: CaseMapper): Array[Byte] = - ju.Arrays.copyOfRange(v.array(), v.position(), v.limit()) - override def to(v: Array[Byte])(cm: CaseMapper): ByteBuffer = ByteBuffer.wrap(v) + override def makeDefault(d: ByteBuffer)(cm: CaseMapper): Array[Byte] = d.array() + override def from(v: ByteBuffer)(cm: CaseMapper): ByteBuffer = v + override def to(v: ByteBuffer)(cm: CaseMapper): ByteBuffer = v } + implicit val afBytes: AvroField[Array[Byte]] = from[ByteBuffer](_.array())(ByteBuffer.wrap) implicit val afCharSequence: AvroField[CharSequence] = id[CharSequence](Schema.Type.STRING) implicit val afString: AvroField[String] = new Aux[String, String, String] { override protected def buildSchema(cm: CaseMapper): Schema = { diff --git a/avro/src/test/scala/magnolify/avro/AvroTypeSuite.scala b/avro/src/test/scala/magnolify/avro/AvroTypeSuite.scala index dee6ce512..181aa5e91 100644 --- a/avro/src/test/scala/magnolify/avro/AvroTypeSuite.scala +++ b/avro/src/test/scala/magnolify/avro/AvroTypeSuite.scala @@ -322,7 +322,7 @@ class AvroTypeSuite extends MagnolifySuite { Map("b" -> 2), JavaEnums.Color.GREEN, ScalaEnums.Color.Green, - BigDecimal(222.222), + BigDecimal(222.222222222), // scale is 9 UUID.fromString("22223333-abcd-abcd-abcd-222233334444"), Instant.ofEpochSecond(22334455L), LocalDate.ofEpochDay(2233), @@ -342,7 +342,7 @@ class AvroTypeSuite extends MagnolifySuite { Map("c" -> 3), JavaEnums.Color.BLUE, ScalaEnums.Color.Blue, - BigDecimal(333.333), + BigDecimal(333.333333333), // scale is 9 UUID.fromString("33334444-abcd-abcd-abcd-333344445555"), Instant.ofEpochSecond(33445566L), LocalDate.ofEpochDay(3344), @@ -354,7 +354,13 @@ class AvroTypeSuite extends MagnolifySuite { } } - testFail(AvroType[SomeDefault])("Option[T] can only default to None") + test("ArrayDefault") { + val at = ensureSerializable(AvroType[DefaultBytes]) + // array use reference equality, convert to Seq + assertEquals(at(new GenericRecordBuilder(at.schema).build()).a.toSeq, DefaultBytes().a.toSeq) + } + + testFail(AvroType[DefaultSome])("Option[T] can only default to None") { implicit val at: AvroType[LowerCamel] = AvroType[LowerCamel](CaseMapper(_.toUpperCase)) @@ -465,7 +471,7 @@ case class DefaultInner( m: Map[String, Int] = Map("a" -> 1), je: JavaEnums.Color = JavaEnums.Color.RED, se: ScalaEnums.Color.Type = ScalaEnums.Color.Red, - bd: BigDecimal = BigDecimal(111.111), + bd: BigDecimal = BigDecimal(111.111111111), // scale is 9 u: UUID = UUID.fromString("11112222-abcd-abcd-abcd-111122223333"), ts: Instant = Instant.ofEpochSecond(11223344), ld: LocalDate = LocalDate.ofEpochDay(1122), @@ -480,7 +486,7 @@ case class DefaultOuter( Map("b" -> 2), JavaEnums.Color.GREEN, ScalaEnums.Color.Green, - BigDecimal(222.222), + BigDecimal(222.222222222), // scale is 9 UUID.fromString("22223333-abcd-abcd-abcd-222233334444"), Instant.ofEpochSecond(22334455L), LocalDate.ofEpochDay(2233), @@ -489,7 +495,13 @@ case class DefaultOuter( ), o: Option[DefaultInner] = None ) -case class SomeDefault(o: Option[Int] = Some(1)) +case class DefaultSome(o: Option[Int] = Some(1)) + +case class DefaultBytes( + a: Array[Byte] = Array(2, 2) + // ByteBuffer is not serializable and can't be used as default value + // bb: ByteBuffer = ByteBuffer.allocate(2).put(2.toByte).put(2.toByte) +) @doc("Avro enum") object Pet extends Enumeration { From 2ff7e54b9838fbb4301fb1767b22ad92a416d4ac Mon Sep 17 00:00:00 2001 From: Michel Davit Date: Fri, 5 May 2023 12:58:44 +0200 Subject: [PATCH 8/9] Use deepEquals for DefaultBytes test --- .../main/scala/magnolify/avro/AvroType.scala | 1 - .../scala/magnolify/avro/AvroTypeSuite.scala | 20 ++++++++++++++----- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/avro/src/main/scala/magnolify/avro/AvroType.scala b/avro/src/main/scala/magnolify/avro/AvroType.scala index 113de4582..d8c85f946 100644 --- a/avro/src/main/scala/magnolify/avro/AvroType.scala +++ b/avro/src/main/scala/magnolify/avro/AvroType.scala @@ -31,7 +31,6 @@ import scala.collection.concurrent import scala.reflect.ClassTag import scala.jdk.CollectionConverters._ import scala.collection.compat._ -import scala.util.chaining._ sealed trait AvroType[T] extends Converter[T, GenericRecord, GenericRecord] { val schema: Schema diff --git a/avro/src/test/scala/magnolify/avro/AvroTypeSuite.scala b/avro/src/test/scala/magnolify/avro/AvroTypeSuite.scala index 181aa5e91..b265518ba 100644 --- a/avro/src/test/scala/magnolify/avro/AvroTypeSuite.scala +++ b/avro/src/test/scala/magnolify/avro/AvroTypeSuite.scala @@ -42,7 +42,8 @@ import java.net.URI import java.nio.ByteBuffer import java.time.{Duration, Instant, LocalDate, LocalDateTime, LocalTime} import java.time.format.DateTimeFormatter -import java.util.UUID +import java.util +import java.util.{Objects, UUID} import scala.jdk.CollectionConverters._ import scala.reflect._ import scala.util.Try @@ -354,10 +355,9 @@ class AvroTypeSuite extends MagnolifySuite { } } - test("ArrayDefault") { + test("DefaultBytes") { val at = ensureSerializable(AvroType[DefaultBytes]) - // array use reference equality, convert to Seq - assertEquals(at(new GenericRecordBuilder(at.schema).build()).a.toSeq, DefaultBytes().a.toSeq) + assertEquals(at(new GenericRecordBuilder(at.schema).build()), DefaultBytes()) } testFail(AvroType[DefaultSome])("Option[T] can only default to None") @@ -501,7 +501,17 @@ case class DefaultBytes( a: Array[Byte] = Array(2, 2) // ByteBuffer is not serializable and can't be used as default value // bb: ByteBuffer = ByteBuffer.allocate(2).put(2.toByte).put(2.toByte) -) +) { + + override def hashCode(): Int = + util.Arrays.hashCode(a) + + override def equals(obj: Any): Boolean = obj match { + case that: DefaultBytes => Objects.deepEquals(this.a, that.a) + case _ => false + } + +} @doc("Avro enum") object Pet extends Enumeration { From f0114e073249d4779d0ef43d32253b8e97c6a6bb Mon Sep 17 00:00:00 2001 From: Michel Davit Date: Thu, 11 May 2023 11:34:42 +0200 Subject: [PATCH 9/9] Import org.joda.time package --- .../main/scala/magnolify/avro/AvroType.scala | 17 ++++---- .../magnolify/avro/logical/package.scala | 40 +++++++++---------- .../scala/magnolify/avro/AvroTypeSuite.scala | 32 +++++++-------- .../test/scala/magnolify/cats/TestEq.scala | 11 ++--- .../magnolify/scalacheck/TestArbitrary.scala | 19 ++++----- 5 files changed, 60 insertions(+), 59 deletions(-) diff --git a/avro/src/main/scala/magnolify/avro/AvroType.scala b/avro/src/main/scala/magnolify/avro/AvroType.scala index d8c85f946..ee0ac6615 100644 --- a/avro/src/main/scala/magnolify/avro/AvroType.scala +++ b/avro/src/main/scala/magnolify/avro/AvroType.scala @@ -16,21 +16,22 @@ package magnolify.avro -import java.nio.{ByteBuffer, ByteOrder} -import java.time._ -import java.{util => ju} import magnolia1._ import magnolify.shared._ import magnolify.shims.FactoryCompat import org.apache.avro.generic.GenericData.EnumSymbol import org.apache.avro.generic._ import org.apache.avro.{JsonProperties, LogicalType, LogicalTypes, Schema} +import org.joda.{time => joda} +import java.nio.{ByteBuffer, ByteOrder} +import java.time._ +import java.{util => ju} import scala.annotation.{implicitNotFound, nowarn} import scala.collection.concurrent -import scala.reflect.ClassTag -import scala.jdk.CollectionConverters._ import scala.collection.compat._ +import scala.jdk.CollectionConverters._ +import scala.reflect.ClassTag sealed trait AvroType[T] extends Converter[T, GenericRecord, GenericRecord] { val schema: Schema @@ -320,12 +321,12 @@ object AvroField { // date implicit val afDate: AvroField[LocalDate] = logicalType[Int](LogicalTypes.date())(x => LocalDate.ofEpochDay(x.toLong))(_.toEpochDay.toInt) - private val EpochJodaDate = new org.joda.time.LocalDate(1970, 1, 1) - implicit val afJodaDate: AvroField[org.joda.time.LocalDate] = + private lazy val EpochJodaDate = new joda.LocalDate(1970, 1, 1) + implicit val afJodaDate: AvroField[joda.LocalDate] = logicalType[Int](LogicalTypes.date()) { daysFromEpoch => EpochJodaDate.plusDays(daysFromEpoch) } { date => - org.joda.time.Days.daysBetween(EpochJodaDate, date).getDays + joda.Days.daysBetween(EpochJodaDate, date).getDays } // duration, as in the avro spec. do not make implicit as there is not a specific type for it diff --git a/avro/src/main/scala/magnolify/avro/logical/package.scala b/avro/src/main/scala/magnolify/avro/logical/package.scala index 03b8f8b02..a6d0d7473 100644 --- a/avro/src/main/scala/magnolify/avro/logical/package.scala +++ b/avro/src/main/scala/magnolify/avro/logical/package.scala @@ -17,11 +17,11 @@ package magnolify.avro import org.apache.avro.LogicalTypes.LogicalTypeFactory - -import java.time.{Instant, LocalDateTime, LocalTime, ZoneOffset} -import java.time.format.{DateTimeFormatter, DateTimeFormatterBuilder} import org.apache.avro.{LogicalType, LogicalTypes, Schema} +import org.joda.{time => joda} +import java.time._ +import java.time.format.{DateTimeFormatter, DateTimeFormatterBuilder} import java.util.concurrent.TimeUnit package object logical { @@ -70,16 +70,16 @@ package object logical { } // avro 1.8 uses joda-time - implicit val afJodaTimestampMicros: AvroField[org.joda.time.DateTime] = + implicit val afJodaTimestampMicros: AvroField[joda.DateTime] = AvroField.logicalType[Long](LogicalTypes.timestampMicros()) { microsFromEpoch => - new org.joda.time.DateTime(microsFromEpoch / 1000, org.joda.time.DateTimeZone.UTC) + new joda.DateTime(microsFromEpoch / 1000, joda.DateTimeZone.UTC) } { timestamp => 1000 * timestamp.getMillis } - implicit val afJodaTimeMicros: AvroField[org.joda.time.LocalTime] = + implicit val afJodaTimeMicros: AvroField[joda.LocalTime] = AvroField.logicalType[Long](LogicalTypes.timeMicros()) { microsFromMidnight => - org.joda.time.LocalTime.fromMillisOfDay(microsFromMidnight / 1000) + joda.LocalTime.fromMillisOfDay(microsFromMidnight / 1000) } { time => // from LossyTimeMicrosConversion 1000L * time.millisOfDay().get() @@ -112,16 +112,16 @@ package object logical { } // avro 1.8 uses joda-time - implicit val afJodaTimestampMillis: AvroField[org.joda.time.DateTime] = + implicit val afJodaTimestampMillis: AvroField[joda.DateTime] = AvroField.logicalType[Long](LogicalTypes.timestampMillis()) { millisFromEpoch => - new org.joda.time.DateTime(millisFromEpoch, org.joda.time.DateTimeZone.UTC) + new joda.DateTime(millisFromEpoch, joda.DateTimeZone.UTC) } { timestamp => timestamp.getMillis } - implicit val afJodaTimeMillis: AvroField[org.joda.time.LocalTime] = + implicit val afJodaTimeMillis: AvroField[joda.LocalTime] = AvroField.logicalType[Int](LogicalTypes.timeMillis()) { millisFromMidnight => - org.joda.time.LocalTime.fromMillisOfDay(millisFromMidnight.toLong) + joda.LocalTime.fromMillisOfDay(millisFromMidnight.toLong) } { time => time.millisOfDay().get() } @@ -166,18 +166,18 @@ package object logical { .toFormatter .withZone(ZoneOffset.UTC) - private val JodaDatetimePrinter = new org.joda.time.format.DateTimeFormatterBuilder() + private val JodaDatetimePrinter = new joda.format.DateTimeFormatterBuilder() .appendPattern(DatetimePattern) .toFormatter - private val JodaDatetimeParser = new org.joda.time.format.DateTimeFormatterBuilder() + private val JodaDatetimeParser = new joda.format.DateTimeFormatterBuilder() .appendPattern(DatePattern) .appendOptional( - new org.joda.time.format.DateTimeFormatterBuilder() + new joda.format.DateTimeFormatterBuilder() .appendLiteral(' ') .appendPattern(TimePattern) .appendOptional( - new org.joda.time.format.DateTimeFormatterBuilder() + new joda.format.DateTimeFormatterBuilder() .appendLiteral('.') .appendPattern(DecimalPattern) .toParser @@ -185,7 +185,7 @@ package object logical { .toParser ) .toFormatter - .withZone(org.joda.time.DateTimeZone.UTC) + .withZone(joda.DateTimeZone.UTC) // NUMERIC // https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types#numeric-type @@ -193,14 +193,14 @@ package object logical { // TIMESTAMP implicit val afBigQueryTimestamp: AvroField[Instant] = micros.afTimestampMicros - implicit val afBigQueryJodaTimestamp: AvroField[org.joda.time.DateTime] = + implicit val afBigQueryJodaTimestamp: AvroField[joda.DateTime] = micros.afJodaTimestampMicros // DATE: `AvroField.afDate` // TIME implicit val afBigQueryTime: AvroField[LocalTime] = micros.afTimeMicros - implicit val afBigQueryJodaTime: AvroField[org.joda.time.LocalTime] = micros.afJodaTimeMicros + implicit val afBigQueryJodaTime: AvroField[joda.LocalTime] = micros.afJodaTimeMicros // DATETIME -> sqlType: DATETIME implicit val afBigQueryDatetime: AvroField[LocalDateTime] = @@ -209,9 +209,9 @@ package object logical { } { datetime => DatetimePrinter.format(datetime) } - implicit val afBigQueryJodaDatetime: AvroField[org.joda.time.LocalDateTime] = + implicit val afBigQueryJodaDatetime: AvroField[joda.LocalDateTime] = AvroField.logicalType[CharSequence](new org.apache.avro.LogicalType(DateTimeTypeName)) { cs => - org.joda.time.LocalDateTime.parse(cs.toString, JodaDatetimeParser) + joda.LocalDateTime.parse(cs.toString, JodaDatetimeParser) } { datetime => JodaDatetimePrinter.print(datetime) } diff --git a/avro/src/test/scala/magnolify/avro/AvroTypeSuite.scala b/avro/src/test/scala/magnolify/avro/AvroTypeSuite.scala index b265518ba..9a0c37f3c 100644 --- a/avro/src/test/scala/magnolify/avro/AvroTypeSuite.scala +++ b/avro/src/test/scala/magnolify/avro/AvroTypeSuite.scala @@ -18,29 +18,27 @@ package magnolify.avro import cats._ import com.fasterxml.jackson.core.JsonFactory -import com.fasterxml.jackson.databind.JsonNode -import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.databind.{JsonNode, ObjectMapper} import magnolify.avro.unsafe._ -import magnolify.cats.auto._ import magnolify.cats.TestEq._ -import magnolify.scalacheck.auto._ +import magnolify.cats.auto._ import magnolify.scalacheck.TestArbitrary._ +import magnolify.scalacheck.auto._ import magnolify.shared.CaseMapper import magnolify.shared.TestEnumType._ import magnolify.test.Simple._ import magnolify.test._ import org.apache.avro.Schema import org.apache.avro.generic._ -import org.apache.avro.io.DecoderFactory -import org.apache.avro.io.EncoderFactory +import org.apache.avro.io.{DecoderFactory, EncoderFactory} import org.apache.avro.util.Utf8 +import org.joda.{time => joda} import org.scalacheck._ -import java.io.ByteArrayInputStream -import java.io.ByteArrayOutputStream +import java.io.{ByteArrayInputStream, ByteArrayOutputStream} import java.net.URI import java.nio.ByteBuffer -import java.time.{Duration, Instant, LocalDate, LocalDateTime, LocalTime} +import java.time._ import java.time.format.DateTimeFormatter import java.util import java.util.{Objects, UUID} @@ -413,29 +411,29 @@ case class MapPrimitive(strMap: Map[String, Int], charSeqMap: Map[CharSequence, case class MapNested(m: Map[String, Nested], charSeqMap: Map[CharSequence, Nested]) case class AvroDuration(months: Long, days: Long, millis: Long) -case class Logical(u: UUID, ld: LocalDate, jld: org.joda.time.LocalDate, d: AvroDuration) +case class Logical(u: UUID, ld: LocalDate, jld: joda.LocalDate, d: AvroDuration) case class LogicalMicros( i: Instant, lt: LocalTime, ldt: LocalDateTime, - jdt: org.joda.time.DateTime, - jlt: org.joda.time.LocalTime + jdt: joda.DateTime, + jlt: joda.LocalTime ) case class LogicalMillis( i: Instant, lt: LocalTime, ldt: LocalDateTime, - jdt: org.joda.time.DateTime, - jlt: org.joda.time.LocalTime + jdt: joda.DateTime, + jlt: joda.LocalTime ) case class LogicalBigQuery( bd: BigDecimal, i: Instant, lt: LocalTime, ldt: LocalDateTime, - jdt: org.joda.time.DateTime, - jlt: org.joda.time.LocalTime, - jldt: org.joda.time.LocalDateTime + jdt: joda.DateTime, + jlt: joda.LocalTime, + jldt: joda.LocalDateTime ) case class BigDec(bd: BigDecimal) diff --git a/cats/src/test/scala/magnolify/cats/TestEq.scala b/cats/src/test/scala/magnolify/cats/TestEq.scala index 6a930cbdc..152cccbd8 100644 --- a/cats/src/test/scala/magnolify/cats/TestEq.scala +++ b/cats/src/test/scala/magnolify/cats/TestEq.scala @@ -18,10 +18,11 @@ package magnolify.cats import cats.Eq import magnolify.cats.semiauto.EqDerivation +import magnolify.shared.UnsafeEnum import magnolify.test.ADT._ import magnolify.test.JavaEnums import magnolify.test.Simple._ -import magnolify.shared.UnsafeEnum +import org.joda.{time => joda} import java.net.URI import java.nio.ByteBuffer @@ -55,10 +56,10 @@ object TestEq { implicit lazy val eqDuration: Eq[Duration] = Eq.fromUniversalEquals // joda-time - implicit val eqJodaDate: Eq[org.joda.time.LocalDate] = Eq.fromUniversalEquals - implicit val eqJodaDateTime: Eq[org.joda.time.DateTime] = Eq.fromUniversalEquals - implicit val eqJodaLocalTime: Eq[org.joda.time.LocalTime] = Eq.fromUniversalEquals - implicit val eqJodaLocalDateTime: Eq[org.joda.time.LocalDateTime] = Eq.fromUniversalEquals + implicit val eqJodaDate: Eq[joda.LocalDate] = Eq.fromUniversalEquals + implicit val eqJodaDateTime: Eq[joda.DateTime] = Eq.fromUniversalEquals + implicit val eqJodaLocalTime: Eq[joda.LocalTime] = Eq.fromUniversalEquals + implicit val eqJodaLocalDateTime: Eq[joda.LocalDateTime] = Eq.fromUniversalEquals // enum implicit lazy val eqJavaEnum: Eq[JavaEnums.Color] = Eq.fromUniversalEquals diff --git a/scalacheck/src/test/scala/magnolify/scalacheck/TestArbitrary.scala b/scalacheck/src/test/scala/magnolify/scalacheck/TestArbitrary.scala index 7bbaf10c1..c0d9de75e 100644 --- a/scalacheck/src/test/scala/magnolify/scalacheck/TestArbitrary.scala +++ b/scalacheck/src/test/scala/magnolify/scalacheck/TestArbitrary.scala @@ -21,11 +21,12 @@ import magnolify.shared.UnsafeEnum import magnolify.test.ADT._ import magnolify.test.JavaEnums import magnolify.test.Simple._ +import org.joda.{time => joda} import org.scalacheck._ -import java.time._ import java.net.URI import java.nio.ByteBuffer +import java.time._ object TestArbitrary { // null @@ -58,24 +59,24 @@ object TestArbitrary { Arbitrary(Gen.posNum[Long].map(Duration.ofMillis)) // joda-time - implicit val arbJodaDate: Arbitrary[org.joda.time.LocalDate] = Arbitrary { + implicit val arbJodaDate: Arbitrary[joda.LocalDate] = Arbitrary { Arbitrary.arbitrary[LocalDate].map { ld => - new org.joda.time.LocalDate(ld.getYear, ld.getMonthValue, ld.getDayOfMonth) + new joda.LocalDate(ld.getYear, ld.getMonthValue, ld.getDayOfMonth) } } - implicit val arbJodaDateTime: Arbitrary[org.joda.time.DateTime] = Arbitrary { + implicit val arbJodaDateTime: Arbitrary[joda.DateTime] = Arbitrary { Arbitrary.arbitrary[Instant].map { i => - new org.joda.time.DateTime(i.toEpochMilli, org.joda.time.DateTimeZone.UTC) + new joda.DateTime(i.toEpochMilli, joda.DateTimeZone.UTC) } } - implicit val arbJodaLocalTime: Arbitrary[org.joda.time.LocalTime] = Arbitrary { + implicit val arbJodaLocalTime: Arbitrary[joda.LocalTime] = Arbitrary { Arbitrary.arbitrary[LocalTime].map { lt => - org.joda.time.LocalTime.fromMillisOfDay(lt.toNanoOfDay / 1000) + joda.LocalTime.fromMillisOfDay(lt.toNanoOfDay / 1000) } } - implicit val arbJodaLocalDateTime: Arbitrary[org.joda.time.LocalDateTime] = Arbitrary { + implicit val arbJodaLocalDateTime: Arbitrary[joda.LocalDateTime] = Arbitrary { Arbitrary.arbitrary[LocalDateTime].map { ldt => - org.joda.time.LocalDateTime.parse(ldt.toString) + joda.LocalDateTime.parse(ldt.toString) } }