Generating enums from IDL (#56)

Resolves #51

We now have separate "storage" and "exported" types
defined for STEF data types. For most types they are
equivalent except for Enums where the "storage" type
is uint64, while "exported" type is user defined.

The generated code now uses either "storage" or "exported"
type, depending on the context and converts between the
types as needed.

The Metric.Type field now uses enum code generation. Previously
manually created enums.go is deleted. enums.go also contains
MetricFlags which is not an enum that we want to generate so
it was moved to internal/metricflags.go.

I will need to refactor internal/metricflags.go, but will do it
in a future PR to keep the size of this PR manageable.

The generator needs more tests to verify how it works with enums
in other data types (oneof,array,multimap). We need to add a better
test suite for generator in the future.

go/otel/otel.stef

@@ -63,13 +63,21 @@ struct Metric dict(Metric) {
   Name string dict(MetricName)
   Description string dict(MetricDescription)
   Unit string dict(MetricUnit)
-  Type uint64
+  Type MetricType
   Metadata Attributes
   HistogramBounds []float64
   AggregationTemporality uint64
   Monotonic bool
 }
 
+enum MetricType {
+	Gauge = 0
+	Sum = 1
+	Histogram = 2
+	ExpHistogram = 3
+	Summary = 4
+}
+
 struct Metrics root {
   Envelope Envelope
   Metric Metric

go/pdata/internal/otlptools/convert.go

@@ -51,11 +51,11 @@ func MetricToOtlp(metric *oteltef.Metric, out pmetric.Metric) error {
 	}
 
 	switch oteltef.MetricType(metric.Type()) {
-	case oteltef.Gauge:
+	case oteltef.MetricTypeGauge:
 		out.SetEmptyGauge()
-	case oteltef.Sum:
+	case oteltef.MetricTypeSum:
 		out.SetEmptySum()
-	case oteltef.Histogram:
+	case oteltef.MetricTypeHistogram:
 		out.SetEmptyHistogram()
 	default:
 		panic("not implemented")

go/pdata/metrics/internal/basesteftotolp.go

@@ -79,28 +79,28 @@ func (s *BaseSTEFToOTLP) AppendOTLPPoint(
 		}
 		otlpAttrs.MoveTo(point.Attributes())
 		dstMetric.Sum().SetIsMonotonic(srcMetric.Monotonic())
-		dstMetric.Sum().SetAggregationTemporality(aggregationTemporalityToOtlp(oteltef.MetricFlags(srcMetric.AggregationTemporality())))
+		dstMetric.Sum().SetAggregationTemporality(aggregationTemporalityToOtlp(MetricFlags(srcMetric.AggregationTemporality())))
 	case pmetric.MetricTypeHistogram:
 		point := dstMetric.Histogram().DataPoints().AppendEmpty()
 		if err := s.convertHistogramPoint(srcMetric, srcPoint, point); err != nil {
 			return err
 		}
 		otlpAttrs.MoveTo(point.Attributes())
-		dstMetric.Histogram().SetAggregationTemporality(aggregationTemporalityToOtlp(oteltef.MetricFlags(srcMetric.AggregationTemporality())))
+		dstMetric.Histogram().SetAggregationTemporality(aggregationTemporalityToOtlp(MetricFlags(srcMetric.AggregationTemporality())))
 	default:
 		panic("not implemented")
 	}
 
 	return nil
 }
 
-func aggregationTemporalityToOtlp(flags oteltef.MetricFlags) pmetric.AggregationTemporality {
-	switch flags & oteltef.MetricTemporalityMask {
-	case oteltef.MetricTemporalityDelta:
+func aggregationTemporalityToOtlp(flags MetricFlags) pmetric.AggregationTemporality {
+	switch flags & MetricTemporalityMask {
+	case MetricTemporalityDelta:
 		return pmetric.AggregationTemporalityDelta
-	case oteltef.MetricTemporalityCumulative:
+	case MetricTemporalityCumulative:
 		return pmetric.AggregationTemporalityCumulative
-	case oteltef.MetricTemporalityUnspecified:
+	case MetricTemporalityUnspecified:
 		return pmetric.AggregationTemporalityUnspecified
 	default:
 		panic("unexpected metric flags")

go/otel/oteltef/enums.go → go/pdata/metrics/internal/metricflags.go

@@ -1,15 +1,4 @@
-package oteltef
-
-type MetricType uint
-
-const (
-	Gauge MetricType = iota
-	Sum
-	Histogram
-	ExpHistogram
-	Summary
-	MetricTypeLimit
-)
+package internal
 
 // MetricFlags is a bitmask
 type MetricFlags uint
@@ -21,11 +10,3 @@ const (
 	MetricTemporalityDelta       MetricFlags = 0b010
 	MetricTemporalityCumulative  MetricFlags = 0b100
 )
-
-//type HistogramFieldPresenceMask int64
-//
-//const (
-//	HistogramHasSum = HistogramFieldPresenceMask(1 << iota)
-//	HistogramHasMin
-//	HistogramHasMax
-//)

go/pdata/metrics/otlp2sortedtree.go

@@ -58,19 +58,19 @@ func (c *OtlpToSortedTree) FromOtlp(rms pmetric.ResourceMetricsSlice) (*sortedby
 	return sm, nil
 }
 
-func calcMetricFlags(monotonic bool, temporality pmetric.AggregationTemporality) oteltef.MetricFlags {
-	var flags oteltef.MetricFlags
+func calcMetricFlags(monotonic bool, temporality pmetric.AggregationTemporality) internal.MetricFlags {
+	var flags internal.MetricFlags
 	if monotonic {
-		flags |= oteltef.MetricMonotonic
+		flags |= internal.MetricMonotonic
 	}
 
 	switch temporality {
 	case pmetric.AggregationTemporalityDelta:
-		flags |= oteltef.MetricTemporalityDelta
+		flags |= internal.MetricTemporalityDelta
 	case pmetric.AggregationTemporalityCumulative:
-		flags |= oteltef.MetricTemporalityCumulative
+		flags |= internal.MetricTemporalityCumulative
 	case pmetric.AggregationTemporalityUnspecified:
-		flags |= oteltef.MetricTemporalityUnspecified
+		flags |= internal.MetricTemporalityUnspecified
 	default:
 		panic("Unknown temporality value")
 	}
@@ -83,7 +83,7 @@ func (c *OtlpToSortedTree) covertNumberDataPoints(
 	sms pmetric.ScopeMetrics,
 	metric pmetric.Metric,
 	srcPoints pmetric.NumberDataPointSlice,
-	flags oteltef.MetricFlags,
+	flags internal.MetricFlags,
 ) {
 	var metricType oteltef.MetricType
 	var byMetric *sortedbymetric.ByMetric
@@ -127,9 +127,9 @@ func (c *OtlpToSortedTree) covertNumberDataPoints(
 func calcNumericMetricType(metric pmetric.Metric) oteltef.MetricType {
 	switch metric.Type() {
 	case pmetric.MetricTypeGauge:
-		return oteltef.Gauge
+		return oteltef.MetricTypeGauge
 	case pmetric.MetricTypeSum:
-		return oteltef.Sum
+		return oteltef.MetricTypeSum
 	default:
 		log.Fatalf("Unsupported value type: %v", metric.Type())
 	}
@@ -153,7 +153,7 @@ func (c *OtlpToSortedTree) covertHistogramDataPoints(
 
 		c.recordCount++
 
-		byMetric = sm.ByMetric(metric, oteltef.Histogram, flags, srcPoint.ExplicitBounds().AsRaw())
+		byMetric = sm.ByMetric(metric, oteltef.MetricTypeHistogram, flags, srcPoint.ExplicitBounds().AsRaw())
 		byResource := byMetric.ByResource(rm.Resource(), rm.SchemaUrl())
 		byScope = byResource.ByScope(sms.Scope(), sms.SchemaUrl())
 		c.Otlp2tef.MapSorted(srcPoint.Attributes(), &c.TempAttrs)

go/pdata/metrics/otlp2stef.go

@@ -14,14 +14,14 @@ type OtlpToSTEFUnsorted struct {
 	internal.BaseOTLPToSTEF
 }
 
-func convertTemporality(temporality pmetric.AggregationTemporality) oteltef.MetricFlags {
+func convertTemporality(temporality pmetric.AggregationTemporality) internal.MetricFlags {
 	switch temporality {
 	case pmetric.AggregationTemporalityCumulative:
-		return oteltef.MetricTemporalityCumulative
+		return internal.MetricTemporalityCumulative
 	case pmetric.AggregationTemporalityDelta:
-		return oteltef.MetricTemporalityDelta
+		return internal.MetricTemporalityDelta
 	case pmetric.AggregationTemporalityUnspecified:
-		return oteltef.MetricTemporalityUnspecified
+		return internal.MetricTemporalityUnspecified
 	default:
 		panic("unhandled default case")
 	}
@@ -30,11 +30,11 @@ func convertTemporality(temporality pmetric.AggregationTemporality) oteltef.Metr
 func metricType(typ pmetric.MetricType) oteltef.MetricType {
 	switch typ {
 	case pmetric.MetricTypeGauge:
-		return oteltef.Gauge
+		return oteltef.MetricTypeGauge
 	case pmetric.MetricTypeSum:
-		return oteltef.Sum
+		return oteltef.MetricTypeSum
 	case pmetric.MetricTypeHistogram:
-		return oteltef.Histogram
+		return oteltef.MetricTypeHistogram
 	default:
 		log.Fatalf("Unsupported value type: %v", typ)
 	}
@@ -50,7 +50,7 @@ func metric2metric(
 	dst.SetName(src.Name())
 	dst.SetDescription(src.Description())
 	dst.SetUnit(src.Unit())
-	dst.SetType(uint64(metricType(src.Type())))
+	dst.SetType(metricType(src.Type()))
 }
 
 func (d *OtlpToSTEFUnsorted) WriteMetrics(src pmetric.Metrics, writer *oteltef.MetricsWriter) error {

go/pdata/metrics/sortedbymetric/sortedmetrics.go

@@ -8,6 +8,7 @@ import (
 	"go.opentelemetry.io/collector/pdata/pmetric"
 	"modernc.org/b/v2"
 
+	"github.com/splunk/stef/go/pdata/metrics/internal"
 	"github.com/splunk/stef/go/pkg"
 
 	"github.com/splunk/stef/go/otel/oteltef"
@@ -88,7 +89,7 @@ func (s *SortedTree) ToTef(writer *oteltef.MetricsWriter) error {
 }
 
 func (s *SortedTree) ByMetric(
-	metric pmetric.Metric, metricType oteltef.MetricType, flags oteltef.MetricFlags,
+	metric pmetric.Metric, metricType oteltef.MetricType, flags internal.MetricFlags,
 	histogramBounds []float64,
 ) *ByMetric {
 	metr := metric2metric(metric, metricType, flags, histogramBounds, &s.otlp2tef)
@@ -103,7 +104,7 @@ func (s *SortedTree) ByMetric(
 }
 
 func metric2metric(
-	metric pmetric.Metric, metricType oteltef.MetricType, flags oteltef.MetricFlags, histogramBounds []float64,
+	metric pmetric.Metric, metricType oteltef.MetricType, flags internal.MetricFlags, histogramBounds []float64,
 	otlp2tef *otlptools.Otlp2Stef,
 ) *oteltef.Metric {
 
@@ -112,11 +113,11 @@ func metric2metric(
 	dst.SetName(metric.Name())
 	dst.SetDescription(metric.Description())
 	dst.SetUnit(metric.Unit())
-	dst.SetType(uint64(metricType))
+	dst.SetType(metricType)
 	//dst.SetFlags(uint64(flags))
 	dst.HistogramBounds().CopyFromSlice(histogramBounds)
-	dst.SetMonotonic(flags&oteltef.MetricMonotonic != 0)
-	dst.SetAggregationTemporality(uint64(flags & oteltef.MetricTemporalityMask))
+	dst.SetMonotonic(flags&internal.MetricMonotonic != 0)
+	dst.SetAggregationTemporality(uint64(flags & internal.MetricTemporalityMask))
 
 	return &dst
 }

stefgen/generator/compileschema.go

@@ -11,6 +11,7 @@ func compileSchema(src *schema.Schema) (*genSchema, error) {
 		PackageName: src.PackageName,
 		Structs:     map[string]*genStructDef{},
 		Multimaps:   map[string]*genMapDef{},
+		Enums:       map[string]*genEnumDef{},
 	}
 
 	for name, struc := range src.Structs {
@@ -21,6 +22,10 @@ func compileSchema(src *schema.Schema) (*genSchema, error) {
 		dst.Multimaps[name] = multimapWireToGen(multimap)
 	}
 
+	for name, enum := range src.Enums {
+		dst.Enums[name] = enumSchemaToGen(enum)
+	}
+
 	if err := dst.resolveRefs(); err != nil {
 		return nil, err
 	}
@@ -200,7 +205,6 @@ func structFieldWireToAst(src *schema.StructField) *genStructFieldDef {
 	dst := &genStructFieldDef{
 		Name:     src.Name,
 		Optional: src.Optional,
-		//Recursive: src.Recursive,
 	}
 
 	dst.Type = typeWireToGen(src.FieldType)
@@ -213,6 +217,7 @@ func typeWireToGen(src schema.FieldType) genFieldTypeRef {
 		return &genPrimitiveTypeRef{
 			Type: *src.Primitive,
 			Dict: src.DictName,
+			Enum: src.Enum,
 		}
 	}
 
@@ -236,3 +241,21 @@ func typeWireToGen(src schema.FieldType) genFieldTypeRef {
 
 	panic("unknown field type")
 }
+
+func enumSchemaToGen(src *schema.Enum) *genEnumDef {
+	dst := &genEnumDef{
+		Name: src.Name,
+	}
+	for i := range src.Fields {
+		dst.Fields = append(dst.Fields, enumFieldSchemaToGen(&src.Fields[i]))
+	}
+	return dst
+}
+
+func enumFieldSchemaToGen(src *schema.EnumField) *genEnumFieldDef {
+	dst := &genEnumFieldDef{
+		Name:  src.Name,
+		Value: src.Value,
+	}
+	return dst
+}

stefgen/generator/enums.go

@@ -0,0 +1,40 @@
+package generator
+
+import "strings"
+
+func (g *Generator) oEnums() error {
+	for _, enum := range g.compiledSchema.Enums {
+		err := g.oEnum(enum)
+		if err != nil {
+			return err
+		}
+	}
+
+	return g.lastErr
+}
+
+func (g *Generator) oEnum(enum *genEnumDef) error {
+	fields := []any{}
+
+	for _, field := range enum.Fields {
+		fieldData := map[string]any{
+			"Name":  field.Name,
+			"Value": field.Value,
+		}
+
+		fields = append(fields, fieldData)
+	}
+
+	data := map[string]any{
+		"EnumName": enum.Name,
+		"Fields":   fields,
+	}
+
+	templateName := "enum.go.tmpl"
+
+	if err := g.oTemplate(templateName, strings.ToLower(enum.Name), data); err != nil {
+		return err
+	}
+
+	return g.lastErr
+}