cmd/cue: fix incorrect position in yaml import

Position information of the CUE AST is represented with bytes, whereas
the YAML decoder represents position information with characters. The
mismatch could result in incorrect and strange positions in the CUE AST
from decoded YAML files.

The simplest thing to do is to change the YAML decoder to represent
position information as bytes. Other solutions may be complex or
computationally expensive. I do not see the character index used in any
meaningful way when decoding, so I believe this to be a safe change.

In addition, it seems position information for decoded YAML documents
may have been off by one, which has also been addressed. Position
information for sequence end tokens were also invalid (or missing) and
is now correctly set to the starting position of the sequence end event.

The Kubernetes tutorial files have been updated to match the changes as
a result of the corrected position information.

	CUE_LONG=true CUE_UPDATE=true go test ./doc/tutorial/kubernetes

Fixes #826

Change-Id: Ie139708698296660b2396bc696e7b753a004a132
Signed-off-by: Thomas Way <[email protected]>
Reviewed-on: https://review.gerrithub.io/c/cue-lang/cue/+/1172017
TryBot-Result: CUEcueckoo <[email protected]>
Reviewed-by: Daniel Martí <[email protected]>

cmd/cue/cmd/testdata/script/def_jsonschema.txtar

@@ -73,10 +73,10 @@ age: twenty
 
 -- expect-stderr2 --
 age: conflicting values "twenty" and int (mismatched types string and int):
-    ./data.yaml:1:7
+    ./data.yaml:1:6
     ./schema.json:18:7
 -- expect-stderr3 --
 age: conflicting values "twenty" and int (mismatched types string and int):
-    ./data.yaml:1:7
+    ./data.yaml:1:6
     ./schema.json:18:7
 -- cue.mod --

cmd/cue/cmd/testdata/script/issue826.txtar

@@ -0,0 +1,18 @@
+exec cue import x.yaml
+cmp x.cue expect.cue
+
+-- x.yaml --
+# Importing YAML with multi-byte characters as CUE resulted in bad formatting
+# since the YAML decoder would count columns by characters and we count by bytes.
+x1:
+  # ああああああああああああああああああ
+  description: ああ
+  type: string
+-- expect.cue --
+// Importing YAML with multi-byte characters as CUE resulted in bad formatting
+// since the YAML decoder would count columns by characters and we count by bytes.
+x1: {
+	// ああああああああああああああああああ
+	description: "ああ"
+	type:        "string"
+}

cmd/cue/cmd/testdata/script/vet_data.txtar

@@ -23,8 +23,8 @@ languages:
 -- vet-stderr --
 languages.1.name: invalid value "dutch" (out of bound =~"^\\p{Lu}"):
     ./schema.cue:3:8
-    ./data.yaml:5:12
+    ./data.yaml:5:11
 -- export-stderr --
 languages.1.name: invalid value "dutch" (out of bound =~"^\\p{Lu}"):
     ./schema.cue:3:8
-    ./data.yaml:5:12
+    ./data.yaml:5:11

cmd/cue/cmd/testdata/script/vet_embed.txtar

@@ -32,7 +32,7 @@ e: 2
 
 -- expect-foo --
 c: field not allowed:
-    ./foo.yaml:2:2
+    ./foo.yaml:2:1
     ./schema.cue:1:1
     ./schema.cue:3:7
     ./schema.cue:7:1
@@ -41,10 +41,10 @@ d: field not allowed:
     ./schema.cue:1:1
     ./schema.cue:3:7
     ./schema.cue:7:1
-    ./stream.yaml:2:2
+    ./stream.yaml:2:1
 -- expect-stream --
 d: field not allowed:
     ./schema.cue:1:1
     ./schema.cue:3:7
     ./schema.cue:7:1
-    ./stream.yaml:2:2
+    ./stream.yaml:2:1

doc/tutorial/basics/0_intro/47_validation.txtar

@@ -28,4 +28,4 @@ languages:
 -- expect-stderr --
 languages.1.name: invalid value "dutch" (out of bound =~"^\\p{Lu}"):
     ./schema.cue:3:8
-    ./data.yaml:5:12
+    ./data.yaml:5:11

doc/tutorial/kubernetes/quick/services/infra/etcd/kube.cue

@@ -20,9 +20,7 @@ statefulSet: etcd: spec: {
 				labelSelector: matchExpressions: [{
 					key:      "app"
 					operator: "In"
-					values: [
-						"etcd",
-					]
+					values: ["etcd"]
 				}]
 				topologyKey: "kubernetes.io/hostname"
 			}]

doc/tutorial/kubernetes/quick/services/infra/events/kube.cue

@@ -15,9 +15,7 @@ deployment: events: spec: {
 				labelSelector: matchExpressions: [{
 					key:      "app"
 					operator: "In"
-					values: [
-						"events",
-					]
+					values: ["events"]
 				}]
 				topologyKey: "kubernetes.io/hostname"
 			}]

doc/tutorial/kubernetes/quick/services/mon/prometheus/configmap.cue

@@ -51,15 +51,11 @@ configMap: prometheus: {
 		"prometheus.yml": yaml656e63.Marshal(_cue_prometheus_yml)
 		let _cue_prometheus_yml = {
 			global: scrape_interval: "15s"
-			rule_files: [
-				"/etc/prometheus/alert.rules",
-			]
+			rule_files: ["/etc/prometheus/alert.rules"]
 			alerting: alertmanagers: [{
 				scheme: "http"
 				static_configs: [{
-					targets: [
-						"alertmanager:9093",
-					]
+					targets: ["alertmanager:9093"]
 				}]
 			}]
 			scrape_configs: [{

internal/third_party/yaml/decode.go

@@ -236,6 +236,8 @@ func (p *parser) sequence() *node {
 	}
 	if len(n.children) > 0 {
 		n.endPos = n.children[len(n.children)-1].endPos
+	} else {
+		n.endPos = p.event.start_mark
 	}
 	p.expect(yaml_SEQUENCE_END_EVENT)
 	return n
@@ -362,7 +364,7 @@ func (d *decoder) attachLineComment(m yaml_mark_t, pos int8, expr ast.Node) {
 }
 
 func (d *decoder) pos(m yaml_mark_t) token.Pos {
-	pos := d.p.info.Pos(m.index+1, token.NoRelPos)
+	pos := d.absPos(m)
 
 	if d.forceNewline {
 		d.forceNewline = false
@@ -390,7 +392,7 @@ func (d *decoder) pos(m yaml_mark_t) token.Pos {
 }
 
 func (d *decoder) absPos(m yaml_mark_t) token.Pos {
-	return d.p.info.Pos(m.index+1, token.NoRelPos)
+	return d.p.info.Pos(m.index, token.NoRelPos)
 }
 
 func (d *decoder) start(n *node) token.Pos {

internal/third_party/yaml/scannerc.go

@@ -489,10 +489,11 @@ func cache(parser *yaml_parser_t, length int) bool {
 
 // Advance the buffer pointer.
 func skip(parser *yaml_parser_t) {
-	parser.mark.index++
+	w := width(parser.buffer[parser.buffer_pos])
+	parser.mark.index += w
 	parser.mark.column++
 	parser.unread--
-	parser.buffer_pos += width(parser.buffer[parser.buffer_pos])
+	parser.buffer_pos += w
 }
 
 func skip_line(parser *yaml_parser_t) {
@@ -503,11 +504,12 @@ func skip_line(parser *yaml_parser_t) {
 		parser.unread -= 2
 		parser.buffer_pos += 2
 	} else if is_break(parser.buffer, parser.buffer_pos) {
-		parser.mark.index++
+		w := width(parser.buffer[parser.buffer_pos])
+		parser.mark.index += w
 		parser.mark.column = 0
 		parser.mark.line++
 		parser.unread--
-		parser.buffer_pos += width(parser.buffer[parser.buffer_pos])
+		parser.buffer_pos += w
 	}
 }
 
@@ -528,7 +530,7 @@ func read(parser *yaml_parser_t, s []byte) []byte {
 		s = append(s, parser.buffer[parser.buffer_pos:parser.buffer_pos+w]...)
 		parser.buffer_pos += w
 	}
-	parser.mark.index++
+	parser.mark.index += w
 	parser.mark.column++
 	parser.unread--
 	return s

pkg/encoding/yaml/testdata/gen.txtar

@@ -32,14 +32,14 @@ Errors:
 t1: error in call to encoding/yaml.Validate: invalid value 4 (out of bound <3):
     ./in.cue:3:5
     ./in.cue:3:49
-    yaml.Validate:3:5
+    yaml.Validate:3:4
 t3: error in call to encoding/yaml.Validate: incomplete value int:
     ./in.cue:5:5
     ./in.cue:5:56
 t4: error in call to encoding/yaml.ValidatePartial: invalid value 4 (out of bound <3):
     ./in.cue:6:5
     ./in.cue:6:49
-    yaml.ValidatePartial:3:5
+    yaml.ValidatePartial:3:4
 
 Result:
 t1: _|_ // t1: error in call to encoding/yaml.Validate: a: invalid value 4 (out of bound <3)