feat: Add ability to override 'type' query parameter name

docs/content/datasources.md

@@ -127,6 +127,13 @@ $ gomplate -d data=file:///tmp/data.txt?type=application/json -i '{{ (ds "data")
 bar
 ```
 
+If you need to provide a query parameter named `type` to the data source, set the `GOMPLATE_TYPE_PARAM` environment variable to another value:
+
+```console
+$ GOMPLATE_TYPE_PARAM=content-type gomplate -d data=https://example.com/mydata?content-type=application/json -i '{{ (ds "data").foo }}'
+bar
+```
+
 ### The `.env` file format
 
 Many applications and frameworks support the use of a ".env" file for providing environment variables. It can also be considerd a simple key/value file format, and as such can be used as a datasource in gomplate.

internal/datafs/mergefs.go

@@ -118,6 +118,17 @@ func (f *mergeFS) Open(name string) (fs.File, error) {
 
 		u := subSource.URL
 
+		// possible type hint in the type query param. Contrary to spec, we allow
+		// unescaped '+' characters to make it simpler to provide types like
+		// "application/array+json"
+		overrideType := typeOverrideParam()
+		mimeType := u.Query().Get(overrideType)
+		mimeType = strings.ReplaceAll(mimeType, " ", "+")
+
+		// now that we have the hint, remove it from the URL - we can't have it
+		// leaking into the filesystem layer
+		u = removeQueryParam(u, overrideType)
+
 		fsURL, base := SplitFSMuxURL(u)
 
 		// need to support absolute paths on local filesystem too
@@ -153,12 +164,6 @@ func (f *mergeFS) Open(name string) (fs.File, error) {
 			modTime = fi.ModTime()
 		}
 
-		// possible type hint in the type query param. Contrary to spec, we allow
-		// unescaped '+' characters to make it simpler to provide types like
-		// "application/array+json"
-		mimeType := u.Query().Get("type")
-		mimeType = strings.ReplaceAll(mimeType, " ", "+")
-
 		if mimeType == "" {
 			mimeType = fsimpl.ContentType(fi)
 		}

internal/datafs/reader.go

@@ -8,6 +8,7 @@ import (
 	"io/fs"
 	"net/http"
 	"net/url"
+	"os"
 	"runtime"
 	"strings"
 
@@ -16,6 +17,17 @@ import (
 	"github.com/hairyhenderson/gomplate/v4/internal/iohelpers"
 )
 
+// typeOverrideParam gets the query parameter used to override the content type
+// used to parse a given datasource - use GOMPLATE_TYPE_PARAM to use a different
+// parameter name.
+func typeOverrideParam() string {
+	if v := os.Getenv("GOMPLATE_TYPE_PARAM"); v != "" {
+		return v
+	}
+
+	return "type"
+}
+
 // DataSourceReader reads content from a datasource
 type DataSourceReader interface {
 	// ReadSource reads the content of a datasource, given an alias and optional
@@ -91,7 +103,25 @@ func (d *dsReader) ReadSource(ctx context.Context, alias string, args ...string)
 	return fc.contentType, fc.b, nil
 }
 
+func removeQueryParam(u *url.URL, key string) *url.URL {
+	q := u.Query()
+	q.Del(key)
+	u.RawQuery = q.Encode()
+	return u
+}
+
 func (d *dsReader) readFileContent(ctx context.Context, u *url.URL, hdr http.Header) (*content, error) {
+	// possible type hint in the type query param. Contrary to spec, we allow
+	// unescaped '+' characters to make it simpler to provide types like
+	// "application/array+json"
+	overrideType := typeOverrideParam()
+	mimeType := u.Query().Get(overrideType)
+	mimeType = strings.ReplaceAll(mimeType, " ", "+")
+
+	// now that we have the hint, remove it from the URL - we can't have it
+	// leaking into the filesystem layer
+	u = removeQueryParam(u, overrideType)
+
 	fsys, err := FSysForPath(ctx, u.String())
 	if err != nil {
 		return nil, fmt.Errorf("fsys for path %v: %w", u, err)
@@ -120,12 +150,6 @@ func (d *dsReader) readFileContent(ctx context.Context, u *url.URL, hdr http.Hea
 		return nil, fmt.Errorf("stat (url: %q, name: %q): %w", u, fname, err)
 	}
 
-	// possible type hint in the type query param. Contrary to spec, we allow
-	// unescaped '+' characters to make it simpler to provide types like
-	// "application/array+json"
-	mimeType := u.Query().Get("type")
-	mimeType = strings.ReplaceAll(mimeType, " ", "+")
-
 	if mimeType == "" {
 		mimeType = fsimpl.ContentType(fi)
 	}

internal/tests/integration/datasources_http_test.go

@@ -15,6 +15,7 @@ func setupDatasourcesHTTPTest(t *testing.T) *httptest.Server {
 	mux.HandleFunc("/actually.json", typeHandler("", `{"value": "json"}`))
 	mux.HandleFunc("/bogus.csv", typeHandler("text/plain", `{"value": "json"}`))
 	mux.HandleFunc("/list", typeHandler("application/array+json", `[1, 2, 3, 4, 5]`))
+	mux.HandleFunc("/params", paramHandler(t))
 
 	srv := httptest.NewServer(mux)
 	t.Cleanup(srv.Close)
@@ -68,6 +69,12 @@ func TestDatasources_HTTP_TypeOverridePrecedence(t *testing.T) {
 		"-c", ".="+srv.URL+"/list?type=application/array+json",
 		"-i", "{{ range . }}{{ . }}{{ end }}").run()
 	assertSuccess(t, o, e, err, "12345")
+
+	o, e, err = cmd(t,
+		"-c", ".="+srv.URL+"/params?foo=bar&type=http&_type=application/json",
+		"-i", "{{ . | toJSON }}").
+		withEnv("GOMPLATE_TYPE_PARAM", "_type").run()
+	assertSuccess(t, o, e, err, `{"foo":["bar"],"type":["http"]}`)
 }
 
 func TestDatasources_HTTP_AppendQueryAfterSubPaths(t *testing.T) {

internal/tests/integration/datasources_merge_test.go

@@ -21,6 +21,10 @@ func setupDatasourcesMergeTest(t *testing.T) (*fs.Dir, *httptest.Server) {
 	mux.HandleFunc("/foo.json", typeHandler("application/json", `{"foo": "bar"}`))
 	mux.HandleFunc("/1.env", typeHandler("application/x-env", "FOO=1\nBAR=2\n"))
 	mux.HandleFunc("/2.env", typeHandler("application/x-env", "FOO=3\n"))
+	// this file is served with a misleading content type and extension, for
+	// testing overriding the type
+	mux.HandleFunc("/wrongtype.txt", typeHandler("text/html", `{"foo": "bar"}`))
+	mux.HandleFunc("/params", paramHandler(t))
 
 	srv := httptest.NewServer(mux)
 	t.Cleanup(srv.Close)
@@ -65,4 +69,28 @@ func TestDatasources_Merge(t *testing.T) {
 {{ ds "merged" | toJSON }}`,
 	).run()
 	assertSuccess(t, o, e, err, `{"foo":"bar","isDefault":true,"isOverride":false,"other":true}`)
+
+	o, e, err = cmd(t,
+		"-d", "default="+tmpDir.Join("default.yml"),
+		"-d", "wrongtype="+srv.URL+"/wrongtype.txt?type=application/json",
+		"-d", "config=merge:wrongtype|default",
+		"-i", `{{ ds "config" | toJSON }}`,
+	).run()
+	assertSuccess(t, o, e, err, `{"foo":"bar","isDefault":true,"isOverride":false,"other":true}`)
+
+	o, e, err = cmd(t,
+		"-d", "default="+tmpDir.Join("default.yml"),
+		"-d", "wrongtype="+srv.URL+"/wrongtype.txt?_=application/json",
+		"-d", "config=merge:wrongtype|default",
+		"-i", `{{ ds "config" | toJSON }}`,
+	).withEnv("GOMPLATE_TYPE_PARAM", "_").run()
+	assertSuccess(t, o, e, err, `{"foo":"bar","isDefault":true,"isOverride":false,"other":true}`)
+
+	o, e, err = cmd(t,
+		"-c", "default="+tmpDir.Join("default.yml"),
+		"-c", "params="+srv.URL+"/params?foo=bar&type=http&_type=application/json",
+		"-c", "merged=merge:params|default",
+		"-i", `{{ .merged | toJSON }}`,
+	).withEnv("GOMPLATE_TYPE_PARAM", "_type").run()
+	assertSuccess(t, o, e, err, `{"foo":["bar"],"isDefault":true,"isOverride":false,"other":true,"type":["http"]}`)
 }

internal/tests/integration/integration_test.go

@@ -95,6 +95,18 @@ func typeHandler(t, body string) func(http.ResponseWriter, *http.Request) {
 	}
 }
 
+func paramHandler(t *testing.T) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		// just returns params as JSON
+		w.Header().Set("Content-Type", "application/json")
+
+		enc := json.NewEncoder(w)
+		if err := enc.Encode(r.URL.Query()); err != nil {
+			t.Fatalf("error encoding: %v", err)
+		}
+	}
+}
+
 // freeport - find a free TCP port for immediate use. No guarantees!
 func freeport(t *testing.T) (port int, addr string) {
 	l, err := net.ListenTCP("tcp", &net.TCPAddr{IP: net.ParseIP("127.0.0.1")})