Integration for minimal flatten mode

* Fixes go-swagger#1572

Contents:
* Enables new swagger CLI options to flatten minimally, fully or expand. Minimal flatten is the default.
* Deprecates --skip-flatten option (replaced by --with-expand)

And also:
* Adapted tests to reflect this new default
* Refactored schema building in operations for body parameters and responses
* More UTs
* Again some more linting
* Cyclo check level for golangCI pushed to 25
* Removed -race option from appveyor CI

* Fix bug with deep maps (rewind ValueExpression)
* Setup parallel testing for models

.golangci.yml

@@ -4,7 +4,7 @@ linters-settings:
   golint:
     min-confidence: 0
   gocyclo:
-    min-complexity: 20
+    min-complexity: 25
   maligned:
     suggest-new: true
   dupl:

appveyor.yml

@@ -50,7 +50,7 @@ build_script:
   - echo %GOPATH%
 
 test_script:
-  - ps: go test -timeout 20m -race -v $(go list ./... | sls -n "vendor|fixtures|examples")
+  - ps: go test -timeout 20m -v $(go list ./... | sls -n "vendor|fixtures|examples")
 
 #artifacts:
 #  - path: '%GOPATH%\bin\*.exe'

cmd/swagger/commands/expand.go

@@ -8,13 +8,18 @@ import (
 
 	"github.com/go-openapi/loads"
 	"github.com/go-openapi/spec"
+	"github.com/go-openapi/swag"
 	flags "github.com/jessevdk/go-flags"
+	yaml "gopkg.in/yaml.v2"
 )
 
-// ExpandSpec is a command that expands the $refs in a swagger document
+// ExpandSpec is a command that expands the $refs in a swagger document.
+//
+// There are no specific options for this expansion.
 type ExpandSpec struct {
-	Compact bool           `long:"compact" description:"when present, doesn't prettify the json"`
+	Compact bool           `long:"compact" description:"applies to JSON formated specs. When present, doesn't prettify the json"`
 	Output  flags.Filename `long:"output" short:"o" description:"the file to write to"`
+	Format  string         `long:"format" description:"the format for the spec document" default:"json" choice:"yaml" choice:"json"`
 }
 
 // Execute expands the spec
@@ -34,16 +39,28 @@ func (c *ExpandSpec) Execute(args []string) error {
 		return err
 	}
 
-	return writeToFile(exp.Spec(), !c.Compact, string(c.Output))
+	return writeToFile(exp.Spec(), !c.Compact, c.Format, string(c.Output))
 }
 
-func writeToFile(swspec *spec.Swagger, pretty bool, output string) error {
+func writeToFile(swspec *spec.Swagger, pretty bool, format string, output string) error {
 	var b []byte
 	var err error
-	if pretty {
+	asJSON := format == "json"
+
+	if pretty && asJSON {
 		b, err = json.MarshalIndent(swspec, "", "  ")
+	} else if asJSON {
+		b, err = json.Marshal(swspec)
 	} else {
+		// marshals as YAML
 		b, err = json.Marshal(swspec)
+		if err == nil {
+			d, ery := swag.BytesToYAMLDoc(b)
+			if ery != nil {
+				return ery
+			}
+			b, err = yaml.Marshal(d)
+		}
 	}
 	if err != nil {
 		return err

cmd/swagger/commands/expand_test.go

@@ -0,0 +1,32 @@
+package commands
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	flags "github.com/jessevdk/go-flags"
+)
+
+// Commands requires at least one arg
+func TestCmd_Expand(t *testing.T) {
+	v := &ExpandSpec{}
+	testRequireParam(t, v)
+}
+
+func TestCmd_Expand_NoError(t *testing.T) {
+	specDoc := filepath.Join(fixtureBase, "bugs", "1536", "fixture-1536.yaml")
+	outDir, output := getOutput(t, specDoc, "flatten", "fixture-1536-flat-expand.json")
+	defer os.RemoveAll(outDir)
+	v := &ExpandSpec{
+		Format:  "json",
+		Compact: false,
+		Output:  flags.Filename(output),
+	}
+	testProduceOutput(t, v, specDoc, output)
+}
+
+func TestCmd_Expand_Error(t *testing.T) {
+	v := &ExpandSpec{}
+	testValidRefs(t, v)
+}

cmd/swagger/commands/flatten.go

@@ -2,19 +2,21 @@ package commands
 
 import (
 	"errors"
-	"log"
 
 	"github.com/go-openapi/analysis"
 	"github.com/go-openapi/loads"
+	"github.com/go-swagger/go-swagger/cmd/swagger/commands/generate"
 	flags "github.com/jessevdk/go-flags"
 )
 
 // FlattenSpec is a command that flattens a swagger document
-// Which will expand the remote references in a spec and move inline schemas to definitions
+// which will expand the remote references in a spec and move inline schemas to definitions
 // after flattening there are no complex inlined anymore
 type FlattenSpec struct {
-	Compact bool           `long:"compact" description:"when present, doesn't prettify the json"`
+	Compact bool           `long:"compact" description:"applies to JSON formated specs. When present, doesn't prettify the json"`
 	Output  flags.Filename `long:"output" short:"o" description:"the file to write to"`
+	Format  string         `long:"format" description:"the format for the spec document" default:"json" choice:"yaml" choice:"json"`
+	generate.FlattenCmdOptions
 }
 
 // Execute expands the spec
@@ -26,15 +28,21 @@ func (c *FlattenSpec) Execute(args []string) error {
 	swaggerDoc := args[0]
 	specDoc, err := loads.Spec(swaggerDoc)
 	if err != nil {
-		log.Fatalln(err)
+		return err
 	}
 
-	if er := analysis.Flatten(analysis.FlattenOpts{
-		BasePath: specDoc.SpecFilePath(),
-		Spec:     analysis.New(specDoc.Spec()),
-	}); er != nil {
-		log.Fatalln(er)
+	flattenOpts := c.FlattenCmdOptions.SetFlattenOptions(&analysis.FlattenOpts{
+		// defaults
+		Minimal:      true,
+		Verbose:      true,
+		Expand:       false,
+		RemoveUnused: false,
+	})
+	flattenOpts.BasePath = specDoc.SpecFilePath()
+	flattenOpts.Spec = analysis.New(specDoc.Spec())
+	if err := analysis.Flatten(*flattenOpts); err != nil {
+		return err
 	}
 
-	return writeToFile(specDoc.Spec(), !c.Compact, string(c.Output))
+	return writeToFile(specDoc.Spec(), !c.Compact, c.Format, string(c.Output))
 }

cmd/swagger/commands/flatten_test.go

@@ -0,0 +1,77 @@
+package commands
+
+import (
+	"io/ioutil"
+	"log"
+	"os"
+	"path/filepath"
+	"testing"
+
+	flags "github.com/jessevdk/go-flags"
+	"github.com/stretchr/testify/assert"
+)
+
+type executable interface {
+	Execute([]string) error
+}
+
+func testValidRefs(t *testing.T, v executable) {
+	log.SetOutput(ioutil.Discard)
+	defer log.SetOutput(os.Stdout)
+
+	specDoc := filepath.Join(fixtureBase, "expansion", "invalid-refs.json")
+	result := v.Execute([]string{specDoc})
+	assert.Error(t, result)
+}
+
+func testRequireParam(t *testing.T, v executable) {
+	log.SetOutput(ioutil.Discard)
+	defer log.SetOutput(os.Stdout)
+
+	result := v.Execute([]string{})
+	assert.Error(t, result)
+
+	result = v.Execute([]string{"nowhere.json"})
+	assert.Error(t, result)
+}
+
+func getOutput(t *testing.T, specDoc, prefix, filename string) (string, string) {
+	outDir, err := ioutil.TempDir(filepath.Dir(specDoc), "flatten")
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+	return outDir, filepath.Join(outDir, filename)
+}
+
+func testProduceOutput(t *testing.T, v executable, specDoc, output string) {
+	log.SetOutput(ioutil.Discard)
+	defer log.SetOutput(os.Stdout)
+
+	result := v.Execute([]string{specDoc})
+	assert.NoError(t, result)
+	_, exists := os.Stat(output)
+	assert.True(t, !os.IsNotExist(exists))
+}
+
+// Commands requires at least one arg
+func TestCmd_Flatten(t *testing.T) {
+	v := &FlattenSpec{}
+	testRequireParam(t, v)
+}
+
+func TestCmd_Flatten_Default(t *testing.T) {
+	specDoc := filepath.Join(fixtureBase, "bugs", "1536", "fixture-1536.yaml")
+	outDir, output := getOutput(t, specDoc, "flatten", "fixture-1536-flat-minimal.json")
+	defer os.RemoveAll(outDir)
+	v := &FlattenSpec{
+		Format:  "json",
+		Compact: true,
+		Output:  flags.Filename(output),
+	}
+	testProduceOutput(t, v, specDoc, output)
+}
+
+func TestCmd_Flatten_Error(t *testing.T) {
+	v := &FlattenSpec{}
+	testValidRefs(t, v)
+}

cmd/swagger/commands/generate/client.go

@@ -15,7 +15,6 @@
 package generate
 
 import (
-	"io/ioutil"
 	"log"
 
 	"github.com/go-swagger/go-swagger/generator"
@@ -35,23 +34,9 @@ type Client struct {
 	SkipOperations  bool     `long:"skip-operations" description:"no operations will be generated when this flag is specified"`
 	DumpData        bool     `long:"dump-data" description:"when present dumps the json for the template generator instead of generating files"`
 	SkipValidation  bool     `long:"skip-validation" description:"skips validation of spec prior to generation"`
-	SkipFlattening  bool     `long:"skip-flatten" description:"skips flattening of spec prior to generation"`
 }
 
 func (c *Client) getOpts() (*generator.GenOpts, error) {
-	var copyrightstr string
-	copyrightfile := string(c.CopyrightFile)
-	if copyrightfile != "" {
-		//Read the Copyright from file path in opts
-		bytebuffer, err := ioutil.ReadFile(copyrightfile)
-		if err != nil {
-			return nil, err
-		}
-		copyrightstr = string(bytebuffer)
-	} else {
-		copyrightstr = ""
-	}
-
 	return &generator.GenOpts{
 		Spec: string(c.Spec),
 
@@ -69,17 +54,19 @@ func (c *Client) getOpts() (*generator.GenOpts, error) {
 		IncludeParameters: !c.SkipOperations,
 		IncludeResponses:  !c.SkipOperations,
 		ValidateSpec:      !c.SkipValidation,
-		FlattenSpec:       !c.SkipFlattening,
 		Tags:              c.Tags,
 		IncludeSupport:    true,
 		TemplateDir:       string(c.TemplateDir),
 		DumpData:          c.DumpData,
 		ExistingModels:    c.ExistingModels,
-		Copyright:         copyrightstr,
 		IsClient:          true,
 	}, nil
 }
 
+func (c *Client) getShared() *shared {
+	return &c.shared
+}
+
 func (c *Client) generate(opts *generator.GenOpts) error {
 	return generator.GenerateClient(c.Name, c.Models, c.Operations, opts)
 }

cmd/swagger/commands/generate/client_test.go

@@ -0,0 +1,59 @@
+package generate_test
+
+import (
+	"io/ioutil"
+	"log"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/go-swagger/go-swagger/cmd/swagger/commands/generate"
+	flags "github.com/jessevdk/go-flags"
+)
+
+func TestGenerateClient(t *testing.T) {
+	specs := []string{
+		"tasklist.basic.yml",
+		"todolist.simplequery.yml",
+	}
+	log.SetOutput(ioutil.Discard)
+	defer log.SetOutput(os.Stdout)
+
+	base := filepath.FromSlash("../../../../")
+	for i, spec := range specs {
+		_ = t.Run(spec, func(t *testing.T) {
+			path := filepath.Join(base, "fixtures/codegen", spec)
+			generated, err := ioutil.TempDir(filepath.Dir(path), "generated")
+			if err != nil {
+				t.Fatalf("TempDir()=%s", generated)
+			}
+			defer func() {
+				_ = os.RemoveAll(generated)
+			}()
+			m := &generate.Client{}
+			_, _ = flags.Parse(m)
+			if i == 0 {
+				m.CopyrightFile = flags.Filename(filepath.Join(base, "LICENSE"))
+			}
+			m.Spec = flags.Filename(path)
+			m.Target = flags.Filename(generated)
+
+			if err := m.Execute([]string{}); err != nil {
+				t.Error(err)
+			}
+		})
+	}
+}
+
+func TestGenerateClient_Check(t *testing.T) {
+	log.SetOutput(ioutil.Discard)
+	defer log.SetOutput(os.Stdout)
+
+	m := &generate.Client{}
+	_, _ = flags.Parse(m)
+	m.CopyrightFile = "nullePart"
+	err := m.Execute([]string{})
+	assert.Error(t, err)
+}

cmd/swagger/commands/generate/model.go

@@ -25,7 +25,6 @@ type Model struct {
 	Name           []string `long:"name" short:"n" description:"the model to generate"`
 	NoStruct       bool     `long:"skip-struct" description:"when present will not generate the model struct"`
 	DumpData       bool     `long:"dump-data" description:"when present dumps the json for the template generator instead of generating files"`
-	SkipFlattening bool     `long:"skip-flatten" description:"skips flattening of spec prior to generation"`
 	SkipValidation bool     `long:"skip-validation" description:"skips validation of spec prior to generation"`
 }
 
@@ -37,7 +36,7 @@ func (m *Model) Execute(args []string) error {
 	}
 
 	if m.ExistingModels != "" {
-		log.Println("Warning: Ignoring existing-models flag when generating models.")
+		log.Println("warning: Ignoring existing-models flag when generating models.")
 	}
 	s := &Server{
 		shared:         m.shared,
@@ -48,7 +47,6 @@ func (m *Model) Execute(args []string) error {
 		SkipSupport:    true,
 		SkipOperations: true,
 		SkipModels:     m.NoStruct,
-		SkipFlattening: m.SkipFlattening,
 		SkipValidation: m.SkipValidation,
 	}
 	return s.Execute(args)