feat: select columns in ingress descriptor (#313)

* feat: select columns in ingress descriptor

* refactor: change table definition only once

* test: fetch hidden keys for relation travel

* feat: start select in ingress descriptor

* test: add missing test file

* fix: set-child-select and set-parent-select commands

* feat(select): add venom tests

* docs: add select feature in README

* test(select): fix venom tests

* test(select): fix venom tests

* test(select): fix venom tests

* docs: update changelog

* test(select): fix venom tests

* test(select): fix venom tests

---------

Co-authored-by: Adrien Aury <[email protected]>

CHANGELOG.md

@@ -14,6 +14,11 @@ Types of changes
 - `Fixed` for any bug fixes.
 - `Security` in case of vulnerabilities.
 
+## [3.1.0]
+
+- `Added` possibility to filter columns via `select` property in ingress descriptors
+- `Added` commands `set-parent-select` and `set-child-select` to `lino id`
+
 ## [3.0.2]
 
 - `Fixed` invalid length for DATE or NUMBER bind variable, bumping oracle driver to 2.8.19

README.md

@@ -267,6 +267,8 @@ To modify the `ingress-descriptor.yml`, some commands can be used instead of edi
 - `lino id set-parent-lookup <relation name> <true or false>` : modify the `lookup` property of the parent object
 - `lino id set-child-where <relation name> <where clause>` : modify the `where` property of the child object
 - `lino id set-parent-where <relation name> <where clause>` : modify the `where` property of the parent object
+- `lino id set-child-select <relation name> <column1> <column2> ...` : modify the `select` property of the child object
+- `lino id set-parent-select <relation name> <column1> <column2> ...` : modify the `select` property of the parent object
 - `lino id set-start-table <table name>` : modify the `startTable` property of the ingress descriptor
 
 Example:

internal/app/id/cli.go

@@ -59,7 +59,9 @@ func NewCommand(fullName string, err *os.File, out *os.File, in *os.File) *cobra
 	cmd.AddCommand(newSetChildLookupCommand(fullName, err, out, in))
 	cmd.AddCommand(newSetParentLookupCommand(fullName, err, out, in))
 	cmd.AddCommand(newSetChildWhereCommand(fullName, err, out, in))
+	cmd.AddCommand(newSetChildSelectCommand(fullName, err, out, in))
 	cmd.AddCommand(newSetParentWhereCommand(fullName, err, out, in))
+	cmd.AddCommand(newSetParentSelectCommand(fullName, err, out, in))
 	cmd.PersistentFlags().StringVarP(&ingressDescriptor, "ingress-descriptor", "i", "ingress-descriptor.yaml", "Ingress descriptor filename")
 	cmd.SetOut(out)
 	cmd.SetErr(err)

internal/app/id/create.go

@@ -45,7 +45,7 @@ func newCreateCommand(fullName string, err *os.File, out *os.File, in *os.File)
 
 			reader := infra.NewRelationReader(relations)
 
-			e := id.Create(table, reader, idStorageFactory(ingressDescriptor))
+			e := id.Create(table, []string{}, reader, idStorageFactory(ingressDescriptor))
 			if e != nil {
 				fmt.Fprintln(err, e.Description)
 				os.Exit(1)

internal/app/id/set_child_select.go

@@ -0,0 +1,53 @@
+// Copyright (C) 2021 CGI France
+//
+// This file is part of LINO.
+//
+// LINO is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// LINO is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with LINO.  If not, see <http://www.gnu.org/licenses/>.
+
+package id
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/cgi-fr/lino/pkg/id"
+	"github.com/spf13/cobra"
+)
+
+// newSetChildSelectCommand implements the cli id set-child-select command
+func newSetChildSelectCommand(fullName string, err *os.File, out *os.File, in *os.File) *cobra.Command {
+	cmd := &cobra.Command{
+		Use:     "set-child-select [relation] [column1] [column2] [column3] ...",
+		Short:   "set child select attribut for relation [relation] in ingress descriptor",
+		Long:    "",
+		Example: fmt.Sprintf("  %[1]s id set-child-select public.store store_id name", fullName),
+		Args:    cobra.MinimumNArgs(2),
+		Run: func(cmd *cobra.Command, args []string) {
+			relation := args[0]
+			selectColumns := args[1:]
+
+			e := id.SetChildSelect(relation, selectColumns, idStorageFactory(ingressDescriptor))
+			if e != nil {
+				fmt.Fprintln(err, e.Description)
+				os.Exit(1)
+			}
+
+			fmt.Fprintf(out, "successfully update relation %s in ingress descriptor\n", relation)
+		},
+	}
+	cmd.SetOut(out)
+	cmd.SetErr(err)
+	cmd.SetIn(in)
+	return cmd
+}

internal/app/id/set_parent_select.go

@@ -0,0 +1,53 @@
+// Copyright (C) 2021 CGI France
+//
+// This file is part of LINO.
+//
+// LINO is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// LINO is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with LINO.  If not, see <http://www.gnu.org/licenses/>.
+
+package id
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/cgi-fr/lino/pkg/id"
+	"github.com/spf13/cobra"
+)
+
+// newSetParentSelectCommand implements the cli id set-parent-select command
+func newSetParentSelectCommand(fullName string, err *os.File, out *os.File, in *os.File) *cobra.Command {
+	cmd := &cobra.Command{
+		Use:     "set-parent-select [relation] [column1] [column2] [column3] ...",
+		Short:   "set parent select attribut for relation [relation] in ingress descriptor",
+		Long:    "",
+		Example: fmt.Sprintf("  %[1]s id set-parent-select public.store store_id name", fullName),
+		Args:    cobra.MinimumNArgs(2),
+		Run: func(cmd *cobra.Command, args []string) {
+			relation := args[0]
+			selectColumns := args[1:]
+
+			e := id.SetParentSelect(relation, selectColumns, idStorageFactory(ingressDescriptor))
+			if e != nil {
+				fmt.Fprintln(err, e.Description)
+				os.Exit(1)
+			}
+
+			fmt.Fprintf(out, "successfully update relation %s in ingress descriptor\n", relation)
+		},
+	}
+	cmd.SetOut(out)
+	cmd.SetErr(err)
+	cmd.SetIn(in)
+	return cmd
+}

internal/app/pull/builder.go

@@ -124,7 +124,8 @@ func (b builder) buildRelation(rel id.IngressRelation) ([]pull.Relation, error)
 					Table: b.getTable(rel.Child().Name()),
 					Keys:  relyaml.Child.Keys,
 				},
-				Where: rel.WhereChild(),
+				Where:  rel.WhereChild(),
+				Select: rel.SelectChild(),
 			}
 		}
 		b.exrmap[name] = exrel
@@ -146,7 +147,8 @@ func (b builder) buildRelation(rel id.IngressRelation) ([]pull.Relation, error)
 					Table: b.getTable(rel.Child().Name()),
 					Keys:  relyaml.Child.Keys,
 				},
-				Where: rel.WhereParent(),
+				Where:  rel.WhereParent(),
+				Select: rel.SelectParent(),
 			}
 		}
 		b.exrmap[name] = exrel

internal/app/pull/cli.go

@@ -117,7 +117,7 @@ func NewCommand(fullName string, err *os.File, out *os.File, in *os.File) *cobra
 				os.Exit(1)
 			}
 
-			plan, start, e2 := getPullerPlan(idStorageFactory(table, ingressDescriptor))
+			plan, start, startSelect, e2 := getPullerPlan(idStorageFactory(table, ingressDescriptor))
 			if e2 != nil {
 				fmt.Fprintln(err, e2.Error())
 				os.Exit(1)
@@ -175,7 +175,7 @@ func NewCommand(fullName string, err *os.File, out *os.File, in *os.File) *cobra
 			}
 
 			puller := pull.NewPullerParallel(plan, datasource, pullExporterFactory(out), tracer, parallel)
-			if e3 := puller.Pull(start, filter, filters, filtersEx); e3 != nil {
+			if e3 := puller.Pull(start, filter, startSelect, filters, filtersEx); e3 != nil {
 				log.Fatal().AnErr("error", e3).Msg("Fatal error stop the pull command")
 				os.Exit(1)
 			}
@@ -222,26 +222,26 @@ func getDataSource(dataconnectorName string, out io.Writer) (pull.DataSource, er
 	return datasourceFactory.New(u.URL.String(), alias.Schema), nil
 }
 
-func getPullerPlan(idStorage id.Storage) (pull.Plan, pull.Table, error) {
-	ep, err1 := id.GetPullerPlan(idStorage)
+func getPullerPlan(idStorage id.Storage) (pull.Plan, pull.Table, []string, error) {
+	pp, err1 := id.GetPullerPlan(idStorage)
 	if err1 != nil {
-		return pull.Plan{}, pull.Table{}, err1
+		return pull.Plan{}, pull.Table{}, []string{}, err1
 	}
 
 	relations, err2 := relStorage.List()
 	if err2 != nil {
-		return pull.Plan{}, pull.Table{}, err2
+		return pull.Plan{}, pull.Table{}, []string{}, err2
 	}
 
 	tables, err3 := tabStorage.List()
 	if err3 != nil {
-		return pull.Plan{}, pull.Table{}, err3
+		return pull.Plan{}, pull.Table{}, []string{}, err3
 	}
 
-	builder := newBuilder(ep, relations, tables)
+	builder := newBuilder(pp, relations, tables)
 	plan, startTable, err4 := builder.plan()
 	if err4 != nil {
-		return pull.Plan{}, pull.Table{}, err4
+		return pull.Plan{}, pull.Table{}, []string{}, err4
 	}
 
 	// Check startTable existe in table.yaml
@@ -256,8 +256,8 @@ func getPullerPlan(idStorage id.Storage) (pull.Plan, pull.Table, error) {
 
 	if !tableExiste {
 		err5 := fmt.Errorf("Table '%s' does not exist in table.yaml", string(startTable.Name))
-		return pull.Plan{}, pull.Table{}, err5
+		return pull.Plan{}, pull.Table{}, []string{}, err5
 	}
 
-	return plan, startTable, nil
+	return plan, startTable, pp.Select(), nil
 }

internal/app/pull/http.go

@@ -126,7 +126,7 @@ func HandlerFactory(ingressDescriptor string) func(w http.ResponseWriter, r *htt
 			return
 		}
 
-		plan, start, e2 := getPullerPlan(idStorageFactory(query.Get("table"), ingressDescriptor))
+		plan, start, startSelect, e2 := getPullerPlan(idStorageFactory(query.Get("table"), ingressDescriptor))
 		if e2 != nil {
 			log.Error().Err(e2).Msg("")
 			w.WriteHeader(http.StatusInternalServerError)
@@ -141,7 +141,7 @@ func HandlerFactory(ingressDescriptor string) func(w http.ResponseWriter, r *htt
 		pullExporter := pullExporterFactory(w)
 		puller := pull.NewPuller(plan, datasource, pullExporter, pull.NoTraceListener{})
 
-		e3 := puller.Pull(start, pull.Filter{Limit: limit, Values: pull.Row{}, Where: where, Distinct: distinct}, nil, nil)
+		e3 := puller.Pull(start, pull.Filter{Limit: limit, Values: pull.Row{}, Where: where, Distinct: distinct}, startSelect, nil, nil)
 		if e3 != nil {
 			log.Error().Err(e3).Msg("")
 			w.WriteHeader(http.StatusInternalServerError)

internal/infra/id/table_storage.go

@@ -38,5 +38,5 @@ func (s *TableStorage) Store(adef id.IngressDescriptor) *id.Error {
 
 // Read create new Ingress Descriptor with table as start table without relations
 func (s *TableStorage) Read() (id.IngressDescriptor, *id.Error) {
-	return id.NewIngressDescriptor(s.table, id.NewIngressRelationList([]id.IngressRelation{})), nil
+	return id.NewIngressDescriptor(s.table, []string{}, id.NewIngressRelationList([]id.IngressRelation{})), nil
 }

internal/infra/id/yaml_storage.go

@@ -34,6 +34,7 @@ type YAMLStructure struct {
 // YAMLIngressDescriptor defines how to store an ingress descriptor in YAML format.
 type YAMLIngressDescriptor struct {
 	StartTable string         `yaml:"startTable"`
+	Select     []string       `yaml:"select"`
 	Relations  []YAMLRelation `yaml:"relations"`
 }
 
@@ -46,9 +47,10 @@ type YAMLRelation struct {
 
 // YAMLTable defines how to store a table in YAML format.
 type YAMLTable struct {
-	Name   string `yaml:"name"`
-	Lookup bool   `yaml:"lookup"`
-	Where  string `yaml:"where,omitempty"`
+	Name   string   `yaml:"name"`
+	Lookup bool     `yaml:"lookup"`
+	Where  string   `yaml:"where,omitempty"`
+	Select []string `yaml:"select,omitempty"`
 }
 
 // YAMLStorage provides storage in a local YAML file
@@ -73,13 +75,14 @@ func (s *YAMLStorage) Store(id id.IngressDescriptor) *id.Error {
 		relation := list.Relation(i)
 		relations = append(relations, YAMLRelation{
 			Name:   relation.Name(),
-			Parent: YAMLTable{Name: relation.Parent().Name(), Lookup: relation.LookUpParent(), Where: relation.WhereParent()},
-			Child:  YAMLTable{Name: relation.Child().Name(), Lookup: relation.LookUpChild(), Where: relation.WhereChild()},
+			Parent: YAMLTable{Name: relation.Parent().Name(), Lookup: relation.LookUpParent(), Where: relation.WhereParent(), Select: relation.SelectParent()},
+			Child:  YAMLTable{Name: relation.Child().Name(), Lookup: relation.LookUpChild(), Where: relation.WhereChild(), Select: relation.SelectChild()},
 		})
 	}
 
 	structure.IngressDescriptor = YAMLIngressDescriptor{
 		StartTable: id.StartTable().Name(),
+		Select:     id.Select(),
 		Relations:  relations,
 	}
 
@@ -107,11 +110,13 @@ func (s *YAMLStorage) Read() (id.IngressDescriptor, *id.Error) {
 					id.NewTable(relation.Child.Name),
 				),
 				relation.Parent.Lookup, relation.Child.Lookup,
-				relation.Parent.Where, relation.Child.Where),
+				relation.Parent.Where, relation.Child.Where,
+				relation.Parent.Select, relation.Child.Select,
+			),
 		)
 	}
 
-	return id.NewIngressDescriptor(id.NewTable(structure.IngressDescriptor.StartTable), id.NewIngressRelationList(relations)), nil
+	return id.NewIngressDescriptor(id.NewTable(structure.IngressDescriptor.StartTable), structure.IngressDescriptor.Select, id.NewIngressRelationList(relations)), nil
 }
 
 func writeFile(structure *YAMLStructure, filename string) *id.Error {