Merge pull request #23 from d-ylee/bugfix/datasetsSQLerror

Fix for Issue #22
dmwm · Apr 11, 2022 · c4179e5 · c4179e5
2 parents 50f05b9 + 92bc8f8
commit c4179e5
Show file tree

Hide file tree

Showing 18 changed files with 531 additions and 254 deletions.
diff --git a/Makefile b/Makefile
@@ -34,7 +34,7 @@ install:
 clean:
 	go clean; rm -rf pkg
 
-test: test-dbs test-sql test-errors test-validator test-bulk test-http test-utils test-migrate test-writer test-lexicon bench
+test: test-dbs test-sql test-errors test-validator test-bulk test-http test-utils test-migrate test-writer test-integration test-lexicon bench
 
 test-github: test-dbs test-sql test-errors test-validator test-bulk test-http test-utils test-writer test-lexicon test-integration bench
 

diff --git a/dbs/acquisitioneras.go b/dbs/acquisitioneras.go
@@ -92,7 +92,7 @@ func (r *AcquisitionEras) Insert(tx *sql.Tx) error {
 		r.CREATE_BY,
 		r.DESCRIPTION)
 	if utils.VERBOSE > 0 {
-		log.Printf("unable to insert AcquisitionEras +v", stm, err)
+		log.Printf("unable to insert AcquisitionEras %s error %+v", stm, err)
 	}
 	if err != nil {
 		return Error(err, InsertErrorCode, "", "dbs.acquisitioneras.Insert")

diff --git a/dbs/datasets.go b/dbs/datasets.go
@@ -96,7 +96,7 @@ func (a *API) Datasets() error {
 	// optional arguments
 	if _, e := getSingleValue(a.Params, "parent_dataset"); e == nil {
 		tmpl["ParentDataset"] = true
-		conds, args = AddParam("parent_dataset", "PDS.DATASET PARENT_DATASET", a.Params, conds, args)
+		conds, args = AddParam("parent_dataset", "PDS.DATASET", a.Params, conds, args)
 	}
 	if _, e := getSingleValue(a.Params, "release_version"); e == nil {
 		tmpl["Version"] = true
@@ -245,11 +245,31 @@ func (a *API) Datasets() error {
 		new(sql.NullString),
 		new(sql.NullInt64),
 		new(sql.NullString)}
+	if tmpl["Version"].(bool) {
+		cols = append(
+			cols,
+			"output_module_label",
+			"global_tag",
+			"release_version",
+			"pset_hash",
+			"app_name")
+		vals = append(
+			vals,
+			new(sql.NullString),
+			new(sql.NullString),
+			new(sql.NullString),
+			new(sql.NullString),
+			new(sql.NullString))
+	}
 	if strings.ToLower(detail) != "true" {
 		//         stm = getSQL("datasets_short")
 		cols = []string{"dataset"}
 		vals = []interface{}{new(sql.NullString)}
 	}
+	if tmpl["ParentDataset"].(bool) {
+		cols = append(cols, "parent_dataset")
+		vals = append(vals, new(sql.NullString))
+	}
 	stm = WhereClause(stm, conds)
 
 	// use generic query API to fetch the results from DB
@@ -432,17 +452,18 @@ type DatasetRecord struct {
 	DATASET         string `json:"dataset" validate:"required"`
 	PRIMARY_DS_NAME string `json:"primary_ds_name" validate:"required"`
 	//     PRIMARY_DS_TYPE        string  `json:"primary_ds_type" validate:"required"`
-	PROCESSED_DS_NAME      string  `json:"processed_ds_name" validate:"required"`
-	DATA_TIER_NAME         string  `json:"data_tier_name" validate:"required"`
-	ACQUISITION_ERA_NAME   string  `json:"acquisition_era_name" validate:"required"`
-	DATASET_ACCESS_TYPE    string  `json:"dataset_access_type" validate:"required"`
-	PROCESSING_VERSION     int64   `json:"processing_version" validate:"required,number,gt=0"`
-	PHYSICS_GROUP_NAME     string  `json:"physics_group_name" validate:"required"`
-	XTCROSSSECTION         float64 `json:"xtcrosssection" validate:"required,number"`
-	CREATION_DATE          int64   `json:"creation_date" validate:"required,number,gt=0"`
-	CREATE_BY              string  `json:"create_by" validate:"required"`
-	LAST_MODIFICATION_DATE int64   `json:"last_modification_date" validate:"required,number,gt=0"`
-	LAST_MODIFIED_BY       string  `json:"last_modified_by" validate:"required"`
+	PROCESSED_DS_NAME      string               `json:"processed_ds_name" validate:"required"`
+	DATA_TIER_NAME         string               `json:"data_tier_name" validate:"required"`
+	ACQUISITION_ERA_NAME   string               `json:"acquisition_era_name" validate:"required"`
+	DATASET_ACCESS_TYPE    string               `json:"dataset_access_type" validate:"required"`
+	PROCESSING_VERSION     int64                `json:"processing_version" validate:"required,number,gt=0"`
+	PHYSICS_GROUP_NAME     string               `json:"physics_group_name" validate:"required"`
+	XTCROSSSECTION         float64              `json:"xtcrosssection" validate:"required,number"`
+	CREATION_DATE          int64                `json:"creation_date" validate:"required,number,gt=0"`
+	CREATE_BY              string               `json:"create_by" validate:"required"`
+	LAST_MODIFICATION_DATE int64                `json:"last_modification_date" validate:"required,number,gt=0"`
+	LAST_MODIFIED_BY       string               `json:"last_modified_by" validate:"required"`
+	OUTPUT_CONFIGS         []OutputConfigRecord `json:"output_configs"`
 }
 
 // InsertDatasets DBS API implements the following logic:
@@ -611,6 +632,23 @@ func (a *API) InsertDatasets() error {
 		return Error(err, InsertErrorCode, "", "dbs.datasets.InsertDatasets")
 	}
 
+	// get current dataset id
+	dsid, err := GetID(tx, "DATASETS", "dataset_id", "dataset", dsrec.DATASET)
+	if err != nil {
+		return Error(err, GetIDErrorCode, "", "dbs.datasets.InsertDatasets")
+	}
+	for _, oc := range rec.OUTPUT_CONFIGS {
+		ocid, err := GetID(tx, "OUTPUT_MODULE_CONFIGS", "output_mod_config_id", "output_module_label", oc.OUTPUT_MODULE_LABEL)
+		if err != nil {
+			return Error(err, GetIDErrorCode, "", "dbs.datasets.InsertDatasets")
+		}
+		r := DatasetOutputModConfigs{OUTPUT_MOD_CONFIG_ID: ocid, DATASET_ID: dsid}
+		err = r.Insert(tx)
+		if err != nil {
+			return Error(err, InsertErrorCode, "", "dbs.files.InsertFiles")
+		}
+	}
+
 	// commit transaction
 	err = tx.Commit()
 	if err != nil {

diff --git a/dbs/filelumis.go b/dbs/filelumis.go
@@ -121,7 +121,7 @@ func (a *API) FileLumis() error {
 
 // FileLumis represents File Lumis DBS DB table
 type FileLumis struct {
-	FILE_ID          int64 `json:"file_id validate:"required,number""`
+	FILE_ID          int64 `json:"file_id" validate:"required,number"`
 	LUMI_SECTION_NUM int64 `json:"lumi_section_num" validate:"required,number"`
 	RUN_NUM          int64 `json:"run_num" validate:"required,number"`
 	EVENT_COUNT      int64 `json:"event_count"`

diff --git a/dbs/migration_requests.go b/dbs/migration_requests.go
@@ -20,7 +20,7 @@ type MigrationRequest struct {
 	CREATE_BY              string `json:"create_by" validate:"required"`
 	CREATION_DATE          int64  `json:"creation_date" validate:"required,number,gt=0"`
 	LAST_MODIFIED_BY       string `json:"last_modified_by" validate:"required"`
-	LAST_MODIFICATION_DATE int64  `json:"last_modification_date" validate:"required,number,gt=0`
+	LAST_MODIFICATION_DATE int64  `json:"last_modification_date" validate:"required,number,gt=0"`
 	RETRY_COUNT            int64  `json:"retry_count"`
 }
 

diff --git a/static/sql/datasets.sql b/static/sql/datasets.sql
@@ -17,19 +17,19 @@ SELECT
         AE.ACQUISITION_ERA_NAME,
         PE.PROCESSING_VERSION,
         PH.PHYSICS_GROUP_NAME 
-{{if .ParentDataset}}
-        ,PDS.DATASET PARENT_DATASET
-{{end}}
 {{if .Version}}
         ,OMC.OUTPUT_MODULE_LABEL
         ,OMC.GLOBAL_TAG
-        ,RV.RELEASE_VERSION,
-        ,PSH.PSET_HASH,
+        ,RV.RELEASE_VERSION
+        ,PSH.PSET_HASH
         ,AEX.APP_NAME
 {{end}}
 {{else}}
         D.DATASET
 {{end}}
+{{if .ParentDataset}}
+        ,PDS.DATASET PARENT_DATASET
+{{end}}
 
 FROM {{.Owner}}.DATASETS D
 {{if .Lfns}}

diff --git a/test/data/integration/integration_data.json b/test/data/integration/integration_data.json
@@ -5,7 +5,8 @@
   "primary_ds_type": "test",
   "procdataset": "acq_era_8268-v8268",
   "physics_group_name": "Tracker",
-  "dataset_access_type": "PRODUCTION",
+  "dataset_access_type": "VALID",
+  "dataset_access_type2": "PRODUCTION",
   "tier": "GEN-SIM-RAW",
   "dataset": "/unittest_web_primary_ds_name_8268/acq_era_8268-v8268/GEN-SIM-RAW",
   "parent_dataset": "/unittest_web_primary_ds_name_8268/acq_era_8268-ptsr-v8268/GEN-SIM-RAW",

diff --git a/test/int_acquisition_eras.go b/test/int_acquisition_eras.go
@@ -42,7 +42,7 @@ func getAcquisitionErasTestTable(t *testing.T) EndpointTestCase {
 				respCode:    http.StatusOK,
 			},
 			{
-				description: "Test POST",
+				description: "Test POST", // DBSClientWriter_t.test06
 				method:      "POST",
 				serverType:  "DBSWriter",
 				input:       acqEraReq,

diff --git a/test/int_data_sets.go b/test/int_data_sets.go