Skip to content

Commit

Permalink
Merge pull request #101 from d-ylee/feature-94-partial_parentage
Browse files Browse the repository at this point in the history 
Allow Partial Parentage for fileparents API

Need to provide `missingFiles` key in JSON body
`child_parent_id_list` that is provided to the API must match the same number as the number of files that were injected from the block. Provide a child-parent id pair with the parent id as -1.
  • Loading branch information
@d-ylee
d-ylee authored Oct 23, 2023
2 parents e71c8a4 + 33ea8b3 commit 4d184ca
Show file tree
Hide file tree
Showing 7 changed files with 219 additions and 44 deletions.
1 change: 1 addition & 0 deletions .github/workflows/build.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,7 @@ jobs:
# - run: docker push cmssw/dbs2go:${{steps.get-ref.outputs.tag}}

- name: Push new image to k8s
if: "!contains(${{steps.get-ref.outputs.tag}}, 'dev')"
run: |
curl -ksLO https://raw.githubusercontent.com/vkuznet/imagebot/main/imagebot.sh
sed -i -e "s,COMMIT,${{github.sha}},g" -e "s,REPOSITORY,${{github.repository}},g" -e "s,NAMESPACE,dbs,g" -e "s,TAG,${{steps.get-ref.outputs.tag}},g" -e "s,IMAGE,registry.cern.ch/cmsweb/dbs2go,g" -e "s,SERVICE,dbs2go,g" -e "s,HOST,${{secrets.IMAGEBOT_URL}},g" imagebot.sh
Expand Down
34 changes: 33 additions & 1 deletion dbs/fileparents.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package dbs
import (
"database/sql"
"encoding/json"
"errors"
"fmt"
"io"
"log"
Expand Down Expand Up @@ -60,10 +61,11 @@ func (a *API) FileParents() error {
// FileParents represents file parents DBS DB table
type FileParents struct {
THIS_FILE_ID int64 `json:"this_file_id" validate:"required,number,gt=0"`
PARENT_FILE_ID int64 `json:"parent_file_id" validate:"required,number,gt=0"`
PARENT_FILE_ID int64 `json:"parent_file_id" validate:"required,number"` // TODO: may need to modify to handle `none` from python
}

// Insert implementation of FileParents
//
//gocyclo:ignore
func (r *FileParents) Insert(tx *sql.Tx) error {
var tid int64
Expand Down Expand Up @@ -206,6 +208,7 @@ func (r *FileParents) Insert(tx *sql.Tx) error {
}

// Validate implementation of FileParents
// TODO: handle this for partial parentage
func (r *FileParents) Validate() error {
if err := RecordValidator.Struct(*r); err != nil {
return DecodeValidatorError(r, err)
Expand Down Expand Up @@ -239,6 +242,7 @@ func (r *FileParents) Decode(reader io.Reader) error {
// err := decoder.Decode(&rec)
if err != nil {
log.Println("fail to decode data", err)
errors.Unwrap(err)
return Error(err, UnmarshalErrorCode, "", "dbs.fileparents.Decode")
}
return nil
Expand Down Expand Up @@ -274,12 +278,17 @@ func (a *API) InsertFileParents() error {
}

// FileParentBlockRecord represents file parent DBS record
// BlockName: name of the block
// ChildParentIDList: list of child and parent file ids
// MissingFiles: Number of missing files in the child and parent file list
type FileParentBlockRecord struct {
BlockName string `json:"block_name"`
ChildParentIDList [][]int64 `json:"child_parent_id_list"`
MissingFiles int64 `json:"missing_files"`
}

// InsertFileParentsBlockTxt DBS API
//
//gocyclo:ignore
func (a *API) InsertFileParentsBlockTxt(tx *sql.Tx) error {
// read given input
Expand Down Expand Up @@ -358,6 +367,8 @@ func (a *API) InsertFileParentsBlockTxt(tx *sql.Tx) error {
}

// now we can loop over provided list and insert file parents
var missingFiles []FileParents
var validatedChildParentIDList []FileParents
for _, v := range rec.ChildParentIDList {
var r FileParents
r.THIS_FILE_ID = v[0]
Expand All @@ -370,6 +381,27 @@ func (a *API) InsertFileParentsBlockTxt(tx *sql.Tx) error {
log.Println("unable to validate the record", r, "error", err)
return Error(err, ValidateErrorCode, "", "dbs.fileparents.InsertFileParentsBlockTxt")
}
// only provide list of child parent that have parentage
// otherwise, skip partial parentage and add to the missingFiles count
if r.PARENT_FILE_ID > 0 {
validatedChildParentIDList = append(validatedChildParentIDList, r)
} else {
missingFiles = append(missingFiles, r)
}
}

if len(missingFiles) > 0 {
log.Printf("files without parentage: %v", missingFiles)
}

// check if provided MissingFiles matches the amount of mising files found
if len(missingFiles) != int(rec.MissingFiles) {
log.Println("provided missingFiles does not match number of pairs with no parentage")
return Error(err, ValidateErrorCode, "", "dbs.fileparents.InsertFileParentsBlockTxt")
}

// only insert valid child parent id lists
for _, r := range validatedChildParentIDList {
err = r.Insert(tx)
if err != nil {
if utils.VERBOSE > 1 {
Expand Down
2 changes: 1 addition & 1 deletion static/parameters.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -156,7 +156,7 @@
{
"api": "fileparents",
"parameters": [
"logical_file_name", "block_name", "block_id"
"logical_file_name", "block_name", "block_id", "missing_files"
]
},
{
Expand Down
64 changes: 32 additions & 32 deletions test/data/integration/bulkblocks_data.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
"release_version": "CMSSW_1_2_3",
"pset_hash": "76e303993a1c2f842159dbfeeed9a0dd",
"pset_name": "",
"lfn": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_/p8268/0.root",
"lfn": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_ptr/p8268/0.root",
"app_name": "cmsRun",
"output_module_label": "merged",
"global_tag": "my-cms-gtag_8268",
Expand All @@ -28,7 +28,7 @@
"release_version": "CMSSW_1_2_3",
"pset_hash": "76e303993a1c2f842159dbfeeed9a0dd",
"pset_name": "",
"lfn": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_/p8268/1.root",
"lfn": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_ptr/p8268/1.root",
"app_name": "cmsRun",
"output_module_label": "merged",
"global_tag": "my-cms-gtag_8268",
Expand All @@ -39,7 +39,7 @@
"release_version": "CMSSW_1_2_3",
"pset_hash": "76e303993a1c2f842159dbfeeed9a0dd",
"pset_name": "",
"lfn": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_/p8268/2.root",
"lfn": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_ptr/p8268/2.root",
"app_name": "cmsRun",
"output_module_label": "merged",
"global_tag": "my-cms-gtag_8268",
Expand All @@ -50,7 +50,7 @@
"release_version": "CMSSW_1_2_3",
"pset_hash": "76e303993a1c2f842159dbfeeed9a0dd",
"pset_name": "",
"lfn": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_/p8268/3.root",
"lfn": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_ptr/p8268/3.root",
"app_name": "cmsRun",
"output_module_label": "merged",
"global_tag": "my-cms-gtag_8268",
Expand All @@ -61,7 +61,7 @@
"release_version": "CMSSW_1_2_3",
"pset_hash": "76e303993a1c2f842159dbfeeed9a0dd",
"pset_name": "",
"lfn": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_/p8268/4.root",
"lfn": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_ptr/p8268/4.root",
"app_name": "cmsRun",
"output_module_label": "merged",
"global_tag": "my-cms-gtag_8268",
Expand Down Expand Up @@ -95,7 +95,7 @@
"file_type": "EDM",
"last_modified_by": "",
"last_modification_date": 0,
"logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_/p8268/0.root",
"logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_ptr/p8268/0.root",
"md5": "",
"auto_cross_section": 0,
"is_file_valid": 1
Expand Down Expand Up @@ -125,7 +125,7 @@
"file_type": "EDM",
"last_modified_by": "",
"last_modification_date": 0,
"logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_/p8268/1.root",
"logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_ptr/p8268/1.root",
"md5": "",
"auto_cross_section": 0,
"is_file_valid": 1
Expand Down Expand Up @@ -155,7 +155,7 @@
"file_type": "EDM",
"last_modified_by": "",
"last_modification_date": 0,
"logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_/p8268/2.root",
"logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_ptr/p8268/2.root",
"md5": "",
"auto_cross_section": 0,
"is_file_valid": 1
Expand Down Expand Up @@ -185,7 +185,7 @@
"file_type": "EDM",
"last_modified_by": "",
"last_modification_date": 0,
"logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_/p8268/3.root",
"logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_ptr/p8268/3.root",
"md5": "",
"auto_cross_section": 0,
"is_file_valid": 1
Expand Down Expand Up @@ -215,7 +215,7 @@
"file_type": "EDM",
"last_modified_by": "",
"last_modification_date": 0,
"logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_/p8268/4.root",
"logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_ptr/p8268/4.root",
"md5": "",
"auto_cross_section": 0,
"is_file_valid": 1
Expand All @@ -232,19 +232,19 @@
"create_by": "WMAgent",
"primary_ds_type": "test",
"primary_ds_name": "unittest_web_primary_ds_name_8268_stepchain",
"creation_date": 1652798955
"creation_date": 1695842582
},
"dataset": {
"dataset_id": 0,
"create_by": "WMAgent",
"creation_date": 1652798955,
"creation_date": 1695842582,
"physics_group_name": "Tracker",
"dataset_access_type": "PRODUCTION",
"data_tier_name": "GEN-SIM-RAW",
"last_modified_by": "WMAgent",
"processed_ds_name": "acq_era_8268-ptsr-v8268",
"xtcrosssection": 0,
"last_modification_date": 1652798955,
"last_modification_date": 1695842582,
"dataset": "/unittest_web_primary_ds_name_8268_stepchain/acq_era_8268-ptsr-v8268/GEN-SIM-RAW",
"prep_id": "TestPrepID"
},
Expand Down Expand Up @@ -507,19 +507,19 @@
"create_by": "WMAgent",
"primary_ds_type": "test",
"primary_ds_name": "unittest_web_primary_ds_name_8268_stepchain",
"creation_date": 1652798955
"creation_date": 1695842582
},
"dataset": {
"dataset_id": 0,
"create_by": "WMAgent",
"creation_date": 1652798955,
"creation_date": 1695842582,
"physics_group_name": "Tracker",
"dataset_access_type": "PRODUCTION",
"data_tier_name": "GEN-SIM-RAW",
"last_modified_by": "WMAgent",
"processed_ds_name": "acq_era_8268-v8268",
"xtcrosssection": 0,
"last_modification_date": 1652798955,
"last_modification_date": 1695842582,
"dataset": "/unittest_web_primary_ds_name_8268_stepchain/acq_era_8268-v8268/GEN-SIM-RAW",
"prep_id": "TestPrepID"
},
Expand Down Expand Up @@ -591,7 +591,7 @@
"file_type": "EDM",
"last_modified_by": "",
"last_modification_date": 0,
"logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_/p8268/5.root",
"logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_ptr/p8268/5.root",
"md5": "",
"auto_cross_section": 0,
"is_file_valid": 1
Expand Down Expand Up @@ -621,7 +621,7 @@
"file_type": "EDM",
"last_modified_by": "",
"last_modification_date": 0,
"logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_/p8268/6.root",
"logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_ptr/p8268/6.root",
"md5": "",
"auto_cross_section": 0,
"is_file_valid": 1
Expand Down Expand Up @@ -651,7 +651,7 @@
"file_type": "EDM",
"last_modified_by": "",
"last_modification_date": 0,
"logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_/p8268/7.root",
"logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_ptr/p8268/7.root",
"md5": "",
"auto_cross_section": 0,
"is_file_valid": 1
Expand Down Expand Up @@ -681,7 +681,7 @@
"file_type": "EDM",
"last_modified_by": "",
"last_modification_date": 0,
"logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_/p8268/8.root",
"logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_ptr/p8268/8.root",
"md5": "",
"auto_cross_section": 0,
"is_file_valid": 1
Expand Down Expand Up @@ -711,7 +711,7 @@
"file_type": "EDM",
"last_modified_by": "",
"last_modification_date": 0,
"logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_/p8268/9.root",
"logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_ptr/p8268/9.root",
"md5": "",
"auto_cross_section": 0,
"is_file_valid": 1
Expand All @@ -728,19 +728,19 @@
"create_by": "WMAgent",
"primary_ds_type": "test",
"primary_ds_name": "unittest_web_primary_ds_name_8268_stepchain2",
"creation_date": 1652798955
"creation_date": 1695842582
},
"dataset": {
"dataset_id": 0,
"create_by": "WMAgent",
"creation_date": 1652798955,
"creation_date": 1695842582,
"physics_group_name": "Tracker",
"dataset_access_type": "PRODUCTION",
"data_tier_name": "GEN-SIM-RAW",
"last_modified_by": "WMAgent",
"processed_ds_name": "acq_era_8268-ptsr-v82682",
"xtcrosssection": 0,
"last_modification_date": 1652798955,
"last_modification_date": 1695842582,
"dataset": "/unittest_web_primary_ds_name_8268_stepchain/acq_era_8268-ptsr-v8268/GEN-SIM-RAW2",
"prep_id": "TestPrepID"
},
Expand Down Expand Up @@ -810,7 +810,7 @@
"file_type": "EDM",
"last_modified_by": "",
"last_modification_date": 0,
"logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_/p8268/5.root",
"logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_ptr/p8268/5.root",
"md5": "",
"auto_cross_section": 0,
"is_file_valid": 1
Expand Down Expand Up @@ -840,7 +840,7 @@
"file_type": "EDM",
"last_modified_by": "",
"last_modification_date": 0,
"logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_/p8268/6.root",
"logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_ptr/p8268/6.root",
"md5": "",
"auto_cross_section": 0,
"is_file_valid": 1
Expand Down Expand Up @@ -870,7 +870,7 @@
"file_type": "EDM",
"last_modified_by": "",
"last_modification_date": 0,
"logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_/p8268/7.root",
"logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_ptr/p8268/7.root",
"md5": "",
"auto_cross_section": 0,
"is_file_valid": 1
Expand Down Expand Up @@ -900,7 +900,7 @@
"file_type": "EDM",
"last_modified_by": "",
"last_modification_date": 0,
"logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_/p8268/8.root",
"logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_ptr/p8268/8.root",
"md5": "",
"auto_cross_section": 0,
"is_file_valid": 1
Expand Down Expand Up @@ -930,7 +930,7 @@
"file_type": "EDM",
"last_modified_by": "",
"last_modification_date": 0,
"logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_/p8268/9.root",
"logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_ptr/p8268/9.root",
"md5": "",
"auto_cross_section": 0,
"is_file_valid": 1
Expand All @@ -947,19 +947,19 @@
"create_by": "WMAgent",
"primary_ds_type": "test",
"primary_ds_name": "unittest_web_primary_ds_name_8268_stepchain2",
"creation_date": 1652798955
"creation_date": 1695842582
},
"dataset": {
"dataset_id": 0,
"create_by": "WMAgent",
"creation_date": 1652798955,
"creation_date": 1695842582,
"physics_group_name": "Tracker",
"dataset_access_type": "PRODUCTION",
"data_tier_name": "GEN-SIM-RAW",
"last_modified_by": "WMAgent",
"processed_ds_name": "acq_era_8268-v82682",
"xtcrosssection": 0,
"last_modification_date": 1652798955,
"last_modification_date": 1695842582,
"dataset": "/unittest_web_primary_ds_name_8268_stepchain/acq_era_8268-v8268/GEN-SIM-RAW2",
"prep_id": "TestPrepID"
},
Expand Down
Loading

0 comments on commit 4d184ca

Please sign in to comment.