diff --git a/cmd/bcda_fetch/bcda_fetch.go b/cmd/bcda_fetch/bcda_fetch.go index cb436b4..f1b05ed 100644 --- a/cmd/bcda_fetch/bcda_fetch.go +++ b/cmd/bcda_fetch/bcda_fetch.go @@ -37,7 +37,7 @@ import ( var ( clientID = flag.String("client_id", "", "BCDA API client ID (required)") clientSecret = flag.String("client_secret", "", "BCDA API client secret (required)") - outputPrefix = flag.String("output_prefix", "claims/claims_data", "Data output prefix") + outputPrefix = flag.String("output_prefix", "", "Data output prefix. If unset, no file output will be written.") useV2 = flag.Bool("use_v2", false, "This indicates if the BCDA V2 API should be used, which returns R4 mapped data.") rectify = flag.Bool("rectify", false, "This indicates that this program should attempt to rectify BCDA FHIR so that it is valid R4 FHIR.") enableFHIRStore = flag.Bool("enable_fhir_store", false, "If true, this enables write to GCP FHIR store. If true, all other fhir_store_* flags must be set.") @@ -100,6 +100,10 @@ func mainWrapper(cfg mainWrapperConfig) error { return errors.New("if enable_fhir_store is true, all other FHIR store related flags must be set") } + if *outputPrefix == "" && !*enableFHIRStore { + log.Warningln("outputPrefix is not set and neither is enableFHIRStore: BCDA fetch will not produce any output.") + } + apiVersion := bcda.V1 if *useV2 { apiVersion = bcda.V2 @@ -166,7 +170,11 @@ func mainWrapper(cfg mainWrapperConfig) error { for r, urls := range jobStatus.ResultURLs { for i, url := range urls { - filePrefix := fmt.Sprintf("%s_%s_%d", *outputPrefix, r, i) + filePrefix := "" + if *outputPrefix != "" { + filePrefix = fmt.Sprintf("%s_%s_%d", *outputPrefix, r, i) + } + r, err := getDataOrExit(cl, url, *clientID, *clientSecret) if err != nil { return err @@ -236,9 +244,13 @@ func writeData(r io.Reader, filePrefix string) { } func rectifyAndWrite(r io.Reader, filePrefix string, cfg mainWrapperConfig) { - w, err := os.OpenFile(fmt.Sprintf("%s.ndjson", filePrefix), os.O_RDWR|os.O_CREATE, 0755) - if err != nil { - log.Exitf("Unable to create output file: %v", err) + var w io.Writer = nil + if filePrefix != "" { + var err error + w, err = os.OpenFile(fmt.Sprintf("%s.ndjson", filePrefix), os.O_RDWR|os.O_CREATE, 0755) + if err != nil { + log.Exitf("Unable to create output file: %v", err) + } } uploader := newFHIRStoreUploader(cfg.fhirStoreEndpoint) @@ -259,8 +271,10 @@ func rectifyAndWrite(r io.Reader, filePrefix string, cfg mainWrapperConfig) { uploader.Upload(fhirOut) } - if _, err := w.Write(fhirOut); err != nil { - log.Exitf("issue during file write: %v", err) + if w != nil { + if _, err := w.Write(fhirOut); err != nil { + log.Exitf("issue during file write: %v", err) + } } } // Since there is only one sender goroutine (this one), it should be safe to diff --git a/cmd/bcda_fetch/bcda_fetch_test.go b/cmd/bcda_fetch/bcda_fetch_test.go index da823e3..4a7d684 100644 --- a/cmd/bcda_fetch/bcda_fetch_test.go +++ b/cmd/bcda_fetch/bcda_fetch_test.go @@ -52,7 +52,9 @@ func TestMainWrapper(t *testing.T) { fhirStoreFailures bool noFailOnUploadErrors bool bcdaJobID string - wantError error + // unsetOutputPrefix sets the outputPrefix to empty string if true. + unsetOutputPrefix bool + wantError error }{ { name: "RectifyEnabledWithoutFHIRStoreBCDAV1", @@ -253,6 +255,61 @@ func TestMainWrapper(t *testing.T) { sinceFileExpectedContent: []byte("2020-12-09T11:00:00.123+00:00\n"), bcdaJobID: "999", }, + // Only testing cases with FHIR Store enabled for setting outputPrefix to "" + { + name: "EmptyOutputPrefixWithRectifyEnabledWithFHIRStoreBCDAV1", + rectify: true, + enableFHIRStore: true, + apiVersion: bcda.V1, + unsetOutputPrefix: true, + }, + { + name: "EmptyOutputPrefixWithRectifyEnabledWithFHIRStoreBCDAV2", + rectify: true, + enableFHIRStore: true, + apiVersion: bcda.V2, + unsetOutputPrefix: true, + }, + { + name: "EmptyOutputPrefixWithSinceProvidedWithRectifyWithFHIRStoreBCDAV2WithBCDAJobId", + rectify: true, + enableFHIRStore: true, + apiVersion: bcda.V2, + since: "2006-01-02T15:04:05.000-07:00", + bcdaJobID: "999", + unsetOutputPrefix: true, + }, + { + name: "EmptyOutputPrefixWithSinceProvidedWithRectifyWithFHIRStoreBCDAV1WithBCDAJobId", + rectify: true, + enableFHIRStore: true, + apiVersion: bcda.V1, + since: "2006-01-02T15:04:05.000-07:00", + bcdaJobID: "999", + unsetOutputPrefix: true, + }, + { + name: "EmptyOutputPrefixWithSinceFileProvidedWithBCDAV2WithBCDAJobIdWithFHIRStore", + rectify: true, + enableFHIRStore: true, + apiVersion: bcda.V2, + sinceFileContent: []byte("2013-12-09T11:00:00.123+00:00\n2015-12-09T11:00:00.123+00:00\n"), + sinceFileLatestTimestamp: "2015-12-09T11:00:00.123+00:00", + sinceFileExpectedContent: []byte("2013-12-09T11:00:00.123+00:00\n2015-12-09T11:00:00.123+00:00\n2020-12-09T11:00:00.123+00:00\n"), + bcdaJobID: "999", + unsetOutputPrefix: true, + }, + { + name: "EmptyOutputPrefixWithSinceFileProvidedWithBCDAV1WithBCDAJobIdWithFHIRStore", + rectify: true, + enableFHIRStore: true, + apiVersion: bcda.V1, + sinceFileContent: []byte("2013-12-09T11:00:00.123+00:00\n2015-12-09T11:00:00.123+00:00\n"), + sinceFileLatestTimestamp: "2015-12-09T11:00:00.123+00:00", + sinceFileExpectedContent: []byte("2013-12-09T11:00:00.123+00:00\n2015-12-09T11:00:00.123+00:00\n2020-12-09T11:00:00.123+00:00\n"), + bcdaJobID: "999", + unsetOutputPrefix: true, + }, // TODO(b/226375559): see if we can generate some of the test cases above // instead of having to spell them out explicitly. // TODO(b/213365276): test that bcda V1 with rectify = true results in an @@ -363,7 +420,9 @@ func TestMainWrapper(t *testing.T) { defer SaveFlags().Restore() flag.Set("client_id", "id") flag.Set("client_secret", "secret") - flag.Set("output_prefix", outputPrefix) + if !tc.unsetOutputPrefix { + flag.Set("output_prefix", outputPrefix) + } flag.Set("bcda_server_url", bcdaServer.URL) flag.Set("fhir_store_gcp_project", gcpProject) @@ -466,19 +525,21 @@ func TestMainWrapper(t *testing.T) { } } - for fileSuffix, wantData := range expectedFileSuffixToData { - fullPath := outputPrefix + fileSuffix - r, err := os.Open(fullPath) - if err != nil { - t.Errorf("unable to open file %s: %s", fullPath, err) - } - defer r.Close() - gotData, err := io.ReadAll(r) - if err != nil { - t.Errorf("error reading file %s: %v", fullPath, err) - } - if !cmp.Equal(normalizeJSON(t, gotData), normalizeJSON(t, wantData)) { - t.Errorf("mainWrapper unexpected ndjson output for file %s. got: %s, want: %s", fullPath, gotData, wantData) + if !tc.unsetOutputPrefix { + for fileSuffix, wantData := range expectedFileSuffixToData { + fullPath := outputPrefix + fileSuffix + r, err := os.Open(fullPath) + if err != nil { + t.Errorf("unable to open file %s: %s", fullPath, err) + } + defer r.Close() + gotData, err := io.ReadAll(r) + if err != nil { + t.Errorf("error reading file %s: %v", fullPath, err) + } + if !cmp.Equal(normalizeJSON(t, gotData), normalizeJSON(t, wantData)) { + t.Errorf("mainWrapper unexpected ndjson output for file %s. got: %s, want: %s", fullPath, gotData, wantData) + } } }