diff --git a/endpointmanager/pkg/chplendpointquerier/betaAfoundriaWebScraper.go b/endpointmanager/pkg/chplendpointquerier/betaAfoundriaWebScraper.go new file mode 100644 index 000000000..eefbf7eff --- /dev/null +++ b/endpointmanager/pkg/chplendpointquerier/betaAfoundriaWebScraper.go @@ -0,0 +1,34 @@ +package chplendpointquerier + +import ( + "github.com/PuerkitoBio/goquery" + "github.com/onc-healthit/lantern-back-end/endpointmanager/pkg/helpers" + log "github.com/sirupsen/logrus" +) + +func BetaAfoundriaWebScraper(CHPLURL string, fileToWriteTo string) { + + var lanternEntryList []LanternEntry + var endpointEntryList EndpointList + + doc, err := helpers.ChromedpQueryEndpointList(CHPLURL, ".container") + if err != nil { + log.Fatal(err) + } + doc.Find(".container h3").Each(func(index int, header *goquery.Selection) { + nextAnchor := header.NextFiltered("a") + href, exists := nextAnchor.Attr("href") + if exists { + var entry LanternEntry + entry.URL = href + lanternEntryList = append(lanternEntryList, entry) + } + }) + + endpointEntryList.Endpoints = lanternEntryList + err = WriteCHPLFile(endpointEntryList, fileToWriteTo) + if err != nil { + log.Fatal(err) + } + +} diff --git a/endpointmanager/pkg/chplendpointquerier/chplendpointquerier.go b/endpointmanager/pkg/chplendpointquerier/chplendpointquerier.go index dcbbbb673..ffe8fc479 100644 --- a/endpointmanager/pkg/chplendpointquerier/chplendpointquerier.go +++ b/endpointmanager/pkg/chplendpointquerier/chplendpointquerier.go @@ -102,7 +102,7 @@ var smartemrURL = "https://smartemr.readme.io/reference/getting-started#base-url var tebraURL = "https://fhir.prd.cloud.tebra.com/fhir-request/swagger-ui/" var landmarkhealthURL = "https://lmdmzprodws.landmarkhealth.org/docs/fhir-base-urls.csv" var nthtechnologyURL = "https://admin.nthtechnology.com/fhir_endpoints.php/json" -var netsmartURL = "https://careconnect-uat.netsmartcloud.com/" +var netsmartURL = "https://careconnect.netsmartcloud.com/csv/service-base-urls-20240905.csv" var omnimdURL = "https://fhirregistration.omnimd.com/#/specification" var pcesystemsURL = "https://www.pcesystems.com/g10APIInfo.html" @@ -194,6 +194,7 @@ var nextgenPracticeURL = "https://www.nextgen.com/api/practice-search" var aspmdURL = "https://fhirapi.asp.md:3030/aspmd/fhirserver/fhir_aspmd.asp" var axeiumURL = "https://apifhir.axeium.net:8443/reference-server/" var emdscloudURL = "https://identity.emdscloud.com/api/api-resource/fhir" +var betaAfoundriaURL = "https://beta.afoundria.com/api/fhir/urls" var ehealthlineURL = "http://ehealthline.com/dev/pdf/FHIR%20API%20Endpoints.htm" var bundleQuerierArray = [30]string{"https://ac-fhir.harrisambulatory.com/endpoints/r4", "https://dynamicfhirpresentation.dynamicfhirsandbox.com/fhir/r4/endpoints", @@ -205,6 +206,11 @@ var bundleQuerierArray = [30]string{"https://ac-fhir.harrisambulatory.com/endpoi "https://appstudio.interopengine.com/partner/fhirR4endpoints-umc.json", "https://testauth.strateqhealth.com/SmartOnFHIR/ValidURLs.json", "https://fhir.ethizo.com/api/4.0.0/service_based_url"} +var ontadaURL = "https://g2fhir-int.mckesson.com/docs/index.html" +var mdlandURL = "https://api-fhir-proxy-2.mdland.net/" +var abeoURL = "https://www.crystalpm.com/FHIRServiceURLs.csv" +var nextechURL2 = "https://www.nextech.com/developers-portal" +var icareURL = "https://www.icare.com/endpoints.csv" var ezemrxURL = "https://www.ezemrx.com/fhir" func contains(arr [30]string, str string) bool { @@ -402,7 +408,7 @@ func QueryCHPLEndpointList(chplURL string, fileToWriteTo string) { } else if URLsEqual(chplURL, nthtechnologyURL) { BundleQuerierParser(chplURL, fileToWriteTo) } else if URLsEqual(chplURL, netsmartURL) { - NetsmartCSVParser("https://careconnect-uat.netsmartcloud.com/baseUrls", fileToWriteTo) + NetsmartCSVParser(chplURL, fileToWriteTo) } else if URLsEqual(chplURL, omnimdURL) { OmniMDWebscraper(chplURL, fileToWriteTo) } else if URLsEqual(chplURL, pcesystemsURL) { @@ -563,6 +569,18 @@ func QueryCHPLEndpointList(chplURL string, fileToWriteTo string) { BundleQuerierParser(chplURL, fileToWriteTo) } else if URLsEqual(emdscloudURL, chplURL) { EmdsCloudWebscraper(emdscloudURL, fileToWriteTo) + } else if URLsEqual(chplURL, betaAfoundriaURL) { + BetaAfoundriaWebScraper(chplURL, fileToWriteTo) + } else if URLsEqual(chplURL, ontadaURL) { + OntadaWebscraper(chplURL, fileToWriteTo) + } else if URLsEqual(chplURL, mdlandURL) { + MdlandWebscraper("https://api.mdland.com/Mdland%20SMART%20on%20FHIR%20OAuth%202.0%20Guide.htm", fileToWriteTo) + } else if URLsEqual(abeoURL, chplURL) { + CustomCSVParser(chplURL, fileToWriteTo, "./FHIRServiceURLs.csv", -1, 0, true, 1, 0) + } else if URLsEqual(nextechURL2, chplURL) { + CSVParser("https://www.nextech.com/hubfs/Nextech%20FHIR%20Base%20URL.csv", fileToWriteTo, "./Nextech FHIR Base URL.csv", -1, 0, true, 1, 0) + } else if URLsEqual(icareURL, chplURL) { + CSVParser(chplURL, fileToWriteTo, "./endpoints.csv", -1, 0, true, 1, 0) } else if URLsEqual(chplURL, ehealthlineURL) { EhealthlineWebscraper(ehealthlineURL, fileToWriteTo) } else { diff --git a/endpointmanager/pkg/chplendpointquerier/commomwebscrapers_test.go b/endpointmanager/pkg/chplendpointquerier/commomwebscrapers_test.go index cbab65ea7..7b26a832b 100644 --- a/endpointmanager/pkg/chplendpointquerier/commomwebscrapers_test.go +++ b/endpointmanager/pkg/chplendpointquerier/commomwebscrapers_test.go @@ -24,6 +24,16 @@ func TestWebScrapers(t *testing.T) { url: "https://fhirapi.asp.md:3030/aspmd/fhirserver/fhir_aspmd.asp", fileName: "ASPMD_Inc_EndpointSources.json", }, + { + scraperFunc: OntadaWebscraper, + url: "https://g2fhir-int.mckesson.com/docs/index.html", + fileName: "McKesson_Specialty_Health_Technology_Products_LLC_Ontada_EndpointSources.json", + }, + { + scraperFunc: MdlandWebscraper, + url: "https://api.mdland.com/Mdland%20SMART%20on%20FHIR%20OAuth%202.0%20Guide.htm", + fileName: "MDLAND_EndpointSources.json", + }, { scraperFunc: EzemrxWebscraper, url: "https://www.ezemrx.com/fhir", diff --git a/endpointmanager/pkg/chplendpointquerier/customcsvparser.go b/endpointmanager/pkg/chplendpointquerier/customcsvparser.go new file mode 100644 index 000000000..980aa8f8f --- /dev/null +++ b/endpointmanager/pkg/chplendpointquerier/customcsvparser.go @@ -0,0 +1,101 @@ +package chplendpointquerier + +import ( + "encoding/csv" + "io" + "os" + "strings" + + "github.com/onc-healthit/lantern-back-end/endpointmanager/pkg/helpers" + log "github.com/sirupsen/logrus" +) + +// CustomCSVParser reads a CSV from a URL or file and writes processed data to an output file. +// Parameters: +// - inputSource: URL or file path of the input CSV. +// - fileToWriteTo: File path to write the processed data. +// - csvFilePath: Temporary file path for storing the downloaded CSV (if applicable). +// - numrecords: Number of records to process (-1 for all records). +// - startrecord: Starting index of records to process. +// - header: Boolean indicating if the CSV has a header to skip. +// - urlIndex: Column index where the URL is located. +// - organizationIndex: Column index where the organization name is located. +func CustomCSVParser(inputSource string, fileToWriteTo string, csvFilePath string, numrecords int, startrecord int, header bool, urlIndex int, organizationIndex int) { + var lanternEntryList []LanternEntry + var endpointEntryList EndpointList + + var csvReader *csv.Reader + var file *os.File + var err error + if strings.HasPrefix(inputSource, "http://") || strings.HasPrefix(inputSource, "https://") { + csvReader, file, err = helpers.QueryAndOpenCSV(inputSource, csvFilePath, header) + if err != nil { + log.Fatal(err) + } + defer file.Close() + } else { + file, err = os.Open(inputSource) + if err != nil { + log.Fatal(err) + } + defer file.Close() + + csvReader = csv.NewReader(file) + if header { + _, err := csvReader.Read() + if err != nil { + log.Fatal(err) + } + } + } + + records := 0 + for { + rec, err := csvReader.Read() + if err == io.EOF { + break + } + if err != nil { + log.Fatal(err) + } + if numrecords >= 0 && records >= numrecords+startrecord { + break + } + if records >= startrecord { + var entry LanternEntry + + organizationName := "" + if organizationIndex >= 0 { + organizationName = strings.TrimSpace(rec[organizationIndex]) + } + + if !strings.Contains(strings.ToLower(organizationName), "auth") { + + URL := strings.TrimSpace(rec[urlIndex]) + URL = strings.Replace(URL, "/metadata", "", 1) + + entry.OrganizationName = organizationName + entry.URL = URL + + lanternEntryList = append(lanternEntryList, entry) + + } + } + + records++ + } + + endpointEntryList.Endpoints = lanternEntryList + + err = WriteCHPLFile(endpointEntryList, fileToWriteTo) + if err != nil { + log.Fatal(err) + } + + if _, err := os.Stat(csvFilePath); err == nil { + err = os.Remove(csvFilePath) + if err != nil { + log.Fatal(err) + } + } +} diff --git a/endpointmanager/pkg/chplendpointquerier/mdlandwebscraper.go b/endpointmanager/pkg/chplendpointquerier/mdlandwebscraper.go new file mode 100644 index 000000000..7d45b5586 --- /dev/null +++ b/endpointmanager/pkg/chplendpointquerier/mdlandwebscraper.go @@ -0,0 +1,41 @@ +package chplendpointquerier + +import ( + "strings" + + "github.com/PuerkitoBio/goquery" + "github.com/onc-healthit/lantern-back-end/endpointmanager/pkg/helpers" + log "github.com/sirupsen/logrus" +) + +func MdlandWebscraper(chplURL string, fileToWriteTo string) { + + var lanternEntryList []LanternEntry + var endpointEntryList EndpointList + + doc, err := helpers.ChromedpQueryEndpointList(chplURL, ".MsoNormal") + if err != nil { + log.Fatal(err) + } + + doc.Find("span").Each(func(index int, spanElem *goquery.Selection) { + if strings.Contains(spanElem.Text(), "https://") && strings.Contains(spanElem.Text(), "metadata") { + str := spanElem.Text() + str = strings.Replace(str, "GET ", "", -1) + str = strings.Replace(str, "/metadata", "", -1) + str = strings.Replace(str, "\nHTTP/1.1", "", -1) + + var entry LanternEntry + entry.URL = str + lanternEntryList = append(lanternEntryList, entry) + } + }) + + endpointEntryList.Endpoints = lanternEntryList + + err = WriteCHPLFile(endpointEntryList, fileToWriteTo) + if err != nil { + log.Fatal(err) + } + +} diff --git a/endpointmanager/pkg/chplendpointquerier/netsmartparser.go b/endpointmanager/pkg/chplendpointquerier/netsmartparser.go index 1d8b45f46..07fcfba75 100644 --- a/endpointmanager/pkg/chplendpointquerier/netsmartparser.go +++ b/endpointmanager/pkg/chplendpointquerier/netsmartparser.go @@ -1,10 +1,11 @@ package chplendpointquerier import ( - "github.com/onc-healthit/lantern-back-end/endpointmanager/pkg/helpers" "io" "strings" + "github.com/onc-healthit/lantern-back-end/endpointmanager/pkg/helpers" + "os" log "github.com/sirupsen/logrus" @@ -14,7 +15,7 @@ func NetsmartCSVParser(CHPLURL string, fileToWriteTo string) { var lanternEntryList []LanternEntry var endpointEntryList EndpointList - csvFilePath := "./netsmart-fhir-base-urls.csv" + csvFilePath := "./service-base-urls-20240905.csv" csvReader, file, err := helpers.QueryAndOpenCSV(CHPLURL, csvFilePath, true) if err != nil { @@ -33,9 +34,11 @@ func NetsmartCSVParser(CHPLURL string, fileToWriteTo string) { var entry LanternEntry orgName := strings.TrimSpace(rec[0]) - URL := strings.TrimSpace(rec[1]) + zipcode := strings.TrimSpace(rec[4]) + URL := strings.TrimSpace(rec[5]) entry.OrganizationName = orgName + entry.OrganizationZipCode = zipcode entry.URL = URL lanternEntryList = append(lanternEntryList, entry) } diff --git a/endpointmanager/pkg/chplendpointquerier/ontadawebscraper.go b/endpointmanager/pkg/chplendpointquerier/ontadawebscraper.go new file mode 100644 index 000000000..c077b19d1 --- /dev/null +++ b/endpointmanager/pkg/chplendpointquerier/ontadawebscraper.go @@ -0,0 +1,36 @@ +package chplendpointquerier + +import ( + "strings" + + "github.com/onc-healthit/lantern-back-end/endpointmanager/pkg/helpers" + log "github.com/sirupsen/logrus" +) + +func OntadaWebscraper(chplURL string, fileToWriteTo string) { + + var lanternEntryList []LanternEntry + var endpointEntryList EndpointList + var entry LanternEntry + + doc, err := helpers.ChromedpQueryEndpointList(chplURL, ".sc-dTSzeu.dfUAUz") + if err != nil { + log.Fatal(err) + } + + divElem := doc.Find(".sc-dTSzeu.dfUAUz").First() + spanElem := divElem.Find("span").First() + + entryURL := strings.TrimSpace(spanElem.Text()) + entry.URL = entryURL + + lanternEntryList = append(lanternEntryList, entry) + + endpointEntryList.Endpoints = lanternEntryList + + err = WriteCHPLFile(endpointEntryList, fileToWriteTo) + if err != nil { + log.Fatal(err) + } + +}