Skip to content

Commit

Permalink
Merge branch 'main' into LANTERN-788-Add-json-parser-emds
Browse files Browse the repository at this point in the history
  • Loading branch information
archita-ekkirala authored Nov 19, 2024
2 parents 1d064c0 + 08c14c3 commit ffdbce2
Show file tree
Hide file tree
Showing 7 changed files with 248 additions and 5 deletions.
34 changes: 34 additions & 0 deletions endpointmanager/pkg/chplendpointquerier/betaAfoundriaWebScraper.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
package chplendpointquerier

import (
"github.com/PuerkitoBio/goquery"
"github.com/onc-healthit/lantern-back-end/endpointmanager/pkg/helpers"
log "github.com/sirupsen/logrus"
)

func BetaAfoundriaWebScraper(CHPLURL string, fileToWriteTo string) {

var lanternEntryList []LanternEntry
var endpointEntryList EndpointList

doc, err := helpers.ChromedpQueryEndpointList(CHPLURL, ".container")
if err != nil {
log.Fatal(err)
}
doc.Find(".container h3").Each(func(index int, header *goquery.Selection) {
nextAnchor := header.NextFiltered("a")
href, exists := nextAnchor.Attr("href")
if exists {
var entry LanternEntry
entry.URL = href
lanternEntryList = append(lanternEntryList, entry)
}
})

endpointEntryList.Endpoints = lanternEntryList
err = WriteCHPLFile(endpointEntryList, fileToWriteTo)
if err != nil {
log.Fatal(err)
}

}
22 changes: 20 additions & 2 deletions endpointmanager/pkg/chplendpointquerier/chplendpointquerier.go
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,7 @@ var smartemrURL = "https://smartemr.readme.io/reference/getting-started#base-url
var tebraURL = "https://fhir.prd.cloud.tebra.com/fhir-request/swagger-ui/"
var landmarkhealthURL = "https://lmdmzprodws.landmarkhealth.org/docs/fhir-base-urls.csv"
var nthtechnologyURL = "https://admin.nthtechnology.com/fhir_endpoints.php/json"
var netsmartURL = "https://careconnect-uat.netsmartcloud.com/"
var netsmartURL = "https://careconnect.netsmartcloud.com/csv/service-base-urls-20240905.csv"

var omnimdURL = "https://fhirregistration.omnimd.com/#/specification"
var pcesystemsURL = "https://www.pcesystems.com/g10APIInfo.html"
Expand Down Expand Up @@ -194,6 +194,7 @@ var nextgenPracticeURL = "https://www.nextgen.com/api/practice-search"
var aspmdURL = "https://fhirapi.asp.md:3030/aspmd/fhirserver/fhir_aspmd.asp"
var axeiumURL = "https://apifhir.axeium.net:8443/reference-server/"
var emdscloudURL = "https://identity.emdscloud.com/api/api-resource/fhir"
var betaAfoundriaURL = "https://beta.afoundria.com/api/fhir/urls"
var ehealthlineURL = "http://ehealthline.com/dev/pdf/FHIR%20API%20Endpoints.htm"

var bundleQuerierArray = [30]string{"https://ac-fhir.harrisambulatory.com/endpoints/r4", "https://dynamicfhirpresentation.dynamicfhirsandbox.com/fhir/r4/endpoints",
Expand All @@ -205,6 +206,11 @@ var bundleQuerierArray = [30]string{"https://ac-fhir.harrisambulatory.com/endpoi
"https://appstudio.interopengine.com/partner/fhirR4endpoints-umc.json", "https://testauth.strateqhealth.com/SmartOnFHIR/ValidURLs.json",
"https://fhir.ethizo.com/api/4.0.0/service_based_url"}

var ontadaURL = "https://g2fhir-int.mckesson.com/docs/index.html"
var mdlandURL = "https://api-fhir-proxy-2.mdland.net/"
var abeoURL = "https://www.crystalpm.com/FHIRServiceURLs.csv"
var nextechURL2 = "https://www.nextech.com/developers-portal"
var icareURL = "https://www.icare.com/endpoints.csv"
var ezemrxURL = "https://www.ezemrx.com/fhir"

func contains(arr [30]string, str string) bool {
Expand Down Expand Up @@ -402,7 +408,7 @@ func QueryCHPLEndpointList(chplURL string, fileToWriteTo string) {
} else if URLsEqual(chplURL, nthtechnologyURL) {
BundleQuerierParser(chplURL, fileToWriteTo)
} else if URLsEqual(chplURL, netsmartURL) {
NetsmartCSVParser("https://careconnect-uat.netsmartcloud.com/baseUrls", fileToWriteTo)
NetsmartCSVParser(chplURL, fileToWriteTo)
} else if URLsEqual(chplURL, omnimdURL) {
OmniMDWebscraper(chplURL, fileToWriteTo)
} else if URLsEqual(chplURL, pcesystemsURL) {
Expand Down Expand Up @@ -563,6 +569,18 @@ func QueryCHPLEndpointList(chplURL string, fileToWriteTo string) {
BundleQuerierParser(chplURL, fileToWriteTo)
} else if URLsEqual(emdscloudURL, chplURL) {
EmdsCloudWebscraper(emdscloudURL, fileToWriteTo)
} else if URLsEqual(chplURL, betaAfoundriaURL) {
BetaAfoundriaWebScraper(chplURL, fileToWriteTo)
} else if URLsEqual(chplURL, ontadaURL) {
OntadaWebscraper(chplURL, fileToWriteTo)
} else if URLsEqual(chplURL, mdlandURL) {
MdlandWebscraper("https://api.mdland.com/Mdland%20SMART%20on%20FHIR%20OAuth%202.0%20Guide.htm", fileToWriteTo)
} else if URLsEqual(abeoURL, chplURL) {
CustomCSVParser(chplURL, fileToWriteTo, "./FHIRServiceURLs.csv", -1, 0, true, 1, 0)
} else if URLsEqual(nextechURL2, chplURL) {
CSVParser("https://www.nextech.com/hubfs/Nextech%20FHIR%20Base%20URL.csv", fileToWriteTo, "./Nextech FHIR Base URL.csv", -1, 0, true, 1, 0)
} else if URLsEqual(icareURL, chplURL) {
CSVParser(chplURL, fileToWriteTo, "./endpoints.csv", -1, 0, true, 1, 0)
} else if URLsEqual(chplURL, ehealthlineURL) {
EhealthlineWebscraper(ehealthlineURL, fileToWriteTo)
} else {
Expand Down
10 changes: 10 additions & 0 deletions endpointmanager/pkg/chplendpointquerier/commomwebscrapers_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,16 @@ func TestWebScrapers(t *testing.T) {
url: "https://fhirapi.asp.md:3030/aspmd/fhirserver/fhir_aspmd.asp",
fileName: "ASPMD_Inc_EndpointSources.json",
},
{
scraperFunc: OntadaWebscraper,
url: "https://g2fhir-int.mckesson.com/docs/index.html",
fileName: "McKesson_Specialty_Health_Technology_Products_LLC_Ontada_EndpointSources.json",
},
{
scraperFunc: MdlandWebscraper,
url: "https://api.mdland.com/Mdland%20SMART%20on%20FHIR%20OAuth%202.0%20Guide.htm",
fileName: "MDLAND_EndpointSources.json",
},
{
scraperFunc: EzemrxWebscraper,
url: "https://www.ezemrx.com/fhir",
Expand Down
101 changes: 101 additions & 0 deletions endpointmanager/pkg/chplendpointquerier/customcsvparser.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,101 @@
package chplendpointquerier

import (
"encoding/csv"
"io"
"os"
"strings"

"github.com/onc-healthit/lantern-back-end/endpointmanager/pkg/helpers"
log "github.com/sirupsen/logrus"
)

// CustomCSVParser reads a CSV from a URL or file and writes processed data to an output file.
// Parameters:
// - inputSource: URL or file path of the input CSV.
// - fileToWriteTo: File path to write the processed data.
// - csvFilePath: Temporary file path for storing the downloaded CSV (if applicable).
// - numrecords: Number of records to process (-1 for all records).
// - startrecord: Starting index of records to process.
// - header: Boolean indicating if the CSV has a header to skip.
// - urlIndex: Column index where the URL is located.
// - organizationIndex: Column index where the organization name is located.
func CustomCSVParser(inputSource string, fileToWriteTo string, csvFilePath string, numrecords int, startrecord int, header bool, urlIndex int, organizationIndex int) {
var lanternEntryList []LanternEntry
var endpointEntryList EndpointList

var csvReader *csv.Reader
var file *os.File
var err error
if strings.HasPrefix(inputSource, "http://") || strings.HasPrefix(inputSource, "https://") {
csvReader, file, err = helpers.QueryAndOpenCSV(inputSource, csvFilePath, header)
if err != nil {
log.Fatal(err)
}
defer file.Close()
} else {
file, err = os.Open(inputSource)
if err != nil {
log.Fatal(err)
}
defer file.Close()

csvReader = csv.NewReader(file)
if header {
_, err := csvReader.Read()
if err != nil {
log.Fatal(err)
}
}
}

records := 0
for {
rec, err := csvReader.Read()
if err == io.EOF {
break
}
if err != nil {
log.Fatal(err)
}
if numrecords >= 0 && records >= numrecords+startrecord {
break
}
if records >= startrecord {
var entry LanternEntry

organizationName := ""
if organizationIndex >= 0 {
organizationName = strings.TrimSpace(rec[organizationIndex])
}

if !strings.Contains(strings.ToLower(organizationName), "auth") {

URL := strings.TrimSpace(rec[urlIndex])
URL = strings.Replace(URL, "/metadata", "", 1)

entry.OrganizationName = organizationName
entry.URL = URL

lanternEntryList = append(lanternEntryList, entry)

}
}

records++
}

endpointEntryList.Endpoints = lanternEntryList

err = WriteCHPLFile(endpointEntryList, fileToWriteTo)
if err != nil {
log.Fatal(err)
}

if _, err := os.Stat(csvFilePath); err == nil {
err = os.Remove(csvFilePath)
if err != nil {
log.Fatal(err)
}
}
}
41 changes: 41 additions & 0 deletions endpointmanager/pkg/chplendpointquerier/mdlandwebscraper.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
package chplendpointquerier

import (
"strings"

"github.com/PuerkitoBio/goquery"
"github.com/onc-healthit/lantern-back-end/endpointmanager/pkg/helpers"
log "github.com/sirupsen/logrus"
)

func MdlandWebscraper(chplURL string, fileToWriteTo string) {

var lanternEntryList []LanternEntry
var endpointEntryList EndpointList

doc, err := helpers.ChromedpQueryEndpointList(chplURL, ".MsoNormal")
if err != nil {
log.Fatal(err)
}

doc.Find("span").Each(func(index int, spanElem *goquery.Selection) {
if strings.Contains(spanElem.Text(), "https://") && strings.Contains(spanElem.Text(), "metadata") {
str := spanElem.Text()
str = strings.Replace(str, "GET ", "", -1)
str = strings.Replace(str, "/metadata", "", -1)
str = strings.Replace(str, "\nHTTP/1.1", "", -1)

var entry LanternEntry
entry.URL = str
lanternEntryList = append(lanternEntryList, entry)
}
})

endpointEntryList.Endpoints = lanternEntryList

err = WriteCHPLFile(endpointEntryList, fileToWriteTo)
if err != nil {
log.Fatal(err)
}

}
9 changes: 6 additions & 3 deletions endpointmanager/pkg/chplendpointquerier/netsmartparser.go
Original file line number Diff line number Diff line change
@@ -1,10 +1,11 @@
package chplendpointquerier

import (
"github.com/onc-healthit/lantern-back-end/endpointmanager/pkg/helpers"
"io"
"strings"

"github.com/onc-healthit/lantern-back-end/endpointmanager/pkg/helpers"

"os"

log "github.com/sirupsen/logrus"
Expand All @@ -14,7 +15,7 @@ func NetsmartCSVParser(CHPLURL string, fileToWriteTo string) {
var lanternEntryList []LanternEntry
var endpointEntryList EndpointList

csvFilePath := "./netsmart-fhir-base-urls.csv"
csvFilePath := "./service-base-urls-20240905.csv"

csvReader, file, err := helpers.QueryAndOpenCSV(CHPLURL, csvFilePath, true)
if err != nil {
Expand All @@ -33,9 +34,11 @@ func NetsmartCSVParser(CHPLURL string, fileToWriteTo string) {

var entry LanternEntry
orgName := strings.TrimSpace(rec[0])
URL := strings.TrimSpace(rec[1])
zipcode := strings.TrimSpace(rec[4])
URL := strings.TrimSpace(rec[5])

entry.OrganizationName = orgName
entry.OrganizationZipCode = zipcode
entry.URL = URL
lanternEntryList = append(lanternEntryList, entry)
}
Expand Down
36 changes: 36 additions & 0 deletions endpointmanager/pkg/chplendpointquerier/ontadawebscraper.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
package chplendpointquerier

import (
"strings"

"github.com/onc-healthit/lantern-back-end/endpointmanager/pkg/helpers"
log "github.com/sirupsen/logrus"
)

func OntadaWebscraper(chplURL string, fileToWriteTo string) {

var lanternEntryList []LanternEntry
var endpointEntryList EndpointList
var entry LanternEntry

doc, err := helpers.ChromedpQueryEndpointList(chplURL, ".sc-dTSzeu.dfUAUz")
if err != nil {
log.Fatal(err)
}

divElem := doc.Find(".sc-dTSzeu.dfUAUz").First()
spanElem := divElem.Find("span").First()

entryURL := strings.TrimSpace(spanElem.Text())
entry.URL = entryURL

lanternEntryList = append(lanternEntryList, entry)

endpointEntryList.Endpoints = lanternEntryList

err = WriteCHPLFile(endpointEntryList, fileToWriteTo)
if err != nil {
log.Fatal(err)
}

}

0 comments on commit ffdbce2

Please sign in to comment.