Add new endpoint list and fixes

api/download/downloadsmodule.R

@@ -53,5 +53,5 @@ download_data <- function(db_tables) {
     rename(api_information_source_name = endpoint_names, certified_api_developer_name = vendor_name) %>%
     rename(created_at = info_created, updated = info_updated) %>%
     rename(http_response_time_second = response_time_seconds)
-  write.csv(csvdata, file, row.names = FALSE)
+  #write.csv(csvdata, file, row.names = FALSE)
 }

capabilityreceiver/pkg/capabilityhandler/supportedProfiles.go

@@ -1,6 +1,7 @@
 package capabilityhandler
 
 import (
+	"fmt"
 	"github.com/onc-healthit/lantern-back-end/endpointmanager/pkg/endpointmanager"
 	"github.com/onc-healthit/lantern-back-end/endpointmanager/pkg/helpers"
 )
@@ -72,16 +73,27 @@ func getCapabilityStatementProfiles(capInt map[string]interface{}, supportedProf
 		}
 
 		if resourceInt["supportedProfile"] != nil {
-			supportedProfileArr := resourceInt["supportedProfile"].([]interface{})
-			for _, profileEntry := range supportedProfileArr {
-				profileURL := profileEntry.(string)
-				var profileInfo endpointmanager.SupportedProfile
-				profileInfo.ProfileURL = profileURL
-				profileInfo.Resource = resourceType
-
-				supportedProfiles = append(supportedProfiles, profileInfo)
+			supportedProfileArr, ok := resourceInt["supportedProfile"].([]interface{})
+			if !ok {
+				// Handle the case where "supportedProfile" is not a []interface{}
+				fmt.Println("Error: 'supportedProfile' is not a []interface{}")
+				// Add appropriate error handling or return from the function
 			}
 
+			for _, profileEntry := range supportedProfileArr {
+				// Check if profileEntry is a string
+				if profileURL, ok := profileEntry.(string); ok {
+					var profileInfo endpointmanager.SupportedProfile
+					profileInfo.ProfileURL = profileURL
+					profileInfo.Resource = resourceType
+
+					supportedProfiles = append(supportedProfiles, profileInfo)
+				} else {
+					// Handle the case where profileEntry is not a string
+					fmt.Println("Error: 'supportedProfile' entry is not a string")
+					// Add appropriate error handling or return from the function
+				}
+			}
 		}
 	}
 

endpointmanager/cmd/CHPLpopulator/main.go

@@ -112,7 +112,8 @@ func main() {
 		var chplJSON CHPLEndpointList
 		err = json.Unmarshal(respBody, &chplJSON)
 		if err != nil {
-			log.Fatal(err)
+			//log.Fatal(err)
+			continue
 		}
 
 		if savedEntries >= chplJSON.RecordCount {

endpointmanager/pkg/chplendpointquerier/NextGenwebscraper.go

@@ -12,30 +12,23 @@ func NextGenwebscraper(CHPLURL string, fileToWriteTo string) {
 
 	var lanternEntryList []LanternEntry
 	var endpointEntryList EndpointList
-
-	doc, err := helpers.ChromedpQueryEndpointList(CHPLURL, ".api-search-result")
+	doc, err := helpers.ChromedpQueryEndpointList(CHPLURL, "#api-search-results")
 	if err != nil {
 		log.Fatal(err)
 	}
 
-	doc.Find("table").Each(func(index int, tablehtml *goquery.Selection) {
-		tablehtml.Find("tbody").Each(func(indextr int, rowhtml *goquery.Selection) {
-			rowhtml.Find("tr").Each(func(indextr int, rowbodyhtml *goquery.Selection) {
+	doc.Find("#api-search-results").Each(func(index int, div1html *goquery.Selection) {
+		div2html := div1html.Find("div").First()
+		ulhtml := div2html.Find("ul").First()
+		ulhtml.Find("li").Each(func(indextr int, lihtml *goquery.Selection) {
+			if strings.Contains(lihtml.Text(), "DSTU2") || strings.Contains(lihtml.Text(), "FHIR R4") {
+				var litext = lihtml.Text()
+				var URL = litext[strings.Index(litext, " https")+1 : len(litext)-1]
 				var entry LanternEntry
+				entry.URL = strings.TrimSpace(URL)
 
-				tableEntries := rowbodyhtml.Find("td")
-				if tableEntries.Length() > 0 {
-					organizationName := strings.TrimSpace(tableEntries.Eq(1).Text())
-					zipCode := strings.TrimSpace(tableEntries.Eq(5).Text())
-					URL := strings.TrimSpace(tableEntries.Eq(6).Text())
-
-					entry.OrganizationName = organizationName
-					entry.URL = URL
-					entry.OrganizationZipCode = zipCode
-
-					lanternEntryList = append(lanternEntryList, entry)
-				}
-			})
+				lanternEntryList = append(lanternEntryList, entry)
+			}
 		})
 	})
 

endpointmanager/pkg/chplendpointquerier/allegiancemdwebscraper.go

@@ -1,11 +1,10 @@
 package chplendpointquerier
 
 import (
-	"strings"
-
 	"github.com/PuerkitoBio/goquery"
 	"github.com/onc-healthit/lantern-back-end/endpointmanager/pkg/helpers"
 	log "github.com/sirupsen/logrus"
+	"strings"
 )
 
 func AllegianceMDWebscraper(chplURL string, fileToWriteTo string) {
@@ -18,7 +17,6 @@ func AllegianceMDWebscraper(chplURL string, fileToWriteTo string) {
 	if err != nil {
 		log.Fatal(err)
 	}
-
 	doc.Each(func(index int, pElem *goquery.Selection) {
 		aElem := pElem.Find("a").First()
 		hrefText, exists := aElem.Attr("href")

endpointmanager/pkg/chplendpointquerier/bridgepatientportalwebscraper.go

@@ -1,11 +1,11 @@
 package chplendpointquerier
 
 import (
-	"strings"
-
+	"fmt"
 	"github.com/PuerkitoBio/goquery"
 	"github.com/onc-healthit/lantern-back-end/endpointmanager/pkg/helpers"
 	log "github.com/sirupsen/logrus"
+	"strings"
 )
 
 func BridgePatientPortalWebscraper(chplURL string, fileToWriteTo string) {
@@ -17,7 +17,7 @@ func BridgePatientPortalWebscraper(chplURL string, fileToWriteTo string) {
 	if err != nil {
 		log.Fatal(err)
 	}
-
+	fmt.Println(doc.Html())
 	fhirEndpointsHeaderElem := doc.Find("#introduction/fhir-bridge-patient-portal/fhir-endpoints")
 	if fhirEndpointsHeaderElem.Length() > 0 {
 		spanElem := fhirEndpointsHeaderElem.Eq(0).Next()

endpointmanager/pkg/chplendpointquerier/broadstreetwebscraper.go

@@ -0,0 +1,44 @@
+package chplendpointquerier
+
+import (
+	"github.com/PuerkitoBio/goquery"
+	"github.com/onc-healthit/lantern-back-end/endpointmanager/pkg/helpers"
+	log "github.com/sirupsen/logrus"
+	"strings"
+)
+
+func BroadStreetURLWebscraper(CHPLURL string, fileToWriteTo string) {
+
+	var lanternEntryList []LanternEntry
+	var endpointEntryList EndpointList
+
+	doc, err := helpers.ChromedpQueryEndpointList(CHPLURL, "article")
+	if err != nil {
+		log.Fatal(err)
+	}
+	doc.Find("article").Each(func(index int, articleElems *goquery.Selection) {
+		articleElems.Find("table").Each(func(index int, tableElems *goquery.Selection) {
+			tableElems.Find("tbody").Each(func(index int, bodyElems *goquery.Selection) {
+				bodyElems.Find("tr").Each(func(index int, trElems *goquery.Selection) {
+					tdEntry := trElems.Find("td").First()
+					if strings.Contains(tdEntry.Text(), "FHIR Production") {
+						tdEntryNext := trElems.Find("td").Next()
+						if tdEntryNext.Length() > 0 {
+							var entry LanternEntry
+							entry.URL = strings.TrimSpace(tdEntryNext.Text())
+							lanternEntryList = append(lanternEntryList, entry)
+						}
+					}
+				})
+			})
+		})
+	})
+
+	endpointEntryList.Endpoints = lanternEntryList
+
+	err = WriteCHPLFile(endpointEntryList, fileToWriteTo)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+}

endpointmanager/pkg/chplendpointquerier/bundleparser.go

@@ -68,6 +68,9 @@ func BundleToLanternFormat(bundle []byte) []LanternEntry {
 		var entry LanternEntry
 
 		if strings.EqualFold(strings.TrimSpace(bundleEntry.Resource.ResourceType), "Endpoint") {
+			if bundleEntry.Resource.Address == nil {
+				continue
+			}
 			entryURL := bundleEntry.Resource.Address.(string)
 			// Do not add entries that do not have URLs
 			if entryURL != "" {

endpointmanager/pkg/chplendpointquerier/chplendpointquerier.go

@@ -18,7 +18,7 @@ type LanternEntry struct {
 }
 
 var MedHostURL = "https://api.mhdi10xasayd.com/medhost-developer-composition/v1/fhir-base-urls.json"
-var NextGenURL = "https://nextgen.com/api/practice-search/"
+var NextGenURL = "https://nextgen.com/api/practice-search"
 var CanvasURL = "https://docs.canvasmedical.com/reference/service-base-urls"
 var AlteraURL = "https://open.allscripts.com/fhirendpoints"
 var EpicURL = "https://open.epic.com/MyApps/Endpoints"
@@ -43,7 +43,6 @@ var healthCare2000URL = "https://www.provider.care/FHIR/MDVitaFHIRUrls.csv"
 var firstInsightURL = "https://www.first-insight.com/maximeyes_fhir_base_url_endpoints/"
 var healthSamuraiURL = "https://cmpl.aidbox.app/smart"
 var triarqURL = "https://fhir.myqone.com/Endpoints"
-var goldblattURL = "https://www.goldblattsystems.com/apis"
 var cyfluentURL = "https://app.swaggerhub.com/apis-docs/Cyfluent/ProviderPortalApi/3.3#/FHIR/fhir"
 var meridianURL = "https://api-datamanager.carecloud.com:8081/fhirurl"
 var qualifactsInsyncURL = "https://qualifacts.com/api-page/platform/insync/insync-fhir-org-list.html"
@@ -64,15 +63,13 @@ var varianmedicalURL = "https://variandev.dynamicfhir.com/"
 var caretrackerURL = "https://hag-fhir.amazingcharts.com/ac/endpoints"
 var zhhealthcareURL = "https://blueehr.com/fhir-urls/"
 var emedpracticeURL = "https://emedpractice.com/Fhir/FhirHelpDocument.html"
-var modernizingmedicineURL = "https://mm-fhir-endpoint-display.qa.fhir.ema-api.com/"
 var doc_torURL = "https://hag-fhir.amazingcharts.com/pc/endpoints"
 var azaleahealthURL = "https://api.azaleahealth.com/fhir/R4/Endpoint"
 var cloudcraftURL = "https://fhirapitest.naiacorp.net/fhir/r4/endpoints/"
 var darenasolutionsURL = "https://api.meldrx.com/Directories/fhir/endpoints"
 var glenwoodsystemsURL = "https://static.glaceemr.com/endpoints/urls.json"
 var practicefusionURL = "https://www.practicefusion.com/assets/static_files/ServiceBaseURLs.json"
 var universalEHRURL = "https://appstudio.interopengine.com/partner/fhirR4endpoints-universalehr.json"
-var welligentURL = "https://mu3test.welligent.com/fhir/r4/endpoints/"
 var astronautURL = "https://astronautehr.com/index.php/fhir-base-urls/"
 var bestpracticesacademyURL = "https://ipatientcare.com/onc-acb-certified-2015-edition"
 var californiamedicalsystemsURL = "https://cal-med.com/fhir/Fhir-base-urls.csv"
@@ -138,6 +135,11 @@ var CarepathsURL = "https://carepaths.com/uploads/org_endpoint_bundle.json"
 var athenaClinicalsURL = "https://docs.athenahealth.com/api/guides/base-fhir-urls"
 var canvasMedicalURL = "https://docs.canvasmedical.com/api/service-base-urls/"
 var veradigmURL = "https://open.platform.veradigm.com/fhirendpoints"
+var broadStreetURL = "https://broadstreetcare.com/docs"
+var officePracticumURL = "https://fhir-documentation.patientmedrecords.com/endpoints"
+var modernizingMedicineURL = "https://mm-fhir-endpoint-display.prod.fhir.ema-api.com/"
+var welligentURL = "https://fhir.qa.welligent.com/"
+var willowURL = "https://www.willowgladetechnologies.com/requirements"
 
 func QueryCHPLEndpointList(chplURL string, fileToWriteTo string) {
 
@@ -149,6 +151,8 @@ func QueryCHPLEndpointList(chplURL string, fileToWriteTo string) {
 		Canvaswebscraper(chplURL, fileToWriteTo)
 	} else if URLsEqual(chplURL, AlteraURL) {
 		AlteraQuerier(chplURL, fileToWriteTo)
+	} else if URLsEqual(chplURL, modernizingMedicineURL) {
+		ModernizingMedicineQuerier(chplURL, fileToWriteTo)
 	} else if URLsEqual(chplURL, EpicURL) {
 		EpicQuerier(chplURL, fileToWriteTo)
 	} else if URLsEqual(chplURL, MeditechURL) {
@@ -192,11 +196,9 @@ func QueryCHPLEndpointList(chplURL string, fileToWriteTo string) {
 	} else if URLsEqual(chplURL, firstInsightURL) {
 		FirstInsightBundleParser(chplURL, fileToWriteTo)
 	} else if URLsEqual(chplURL, healthSamuraiURL) {
-		HealthSamuraiWebscraper(chplURL, fileToWriteTo)
+		HealthSamuraiWebscraper("https://smartbox.aidbox.app/service-base-urls", fileToWriteTo)
 	} else if URLsEqual(chplURL, triarqURL) {
 		TRIARQPracticeWebscraper(chplURL, fileToWriteTo)
-	} else if URLsEqual(chplURL, goldblattURL) {
-		BundleQuerierParser("https://fhir-test.csn.health/gs-fhir-domain-server/public-base-service-endpoints.json", fileToWriteTo)
 	} else if URLsEqual(chplURL, cyfluentURL) {
 		SwaggerUIWebscraper(chplURL, fileToWriteTo)
 	} else if URLsEqual(chplURL, meridianURL) {
@@ -239,8 +241,6 @@ func QueryCHPLEndpointList(chplURL string, fileToWriteTo string) {
 		MedicalInformaticsEngineeringWebscraper(chplURL, fileToWriteTo)
 	} else if chplURL == emedpracticeURL {
 		eMedPracticeWebscraper("https://servicebackup.emedpractice.com:8443/helpdoc/fhir_helpdoc.html", fileToWriteTo)
-	} else if chplURL == modernizingmedicineURL {
-		ModernizingMedicineQuerier("qa.fhir.ema-api.com/fhir/r4/Endpoint?connection-type=hl7-fhir-rest", fileToWriteTo)
 	} else if chplURL == doc_torURL {
 		BundleQuerierParser(chplURL+"/r4", fileToWriteTo)
 	} else if URLsEqual(chplURL, azaleahealthURL) {
@@ -256,7 +256,7 @@ func QueryCHPLEndpointList(chplURL string, fileToWriteTo string) {
 	} else if URLsEqual(chplURL, universalEHRURL) {
 		BundleQuerierParser(chplURL, fileToWriteTo)
 	} else if URLsEqual(chplURL, welligentURL) {
-		BundleQuerierParser(chplURL, fileToWriteTo)
+		BundleQuerierParser("https://fhir.qa.welligent.com/fhir/r4/endpoints", fileToWriteTo)
 	} else if URLsEqual(chplURL, astronautURL) {
 		CSVParser("https://astronautehr.com/wp-content/uploads/2022/12/Astronaut-fhir-base-urls.csv", fileToWriteTo, "./astronaut_fhir_base_urls.csv", 1, 2, true, 1, -1)
 	} else if URLsEqual(chplURL, bestpracticesacademyURL) {
@@ -385,6 +385,12 @@ func QueryCHPLEndpointList(chplURL string, fileToWriteTo string) {
 		CSVParser(MEDENTURL, fileToWriteTo, "./ServiceBaseURL.csv", 1, 0, true, 1, 0)
 	} else if URLsEqual(chplURL, canvasMedicalURL) {
 		CanvasMedicalURLWebscraper(chplURL, fileToWriteTo)
+	} else if URLsEqual(chplURL, broadStreetURL) {
+		BroadStreetURLWebscraper(chplURL, fileToWriteTo)
+	} else if URLsEqual(chplURL, officePracticumURL) {
+		OfficePracticumURLWebscraper(chplURL, fileToWriteTo)
+	} else if URLsEqual(chplURL, willowURL) {
+		WillowQuerierParser("https://ccdoc.phn.care/service-base-urls", fileToWriteTo)
 	} else if URLsEqual(chplURL, veradigmURL) {
 		BundleQuerierParser("https://open.platform.veradigm.com/fhirendpoints/download/R4", fileToWriteTo)
 	}

endpointmanager/pkg/chplendpointquerier/modernizingmedicineparser.go

@@ -6,29 +6,46 @@ import (
 )
 
 func ModernizingMedicineQuerier(chplURL string, fileToWriteTo string) {
-
-	emaURL := "https://fhir.m2qa." + chplURL
-	gastroURL := "https://fhir.gastro." + chplURL
+	emaURL := "https://fhir.mmi.prod.fhir.ema-api.com/fhir/r4/Endpoint?connection-type=hl7-fhir-rest"
+	gastroURL := "https://fhir.gastro.prod.fhir.ema-api.com/fhir/r4/Endpoint?connection-type=hl7-fhir-rest"
+	exscribeURL := "https://ehrapi-exscribe-prod-fhir.ema-api.com/api/Endpoint"
+	//traknetURL := "https://ehrapi-traknet-prod-fhir.ema-api.com/api/Endpoint"
+	//sammyURL := "https://ehrapi-sammyehr-prod-fhir.ema-api.com/api/Endpoint"
 
 	var endpointEntryList EndpointList
-
 	respBody, err := helpers.QueryEndpointList(emaURL)
 	if err != nil {
 		log.Fatal(err)
 	}
-
 	endpointEntryList.Endpoints = BundleToLanternFormat(respBody)
 
 	respBody, err = helpers.QueryEndpointList(gastroURL)
 	if err != nil {
 		log.Fatal(err)
 	}
+	endpointEntryList.Endpoints = append(endpointEntryList.Endpoints, BundleToLanternFormat(respBody)...)
 
+	respBody, err = helpers.QueryEndpointList(exscribeURL)
+	if err != nil {
+		log.Fatal(err)
+	}
 	endpointEntryList.Endpoints = append(endpointEntryList.Endpoints, BundleToLanternFormat(respBody)...)
 
+	//respBody, err = helpers.QueryEndpointList(traknetURL)
+	//if err != nil {
+	//	log.Fatal(err)
+	//}
+	//endpointEntryList.Endpoints = append(endpointEntryList.Endpoints, BundleToLanternFormat(respBody)...)
+
+	// respBody, err = helpers.QueryEndpointList(sammyURL)
+	// if err != nil {
+	// 	log.Fatal(err)
+	// }
+
+	// endpointEntryList.Endpoints = append(endpointEntryList.Endpoints, BundleToLanternFormat(respBody)...)
+
 	err = WriteCHPLFile(endpointEntryList, fileToWriteTo)
 	if err != nil {
 		log.Fatal(err)
 	}
-
 }

endpointmanager/pkg/chplendpointquerier/officepracticumwebscraper.go

@@ -0,0 +1,38 @@
+package chplendpointquerier
+
+import (
+	"github.com/PuerkitoBio/goquery"
+	"github.com/onc-healthit/lantern-back-end/endpointmanager/pkg/helpers"
+	log "github.com/sirupsen/logrus"
+	"strings"
+)
+
+func OfficePracticumURLWebscraper(CHPLURL string, fileToWriteTo string) {
+
+	var lanternEntryList []LanternEntry
+	var endpointEntryList EndpointList
+
+	doc, err := helpers.ChromedpQueryEndpointList(CHPLURL, ".MuiTableCell-root.MuiTableCell-body.MuiTableCell-sizeMedium.css-q34dxg")
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	doc.Find("tbody.MuiTableBody-root.css-1xnox0e").Each(func(i int, bodyEle *goquery.Selection) {
+		bodyEle.Find("tr").Each(func(index int, trEle *goquery.Selection) {
+			organizationName := trEle.Find("td").First().Text()
+			URL := trEle.Find("td").Next().Text()
+			var entry LanternEntry
+			entry.OrganizationName = strings.TrimSpace(organizationName)
+			entry.URL = strings.TrimSpace(URL)
+			lanternEntryList = append(lanternEntryList, entry)
+		})
+	})
+
+	endpointEntryList.Endpoints = lanternEntryList
+
+	err = WriteCHPLFile(endpointEntryList, fileToWriteTo)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+}