Merge pull request #440 from LibreQoE/main

Update Develop

.github/workflows/lint_python.yml

@@ -24,7 +24,7 @@ jobs:
       - run: pytest . || true
       - run: pytest --doctest-modules . || true
       - run: shopt -s globstar && pyupgrade --py37-plus **/*.py || true
-      - run: safety check
+      - run: safety check --ignore=62044
       #- uses: pyupio/[email protected]
       #  with:
       #    api-key: ${{secrets.SAFETY_API_KEY}}

README.md

@@ -8,7 +8,11 @@ Learn more at [LibreQoS.io](https://libreqos.io/)!
 
 ## Sponsors
 
-Special thanks to Equinix for providing server resources to support the development of LibreQoS.
+LibreQoS' development is made possible by our sponsors, the NLnet Foundation and Equinix.
+
+LibreQoS has been funded through the NGI0 Entrust Fund, a fund established by NLnet with financial support from the European Commission’s Next Generation Internet programme, under the aegis of DG Communications Networks, Content and Technology under grant agreement No 101069594. Learn more at https://nlnet.nl/project/LibreQoS/
+
+Equinix supports LibreQoS through its Open Source program – providing access to hardware resources on its Equinix Metal infrastructure. Equinix’ support has been crucial for LibreQoS to scale past 10Gbps for higher-bandwidth networks. Learn more about Equinix Metal here.
 Learn more about [Equinix Metal here](https://deploy.equinix.com/metal/).
 
 ## Support LibreQoS

src/integrationPowercode.py

@@ -0,0 +1,127 @@
+from pythonCheck import checkPythonVersion
+checkPythonVersion()
+import requests
+import warnings
+from ispConfig import excludeSites, findIPv6usingMikrotik, bandwidthOverheadFactor, exceptionCPEs, powercode_api_key, powercode_api_url
+from integrationCommon import isIpv4Permitted
+import base64
+from requests.auth import HTTPBasicAuth
+if findIPv6usingMikrotik == True:
+	from mikrotikFindIPv6 import pullMikrotikIPv6  
+from integrationCommon import NetworkGraph, NetworkNode, NodeType
+from urllib3.exceptions import InsecureRequestWarning
+
+def getCustomerInfo():
+	headers= {'Content-Type': 'application/x-www-form-urlencoded'}
+	url = powercode_api_url + ":444/api/preseem/index.php"
+	data = {}
+	data['apiKey'] = powercode_api_key
+	data['action'] = 'list_customers'
+
+	r = requests.post(url, data=data, headers=headers, verify=False, timeout=10)
+	return r.json()
+
+def getServiceInfo(customerID):
+	headers= {'Content-Type': 'application/x-www-form-urlencoded'}
+	url = powercode_api_url + ":444/api/1/index.php"
+	data = {}
+	data['apiKey'] = powercode_api_key
+	data['action'] = 'readCustomerService'
+	data['customerID'] = customerID
+
+	r = requests.post(url, data=data, headers=headers, verify=False, timeout=10)
+	servicesDict = {}
+	for service in r.json()['services']:
+		if 'internetInfo' in service:
+			servicesDict[service['serviceID']] = {}
+			servicesDict[service['serviceID']]['downloadMbps'] = int(round(int(service['internetInfo']['maxIn']) / 1000))
+			servicesDict[service['serviceID']]['uploadMbps'] = int(round(int(service['internetInfo']['maxOut']) / 1000))
+	return servicesDict
+
+def createShaper():
+	net = NetworkGraph()
+	requests.packages.urllib3.disable_warnings(category=InsecureRequestWarning)
+	print("Fetching data from Powercode")
+
+	customerInfo = getCustomerInfo()
+
+	customerIDs = []
+	for customer in customerInfo:
+		if customer['id'] != '1':
+			if customer['id'] != '':
+				if customer['status'] == 'Active':
+					customerIDint = int(customer['id'])
+					if customerIDint != 0:
+						if customerIDint != None:
+							if customerIDint not in customerIDs:
+								customerIDs.append(customerIDint)
+
+	allServices = {}
+	for customerID in customerIDs:
+		allServices.update(getServiceInfo(customerID))
+
+	acceptableEquipment = ['Customer Owned Equipment', 'Router', 'Customer Owned Equipment', 'Managed Routers'] #'CPE'
+
+	devicesByCustomerID = {}
+	for customer in customerInfo:
+		if customer['status'] == 'Active':
+			chosenName = ''
+			if customer['name'] != '':
+				chosenName = customer['name']
+			elif customer['company_name'] != '':
+				chosenName = customer['company_name']
+			else:
+				chosenName = customer['id']
+			for equipment in customer['equipment']:
+				if equipment['type'] in acceptableEquipment:
+					if equipment['service_id'] in allServices:
+						device = {}
+						device['id'] = "c_" + customer['id'] + "_s_" + "_d_" + equipment['id']
+						device['name'] = equipment['name']
+						device['ipv4'] = equipment['ip_address']
+						device['mac'] = equipment['mac_address']
+						if customer['id'] not in devicesByCustomerID:
+							devicesByCustomerID[customer['id']] = {}
+							devicesByCustomerID[customer['id']]['name'] = chosenName
+						devicesByCustomerID[customer['id']]['downloadMbps'] = allServices[equipment['service_id']]['downloadMbps']
+						devicesByCustomerID[customer['id']]['uploadMbps'] = allServices[equipment['service_id']]['uploadMbps']
+						if 'devices' not in devicesByCustomerID[customer['id']]:
+							devicesByCustomerID[customer['id']]['devices'] = []
+						devicesByCustomerID[customer['id']]['devices'].append(device)
+
+	for customerID in devicesByCustomerID:
+		customer = NetworkNode(
+				type=NodeType.client,
+				id=customerID,
+				displayName=devicesByCustomerID[customerID]['name'],
+				address='',
+				customerName=devicesByCustomerID[customerID]['name'],
+				download=devicesByCustomerID[customerID]['downloadMbps'],
+				upload=devicesByCustomerID[customerID]['uploadMbps'],
+			)
+		net.addRawNode(customer)
+		for device in devicesByCustomerID[customerID]['devices']:
+			newDevice = NetworkNode(
+				id=device['id'],
+				displayName=device["name"],
+				type=NodeType.device,
+				parentId=customerID,
+				mac=device["mac"],
+				ipv4=[device['ipv4']],
+				ipv6=[]
+			)
+			net.addRawNode(newDevice)
+	net.prepareTree()
+	net.plotNetworkGraph(False)
+	if net.doesNetworkJsonExist():
+		print("network.json already exists. Leaving in-place.")
+	else:
+		net.createNetworkJson()
+	net.createShapedDevices()
+
+def importFromPowercode():
+	#createNetworkJSON()
+	createShaper()
+
+if __name__ == '__main__':
+	importFromPowercode()

src/integrationSplynx.py

@@ -50,7 +50,6 @@ def getRouters(headers):
 	for router in data:
 		routerID = router['id']
 		ipForRouter[routerID] = router['ip']
-
 	print("Router IPs found: " + str(len(ipForRouter)))
 	return ipForRouter
 
@@ -62,6 +61,30 @@ def combineAddress(json):
 	else:
 		return json["street_1"] + " " + json["city"] + " " + json["zip_code"]
 
+def getAllServices(headers):
+	services = spylnxRequest("admin/customers/customer/0/internet-services?main_attributes%5Bstatus%5D=active", headers)
+	return services
+
+def getAllIPs(headers):
+	ipv4ByCustomerID = {}
+	ipv6ByCustomerID = {}
+	allIPv4 = spylnxRequest("admin/networking/ipv4-ip?main_attributes%5Bis_used%5D=1", headers)
+	allIPv6 = spylnxRequest("admin/networking/ipv6-ip", headers)
+	for ipv4 in allIPv4:
+		if ipv4['customer_id'] not in ipv4ByCustomerID:
+			ipv4ByCustomerID[ipv4['customer_id']] = []
+		temp = ipv4ByCustomerID[ipv4['customer_id']]
+		temp.append(ipv4['ip'])
+		ipv4ByCustomerID[ipv4['customer_id']] = temp
+	for ipv6 in allIPv6:
+		if ipv6['is_used'] == 1:
+			if ipv6['customer_id'] not in ipv6ByCustomerID:
+				ipv6ByCustomerID[ipv6['customer_id']] = []
+			temp = ipv6ByCustomerID[ipv6['customer_id']]
+			temp.append(ipv6['ip'])
+			ipv6ByCustomerID[ipv6['customer_id']] = temp
+	return (ipv4ByCustomerID, ipv6ByCustomerID)
+
 def createShaper():
 	net = NetworkGraph()
 
@@ -70,17 +93,28 @@ def createShaper():
 	tariff, downloadForTariffID, uploadForTariffID = getTariffs(headers)
 	customers = getCustomers(headers)
 	ipForRouter = getRouters(headers)
-
-	# It's not very clear how a service is meant to handle multiple
-	# devices on a shared tariff. Creating each service as a combined
-	# entity including the customer, to be on the safe side.
+	allServices = getAllServices(headers)
+	ipv4ByCustomerID, ipv6ByCustomerID = getAllIPs(headers)
+
+	allServicesDict = {}
+	for serviceItem in allServices:
+		if (serviceItem['status'] == 'active'):
+			if serviceItem["customer_id"] not in allServicesDict:
+				allServicesDict[serviceItem["customer_id"]] = []
+			temp = allServicesDict[serviceItem["customer_id"]]
+			temp.append(serviceItem)
+			allServicesDict[serviceItem["customer_id"]] = temp
+
+	#It's not very clear how a service is meant to handle multiple
+	#devices on a shared tariff. Creating each service as a combined
+	#entity including the customer, to be on the safe side.
 	for customerJson in customers:
 		if customerJson['status'] == 'active':
-			services = spylnxRequest("admin/customers/customer/" + customerJson["id"] + "/internet-services", headers)
-			for serviceJson in services:
-				if (serviceJson['status'] == 'active'):
-					combinedId = "c_" + str(customerJson["id"]) + "_s_" + str(serviceJson["id"])
-					tariff_id = serviceJson['tariff_id']
+			if customerJson['id'] in allServicesDict:
+				servicesForCustomer = allServicesDict[customerJson['id']]
+				for service in servicesForCustomer:
+					combinedId = "c_" + str(customerJson["id"]) + "_s_" + str(service["id"])
+					tariff_id = service['tariff_id']
 					customer = NetworkNode(
 						type=NodeType.client,
 						id=combinedId,
@@ -92,39 +126,42 @@ def createShaper():
 					)
 					net.addRawNode(customer)
 
-					ipv4 = ''
-					ipv6 = ''
-					routerID = serviceJson['router_id']
+					ipv4 = []
+					ipv6 = []
+					routerID = service['router_id']
+
 					# If not "Taking IPv4" (Router will assign IP), then use router's set IP
-					# Debug
-					taking_ipv4 = int(serviceJson['taking_ipv4'])
+					taking_ipv4 = int(service['taking_ipv4'])
 					if taking_ipv4 == 0:
-						try:
-							ipv4 = ipForRouter[routerID]
-						except:
-							warnings.warn("taking_ipv4 was 0 for client " + combinedId + " but router ID was not found in ipForRouter", stacklevel=2)
-							ipv4 = ''
+						if routerID in ipForRouter:
+								ipv4 = [ipForRouter[routerID]]
+
 					elif taking_ipv4 == 1:
-						ipv4 = serviceJson['ipv4']
+						ipv4 = [service['ipv4']]
+					if len(ipv4) == 0:
+						#Only do this if single service for a customer
+						if len(servicesForCustomer) == 1:
+							if customerJson['id'] in ipv4ByCustomerID:
+								ipv4 = ipv4ByCustomerID[customerJson['id']]
 
 					# If not "Taking IPv6" (Router will assign IP), then use router's set IP
-					if isinstance(serviceJson['taking_ipv6'], str):
-						taking_ipv6 = int(serviceJson['taking_ipv6'])
+					if isinstance(service['taking_ipv6'], str):
+						taking_ipv6 = int(service['taking_ipv6'])
 					else:
-						taking_ipv6 = serviceJson['taking_ipv6']
+						taking_ipv6 = service['taking_ipv6']
 					if taking_ipv6 == 0:
-						ipv6 = ''
+						ipv6 = []
 					elif taking_ipv6 == 1:
-						ipv6 = serviceJson['ipv6']
+						ipv6 = [service['ipv6']]
 
 					device = NetworkNode(
-						id=combinedId+"_d" + str(serviceJson["id"]),
-						displayName=serviceJson["id"],
+						id=combinedId+"_d" + str(service["id"]),
+						displayName=service["id"],
 						type=NodeType.device,
 						parentId=combinedId,
-						mac=serviceJson["mac"],
-						ipv4=[ipv4],
-						ipv6=[ipv6]
+						mac=service["mac"],
+						ipv4=ipv4,
+						ipv6=ipv6
 					)
 					net.addRawNode(device)
 

src/ispConfig.example.py

@@ -77,6 +77,12 @@
 ignoreSubnets = ['192.168.0.0/16']
 allowedSubnets = ['100.64.0.0/10']
 
+# Powercode Integration
+automaticImportPowercode = False
+powercode_api_key = ''
+# Everything before :444/api/ in your Powercode instance URL
+powercode_api_url = ''
+
 # Splynx Integration
 automaticImportSplynx = False
 splynx_api_key = ''

src/scheduler.py

@@ -11,6 +11,12 @@
 	from integrationUISP import importFromUISP
 if automaticImportSplynx:
 	from integrationSplynx import importFromSplynx
+try:
+	from ispConfig import automaticImportPowercode
+except:
+	automaticImportPowercode = False
+if automaticImportPowercode:
+	from integrationPowercode import importFromPowercode
 from apscheduler.schedulers.background import BlockingScheduler
 from apscheduler.executors.pool import ThreadPoolExecutor
 
@@ -27,6 +33,11 @@ def importFromCRM():
 			importFromSplynx()
 		except:
 			print("Failed to import from Splynx")
+	elif automaticImportPowercode:
+		try:
+			importFromPowercode()
+		except:
+			print("Failed to import from Powercode")
 
 def graphHandler():
 	try: