Merge pull request #680 from yogeshojha/release/1.3.1

Release/1.3.1

CHANGELOG.md

@@ -1,5 +1,14 @@
 # Changelog
 
+## 1.3.1
+**Release Date: August 12, 2022**
+
+# Fixes
+- Fix for #643 Downloading issue for Subdomain and Endpoints
+- Fix for #627 Too many Targets causes issues while loading datatable
+- Fix version Numbering issue
+
+
 ## 1.3.0
 **Release Date: July 11, 2022**
 

docker-compose.dev.yml

@@ -96,7 +96,7 @@ services:
       - POSTGRES_HOST=${POSTGRES_HOST}
       # THIS IS A MUST FOR CHECKING UPDATE, EVERYTIME A COMMIT IS MERGED INTO
       # MASTER, UPDATE THIS!!! MAJOR.MINOR.PATCH https://semver.org/
-      - RENGINE_CURRENT_VERSION='1.3.0'
+      - RENGINE_CURRENT_VERSION='1.3.1'
     volumes:
       - ./web:/usr/src/app
       - github_repos:/usr/src/github

docker-compose.yml

@@ -95,7 +95,7 @@ services:
       - POSTGRES_HOST=${POSTGRES_HOST}
       # THIS IS A MUST FOR CHECKING UPDATE, EVERYTIME A COMMIT IS MERGED INTO
       # MASTER, UPDATE THIS!!! MAJOR.MINOR.PATCH https://semver.org/
-      - RENGINE_CURRENT_VERSION='1.3.0'
+      - RENGINE_CURRENT_VERSION='1.3.1'
     volumes:
       - ./web:/usr/src/app
       - github_repos:/usr/src/github

web/api/serializers.py

@@ -7,6 +7,8 @@
 from recon_note.models import *
 
 from django.db.models import F, JSONField, Value
+from django.contrib.humanize.templatetags.humanize import naturalday, naturaltime
+
 
 
 class SearchHistorySerializer(serializers.ModelSerializer):
@@ -17,17 +19,45 @@ class Meta:
 
 class DomainSerializer(serializers.ModelSerializer):
 	vuln_count = serializers.SerializerMethodField()
+	organization = serializers.SerializerMethodField()
+	most_recent_scan = serializers.SerializerMethodField()
+	insert_date = serializers.SerializerMethodField()
+	insert_date_humanized = serializers.SerializerMethodField()
+	start_scan_date = serializers.SerializerMethodField()
+	start_scan_date_humanized = serializers.SerializerMethodField()
 
 	class Meta:
 		model = Domain
 		fields = '__all__'
+		depth = 2
 
 	def get_vuln_count(self, obj):
 		try:
 			return obj.vuln_count
 		except:
 			return None
 
+	def get_organization(self, obj):
+		if Organization.objects.filter(domains__id=obj.id).exists():
+			return [org.name for org in Organization.objects.filter(domains__id=obj.id)]
+
+	def get_most_recent_scan(self, obj):
+		return obj.get_recent_scan_id()
+
+	def get_insert_date(self, obj):
+		return naturalday(obj.insert_date).title()
+
+	def get_insert_date_humanized(self, obj):
+		return naturaltime(obj.insert_date).title()
+
+	def get_start_scan_date(self, obj):
+		if obj.start_scan_date:
+			return naturalday(obj.start_scan_date).title()
+
+	def get_start_scan_date_humanized(self, obj):
+		if obj.start_scan_date:
+			return naturaltime(obj.start_scan_date).title()
+
 
 class SubScanResultSerializer(serializers.ModelSerializer):
 

web/api/urls.py

@@ -8,6 +8,8 @@
 
 router.register(r'listDatatableSubdomain', SubdomainDatatableViewSet)
 
+router.register(r'listTargets', ListTargetsDatatableViewSet)
+
 router.register(r'listSubdomains', SubdomainsViewSet)
 
 router.register(r'listEndpoints', EndPointViewSet)
@@ -48,6 +50,10 @@
         'queryIps/',
         ListIPs.as_view(),
         name='listIPs'),
+    path(
+        'queryInterestingSubdomains/',
+        QueryInterestingSubdomains.as_view(),
+        name='queryInterestingSubdomains'),
     path(
         'querySubdomains/',
         ListSubdomains.as_view(),

web/api/views.py

@@ -14,7 +14,7 @@
 from selenium.webdriver.common.by import By
 from selenium.webdriver.support import expected_conditions as EC
 
-from django.db.models import Q
+from django.db.models import Q, F
 from django.db.models import CharField, Value, Count
 from django.core import serializers
 from django.shortcuts import get_object_or_404
@@ -40,6 +40,64 @@
 from reNgine.celery import app
 from django.utils import timezone
 
+
+class QueryInterestingSubdomains(APIView):
+	def get(self, request):
+		req = self.request
+		scan_id = req.query_params.get('scan_id')
+		target_id = req.query_params.get('target_id')
+
+		if scan_id:
+			queryset = get_interesting_subdomains(scan_history=scan_id)
+		elif target_id:
+			queryset = get_interesting_subdomains(target=target_id)
+		else:
+			queryset = get_interesting_subdomains()
+
+		queryset = queryset.distinct('name')
+
+		return Response(InterestingSubdomainSerializer(queryset, many=True).data)
+
+
+class ListTargetsDatatableViewSet(viewsets.ModelViewSet):
+	queryset = Domain.objects.all()
+	serializer_class = DomainSerializer
+
+	def get_queryset(self):
+		return self.queryset
+
+	def filter_queryset(self, qs):
+		qs = self.queryset.filter()
+		search_value = self.request.GET.get(u'search[value]', None)
+		_order_col = self.request.GET.get(u'order[0][column]', None)
+		_order_direction = self.request.GET.get(u'order[0][dir]', None)
+		if search_value or _order_col or _order_direction:
+			order_col = 'id'
+			if _order_col == '2':
+				order_col = 'name'
+			elif _order_col == '4':
+				order_col = 'insert_date'
+			elif _order_col == '5':
+				order_col = 'start_scan_date'
+				if _order_direction == 'desc':
+					return qs.order_by(F('start_scan_date').desc(nulls_last=True))
+				return qs.order_by(F('start_scan_date').asc(nulls_last=True))
+
+
+			if _order_direction == 'desc':
+				order_col = '-{}'.format(order_col)
+
+			qs = self.queryset.filter(
+				Q(name__icontains=search_value) |
+				Q(description__icontains=search_value) |
+				Q(domains__name__icontains=search_value)
+			)
+			return qs.order_by(order_col)
+
+		return qs.order_by('-id')
+
+
+
 class WafDetector(APIView):
 	def get(self, request):
 		req = self.request
@@ -619,7 +677,10 @@ def get(self, request):
 
 		# get current version_number
 		# remove quotes from current_version
-		current_version = '1.2.0'
+		current_version = ((os.environ['RENGINE_CURRENT_VERSION'
+							])[1:] if os.environ['RENGINE_CURRENT_VERSION'
+							][0] == 'v'
+							else os.environ['RENGINE_CURRENT_VERSION']).replace("'", "")
 
 		# for consistency remove v from both if exists
 		latest_version = re.search(r'v(\d+\.)?(\d+\.)?(\*|\d+)',
@@ -1486,7 +1547,7 @@ def get_queryset(self):
 			self.serializer_class = InterestingSubdomainSerializer
 
 		if scan_id:
-			self. queryset = get_interesting_subdomains(scan_history=scan_id)
+			self.queryset = get_interesting_subdomains(scan_history=scan_id)
 		elif target_id:
 			self.queryset = get_interesting_subdomains(target=target_id)
 		else:

web/art/reNgine.txt

@@ -3,6 +3,6 @@
   _ __ ___|  \| | __ _ _ _ __   ___
  | '__/ _ \ . ` |/ _` | | '_ \ / _ \
  | | |  __/ |\  | (_| | | | | |  __/
- |_|  \___|_| \_|\__, |_|_| |_|\___| v1.3.0
+ |_|  \___|_| \_|\__, |_|_| |_|\___| v1.3.1
                   __/ |
                  |___/

web/dashboard/templates/dashboard/index.html

@@ -17,7 +17,7 @@
 {% endblock custom_js_css_link %}
 
 {% block breadcrumb_title %}
-<span class="badge badge-soft-info">reNgine 1.3.0</span>
+<span class="badge badge-soft-info">reNgine 1.3.1</span>
 {% endblock breadcrumb_title %}
 
 {% block main_content %}

web/reNgine/common_func.py

@@ -71,7 +71,7 @@ def get_interesting_subdomains(scan_history=None, target=None):
     title_lookup = Subdomain.objects.none()
 
     if target:
-        subdomains = Subdomain.objects.filter(target_domain__id=target).distinct('name')
+        subdomains = Subdomain.objects.filter(target_domain__id=target)
         if subdomain_lookup_query:
             subdomain_lookup = subdomains.filter(subdomain_lookup_query)
         if page_title_lookup_query:

web/startScan/static/startScan/js/detail_scan.js

@@ -1145,18 +1145,21 @@ function download_subdomains(scan_id=null, domain_id=null, domain_name=null){
 	});
 }
 
-function download_interesting_subdomains(scan_id, domain_name){
+function download_interesting_subdomains(scan_id=null, domain_id=null, domain_name=null){
 	Swal.fire({
 		title: 'Querying Interesting Subdomains...'
 	});
 	swal.showLoading();
 	count = `<span class="modal_count"></span>`;
+	var url = `/api/queryInterestingSubdomains/?format=json`;
 	if (scan_id) {
-		url = `/api/listInterestingSubdomains/?scan_id=${scan_id}&format=json&only_subdomains&no_page`;
+		url = `/api/queryInterestingSubdomains/?scan_id=${scan_id}&format=json`;
 	}
-	else{
-		url = `/api/listInterestingSubdomains/?format=json&only_subdomains&no_page`;
+	else if(domain_id){
+		url = `/api/queryInterestingSubdomains/?target_id=${domain_id}&format=json`;
 	}
+	console.log(url);
+
 	if (domain_name) {
 		$('.modal-title').html( count + ' Interesting Subdomains for : <b>' + domain_name + '</b>');
 	}
@@ -1244,20 +1247,21 @@ function download_interesting_endpoints(scan_id, domain_name){
 }
 
 
-function download_important_subdomains(scan_id, domain_name){
+function download_important_subdomains(scan_id=null, domain_id=null, domain_name=null){
 	Swal.fire({
 		title: 'Querying Interesting Subdomains...'
 	});
 	swal.showLoading();
 	count = `<span class="modal_count"></span>`;
+	var url = `/api/querySubdomains?format=json&no_lookup_interesting&only_important`;
 	if (scan_id) {
 		url = `/api/querySubdomains?format=json&no_lookup_interesting&only_important&scan_id=${scan_id}`;
 	}
-	else{
-		url = `/api/querySubdomains?format=json&no_lookup_interesting&only_important`;
+	else if (domain_id){
+		url = `/api/querySubdomains?format=json&no_lookup_interesting&only_important&target_id=${domain_id}`;
 	}
 	if (domain_name) {
-		$('.modal-title').html(count + 'Subdomains marked as important : <b>' + domain_name + '</b>');
+		$('.modal-title').html(count + ' Subdomains marked as important : <b>' + domain_name + '</b>');
 	}
 	else{
 		$('.modal-title').html(count + ' Subdomains marked as important');

web/startScan/templates/startScan/endpoints.html

@@ -15,7 +15,7 @@
 
 {% block main_content %}
 <div class="row justify-content-center">
-  {% include 'base/_items/endpoint_tab_content.html' with detail_scan=False%}
+  {% include 'base/_items/endpoint_tab_content.html' with all_endpoints=True%}
 </div>
 {% endblock main_content %}
 

web/startScan/templates/startScan/subdomains.html

@@ -16,7 +16,7 @@
 
 {% block main_content %}
 <div class="row justify-content-center">
-	{% include 'base/_items/subdomain_tab_content.html' %}
+	{% include 'base/_items/subdomain_tab_content.html' with all_subdomains=True%}
 </div>
 {% include "base/_items/recon_note_modal.html" %}
 {% include "base/_items/subscan_modal.html" %}

web/targetApp/models.py

@@ -227,4 +227,4 @@ def get_recent_scan_id(self):
 	          return obj[0].id
 
     def __str__(self):
-        return self.name
+        return str(self.name)