Merge Dev into Master

docs/source/change_log.rst

@@ -10,12 +10,14 @@ Version 1.x.x
 Version 1.8.0
 ^^^^^^^^^^^^^
 
-*In Progress*
+Released :release:`1.8.0` on June 6th, 2017
 
 * Install script now supports Red Hat Server 7
 * Support the client on OS X by using Docker
 * Support for issuing certificates  with acme while the server is running
 * Add a wrapping tool for certbot to make the process easier
+* Updated `tools/cx_freeze.py` to build the King Phisher client in Python 3.4
+* Updated documentation for the Windows build
 
 Version 1.7.1
 ^^^^^^^^^^^^^

docs/source/development/windows_build.rst

@@ -31,16 +31,64 @@ folder.
 Version Information
 -------------------
 
-After building the MSI file you will need to add custom properties.
-By right clicking on the MSI file, select properties, and then the
-custom tab you can add custom fields. You will need to add
-the Python Version, and PyGI-AIO version utilized in making the build
-as text entries. Below is the name fields and example values.
+After building the MSI file, the custom properties will need to be added. These
+are added by right clicking on the MSI file, selecting properties, and then the
+custom tab where custom fields can be created. These need to include the Python
+version, and PyGI-AIO version utilized in making the build as text entries.
+Below is the name fields and example values.
 
 +--------------------------------+---------------------------------+
 | Name                           | Example Value                   |
 +================================+=================================+
-| Python Version                 | 2.7.11                          |
+| Python Version                 | 3.4                             |
 +--------------------------------+---------------------------------+
 | PyGI-AIO Version               | 3.14.0 rev22                    |
 +--------------------------------+---------------------------------+
+
+Python 3.4 Build
+----------------
+
+As of King Phisher :release:`1.8.0`, the Windows client is built with Python
+3.4. To install basemaps for Python 3.4 geos will need to be compiled for
+Windows. In addition to the packages in the "requirements.txt" file,
+``pypiwin32api``, and ``numpy`` will need to be installed manually.
+
+For information on how to build geos on Windows with CMake visit:
+`<https://trac.osgeo.org/geos/wiki/BuildingOnWindowsWithCMake>`_.
+
+It is important that the same version of geos be built that is used with
+basemaps.
+
+Once geos is complied the two generated DLLs ``geos.dll`` and ``geos_c.dll``
+need to be copied to "[python34]\libs\site-packages\".
+
+.. note::
+   C++ 2010 Express and older will need to have the ``floor`` and ``ceil``
+   functions defined. These two functions are required by the geos library but
+   are unavailable in older versions of the standard library.
+
+CX Freeze version 5.0.1
+-----------------------
+
+After building and installing the MSI file, if the short cut link fails because
+it cannot ``from . import xxx``, it is because the working directory for the
+shortcut is not set. To change this so builds have the working directory set
+automatically, the last line of
+"[python34]\Lib\site-packages\cx_Freeze\windist.py" needs to be updated from
+``None`` to ``"TARGETDIR"``.
+
+The ouput example of lines 52-62 of cx_freeze's "windist.py" file, with change
+applied.
+
+.. code-block:: python
+
+   for index, executable in enumerate(self.distribution.executables):
+       if executable.shortcutName is not None \
+               and executable.shortcutDir is not None:
+           baseName = os.path.basename(executable.targetName)
+           msilib.add_data(self.db, "Shortcut",
+                   [("S_APP_%s" % index, executable.shortcutDir,
+                           executable.shortcutName, "TARGETDIR",
+                           "[TARGETDIR]%s" % baseName, None, None, None,
+                           None, None, None, "TARGETDIR")])
+

king_phisher/client/dialogs/ssh_host_key.py

@@ -178,7 +178,8 @@ def missing_host_key(self, client, hostname, key):
 
 		if add_host_key:
 			self.logger.debug("setting ssh host key {0} for {1}".format(key.get_name(), hostname))
-			host_keys.pop(hostname)
+			if hostname in host_keys:
+				host_keys.pop(hostname)
 			host_keys.add(hostname, key.get_name(), key)
 			try:
 				host_keys.save(known_hosts_file)

king_phisher/server/server.py

@@ -59,18 +59,20 @@
 
 import advancedhttpserver
 import jinja2
-from smoke_zephyr import job
+import smoke_zephyr.job
+import smoke_zephyr.utilities
 
 class KingPhisherRequestHandler(advancedhttpserver.RequestHandler):
 	logger = logging.getLogger('KingPhisher.Server.RequestHandler')
 	def __init__(self, *args, **kwargs):
-		self.logger.debug("request handler running in tid: 0x{0:x}".format(threading.current_thread().ident))
+		self.logger.debug("tid: 0x{0:x} running http request handler".format(threading.current_thread().ident))
 		# this is for attribute documentation
 		self.config = None
 		"""A reference to the main server instance :py:attr:`.KingPhisherServer.config`."""
 		self.path = None
 		"""The resource path of the current HTTP request."""
 		self.rpc_session = None
+		self.semaphore_acquired = False
 		super(KingPhisherRequestHandler, self).__init__(*args, **kwargs)
 
 	def on_init(self):
@@ -133,18 +135,45 @@ def adjust_path(self):
 			raise errors.KingPhisherAbortRequestError()
 		self.path = '/' + self.vhost + self.path
 
+	def semaphore_acquire(self):
+		if self.semaphore_acquired:
+			raise RuntimeError('the request semaphore has already been acquired')
+		self.server.throttle_semaphore.acquire()
+		self.semaphore_acquired = True
+
+	def semaphore_release(self):
+		if not self.semaphore_acquired:
+			raise RuntimeError('the request semaphore has not been acquired')
+		self.server.throttle_semaphore.release()
+		self.semaphore_acquired = False
+
 	def _do_http_method(self, *args, **kwargs):
-		if self.command != 'RPC':
+		# This method wraps all of the default do_* HTTP verb handlers to
+		# provide error handling and (for non-RPC requests) path adjustments.
+		# This also is also a high level location where the throttle semaphore
+		# is managed which is acquired for all RPC requests. Non-RPC requests
+		# can acquire it as necessary and *should* release it when they are
+		# finished with it, however if they fail to do so or encounter an error
+		# the semaphore will be released here as a fail safe.
+		self.connection.settimeout(smoke_zephyr.utilities.parse_timespan('20s'))  # set a timeout as a fail safe
+		if self.command == 'RPC':
+			self.semaphore_acquire()
+		else:
 			self.adjust_path()
 		http_method_handler = getattr(super(KingPhisherRequestHandler, self), 'do_' + self.command)
-		self.server.throttle_semaphore.acquire()
 		try:
 			http_method_handler(*args, **kwargs)
 		except errors.KingPhisherAbortRequestError as error:
+			self.logger.info('http request aborted')
 			if not error.response_sent:
 				self.respond_not_found()
 		finally:
-			self.server.throttle_semaphore.release()
+			if self.semaphore_acquired:
+				if self.command != 'RPC':
+					self.logger.warning('http request failed to cleanly release resources')
+				self.semaphore_release()
+		self.connection.settimeout(None)
+
 	do_GET = _do_http_method
 	do_HEAD = _do_http_method
 	do_POST = _do_http_method
@@ -394,7 +423,9 @@ def send_response(self, code, message=None):
 		signals.safe_send('response-sent', self.logger, self, code=code, message=message)
 
 	def respond_file(self, file_path, attachment=False, query=None):
+		self.semaphore_acquire()
 		self._respond_file_check_id()
+		self.semaphore_release()
 		file_path = os.path.abspath(file_path)
 		mime_type = self.guess_mime_type(file_path)
 		if attachment or (mime_type != 'text/html' and mime_type != 'text/plain'):
@@ -411,6 +442,7 @@ def respond_file(self, file_path, attachment=False, query=None):
 			self.server.logger.error("unicode error {0} in template file: {1}:{2}-{3}".format(error.reason, file_path, error.start, error.end))
 			raise errors.KingPhisherAbortRequestError()
 
+		self.semaphore_acquire()
 		template_data = b''
 		headers = []
 		template_vars = {
@@ -430,6 +462,7 @@ def respond_file(self, file_path, attachment=False, query=None):
 		try:
 			template_module = template.make_module(template_vars)
 		except (TypeError, jinja2.TemplateError) as error:
+			self.semaphore_release()
 			self.server.logger.error("jinja2 template {0} render failed: {1} {2}".format(template.filename, error.__class__.__name__, error.message))
 			raise errors.KingPhisherAbortRequestError()
 
@@ -444,6 +477,7 @@ def respond_file(self, file_path, attachment=False, query=None):
 			try:
 				template_data = template.render(template_vars)
 			except (TypeError, jinja2.TemplateError) as error:
+				self.semaphore_release()
 				self.server.logger.error("jinja2 template {0} render failed: {1} {2}".format(template.filename, error.__class__.__name__, error.message))
 				raise errors.KingPhisherAbortRequestError()
 			self.send_response(200)
@@ -461,6 +495,8 @@ def respond_file(self, file_path, attachment=False, query=None):
 			self.handle_page_visit()
 		except Exception as error:
 			self.server.logger.error('handle_page_visit raised error: {0}.{1}'.format(error.__class__.__module__, error.__class__.__name__), exc_info=True)
+		finally:
+			self.semaphore_release()
 
 		self.end_headers()
 		self.wfile.write(template_data)
@@ -561,21 +597,25 @@ def handle_deaddrop_visit(self, query):
 			self.logger.error('dead drop request received with invalid \'token\' data')
 			return
 
+		self.semaphore_acquire()
 		session = db_manager.Session()
 		deployment = db_manager.get_row_by_id(session, db_models.DeaddropDeployment, data.get('deaddrop_id'))
 		if not deployment:
 			session.close()
+			self.semaphore_release()
 			self.logger.error('dead drop request received for an unknown campaign')
 			return
 		if deployment.campaign.has_expired:
 			session.close()
+			self.semaphore_release()
 			self.logger.info('dead drop request received for an expired campaign')
 			return
 
 		local_username = data.get('local_username')
 		local_hostname = data.get('local_hostname')
 		if local_username is None or local_hostname is None:
 			session.close()
+			self.semaphore_release()
 			self.logger.error('dead drop request received with missing data')
 			return
 		local_ip_addresses = data.get('local_ip_addresses')
@@ -602,6 +642,7 @@ def handle_deaddrop_visit(self, query):
 		query = query.filter_by(campaign_id=deployment.campaign_id)
 		visit_count = query.count()
 		session.close()
+		self.semaphore_release()
 		if new_connection and visit_count > 0 and ((visit_count in [1, 3, 5]) or ((visit_count % 10) == 0)):
 			alert_text = "{0} deaddrop connections reached for campaign: {{campaign_name}}".format(visit_count)
 			self.server.job_manager.job_run(self.issue_alert, (alert_text, deployment.campaign_id))
@@ -621,6 +662,7 @@ def handle_email_opened(self, query):
 		msg_id = self.get_query('id')
 		if not msg_id:
 			return
+		self.semaphore_acquire()
 		session = db_manager.Session()
 		query = session.query(db_models.Message)
 		query = query.filter_by(id=msg_id, opened=None)
@@ -632,6 +674,7 @@ def handle_email_opened(self, query):
 			session.commit()
 		session.close()
 		signals.safe_send('email-opened', self.logger, self)
+		self.semaphore_release()
 
 	def handle_javascript_hook(self, query):
 		kp_hook_js = find.data_file('javascript_hook.js')
@@ -776,7 +819,7 @@ def __init__(self, config, plugin_manager, handler_klass, *args, **kwargs):
 		self.serve_robots_txt = True
 		self.database_engine = db_manager.init_database(config.get('server.database'), extra_init=True)
 
-		self.throttle_semaphore = threading.Semaphore()
+		self.throttle_semaphore = threading.BoundedSemaphore()
 		self.session_manager = aaa.AuthenticatedSessionManager(
 			timeout=config.get_if_exists('server.authentication.cache_timeout', '30m')
 		)
@@ -785,7 +828,7 @@ def __init__(self, config, plugin_manager, handler_klass, *args, **kwargs):
 			required_group=config.get_if_exists('server.authentication.group'),
 			pam_service=config.get_if_exists('server.authentication.pam_service', 'sshd')
 		)
-		self.job_manager = job.JobManager(logger_name='KingPhisher.Server.JobManager')
+		self.job_manager = smoke_zephyr.job.JobManager(logger_name='KingPhisher.Server.JobManager')
 		"""A :py:class:`~smoke_zephyr.job.JobManager` instance for scheduling tasks."""
 		self.job_manager.start()
 		loader = jinja2.FileSystemLoader(config.get('server.web_root'))

king_phisher/server/web_sockets.py

@@ -79,7 +79,7 @@ def __init__(self, handler, manager):
 		:param manager: The manager that this event socket should register with.
 		:type manager: :py:class:`.WebSocketsManager`
 		"""
-		handler.server.throttle_semaphore.release()
+		handler.connection.settimeout(None)
 		self._subscriptions = {}
 		self.rpc_session = handler.rpc_session
 		if self.rpc_session.event_socket is not None:
@@ -103,7 +103,6 @@ def is_subscribed(self, event_id, event_type):
 		return event_type in self._subscriptions[event_id].event_types
 
 	def on_closed(self):
-		self.handler.server.throttle_semaphore.acquire()
 		manager = self._manager_ref()
 		if manager is not None:
 			manager.remove(self)

king_phisher/version.py

@@ -66,7 +66,7 @@ def get_revision():
 version_info = collections.namedtuple('version_info', ('major', 'minor', 'micro'))(1, 8, 0)
 """A tuple representing the version information in the format ('major', 'minor', 'micro')"""
 
-version_label = 'beta'
+version_label = ''
 """A version label such as alpha or beta."""
 
 version = "{0}.{1}.{2}".format(version_info.major, version_info.minor, version_info.micro)