[rqd] [cuegui] Consolidate log read and write into a single package

ci/build_sphinx_docs.sh

@@ -7,10 +7,12 @@ pip install --user -r requirements.txt -r docs/requirements.txt
 
 # Must generate Python code from Protos in order for Sphinx to build the docs.
 python -m grpc_tools.protoc -I=proto/ --python_out=pycue/opencue/compiled_proto --grpc_python_out=pycue/opencue/compiled_proto proto/*.proto
+python -m grpc_tools.protoc -I=proto/ --python_out=rqd/rqd/compiled_proto --grpc_python_out=rqd/rqd/compiled_proto proto/*.proto
 
 # Fix imports to work in both Python 2 and 3. See
 # <https://github.com/protocolbuffers/protobuf/issues/1491> for more info.
 2to3 -wn -f import pycue/opencue/compiled_proto/*_pb2*.py
+2to3 -wn -f import rqd/rqd/compiled_proto/*_pb2*.py
 
 # Build the docs and treat warnings as errors
 ~/.local/bin/sphinx-build -W -b html -d docs/_build/doctrees docs docs/_build/html

ci/pylintrc_main

@@ -204,11 +204,15 @@ ignored-classes=optparse.Values,thread._local,_thread._local
 # and thus existing member attributes cannot be deduced by static analysis). It
 # supports qualified module names, as well as Unix pattern matching.
 ignored-modules=opencue.compiled_proto,
-            opencue.compiled_proto.filter_pb2,
-            opencue.compiled_proto.host_pb2,
-            rqd.compiled_proto.rqd_pb2,
-            rqd.compiled_proto.host_pb2,
-            rqd.compiled_proto.report_pb2
+                opencue.compiled_proto.filter_pb2,
+                opencue.compiled_proto.host_pb2,
+                opencue.compiled_proto.rqd_pb2,
+                opencue.compiled_proto.report_pb2,
+                rqd.compiled_proto,
+                rqd.compiled_proto.filter_pb2,
+                rqd.compiled_proto.host_pb2,
+                rqd.compiled_proto.rqd_pb2,
+                rqd.compiled_proto.report_pb2
 
 # Show a hint with possible names when a member name was not found. The aspect
 # of finding the hint is based on edit distance.

ci/pylintrc_test

@@ -206,8 +206,12 @@ ignored-classes=optparse.Values,thread._local,_thread._local
 ignored-modules=opencue.compiled_proto,
                 opencue.compiled_proto.filter_pb2,
                 opencue.compiled_proto.host_pb2,
-                rqd.compiled_proto.rqd_pb2,
+                opencue.compiled_proto.rqd_pb2,
+                opencue.compiled_proto.report_pb2,
+                rqd.compiled_proto,
+                rqd.compiled_proto.filter_pb2,
                 rqd.compiled_proto.host_pb2,
+                rqd.compiled_proto.rqd_pb2,
                 rqd.compiled_proto.report_pb2
 
 # Show a hint with possible names when a member name was not found. The aspect

ci/run_python_lint.sh

@@ -49,6 +49,6 @@ cd ..
 
 echo "Running lint for rqd/..."
 cd rqd
-python -m pylint --rcfile=../ci/pylintrc_main rqd --ignore=rqd/compiled_proto
-python -m pylint --rcfile=../ci/pylintrc_test tests
+PYTHONPATH=../pycue python -m pylint --rcfile=../ci/pylintrc_main rqd --ignore=rqd/compiled_proto
+PYTHONPATH=../pycue python -m pylint --rcfile=../ci/pylintrc_test tests
 cd ..

ci/run_python_tests.sh

@@ -26,7 +26,7 @@ python pycue/setup.py test
 PYTHONPATH=pycue python pyoutline/setup.py test
 PYTHONPATH=pycue python cueadmin/setup.py test
 PYTHONPATH=pycue:pyoutline python cuesubmit/setup.py test
-python rqd/setup.py test
+PYTHONPATH=pycue python rqd/setup.py test
 
 # Xvfb no longer supports Python 2.
 if [[ "$python_version" =~ "Python 3" && ${args[0]} != "--no-gui" ]]; then

cuegui/cuegui/plugins/LogViewPlugin.py

@@ -22,7 +22,6 @@
 
 from builtins import str
 from builtins import range
-import os
 import re
 import string
 import sys
@@ -33,53 +32,17 @@
 from qtpy import QtCore
 from qtpy import QtWidgets
 
+import cuelogging
 import cuegui.Constants
 import cuegui.AbstractDockWidget
 
-
 PLUGIN_NAME = 'LogView'
 PLUGIN_CATEGORY = 'Other'
 PLUGIN_DESCRIPTION = 'Displays Frame Log'
 PLUGIN_PROVIDES = 'LogViewPlugin'
 PRINTABLE = set(string.printable)
 
 
-class LogReader(object):
-    """
-    Custom class to abstract reading log files from multiple backends
-    """
-    filepath = None
-    type = None
-
-    def __init__(self, filepath):
-        """LogReader class initialization
-           @type    filepath: string
-           @param   filepath: The filepath to log to
-        """
-        self.filepath = filepath
-
-    def size(self):
-        """Return the size of the file"""
-        return int(os.stat(self.filepath).st_size)
-
-    def getMtime(self):
-        """Return modification time of the file"""
-        return os.path.getmtime(self.filepath)
-
-    def exists(self):
-        """Check if the file exists"""
-        return os.path.exists(self.filepath)
-
-    def read(self):
-        """Read the data from the backend"""
-        content = None
-        if self.exists() is True:
-            with open(self.filepath, "r", encoding='utf-8') as fp:
-                content = fp.read()
-
-        return content
-
-
 class LineNumberArea(QtWidgets.QWidget):
     """
     Custom widget for the line numbers. This widget is designed to be attached
@@ -850,7 +813,7 @@ def _display_log_content(self):
         @postcondition: The _update_log method is scheduled to run again
                         after 5 seconds
         """
-        log_reader = LogReader(self._log_file)
+        log_reader = cuelogging.CueLogReader(self._log_file)
 
         try:
             if log_reader.exists() is not True:

pycue/cuelogging/CueLogReader.py

@@ -0,0 +1,59 @@
+#  Copyright Contributors to the OpenCue Project
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+"""Module for reading files"""
+
+import os
+
+class CueLogReader(object):
+    """Class to abstract file log reading, this class tries to act as a file object"""
+
+    filepath = None
+
+    def __init__(self, filepath):
+        """CueLogWriter class initialization
+           @type    filepath: string
+           @param   filepath: The filepath to log to
+        """
+        self.filepath = filepath
+
+    def size(self):
+        """Return the size of the file"""
+        return int(os.stat(self.filepath).st_size)
+
+    def getMtime(self):
+        """Return modification time of the file"""
+        return os.path.getmtime(self.filepath)
+
+    def exists(self):
+        """Check if the file exists"""
+        return os.path.exists(self.filepath)
+
+    def read(self):
+        """Read the data from the backend"""
+
+        content = None
+        if self.exists() is True:
+            with open(self.filepath, "r", encoding='utf-8') as fp:
+                content = fp.read()
+        else:
+            raise IOError("Failed to open %s" % self.filepath)
+
+        return content
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        pass

rqd/rqd/rqlogging.py → pycue/cuelogging/CueLogWriter.py

@@ -12,64 +12,56 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
-
-"""Logging module, handles logging to files and non-files"""
-
+"""Module for writing files"""
 
 import logging
-import time
 import os
-import datetime
 import platform
-
-import rqd.rqconstants
+import datetime
+import time
 
 log = logging.getLogger(__name__)
-log.setLevel(rqd.rqconstants.CONSOLE_LOG_LEVEL)
 
+class CueLogWriter(object):
+    """Class to abstract file log writing, this class tries to act as a file object"""
 
-class RqdLogger(object):
-    """Class to abstract file logging, this class tries to act as a file object"""
     filepath = None
-    fd = None
-    type = 0
 
-    def __init__(self, filepath):
-        """RQDLogger class initialization
+    def __init__(self, filepath, maxLogFiles=1):
+        """CueLogWriter class initialization
            @type    filepath: string
            @param   filepath: The filepath to log to
+           @type    maxLogFiles: int
+           @param   maxLogFiles: number of files to rotate, when in write mode
         """
 
         self.filepath = filepath
+        self.__log_dir = os.path.dirname(self.filepath)
+        self.__maxLogFiles = maxLogFiles
+        if not os.access(self.__log_dir, os.F_OK):
+            self.__makeLogDir()
 
-        log_dir = os.path.dirname(self.filepath)
-        if not os.access(log_dir, os.F_OK):
-            # Attempting mkdir for missing logdir
-            msg = "No Error"
-            try:
-                os.makedirs(log_dir)
-                os.chmod(log_dir, 0o777)
-            # pylint: disable=broad-except
-            except Exception as e:
-                # This is expected to fail when called in abq
-                # But the directory should now be visible
-                msg = e
-
-            if not os.access(log_dir, os.F_OK):
-                err = "Unable to see log directory: %s, mkdir failed with: %s" % (
-                    log_dir, msg)
-                raise RuntimeError(err)
-
-        if not os.access(log_dir, os.W_OK):
-            err = "Unable to write to log directory %s" % log_dir
+        if not os.access(self.__log_dir, os.W_OK):
+            err = "Unable to write to log directory %s" % self.__log_dir
             raise RuntimeError(err)
 
+        self.__attemptRotateLogs()
+        # pylint: disable=consider-using-with
+        self.__fd = open(self.filepath, "w+", 1, encoding='utf-8')
+        try:
+            os.chmod(self.filepath, 0o666)
+        # pylint: disable=broad-except
+        except Exception as e:
+            err = "Failed to chmod log file! %s due to %s" % (self.filepath, e)
+            log.warning(err)
+
+    def __attemptRotateLogs(self):
         try:
             # Rotate any old logs to a max of MAX_LOG_FILES:
             if os.path.isfile(self.filepath):
                 rotateCount = 1
                 while (os.path.isfile("%s.%s" % (self.filepath, rotateCount))
-                       and rotateCount < rqd.rqconstants.MAX_LOG_FILES):
+                       and rotateCount < self.__maxLogFiles):
                     rotateCount += 1
                 os.rename(self.filepath,
                           "%s.%s" % (self.filepath, rotateCount))
@@ -83,14 +75,22 @@ def __init__(self, filepath):
                 log.warning(err)
             else:
                 raise RuntimeError(err)
-        # pylint: disable=consider-using-with
-        self.fd = open(self.filepath, "w+", 1, encoding='utf-8')
+
+    def __makeLogDir(self):
+        # Attempting mkdir for missing logdir
+        msg = "No Error"
         try:
-            os.chmod(self.filepath, 0o666)
+            os.makedirs(self.__log_dir)
+            os.chmod(self.__log_dir, 0o777)
         # pylint: disable=broad-except
         except Exception as e:
-            err = "Failed to chmod log file! %s due to %s" % (self.filepath, e)
-            log.warning(err)
+            # This is expected to fail when called in abq
+            # But the directory should now be visible
+            msg = e
+
+        if not os.access(self.__log_dir, os.F_OK):
+            err = "Unable to see log directory: %s, mkdir failed with: %s" % (self.__log_dir, msg)
+            raise RuntimeError(err)
 
     # pylint: disable=arguments-differ
     def write(self, data, prependTimestamp=False):
@@ -104,7 +104,7 @@ def write(self, data, prependTimestamp=False):
             for line in lines:
                 print("[%s] %s" % (curr_line_timestamp, line), file=self)
         else:
-            self.fd.write(data)
+            self.__fd.write(data)
 
     def writelines(self, __lines):
         """Provides support for writing mutliple lines at a time"""
@@ -113,7 +113,7 @@ def writelines(self, __lines):
 
     def close(self):
         """Closes the file if the backend is file based"""
-        self.fd.close()
+        self.__fd.close()
 
     def waitForFile(self, maxTries=5):
         """Waits for the file to exist before continuing when using a file backend"""
@@ -126,8 +126,4 @@ def waitForFile(self, maxTries=5):
             time.sleep(0.5 * tries)
         raise IOError("Failed to create %s" % self.filepath)
 
-    def __enter__(self):
-        return self
 
-    def __exit__(self, exc_type, exc_val, exc_tb):
-        pass

pycue/cuelogging/__init__.py

@@ -0,0 +1,19 @@
+#  Copyright Contributors to the OpenCue Project
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+
+"""Package for reading and writing log files"""
+
+from .CueLogReader import CueLogReader
+from .CueLogWriter import CueLogWriter

rqd/rqd/rqcore.py

@@ -36,6 +36,7 @@
 import traceback
 import select
 
+import cuelogging
 import rqd.compiled_proto.host_pb2
 import rqd.compiled_proto.report_pb2
 import rqd.rqconstants
@@ -44,7 +45,6 @@
 import rqd.rqnetwork
 import rqd.rqnimby
 import rqd.rqutil
-import rqd.rqlogging
 
 INT32_MAX = 2147483647
 INT32_MIN = -2147483648
@@ -490,7 +490,10 @@ def run(self):
 
                     # Setup frame logging
                     try:
-                        self.rqlog = rqd.rqlogging.RqdLogger(runFrame.log_dir_file)
+                        self.rqlog = cuelogging.CueLogWriter(
+                            runFrame.log_dir_file,
+                            maxLogFiles=rqd.rqconstants.MAX_LOG_FILES
+                        )
                         self.rqlog.waitForFile()
                     # pylint: disable=broad-except
                     except Exception as e: