diff --git a/ci/build_sphinx_docs.sh b/ci/build_sphinx_docs.sh index b39c68fc2..04cbd26d8 100755 --- a/ci/build_sphinx_docs.sh +++ b/ci/build_sphinx_docs.sh @@ -7,10 +7,12 @@ pip install --user -r requirements.txt -r docs/requirements.txt # Must generate Python code from Protos in order for Sphinx to build the docs. python -m grpc_tools.protoc -I=proto/ --python_out=pycue/opencue/compiled_proto --grpc_python_out=pycue/opencue/compiled_proto proto/*.proto +python -m grpc_tools.protoc -I=proto/ --python_out=rqd/rqd/compiled_proto --grpc_python_out=rqd/rqd/compiled_proto proto/*.proto # Fix imports to work in both Python 2 and 3. See # for more info. python ci/fix_compiled_proto.py pycue/opencue/compiled_proto +python ci/fix_compiled_proto.py rqd/rqd/compiled_proto # Build the docs and treat warnings as errors ~/.local/bin/sphinx-build -W -b html -d docs/_build/doctrees docs docs/_build/html diff --git a/ci/pylintrc_main b/ci/pylintrc_main index cdd7f1ab6..8bf017821 100644 --- a/ci/pylintrc_main +++ b/ci/pylintrc_main @@ -204,11 +204,15 @@ ignored-classes=optparse.Values,thread._local,_thread._local # and thus existing member attributes cannot be deduced by static analysis). It # supports qualified module names, as well as Unix pattern matching. ignored-modules=opencue.compiled_proto, - opencue.compiled_proto.filter_pb2, - opencue.compiled_proto.host_pb2, - rqd.compiled_proto.rqd_pb2, - rqd.compiled_proto.host_pb2, - rqd.compiled_proto.report_pb2 + opencue.compiled_proto.filter_pb2, + opencue.compiled_proto.host_pb2, + opencue.compiled_proto.rqd_pb2, + opencue.compiled_proto.report_pb2, + rqd.compiled_proto, + rqd.compiled_proto.filter_pb2, + rqd.compiled_proto.host_pb2, + rqd.compiled_proto.rqd_pb2, + rqd.compiled_proto.report_pb2 # Show a hint with possible names when a member name was not found. The aspect # of finding the hint is based on edit distance. diff --git a/ci/pylintrc_test b/ci/pylintrc_test index 434a1d754..4603f50a4 100644 --- a/ci/pylintrc_test +++ b/ci/pylintrc_test @@ -206,8 +206,12 @@ ignored-classes=optparse.Values,thread._local,_thread._local ignored-modules=opencue.compiled_proto, opencue.compiled_proto.filter_pb2, opencue.compiled_proto.host_pb2, - rqd.compiled_proto.rqd_pb2, + opencue.compiled_proto.rqd_pb2, + opencue.compiled_proto.report_pb2, + rqd.compiled_proto, + rqd.compiled_proto.filter_pb2, rqd.compiled_proto.host_pb2, + rqd.compiled_proto.rqd_pb2, rqd.compiled_proto.report_pb2 # Show a hint with possible names when a member name was not found. The aspect diff --git a/ci/python_coverage_report.sh b/ci/python_coverage_report.sh index 4477c8d44..480d40cea 100755 --- a/ci/python_coverage_report.sh +++ b/ci/python_coverage_report.sh @@ -19,7 +19,7 @@ PYTHONPATH=pycue python -m coverage run -a --source=cueadmin/cueadmin/ cueadmin/ # TODO: re-enable cuegui tests when xvfb-run gets configured to execute on the new vfx-platform # PYTHONPATH=pycue xvfb-run -d python -m coverage run -a --source=cuegui/cuegui/ cuegui/setup.py test PYTHONPATH=pycue:pyoutline python -m coverage run -a --source=cuesubmit/cuesubmit/ cuesubmit/setup.py test -python -m coverage run -a --source=rqd/rqd/ --omit=rqd/rqd/compiled_proto/* rqd/setup.py test +PYTHONPATH=pycue python -m coverage run -a --source=rqd/rqd/ --omit=rqd/rqd/compiled_proto/* rqd/setup.py test # SonarCloud needs the report in XML. python -m coverage xml diff --git a/ci/run_python_lint.sh b/ci/run_python_lint.sh index ba98d2b6c..1dd3bef39 100755 --- a/ci/run_python_lint.sh +++ b/ci/run_python_lint.sh @@ -49,6 +49,6 @@ cd .. echo "Running lint for rqd/..." cd rqd -python -m pylint --rcfile=../ci/pylintrc_main rqd --ignore=rqd/compiled_proto -python -m pylint --rcfile=../ci/pylintrc_test tests +PYTHONPATH=../pycue python -m pylint --rcfile=../ci/pylintrc_main rqd --ignore=rqd/compiled_proto +PYTHONPATH=../pycue python -m pylint --rcfile=../ci/pylintrc_test tests cd .. diff --git a/ci/run_python_tests.sh b/ci/run_python_tests.sh index 101d75288..b8452d02f 100755 --- a/ci/run_python_tests.sh +++ b/ci/run_python_tests.sh @@ -26,7 +26,7 @@ python -m unittest discover -s pycue/tests -t pycue -p "*.py" PYTHONPATH=pycue python -m unittest discover -s pyoutline/tests -t pyoutline -p "*.py" PYTHONPATH=pycue python -m unittest discover -s cueadmin/tests -t cueadmin -p "*.py" PYTHONPATH=pycue:pyoutline python -m unittest discover -s cuesubmit/tests -t cuesubmit -p "*.py" -python -m pytest rqd/tests +PYTHONPATH=pycue python -m pytest rqd/tests # Xvfb no longer supports Python 2. diff --git a/cuegui/cuegui/plugins/LogViewPlugin.py b/cuegui/cuegui/plugins/LogViewPlugin.py index a59f20251..52787fa91 100644 --- a/cuegui/cuegui/plugins/LogViewPlugin.py +++ b/cuegui/cuegui/plugins/LogViewPlugin.py @@ -22,7 +22,6 @@ from builtins import str from builtins import range -import os import re import string import sys @@ -33,10 +32,10 @@ from qtpy import QtCore from qtpy import QtWidgets +import cuelogging import cuegui.Constants import cuegui.AbstractDockWidget - PLUGIN_NAME = 'LogView' PLUGIN_CATEGORY = 'Other' PLUGIN_DESCRIPTION = 'Displays Frame Log' @@ -44,42 +43,6 @@ PRINTABLE = set(string.printable) -class LogReader(object): - """ - Custom class to abstract reading log files from multiple backends - """ - filepath = None - type = None - - def __init__(self, filepath): - """LogReader class initialization - @type filepath: string - @param filepath: The filepath to log to - """ - self.filepath = filepath - - def size(self): - """Return the size of the file""" - return int(os.stat(self.filepath).st_size) - - def getMtime(self): - """Return modification time of the file""" - return os.path.getmtime(self.filepath) - - def exists(self): - """Check if the file exists""" - return os.path.exists(self.filepath) - - def read(self): - """Read the data from the backend""" - content = None - if self.exists() is True: - with open(self.filepath, "r", encoding='utf-8') as fp: - content = fp.read() - - return content - - class LineNumberArea(QtWidgets.QWidget): """ Custom widget for the line numbers. This widget is designed to be attached @@ -850,7 +813,7 @@ def _display_log_content(self): @postcondition: The _update_log method is scheduled to run again after 5 seconds """ - log_reader = LogReader(self._log_file) + log_reader = cuelogging.CueLogReader(self._log_file) try: if log_reader.exists() is not True: diff --git a/pycue/cuelogging/CueLogReader.py b/pycue/cuelogging/CueLogReader.py new file mode 100644 index 000000000..4bfb6d5bd --- /dev/null +++ b/pycue/cuelogging/CueLogReader.py @@ -0,0 +1,59 @@ +# Copyright Contributors to the OpenCue Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Module for reading files""" + +import os + +class CueLogReader(object): + """Class to abstract file log reading, this class tries to act as a file object""" + + filepath = None + + def __init__(self, filepath): + """CueLogWriter class initialization + @type filepath: string + @param filepath: The filepath to log to + """ + self.filepath = filepath + + def size(self): + """Return the size of the file""" + return int(os.stat(self.filepath).st_size) + + def getMtime(self): + """Return modification time of the file""" + return os.path.getmtime(self.filepath) + + def exists(self): + """Check if the file exists""" + return os.path.exists(self.filepath) + + def read(self): + """Read the data from the backend""" + + content = None + if self.exists() is True: + with open(self.filepath, "r", encoding='utf-8') as fp: + content = fp.read() + else: + raise IOError("Failed to open %s" % self.filepath) + + return content + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + pass diff --git a/rqd/rqd/rqlogging.py b/pycue/cuelogging/CueLogWriter.py similarity index 68% rename from rqd/rqd/rqlogging.py rename to pycue/cuelogging/CueLogWriter.py index e8878c0d9..d58b2c8ed 100644 --- a/rqd/rqd/rqlogging.py +++ b/pycue/cuelogging/CueLogWriter.py @@ -12,64 +12,56 @@ # See the License for the specific language governing permissions and # limitations under the License. - -"""Logging module, handles logging to files and non-files""" - +"""Module for writing files""" import logging -import time import os -import datetime import platform - -import rqd.rqconstants +import datetime +import time log = logging.getLogger(__name__) -log.setLevel(rqd.rqconstants.CONSOLE_LOG_LEVEL) +class CueLogWriter(object): + """Class to abstract file log writing, this class tries to act as a file object""" -class RqdLogger(object): - """Class to abstract file logging, this class tries to act as a file object""" filepath = None - fd = None - type = 0 - def __init__(self, filepath): - """RQDLogger class initialization + def __init__(self, filepath, maxLogFiles=1): + """CueLogWriter class initialization @type filepath: string @param filepath: The filepath to log to + @type maxLogFiles: int + @param maxLogFiles: number of files to rotate, when in write mode """ self.filepath = filepath + self.__log_dir = os.path.dirname(self.filepath) + self.__maxLogFiles = maxLogFiles + if not os.access(self.__log_dir, os.F_OK): + self.__makeLogDir() - log_dir = os.path.dirname(self.filepath) - if not os.access(log_dir, os.F_OK): - # Attempting mkdir for missing logdir - msg = "No Error" - try: - os.makedirs(log_dir) - os.chmod(log_dir, 0o777) - # pylint: disable=broad-except - except Exception as e: - # This is expected to fail when called in abq - # But the directory should now be visible - msg = e - - if not os.access(log_dir, os.F_OK): - err = "Unable to see log directory: %s, mkdir failed with: %s" % ( - log_dir, msg) - raise RuntimeError(err) - - if not os.access(log_dir, os.W_OK): - err = "Unable to write to log directory %s" % log_dir + if not os.access(self.__log_dir, os.W_OK): + err = "Unable to write to log directory %s" % self.__log_dir raise RuntimeError(err) + self.__attemptRotateLogs() + # pylint: disable=consider-using-with + self.__fd = open(self.filepath, "w+", 1, encoding='utf-8') + try: + os.chmod(self.filepath, 0o666) + # pylint: disable=broad-except + except Exception as e: + err = "Failed to chmod log file! %s due to %s" % (self.filepath, e) + log.warning(err) + + def __attemptRotateLogs(self): try: # Rotate any old logs to a max of MAX_LOG_FILES: if os.path.isfile(self.filepath): rotateCount = 1 while (os.path.isfile("%s.%s" % (self.filepath, rotateCount)) - and rotateCount < rqd.rqconstants.MAX_LOG_FILES): + and rotateCount < self.__maxLogFiles): rotateCount += 1 os.rename(self.filepath, "%s.%s" % (self.filepath, rotateCount)) @@ -83,14 +75,22 @@ def __init__(self, filepath): log.warning(err) else: raise RuntimeError(err) - # pylint: disable=consider-using-with - self.fd = open(self.filepath, "w+", 1, encoding='utf-8') + + def __makeLogDir(self): + # Attempting mkdir for missing logdir + msg = "No Error" try: - os.chmod(self.filepath, 0o666) + os.makedirs(self.__log_dir) + os.chmod(self.__log_dir, 0o777) # pylint: disable=broad-except except Exception as e: - err = "Failed to chmod log file! %s due to %s" % (self.filepath, e) - log.warning(err) + # This is expected to fail when called in abq + # But the directory should now be visible + msg = e + + if not os.access(self.__log_dir, os.F_OK): + err = "Unable to see log directory: %s, mkdir failed with: %s" % (self.__log_dir, msg) + raise RuntimeError(err) # pylint: disable=arguments-differ def write(self, data, prependTimestamp=False): @@ -104,7 +104,7 @@ def write(self, data, prependTimestamp=False): for line in lines: print("[%s] %s" % (curr_line_timestamp, line), file=self) else: - self.fd.write(data) + self.__fd.write(data) def writelines(self, __lines): """Provides support for writing mutliple lines at a time""" @@ -113,7 +113,7 @@ def writelines(self, __lines): def close(self): """Closes the file if the backend is file based""" - self.fd.close() + self.__fd.close() def waitForFile(self, maxTries=5): """Waits for the file to exist before continuing when using a file backend""" @@ -126,8 +126,4 @@ def waitForFile(self, maxTries=5): time.sleep(0.5 * tries) raise IOError("Failed to create %s" % self.filepath) - def __enter__(self): - return self - def __exit__(self, exc_type, exc_val, exc_tb): - pass diff --git a/pycue/cuelogging/__init__.py b/pycue/cuelogging/__init__.py new file mode 100644 index 000000000..06b40afe2 --- /dev/null +++ b/pycue/cuelogging/__init__.py @@ -0,0 +1,19 @@ +# Copyright Contributors to the OpenCue Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +"""Package for reading and writing log files""" + +from .CueLogReader import CueLogReader +from .CueLogWriter import CueLogWriter diff --git a/rqd/rqd/rqcore.py b/rqd/rqd/rqcore.py index abff2aed0..3d0928ee7 100644 --- a/rqd/rqd/rqcore.py +++ b/rqd/rqd/rqcore.py @@ -36,6 +36,7 @@ import traceback import select +import cuelogging import rqd.compiled_proto.host_pb2 import rqd.compiled_proto.report_pb2 import rqd.rqconstants @@ -44,7 +45,6 @@ import rqd.rqnetwork import rqd.rqnimby import rqd.rqutil -import rqd.rqlogging INT32_MAX = 2147483647 INT32_MIN = -2147483648 @@ -491,7 +491,10 @@ def run(self): # Setup frame logging try: - self.rqlog = rqd.rqlogging.RqdLogger(runFrame.log_dir_file) + self.rqlog = cuelogging.CueLogWriter( + runFrame.log_dir_file, + maxLogFiles=rqd.rqconstants.MAX_LOG_FILES + ) self.rqlog.waitForFile() # pylint: disable=broad-except except Exception as e: