From e840254f1b5f264d72b3027fc5583d3cc0117521 Mon Sep 17 00:00:00 2001 From: Jimmy Christensen Date: Wed, 14 Aug 2024 22:14:05 +0200 Subject: [PATCH 01/32] Remove rqd.compiled_proto and use from opencue.compiled_proto instead. --- rqd/rqd/compiled_proto/.gitignore | 2 -- rqd/rqd/compiled_proto/__init__.py | 13 --------- rqd/rqd/cuerqd.py | 42 +++++++++++++++--------------- rqd/rqd/rqcore.py | 20 +++++++------- rqd/rqd/rqdservicers.py | 40 ++++++++++++++-------------- rqd/rqd/rqmachine.py | 14 +++++----- rqd/rqd/rqnetwork.py | 28 ++++++++++---------- 7 files changed, 72 insertions(+), 87 deletions(-) delete mode 100644 rqd/rqd/compiled_proto/.gitignore delete mode 100644 rqd/rqd/compiled_proto/__init__.py diff --git a/rqd/rqd/compiled_proto/.gitignore b/rqd/rqd/compiled_proto/.gitignore deleted file mode 100644 index 49d7060ef..000000000 --- a/rqd/rqd/compiled_proto/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -*_pb2.py -*_pb2_grpc.py diff --git a/rqd/rqd/compiled_proto/__init__.py b/rqd/rqd/compiled_proto/__init__.py deleted file mode 100644 index d9c80f13d..000000000 --- a/rqd/rqd/compiled_proto/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright Contributors to the OpenCue Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/rqd/rqd/cuerqd.py b/rqd/rqd/cuerqd.py index 6461d7b22..7decf3486 100755 --- a/rqd/rqd/cuerqd.py +++ b/rqd/rqd/cuerqd.py @@ -31,8 +31,8 @@ import grpc -import rqd.compiled_proto.rqd_pb2 -import rqd.compiled_proto.rqd_pb2_grpc +import opencue.compiled_proto.rqd_pb2 +import opencue.compiled_proto.rqd_pb2_grpc import rqd.rqconstants @@ -46,62 +46,62 @@ def __init__(self, rqdHost, rqdPort=rqd.rqconstants.RQD_GRPC_PORT): self.rqdPort = rqdPort channel = grpc.insecure_channel('%s:%s' % (self.rqdHost, self.rqdPort)) - self.stub = rqd.compiled_proto.rqd_pb2_grpc.RqdInterfaceStub(channel) - self.frameStub = rqd.compiled_proto.rqd_pb2_grpc.RunningFrameStub(channel) + self.stub = opencue.compiled_proto.rqd_pb2_grpc.RqdInterfaceStub(channel) + self.frameStub = opencue.compiled_proto.rqd_pb2_grpc.RunningFrameStub(channel) def status(self): """Fetches and returns the host status report.""" - return self.stub.ReportStatus(rqd.compiled_proto.rqd_pb2.RqdStaticReportStatusRequest()) + return self.stub.ReportStatus(opencue.compiled_proto.rqd_pb2.RqdStaticReportStatusRequest()) def getRunningFrame(self, frameId): """Returns the host's currently running frame.""" return self.stub.GetRunFrame( - rqd.compiled_proto.rqd_pb2.RqdStaticGetRunFrameRequest(frame_id=frameId)) + opencue.compiled_proto.rqd_pb2.RqdStaticGetRunFrameRequest(frame_id=frameId)) def nimbyOff(self): """Disables Nimby on the host.""" log.info(self.rqdHost, "Turning off Nimby") log.info("rqd nimbyoff by %s", os.environ.get("USER")) - self.stub.NimbyOff(rqd.compiled_proto.rqd_pb2.RqdStaticNimbyOffRequest()) + self.stub.NimbyOff(opencue.compiled_proto.rqd_pb2.RqdStaticNimbyOffRequest()) def nimbyOn(self): """Enables Nimby on the host.""" log.info(self.rqdHost, "Turning on Nimby") log.info("rqd nimbyon by %s", os.environ.get("USER")) - self.stub.NimbyOn(rqd.compiled_proto.rqd_pb2.RqdStaticNimbyOnRequest()) + self.stub.NimbyOn(opencue.compiled_proto.rqd_pb2.RqdStaticNimbyOnRequest()) def lockAll(self): """Locks all of the host's cores.""" print(self.rqdHost, "Locking all cores") - self.stub.LockAll(rqd.compiled_proto.rqd_pb2.RqdStaticLockAllRequest()) + self.stub.LockAll(opencue.compiled_proto.rqd_pb2.RqdStaticLockAllRequest()) def unlockAll(self): """Unlocks all of the host's cores.""" print(self.rqdHost, "Unlocking all cores") - self.stub.UnlockAll(rqd.compiled_proto.rqd_pb2.RqdStaticUnlockAllRequest()) + self.stub.UnlockAll(opencue.compiled_proto.rqd_pb2.RqdStaticUnlockAllRequest()) def lock(self, cores): """Locks the given number of cores.""" cores = int(cores) print(self.rqdHost, "Locking %d cores" % cores) - self.stub.Lock(rqd.compiled_proto.rqd_pb2.RqdStaticLockRequest(cores=cores)) + self.stub.Lock(opencue.compiled_proto.rqd_pb2.RqdStaticLockRequest(cores=cores)) def unlock(self, cores): """Unlocks the given number of cores.""" cores = int(cores) print(self.rqdHost, "Unlocking %d cores" % cores) - self.stub.Unlock(rqd.compiled_proto.rqd_pb2.RqdStaticUnlockRequest(cores=cores)) + self.stub.Unlock(opencue.compiled_proto.rqd_pb2.RqdStaticUnlockRequest(cores=cores)) def shutdownRqdIdle(self): """Shuts down the host when idle.""" print(self.rqdHost, "Sending shutdownRqdIdle command") - self.stub.ShutdownRqdIdle(rqd.compiled_proto.rqd_pb2.RqdStaticShutdownIdleRequest()) + self.stub.ShutdownRqdIdle(opencue.compiled_proto.rqd_pb2.RqdStaticShutdownIdleRequest()) def shutdownRqdNow(self): """Shuts down the host now.""" print(self.rqdHost, "Sending shutdownRqdNow command") try: - self.stub.ShutdownRqdNow(rqd.compiled_proto.rqd_pb2.RqdStaticShutdownNowRequest()) + self.stub.ShutdownRqdNow(opencue.compiled_proto.rqd_pb2.RqdStaticShutdownNowRequest()) # pylint: disable=broad-except except Exception: # Shutting down the service from inside means this request will receive @@ -111,22 +111,22 @@ def shutdownRqdNow(self): def restartRqdIdle(self): """Restarts RQD on the host when idle.""" print(self.rqdHost, "Sending restartRqdIdle command") - self.stub.RestartRqdIdle(rqd.compiled_proto.rqd_pb2.RqdStaticRestartIdleRequest()) + self.stub.RestartRqdIdle(opencue.compiled_proto.rqd_pb2.RqdStaticRestartIdleRequest()) def rebootIdle(self): """Reboots the host when idle.""" print(self.rqdHost, "Sending rebootIdle command") - self.stub.RebootIdle(rqd.compiled_proto.rqd_pb2.RqdStaticRebootIdleRequest()) + self.stub.RebootIdle(opencue.compiled_proto.rqd_pb2.RqdStaticRebootIdleRequest()) def rebootNow(self): """Reboots the host now.""" print(self.rqdHost, "Sending rebootNow command") - self.stub.RebootNow(rqd.compiled_proto.rqd_pb2.RqdStaticRebootNowRequest()) + self.stub.RebootNow(opencue.compiled_proto.rqd_pb2.RqdStaticRebootNowRequest()) def launchFrame(self, frame): """Launches a frame on the host.""" self.stub.LaunchFrame( - rqd.compiled_proto.rqd_pb2.RqdStaticLaunchFrameRequest(run_frame=frame)) + opencue.compiled_proto.rqd_pb2.RqdStaticLaunchFrameRequest(run_frame=frame)) def killFrame(self, frameId, message): """Kills a frame on the host.""" @@ -251,7 +251,7 @@ def main(): if args.test_edu_frame: print("Launching edu test frame (logs to /mcp)") frameNum = "0001" - runFrame = rqd.compiled_proto.rqd_pb2.RunFrame() + runFrame = opencue.compiled_proto.rqd_pb2.RunFrame() runFrame.job_id = "SD6F3S72DJ26236KFS" runFrame.job_name = "edu-trn_jwelborn-jwelborn_teapot_bty" runFrame.frame_id = "FD1S3I154O646UGSNN%s" % frameNum @@ -272,7 +272,7 @@ def main(): if args.test_script_frame: print("Launching script test frame (logs to /mcp)") - runFrame = rqd.compiled_proto.rqd_pb2.RunFrame() + runFrame = opencue.compiled_proto.rqd_pb2.RunFrame() runFrame.resource_id = "8888888877777755555" runFrame.job_id = "SD6F3S72DJ26236KFS" runFrame.job_name = "swtest-home-jwelborn_rqd_test" @@ -290,7 +290,7 @@ def main(): if args.test_script_frame_mac: print("Launching script test frame (logs to /tmp)") - runFrame = rqd.compiled_proto.rqd_pb2.RunFrame() + runFrame = opencue.compiled_proto.rqd_pb2.RunFrame() runFrame.resource_id = "2222222277777755555" runFrame.job_id = "SD6F3S72DJ26236KFS" runFrame.job_name = "swtest-home-jwelborn_rqd_test" diff --git a/rqd/rqd/rqcore.py b/rqd/rqd/rqcore.py index 8210cfcc3..b6e9837dd 100644 --- a/rqd/rqd/rqcore.py +++ b/rqd/rqd/rqcore.py @@ -36,8 +36,8 @@ import traceback import select -import rqd.compiled_proto.host_pb2 -import rqd.compiled_proto.report_pb2 +import opencue.compiled_proto.host_pb2 +import opencue.compiled_proto.report_pb2 import rqd.rqconstants import rqd.rqexceptions import rqd.rqmachine @@ -242,7 +242,7 @@ def __writeFooter(self): def __sendFrameCompleteReport(self): """Send report to cuebot that frame has finished""" - report = rqd.compiled_proto.report_pb2.FrameCompleteReport() + report = opencue.compiled_proto.report_pb2.FrameCompleteReport() # pylint: disable=no-member report.host.CopyFrom(self.rqCore.machine.getHostInfo()) report.frame.CopyFrom(self.frameInfo.runningFrameInfo()) @@ -581,7 +581,7 @@ def __init__(self, optNimbyoff=False): self.__optNimbyoff = optNimbyoff - self.cores = rqd.compiled_proto.report_pb2.CoreDetail( + self.cores = opencue.compiled_proto.report_pb2.CoreDetail( total_cores=0, idle_cores=0, locked_cores=0, @@ -825,7 +825,7 @@ def launchFrame(self, runFrame): # Check for reasons to abort launch # - if self.machine.state != rqd.compiled_proto.host_pb2.UP: + if self.machine.state != opencue.compiled_proto.host_pb2.UP: err = "Not launching, rqd HardwareState is not Up" log.info(err) raise rqd.rqexceptions.CoreReservationFailureException(err) @@ -907,7 +907,7 @@ def reportStatus(self): def shutdownRqdNow(self): """Kill all running frames and shutdown RQD""" - self.machine.state = rqd.compiled_proto.host_pb2.DOWN + self.machine.state = opencue.compiled_proto.host_pb2.DOWN try: self.lockAll() self.killAllFrame("shutdownRqdNow Command") @@ -1030,12 +1030,12 @@ def unlock(self, reqUnlock): sendUpdate = False if (self.__whenIdle or self.__reboot or - self.machine.state != rqd.compiled_proto.host_pb2.UP): + self.machine.state != opencue.compiled_proto.host_pb2.UP): sendUpdate = True self.__whenIdle = False self.__reboot = False - self.machine.state = rqd.compiled_proto.host_pb2.UP + self.machine.state = opencue.compiled_proto.host_pb2.UP with self.__threadLock: # pylint: disable=no-member @@ -1059,12 +1059,12 @@ def unlockAll(self): sendUpdate = False if (self.__whenIdle or self.__reboot - or self.machine.state != rqd.compiled_proto.host_pb2.UP): + or self.machine.state != opencue.compiled_proto.host_pb2.UP): sendUpdate = True self.__whenIdle = False self.__reboot = False - self.machine.state = rqd.compiled_proto.host_pb2.UP + self.machine.state = opencue.compiled_proto.host_pb2.UP with self.__threadLock: # pylint: disable=no-member diff --git a/rqd/rqd/rqdservicers.py b/rqd/rqd/rqdservicers.py index 324a5a51a..f7773160c 100644 --- a/rqd/rqd/rqdservicers.py +++ b/rqd/rqd/rqdservicers.py @@ -24,14 +24,14 @@ import grpc -import rqd.compiled_proto.rqd_pb2 -import rqd.compiled_proto.rqd_pb2_grpc +import opencue.compiled_proto.rqd_pb2 +import opencue.compiled_proto.rqd_pb2_grpc log = logging.getLogger(__name__) -class RqdInterfaceServicer(rqd.compiled_proto.rqd_pb2_grpc.RqdInterfaceServicer): +class RqdInterfaceServicer(opencue.compiled_proto.rqd_pb2_grpc.RqdInterfaceServicer): """Service interface for RqdStatic gRPC definition.""" def __init__(self, rqCore): @@ -41,12 +41,12 @@ def LaunchFrame(self, request, context): """RPC call that launches the given frame""" log.info("Request received: launchFrame") self.rqCore.launchFrame(request.run_frame) - return rqd.compiled_proto.rqd_pb2.RqdStaticLaunchFrameResponse() + return opencue.compiled_proto.rqd_pb2.RqdStaticLaunchFrameResponse() def ReportStatus(self, request, context): """RPC call that returns reportStatus""" log.info("Request received: reportStatus") - return rqd.compiled_proto.rqd_pb2.RqdStaticReportStatusResponse( + return opencue.compiled_proto.rqd_pb2.RqdStaticReportStatusResponse( host_report=self.rqCore.reportStatus()) def GetRunningFrameStatus(self, request, context): @@ -54,12 +54,12 @@ def GetRunningFrameStatus(self, request, context): log.info("Request received: getRunningFrameStatus") frame = self.rqCore.getRunningFrame(request.frameId) if frame: - return rqd.compiled_proto.rqd_pb2.RqdStaticGetRunningFrameStatusResponse( + return opencue.compiled_proto.rqd_pb2.RqdStaticGetRunningFrameStatusResponse( running_frame_info=frame.runningFrameInfo()) context.set_details( "The requested frame was not found. frameId: {}".format(request.frameId)) context.set_code(grpc.StatusCode.NOT_FOUND) - return rqd.compiled_proto.rqd_pb2.RqdStaticGetRunningFrameStatusResponse() + return opencue.compiled_proto.rqd_pb2.RqdStaticGetRunningFrameStatusResponse() def KillRunningFrame(self, request, context): """RPC call that kills the running frame with the given id""" @@ -69,77 +69,77 @@ def KillRunningFrame(self, request, context): frame.kill(message=request.message) else: log.warning("Wasn't able to find frame(%s) to kill", request.frame_id) - return rqd.compiled_proto.rqd_pb2.RqdStaticKillRunningFrameResponse() + return opencue.compiled_proto.rqd_pb2.RqdStaticKillRunningFrameResponse() def ShutdownRqdNow(self, request, context): """RPC call that kills all running frames and shuts down rqd""" log.info("Request received: shutdownRqdNow") self.rqCore.shutdownRqdNow() - return rqd.compiled_proto.rqd_pb2.RqdStaticShutdownNowResponse() + return opencue.compiled_proto.rqd_pb2.RqdStaticShutdownNowResponse() def ShutdownRqdIdle(self, request, context): """RPC call that locks all cores and shuts down rqd when it is idle. unlockAll will abort the request.""" log.info("Request received: shutdownRqdIdle") self.rqCore.shutdownRqdIdle() - return rqd.compiled_proto.rqd_pb2.RqdStaticShutdownIdleResponse() + return opencue.compiled_proto.rqd_pb2.RqdStaticShutdownIdleResponse() def RestartRqdNow(self, request, context): """RPC call that kills all running frames and restarts rqd""" log.warning("Deprecated Request received: restartRqdNow. This request has no effect.") - return rqd.compiled_proto.rqd_pb2.RqdStaticRestartNowResponse() + return opencue.compiled_proto.rqd_pb2.RqdStaticRestartNowResponse() def RestartRqdIdle(self, request, context): """RPC call that that locks all cores and restarts rqd when idle. unlockAll will abort the request.""" log.warning("Deprecated Request received: restartRqdIdle. This request has no effect.") - return rqd.compiled_proto.rqd_pb2.RqdStaticRestartIdleResponse() + return opencue.compiled_proto.rqd_pb2.RqdStaticRestartIdleResponse() def RebootNow(self, request, context): """RPC call that kills all running frames and reboots the host.""" log.info("Request received: rebootNow") self.rqCore.rebootNow() - return rqd.compiled_proto.rqd_pb2.RqdStaticRebootNowResponse() + return opencue.compiled_proto.rqd_pb2.RqdStaticRebootNowResponse() def RebootIdle(self, request, context): """RPC call that that locks all cores and reboots the host when idle. unlockAll will abort the request.""" log.info("Request received: rebootIdle") self.rqCore.rebootIdle() - return rqd.compiled_proto.rqd_pb2.RqdStaticRebootIdleResponse() + return opencue.compiled_proto.rqd_pb2.RqdStaticRebootIdleResponse() def NimbyOn(self, request, context): """RPC call that activates nimby""" log.info("Request received: nimbyOn") self.rqCore.nimbyOn() - return rqd.compiled_proto.rqd_pb2.RqdStaticNimbyOnResponse() + return opencue.compiled_proto.rqd_pb2.RqdStaticNimbyOnResponse() def NimbyOff(self, request, context): """RPC call that deactivates nimby""" log.info("Request received: nimbyOff") self.rqCore.nimbyOff() - return rqd.compiled_proto.rqd_pb2.RqdStaticNimbyOffResponse() + return opencue.compiled_proto.rqd_pb2.RqdStaticNimbyOffResponse() def Lock(self, request, context): """RPC call that locks a specific number of cores""" log.info("Request received: lock %d", request.cores) self.rqCore.lock(request.cores) - return rqd.compiled_proto.rqd_pb2.RqdStaticLockResponse() + return opencue.compiled_proto.rqd_pb2.RqdStaticLockResponse() def LockAll(self, request, context): """RPC call that locks all cores""" log.info("Request received: lockAll") self.rqCore.lockAll() - return rqd.compiled_proto.rqd_pb2.RqdStaticLockAllResponse() + return opencue.compiled_proto.rqd_pb2.RqdStaticLockAllResponse() def Unlock(self, request, context): """RPC call that unlocks a specific number of cores""" log.info("Request received: unlock %d", request.cores) self.rqCore.unlock(request.cores) - return rqd.compiled_proto.rqd_pb2.RqdStaticUnlockResponse() + return opencue.compiled_proto.rqd_pb2.RqdStaticUnlockResponse() def UnlockAll(self, request, context): """RPC call that unlocks all cores""" log.info("Request received: unlockAll") self.rqCore.unlockAll() - return rqd.compiled_proto.rqd_pb2.RqdStaticUnlockAllResponse() + return opencue.compiled_proto.rqd_pb2.RqdStaticUnlockAllResponse() diff --git a/rqd/rqd/rqmachine.py b/rqd/rqd/rqmachine.py index 6874bd25b..acd2d55c2 100644 --- a/rqd/rqd/rqmachine.py +++ b/rqd/rqd/rqmachine.py @@ -57,8 +57,8 @@ import psutil -import rqd.compiled_proto.host_pb2 -import rqd.compiled_proto.report_pb2 +import opencue.compiled_proto.host_pb2 +import opencue.compiled_proto.report_pb2 import rqd.rqconstants import rqd.rqexceptions import rqd.rqswap @@ -75,7 +75,7 @@ def __init__(self, rqCore, coreInfo): """Machine class initialization @type rqCore: rqd.rqcore.RqCore @param rqCore: Main RQD Object, used to access frames and nimby states - @type coreInfo: rqd.compiled_proto.report_pb2.CoreDetail + @type coreInfo: opencue.compiled_proto.report_pb2.CoreDetail @param coreInfo: Object contains information on the state of all cores """ self.__rqCore = rqCore @@ -93,18 +93,18 @@ def __init__(self, rqCore, coreInfo): if platform.system() == 'Linux': self.__vmstat = rqd.rqswap.VmStat() - self.state = rqd.compiled_proto.host_pb2.UP + self.state = opencue.compiled_proto.host_pb2.UP - self.__renderHost = rqd.compiled_proto.report_pb2.RenderHost() + self.__renderHost = opencue.compiled_proto.report_pb2.RenderHost() self.__initMachineTags() self.__initMachineStats() - self.__bootReport = rqd.compiled_proto.report_pb2.BootReport() + self.__bootReport = opencue.compiled_proto.report_pb2.BootReport() # pylint: disable=no-member self.__bootReport.core_info.CopyFrom(self.__coreInfo) # pylint: enable=no-member - self.__hostReport = rqd.compiled_proto.report_pb2.HostReport() + self.__hostReport = opencue.compiled_proto.report_pb2.HostReport() # pylint: disable=no-member self.__hostReport.core_info.CopyFrom(self.__coreInfo) # pylint: enable=no-member diff --git a/rqd/rqd/rqnetwork.py b/rqd/rqd/rqnetwork.py index fa1fb8060..63aceca83 100644 --- a/rqd/rqd/rqnetwork.py +++ b/rqd/rqd/rqnetwork.py @@ -34,9 +34,9 @@ import grpc -import rqd.compiled_proto.report_pb2 -import rqd.compiled_proto.report_pb2_grpc -import rqd.compiled_proto.rqd_pb2_grpc +import opencue.compiled_proto.report_pb2 +import opencue.compiled_proto.report_pb2_grpc +import opencue.compiled_proto.rqd_pb2_grpc import rqd.rqconstants import rqd.rqexceptions import rqd.rqdservicers @@ -80,7 +80,7 @@ def __init__(self, rqCore, runFrame): def runningFrameInfo(self): """Returns the RunningFrameInfo object""" - runningFrameInfo = rqd.compiled_proto.report_pb2.RunningFrameInfo( + runningFrameInfo = opencue.compiled_proto.report_pb2.RunningFrameInfo( resource_id=self.runFrame.resource_id, job_id=self.runFrame.job_id, job_name=self.runFrame.job_name, @@ -111,13 +111,13 @@ def _serializeChildrenProcs(self): :param data: dictionary :return: serialized children proc host stats - :rtype: rqd.compiled_proto.report_pb2.ChildrenProcStats + :rtype: opencue.compiled_proto.report_pb2.ChildrenProcStats """ - childrenProc = rqd.compiled_proto.report_pb2.ChildrenProcStats() + childrenProc = opencue.compiled_proto.report_pb2.ChildrenProcStats() for proc, values in self.childrenProcs.items(): - procStats = rqd.compiled_proto.report_pb2.ProcStats() - procStatFile = rqd.compiled_proto.report_pb2.Stat() - procStatmFile = rqd.compiled_proto.report_pb2.Statm() + procStats = opencue.compiled_proto.report_pb2.ProcStats() + procStatFile = opencue.compiled_proto.report_pb2.Stat() + procStatmFile = opencue.compiled_proto.report_pb2.Statm() procStatFile.pid = proc procStatFile.name = values["name"] if values["name"] else "" @@ -194,7 +194,7 @@ def __init__(self, rqCore): def addServicers(self): """Registers the gRPC servicers defined in rqdservicers.py.""" for servicer in self.servicers: - addFunc = getattr(rqd.compiled_proto.rqd_pb2_grpc, 'add_{0}_to_server'.format(servicer)) + addFunc = getattr(opencue.compiled_proto.rqd_pb2_grpc, 'add_{0}_to_server'.format(servicer)) servicerClass = getattr(rqd.rqdservicers, servicer) addFunc(servicerClass(self.rqCore), self.server) @@ -293,25 +293,25 @@ def __getChannel(self): def __getReportStub(self): self.__getChannel() - return rqd.compiled_proto.report_pb2_grpc.RqdReportInterfaceStub(self.channel) + return opencue.compiled_proto.report_pb2_grpc.RqdReportInterfaceStub(self.channel) def reportRqdStartup(self, report): """Wraps the ability to send a startup report to rqd via grpc""" stub = self.__getReportStub() - request = rqd.compiled_proto.report_pb2.RqdReportRqdStartupRequest(boot_report=report) + request = opencue.compiled_proto.report_pb2.RqdReportRqdStartupRequest(boot_report=report) stub.ReportRqdStartup(request, timeout=rqd.rqconstants.RQD_TIMEOUT) def reportStatus(self, report): """Wraps the ability to send a status report to the cuebot via grpc""" stub = self.__getReportStub() - request = rqd.compiled_proto.report_pb2.RqdReportStatusRequest(host_report=report) + request = opencue.compiled_proto.report_pb2.RqdReportStatusRequest(host_report=report) stub.ReportStatus(request, timeout=rqd.rqconstants.RQD_TIMEOUT) def reportRunningFrameCompletion(self, report): """Wraps the ability to send a running frame completion report to the cuebot via grpc""" stub = self.__getReportStub() - request = rqd.compiled_proto.report_pb2.RqdReportRunningFrameCompletionRequest( + request = opencue.compiled_proto.report_pb2.RqdReportRunningFrameCompletionRequest( frame_complete_report=report) stub.ReportRunningFrameCompletion(request, timeout=rqd.rqconstants.RQD_TIMEOUT) From 641e74de91014646aa8936e47acb9a06ae981198 Mon Sep 17 00:00:00 2001 From: Jimmy Christensen Date: Wed, 14 Aug 2024 22:31:38 +0200 Subject: [PATCH 02/32] Move RQDLogger into opencue.cuelogging --- pycue/opencue/cuelogging.py | 123 +++++++++++++++++++++++++++++++++ rqd/rqd/rqcore.py | 8 ++- rqd/rqd/rqlogging.py | 133 ------------------------------------ 3 files changed, 129 insertions(+), 135 deletions(-) create mode 100644 pycue/opencue/cuelogging.py delete mode 100644 rqd/rqd/rqlogging.py diff --git a/pycue/opencue/cuelogging.py b/pycue/opencue/cuelogging.py new file mode 100644 index 000000000..517040896 --- /dev/null +++ b/pycue/opencue/cuelogging.py @@ -0,0 +1,123 @@ + +import logging +import os +import platform +import datetime +import time + +log = logging.getLogger(__name__) + +MODE_READ = 0 +MODE_WRITE = 1 + +class CueLogger(object): + """Class to abstract file logging, this class tries to act as a file object""" + filepath = None + fd = None + type = 0 + mode = 0 # 0 for read, 1 for write + + def __init__(self, filepath, mode, maxLogFiles=1): + """RQDLogger class initialization + @type filepath: string + @param filepath: The filepath to log to + """ + + self.filepath = filepath + self.mode = mode + if self.mode == MODE_WRITE: + log_dir = os.path.dirname(self.filepath) + if not os.access(log_dir, os.F_OK): + # Attempting mkdir for missing logdir + msg = "No Error" + try: + os.makedirs(log_dir) + os.chmod(log_dir, 0o777) + # pylint: disable=broad-except + except Exception as e: + # This is expected to fail when called in abq + # But the directory should now be visible + msg = e + + if not os.access(log_dir, os.F_OK): + err = "Unable to see log directory: %s, mkdir failed with: %s" % ( + log_dir, msg) + raise RuntimeError(err) + + if not os.access(log_dir, os.W_OK): + err = "Unable to write to log directory %s" % log_dir + raise RuntimeError(err) + + try: + # Rotate any old logs to a max of MAX_LOG_FILES: + if os.path.isfile(self.filepath): + rotateCount = 1 + while (os.path.isfile("%s.%s" % (self.filepath, rotateCount)) + and rotateCount < maxLogFiles): + rotateCount += 1 + os.rename(self.filepath, + "%s.%s" % (self.filepath, rotateCount)) + # pylint: disable=broad-except + except Exception as e: + err = "Unable to rotate previous log file due to %s" % e + # Windows might fail while trying to rotate logs for checking if file is + # being used by another process. Frame execution doesn't need to + # be halted for this. + if platform.system() == "Windows": + log.warning(err) + else: + raise RuntimeError(err) + # pylint: disable=consider-using-with + self.fd = open(self.filepath, "w+", 1, encoding='utf-8') + try: + os.chmod(self.filepath, 0o666) + # pylint: disable=broad-except + except Exception as e: + err = "Failed to chmod log file! %s due to %s" % (self.filepath, e) + log.warning(err) + elif mode == MODE_READ: + self.fd = open(self.filepath, "r", encoding='utf-8') + else: + log.error("Unknown mode for log file!") + + # pylint: disable=arguments-differ + def write(self, data, prependTimestamp=False): + """Abstract write function that will write to the correct backend""" + # Only work if in write mode, otherwise ignore + if self.mode == MODE_WRITE: + # Convert data to unicode + if isinstance(data, bytes): + data = data.decode('utf-8', errors='ignore') + if prependTimestamp is True: + lines = data.splitlines() + curr_line_timestamp = datetime.datetime.now().strftime("%H:%M:%S") + for line in lines: + print("[%s] %s" % (curr_line_timestamp, line), file=self) + else: + self.fd.write(data) + + def writelines(self, __lines): + """Provides support for writing mutliple lines at a time""" + for line in __lines: + self.write(line) + + def close(self): + """Closes the file if the backend is file based""" + self.fd.close() + + def waitForFile(self, maxTries=5): + """Waits for the file to exist before continuing when using a file backend""" + # Waits for a file to exist + tries = 0 + while tries < maxTries: + if os.path.exists(self.filepath): + return + tries += 1 + time.sleep(0.5 * tries) + raise IOError("Failed to create %s" % self.filepath) + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + pass diff --git a/rqd/rqd/rqcore.py b/rqd/rqd/rqcore.py index b6e9837dd..e9aa77c55 100644 --- a/rqd/rqd/rqcore.py +++ b/rqd/rqd/rqcore.py @@ -36,6 +36,7 @@ import traceback import select +import opencue.cuelogging import opencue.compiled_proto.host_pb2 import opencue.compiled_proto.report_pb2 import rqd.rqconstants @@ -44,7 +45,6 @@ import rqd.rqnetwork import rqd.rqnimby import rqd.rqutil -import rqd.rqlogging INT32_MAX = 2147483647 INT32_MIN = -2147483648 @@ -520,7 +520,11 @@ def run(self): # Setup proc to allow launching of frame # try: - self.rqlog = rqd.rqlogging.RqdLogger(runFrame.log_dir_file) + self.rqlog = opencue.cuelogging.CueLogger( + runFrame.log_dir_file, + mode=opencue.cuelogging.MODE_WRITE, + maxLogFiles=rqd.rqconstants.MAX_LOG_FILES + ) self.rqlog.waitForFile() # pylint: disable=broad-except except Exception as e: diff --git a/rqd/rqd/rqlogging.py b/rqd/rqd/rqlogging.py deleted file mode 100644 index e8878c0d9..000000000 --- a/rqd/rqd/rqlogging.py +++ /dev/null @@ -1,133 +0,0 @@ -# Copyright Contributors to the OpenCue Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -"""Logging module, handles logging to files and non-files""" - - -import logging -import time -import os -import datetime -import platform - -import rqd.rqconstants - -log = logging.getLogger(__name__) -log.setLevel(rqd.rqconstants.CONSOLE_LOG_LEVEL) - - -class RqdLogger(object): - """Class to abstract file logging, this class tries to act as a file object""" - filepath = None - fd = None - type = 0 - - def __init__(self, filepath): - """RQDLogger class initialization - @type filepath: string - @param filepath: The filepath to log to - """ - - self.filepath = filepath - - log_dir = os.path.dirname(self.filepath) - if not os.access(log_dir, os.F_OK): - # Attempting mkdir for missing logdir - msg = "No Error" - try: - os.makedirs(log_dir) - os.chmod(log_dir, 0o777) - # pylint: disable=broad-except - except Exception as e: - # This is expected to fail when called in abq - # But the directory should now be visible - msg = e - - if not os.access(log_dir, os.F_OK): - err = "Unable to see log directory: %s, mkdir failed with: %s" % ( - log_dir, msg) - raise RuntimeError(err) - - if not os.access(log_dir, os.W_OK): - err = "Unable to write to log directory %s" % log_dir - raise RuntimeError(err) - - try: - # Rotate any old logs to a max of MAX_LOG_FILES: - if os.path.isfile(self.filepath): - rotateCount = 1 - while (os.path.isfile("%s.%s" % (self.filepath, rotateCount)) - and rotateCount < rqd.rqconstants.MAX_LOG_FILES): - rotateCount += 1 - os.rename(self.filepath, - "%s.%s" % (self.filepath, rotateCount)) - # pylint: disable=broad-except - except Exception as e: - err = "Unable to rotate previous log file due to %s" % e - # Windows might fail while trying to rotate logs for checking if file is - # being used by another process. Frame execution doesn't need to - # be halted for this. - if platform.system() == "Windows": - log.warning(err) - else: - raise RuntimeError(err) - # pylint: disable=consider-using-with - self.fd = open(self.filepath, "w+", 1, encoding='utf-8') - try: - os.chmod(self.filepath, 0o666) - # pylint: disable=broad-except - except Exception as e: - err = "Failed to chmod log file! %s due to %s" % (self.filepath, e) - log.warning(err) - - # pylint: disable=arguments-differ - def write(self, data, prependTimestamp=False): - """Abstract write function that will write to the correct backend""" - # Convert data to unicode - if isinstance(data, bytes): - data = data.decode('utf-8', errors='ignore') - if prependTimestamp is True: - lines = data.splitlines() - curr_line_timestamp = datetime.datetime.now().strftime("%H:%M:%S") - for line in lines: - print("[%s] %s" % (curr_line_timestamp, line), file=self) - else: - self.fd.write(data) - - def writelines(self, __lines): - """Provides support for writing mutliple lines at a time""" - for line in __lines: - self.write(line) - - def close(self): - """Closes the file if the backend is file based""" - self.fd.close() - - def waitForFile(self, maxTries=5): - """Waits for the file to exist before continuing when using a file backend""" - # Waits for a file to exist - tries = 0 - while tries < maxTries: - if os.path.exists(self.filepath): - return - tries += 1 - time.sleep(0.5 * tries) - raise IOError("Failed to create %s" % self.filepath) - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - pass From 4612cc0b45647da06338fcc790662de8bb0fca3a Mon Sep 17 00:00:00 2001 From: Jimmy Christensen Date: Wed, 14 Aug 2024 22:39:37 +0200 Subject: [PATCH 03/32] Replace LogViewPlugin.LogReader with opencue.cuelogging.CueLogger --- cuegui/cuegui/plugins/LogViewPlugin.py | 40 ++------------------------ pycue/opencue/cuelogging.py | 23 +++++++++++++++ 2 files changed, 25 insertions(+), 38 deletions(-) diff --git a/cuegui/cuegui/plugins/LogViewPlugin.py b/cuegui/cuegui/plugins/LogViewPlugin.py index d69f5cf80..daee12ae0 100644 --- a/cuegui/cuegui/plugins/LogViewPlugin.py +++ b/cuegui/cuegui/plugins/LogViewPlugin.py @@ -35,7 +35,7 @@ import cuegui.Constants import cuegui.AbstractDockWidget - +import opencue.cuelogging PLUGIN_NAME = 'LogView' PLUGIN_CATEGORY = 'Other' @@ -44,42 +44,6 @@ PRINTABLE = set(string.printable) -class LogReader(object): - """ - Custom class to abstract reading log files from multiple backends - """ - filepath = None - type = None - - def __init__(self, filepath): - """LogReader class initialization - @type filepath: string - @param filepath: The filepath to log to - """ - self.filepath = filepath - - def size(self): - """Return the size of the file""" - return int(os.stat(self.filepath).st_size) - - def getMtime(self): - """Return modification time of the file""" - return os.path.getmtime(self.filepath) - - def exists(self): - """Check if the file exists""" - return os.path.exists(self.filepath) - - def read(self): - """Read the data from the backend""" - content = None - if self.exists() is True: - with open(self.filepath, "r", encoding='utf-8') as fp: - content = fp.read() - - return content - - class LineNumberArea(QtWidgets.QWidget): """ Custom widget for the line numbers. This widget is designed to be attached @@ -849,7 +813,7 @@ def _display_log_content(self): @postcondition: The _update_log method is scheduled to run again after 5 seconds """ - log_reader = LogReader(self._log_file) + log_reader = opencue.cuelogging.CueLogger(self._log_file, mode=opencue.cuelogging.MODE_READ) try: if log_reader.exists() is not True: diff --git a/pycue/opencue/cuelogging.py b/pycue/opencue/cuelogging.py index 517040896..86e60fbd3 100644 --- a/pycue/opencue/cuelogging.py +++ b/pycue/opencue/cuelogging.py @@ -116,6 +116,29 @@ def waitForFile(self, maxTries=5): time.sleep(0.5 * tries) raise IOError("Failed to create %s" % self.filepath) + def size(self): + """Return the size of the file""" + return int(os.stat(self.filepath).st_size) + + def getMtime(self): + """Return modification time of the file""" + return os.path.getmtime(self.filepath) + + def exists(self): + """Check if the file exists""" + return os.path.exists(self.filepath) + + def read(self): + """Read the data from the backend""" + # Only allow reading when in read mode + if self.mode == MODE_READ: + content = None + if self.exists() is True: + with open(self.filepath, "r", encoding='utf-8') as fp: + content = fp.read() + + return content + def __enter__(self): return self From 9a7742f9c883f305d661c4d1c93e375313dc7fa5 Mon Sep 17 00:00:00 2001 From: Jimmy Christensen Date: Wed, 14 Aug 2024 22:46:59 +0200 Subject: [PATCH 04/32] Small change to default mode --- pycue/opencue/cuelogging.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pycue/opencue/cuelogging.py b/pycue/opencue/cuelogging.py index 86e60fbd3..4385ccc44 100644 --- a/pycue/opencue/cuelogging.py +++ b/pycue/opencue/cuelogging.py @@ -15,7 +15,7 @@ class CueLogger(object): filepath = None fd = None type = 0 - mode = 0 # 0 for read, 1 for write + mode = MODE_READ def __init__(self, filepath, mode, maxLogFiles=1): """RQDLogger class initialization From d87dacb609010a968d888da919db91e83e8cb4b5 Mon Sep 17 00:00:00 2001 From: Jimmy Christensen Date: Wed, 14 Aug 2024 22:55:11 +0200 Subject: [PATCH 05/32] Remove rqd/compiled_proto from tests and documentation --- ci/python_coverage_report.sh | 4 +--- ci/run_python_lint.sh | 4 +--- ci/run_python_tests.sh | 2 -- ci/run_python_tests_pyside6.sh | 2 -- proto/README.md | 18 ------------------ rqd/Dockerfile | 10 ---------- 6 files changed, 2 insertions(+), 38 deletions(-) diff --git a/ci/python_coverage_report.sh b/ci/python_coverage_report.sh index e21f0259e..dbdb88cc7 100755 --- a/ci/python_coverage_report.sh +++ b/ci/python_coverage_report.sh @@ -6,9 +6,7 @@ pip install --user -r requirements.txt -r requirements_gui.txt # Protos need to have their Python code generated in order for tests to pass. python -m grpc_tools.protoc -I=proto/ --python_out=pycue/opencue/compiled_proto --grpc_python_out=pycue/opencue/compiled_proto proto/*.proto -python -m grpc_tools.protoc -I=proto/ --python_out=rqd/rqd/compiled_proto --grpc_python_out=rqd/rqd/compiled_proto proto/*.proto 2to3 -wn -f import pycue/opencue/compiled_proto/*_pb2*.py -2to3 -wn -f import rqd/rqd/compiled_proto/*_pb2*.py # Run coverage for each component individually, but append it all into the same report. coverage run --source=pycue/opencue/,pycue/FileSequence/ --omit=pycue/opencue/compiled_proto/* pycue/setup.py test @@ -16,7 +14,7 @@ PYTHONPATH=pycue coverage run -a --source=pyoutline/outline/ pyoutline/setup.py PYTHONPATH=pycue coverage run -a --source=cueadmin/cueadmin/ cueadmin/setup.py test PYTHONPATH=pycue xvfb-run -d coverage run -a --source=cuegui/cuegui/ cuegui/setup.py test PYTHONPATH=pycue:pyoutline coverage run -a --source=cuesubmit/cuesubmit/ cuesubmit/setup.py test -coverage run -a --source=rqd/rqd/ --omit=rqd/rqd/compiled_proto/* rqd/setup.py test +coverage run -a --source=rqd/rqd/ --omit=pycue/opencue/compiled_proto/* rqd/setup.py test # SonarCloud needs the report in XML. coverage xml diff --git a/ci/run_python_lint.sh b/ci/run_python_lint.sh index bd86c9188..70fe861bf 100755 --- a/ci/run_python_lint.sh +++ b/ci/run_python_lint.sh @@ -9,12 +9,10 @@ pip install --user -r requirements.txt -r requirements_gui.txt # Protos need to have their Python code generated in order for tests to pass. python -m grpc_tools.protoc -I=proto/ --python_out=pycue/opencue/compiled_proto --grpc_python_out=pycue/opencue/compiled_proto proto/*.proto -python -m grpc_tools.protoc -I=proto/ --python_out=rqd/rqd/compiled_proto --grpc_python_out=rqd/rqd/compiled_proto proto/*.proto # Fix imports to work in both Python 2 and 3. See # for more info. 2to3 -wn -f import pycue/opencue/compiled_proto/*_pb2*.py -2to3 -wn -f import rqd/rqd/compiled_proto/*_pb2*.py echo "Running lint for pycue/..." cd pycue @@ -49,6 +47,6 @@ cd .. echo "Running lint for rqd/..." cd rqd -python -m pylint --rcfile=../ci/pylintrc_main rqd --ignore=rqd/compiled_proto +python -m pylint --rcfile=../ci/pylintrc_main rqd --ignore=pycue/opencue/compiled_proto python -m pylint --rcfile=../ci/pylintrc_test tests cd .. diff --git a/ci/run_python_tests.sh b/ci/run_python_tests.sh index 5f1bfe294..048300c3b 100755 --- a/ci/run_python_tests.sh +++ b/ci/run_python_tests.sh @@ -15,12 +15,10 @@ pip install --user -r requirements.txt -r requirements_gui.txt # Protos need to have their Python code generated in order for tests to pass. python -m grpc_tools.protoc -I=proto/ --python_out=pycue/opencue/compiled_proto --grpc_python_out=pycue/opencue/compiled_proto proto/*.proto -python -m grpc_tools.protoc -I=proto/ --python_out=rqd/rqd/compiled_proto --grpc_python_out=rqd/rqd/compiled_proto proto/*.proto # Fix imports to work in both Python 2 and 3. See # for more info. 2to3 -wn -f import pycue/opencue/compiled_proto/*_pb2*.py -2to3 -wn -f import rqd/rqd/compiled_proto/*_pb2*.py python pycue/setup.py test PYTHONPATH=pycue python pyoutline/setup.py test diff --git a/ci/run_python_tests_pyside6.sh b/ci/run_python_tests_pyside6.sh index 384841cfe..8f4c46941 100755 --- a/ci/run_python_tests_pyside6.sh +++ b/ci/run_python_tests_pyside6.sh @@ -33,11 +33,9 @@ python3 -m pip install --user PySide6==6.3.2 # Protos need to have their Python code generated in order for tests to pass. python -m grpc_tools.protoc -I=proto/ --python_out=pycue/opencue/compiled_proto --grpc_python_out=pycue/opencue/compiled_proto proto/*.proto -python -m grpc_tools.protoc -I=proto/ --python_out=rqd/rqd/compiled_proto --grpc_python_out=rqd/rqd/compiled_proto proto/*.proto # Fix compiled proto code for Python 3. 2to3 -wn -f import pycue/opencue/compiled_proto/*_pb2*.py -2to3 -wn -f import rqd/rqd/compiled_proto/*_pb2*.py python pycue/setup.py test PYTHONPATH=pycue python pyoutline/setup.py test diff --git a/proto/README.md b/proto/README.md index 0b039b462..cc7361bc6 100644 --- a/proto/README.md +++ b/proto/README.md @@ -8,24 +8,6 @@ To use them, they must first be compiled into the native language of the compone Gradle automatically compiles these proto files, no further action is needed. -## RQD - -To generate: - -```sh -python -m grpc_tools.protoc -I=. --python_out=../rqd/rqd/compiled_proto --grpc_python_out=../rqd/rqd/compiled_proto ./*.proto -2to3 -wn -f import ../rqd/rqd/compiled_proto/*_pb2*.py -``` - -For Windows (Powershell): - -```powershell -python -m grpc_tools.protoc --proto_path=. --python_out=../rqd/rqd/compiled_proto --grpc_python_out=../rqd/rqd/compiled_proto (ls *.proto).Name -cd ..\rqd\rqd\compiled_proto\ -2to3 -wn -f import (ls *_pb2*.py).Name -``` - - ## pycue To generate: diff --git a/rqd/Dockerfile b/rqd/Dockerfile index c3a0a0dc1..d3c1bca9f 100644 --- a/rqd/Dockerfile +++ b/rqd/Dockerfile @@ -27,16 +27,6 @@ COPY rqd/setup.py ./rqd/ COPY rqd/tests/ ./rqd/tests COPY rqd/rqd/ ./rqd/rqd -RUN python3.9 -m grpc_tools.protoc \ - -I=./proto \ - --python_out=./rqd/rqd/compiled_proto \ - --grpc_python_out=./rqd/rqd/compiled_proto \ - ./proto/*.proto - -# Fix imports to work in both Python 2 and 3. See -# for more info. -RUN 2to3 -wn -f import rqd/rqd/compiled_proto/*_pb2*.py - COPY VERSION.in VERSIO[N] ./ RUN test -e VERSION || echo "$(cat VERSION.in)" | tee VERSION From c27b60874685089d637a2b28ddd9ec6da94940ab Mon Sep 17 00:00:00 2001 From: Jimmy Christensen Date: Thu, 15 Aug 2024 00:04:11 +0200 Subject: [PATCH 06/32] Remove rqd/compiled_proto from tests and documentation --- ci/pylintrc_main | 5 ++--- ci/pylintrc_test | 5 ++--- pycue/opencue/cuelogging.py | 16 ++++++++++++++++ rqd/tests/cuerqd_tests.py | 6 +++--- rqd/tests/test_cuebot_listener.py | 6 +++--- 5 files changed, 26 insertions(+), 12 deletions(-) diff --git a/ci/pylintrc_main b/ci/pylintrc_main index cdd7f1ab6..b53ad2ae5 100644 --- a/ci/pylintrc_main +++ b/ci/pylintrc_main @@ -206,9 +206,8 @@ ignored-classes=optparse.Values,thread._local,_thread._local ignored-modules=opencue.compiled_proto, opencue.compiled_proto.filter_pb2, opencue.compiled_proto.host_pb2, - rqd.compiled_proto.rqd_pb2, - rqd.compiled_proto.host_pb2, - rqd.compiled_proto.report_pb2 + opencue.compiled_proto.rqd_pb2, + opencue.compiled_proto.report_pb2 # Show a hint with possible names when a member name was not found. The aspect # of finding the hint is based on edit distance. diff --git a/ci/pylintrc_test b/ci/pylintrc_test index 434a1d754..6578bab2b 100644 --- a/ci/pylintrc_test +++ b/ci/pylintrc_test @@ -206,9 +206,8 @@ ignored-classes=optparse.Values,thread._local,_thread._local ignored-modules=opencue.compiled_proto, opencue.compiled_proto.filter_pb2, opencue.compiled_proto.host_pb2, - rqd.compiled_proto.rqd_pb2, - rqd.compiled_proto.host_pb2, - rqd.compiled_proto.report_pb2 + opencue.compiled_proto.rqd_pb2, + opencue.compiled_proto.report_pb2 # Show a hint with possible names when a member name was not found. The aspect # of finding the hint is based on edit distance. diff --git a/pycue/opencue/cuelogging.py b/pycue/opencue/cuelogging.py index 4385ccc44..620f379c4 100644 --- a/pycue/opencue/cuelogging.py +++ b/pycue/opencue/cuelogging.py @@ -1,3 +1,19 @@ +# Copyright Contributors to the OpenCue Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +"""Module for reading and writing log files""" import logging import os diff --git a/rqd/tests/cuerqd_tests.py b/rqd/tests/cuerqd_tests.py index ad647d770..18b646a45 100644 --- a/rqd/tests/cuerqd_tests.py +++ b/rqd/tests/cuerqd_tests.py @@ -29,15 +29,15 @@ import mock import rqd.cuerqd -import rqd.compiled_proto.rqd_pb2 +import opencue.compiled_proto.rqd_pb2 SCRIPT_NAME = '/arbitrary/path/to/script' RQD_HOSTNAME = 'arbitrary-rqd-hostname' -@mock.patch('rqd.compiled_proto.rqd_pb2_grpc.RunningFrameStub') -@mock.patch('rqd.compiled_proto.rqd_pb2_grpc.RqdInterfaceStub') +@mock.patch('opencue.compiled_proto.rqd_pb2_grpc.RunningFrameStub') +@mock.patch('opencue.compiled_proto.rqd_pb2_grpc.RqdInterfaceStub') @mock.patch('grpc.insecure_channel', new=mock.MagicMock()) class CueRqdTests(unittest.TestCase): diff --git a/rqd/tests/test_cuebot_listener.py b/rqd/tests/test_cuebot_listener.py index 160951f05..90e7dcc5a 100644 --- a/rqd/tests/test_cuebot_listener.py +++ b/rqd/tests/test_cuebot_listener.py @@ -32,7 +32,7 @@ import grpc -import rqd.compiled_proto.report_pb2_grpc +import opencue.compiled_proto.report_pb2_grpc import rqd.rqconstants @@ -46,7 +46,7 @@ def __init__(self): def addServicers(self): addFunc = getattr( - rqd.compiled_proto.report_pb2_grpc, 'add_{0}_to_server'.format(self.servicerName)) + opencue.compiled_proto.report_pb2_grpc, 'add_{0}_to_server'.format(self.servicerName)) servicerClass = globals()[self.servicerName] self.servicer = servicerClass() addFunc(self.servicer, self.server) @@ -70,7 +70,7 @@ def stayAlive(self): self.server.stop(0) -class RqdReportInterfaceServicer(rqd.compiled_proto.report_pb2_grpc.RqdReportInterfaceServicer): +class RqdReportInterfaceServicer(opencue.compiled_proto.report_pb2_grpc.RqdReportInterfaceServicer): """Test class implements RqdReportStatic interface. Received reports are stored in the variables listed below. Create as an object to connect. From beaf975e848aa3727ac33742ff93aed3a33d8865 Mon Sep 17 00:00:00 2001 From: Jimmy Christensen Date: Thu, 15 Aug 2024 00:10:11 +0200 Subject: [PATCH 07/32] Some linting fixes --- cuegui/cuegui/plugins/LogViewPlugin.py | 3 +-- pycue/opencue/cuelogging.py | 5 +++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cuegui/cuegui/plugins/LogViewPlugin.py b/cuegui/cuegui/plugins/LogViewPlugin.py index daee12ae0..ba0a2d6d1 100644 --- a/cuegui/cuegui/plugins/LogViewPlugin.py +++ b/cuegui/cuegui/plugins/LogViewPlugin.py @@ -22,7 +22,6 @@ from builtins import str from builtins import range -import os import re import string import sys @@ -33,9 +32,9 @@ from qtpy import QtCore from qtpy import QtWidgets +import opencue.cuelogging import cuegui.Constants import cuegui.AbstractDockWidget -import opencue.cuelogging PLUGIN_NAME = 'LogView' PLUGIN_CATEGORY = 'Other' diff --git a/pycue/opencue/cuelogging.py b/pycue/opencue/cuelogging.py index 620f379c4..868cde035 100644 --- a/pycue/opencue/cuelogging.py +++ b/pycue/opencue/cuelogging.py @@ -147,13 +147,14 @@ def exists(self): def read(self): """Read the data from the backend""" # Only allow reading when in read mode + + content = None if self.mode == MODE_READ: - content = None if self.exists() is True: with open(self.filepath, "r", encoding='utf-8') as fp: content = fp.read() - return content + return content def __enter__(self): return self From d13e589e03c8f556993e6fa996b36bd3993ee76d Mon Sep 17 00:00:00 2001 From: Jimmy Christensen Date: Thu, 15 Aug 2024 00:21:32 +0200 Subject: [PATCH 08/32] Fix more tests --- ci/run_python_lint.sh | 4 +- ci/run_python_tests.sh | 2 +- rqd/tests/cuerqd_tests.py | 30 +++++----- rqd/tests/rqconstants_tests.py | 4 +- rqd/tests/rqcore_tests.py | 104 ++++++++++++++++----------------- rqd/tests/rqmachine_tests.py | 18 +++--- 6 files changed, 81 insertions(+), 81 deletions(-) diff --git a/ci/run_python_lint.sh b/ci/run_python_lint.sh index 70fe861bf..f950a7a4c 100755 --- a/ci/run_python_lint.sh +++ b/ci/run_python_lint.sh @@ -47,6 +47,6 @@ cd .. echo "Running lint for rqd/..." cd rqd -python -m pylint --rcfile=../ci/pylintrc_main rqd --ignore=pycue/opencue/compiled_proto -python -m pylint --rcfile=../ci/pylintrc_test tests +PYTHONPATH=../pycue python -m pylint --rcfile=../ci/pylintrc_main rqd --ignore=pycue/opencue/compiled_proto +PYTHONPATH=../pycue python -m pylint --rcfile=../ci/pylintrc_test tests cd .. diff --git a/ci/run_python_tests.sh b/ci/run_python_tests.sh index 048300c3b..bf0d7506d 100755 --- a/ci/run_python_tests.sh +++ b/ci/run_python_tests.sh @@ -24,7 +24,7 @@ python pycue/setup.py test PYTHONPATH=pycue python pyoutline/setup.py test PYTHONPATH=pycue python cueadmin/setup.py test PYTHONPATH=pycue:pyoutline python cuesubmit/setup.py test -python rqd/setup.py test +PYTHONPATH=pycue python rqd/setup.py test # Xvfb no longer supports Python 2. if [[ "$python_version" =~ "Python 3" && ${args[0]} != "--no-gui" ]]; then diff --git a/rqd/tests/cuerqd_tests.py b/rqd/tests/cuerqd_tests.py index 18b646a45..b8e42866c 100644 --- a/rqd/tests/cuerqd_tests.py +++ b/rqd/tests/cuerqd_tests.py @@ -59,7 +59,7 @@ def test_initWithLocalhost(self, rqdHostMock, stubMock, frameStubMock): def test_status(self, stubMock, frameStubMock): sys.argv = [SCRIPT_NAME, RQD_HOSTNAME, '-s'] - statusRequest = rqd.compiled_proto.rqd_pb2.RqdStaticReportStatusRequest() + statusRequest = opencue.compiled_proto.rqd_pb2.RqdStaticReportStatusRequest() rqd.cuerqd.main() @@ -68,7 +68,7 @@ def test_status(self, stubMock, frameStubMock): def test_getRunningFrame(self, stubMock, frameStubMock): frameId = 'arbitrary-frame-id' sys.argv = [SCRIPT_NAME, RQD_HOSTNAME, '--getproxy', frameId] - runFrameRequest = rqd.compiled_proto.rqd_pb2.RqdStaticGetRunFrameRequest(frame_id=frameId) + runFrameRequest = opencue.compiled_proto.rqd_pb2.RqdStaticGetRunFrameRequest(frame_id=frameId) rqd.cuerqd.main() @@ -76,7 +76,7 @@ def test_getRunningFrame(self, stubMock, frameStubMock): def test_nimbyOff(self, stubMock, frameStubMock): sys.argv = [SCRIPT_NAME, RQD_HOSTNAME, '--nimbyoff'] - nimbyOffRequest = rqd.compiled_proto.rqd_pb2.RqdStaticNimbyOffRequest() + nimbyOffRequest = opencue.compiled_proto.rqd_pb2.RqdStaticNimbyOffRequest() rqd.cuerqd.main() @@ -84,7 +84,7 @@ def test_nimbyOff(self, stubMock, frameStubMock): def test_nimbyOn(self, stubMock, frameStubMock): sys.argv = [SCRIPT_NAME, RQD_HOSTNAME, '--nimbyon'] - nimbyOnRequest = rqd.compiled_proto.rqd_pb2.RqdStaticNimbyOnRequest() + nimbyOnRequest = opencue.compiled_proto.rqd_pb2.RqdStaticNimbyOnRequest() rqd.cuerqd.main() @@ -92,7 +92,7 @@ def test_nimbyOn(self, stubMock, frameStubMock): def test_lockAll(self, stubMock, frameStubMock): sys.argv = [SCRIPT_NAME, RQD_HOSTNAME, '--lh'] - lockAllRequest = rqd.compiled_proto.rqd_pb2.RqdStaticLockAllRequest() + lockAllRequest = opencue.compiled_proto.rqd_pb2.RqdStaticLockAllRequest() rqd.cuerqd.main() @@ -100,7 +100,7 @@ def test_lockAll(self, stubMock, frameStubMock): def test_unlockAll(self, stubMock, frameStubMock): sys.argv = [SCRIPT_NAME, RQD_HOSTNAME, '--ulh'] - unlockAllRequest = rqd.compiled_proto.rqd_pb2.RqdStaticUnlockAllRequest() + unlockAllRequest = opencue.compiled_proto.rqd_pb2.RqdStaticUnlockAllRequest() rqd.cuerqd.main() @@ -109,7 +109,7 @@ def test_unlockAll(self, stubMock, frameStubMock): def test_lock(self, stubMock, frameStubMock): numCoresToLock = 85 sys.argv = [SCRIPT_NAME, RQD_HOSTNAME, '--lp', str(numCoresToLock)] - lockRequest = rqd.compiled_proto.rqd_pb2.RqdStaticLockRequest(cores=numCoresToLock) + lockRequest = opencue.compiled_proto.rqd_pb2.RqdStaticLockRequest(cores=numCoresToLock) rqd.cuerqd.main() @@ -118,7 +118,7 @@ def test_lock(self, stubMock, frameStubMock): def test_unlock(self, stubMock, frameStubMock): numCoresToUnlock = 52 sys.argv = [SCRIPT_NAME, RQD_HOSTNAME, '--ulp', str(numCoresToUnlock)] - unlockRequest = rqd.compiled_proto.rqd_pb2.RqdStaticUnlockRequest(cores=numCoresToUnlock) + unlockRequest = opencue.compiled_proto.rqd_pb2.RqdStaticUnlockRequest(cores=numCoresToUnlock) rqd.cuerqd.main() @@ -126,7 +126,7 @@ def test_unlock(self, stubMock, frameStubMock): def test_shutdownRqdIdle(self, stubMock, frameStubMock): sys.argv = [SCRIPT_NAME, RQD_HOSTNAME, '--exit'] - shutdownIdleRequest = rqd.compiled_proto.rqd_pb2.RqdStaticShutdownIdleRequest() + shutdownIdleRequest = opencue.compiled_proto.rqd_pb2.RqdStaticShutdownIdleRequest() rqd.cuerqd.main() @@ -134,7 +134,7 @@ def test_shutdownRqdIdle(self, stubMock, frameStubMock): def test_shutdownRqdNow(self, stubMock, frameStubMock): sys.argv = [SCRIPT_NAME, RQD_HOSTNAME, '--exit_now'] - shutdownNowRequest = rqd.compiled_proto.rqd_pb2.RqdStaticShutdownNowRequest() + shutdownNowRequest = opencue.compiled_proto.rqd_pb2.RqdStaticShutdownNowRequest() rqd.cuerqd.main() @@ -142,7 +142,7 @@ def test_shutdownRqdNow(self, stubMock, frameStubMock): def test_rebootIdle(self, stubMock, frameStubMock): sys.argv = [SCRIPT_NAME, RQD_HOSTNAME, '--reboot'] - rebootIdleRequest = rqd.compiled_proto.rqd_pb2.RqdStaticRebootIdleRequest() + rebootIdleRequest = opencue.compiled_proto.rqd_pb2.RqdStaticRebootIdleRequest() rqd.cuerqd.main() @@ -150,18 +150,18 @@ def test_rebootIdle(self, stubMock, frameStubMock): def test_rebootNow(self, stubMock, frameStubMock): sys.argv = [SCRIPT_NAME, RQD_HOSTNAME, '--reboot_now'] - rebootNowRequest = rqd.compiled_proto.rqd_pb2.RqdStaticRebootNowRequest() + rebootNowRequest = opencue.compiled_proto.rqd_pb2.RqdStaticRebootNowRequest() rqd.cuerqd.main() stubMock.return_value.RebootNow.assert_called_with(rebootNowRequest) def test_launchFrame(self, stubMock, frameStubMock): - runFrame = rqd.compiled_proto.rqd_pb2.RunFrame() + runFrame = opencue.compiled_proto.rqd_pb2.RunFrame() runFrame.job_id = "SD6F3S72DJ26236KFS" runFrame.job_name = "edu-trn_job-name" runFrame.frame_id = "FD1S3I154O646UGSNN" - runFrameRequest = rqd.compiled_proto.rqd_pb2.RqdStaticLaunchFrameRequest(run_frame=runFrame) + runFrameRequest = opencue.compiled_proto.rqd_pb2.RqdStaticLaunchFrameRequest(run_frame=runFrame) rqdHost = rqd.cuerqd.RqdHost(RQD_HOSTNAME) rqdHost.active = False @@ -171,7 +171,7 @@ def test_launchFrame(self, stubMock, frameStubMock): def test_killFrame(self, stubMock, frameStubMock): frameId = 'arbitrary-frame-id' - runFrame = rqd.compiled_proto.rqd_pb2.RunFrame(frame_id=frameId) + runFrame = opencue.compiled_proto.rqd_pb2.RunFrame(frame_id=frameId) stubMock.return_value.GetRunFrame.return_value = runFrame sys.argv = [SCRIPT_NAME, RQD_HOSTNAME, '--kill', frameId] diff --git a/rqd/tests/rqconstants_tests.py b/rqd/tests/rqconstants_tests.py index 45e52c0b1..5a13651af 100644 --- a/rqd/tests/rqconstants_tests.py +++ b/rqd/tests/rqconstants_tests.py @@ -37,7 +37,7 @@ import rqd.rqmachine import rqd.rqnimby import rqd.rqutil -import rqd.compiled_proto.report_pb2 +import opencue.compiled_proto.report_pb2 from .rqmachine_tests import ( CPUINFO, @@ -109,7 +109,7 @@ def makeRqMachine(self): rqCore.nimby = nimby nimby.active = False nimby.locked = False - coreDetail = rqd.compiled_proto.report_pb2.CoreDetail(total_cores=2) + coreDetail = opencue.compiled_proto.report_pb2.CoreDetail(total_cores=2) machine = rqd.rqmachine.Machine(rqCore, coreDetail) machine.renderHost = machine.__dict__["_Machine__renderHost"] diff --git a/rqd/tests/rqcore_tests.py b/rqd/tests/rqcore_tests.py index 09f06d23f..6fe67bfee 100644 --- a/rqd/tests/rqcore_tests.py +++ b/rqd/tests/rqcore_tests.py @@ -28,9 +28,9 @@ import mock import pyfakefs.fake_filesystem_unittest -import rqd.compiled_proto.host_pb2 -import rqd.compiled_proto.report_pb2 -import rqd.compiled_proto.rqd_pb2 +import opencue.compiled_proto.host_pb2 +import opencue.compiled_proto.report_pb2 +import opencue.compiled_proto.rqd_pb2 import rqd.rqconstants import rqd.rqcore import rqd.rqexceptions @@ -211,13 +211,13 @@ def test_killAllFrame(self): frame2Id = 'frame2' frame3Id = 'frame3' frame1 = rqd.rqnetwork.RunningFrame( - self.rqcore, rqd.compiled_proto.rqd_pb2.RunFrame(frame_id=frame1Id)) + self.rqcore, opencue.compiled_proto.rqd_pb2.RunFrame(frame_id=frame1Id)) frame1.frameAttendantThread = frameAttendantThread frame2 = rqd.rqnetwork.RunningFrame( - self.rqcore, rqd.compiled_proto.rqd_pb2.RunFrame(frame_id=frame2Id)) + self.rqcore, opencue.compiled_proto.rqd_pb2.RunFrame(frame_id=frame2Id)) frame2.frameAttendantThread = frameAttendantThread frame3 = rqd.rqnetwork.RunningFrame( - self.rqcore, rqd.compiled_proto.rqd_pb2.RunFrame(frame_id=frame3Id)) + self.rqcore, opencue.compiled_proto.rqd_pb2.RunFrame(frame_id=frame3Id)) frame3.frameAttendantThread = frameAttendantThread self.rqcore.storeFrame(frame1Id, frame1) self.rqcore.storeFrame(frame2Id, frame2) @@ -234,10 +234,10 @@ def test_killAllFrameIgnoreNimby(self): frame1Id = 'frame1' frame2Id = 'frame2' frame1 = rqd.rqnetwork.RunningFrame( - self.rqcore, rqd.compiled_proto.rqd_pb2.RunFrame(frame_id=frame1Id)) + self.rqcore, opencue.compiled_proto.rqd_pb2.RunFrame(frame_id=frame1Id)) frame1.frameAttendantThread = frameAttendantThread frame2 = rqd.rqnetwork.RunningFrame( - self.rqcore, rqd.compiled_proto.rqd_pb2.RunFrame(frame_id=frame2Id, ignore_nimby=True)) + self.rqcore, opencue.compiled_proto.rqd_pb2.RunFrame(frame_id=frame2Id, ignore_nimby=True)) frame2.frameAttendantThread = frameAttendantThread self.rqcore.storeFrame(frame1Id, frame1) self.rqcore.storeFrame(frame2Id, frame2) @@ -250,7 +250,7 @@ def test_releaseCores(self): num_idle_cores = 10 num_booked_cores = 7 num_cores_to_release = 5 - self.rqcore.cores = rqd.compiled_proto.report_pb2.CoreDetail( + self.rqcore.cores = opencue.compiled_proto.report_pb2.CoreDetail( total_cores=50, idle_cores=num_idle_cores, locked_cores=2, booked_cores=num_booked_cores) @@ -282,10 +282,10 @@ def test_handleExit(self, networkMock, exitMock): @mock.patch('rqd.rqcore.FrameAttendantThread') def test_launchFrame(self, frameThreadMock): - self.rqcore.cores = rqd.compiled_proto.report_pb2.CoreDetail(total_cores=100, idle_cores=20) - self.machineMock.return_value.state = rqd.compiled_proto.host_pb2.UP + self.rqcore.cores = opencue.compiled_proto.report_pb2.CoreDetail(total_cores=100, idle_cores=20) + self.machineMock.return_value.state = opencue.compiled_proto.host_pb2.UP self.nimbyMock.return_value.locked = False - frame = rqd.compiled_proto.rqd_pb2.RunFrame(uid=22, num_cores=10) + frame = opencue.compiled_proto.rqd_pb2.RunFrame(uid=22, num_cores=10) rqd.rqconstants.OVERRIDE_NIMBY = None self.rqcore.launchFrame(frame) @@ -293,16 +293,16 @@ def test_launchFrame(self, frameThreadMock): frameThreadMock.return_value.start.assert_called() def test_launchFrameOnDownHost(self): - self.machineMock.return_value.state = rqd.compiled_proto.host_pb2.DOWN - frame = rqd.compiled_proto.rqd_pb2.RunFrame() + self.machineMock.return_value.state = opencue.compiled_proto.host_pb2.DOWN + frame = opencue.compiled_proto.rqd_pb2.RunFrame() with self.assertRaises(rqd.rqexceptions.CoreReservationFailureException): self.rqcore.launchFrame(frame) def test_launchFrameOnHostWaitingForShutdown(self): - self.machineMock.return_value.state = rqd.compiled_proto.host_pb2.UP + self.machineMock.return_value.state = opencue.compiled_proto.host_pb2.UP self.nimbyMock.return_value.active = False - frame = rqd.compiled_proto.rqd_pb2.RunFrame() + frame = opencue.compiled_proto.rqd_pb2.RunFrame() self.rqcore.shutdownRqdIdle() with self.assertRaises(rqd.rqexceptions.CoreReservationFailureException): @@ -310,10 +310,10 @@ def test_launchFrameOnHostWaitingForShutdown(self): @mock.patch('rqd.rqcore.FrameAttendantThread') def test_launchFrameOnNimbyHost(self, frameThreadMock): - self.rqcore.cores = rqd.compiled_proto.report_pb2.CoreDetail(total_cores=100, idle_cores=20) - self.machineMock.return_value.state = rqd.compiled_proto.host_pb2.UP - frame = rqd.compiled_proto.rqd_pb2.RunFrame(uid=22, num_cores=10) - frameIgnoreNimby = rqd.compiled_proto.rqd_pb2.RunFrame( + self.rqcore.cores = opencue.compiled_proto.report_pb2.CoreDetail(total_cores=100, idle_cores=20) + self.machineMock.return_value.state = opencue.compiled_proto.host_pb2.UP + frame = opencue.compiled_proto.rqd_pb2.RunFrame(uid=22, num_cores=10) + frameIgnoreNimby = opencue.compiled_proto.rqd_pb2.RunFrame( uid=22, num_cores=10, ignore_nimby=True) self.rqcore.nimby = mock.create_autospec(rqd.rqnimby.NimbySelect) self.rqcore.nimby.locked = True @@ -326,45 +326,45 @@ def test_launchFrameOnNimbyHost(self, frameThreadMock): frameThreadMock.return_value.start.assert_called() def test_launchDuplicateFrame(self): - self.rqcore.cores = rqd.compiled_proto.report_pb2.CoreDetail(total_cores=100, idle_cores=20) - self.machineMock.return_value.state = rqd.compiled_proto.host_pb2.UP + self.rqcore.cores = opencue.compiled_proto.report_pb2.CoreDetail(total_cores=100, idle_cores=20) + self.machineMock.return_value.state = opencue.compiled_proto.host_pb2.UP self.nimbyMock.return_value.locked = False frameId = 'arbitrary-frame-id' - self.rqcore.storeFrame(frameId, rqd.compiled_proto.rqd_pb2.RunFrame(frame_id=frameId)) - frameToLaunch = rqd.compiled_proto.rqd_pb2.RunFrame(frame_id=frameId) + self.rqcore.storeFrame(frameId, opencue.compiled_proto.rqd_pb2.RunFrame(frame_id=frameId)) + frameToLaunch = opencue.compiled_proto.rqd_pb2.RunFrame(frame_id=frameId) rqd.rqconstants.OVERRIDE_NIMBY = None with self.assertRaises(rqd.rqexceptions.DuplicateFrameViolationException): self.rqcore.launchFrame(frameToLaunch) def test_launchFrameWithInvalidUid(self): - self.machineMock.return_value.state = rqd.compiled_proto.host_pb2.UP + self.machineMock.return_value.state = opencue.compiled_proto.host_pb2.UP self.nimbyMock.return_value.locked = False - frame = rqd.compiled_proto.rqd_pb2.RunFrame(uid=0) + frame = opencue.compiled_proto.rqd_pb2.RunFrame(uid=0) with self.assertRaises(rqd.rqexceptions.InvalidUserException): self.rqcore.launchFrame(frame) def test_launchFrameWithInvalidCoreCount(self): - self.machineMock.return_value.state = rqd.compiled_proto.host_pb2.UP + self.machineMock.return_value.state = opencue.compiled_proto.host_pb2.UP self.nimbyMock.return_value.locked = False - frame = rqd.compiled_proto.rqd_pb2.RunFrame(uid=22, num_cores=0) + frame = opencue.compiled_proto.rqd_pb2.RunFrame(uid=22, num_cores=0) with self.assertRaises(rqd.rqexceptions.CoreReservationFailureException): self.rqcore.launchFrame(frame) def test_launchFrameWithInsufficientCores(self): - self.rqcore.cores = rqd.compiled_proto.report_pb2.CoreDetail(total_cores=100, idle_cores=5) - self.machineMock.return_value.state = rqd.compiled_proto.host_pb2.UP + self.rqcore.cores = opencue.compiled_proto.report_pb2.CoreDetail(total_cores=100, idle_cores=5) + self.machineMock.return_value.state = opencue.compiled_proto.host_pb2.UP self.nimbyMock.return_value.locked = False - frame = rqd.compiled_proto.rqd_pb2.RunFrame(uid=22, num_cores=10) + frame = opencue.compiled_proto.rqd_pb2.RunFrame(uid=22, num_cores=10) with self.assertRaises(rqd.rqexceptions.CoreReservationFailureException): self.rqcore.launchFrame(frame) def test_getRunningFrame(self): frameId = 'arbitrary-frame-id' - frame = rqd.compiled_proto.rqd_pb2.RunFrame(frame_id=frameId) + frame = opencue.compiled_proto.rqd_pb2.RunFrame(frame_id=frameId) self.rqcore.storeFrame(frameId, frame) self.assertEqual(frame, self.rqcore.getRunningFrame(frameId)) @@ -395,7 +395,7 @@ def test_rebootIdleNoFrames(self): def test_rebootIdleWithFrames(self): frame1Id = 'frame1' frame1 = rqd.rqnetwork.RunningFrame( - self.rqcore, rqd.compiled_proto.rqd_pb2.RunFrame(frame_id=frame1Id)) + self.rqcore, opencue.compiled_proto.rqd_pb2.RunFrame(frame_id=frame1Id)) self.rqcore.storeFrame(frame1Id, frame1) self.rqcore.rebootIdle() @@ -465,7 +465,7 @@ def test_lockAll(self): self.assertEqual(50, self.rqcore.cores.locked_cores) def test_unlock(self): - self.machineMock.return_value.state = rqd.compiled_proto.host_pb2.UP + self.machineMock.return_value.state = opencue.compiled_proto.host_pb2.UP self.rqcore.cores.total_cores = 50 self.rqcore.cores.idle_cores = 10 self.rqcore.cores.locked_cores = 40 @@ -477,7 +477,7 @@ def test_unlock(self): self.assertEqual(20, self.rqcore.cores.locked_cores) def test_unlockMoreCoresThanThereAre(self): - self.machineMock.return_value.state = rqd.compiled_proto.host_pb2.UP + self.machineMock.return_value.state = opencue.compiled_proto.host_pb2.UP self.rqcore.cores.total_cores = 50 self.rqcore.cores.idle_cores = 40 self.rqcore.cores.locked_cores = 10 @@ -489,7 +489,7 @@ def test_unlockMoreCoresThanThereAre(self): self.assertEqual(0, self.rqcore.cores.locked_cores) def test_unlockAll(self): - self.machineMock.return_value.state = rqd.compiled_proto.host_pb2.UP + self.machineMock.return_value.state = opencue.compiled_proto.host_pb2.UP self.nimbyMock.return_value.locked = False self.rqcore.cores.total_cores = 50 self.rqcore.cores.idle_cores = 40 @@ -502,7 +502,7 @@ def test_unlockAll(self): self.assertEqual(0, self.rqcore.cores.locked_cores) def test_unlockAllWhenNimbyLocked(self): - self.machineMock.return_value.state = rqd.compiled_proto.host_pb2.UP + self.machineMock.return_value.state = opencue.compiled_proto.host_pb2.UP self.nimbyMock.return_value.locked = True self.rqcore.cores.total_cores = 50 self.rqcore.cores.idle_cores = 40 @@ -541,7 +541,7 @@ def test_runLinux(self, getTempDirMock, permsUser, timeMock, popenMock): # mkdir frameUid = 928 frameUsername = 'my-random-user' returnCode = 0 - renderHost = rqd.compiled_proto.report_pb2.RenderHost(name='arbitrary-host-name') + renderHost = opencue.compiled_proto.report_pb2.RenderHost(name='arbitrary-host-name') logFile = os.path.join(logDir, '%s.%s.rqlog' % (jobName, frameName)) self.fs.create_dir(tempDir) @@ -557,9 +557,9 @@ def test_runLinux(self, getTempDirMock, permsUser, timeMock, popenMock): # mkdir rqCore.machine.isDesktop.return_value = True rqCore.machine.getHostInfo.return_value = renderHost rqCore.nimby.locked = False - children = rqd.compiled_proto.report_pb2.ChildrenProcStats() + children = opencue.compiled_proto.report_pb2.ChildrenProcStats() - runFrame = rqd.compiled_proto.rqd_pb2.RunFrame( + runFrame = opencue.compiled_proto.rqd_pb2.RunFrame( frame_id=frameId, job_name=jobName, frame_name=frameName, @@ -596,9 +596,9 @@ def test_runLinux(self, getTempDirMock, permsUser, timeMock, popenMock): # mkdir _, kwargs = popenMock.call_args rqCore.network.reportRunningFrameCompletion.assert_called_with( - rqd.compiled_proto.report_pb2.FrameCompleteReport( + opencue.compiled_proto.report_pb2.FrameCompleteReport( host=renderHost, - frame=rqd.compiled_proto.report_pb2.RunningFrameInfo( + frame=opencue.compiled_proto.report_pb2.RunningFrameInfo( job_name=jobName, frame_id=frameId, frame_name=frameName, children=children), exit_status=returnCode)) @@ -617,7 +617,7 @@ def disabled__test_runWindows(self, permsUser, timeMock, popenMock): frameUid = 928 frameUsername = 'my-random-user' returnCode = 0 - renderHost = rqd.compiled_proto.report_pb2.RenderHost(name='arbitrary-host-name') + renderHost = opencue.compiled_proto.report_pb2.RenderHost(name='arbitrary-host-name') timeMock.return_value = currentTime popenMock.return_value.returncode = returnCode @@ -629,9 +629,9 @@ def disabled__test_runWindows(self, permsUser, timeMock, popenMock): rqCore.machine.isDesktop.return_value = True rqCore.machine.getHostInfo.return_value = renderHost rqCore.nimby.locked = False - children = rqd.compiled_proto.report_pb2.ChildrenProcStats() + children = opencue.compiled_proto.report_pb2.ChildrenProcStats() - runFrame = rqd.compiled_proto.rqd_pb2.RunFrame( + runFrame = opencue.compiled_proto.rqd_pb2.RunFrame( frame_id=frameId, job_id=jobId, job_name=jobName, @@ -656,9 +656,9 @@ def disabled__test_runWindows(self, permsUser, timeMock, popenMock): # TODO(bcipriano) Verify the log directory was created and used for stdout/stderr. rqCore.network.reportRunningFrameCompletion.assert_called_with( - rqd.compiled_proto.report_pb2.FrameCompleteReport( + opencue.compiled_proto.report_pb2.FrameCompleteReport( host=renderHost, - frame=rqd.compiled_proto.report_pb2.RunningFrameInfo( + frame=opencue.compiled_proto.report_pb2.RunningFrameInfo( job_name=jobName, frame_id=frameId, frame_name=frameName, children=children), exit_status=returnCode)) @@ -676,7 +676,7 @@ def test_runDarwin(self, getTempDirMock, permsUser, timeMock, popenMock): frameUid = 928 frameUsername = 'my-random-user' returnCode = 0 - renderHost = rqd.compiled_proto.report_pb2.RenderHost(name='arbitrary-host-name') + renderHost = opencue.compiled_proto.report_pb2.RenderHost(name='arbitrary-host-name') logFile = os.path.join(logDir, '%s.%s.rqlog' % (jobName, frameName)) self.fs.create_dir(tempDir) @@ -692,9 +692,9 @@ def test_runDarwin(self, getTempDirMock, permsUser, timeMock, popenMock): rqCore.machine.isDesktop.return_value = True rqCore.machine.getHostInfo.return_value = renderHost rqCore.nimby.locked = False - children = rqd.compiled_proto.report_pb2.ChildrenProcStats() + children = opencue.compiled_proto.report_pb2.ChildrenProcStats() - runFrame = rqd.compiled_proto.rqd_pb2.RunFrame( + runFrame = opencue.compiled_proto.rqd_pb2.RunFrame( frame_id=frameId, job_name=jobName, frame_name=frameName, @@ -730,9 +730,9 @@ def test_runDarwin(self, getTempDirMock, permsUser, timeMock, popenMock): self.assertEqual(logFile, kwargs['stderr'].name) rqCore.network.reportRunningFrameCompletion.assert_called_with( - rqd.compiled_proto.report_pb2.FrameCompleteReport( + opencue.compiled_proto.report_pb2.FrameCompleteReport( host=renderHost, - frame=rqd.compiled_proto.report_pb2.RunningFrameInfo( + frame=opencue.compiled_proto.report_pb2.RunningFrameInfo( job_name=jobName, frame_id=frameId, frame_name=frameName, children=children), exit_status=returnCode)) diff --git a/rqd/tests/rqmachine_tests.py b/rqd/tests/rqmachine_tests.py index 1b1bdaf4a..b3f878b5f 100644 --- a/rqd/tests/rqmachine_tests.py +++ b/rqd/tests/rqmachine_tests.py @@ -33,9 +33,9 @@ import rqd.rqnetwork import rqd.rqnimby import rqd.rqutil -import rqd.compiled_proto.host_pb2 -import rqd.compiled_proto.report_pb2 -import rqd.compiled_proto.rqd_pb2 +import opencue.compiled_proto.host_pb2 +import opencue.compiled_proto.report_pb2 +import opencue.compiled_proto.rqd_pb2 CPUINFO = """processor : 0 @@ -180,7 +180,7 @@ def setUp(self): self.rqCore.nimby = self.nimby self.nimby.active = False self.nimby.locked = False - self.coreDetail = rqd.compiled_proto.report_pb2.CoreDetail(total_cores=2) + self.coreDetail = opencue.compiled_proto.report_pb2.CoreDetail(total_cores=2) self.machine = rqd.rqmachine.Machine(self.rqCore, self.coreDetail) @@ -288,7 +288,7 @@ def _test_rssUpdate(self, proc_stat): self.fs.create_file('/proc/%s/cmdline' % pid, contents=PROC_PID_CMDLINE) self.fs.create_file('/proc/%s/statm' % pid, contents=PROC_PID_STATM) runningFrame = rqd.rqnetwork.RunningFrame(self.rqCore, - rqd.compiled_proto.rqd_pb2.RunFrame()) + opencue.compiled_proto.rqd_pb2.RunFrame()) runningFrame.pid = pid frameCache = {frameId: runningFrame} @@ -403,14 +403,14 @@ def test_getHostInfo(self): self.assertEqual(25, hostInfo.load) self.assertEqual(False, hostInfo.nimby_enabled) self.assertEqual(False, hostInfo.nimby_locked) - self.assertEqual(rqd.compiled_proto.host_pb2.UP, hostInfo.state) + self.assertEqual(opencue.compiled_proto.host_pb2.UP, hostInfo.state) def test_getHostReport(self): frame1 = mock.MagicMock(spec=rqd.rqnetwork.RunningFrame) - frame1Info = rqd.compiled_proto.report_pb2.RunningFrameInfo(resource_id='arbitrary-id-1') + frame1Info = opencue.compiled_proto.report_pb2.RunningFrameInfo(resource_id='arbitrary-id-1') frame1.runningFrameInfo.return_value = frame1Info frame2 = mock.MagicMock(spec=rqd.rqnetwork.RunningFrame) - frame2Info = rqd.compiled_proto.report_pb2.RunningFrameInfo(resource_id='arbitrary-id-2') + frame2Info = opencue.compiled_proto.report_pb2.RunningFrameInfo(resource_id='arbitrary-id-2') frame2.runningFrameInfo.return_value = frame2Info frameIds = ['frame1', 'frame2'] frames = { @@ -419,7 +419,7 @@ def test_getHostReport(self): } self.rqCore.getFrameKeys.return_value = frameIds self.rqCore.getFrame.side_effect = lambda frameId: frames[frameId] - coreDetail = rqd.compiled_proto.report_pb2.CoreDetail( + coreDetail = opencue.compiled_proto.report_pb2.CoreDetail( total_cores=152, idle_cores=57, locked_cores=30, booked_cores=65) self.rqCore.getCoreInfo.return_value = coreDetail From 9027437221e1335600030e72a979eb2520045424 Mon Sep 17 00:00:00 2001 From: Jimmy Christensen Date: Thu, 15 Aug 2024 00:25:10 +0200 Subject: [PATCH 09/32] Fix linting --- rqd/rqd/rqnetwork.py | 3 ++- rqd/tests/cuerqd_tests.py | 11 +++++++---- rqd/tests/rqconstants_tests.py | 2 +- rqd/tests/rqcore_tests.py | 15 ++++++++++----- rqd/tests/rqmachine_tests.py | 12 +++++++----- 5 files changed, 27 insertions(+), 16 deletions(-) diff --git a/rqd/rqd/rqnetwork.py b/rqd/rqd/rqnetwork.py index 63aceca83..4943d5641 100644 --- a/rqd/rqd/rqnetwork.py +++ b/rqd/rqd/rqnetwork.py @@ -194,7 +194,8 @@ def __init__(self, rqCore): def addServicers(self): """Registers the gRPC servicers defined in rqdservicers.py.""" for servicer in self.servicers: - addFunc = getattr(opencue.compiled_proto.rqd_pb2_grpc, 'add_{0}_to_server'.format(servicer)) + addFunc = getattr(opencue.compiled_proto.rqd_pb2_grpc, + 'add_{0}_to_server'.format(servicer)) servicerClass = getattr(rqd.rqdservicers, servicer) addFunc(servicerClass(self.rqCore), self.server) diff --git a/rqd/tests/cuerqd_tests.py b/rqd/tests/cuerqd_tests.py index b8e42866c..6081e7d9b 100644 --- a/rqd/tests/cuerqd_tests.py +++ b/rqd/tests/cuerqd_tests.py @@ -28,8 +28,8 @@ import mock -import rqd.cuerqd import opencue.compiled_proto.rqd_pb2 +import rqd.cuerqd SCRIPT_NAME = '/arbitrary/path/to/script' @@ -68,7 +68,8 @@ def test_status(self, stubMock, frameStubMock): def test_getRunningFrame(self, stubMock, frameStubMock): frameId = 'arbitrary-frame-id' sys.argv = [SCRIPT_NAME, RQD_HOSTNAME, '--getproxy', frameId] - runFrameRequest = opencue.compiled_proto.rqd_pb2.RqdStaticGetRunFrameRequest(frame_id=frameId) + runFrameRequest = opencue.compiled_proto.rqd_pb2.RqdStaticGetRunFrameRequest( + frame_id=frameId) rqd.cuerqd.main() @@ -118,7 +119,8 @@ def test_lock(self, stubMock, frameStubMock): def test_unlock(self, stubMock, frameStubMock): numCoresToUnlock = 52 sys.argv = [SCRIPT_NAME, RQD_HOSTNAME, '--ulp', str(numCoresToUnlock)] - unlockRequest = opencue.compiled_proto.rqd_pb2.RqdStaticUnlockRequest(cores=numCoresToUnlock) + unlockRequest = opencue.compiled_proto.rqd_pb2.RqdStaticUnlockRequest( + cores=numCoresToUnlock) rqd.cuerqd.main() @@ -161,7 +163,8 @@ def test_launchFrame(self, stubMock, frameStubMock): runFrame.job_id = "SD6F3S72DJ26236KFS" runFrame.job_name = "edu-trn_job-name" runFrame.frame_id = "FD1S3I154O646UGSNN" - runFrameRequest = opencue.compiled_proto.rqd_pb2.RqdStaticLaunchFrameRequest(run_frame=runFrame) + runFrameRequest = opencue.compiled_proto.rqd_pb2.RqdStaticLaunchFrameRequest( + run_frame=runFrame) rqdHost = rqd.cuerqd.RqdHost(RQD_HOSTNAME) rqdHost.active = False diff --git a/rqd/tests/rqconstants_tests.py b/rqd/tests/rqconstants_tests.py index 5a13651af..761d9be14 100644 --- a/rqd/tests/rqconstants_tests.py +++ b/rqd/tests/rqconstants_tests.py @@ -32,12 +32,12 @@ import six +import opencue.compiled_proto.report_pb2 import rqd.rqconstants import rqd.rqcore import rqd.rqmachine import rqd.rqnimby import rqd.rqutil -import opencue.compiled_proto.report_pb2 from .rqmachine_tests import ( CPUINFO, diff --git a/rqd/tests/rqcore_tests.py b/rqd/tests/rqcore_tests.py index 6fe67bfee..f7a3ec891 100644 --- a/rqd/tests/rqcore_tests.py +++ b/rqd/tests/rqcore_tests.py @@ -237,7 +237,8 @@ def test_killAllFrameIgnoreNimby(self): self.rqcore, opencue.compiled_proto.rqd_pb2.RunFrame(frame_id=frame1Id)) frame1.frameAttendantThread = frameAttendantThread frame2 = rqd.rqnetwork.RunningFrame( - self.rqcore, opencue.compiled_proto.rqd_pb2.RunFrame(frame_id=frame2Id, ignore_nimby=True)) + self.rqcore, opencue.compiled_proto.rqd_pb2.RunFrame(frame_id=frame2Id, + ignore_nimby=True)) frame2.frameAttendantThread = frameAttendantThread self.rqcore.storeFrame(frame1Id, frame1) self.rqcore.storeFrame(frame2Id, frame2) @@ -282,7 +283,8 @@ def test_handleExit(self, networkMock, exitMock): @mock.patch('rqd.rqcore.FrameAttendantThread') def test_launchFrame(self, frameThreadMock): - self.rqcore.cores = opencue.compiled_proto.report_pb2.CoreDetail(total_cores=100, idle_cores=20) + self.rqcore.cores = opencue.compiled_proto.report_pb2.CoreDetail(total_cores=100, + idle_cores=20) self.machineMock.return_value.state = opencue.compiled_proto.host_pb2.UP self.nimbyMock.return_value.locked = False frame = opencue.compiled_proto.rqd_pb2.RunFrame(uid=22, num_cores=10) @@ -310,7 +312,8 @@ def test_launchFrameOnHostWaitingForShutdown(self): @mock.patch('rqd.rqcore.FrameAttendantThread') def test_launchFrameOnNimbyHost(self, frameThreadMock): - self.rqcore.cores = opencue.compiled_proto.report_pb2.CoreDetail(total_cores=100, idle_cores=20) + self.rqcore.cores = opencue.compiled_proto.report_pb2.CoreDetail(total_cores=100, + idle_cores=20) self.machineMock.return_value.state = opencue.compiled_proto.host_pb2.UP frame = opencue.compiled_proto.rqd_pb2.RunFrame(uid=22, num_cores=10) frameIgnoreNimby = opencue.compiled_proto.rqd_pb2.RunFrame( @@ -326,7 +329,8 @@ def test_launchFrameOnNimbyHost(self, frameThreadMock): frameThreadMock.return_value.start.assert_called() def test_launchDuplicateFrame(self): - self.rqcore.cores = opencue.compiled_proto.report_pb2.CoreDetail(total_cores=100, idle_cores=20) + self.rqcore.cores = opencue.compiled_proto.report_pb2.CoreDetail(total_cores=100, + idle_cores=20) self.machineMock.return_value.state = opencue.compiled_proto.host_pb2.UP self.nimbyMock.return_value.locked = False frameId = 'arbitrary-frame-id' @@ -354,7 +358,8 @@ def test_launchFrameWithInvalidCoreCount(self): self.rqcore.launchFrame(frame) def test_launchFrameWithInsufficientCores(self): - self.rqcore.cores = opencue.compiled_proto.report_pb2.CoreDetail(total_cores=100, idle_cores=5) + self.rqcore.cores = opencue.compiled_proto.report_pb2.CoreDetail(total_cores=100, + idle_cores=5) self.machineMock.return_value.state = opencue.compiled_proto.host_pb2.UP self.nimbyMock.return_value.locked = False frame = opencue.compiled_proto.rqd_pb2.RunFrame(uid=22, num_cores=10) diff --git a/rqd/tests/rqmachine_tests.py b/rqd/tests/rqmachine_tests.py index b3f878b5f..566668fea 100644 --- a/rqd/tests/rqmachine_tests.py +++ b/rqd/tests/rqmachine_tests.py @@ -27,15 +27,15 @@ import mock import pyfakefs.fake_filesystem_unittest +import opencue.compiled_proto.host_pb2 +import opencue.compiled_proto.report_pb2 +import opencue.compiled_proto.rqd_pb2 import rqd.rqconstants import rqd.rqcore import rqd.rqmachine import rqd.rqnetwork import rqd.rqnimby import rqd.rqutil -import opencue.compiled_proto.host_pb2 -import opencue.compiled_proto.report_pb2 -import opencue.compiled_proto.rqd_pb2 CPUINFO = """processor : 0 @@ -407,10 +407,12 @@ def test_getHostInfo(self): def test_getHostReport(self): frame1 = mock.MagicMock(spec=rqd.rqnetwork.RunningFrame) - frame1Info = opencue.compiled_proto.report_pb2.RunningFrameInfo(resource_id='arbitrary-id-1') + frame1Info = opencue.compiled_proto.report_pb2.RunningFrameInfo( + resource_id='arbitrary-id-1') frame1.runningFrameInfo.return_value = frame1Info frame2 = mock.MagicMock(spec=rqd.rqnetwork.RunningFrame) - frame2Info = opencue.compiled_proto.report_pb2.RunningFrameInfo(resource_id='arbitrary-id-2') + frame2Info = opencue.compiled_proto.report_pb2.RunningFrameInfo( + resource_id='arbitrary-id-2') frame2.runningFrameInfo.return_value = frame2Info frameIds = ['frame1', 'frame2'] frames = { From b4d3638544d067cfabf69bad9475e46cb59374e6 Mon Sep 17 00:00:00 2001 From: Jimmy Christensen Date: Thu, 15 Aug 2024 00:35:06 +0200 Subject: [PATCH 10/32] Add documentation to new parameters --- pycue/opencue/cuelogging.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pycue/opencue/cuelogging.py b/pycue/opencue/cuelogging.py index 868cde035..325c27f71 100644 --- a/pycue/opencue/cuelogging.py +++ b/pycue/opencue/cuelogging.py @@ -37,6 +37,10 @@ def __init__(self, filepath, mode, maxLogFiles=1): """RQDLogger class initialization @type filepath: string @param filepath: The filepath to log to + @type mode: int + @param mode: read or write mode + @type maxLogFiles: int + @param maxLogFiles: number of files to rotate, when in write mode """ self.filepath = filepath From 8c9944b44c9e8bf61dd84081010165563d0b6de8 Mon Sep 17 00:00:00 2001 From: Jimmy Christensen Date: Thu, 15 Aug 2024 00:35:35 +0200 Subject: [PATCH 11/32] Removed unused type attribute --- pycue/opencue/cuelogging.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pycue/opencue/cuelogging.py b/pycue/opencue/cuelogging.py index 325c27f71..cc2306530 100644 --- a/pycue/opencue/cuelogging.py +++ b/pycue/opencue/cuelogging.py @@ -30,7 +30,6 @@ class CueLogger(object): """Class to abstract file logging, this class tries to act as a file object""" filepath = None fd = None - type = 0 mode = MODE_READ def __init__(self, filepath, mode, maxLogFiles=1): From 8873acdb8f66cdb3acb1180f0bf5d7d4e37c275c Mon Sep 17 00:00:00 2001 From: Jimmy Christensen Date: Thu, 15 Aug 2024 00:48:07 +0200 Subject: [PATCH 12/32] Add small comment about default mode --- pycue/opencue/cuelogging.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pycue/opencue/cuelogging.py b/pycue/opencue/cuelogging.py index cc2306530..19f239305 100644 --- a/pycue/opencue/cuelogging.py +++ b/pycue/opencue/cuelogging.py @@ -30,7 +30,7 @@ class CueLogger(object): """Class to abstract file logging, this class tries to act as a file object""" filepath = None fd = None - mode = MODE_READ + mode = MODE_READ # Default in read mode def __init__(self, filepath, mode, maxLogFiles=1): """RQDLogger class initialization From 958e462e4f8c7865af56a4f879858ec2fdcb7c0c Mon Sep 17 00:00:00 2001 From: Jimmy Christensen Date: Tue, 20 Aug 2024 21:21:47 +0200 Subject: [PATCH 13/32] Split log reading and writing into separate classes --- cuegui/cuegui/plugins/LogViewPlugin.py | 2 +- pycue/opencue/cuelogging.py | 154 ++++++++++++------------- rqd/rqd/rqcore.py | 3 +- 3 files changed, 78 insertions(+), 81 deletions(-) diff --git a/cuegui/cuegui/plugins/LogViewPlugin.py b/cuegui/cuegui/plugins/LogViewPlugin.py index b7a0b6508..9da2cc919 100644 --- a/cuegui/cuegui/plugins/LogViewPlugin.py +++ b/cuegui/cuegui/plugins/LogViewPlugin.py @@ -812,7 +812,7 @@ def _display_log_content(self): @postcondition: The _update_log method is scheduled to run again after 5 seconds """ - log_reader = opencue.cuelogging.CueLogger(self._log_file, mode=opencue.cuelogging.MODE_READ) + log_reader = opencue.cuelogging.CueLogReader(self._log_file) try: if log_reader.exists() is not True: diff --git a/pycue/opencue/cuelogging.py b/pycue/opencue/cuelogging.py index 19f239305..1917fee13 100644 --- a/pycue/opencue/cuelogging.py +++ b/pycue/opencue/cuelogging.py @@ -23,97 +23,83 @@ log = logging.getLogger(__name__) -MODE_READ = 0 -MODE_WRITE = 1 - -class CueLogger(object): - """Class to abstract file logging, this class tries to act as a file object""" +class CueLogWriter(object): + """Class to abstract file log writing, this class tries to act as a file object""" filepath = None fd = None - mode = MODE_READ # Default in read mode - def __init__(self, filepath, mode, maxLogFiles=1): - """RQDLogger class initialization + def __init__(self, filepath, maxLogFiles=1): + """CueLogWriter class initialization @type filepath: string @param filepath: The filepath to log to - @type mode: int - @param mode: read or write mode @type maxLogFiles: int @param maxLogFiles: number of files to rotate, when in write mode """ self.filepath = filepath - self.mode = mode - if self.mode == MODE_WRITE: - log_dir = os.path.dirname(self.filepath) - if not os.access(log_dir, os.F_OK): - # Attempting mkdir for missing logdir - msg = "No Error" - try: - os.makedirs(log_dir) - os.chmod(log_dir, 0o777) - # pylint: disable=broad-except - except Exception as e: - # This is expected to fail when called in abq - # But the directory should now be visible - msg = e - - if not os.access(log_dir, os.F_OK): - err = "Unable to see log directory: %s, mkdir failed with: %s" % ( - log_dir, msg) - raise RuntimeError(err) - - if not os.access(log_dir, os.W_OK): - err = "Unable to write to log directory %s" % log_dir - raise RuntimeError(err) - - try: - # Rotate any old logs to a max of MAX_LOG_FILES: - if os.path.isfile(self.filepath): - rotateCount = 1 - while (os.path.isfile("%s.%s" % (self.filepath, rotateCount)) - and rotateCount < maxLogFiles): - rotateCount += 1 - os.rename(self.filepath, - "%s.%s" % (self.filepath, rotateCount)) - # pylint: disable=broad-except - except Exception as e: - err = "Unable to rotate previous log file due to %s" % e - # Windows might fail while trying to rotate logs for checking if file is - # being used by another process. Frame execution doesn't need to - # be halted for this. - if platform.system() == "Windows": - log.warning(err) - else: - raise RuntimeError(err) - # pylint: disable=consider-using-with - self.fd = open(self.filepath, "w+", 1, encoding='utf-8') + log_dir = os.path.dirname(self.filepath) + if not os.access(log_dir, os.F_OK): + # Attempting mkdir for missing logdir + msg = "No Error" try: - os.chmod(self.filepath, 0o666) + os.makedirs(log_dir) + os.chmod(log_dir, 0o777) # pylint: disable=broad-except except Exception as e: - err = "Failed to chmod log file! %s due to %s" % (self.filepath, e) + # This is expected to fail when called in abq + # But the directory should now be visible + msg = e + + if not os.access(log_dir, os.F_OK): + err = "Unable to see log directory: %s, mkdir failed with: %s" % ( + log_dir, msg) + raise RuntimeError(err) + + if not os.access(log_dir, os.W_OK): + err = "Unable to write to log directory %s" % log_dir + raise RuntimeError(err) + + try: + # Rotate any old logs to a max of MAX_LOG_FILES: + if os.path.isfile(self.filepath): + rotateCount = 1 + while (os.path.isfile("%s.%s" % (self.filepath, rotateCount)) + and rotateCount < maxLogFiles): + rotateCount += 1 + os.rename(self.filepath, + "%s.%s" % (self.filepath, rotateCount)) + # pylint: disable=broad-except + except Exception as e: + err = "Unable to rotate previous log file due to %s" % e + # Windows might fail while trying to rotate logs for checking if file is + # being used by another process. Frame execution doesn't need to + # be halted for this. + if platform.system() == "Windows": log.warning(err) - elif mode == MODE_READ: - self.fd = open(self.filepath, "r", encoding='utf-8') - else: - log.error("Unknown mode for log file!") + else: + raise RuntimeError(err) + # pylint: disable=consider-using-with + self.fd = open(self.filepath, "w+", 1, encoding='utf-8') + try: + os.chmod(self.filepath, 0o666) + # pylint: disable=broad-except + except Exception as e: + err = "Failed to chmod log file! %s due to %s" % (self.filepath, e) + log.warning(err) # pylint: disable=arguments-differ def write(self, data, prependTimestamp=False): """Abstract write function that will write to the correct backend""" - # Only work if in write mode, otherwise ignore - if self.mode == MODE_WRITE: - # Convert data to unicode - if isinstance(data, bytes): - data = data.decode('utf-8', errors='ignore') - if prependTimestamp is True: - lines = data.splitlines() - curr_line_timestamp = datetime.datetime.now().strftime("%H:%M:%S") - for line in lines: - print("[%s] %s" % (curr_line_timestamp, line), file=self) - else: - self.fd.write(data) + # Convert data to unicode + if isinstance(data, bytes): + data = data.decode('utf-8', errors='ignore') + if prependTimestamp is True: + lines = data.splitlines() + curr_line_timestamp = datetime.datetime.now().strftime("%H:%M:%S") + for line in lines: + print("[%s] %s" % (curr_line_timestamp, line), file=self) + else: + self.fd.write(data) def writelines(self, __lines): """Provides support for writing mutliple lines at a time""" @@ -135,6 +121,20 @@ def waitForFile(self, maxTries=5): time.sleep(0.5 * tries) raise IOError("Failed to create %s" % self.filepath) + +class CueLogReader(object): + """Class to abstract file log reading, this class tries to act as a file object""" + filepath = None + fd = None + + def __init__(self, filepath): + """CueLogWriter class initialization + @type filepath: string + @param filepath: The filepath to log to + """ + self.filepath = filepath + self.fd = open(self.filepath, "r", encoding='utf-8') + def size(self): """Return the size of the file""" return int(os.stat(self.filepath).st_size) @@ -149,13 +149,11 @@ def exists(self): def read(self): """Read the data from the backend""" - # Only allow reading when in read mode content = None - if self.mode == MODE_READ: - if self.exists() is True: - with open(self.filepath, "r", encoding='utf-8') as fp: - content = fp.read() + if self.exists() is True: + with open(self.filepath, "r", encoding='utf-8') as fp: + content = fp.read() return content diff --git a/rqd/rqd/rqcore.py b/rqd/rqd/rqcore.py index 044bd08b3..90bb57cdb 100644 --- a/rqd/rqd/rqcore.py +++ b/rqd/rqd/rqcore.py @@ -520,9 +520,8 @@ def run(self): # Setup proc to allow launching of frame # try: - self.rqlog = opencue.cuelogging.CueLogger( + self.rqlog = opencue.cuelogging.CueLogWriter( runFrame.log_dir_file, - mode=opencue.cuelogging.MODE_WRITE, maxLogFiles=rqd.rqconstants.MAX_LOG_FILES ) self.rqlog.waitForFile() From 9c38c8dc21e9cf87118c6d6c7f90b093982e84d7 Mon Sep 17 00:00:00 2001 From: Jimmy Christensen Date: Tue, 20 Aug 2024 21:33:26 +0200 Subject: [PATCH 14/32] Split up init of CueLogWriter into smaller functions and internalize some class attributes --- pycue/opencue/cuelogging.py | 76 +++++++++++++++++++------------------ 1 file changed, 39 insertions(+), 37 deletions(-) diff --git a/pycue/opencue/cuelogging.py b/pycue/opencue/cuelogging.py index 1917fee13..cfbe68198 100644 --- a/pycue/opencue/cuelogging.py +++ b/pycue/opencue/cuelogging.py @@ -26,8 +26,6 @@ class CueLogWriter(object): """Class to abstract file log writing, this class tries to act as a file object""" filepath = None - fd = None - def __init__(self, filepath, maxLogFiles=1): """CueLogWriter class initialization @type filepath: string @@ -37,34 +35,32 @@ def __init__(self, filepath, maxLogFiles=1): """ self.filepath = filepath - log_dir = os.path.dirname(self.filepath) - if not os.access(log_dir, os.F_OK): - # Attempting mkdir for missing logdir - msg = "No Error" - try: - os.makedirs(log_dir) - os.chmod(log_dir, 0o777) - # pylint: disable=broad-except - except Exception as e: - # This is expected to fail when called in abq - # But the directory should now be visible - msg = e - - if not os.access(log_dir, os.F_OK): - err = "Unable to see log directory: %s, mkdir failed with: %s" % ( - log_dir, msg) - raise RuntimeError(err) + self.__log_dir = os.path.dirname(self.filepath) + self.__maxLogFiles = maxLogFiles + if not os.access(self.__log_dir, os.F_OK): + self.__makeLogDir() - if not os.access(log_dir, os.W_OK): - err = "Unable to write to log directory %s" % log_dir + if not os.access(self.__log_dir, os.W_OK): + err = "Unable to write to log directory %s" % self.__log_dir raise RuntimeError(err) + self.__attemptRotateLogs() + # pylint: disable=consider-using-with + self.__fd = open(self.filepath, "w+", 1, encoding='utf-8') + try: + os.chmod(self.filepath, 0o666) + # pylint: disable=broad-except + except Exception as e: + err = "Failed to chmod log file! %s due to %s" % (self.filepath, e) + log.warning(err) + + def __attemptRotateLogs(self): try: # Rotate any old logs to a max of MAX_LOG_FILES: if os.path.isfile(self.filepath): rotateCount = 1 while (os.path.isfile("%s.%s" % (self.filepath, rotateCount)) - and rotateCount < maxLogFiles): + and rotateCount < self.__maxLogFiles): rotateCount += 1 os.rename(self.filepath, "%s.%s" % (self.filepath, rotateCount)) @@ -78,14 +74,22 @@ def __init__(self, filepath, maxLogFiles=1): log.warning(err) else: raise RuntimeError(err) - # pylint: disable=consider-using-with - self.fd = open(self.filepath, "w+", 1, encoding='utf-8') + + def __makeLogDir(self): + # Attempting mkdir for missing logdir + msg = "No Error" try: - os.chmod(self.filepath, 0o666) + os.makedirs(self.__log_dir) + os.chmod(self.__log_dir, 0o777) # pylint: disable=broad-except except Exception as e: - err = "Failed to chmod log file! %s due to %s" % (self.filepath, e) - log.warning(err) + # This is expected to fail when called in abq + # But the directory should now be visible + msg = e + + if not os.access(self.__log_dir, os.F_OK): + err = "Unable to see log directory: %s, mkdir failed with: %s" % (self.__log_dir, msg) + raise RuntimeError(err) # pylint: disable=arguments-differ def write(self, data, prependTimestamp=False): @@ -99,7 +103,7 @@ def write(self, data, prependTimestamp=False): for line in lines: print("[%s] %s" % (curr_line_timestamp, line), file=self) else: - self.fd.write(data) + self.__fd.write(data) def writelines(self, __lines): """Provides support for writing mutliple lines at a time""" @@ -108,7 +112,7 @@ def writelines(self, __lines): def close(self): """Closes the file if the backend is file based""" - self.fd.close() + self.__fd.close() def waitForFile(self, maxTries=5): """Waits for the file to exist before continuing when using a file backend""" @@ -124,35 +128,33 @@ def waitForFile(self, maxTries=5): class CueLogReader(object): """Class to abstract file log reading, this class tries to act as a file object""" - filepath = None - fd = None def __init__(self, filepath): """CueLogWriter class initialization @type filepath: string @param filepath: The filepath to log to """ - self.filepath = filepath - self.fd = open(self.filepath, "r", encoding='utf-8') + self.__filepath = filepath + self.__fd = open(self.__filepath, "r", encoding='utf-8') def size(self): """Return the size of the file""" - return int(os.stat(self.filepath).st_size) + return int(os.stat(self.__filepath).st_size) def getMtime(self): """Return modification time of the file""" - return os.path.getmtime(self.filepath) + return os.path.getmtime(self.__filepath) def exists(self): """Check if the file exists""" - return os.path.exists(self.filepath) + return os.path.exists(self.__filepath) def read(self): """Read the data from the backend""" content = None if self.exists() is True: - with open(self.filepath, "r", encoding='utf-8') as fp: + with open(self.__filepath, "r", encoding='utf-8') as fp: content = fp.read() return content From 84d04626a00fdd8597040e105dac7c7d5b1da097 Mon Sep 17 00:00:00 2001 From: Jimmy Christensen Date: Tue, 20 Aug 2024 21:40:03 +0200 Subject: [PATCH 15/32] Make CueLogReader.filepath public and raise IOError if file doesn't exist when attempting to read it. --- pycue/opencue/cuelogging.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/pycue/opencue/cuelogging.py b/pycue/opencue/cuelogging.py index cfbe68198..385de2f92 100644 --- a/pycue/opencue/cuelogging.py +++ b/pycue/opencue/cuelogging.py @@ -25,7 +25,9 @@ class CueLogWriter(object): """Class to abstract file log writing, this class tries to act as a file object""" + filepath = None + def __init__(self, filepath, maxLogFiles=1): """CueLogWriter class initialization @type filepath: string @@ -129,33 +131,37 @@ def waitForFile(self, maxTries=5): class CueLogReader(object): """Class to abstract file log reading, this class tries to act as a file object""" + filepath = None + def __init__(self, filepath): """CueLogWriter class initialization @type filepath: string @param filepath: The filepath to log to """ - self.__filepath = filepath - self.__fd = open(self.__filepath, "r", encoding='utf-8') + self.filepath = filepath + self.__fd = open(self.filepath, "r", encoding='utf-8') def size(self): """Return the size of the file""" - return int(os.stat(self.__filepath).st_size) + return int(os.stat(self.filepath).st_size) def getMtime(self): """Return modification time of the file""" - return os.path.getmtime(self.__filepath) + return os.path.getmtime(self.filepath) def exists(self): """Check if the file exists""" - return os.path.exists(self.__filepath) + return os.path.exists(self.filepath) def read(self): """Read the data from the backend""" content = None if self.exists() is True: - with open(self.__filepath, "r", encoding='utf-8') as fp: + with open(self.filepath, "r", encoding='utf-8') as fp: content = fp.read() + else: + raise IOError("Failed to open %s" % self.filepath) return content From b3e637054cc42c45287b8a48c84b266be14468fd Mon Sep 17 00:00:00 2001 From: Jimmy Christensen Date: Tue, 20 Aug 2024 21:41:24 +0200 Subject: [PATCH 16/32] Remove file handler creation in CueLogReader.__init__() --- pycue/opencue/cuelogging.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pycue/opencue/cuelogging.py b/pycue/opencue/cuelogging.py index 385de2f92..4564cf457 100644 --- a/pycue/opencue/cuelogging.py +++ b/pycue/opencue/cuelogging.py @@ -139,7 +139,6 @@ def __init__(self, filepath): @param filepath: The filepath to log to """ self.filepath = filepath - self.__fd = open(self.filepath, "r", encoding='utf-8') def size(self): """Return the size of the file""" From 956a58dad7631903829e6ca1ae0d8ac84e015296 Mon Sep 17 00:00:00 2001 From: Jimmy Christensen Date: Thu, 12 Sep 2024 01:14:41 +0200 Subject: [PATCH 17/32] Move cuelogging into separate package so the compiled_prot can be duplicated --- pycue/{opencue/cuelogging.py => cuelogging/__init__.py} | 0 rqd/rqd/compiled_proto/.gitignore | 2 ++ rqd/rqd/compiled_proto/__init__.py | 0 3 files changed, 2 insertions(+) rename pycue/{opencue/cuelogging.py => cuelogging/__init__.py} (100%) create mode 100644 rqd/rqd/compiled_proto/.gitignore create mode 100644 rqd/rqd/compiled_proto/__init__.py diff --git a/pycue/opencue/cuelogging.py b/pycue/cuelogging/__init__.py similarity index 100% rename from pycue/opencue/cuelogging.py rename to pycue/cuelogging/__init__.py diff --git a/rqd/rqd/compiled_proto/.gitignore b/rqd/rqd/compiled_proto/.gitignore new file mode 100644 index 000000000..49d7060ef --- /dev/null +++ b/rqd/rqd/compiled_proto/.gitignore @@ -0,0 +1,2 @@ +*_pb2.py +*_pb2_grpc.py diff --git a/rqd/rqd/compiled_proto/__init__.py b/rqd/rqd/compiled_proto/__init__.py new file mode 100644 index 000000000..e69de29bb From a7f3445c33557c5e20f22fd0c5e951857e046a18 Mon Sep 17 00:00:00 2001 From: Jimmy Christensen Date: Thu, 12 Sep 2024 01:15:20 +0200 Subject: [PATCH 18/32] Re-duplicate compiled_proto into rqd again and update test --- ci/run_python_tests.sh | 2 ++ cuegui/cuegui/plugins/LogViewPlugin.py | 4 +-- rqd/rqd/cuerqd.py | 42 +++++++++++++------------- rqd/rqd/rqcore.py | 24 +++++++-------- rqd/rqd/rqdservicers.py | 40 ++++++++++++------------ rqd/rqd/rqmachine.py | 14 ++++----- rqd/rqd/rqnetwork.py | 28 ++++++++--------- 7 files changed, 78 insertions(+), 76 deletions(-) diff --git a/ci/run_python_tests.sh b/ci/run_python_tests.sh index bf0d7506d..af1635005 100755 --- a/ci/run_python_tests.sh +++ b/ci/run_python_tests.sh @@ -15,10 +15,12 @@ pip install --user -r requirements.txt -r requirements_gui.txt # Protos need to have their Python code generated in order for tests to pass. python -m grpc_tools.protoc -I=proto/ --python_out=pycue/opencue/compiled_proto --grpc_python_out=pycue/opencue/compiled_proto proto/*.proto +python -m grpc_tools.protoc -I=proto/ --python_out=rqd/rqd/compiled_proto --grpc_python_out=rqd/rqd/compiled_proto proto/*.proto # Fix imports to work in both Python 2 and 3. See # for more info. 2to3 -wn -f import pycue/opencue/compiled_proto/*_pb2*.py +2to3 -wn -f import rqd/rqd/compiled_proto/*_pb2*.py python pycue/setup.py test PYTHONPATH=pycue python pyoutline/setup.py test diff --git a/cuegui/cuegui/plugins/LogViewPlugin.py b/cuegui/cuegui/plugins/LogViewPlugin.py index 1285a866b..52787fa91 100644 --- a/cuegui/cuegui/plugins/LogViewPlugin.py +++ b/cuegui/cuegui/plugins/LogViewPlugin.py @@ -32,7 +32,7 @@ from qtpy import QtCore from qtpy import QtWidgets -import opencue.cuelogging +import cuelogging import cuegui.Constants import cuegui.AbstractDockWidget @@ -813,7 +813,7 @@ def _display_log_content(self): @postcondition: The _update_log method is scheduled to run again after 5 seconds """ - log_reader = opencue.cuelogging.CueLogReader(self._log_file) + log_reader = cuelogging.CueLogReader(self._log_file) try: if log_reader.exists() is not True: diff --git a/rqd/rqd/cuerqd.py b/rqd/rqd/cuerqd.py index 7decf3486..6461d7b22 100755 --- a/rqd/rqd/cuerqd.py +++ b/rqd/rqd/cuerqd.py @@ -31,8 +31,8 @@ import grpc -import opencue.compiled_proto.rqd_pb2 -import opencue.compiled_proto.rqd_pb2_grpc +import rqd.compiled_proto.rqd_pb2 +import rqd.compiled_proto.rqd_pb2_grpc import rqd.rqconstants @@ -46,62 +46,62 @@ def __init__(self, rqdHost, rqdPort=rqd.rqconstants.RQD_GRPC_PORT): self.rqdPort = rqdPort channel = grpc.insecure_channel('%s:%s' % (self.rqdHost, self.rqdPort)) - self.stub = opencue.compiled_proto.rqd_pb2_grpc.RqdInterfaceStub(channel) - self.frameStub = opencue.compiled_proto.rqd_pb2_grpc.RunningFrameStub(channel) + self.stub = rqd.compiled_proto.rqd_pb2_grpc.RqdInterfaceStub(channel) + self.frameStub = rqd.compiled_proto.rqd_pb2_grpc.RunningFrameStub(channel) def status(self): """Fetches and returns the host status report.""" - return self.stub.ReportStatus(opencue.compiled_proto.rqd_pb2.RqdStaticReportStatusRequest()) + return self.stub.ReportStatus(rqd.compiled_proto.rqd_pb2.RqdStaticReportStatusRequest()) def getRunningFrame(self, frameId): """Returns the host's currently running frame.""" return self.stub.GetRunFrame( - opencue.compiled_proto.rqd_pb2.RqdStaticGetRunFrameRequest(frame_id=frameId)) + rqd.compiled_proto.rqd_pb2.RqdStaticGetRunFrameRequest(frame_id=frameId)) def nimbyOff(self): """Disables Nimby on the host.""" log.info(self.rqdHost, "Turning off Nimby") log.info("rqd nimbyoff by %s", os.environ.get("USER")) - self.stub.NimbyOff(opencue.compiled_proto.rqd_pb2.RqdStaticNimbyOffRequest()) + self.stub.NimbyOff(rqd.compiled_proto.rqd_pb2.RqdStaticNimbyOffRequest()) def nimbyOn(self): """Enables Nimby on the host.""" log.info(self.rqdHost, "Turning on Nimby") log.info("rqd nimbyon by %s", os.environ.get("USER")) - self.stub.NimbyOn(opencue.compiled_proto.rqd_pb2.RqdStaticNimbyOnRequest()) + self.stub.NimbyOn(rqd.compiled_proto.rqd_pb2.RqdStaticNimbyOnRequest()) def lockAll(self): """Locks all of the host's cores.""" print(self.rqdHost, "Locking all cores") - self.stub.LockAll(opencue.compiled_proto.rqd_pb2.RqdStaticLockAllRequest()) + self.stub.LockAll(rqd.compiled_proto.rqd_pb2.RqdStaticLockAllRequest()) def unlockAll(self): """Unlocks all of the host's cores.""" print(self.rqdHost, "Unlocking all cores") - self.stub.UnlockAll(opencue.compiled_proto.rqd_pb2.RqdStaticUnlockAllRequest()) + self.stub.UnlockAll(rqd.compiled_proto.rqd_pb2.RqdStaticUnlockAllRequest()) def lock(self, cores): """Locks the given number of cores.""" cores = int(cores) print(self.rqdHost, "Locking %d cores" % cores) - self.stub.Lock(opencue.compiled_proto.rqd_pb2.RqdStaticLockRequest(cores=cores)) + self.stub.Lock(rqd.compiled_proto.rqd_pb2.RqdStaticLockRequest(cores=cores)) def unlock(self, cores): """Unlocks the given number of cores.""" cores = int(cores) print(self.rqdHost, "Unlocking %d cores" % cores) - self.stub.Unlock(opencue.compiled_proto.rqd_pb2.RqdStaticUnlockRequest(cores=cores)) + self.stub.Unlock(rqd.compiled_proto.rqd_pb2.RqdStaticUnlockRequest(cores=cores)) def shutdownRqdIdle(self): """Shuts down the host when idle.""" print(self.rqdHost, "Sending shutdownRqdIdle command") - self.stub.ShutdownRqdIdle(opencue.compiled_proto.rqd_pb2.RqdStaticShutdownIdleRequest()) + self.stub.ShutdownRqdIdle(rqd.compiled_proto.rqd_pb2.RqdStaticShutdownIdleRequest()) def shutdownRqdNow(self): """Shuts down the host now.""" print(self.rqdHost, "Sending shutdownRqdNow command") try: - self.stub.ShutdownRqdNow(opencue.compiled_proto.rqd_pb2.RqdStaticShutdownNowRequest()) + self.stub.ShutdownRqdNow(rqd.compiled_proto.rqd_pb2.RqdStaticShutdownNowRequest()) # pylint: disable=broad-except except Exception: # Shutting down the service from inside means this request will receive @@ -111,22 +111,22 @@ def shutdownRqdNow(self): def restartRqdIdle(self): """Restarts RQD on the host when idle.""" print(self.rqdHost, "Sending restartRqdIdle command") - self.stub.RestartRqdIdle(opencue.compiled_proto.rqd_pb2.RqdStaticRestartIdleRequest()) + self.stub.RestartRqdIdle(rqd.compiled_proto.rqd_pb2.RqdStaticRestartIdleRequest()) def rebootIdle(self): """Reboots the host when idle.""" print(self.rqdHost, "Sending rebootIdle command") - self.stub.RebootIdle(opencue.compiled_proto.rqd_pb2.RqdStaticRebootIdleRequest()) + self.stub.RebootIdle(rqd.compiled_proto.rqd_pb2.RqdStaticRebootIdleRequest()) def rebootNow(self): """Reboots the host now.""" print(self.rqdHost, "Sending rebootNow command") - self.stub.RebootNow(opencue.compiled_proto.rqd_pb2.RqdStaticRebootNowRequest()) + self.stub.RebootNow(rqd.compiled_proto.rqd_pb2.RqdStaticRebootNowRequest()) def launchFrame(self, frame): """Launches a frame on the host.""" self.stub.LaunchFrame( - opencue.compiled_proto.rqd_pb2.RqdStaticLaunchFrameRequest(run_frame=frame)) + rqd.compiled_proto.rqd_pb2.RqdStaticLaunchFrameRequest(run_frame=frame)) def killFrame(self, frameId, message): """Kills a frame on the host.""" @@ -251,7 +251,7 @@ def main(): if args.test_edu_frame: print("Launching edu test frame (logs to /mcp)") frameNum = "0001" - runFrame = opencue.compiled_proto.rqd_pb2.RunFrame() + runFrame = rqd.compiled_proto.rqd_pb2.RunFrame() runFrame.job_id = "SD6F3S72DJ26236KFS" runFrame.job_name = "edu-trn_jwelborn-jwelborn_teapot_bty" runFrame.frame_id = "FD1S3I154O646UGSNN%s" % frameNum @@ -272,7 +272,7 @@ def main(): if args.test_script_frame: print("Launching script test frame (logs to /mcp)") - runFrame = opencue.compiled_proto.rqd_pb2.RunFrame() + runFrame = rqd.compiled_proto.rqd_pb2.RunFrame() runFrame.resource_id = "8888888877777755555" runFrame.job_id = "SD6F3S72DJ26236KFS" runFrame.job_name = "swtest-home-jwelborn_rqd_test" @@ -290,7 +290,7 @@ def main(): if args.test_script_frame_mac: print("Launching script test frame (logs to /tmp)") - runFrame = opencue.compiled_proto.rqd_pb2.RunFrame() + runFrame = rqd.compiled_proto.rqd_pb2.RunFrame() runFrame.resource_id = "2222222277777755555" runFrame.job_id = "SD6F3S72DJ26236KFS" runFrame.job_name = "swtest-home-jwelborn_rqd_test" diff --git a/rqd/rqd/rqcore.py b/rqd/rqd/rqcore.py index 665812091..c0d6d8397 100644 --- a/rqd/rqd/rqcore.py +++ b/rqd/rqd/rqcore.py @@ -36,9 +36,9 @@ import traceback import select -import opencue.cuelogging -import opencue.compiled_proto.host_pb2 -import opencue.compiled_proto.report_pb2 +import cuelogging +import rqd.compiled_proto.host_pb2 as host_pb3 +import rqd.compiled_proto.report_pb2 as report_pb3 import rqd.rqconstants import rqd.rqexceptions import rqd.rqmachine @@ -242,7 +242,7 @@ def __writeFooter(self): def __sendFrameCompleteReport(self): """Send report to cuebot that frame has finished""" - report = opencue.compiled_proto.report_pb2.FrameCompleteReport() + report = report_pb3.FrameCompleteReport() # pylint: disable=no-member report.host.CopyFrom(self.rqCore.machine.getHostInfo()) report.frame.CopyFrom(self.frameInfo.runningFrameInfo()) @@ -516,7 +516,7 @@ def run(self): # Setup frame logging try: - self.rqlog = opencue.cuelogging.CueLogWriter( + self.rqlog = cuelogging.CueLogWriter( runFrame.log_dir_file, maxLogFiles=rqd.rqconstants.MAX_LOG_FILES ) @@ -580,7 +580,7 @@ def __init__(self, optNimbyoff=False): self.__optNimbyoff = optNimbyoff - self.cores = opencue.compiled_proto.report_pb2.CoreDetail( + self.cores = report_pb3.CoreDetail( total_cores=0, idle_cores=0, locked_cores=0, @@ -824,7 +824,7 @@ def launchFrame(self, runFrame): # Check for reasons to abort launch # - if self.machine.state != opencue.compiled_proto.host_pb2.UP: + if self.machine.state != host_pb3.UP: err = "Not launching, rqd HardwareState is not Up" log.info(err) raise rqd.rqexceptions.CoreReservationFailureException(err) @@ -906,7 +906,7 @@ def reportStatus(self): def shutdownRqdNow(self): """Kill all running frames and shutdown RQD""" - self.machine.state = opencue.compiled_proto.host_pb2.DOWN + self.machine.state = host_pb3.DOWN try: self.lockAll() self.killAllFrame("shutdownRqdNow Command") @@ -1029,12 +1029,12 @@ def unlock(self, reqUnlock): sendUpdate = False if (self.__whenIdle or self.__reboot or - self.machine.state != opencue.compiled_proto.host_pb2.UP): + self.machine.state != host_pb3.UP): sendUpdate = True self.__whenIdle = False self.__reboot = False - self.machine.state = opencue.compiled_proto.host_pb2.UP + self.machine.state = host_pb3.UP with self.__threadLock: # pylint: disable=no-member @@ -1058,12 +1058,12 @@ def unlockAll(self): sendUpdate = False if (self.__whenIdle or self.__reboot - or self.machine.state != opencue.compiled_proto.host_pb2.UP): + or self.machine.state != host_pb3.UP): sendUpdate = True self.__whenIdle = False self.__reboot = False - self.machine.state = opencue.compiled_proto.host_pb2.UP + self.machine.state = host_pb3.UP with self.__threadLock: # pylint: disable=no-member diff --git a/rqd/rqd/rqdservicers.py b/rqd/rqd/rqdservicers.py index f7773160c..324a5a51a 100644 --- a/rqd/rqd/rqdservicers.py +++ b/rqd/rqd/rqdservicers.py @@ -24,14 +24,14 @@ import grpc -import opencue.compiled_proto.rqd_pb2 -import opencue.compiled_proto.rqd_pb2_grpc +import rqd.compiled_proto.rqd_pb2 +import rqd.compiled_proto.rqd_pb2_grpc log = logging.getLogger(__name__) -class RqdInterfaceServicer(opencue.compiled_proto.rqd_pb2_grpc.RqdInterfaceServicer): +class RqdInterfaceServicer(rqd.compiled_proto.rqd_pb2_grpc.RqdInterfaceServicer): """Service interface for RqdStatic gRPC definition.""" def __init__(self, rqCore): @@ -41,12 +41,12 @@ def LaunchFrame(self, request, context): """RPC call that launches the given frame""" log.info("Request received: launchFrame") self.rqCore.launchFrame(request.run_frame) - return opencue.compiled_proto.rqd_pb2.RqdStaticLaunchFrameResponse() + return rqd.compiled_proto.rqd_pb2.RqdStaticLaunchFrameResponse() def ReportStatus(self, request, context): """RPC call that returns reportStatus""" log.info("Request received: reportStatus") - return opencue.compiled_proto.rqd_pb2.RqdStaticReportStatusResponse( + return rqd.compiled_proto.rqd_pb2.RqdStaticReportStatusResponse( host_report=self.rqCore.reportStatus()) def GetRunningFrameStatus(self, request, context): @@ -54,12 +54,12 @@ def GetRunningFrameStatus(self, request, context): log.info("Request received: getRunningFrameStatus") frame = self.rqCore.getRunningFrame(request.frameId) if frame: - return opencue.compiled_proto.rqd_pb2.RqdStaticGetRunningFrameStatusResponse( + return rqd.compiled_proto.rqd_pb2.RqdStaticGetRunningFrameStatusResponse( running_frame_info=frame.runningFrameInfo()) context.set_details( "The requested frame was not found. frameId: {}".format(request.frameId)) context.set_code(grpc.StatusCode.NOT_FOUND) - return opencue.compiled_proto.rqd_pb2.RqdStaticGetRunningFrameStatusResponse() + return rqd.compiled_proto.rqd_pb2.RqdStaticGetRunningFrameStatusResponse() def KillRunningFrame(self, request, context): """RPC call that kills the running frame with the given id""" @@ -69,77 +69,77 @@ def KillRunningFrame(self, request, context): frame.kill(message=request.message) else: log.warning("Wasn't able to find frame(%s) to kill", request.frame_id) - return opencue.compiled_proto.rqd_pb2.RqdStaticKillRunningFrameResponse() + return rqd.compiled_proto.rqd_pb2.RqdStaticKillRunningFrameResponse() def ShutdownRqdNow(self, request, context): """RPC call that kills all running frames and shuts down rqd""" log.info("Request received: shutdownRqdNow") self.rqCore.shutdownRqdNow() - return opencue.compiled_proto.rqd_pb2.RqdStaticShutdownNowResponse() + return rqd.compiled_proto.rqd_pb2.RqdStaticShutdownNowResponse() def ShutdownRqdIdle(self, request, context): """RPC call that locks all cores and shuts down rqd when it is idle. unlockAll will abort the request.""" log.info("Request received: shutdownRqdIdle") self.rqCore.shutdownRqdIdle() - return opencue.compiled_proto.rqd_pb2.RqdStaticShutdownIdleResponse() + return rqd.compiled_proto.rqd_pb2.RqdStaticShutdownIdleResponse() def RestartRqdNow(self, request, context): """RPC call that kills all running frames and restarts rqd""" log.warning("Deprecated Request received: restartRqdNow. This request has no effect.") - return opencue.compiled_proto.rqd_pb2.RqdStaticRestartNowResponse() + return rqd.compiled_proto.rqd_pb2.RqdStaticRestartNowResponse() def RestartRqdIdle(self, request, context): """RPC call that that locks all cores and restarts rqd when idle. unlockAll will abort the request.""" log.warning("Deprecated Request received: restartRqdIdle. This request has no effect.") - return opencue.compiled_proto.rqd_pb2.RqdStaticRestartIdleResponse() + return rqd.compiled_proto.rqd_pb2.RqdStaticRestartIdleResponse() def RebootNow(self, request, context): """RPC call that kills all running frames and reboots the host.""" log.info("Request received: rebootNow") self.rqCore.rebootNow() - return opencue.compiled_proto.rqd_pb2.RqdStaticRebootNowResponse() + return rqd.compiled_proto.rqd_pb2.RqdStaticRebootNowResponse() def RebootIdle(self, request, context): """RPC call that that locks all cores and reboots the host when idle. unlockAll will abort the request.""" log.info("Request received: rebootIdle") self.rqCore.rebootIdle() - return opencue.compiled_proto.rqd_pb2.RqdStaticRebootIdleResponse() + return rqd.compiled_proto.rqd_pb2.RqdStaticRebootIdleResponse() def NimbyOn(self, request, context): """RPC call that activates nimby""" log.info("Request received: nimbyOn") self.rqCore.nimbyOn() - return opencue.compiled_proto.rqd_pb2.RqdStaticNimbyOnResponse() + return rqd.compiled_proto.rqd_pb2.RqdStaticNimbyOnResponse() def NimbyOff(self, request, context): """RPC call that deactivates nimby""" log.info("Request received: nimbyOff") self.rqCore.nimbyOff() - return opencue.compiled_proto.rqd_pb2.RqdStaticNimbyOffResponse() + return rqd.compiled_proto.rqd_pb2.RqdStaticNimbyOffResponse() def Lock(self, request, context): """RPC call that locks a specific number of cores""" log.info("Request received: lock %d", request.cores) self.rqCore.lock(request.cores) - return opencue.compiled_proto.rqd_pb2.RqdStaticLockResponse() + return rqd.compiled_proto.rqd_pb2.RqdStaticLockResponse() def LockAll(self, request, context): """RPC call that locks all cores""" log.info("Request received: lockAll") self.rqCore.lockAll() - return opencue.compiled_proto.rqd_pb2.RqdStaticLockAllResponse() + return rqd.compiled_proto.rqd_pb2.RqdStaticLockAllResponse() def Unlock(self, request, context): """RPC call that unlocks a specific number of cores""" log.info("Request received: unlock %d", request.cores) self.rqCore.unlock(request.cores) - return opencue.compiled_proto.rqd_pb2.RqdStaticUnlockResponse() + return rqd.compiled_proto.rqd_pb2.RqdStaticUnlockResponse() def UnlockAll(self, request, context): """RPC call that unlocks all cores""" log.info("Request received: unlockAll") self.rqCore.unlockAll() - return opencue.compiled_proto.rqd_pb2.RqdStaticUnlockAllResponse() + return rqd.compiled_proto.rqd_pb2.RqdStaticUnlockAllResponse() diff --git a/rqd/rqd/rqmachine.py b/rqd/rqd/rqmachine.py index c88cd135d..f67b8955d 100644 --- a/rqd/rqd/rqmachine.py +++ b/rqd/rqd/rqmachine.py @@ -57,8 +57,8 @@ import psutil -import opencue.compiled_proto.host_pb2 -import opencue.compiled_proto.report_pb2 +import rqd.compiled_proto.host_pb2 +import rqd.compiled_proto.report_pb2 import rqd.rqconstants import rqd.rqexceptions import rqd.rqswap @@ -75,7 +75,7 @@ def __init__(self, rqCore, coreInfo): """Machine class initialization @type rqCore: rqd.rqcore.RqCore @param rqCore: Main RQD Object, used to access frames and nimby states - @type coreInfo: opencue.compiled_proto.report_pb2.CoreDetail + @type coreInfo: rqd.compiled_proto.report_pb2.CoreDetail @param coreInfo: Object contains information on the state of all cores """ self.__rqCore = rqCore @@ -93,18 +93,18 @@ def __init__(self, rqCore, coreInfo): if platform.system() == 'Linux': self.__vmstat = rqd.rqswap.VmStat() - self.state = opencue.compiled_proto.host_pb2.UP + self.state = rqd.compiled_proto.host_pb2.UP - self.__renderHost = opencue.compiled_proto.report_pb2.RenderHost() + self.__renderHost = rqd.compiled_proto.report_pb2.RenderHost() self.__initMachineTags() self.__initMachineStats() - self.__bootReport = opencue.compiled_proto.report_pb2.BootReport() + self.__bootReport = rqd.compiled_proto.report_pb2.BootReport() # pylint: disable=no-member self.__bootReport.core_info.CopyFrom(self.__coreInfo) # pylint: enable=no-member - self.__hostReport = opencue.compiled_proto.report_pb2.HostReport() + self.__hostReport = rqd.compiled_proto.report_pb2.HostReport() # pylint: disable=no-member self.__hostReport.core_info.CopyFrom(self.__coreInfo) # pylint: enable=no-member diff --git a/rqd/rqd/rqnetwork.py b/rqd/rqd/rqnetwork.py index b2b610ecb..d3d0ce95f 100644 --- a/rqd/rqd/rqnetwork.py +++ b/rqd/rqd/rqnetwork.py @@ -34,9 +34,9 @@ import grpc -import opencue.compiled_proto.report_pb2 -import opencue.compiled_proto.report_pb2_grpc -import opencue.compiled_proto.rqd_pb2_grpc +import rqd.compiled_proto.report_pb2 +import rqd.compiled_proto.report_pb2_grpc +import rqd.compiled_proto.rqd_pb2_grpc import rqd.rqconstants import rqd.rqexceptions import rqd.rqdservicers @@ -82,7 +82,7 @@ def __init__(self, rqCore, runFrame): def runningFrameInfo(self): """Returns the RunningFrameInfo object""" - runningFrameInfo = opencue.compiled_proto.report_pb2.RunningFrameInfo( + runningFrameInfo = rqd.compiled_proto.report_pb2.RunningFrameInfo( resource_id=self.runFrame.resource_id, job_id=self.runFrame.job_id, job_name=self.runFrame.job_name, @@ -114,13 +114,13 @@ def _serializeChildrenProcs(self): :param data: dictionary :return: serialized children proc host stats - :rtype: opencue.compiled_proto.report_pb2.ChildrenProcStats + :rtype: rqd.compiled_proto.report_pb2.ChildrenProcStats """ - childrenProc = opencue.compiled_proto.report_pb2.ChildrenProcStats() + childrenProc = rqd.compiled_proto.report_pb2.ChildrenProcStats() for proc, values in self.childrenProcs.items(): - procStats = opencue.compiled_proto.report_pb2.ProcStats() - procStatFile = opencue.compiled_proto.report_pb2.Stat() - procStatmFile = opencue.compiled_proto.report_pb2.Statm() + procStats = rqd.compiled_proto.report_pb2.ProcStats() + procStatFile = rqd.compiled_proto.report_pb2.Stat() + procStatmFile = rqd.compiled_proto.report_pb2.Statm() procStatFile.pid = proc procStatFile.name = values["name"] if values["name"] else "" @@ -197,7 +197,7 @@ def __init__(self, rqCore): def addServicers(self): """Registers the gRPC servicers defined in rqdservicers.py.""" for servicer in self.servicers: - addFunc = getattr(opencue.compiled_proto.rqd_pb2_grpc, + addFunc = getattr(rqd.compiled_proto.rqd_pb2_grpc, 'add_{0}_to_server'.format(servicer)) servicerClass = getattr(rqd.rqdservicers, servicer) addFunc(servicerClass(self.rqCore), self.server) @@ -297,25 +297,25 @@ def __getChannel(self): def __getReportStub(self): self.__getChannel() - return opencue.compiled_proto.report_pb2_grpc.RqdReportInterfaceStub(self.channel) + return rqd.compiled_proto.report_pb2_grpc.RqdReportInterfaceStub(self.channel) def reportRqdStartup(self, report): """Wraps the ability to send a startup report to rqd via grpc""" stub = self.__getReportStub() - request = opencue.compiled_proto.report_pb2.RqdReportRqdStartupRequest(boot_report=report) + request = rqd.compiled_proto.report_pb2.RqdReportRqdStartupRequest(boot_report=report) stub.ReportRqdStartup(request, timeout=rqd.rqconstants.RQD_TIMEOUT) def reportStatus(self, report): """Wraps the ability to send a status report to the cuebot via grpc""" stub = self.__getReportStub() - request = opencue.compiled_proto.report_pb2.RqdReportStatusRequest(host_report=report) + request = rqd.compiled_proto.report_pb2.RqdReportStatusRequest(host_report=report) stub.ReportStatus(request, timeout=rqd.rqconstants.RQD_TIMEOUT) def reportRunningFrameCompletion(self, report): """Wraps the ability to send a running frame completion report to the cuebot via grpc""" stub = self.__getReportStub() - request = opencue.compiled_proto.report_pb2.RqdReportRunningFrameCompletionRequest( + request = rqd.compiled_proto.report_pb2.RqdReportRunningFrameCompletionRequest( frame_complete_report=report) stub.ReportRunningFrameCompletion(request, timeout=rqd.rqconstants.RQD_TIMEOUT) From d10d012d4b89ce6167d30a81983eed5f34639c71 Mon Sep 17 00:00:00 2001 From: Jimmy Christensen Date: Thu, 12 Sep 2024 01:18:30 +0200 Subject: [PATCH 19/32] Fix linting and test scripts --- ci/build_sphinx_docs.sh | 2 ++ ci/python_coverage_report.sh | 2 ++ ci/run_python_lint.sh | 2 ++ ci/run_python_tests_pyside6.sh | 2 ++ ci/test_pyside6.sh | 2 ++ 5 files changed, 10 insertions(+) diff --git a/ci/build_sphinx_docs.sh b/ci/build_sphinx_docs.sh index 9f207d51e..412dcd6bb 100755 --- a/ci/build_sphinx_docs.sh +++ b/ci/build_sphinx_docs.sh @@ -7,10 +7,12 @@ pip install --user -r requirements.txt -r docs/requirements.txt # Must generate Python code from Protos in order for Sphinx to build the docs. python -m grpc_tools.protoc -I=proto/ --python_out=pycue/opencue/compiled_proto --grpc_python_out=pycue/opencue/compiled_proto proto/*.proto +python -m grpc_tools.protoc -I=proto/ --python_out=rqd/rqd/compiled_proto --grpc_python_out=rqd/rqd/compiled_proto proto/*.proto # Fix imports to work in both Python 2 and 3. See # for more info. 2to3 -wn -f import pycue/opencue/compiled_proto/*_pb2*.py +2to3 -wn -f import rqd/rqd/compiled_proto/*_pb2*.py # Build the docs and treat warnings as errors ~/.local/bin/sphinx-build -W -b html -d docs/_build/doctrees docs docs/_build/html diff --git a/ci/python_coverage_report.sh b/ci/python_coverage_report.sh index dbdb88cc7..d1ab1d2bd 100755 --- a/ci/python_coverage_report.sh +++ b/ci/python_coverage_report.sh @@ -6,7 +6,9 @@ pip install --user -r requirements.txt -r requirements_gui.txt # Protos need to have their Python code generated in order for tests to pass. python -m grpc_tools.protoc -I=proto/ --python_out=pycue/opencue/compiled_proto --grpc_python_out=pycue/opencue/compiled_proto proto/*.proto +python -m grpc_tools.protoc -I=proto/ --python_out=rqd/rqd/compiled_proto --grpc_python_out=rqd/rqd/compiled_proto proto/*.proto 2to3 -wn -f import pycue/opencue/compiled_proto/*_pb2*.py +2to3 -wn -f import rqd/rqd/compiled_proto/*_pb2*.py # Run coverage for each component individually, but append it all into the same report. coverage run --source=pycue/opencue/,pycue/FileSequence/ --omit=pycue/opencue/compiled_proto/* pycue/setup.py test diff --git a/ci/run_python_lint.sh b/ci/run_python_lint.sh index f950a7a4c..8c68efb49 100755 --- a/ci/run_python_lint.sh +++ b/ci/run_python_lint.sh @@ -9,10 +9,12 @@ pip install --user -r requirements.txt -r requirements_gui.txt # Protos need to have their Python code generated in order for tests to pass. python -m grpc_tools.protoc -I=proto/ --python_out=pycue/opencue/compiled_proto --grpc_python_out=pycue/opencue/compiled_proto proto/*.proto +python -m grpc_tools.protoc -I=proto/ --python_out=rqd/rqd/compiled_proto --grpc_python_out=rqd/rqd/compiled_proto proto/*.proto # Fix imports to work in both Python 2 and 3. See # for more info. 2to3 -wn -f import pycue/opencue/compiled_proto/*_pb2*.py +2to3 -wn -f import rqd/rqd/compiled_proto/*_pb2*.py echo "Running lint for pycue/..." cd pycue diff --git a/ci/run_python_tests_pyside6.sh b/ci/run_python_tests_pyside6.sh index 8f4c46941..384841cfe 100755 --- a/ci/run_python_tests_pyside6.sh +++ b/ci/run_python_tests_pyside6.sh @@ -33,9 +33,11 @@ python3 -m pip install --user PySide6==6.3.2 # Protos need to have their Python code generated in order for tests to pass. python -m grpc_tools.protoc -I=proto/ --python_out=pycue/opencue/compiled_proto --grpc_python_out=pycue/opencue/compiled_proto proto/*.proto +python -m grpc_tools.protoc -I=proto/ --python_out=rqd/rqd/compiled_proto --grpc_python_out=rqd/rqd/compiled_proto proto/*.proto # Fix compiled proto code for Python 3. 2to3 -wn -f import pycue/opencue/compiled_proto/*_pb2*.py +2to3 -wn -f import rqd/rqd/compiled_proto/*_pb2*.py python pycue/setup.py test PYTHONPATH=pycue python pyoutline/setup.py test diff --git a/ci/test_pyside6.sh b/ci/test_pyside6.sh index 05bd4c173..b6ac1319a 100755 --- a/ci/test_pyside6.sh +++ b/ci/test_pyside6.sh @@ -30,7 +30,9 @@ python3 -m pip install --user PySide6==6.3.2 # Fix compiled proto code for Python 3. python3 -m grpc_tools.protoc -I=proto/ --python_out=pycue/opencue/compiled_proto --grpc_python_out=pycue/opencue/compiled_proto proto/*.proto +python -m grpc_tools.protoc -I=proto/ --python_out=rqd/rqd/compiled_proto --grpc_python_out=rqd/rqd/compiled_proto proto/*.proto 2to3 -wn -f import pycue/opencue/compiled_proto/*_pb2*.py +2to3 -wn -f import rqd/rqd/compiled_proto/*_pb2*.py # Run tests. ci/run_gui_test.sh From 6253faef96f12778690c86f5b28eacdf9fee5999 Mon Sep 17 00:00:00 2001 From: Jimmy Christensen Date: Thu, 12 Sep 2024 01:23:25 +0200 Subject: [PATCH 20/32] Small type fix to the pyside6 test --- ci/test_pyside6.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/test_pyside6.sh b/ci/test_pyside6.sh index b6ac1319a..6bb79d1a3 100755 --- a/ci/test_pyside6.sh +++ b/ci/test_pyside6.sh @@ -30,7 +30,7 @@ python3 -m pip install --user PySide6==6.3.2 # Fix compiled proto code for Python 3. python3 -m grpc_tools.protoc -I=proto/ --python_out=pycue/opencue/compiled_proto --grpc_python_out=pycue/opencue/compiled_proto proto/*.proto -python -m grpc_tools.protoc -I=proto/ --python_out=rqd/rqd/compiled_proto --grpc_python_out=rqd/rqd/compiled_proto proto/*.proto +python3 -m grpc_tools.protoc -I=proto/ --python_out=rqd/rqd/compiled_proto --grpc_python_out=rqd/rqd/compiled_proto proto/*.proto 2to3 -wn -f import pycue/opencue/compiled_proto/*_pb2*.py 2to3 -wn -f import rqd/rqd/compiled_proto/*_pb2*.py From fde5d48190c5f6e54500661d089be32d60697814 Mon Sep 17 00:00:00 2001 From: Jimmy Christensen Date: Thu, 12 Sep 2024 01:32:36 +0200 Subject: [PATCH 21/32] Split cuelogging into separate files/modules --- pycue/cuelogging/CueLogReader.py | 59 ++++++++++++ pycue/cuelogging/CueLogWriter.py | 126 +++++++++++++++++++++++++ pycue/cuelogging/__init__.py | 155 +------------------------------ 3 files changed, 188 insertions(+), 152 deletions(-) create mode 100644 pycue/cuelogging/CueLogReader.py create mode 100644 pycue/cuelogging/CueLogWriter.py diff --git a/pycue/cuelogging/CueLogReader.py b/pycue/cuelogging/CueLogReader.py new file mode 100644 index 000000000..4bfb6d5bd --- /dev/null +++ b/pycue/cuelogging/CueLogReader.py @@ -0,0 +1,59 @@ +# Copyright Contributors to the OpenCue Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Module for reading files""" + +import os + +class CueLogReader(object): + """Class to abstract file log reading, this class tries to act as a file object""" + + filepath = None + + def __init__(self, filepath): + """CueLogWriter class initialization + @type filepath: string + @param filepath: The filepath to log to + """ + self.filepath = filepath + + def size(self): + """Return the size of the file""" + return int(os.stat(self.filepath).st_size) + + def getMtime(self): + """Return modification time of the file""" + return os.path.getmtime(self.filepath) + + def exists(self): + """Check if the file exists""" + return os.path.exists(self.filepath) + + def read(self): + """Read the data from the backend""" + + content = None + if self.exists() is True: + with open(self.filepath, "r", encoding='utf-8') as fp: + content = fp.read() + else: + raise IOError("Failed to open %s" % self.filepath) + + return content + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + pass diff --git a/pycue/cuelogging/CueLogWriter.py b/pycue/cuelogging/CueLogWriter.py new file mode 100644 index 000000000..923849642 --- /dev/null +++ b/pycue/cuelogging/CueLogWriter.py @@ -0,0 +1,126 @@ +# Copyright Contributors to the OpenCue Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Module for writing files""" + +import os +import platform +import datetime +import time + +class CueLogWriter(object): + """Class to abstract file log writing, this class tries to act as a file object""" + + filepath = None + + def __init__(self, filepath, maxLogFiles=1): + """CueLogWriter class initialization + @type filepath: string + @param filepath: The filepath to log to + @type maxLogFiles: int + @param maxLogFiles: number of files to rotate, when in write mode + """ + + self.filepath = filepath + self.__log_dir = os.path.dirname(self.filepath) + self.__maxLogFiles = maxLogFiles + if not os.access(self.__log_dir, os.F_OK): + self.__makeLogDir() + + if not os.access(self.__log_dir, os.W_OK): + err = "Unable to write to log directory %s" % self.__log_dir + raise RuntimeError(err) + + self.__attemptRotateLogs() + # pylint: disable=consider-using-with + self.__fd = open(self.filepath, "w+", 1, encoding='utf-8') + try: + os.chmod(self.filepath, 0o666) + # pylint: disable=broad-except + except Exception as e: + err = "Failed to chmod log file! %s due to %s" % (self.filepath, e) + log.warning(err) + + def __attemptRotateLogs(self): + try: + # Rotate any old logs to a max of MAX_LOG_FILES: + if os.path.isfile(self.filepath): + rotateCount = 1 + while (os.path.isfile("%s.%s" % (self.filepath, rotateCount)) + and rotateCount < self.__maxLogFiles): + rotateCount += 1 + os.rename(self.filepath, + "%s.%s" % (self.filepath, rotateCount)) + # pylint: disable=broad-except + except Exception as e: + err = "Unable to rotate previous log file due to %s" % e + # Windows might fail while trying to rotate logs for checking if file is + # being used by another process. Frame execution doesn't need to + # be halted for this. + if platform.system() == "Windows": + log.warning(err) + else: + raise RuntimeError(err) + + def __makeLogDir(self): + # Attempting mkdir for missing logdir + msg = "No Error" + try: + os.makedirs(self.__log_dir) + os.chmod(self.__log_dir, 0o777) + # pylint: disable=broad-except + except Exception as e: + # This is expected to fail when called in abq + # But the directory should now be visible + msg = e + + if not os.access(self.__log_dir, os.F_OK): + err = "Unable to see log directory: %s, mkdir failed with: %s" % (self.__log_dir, msg) + raise RuntimeError(err) + + # pylint: disable=arguments-differ + def write(self, data, prependTimestamp=False): + """Abstract write function that will write to the correct backend""" + # Convert data to unicode + if isinstance(data, bytes): + data = data.decode('utf-8', errors='ignore') + if prependTimestamp is True: + lines = data.splitlines() + curr_line_timestamp = datetime.datetime.now().strftime("%H:%M:%S") + for line in lines: + print("[%s] %s" % (curr_line_timestamp, line), file=self) + else: + self.__fd.write(data) + + def writelines(self, __lines): + """Provides support for writing mutliple lines at a time""" + for line in __lines: + self.write(line) + + def close(self): + """Closes the file if the backend is file based""" + self.__fd.close() + + def waitForFile(self, maxTries=5): + """Waits for the file to exist before continuing when using a file backend""" + # Waits for a file to exist + tries = 0 + while tries < maxTries: + if os.path.exists(self.filepath): + return + tries += 1 + time.sleep(0.5 * tries) + raise IOError("Failed to create %s" % self.filepath) + + diff --git a/pycue/cuelogging/__init__.py b/pycue/cuelogging/__init__.py index 4564cf457..caa9a9c6f 100644 --- a/pycue/cuelogging/__init__.py +++ b/pycue/cuelogging/__init__.py @@ -13,159 +13,10 @@ # limitations under the License. -"""Module for reading and writing log files""" +"""Package for reading and writing log files""" import logging -import os -import platform -import datetime -import time +from .CueLogReader import CueLogReader +from .CueLogWriter import CueLogWriter log = logging.getLogger(__name__) - -class CueLogWriter(object): - """Class to abstract file log writing, this class tries to act as a file object""" - - filepath = None - - def __init__(self, filepath, maxLogFiles=1): - """CueLogWriter class initialization - @type filepath: string - @param filepath: The filepath to log to - @type maxLogFiles: int - @param maxLogFiles: number of files to rotate, when in write mode - """ - - self.filepath = filepath - self.__log_dir = os.path.dirname(self.filepath) - self.__maxLogFiles = maxLogFiles - if not os.access(self.__log_dir, os.F_OK): - self.__makeLogDir() - - if not os.access(self.__log_dir, os.W_OK): - err = "Unable to write to log directory %s" % self.__log_dir - raise RuntimeError(err) - - self.__attemptRotateLogs() - # pylint: disable=consider-using-with - self.__fd = open(self.filepath, "w+", 1, encoding='utf-8') - try: - os.chmod(self.filepath, 0o666) - # pylint: disable=broad-except - except Exception as e: - err = "Failed to chmod log file! %s due to %s" % (self.filepath, e) - log.warning(err) - - def __attemptRotateLogs(self): - try: - # Rotate any old logs to a max of MAX_LOG_FILES: - if os.path.isfile(self.filepath): - rotateCount = 1 - while (os.path.isfile("%s.%s" % (self.filepath, rotateCount)) - and rotateCount < self.__maxLogFiles): - rotateCount += 1 - os.rename(self.filepath, - "%s.%s" % (self.filepath, rotateCount)) - # pylint: disable=broad-except - except Exception as e: - err = "Unable to rotate previous log file due to %s" % e - # Windows might fail while trying to rotate logs for checking if file is - # being used by another process. Frame execution doesn't need to - # be halted for this. - if platform.system() == "Windows": - log.warning(err) - else: - raise RuntimeError(err) - - def __makeLogDir(self): - # Attempting mkdir for missing logdir - msg = "No Error" - try: - os.makedirs(self.__log_dir) - os.chmod(self.__log_dir, 0o777) - # pylint: disable=broad-except - except Exception as e: - # This is expected to fail when called in abq - # But the directory should now be visible - msg = e - - if not os.access(self.__log_dir, os.F_OK): - err = "Unable to see log directory: %s, mkdir failed with: %s" % (self.__log_dir, msg) - raise RuntimeError(err) - - # pylint: disable=arguments-differ - def write(self, data, prependTimestamp=False): - """Abstract write function that will write to the correct backend""" - # Convert data to unicode - if isinstance(data, bytes): - data = data.decode('utf-8', errors='ignore') - if prependTimestamp is True: - lines = data.splitlines() - curr_line_timestamp = datetime.datetime.now().strftime("%H:%M:%S") - for line in lines: - print("[%s] %s" % (curr_line_timestamp, line), file=self) - else: - self.__fd.write(data) - - def writelines(self, __lines): - """Provides support for writing mutliple lines at a time""" - for line in __lines: - self.write(line) - - def close(self): - """Closes the file if the backend is file based""" - self.__fd.close() - - def waitForFile(self, maxTries=5): - """Waits for the file to exist before continuing when using a file backend""" - # Waits for a file to exist - tries = 0 - while tries < maxTries: - if os.path.exists(self.filepath): - return - tries += 1 - time.sleep(0.5 * tries) - raise IOError("Failed to create %s" % self.filepath) - - -class CueLogReader(object): - """Class to abstract file log reading, this class tries to act as a file object""" - - filepath = None - - def __init__(self, filepath): - """CueLogWriter class initialization - @type filepath: string - @param filepath: The filepath to log to - """ - self.filepath = filepath - - def size(self): - """Return the size of the file""" - return int(os.stat(self.filepath).st_size) - - def getMtime(self): - """Return modification time of the file""" - return os.path.getmtime(self.filepath) - - def exists(self): - """Check if the file exists""" - return os.path.exists(self.filepath) - - def read(self): - """Read the data from the backend""" - - content = None - if self.exists() is True: - with open(self.filepath, "r", encoding='utf-8') as fp: - content = fp.read() - else: - raise IOError("Failed to open %s" % self.filepath) - - return content - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - pass From 59b6b71b00118c1021b45ab1489f5ec121639e0a Mon Sep 17 00:00:00 2001 From: Jimmy Christensen Date: Thu, 12 Sep 2024 01:40:38 +0200 Subject: [PATCH 22/32] Fix linting of the new proto modules --- ci/pylintrc_main | 7 ++++++- ci/pylintrc_test | 7 ++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/ci/pylintrc_main b/ci/pylintrc_main index b53ad2ae5..8b19497f9 100644 --- a/ci/pylintrc_main +++ b/ci/pylintrc_main @@ -207,7 +207,12 @@ ignored-modules=opencue.compiled_proto, opencue.compiled_proto.filter_pb2, opencue.compiled_proto.host_pb2, opencue.compiled_proto.rqd_pb2, - opencue.compiled_proto.report_pb2 + opencue.compiled_proto.report_pb2, + rqd.compiled_proto, + rqd.compiled_proto.filter_pb2, + rqd.compiled_proto.host_pb2, + rqd.compiled_proto.rqd_pb2, + rqd.compiled_proto.report_pb2 # Show a hint with possible names when a member name was not found. The aspect # of finding the hint is based on edit distance. diff --git a/ci/pylintrc_test b/ci/pylintrc_test index 6578bab2b..4603f50a4 100644 --- a/ci/pylintrc_test +++ b/ci/pylintrc_test @@ -207,7 +207,12 @@ ignored-modules=opencue.compiled_proto, opencue.compiled_proto.filter_pb2, opencue.compiled_proto.host_pb2, opencue.compiled_proto.rqd_pb2, - opencue.compiled_proto.report_pb2 + opencue.compiled_proto.report_pb2, + rqd.compiled_proto, + rqd.compiled_proto.filter_pb2, + rqd.compiled_proto.host_pb2, + rqd.compiled_proto.rqd_pb2, + rqd.compiled_proto.report_pb2 # Show a hint with possible names when a member name was not found. The aspect # of finding the hint is based on edit distance. From c524aa19f7813bf0c65b422ca091fd861debdb07 Mon Sep 17 00:00:00 2001 From: Jimmy Christensen Date: Thu, 12 Sep 2024 01:47:47 +0200 Subject: [PATCH 23/32] More fixes to the linting --- ci/pylintrc_main | 18 +++++++++--------- ci/run_python_lint.sh | 2 +- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/ci/pylintrc_main b/ci/pylintrc_main index 8b19497f9..8bf017821 100644 --- a/ci/pylintrc_main +++ b/ci/pylintrc_main @@ -204,15 +204,15 @@ ignored-classes=optparse.Values,thread._local,_thread._local # and thus existing member attributes cannot be deduced by static analysis). It # supports qualified module names, as well as Unix pattern matching. ignored-modules=opencue.compiled_proto, - opencue.compiled_proto.filter_pb2, - opencue.compiled_proto.host_pb2, - opencue.compiled_proto.rqd_pb2, - opencue.compiled_proto.report_pb2, - rqd.compiled_proto, - rqd.compiled_proto.filter_pb2, - rqd.compiled_proto.host_pb2, - rqd.compiled_proto.rqd_pb2, - rqd.compiled_proto.report_pb2 + opencue.compiled_proto.filter_pb2, + opencue.compiled_proto.host_pb2, + opencue.compiled_proto.rqd_pb2, + opencue.compiled_proto.report_pb2, + rqd.compiled_proto, + rqd.compiled_proto.filter_pb2, + rqd.compiled_proto.host_pb2, + rqd.compiled_proto.rqd_pb2, + rqd.compiled_proto.report_pb2 # Show a hint with possible names when a member name was not found. The aspect # of finding the hint is based on edit distance. diff --git a/ci/run_python_lint.sh b/ci/run_python_lint.sh index 8c68efb49..2cde836ea 100755 --- a/ci/run_python_lint.sh +++ b/ci/run_python_lint.sh @@ -49,6 +49,6 @@ cd .. echo "Running lint for rqd/..." cd rqd -PYTHONPATH=../pycue python -m pylint --rcfile=../ci/pylintrc_main rqd --ignore=pycue/opencue/compiled_proto +PYTHONPATH=../pycue python -m pylint --rcfile=../ci/pylintrc_main rqd --ignore=rqd/compiled_proto PYTHONPATH=../pycue python -m pylint --rcfile=../ci/pylintrc_test tests cd .. From d6a3310916a331bf697bd461eff96ce73802466b Mon Sep 17 00:00:00 2001 From: Jimmy Christensen Date: Thu, 12 Sep 2024 01:51:29 +0200 Subject: [PATCH 24/32] Small change to coverage --- ci/python_coverage_report.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/python_coverage_report.sh b/ci/python_coverage_report.sh index d1ab1d2bd..e21f0259e 100755 --- a/ci/python_coverage_report.sh +++ b/ci/python_coverage_report.sh @@ -16,7 +16,7 @@ PYTHONPATH=pycue coverage run -a --source=pyoutline/outline/ pyoutline/setup.py PYTHONPATH=pycue coverage run -a --source=cueadmin/cueadmin/ cueadmin/setup.py test PYTHONPATH=pycue xvfb-run -d coverage run -a --source=cuegui/cuegui/ cuegui/setup.py test PYTHONPATH=pycue:pyoutline coverage run -a --source=cuesubmit/cuesubmit/ cuesubmit/setup.py test -coverage run -a --source=rqd/rqd/ --omit=pycue/opencue/compiled_proto/* rqd/setup.py test +coverage run -a --source=rqd/rqd/ --omit=rqd/rqd/compiled_proto/* rqd/setup.py test # SonarCloud needs the report in XML. coverage xml From 53840137aa5c28a75446275d0e3730f8ed74a3c2 Mon Sep 17 00:00:00 2001 From: Jimmy Christensen Date: Thu, 12 Sep 2024 01:52:47 +0200 Subject: [PATCH 25/32] Revert changes for proto/README.md --- proto/README.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/proto/README.md b/proto/README.md index cc7361bc6..0b039b462 100644 --- a/proto/README.md +++ b/proto/README.md @@ -8,6 +8,24 @@ To use them, they must first be compiled into the native language of the compone Gradle automatically compiles these proto files, no further action is needed. +## RQD + +To generate: + +```sh +python -m grpc_tools.protoc -I=. --python_out=../rqd/rqd/compiled_proto --grpc_python_out=../rqd/rqd/compiled_proto ./*.proto +2to3 -wn -f import ../rqd/rqd/compiled_proto/*_pb2*.py +``` + +For Windows (Powershell): + +```powershell +python -m grpc_tools.protoc --proto_path=. --python_out=../rqd/rqd/compiled_proto --grpc_python_out=../rqd/rqd/compiled_proto (ls *.proto).Name +cd ..\rqd\rqd\compiled_proto\ +2to3 -wn -f import (ls *_pb2*.py).Name +``` + + ## pycue To generate: From 21b56c372c5d898abc0aa277debc941d7485a089 Mon Sep 17 00:00:00 2001 From: Jimmy Christensen Date: Thu, 12 Sep 2024 01:53:45 +0200 Subject: [PATCH 26/32] Revert changes --- rqd/Dockerfile | 10 +++ rqd/rqd/compiled_proto/__init__.py | 13 ++++ rqd/rqd/rqcore.py | 20 +++--- rqd/rqd/rqnetwork.py | 3 +- rqd/tests/cuerqd_tests.py | 39 +++++------ rqd/tests/rqconstants_tests.py | 4 +- rqd/tests/rqcore_tests.py | 109 ++++++++++++++--------------- rqd/tests/rqmachine_tests.py | 20 +++--- rqd/tests/test_cuebot_listener.py | 6 +- 9 files changed, 118 insertions(+), 106 deletions(-) diff --git a/rqd/Dockerfile b/rqd/Dockerfile index d3c1bca9f..c3a0a0dc1 100644 --- a/rqd/Dockerfile +++ b/rqd/Dockerfile @@ -27,6 +27,16 @@ COPY rqd/setup.py ./rqd/ COPY rqd/tests/ ./rqd/tests COPY rqd/rqd/ ./rqd/rqd +RUN python3.9 -m grpc_tools.protoc \ + -I=./proto \ + --python_out=./rqd/rqd/compiled_proto \ + --grpc_python_out=./rqd/rqd/compiled_proto \ + ./proto/*.proto + +# Fix imports to work in both Python 2 and 3. See +# for more info. +RUN 2to3 -wn -f import rqd/rqd/compiled_proto/*_pb2*.py + COPY VERSION.in VERSIO[N] ./ RUN test -e VERSION || echo "$(cat VERSION.in)" | tee VERSION diff --git a/rqd/rqd/compiled_proto/__init__.py b/rqd/rqd/compiled_proto/__init__.py index e69de29bb..d9c80f13d 100644 --- a/rqd/rqd/compiled_proto/__init__.py +++ b/rqd/rqd/compiled_proto/__init__.py @@ -0,0 +1,13 @@ +# Copyright Contributors to the OpenCue Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/rqd/rqd/rqcore.py b/rqd/rqd/rqcore.py index c0d6d8397..4efdbab4a 100644 --- a/rqd/rqd/rqcore.py +++ b/rqd/rqd/rqcore.py @@ -37,8 +37,8 @@ import select import cuelogging -import rqd.compiled_proto.host_pb2 as host_pb3 -import rqd.compiled_proto.report_pb2 as report_pb3 +import rqd.compiled_proto.host_pb2 +import rqd.compiled_proto.report_pb2 import rqd.rqconstants import rqd.rqexceptions import rqd.rqmachine @@ -242,7 +242,7 @@ def __writeFooter(self): def __sendFrameCompleteReport(self): """Send report to cuebot that frame has finished""" - report = report_pb3.FrameCompleteReport() + report = rqd.compiled_proto.report_pb2.FrameCompleteReport() # pylint: disable=no-member report.host.CopyFrom(self.rqCore.machine.getHostInfo()) report.frame.CopyFrom(self.frameInfo.runningFrameInfo()) @@ -580,7 +580,7 @@ def __init__(self, optNimbyoff=False): self.__optNimbyoff = optNimbyoff - self.cores = report_pb3.CoreDetail( + self.cores = rqd.compiled_proto.report_pb2.CoreDetail( total_cores=0, idle_cores=0, locked_cores=0, @@ -824,7 +824,7 @@ def launchFrame(self, runFrame): # Check for reasons to abort launch # - if self.machine.state != host_pb3.UP: + if self.machine.state != rqd.compiled_proto.host_pb2.UP: err = "Not launching, rqd HardwareState is not Up" log.info(err) raise rqd.rqexceptions.CoreReservationFailureException(err) @@ -906,7 +906,7 @@ def reportStatus(self): def shutdownRqdNow(self): """Kill all running frames and shutdown RQD""" - self.machine.state = host_pb3.DOWN + self.machine.state = rqd.compiled_proto.host_pb2.DOWN try: self.lockAll() self.killAllFrame("shutdownRqdNow Command") @@ -1029,12 +1029,12 @@ def unlock(self, reqUnlock): sendUpdate = False if (self.__whenIdle or self.__reboot or - self.machine.state != host_pb3.UP): + self.machine.state != rqd.compiled_proto.host_pb2.UP): sendUpdate = True self.__whenIdle = False self.__reboot = False - self.machine.state = host_pb3.UP + self.machine.state = rqd.compiled_proto.host_pb2.UP with self.__threadLock: # pylint: disable=no-member @@ -1058,12 +1058,12 @@ def unlockAll(self): sendUpdate = False if (self.__whenIdle or self.__reboot - or self.machine.state != host_pb3.UP): + or self.machine.state != rqd.compiled_proto.host_pb2.UP): sendUpdate = True self.__whenIdle = False self.__reboot = False - self.machine.state = host_pb3.UP + self.machine.state = rqd.compiled_proto.host_pb2.UP with self.__threadLock: # pylint: disable=no-member diff --git a/rqd/rqd/rqnetwork.py b/rqd/rqd/rqnetwork.py index d3d0ce95f..de1b38475 100644 --- a/rqd/rqd/rqnetwork.py +++ b/rqd/rqd/rqnetwork.py @@ -197,8 +197,7 @@ def __init__(self, rqCore): def addServicers(self): """Registers the gRPC servicers defined in rqdservicers.py.""" for servicer in self.servicers: - addFunc = getattr(rqd.compiled_proto.rqd_pb2_grpc, - 'add_{0}_to_server'.format(servicer)) + addFunc = getattr(rqd.compiled_proto.rqd_pb2_grpc, 'add_{0}_to_server'.format(servicer)) servicerClass = getattr(rqd.rqdservicers, servicer) addFunc(servicerClass(self.rqCore), self.server) diff --git a/rqd/tests/cuerqd_tests.py b/rqd/tests/cuerqd_tests.py index 6081e7d9b..ad647d770 100644 --- a/rqd/tests/cuerqd_tests.py +++ b/rqd/tests/cuerqd_tests.py @@ -28,16 +28,16 @@ import mock -import opencue.compiled_proto.rqd_pb2 import rqd.cuerqd +import rqd.compiled_proto.rqd_pb2 SCRIPT_NAME = '/arbitrary/path/to/script' RQD_HOSTNAME = 'arbitrary-rqd-hostname' -@mock.patch('opencue.compiled_proto.rqd_pb2_grpc.RunningFrameStub') -@mock.patch('opencue.compiled_proto.rqd_pb2_grpc.RqdInterfaceStub') +@mock.patch('rqd.compiled_proto.rqd_pb2_grpc.RunningFrameStub') +@mock.patch('rqd.compiled_proto.rqd_pb2_grpc.RqdInterfaceStub') @mock.patch('grpc.insecure_channel', new=mock.MagicMock()) class CueRqdTests(unittest.TestCase): @@ -59,7 +59,7 @@ def test_initWithLocalhost(self, rqdHostMock, stubMock, frameStubMock): def test_status(self, stubMock, frameStubMock): sys.argv = [SCRIPT_NAME, RQD_HOSTNAME, '-s'] - statusRequest = opencue.compiled_proto.rqd_pb2.RqdStaticReportStatusRequest() + statusRequest = rqd.compiled_proto.rqd_pb2.RqdStaticReportStatusRequest() rqd.cuerqd.main() @@ -68,8 +68,7 @@ def test_status(self, stubMock, frameStubMock): def test_getRunningFrame(self, stubMock, frameStubMock): frameId = 'arbitrary-frame-id' sys.argv = [SCRIPT_NAME, RQD_HOSTNAME, '--getproxy', frameId] - runFrameRequest = opencue.compiled_proto.rqd_pb2.RqdStaticGetRunFrameRequest( - frame_id=frameId) + runFrameRequest = rqd.compiled_proto.rqd_pb2.RqdStaticGetRunFrameRequest(frame_id=frameId) rqd.cuerqd.main() @@ -77,7 +76,7 @@ def test_getRunningFrame(self, stubMock, frameStubMock): def test_nimbyOff(self, stubMock, frameStubMock): sys.argv = [SCRIPT_NAME, RQD_HOSTNAME, '--nimbyoff'] - nimbyOffRequest = opencue.compiled_proto.rqd_pb2.RqdStaticNimbyOffRequest() + nimbyOffRequest = rqd.compiled_proto.rqd_pb2.RqdStaticNimbyOffRequest() rqd.cuerqd.main() @@ -85,7 +84,7 @@ def test_nimbyOff(self, stubMock, frameStubMock): def test_nimbyOn(self, stubMock, frameStubMock): sys.argv = [SCRIPT_NAME, RQD_HOSTNAME, '--nimbyon'] - nimbyOnRequest = opencue.compiled_proto.rqd_pb2.RqdStaticNimbyOnRequest() + nimbyOnRequest = rqd.compiled_proto.rqd_pb2.RqdStaticNimbyOnRequest() rqd.cuerqd.main() @@ -93,7 +92,7 @@ def test_nimbyOn(self, stubMock, frameStubMock): def test_lockAll(self, stubMock, frameStubMock): sys.argv = [SCRIPT_NAME, RQD_HOSTNAME, '--lh'] - lockAllRequest = opencue.compiled_proto.rqd_pb2.RqdStaticLockAllRequest() + lockAllRequest = rqd.compiled_proto.rqd_pb2.RqdStaticLockAllRequest() rqd.cuerqd.main() @@ -101,7 +100,7 @@ def test_lockAll(self, stubMock, frameStubMock): def test_unlockAll(self, stubMock, frameStubMock): sys.argv = [SCRIPT_NAME, RQD_HOSTNAME, '--ulh'] - unlockAllRequest = opencue.compiled_proto.rqd_pb2.RqdStaticUnlockAllRequest() + unlockAllRequest = rqd.compiled_proto.rqd_pb2.RqdStaticUnlockAllRequest() rqd.cuerqd.main() @@ -110,7 +109,7 @@ def test_unlockAll(self, stubMock, frameStubMock): def test_lock(self, stubMock, frameStubMock): numCoresToLock = 85 sys.argv = [SCRIPT_NAME, RQD_HOSTNAME, '--lp', str(numCoresToLock)] - lockRequest = opencue.compiled_proto.rqd_pb2.RqdStaticLockRequest(cores=numCoresToLock) + lockRequest = rqd.compiled_proto.rqd_pb2.RqdStaticLockRequest(cores=numCoresToLock) rqd.cuerqd.main() @@ -119,8 +118,7 @@ def test_lock(self, stubMock, frameStubMock): def test_unlock(self, stubMock, frameStubMock): numCoresToUnlock = 52 sys.argv = [SCRIPT_NAME, RQD_HOSTNAME, '--ulp', str(numCoresToUnlock)] - unlockRequest = opencue.compiled_proto.rqd_pb2.RqdStaticUnlockRequest( - cores=numCoresToUnlock) + unlockRequest = rqd.compiled_proto.rqd_pb2.RqdStaticUnlockRequest(cores=numCoresToUnlock) rqd.cuerqd.main() @@ -128,7 +126,7 @@ def test_unlock(self, stubMock, frameStubMock): def test_shutdownRqdIdle(self, stubMock, frameStubMock): sys.argv = [SCRIPT_NAME, RQD_HOSTNAME, '--exit'] - shutdownIdleRequest = opencue.compiled_proto.rqd_pb2.RqdStaticShutdownIdleRequest() + shutdownIdleRequest = rqd.compiled_proto.rqd_pb2.RqdStaticShutdownIdleRequest() rqd.cuerqd.main() @@ -136,7 +134,7 @@ def test_shutdownRqdIdle(self, stubMock, frameStubMock): def test_shutdownRqdNow(self, stubMock, frameStubMock): sys.argv = [SCRIPT_NAME, RQD_HOSTNAME, '--exit_now'] - shutdownNowRequest = opencue.compiled_proto.rqd_pb2.RqdStaticShutdownNowRequest() + shutdownNowRequest = rqd.compiled_proto.rqd_pb2.RqdStaticShutdownNowRequest() rqd.cuerqd.main() @@ -144,7 +142,7 @@ def test_shutdownRqdNow(self, stubMock, frameStubMock): def test_rebootIdle(self, stubMock, frameStubMock): sys.argv = [SCRIPT_NAME, RQD_HOSTNAME, '--reboot'] - rebootIdleRequest = opencue.compiled_proto.rqd_pb2.RqdStaticRebootIdleRequest() + rebootIdleRequest = rqd.compiled_proto.rqd_pb2.RqdStaticRebootIdleRequest() rqd.cuerqd.main() @@ -152,19 +150,18 @@ def test_rebootIdle(self, stubMock, frameStubMock): def test_rebootNow(self, stubMock, frameStubMock): sys.argv = [SCRIPT_NAME, RQD_HOSTNAME, '--reboot_now'] - rebootNowRequest = opencue.compiled_proto.rqd_pb2.RqdStaticRebootNowRequest() + rebootNowRequest = rqd.compiled_proto.rqd_pb2.RqdStaticRebootNowRequest() rqd.cuerqd.main() stubMock.return_value.RebootNow.assert_called_with(rebootNowRequest) def test_launchFrame(self, stubMock, frameStubMock): - runFrame = opencue.compiled_proto.rqd_pb2.RunFrame() + runFrame = rqd.compiled_proto.rqd_pb2.RunFrame() runFrame.job_id = "SD6F3S72DJ26236KFS" runFrame.job_name = "edu-trn_job-name" runFrame.frame_id = "FD1S3I154O646UGSNN" - runFrameRequest = opencue.compiled_proto.rqd_pb2.RqdStaticLaunchFrameRequest( - run_frame=runFrame) + runFrameRequest = rqd.compiled_proto.rqd_pb2.RqdStaticLaunchFrameRequest(run_frame=runFrame) rqdHost = rqd.cuerqd.RqdHost(RQD_HOSTNAME) rqdHost.active = False @@ -174,7 +171,7 @@ def test_launchFrame(self, stubMock, frameStubMock): def test_killFrame(self, stubMock, frameStubMock): frameId = 'arbitrary-frame-id' - runFrame = opencue.compiled_proto.rqd_pb2.RunFrame(frame_id=frameId) + runFrame = rqd.compiled_proto.rqd_pb2.RunFrame(frame_id=frameId) stubMock.return_value.GetRunFrame.return_value = runFrame sys.argv = [SCRIPT_NAME, RQD_HOSTNAME, '--kill', frameId] diff --git a/rqd/tests/rqconstants_tests.py b/rqd/tests/rqconstants_tests.py index 761d9be14..45e52c0b1 100644 --- a/rqd/tests/rqconstants_tests.py +++ b/rqd/tests/rqconstants_tests.py @@ -32,12 +32,12 @@ import six -import opencue.compiled_proto.report_pb2 import rqd.rqconstants import rqd.rqcore import rqd.rqmachine import rqd.rqnimby import rqd.rqutil +import rqd.compiled_proto.report_pb2 from .rqmachine_tests import ( CPUINFO, @@ -109,7 +109,7 @@ def makeRqMachine(self): rqCore.nimby = nimby nimby.active = False nimby.locked = False - coreDetail = opencue.compiled_proto.report_pb2.CoreDetail(total_cores=2) + coreDetail = rqd.compiled_proto.report_pb2.CoreDetail(total_cores=2) machine = rqd.rqmachine.Machine(rqCore, coreDetail) machine.renderHost = machine.__dict__["_Machine__renderHost"] diff --git a/rqd/tests/rqcore_tests.py b/rqd/tests/rqcore_tests.py index f7a3ec891..09f06d23f 100644 --- a/rqd/tests/rqcore_tests.py +++ b/rqd/tests/rqcore_tests.py @@ -28,9 +28,9 @@ import mock import pyfakefs.fake_filesystem_unittest -import opencue.compiled_proto.host_pb2 -import opencue.compiled_proto.report_pb2 -import opencue.compiled_proto.rqd_pb2 +import rqd.compiled_proto.host_pb2 +import rqd.compiled_proto.report_pb2 +import rqd.compiled_proto.rqd_pb2 import rqd.rqconstants import rqd.rqcore import rqd.rqexceptions @@ -211,13 +211,13 @@ def test_killAllFrame(self): frame2Id = 'frame2' frame3Id = 'frame3' frame1 = rqd.rqnetwork.RunningFrame( - self.rqcore, opencue.compiled_proto.rqd_pb2.RunFrame(frame_id=frame1Id)) + self.rqcore, rqd.compiled_proto.rqd_pb2.RunFrame(frame_id=frame1Id)) frame1.frameAttendantThread = frameAttendantThread frame2 = rqd.rqnetwork.RunningFrame( - self.rqcore, opencue.compiled_proto.rqd_pb2.RunFrame(frame_id=frame2Id)) + self.rqcore, rqd.compiled_proto.rqd_pb2.RunFrame(frame_id=frame2Id)) frame2.frameAttendantThread = frameAttendantThread frame3 = rqd.rqnetwork.RunningFrame( - self.rqcore, opencue.compiled_proto.rqd_pb2.RunFrame(frame_id=frame3Id)) + self.rqcore, rqd.compiled_proto.rqd_pb2.RunFrame(frame_id=frame3Id)) frame3.frameAttendantThread = frameAttendantThread self.rqcore.storeFrame(frame1Id, frame1) self.rqcore.storeFrame(frame2Id, frame2) @@ -234,11 +234,10 @@ def test_killAllFrameIgnoreNimby(self): frame1Id = 'frame1' frame2Id = 'frame2' frame1 = rqd.rqnetwork.RunningFrame( - self.rqcore, opencue.compiled_proto.rqd_pb2.RunFrame(frame_id=frame1Id)) + self.rqcore, rqd.compiled_proto.rqd_pb2.RunFrame(frame_id=frame1Id)) frame1.frameAttendantThread = frameAttendantThread frame2 = rqd.rqnetwork.RunningFrame( - self.rqcore, opencue.compiled_proto.rqd_pb2.RunFrame(frame_id=frame2Id, - ignore_nimby=True)) + self.rqcore, rqd.compiled_proto.rqd_pb2.RunFrame(frame_id=frame2Id, ignore_nimby=True)) frame2.frameAttendantThread = frameAttendantThread self.rqcore.storeFrame(frame1Id, frame1) self.rqcore.storeFrame(frame2Id, frame2) @@ -251,7 +250,7 @@ def test_releaseCores(self): num_idle_cores = 10 num_booked_cores = 7 num_cores_to_release = 5 - self.rqcore.cores = opencue.compiled_proto.report_pb2.CoreDetail( + self.rqcore.cores = rqd.compiled_proto.report_pb2.CoreDetail( total_cores=50, idle_cores=num_idle_cores, locked_cores=2, booked_cores=num_booked_cores) @@ -283,11 +282,10 @@ def test_handleExit(self, networkMock, exitMock): @mock.patch('rqd.rqcore.FrameAttendantThread') def test_launchFrame(self, frameThreadMock): - self.rqcore.cores = opencue.compiled_proto.report_pb2.CoreDetail(total_cores=100, - idle_cores=20) - self.machineMock.return_value.state = opencue.compiled_proto.host_pb2.UP + self.rqcore.cores = rqd.compiled_proto.report_pb2.CoreDetail(total_cores=100, idle_cores=20) + self.machineMock.return_value.state = rqd.compiled_proto.host_pb2.UP self.nimbyMock.return_value.locked = False - frame = opencue.compiled_proto.rqd_pb2.RunFrame(uid=22, num_cores=10) + frame = rqd.compiled_proto.rqd_pb2.RunFrame(uid=22, num_cores=10) rqd.rqconstants.OVERRIDE_NIMBY = None self.rqcore.launchFrame(frame) @@ -295,16 +293,16 @@ def test_launchFrame(self, frameThreadMock): frameThreadMock.return_value.start.assert_called() def test_launchFrameOnDownHost(self): - self.machineMock.return_value.state = opencue.compiled_proto.host_pb2.DOWN - frame = opencue.compiled_proto.rqd_pb2.RunFrame() + self.machineMock.return_value.state = rqd.compiled_proto.host_pb2.DOWN + frame = rqd.compiled_proto.rqd_pb2.RunFrame() with self.assertRaises(rqd.rqexceptions.CoreReservationFailureException): self.rqcore.launchFrame(frame) def test_launchFrameOnHostWaitingForShutdown(self): - self.machineMock.return_value.state = opencue.compiled_proto.host_pb2.UP + self.machineMock.return_value.state = rqd.compiled_proto.host_pb2.UP self.nimbyMock.return_value.active = False - frame = opencue.compiled_proto.rqd_pb2.RunFrame() + frame = rqd.compiled_proto.rqd_pb2.RunFrame() self.rqcore.shutdownRqdIdle() with self.assertRaises(rqd.rqexceptions.CoreReservationFailureException): @@ -312,11 +310,10 @@ def test_launchFrameOnHostWaitingForShutdown(self): @mock.patch('rqd.rqcore.FrameAttendantThread') def test_launchFrameOnNimbyHost(self, frameThreadMock): - self.rqcore.cores = opencue.compiled_proto.report_pb2.CoreDetail(total_cores=100, - idle_cores=20) - self.machineMock.return_value.state = opencue.compiled_proto.host_pb2.UP - frame = opencue.compiled_proto.rqd_pb2.RunFrame(uid=22, num_cores=10) - frameIgnoreNimby = opencue.compiled_proto.rqd_pb2.RunFrame( + self.rqcore.cores = rqd.compiled_proto.report_pb2.CoreDetail(total_cores=100, idle_cores=20) + self.machineMock.return_value.state = rqd.compiled_proto.host_pb2.UP + frame = rqd.compiled_proto.rqd_pb2.RunFrame(uid=22, num_cores=10) + frameIgnoreNimby = rqd.compiled_proto.rqd_pb2.RunFrame( uid=22, num_cores=10, ignore_nimby=True) self.rqcore.nimby = mock.create_autospec(rqd.rqnimby.NimbySelect) self.rqcore.nimby.locked = True @@ -329,47 +326,45 @@ def test_launchFrameOnNimbyHost(self, frameThreadMock): frameThreadMock.return_value.start.assert_called() def test_launchDuplicateFrame(self): - self.rqcore.cores = opencue.compiled_proto.report_pb2.CoreDetail(total_cores=100, - idle_cores=20) - self.machineMock.return_value.state = opencue.compiled_proto.host_pb2.UP + self.rqcore.cores = rqd.compiled_proto.report_pb2.CoreDetail(total_cores=100, idle_cores=20) + self.machineMock.return_value.state = rqd.compiled_proto.host_pb2.UP self.nimbyMock.return_value.locked = False frameId = 'arbitrary-frame-id' - self.rqcore.storeFrame(frameId, opencue.compiled_proto.rqd_pb2.RunFrame(frame_id=frameId)) - frameToLaunch = opencue.compiled_proto.rqd_pb2.RunFrame(frame_id=frameId) + self.rqcore.storeFrame(frameId, rqd.compiled_proto.rqd_pb2.RunFrame(frame_id=frameId)) + frameToLaunch = rqd.compiled_proto.rqd_pb2.RunFrame(frame_id=frameId) rqd.rqconstants.OVERRIDE_NIMBY = None with self.assertRaises(rqd.rqexceptions.DuplicateFrameViolationException): self.rqcore.launchFrame(frameToLaunch) def test_launchFrameWithInvalidUid(self): - self.machineMock.return_value.state = opencue.compiled_proto.host_pb2.UP + self.machineMock.return_value.state = rqd.compiled_proto.host_pb2.UP self.nimbyMock.return_value.locked = False - frame = opencue.compiled_proto.rqd_pb2.RunFrame(uid=0) + frame = rqd.compiled_proto.rqd_pb2.RunFrame(uid=0) with self.assertRaises(rqd.rqexceptions.InvalidUserException): self.rqcore.launchFrame(frame) def test_launchFrameWithInvalidCoreCount(self): - self.machineMock.return_value.state = opencue.compiled_proto.host_pb2.UP + self.machineMock.return_value.state = rqd.compiled_proto.host_pb2.UP self.nimbyMock.return_value.locked = False - frame = opencue.compiled_proto.rqd_pb2.RunFrame(uid=22, num_cores=0) + frame = rqd.compiled_proto.rqd_pb2.RunFrame(uid=22, num_cores=0) with self.assertRaises(rqd.rqexceptions.CoreReservationFailureException): self.rqcore.launchFrame(frame) def test_launchFrameWithInsufficientCores(self): - self.rqcore.cores = opencue.compiled_proto.report_pb2.CoreDetail(total_cores=100, - idle_cores=5) - self.machineMock.return_value.state = opencue.compiled_proto.host_pb2.UP + self.rqcore.cores = rqd.compiled_proto.report_pb2.CoreDetail(total_cores=100, idle_cores=5) + self.machineMock.return_value.state = rqd.compiled_proto.host_pb2.UP self.nimbyMock.return_value.locked = False - frame = opencue.compiled_proto.rqd_pb2.RunFrame(uid=22, num_cores=10) + frame = rqd.compiled_proto.rqd_pb2.RunFrame(uid=22, num_cores=10) with self.assertRaises(rqd.rqexceptions.CoreReservationFailureException): self.rqcore.launchFrame(frame) def test_getRunningFrame(self): frameId = 'arbitrary-frame-id' - frame = opencue.compiled_proto.rqd_pb2.RunFrame(frame_id=frameId) + frame = rqd.compiled_proto.rqd_pb2.RunFrame(frame_id=frameId) self.rqcore.storeFrame(frameId, frame) self.assertEqual(frame, self.rqcore.getRunningFrame(frameId)) @@ -400,7 +395,7 @@ def test_rebootIdleNoFrames(self): def test_rebootIdleWithFrames(self): frame1Id = 'frame1' frame1 = rqd.rqnetwork.RunningFrame( - self.rqcore, opencue.compiled_proto.rqd_pb2.RunFrame(frame_id=frame1Id)) + self.rqcore, rqd.compiled_proto.rqd_pb2.RunFrame(frame_id=frame1Id)) self.rqcore.storeFrame(frame1Id, frame1) self.rqcore.rebootIdle() @@ -470,7 +465,7 @@ def test_lockAll(self): self.assertEqual(50, self.rqcore.cores.locked_cores) def test_unlock(self): - self.machineMock.return_value.state = opencue.compiled_proto.host_pb2.UP + self.machineMock.return_value.state = rqd.compiled_proto.host_pb2.UP self.rqcore.cores.total_cores = 50 self.rqcore.cores.idle_cores = 10 self.rqcore.cores.locked_cores = 40 @@ -482,7 +477,7 @@ def test_unlock(self): self.assertEqual(20, self.rqcore.cores.locked_cores) def test_unlockMoreCoresThanThereAre(self): - self.machineMock.return_value.state = opencue.compiled_proto.host_pb2.UP + self.machineMock.return_value.state = rqd.compiled_proto.host_pb2.UP self.rqcore.cores.total_cores = 50 self.rqcore.cores.idle_cores = 40 self.rqcore.cores.locked_cores = 10 @@ -494,7 +489,7 @@ def test_unlockMoreCoresThanThereAre(self): self.assertEqual(0, self.rqcore.cores.locked_cores) def test_unlockAll(self): - self.machineMock.return_value.state = opencue.compiled_proto.host_pb2.UP + self.machineMock.return_value.state = rqd.compiled_proto.host_pb2.UP self.nimbyMock.return_value.locked = False self.rqcore.cores.total_cores = 50 self.rqcore.cores.idle_cores = 40 @@ -507,7 +502,7 @@ def test_unlockAll(self): self.assertEqual(0, self.rqcore.cores.locked_cores) def test_unlockAllWhenNimbyLocked(self): - self.machineMock.return_value.state = opencue.compiled_proto.host_pb2.UP + self.machineMock.return_value.state = rqd.compiled_proto.host_pb2.UP self.nimbyMock.return_value.locked = True self.rqcore.cores.total_cores = 50 self.rqcore.cores.idle_cores = 40 @@ -546,7 +541,7 @@ def test_runLinux(self, getTempDirMock, permsUser, timeMock, popenMock): # mkdir frameUid = 928 frameUsername = 'my-random-user' returnCode = 0 - renderHost = opencue.compiled_proto.report_pb2.RenderHost(name='arbitrary-host-name') + renderHost = rqd.compiled_proto.report_pb2.RenderHost(name='arbitrary-host-name') logFile = os.path.join(logDir, '%s.%s.rqlog' % (jobName, frameName)) self.fs.create_dir(tempDir) @@ -562,9 +557,9 @@ def test_runLinux(self, getTempDirMock, permsUser, timeMock, popenMock): # mkdir rqCore.machine.isDesktop.return_value = True rqCore.machine.getHostInfo.return_value = renderHost rqCore.nimby.locked = False - children = opencue.compiled_proto.report_pb2.ChildrenProcStats() + children = rqd.compiled_proto.report_pb2.ChildrenProcStats() - runFrame = opencue.compiled_proto.rqd_pb2.RunFrame( + runFrame = rqd.compiled_proto.rqd_pb2.RunFrame( frame_id=frameId, job_name=jobName, frame_name=frameName, @@ -601,9 +596,9 @@ def test_runLinux(self, getTempDirMock, permsUser, timeMock, popenMock): # mkdir _, kwargs = popenMock.call_args rqCore.network.reportRunningFrameCompletion.assert_called_with( - opencue.compiled_proto.report_pb2.FrameCompleteReport( + rqd.compiled_proto.report_pb2.FrameCompleteReport( host=renderHost, - frame=opencue.compiled_proto.report_pb2.RunningFrameInfo( + frame=rqd.compiled_proto.report_pb2.RunningFrameInfo( job_name=jobName, frame_id=frameId, frame_name=frameName, children=children), exit_status=returnCode)) @@ -622,7 +617,7 @@ def disabled__test_runWindows(self, permsUser, timeMock, popenMock): frameUid = 928 frameUsername = 'my-random-user' returnCode = 0 - renderHost = opencue.compiled_proto.report_pb2.RenderHost(name='arbitrary-host-name') + renderHost = rqd.compiled_proto.report_pb2.RenderHost(name='arbitrary-host-name') timeMock.return_value = currentTime popenMock.return_value.returncode = returnCode @@ -634,9 +629,9 @@ def disabled__test_runWindows(self, permsUser, timeMock, popenMock): rqCore.machine.isDesktop.return_value = True rqCore.machine.getHostInfo.return_value = renderHost rqCore.nimby.locked = False - children = opencue.compiled_proto.report_pb2.ChildrenProcStats() + children = rqd.compiled_proto.report_pb2.ChildrenProcStats() - runFrame = opencue.compiled_proto.rqd_pb2.RunFrame( + runFrame = rqd.compiled_proto.rqd_pb2.RunFrame( frame_id=frameId, job_id=jobId, job_name=jobName, @@ -661,9 +656,9 @@ def disabled__test_runWindows(self, permsUser, timeMock, popenMock): # TODO(bcipriano) Verify the log directory was created and used for stdout/stderr. rqCore.network.reportRunningFrameCompletion.assert_called_with( - opencue.compiled_proto.report_pb2.FrameCompleteReport( + rqd.compiled_proto.report_pb2.FrameCompleteReport( host=renderHost, - frame=opencue.compiled_proto.report_pb2.RunningFrameInfo( + frame=rqd.compiled_proto.report_pb2.RunningFrameInfo( job_name=jobName, frame_id=frameId, frame_name=frameName, children=children), exit_status=returnCode)) @@ -681,7 +676,7 @@ def test_runDarwin(self, getTempDirMock, permsUser, timeMock, popenMock): frameUid = 928 frameUsername = 'my-random-user' returnCode = 0 - renderHost = opencue.compiled_proto.report_pb2.RenderHost(name='arbitrary-host-name') + renderHost = rqd.compiled_proto.report_pb2.RenderHost(name='arbitrary-host-name') logFile = os.path.join(logDir, '%s.%s.rqlog' % (jobName, frameName)) self.fs.create_dir(tempDir) @@ -697,9 +692,9 @@ def test_runDarwin(self, getTempDirMock, permsUser, timeMock, popenMock): rqCore.machine.isDesktop.return_value = True rqCore.machine.getHostInfo.return_value = renderHost rqCore.nimby.locked = False - children = opencue.compiled_proto.report_pb2.ChildrenProcStats() + children = rqd.compiled_proto.report_pb2.ChildrenProcStats() - runFrame = opencue.compiled_proto.rqd_pb2.RunFrame( + runFrame = rqd.compiled_proto.rqd_pb2.RunFrame( frame_id=frameId, job_name=jobName, frame_name=frameName, @@ -735,9 +730,9 @@ def test_runDarwin(self, getTempDirMock, permsUser, timeMock, popenMock): self.assertEqual(logFile, kwargs['stderr'].name) rqCore.network.reportRunningFrameCompletion.assert_called_with( - opencue.compiled_proto.report_pb2.FrameCompleteReport( + rqd.compiled_proto.report_pb2.FrameCompleteReport( host=renderHost, - frame=opencue.compiled_proto.report_pb2.RunningFrameInfo( + frame=rqd.compiled_proto.report_pb2.RunningFrameInfo( job_name=jobName, frame_id=frameId, frame_name=frameName, children=children), exit_status=returnCode)) diff --git a/rqd/tests/rqmachine_tests.py b/rqd/tests/rqmachine_tests.py index 566668fea..1b1bdaf4a 100644 --- a/rqd/tests/rqmachine_tests.py +++ b/rqd/tests/rqmachine_tests.py @@ -27,15 +27,15 @@ import mock import pyfakefs.fake_filesystem_unittest -import opencue.compiled_proto.host_pb2 -import opencue.compiled_proto.report_pb2 -import opencue.compiled_proto.rqd_pb2 import rqd.rqconstants import rqd.rqcore import rqd.rqmachine import rqd.rqnetwork import rqd.rqnimby import rqd.rqutil +import rqd.compiled_proto.host_pb2 +import rqd.compiled_proto.report_pb2 +import rqd.compiled_proto.rqd_pb2 CPUINFO = """processor : 0 @@ -180,7 +180,7 @@ def setUp(self): self.rqCore.nimby = self.nimby self.nimby.active = False self.nimby.locked = False - self.coreDetail = opencue.compiled_proto.report_pb2.CoreDetail(total_cores=2) + self.coreDetail = rqd.compiled_proto.report_pb2.CoreDetail(total_cores=2) self.machine = rqd.rqmachine.Machine(self.rqCore, self.coreDetail) @@ -288,7 +288,7 @@ def _test_rssUpdate(self, proc_stat): self.fs.create_file('/proc/%s/cmdline' % pid, contents=PROC_PID_CMDLINE) self.fs.create_file('/proc/%s/statm' % pid, contents=PROC_PID_STATM) runningFrame = rqd.rqnetwork.RunningFrame(self.rqCore, - opencue.compiled_proto.rqd_pb2.RunFrame()) + rqd.compiled_proto.rqd_pb2.RunFrame()) runningFrame.pid = pid frameCache = {frameId: runningFrame} @@ -403,16 +403,14 @@ def test_getHostInfo(self): self.assertEqual(25, hostInfo.load) self.assertEqual(False, hostInfo.nimby_enabled) self.assertEqual(False, hostInfo.nimby_locked) - self.assertEqual(opencue.compiled_proto.host_pb2.UP, hostInfo.state) + self.assertEqual(rqd.compiled_proto.host_pb2.UP, hostInfo.state) def test_getHostReport(self): frame1 = mock.MagicMock(spec=rqd.rqnetwork.RunningFrame) - frame1Info = opencue.compiled_proto.report_pb2.RunningFrameInfo( - resource_id='arbitrary-id-1') + frame1Info = rqd.compiled_proto.report_pb2.RunningFrameInfo(resource_id='arbitrary-id-1') frame1.runningFrameInfo.return_value = frame1Info frame2 = mock.MagicMock(spec=rqd.rqnetwork.RunningFrame) - frame2Info = opencue.compiled_proto.report_pb2.RunningFrameInfo( - resource_id='arbitrary-id-2') + frame2Info = rqd.compiled_proto.report_pb2.RunningFrameInfo(resource_id='arbitrary-id-2') frame2.runningFrameInfo.return_value = frame2Info frameIds = ['frame1', 'frame2'] frames = { @@ -421,7 +419,7 @@ def test_getHostReport(self): } self.rqCore.getFrameKeys.return_value = frameIds self.rqCore.getFrame.side_effect = lambda frameId: frames[frameId] - coreDetail = opencue.compiled_proto.report_pb2.CoreDetail( + coreDetail = rqd.compiled_proto.report_pb2.CoreDetail( total_cores=152, idle_cores=57, locked_cores=30, booked_cores=65) self.rqCore.getCoreInfo.return_value = coreDetail diff --git a/rqd/tests/test_cuebot_listener.py b/rqd/tests/test_cuebot_listener.py index 90e7dcc5a..160951f05 100644 --- a/rqd/tests/test_cuebot_listener.py +++ b/rqd/tests/test_cuebot_listener.py @@ -32,7 +32,7 @@ import grpc -import opencue.compiled_proto.report_pb2_grpc +import rqd.compiled_proto.report_pb2_grpc import rqd.rqconstants @@ -46,7 +46,7 @@ def __init__(self): def addServicers(self): addFunc = getattr( - opencue.compiled_proto.report_pb2_grpc, 'add_{0}_to_server'.format(self.servicerName)) + rqd.compiled_proto.report_pb2_grpc, 'add_{0}_to_server'.format(self.servicerName)) servicerClass = globals()[self.servicerName] self.servicer = servicerClass() addFunc(self.servicer, self.server) @@ -70,7 +70,7 @@ def stayAlive(self): self.server.stop(0) -class RqdReportInterfaceServicer(opencue.compiled_proto.report_pb2_grpc.RqdReportInterfaceServicer): +class RqdReportInterfaceServicer(rqd.compiled_proto.report_pb2_grpc.RqdReportInterfaceServicer): """Test class implements RqdReportStatic interface. Received reports are stored in the variables listed below. Create as an object to connect. From 39c578205dd7bc9b4968d18ec4b49efab5a4692b Mon Sep 17 00:00:00 2001 From: Jimmy Christensen Date: Fri, 20 Sep 2024 00:02:27 +0200 Subject: [PATCH 27/32] Fix logger in CueLogWriter.py --- pycue/cuelogging/CueLogWriter.py | 3 +++ pycue/cuelogging/__init__.py | 3 --- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pycue/cuelogging/CueLogWriter.py b/pycue/cuelogging/CueLogWriter.py index 923849642..d58b2c8ed 100644 --- a/pycue/cuelogging/CueLogWriter.py +++ b/pycue/cuelogging/CueLogWriter.py @@ -14,11 +14,14 @@ """Module for writing files""" +import logging import os import platform import datetime import time +log = logging.getLogger(__name__) + class CueLogWriter(object): """Class to abstract file log writing, this class tries to act as a file object""" diff --git a/pycue/cuelogging/__init__.py b/pycue/cuelogging/__init__.py index caa9a9c6f..06b40afe2 100644 --- a/pycue/cuelogging/__init__.py +++ b/pycue/cuelogging/__init__.py @@ -15,8 +15,5 @@ """Package for reading and writing log files""" -import logging from .CueLogReader import CueLogReader from .CueLogWriter import CueLogWriter - -log = logging.getLogger(__name__) From c1d344dc099c4a437b9f1a29fc92af85b9f6b0c0 Mon Sep 17 00:00:00 2001 From: Jimmy Christensen Date: Wed, 9 Oct 2024 22:33:00 +0200 Subject: [PATCH 28/32] Add coverage report temporarily to check if it works --- .github/workflows/testing-pipeline.yml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/.github/workflows/testing-pipeline.yml b/.github/workflows/testing-pipeline.yml index 63f09afe2..2b9781760 100644 --- a/.github/workflows/testing-pipeline.yml +++ b/.github/workflows/testing-pipeline.yml @@ -124,3 +124,26 @@ jobs: uses: tj-actions/changed-files@v41 - name: Check for Version Change run: ci/check_version_bump.py ${{ steps.get_changed_files.outputs.all_changed_and_modified_files }} + + analyze_python: + runs-on: ubuntu-latest + container: aswf/ci-opencue:2024.1 + name: Analyze Python Components + env: + ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + # Fetch all Git history, otherwise the current version number will + # not be correctly calculated. + fetch-depth: 0 + + - name: Generate coverage report + run: ci/python_coverage_report.sh + + - name: Analyze and send to SonarCloud + uses: sonarsource/sonarcloud-github-action@master + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} From af5648f5dd64a9beadeebc7b12f8f34da9d31f70 Mon Sep 17 00:00:00 2001 From: Jimmy Christensen Date: Wed, 9 Oct 2024 22:38:37 +0200 Subject: [PATCH 29/32] Add pycue to pythonpath when testing rqd --- ci/python_coverage_report.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/python_coverage_report.sh b/ci/python_coverage_report.sh index e0c65328f..df750a4b8 100755 --- a/ci/python_coverage_report.sh +++ b/ci/python_coverage_report.sh @@ -19,7 +19,7 @@ PYTHONPATH=pycue python -m coverage run -a --source=cueadmin/cueadmin/ cueadmin/ # TODO: re-enable cuegui tests when xvfb-run gets configured to execute on the new vfx-platform # PYTHONPATH=pycue xvfb-run -d python -m coverage run -a --source=cuegui/cuegui/ cuegui/setup.py test PYTHONPATH=pycue:pyoutline python -m coverage run -a --source=cuesubmit/cuesubmit/ cuesubmit/setup.py test -python -m coverage run -a --source=rqd/rqd/ --omit=rqd/rqd/compiled_proto/* rqd/setup.py test +PYTHONPATH=pycue python -m coverage run -a --source=rqd/rqd/ --omit=rqd/rqd/compiled_proto/* rqd/setup.py test # SonarCloud needs the report in XML. python -m coverage xml From ed212de59ddec0c7c7d97d0a497fe3a1d9876bbc Mon Sep 17 00:00:00 2001 From: Jimmy Christensen Date: Wed, 9 Oct 2024 22:44:31 +0200 Subject: [PATCH 30/32] Dummy change to force CI to run again --- .github/workflows/testing-pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/testing-pipeline.yml b/.github/workflows/testing-pipeline.yml index 2b9781760..f9efed14e 100644 --- a/.github/workflows/testing-pipeline.yml +++ b/.github/workflows/testing-pipeline.yml @@ -124,7 +124,7 @@ jobs: uses: tj-actions/changed-files@v41 - name: Check for Version Change run: ci/check_version_bump.py ${{ steps.get_changed_files.outputs.all_changed_and_modified_files }} - + analyze_python: runs-on: ubuntu-latest container: aswf/ci-opencue:2024.1 From c2efce686b2b9a904e45e3b24cfb70668996d61d Mon Sep 17 00:00:00 2001 From: Jimmy Christensen Date: Wed, 9 Oct 2024 23:08:21 +0200 Subject: [PATCH 31/32] Remove temporary test --- .github/workflows/testing-pipeline.yml | 23 ----------------------- 1 file changed, 23 deletions(-) diff --git a/.github/workflows/testing-pipeline.yml b/.github/workflows/testing-pipeline.yml index f9efed14e..63f09afe2 100644 --- a/.github/workflows/testing-pipeline.yml +++ b/.github/workflows/testing-pipeline.yml @@ -124,26 +124,3 @@ jobs: uses: tj-actions/changed-files@v41 - name: Check for Version Change run: ci/check_version_bump.py ${{ steps.get_changed_files.outputs.all_changed_and_modified_files }} - - analyze_python: - runs-on: ubuntu-latest - container: aswf/ci-opencue:2024.1 - name: Analyze Python Components - env: - ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - # Fetch all Git history, otherwise the current version number will - # not be correctly calculated. - fetch-depth: 0 - - - name: Generate coverage report - run: ci/python_coverage_report.sh - - - name: Analyze and send to SonarCloud - uses: sonarsource/sonarcloud-github-action@master - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} From 6527d93a392934a8814b802e2b592e6361bb0467 Mon Sep 17 00:00:00 2001 From: Jimmy Christensen Date: Tue, 5 Nov 2024 20:57:41 +0100 Subject: [PATCH 32/32] Add pycue to rqd tests --- ci/run_python_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/run_python_tests.sh b/ci/run_python_tests.sh index 101d75288..b8452d02f 100755 --- a/ci/run_python_tests.sh +++ b/ci/run_python_tests.sh @@ -26,7 +26,7 @@ python -m unittest discover -s pycue/tests -t pycue -p "*.py" PYTHONPATH=pycue python -m unittest discover -s pyoutline/tests -t pyoutline -p "*.py" PYTHONPATH=pycue python -m unittest discover -s cueadmin/tests -t cueadmin -p "*.py" PYTHONPATH=pycue:pyoutline python -m unittest discover -s cuesubmit/tests -t cuesubmit -p "*.py" -python -m pytest rqd/tests +PYTHONPATH=pycue python -m pytest rqd/tests # Xvfb no longer supports Python 2.