Cuesubmit jobs from config file

cuesubmit/cuesubmit/Config.py

@@ -35,19 +35,61 @@
 
 def getConfigValues():
     """Reads the config file from disk and returns the values it defines."""
-    configData = {}
-    configFile = os.environ.get(CONFIG_FILE_ENV_VAR)
-    if not configFile:
-        configFile = os.path.join(opencue.config.config_base_directory(), 'cuesubmit.yaml')
-    if os.path.exists(configFile):
-        with open(configFile, 'r', encoding='utf-8') as data:
-            try:
-                configData = yaml.load(data, Loader=yaml.SafeLoader)
-            except yaml.YAMLError:
-                raise CuesubmitConfigError("Could not load yaml file: {}. Please check its "
-                                           "formatting".format(configFile))
+    configFile = os.environ.get(CONFIG_FILE_ENV_VAR)\
+                 or os.path.join(opencue.config.config_base_directory(),
+                                 'cuesubmit.yaml')
+    if not os.path.exists(configFile):
+        return {}
+    configData = _loadYamlFile(yaml_file=configFile)
+    if 'RENDER_CMDS' in configData:
+        # look for any sub-config files and load them
+        configData['RENDER_CMDS'] = _expandRenderConfigValues(configData['RENDER_CMDS'])
     return configData
 
 
+def _loadYamlFile(yaml_file):
+    """ Load config yaml as dict
+    :param yaml_file: path to a config.yaml file (path can be an env var)
+    :type yaml_file: str
+    :returns: yaml content
+    :rtype: dict
+    """
+    _yaml_file = os.path.expandvars(yaml_file)
+    if not os.path.exists(_yaml_file):
+        raise FileNotFoundError(f"yaml_file:{_yaml_file} not found")
+    config_data = {}
+    with open(_yaml_file, 'r', encoding='utf-8') as data:
+        try:
+            config_data = yaml.load(data, Loader=yaml.SafeLoader)
+        except yaml.YAMLError:
+            raise CuesubmitConfigError("Could not load yaml file: {}. Please check its "
+                                       "formatting".format(_yaml_file))
+    return config_data
+
+
+def _expandRenderConfigValues(RENDER_CMDS):
+    """ Looks through each render command and loads their 'config_file' if any
+    If 'config_file' is set but does not exist, replace its content with error for proper feedback
+
+    :param RENDER_CMDS: all render commands from the cuesubmit_config.yaml file
+    :type RENDER_CMDS: dict
+    :returns: Updated RENDER_CMDS
+    :rtype: dict
+    """
+    for job_type, _options in RENDER_CMDS.items():
+        _sub_config_file = _options.get('config_file')
+        if not _sub_config_file:
+            continue
+        try:
+            RENDER_CMDS[job_type] = _loadYamlFile(yaml_file=_sub_config_file)
+        except FileNotFoundError as error:
+            RENDER_CMDS[job_type] = {
+                'command': 'error',
+                'options': {
+                    '{ERROR}': error}
+            }
+    return RENDER_CMDS
+
+
 class CuesubmitConfigError(Exception):
     """Thrown when an error occurs reading the config file."""

cuesubmit/cuesubmit/Constants.py

@@ -36,6 +36,10 @@
 MAYA_RENDER_CMD = config.get('MAYA_RENDER_CMD', 'Render')
 NUKE_RENDER_CMD = config.get('NUKE_RENDER_CMD', 'nuke')
 BLENDER_RENDER_CMD = config.get('BLENDER_RENDER_CMD', 'blender')
+RENDER_CMDS = config.get('RENDER_CMDS', {})
+
+DEFAULT_SHOW = config.get('DEFAULT_SHOW') or os.environ.get('PROJECT', 'default')
+
 FRAME_TOKEN = config.get('FRAME_TOKEN', '#IFRAME#')
 FRAME_START_TOKEN = config.get('FRAME_START', '#FRAME_START#')
 FRAME_END_TOKEN = config.get('FRAME_END', '#FRAME_END#')
@@ -68,6 +72,16 @@
 BLENDER_OUTPUT_OPTIONS_URL = \
     'https://docs.blender.org/manual/en/latest/advanced/command_line/arguments.html#render-options'
 
+REGEX_CUETOKEN = r'^#.*#$' #FRAME_START#
+REGEX_COMMAND_OPTIONS = (r'(?P<command_flag>-+\w*)?'   # -optionFlag
+                         r'(?P<hidden>\~)?'            # -hiddenFlag~
+                         r'\s?'
+                         r'({'
+                           r'(?P<mandatory>\!)?'      # {!Mandatory argument}
+                           r'(?P<label>[^{}\*\/\!]+)' # {Nice name}
+                           r'(?P<browsable>\*?\/?)'   # {browseFile*} or {browseFolder/}
+                         r'})?')
+
 DIR_PATH = os.path.dirname(__file__)
 
 # Dropdown label to specify the default Facility, i.e. let Cuebot decide.

cuesubmit/cuesubmit/JobTypes.py

@@ -24,7 +24,8 @@
 
 from builtins import object
 from cuesubmit.ui import SettingsWidgets
-
+from cuesubmit import Constants
+from cuesubmit import Util
 
 class JobTypes(object):
     """Base Job Types available in the UI.
@@ -36,23 +37,41 @@ class JobTypes(object):
     MAYA = 'Maya'
     NUKE = 'Nuke'
     BLENDER = 'Blender'
+    FROM_CONFIG_FILE = Constants.RENDER_CMDS.keys()
 
     SETTINGS_MAP = {
         SHELL: SettingsWidgets.ShellSettings,
         MAYA: SettingsWidgets.BaseMayaSettings,
         NUKE: SettingsWidgets.BaseNukeSettings,
         BLENDER: SettingsWidgets.BaseBlenderSettings,
     }
+    for jobType in FROM_CONFIG_FILE:
+        SETTINGS_MAP[jobType] = SettingsWidgets.DynamicSettingsWidget
 
     def __init__(self):
         pass
 
     @classmethod
     def build(cls, jobType, *args, **kwargs):
         """Factory method for creating a settings widget."""
+        if jobType in cls.FROM_CONFIG_FILE:
+            jobOptions = Constants.RENDER_CMDS[jobType].get('options')
+            parameters = Util.convertCommandOptions(options=jobOptions)
+            kwargs.update({'tool_name': jobType,
+                           'parameters': parameters})
         return cls.SETTINGS_MAP[jobType](*args, **kwargs)
 
     @classmethod
     def types(cls):
-        """return a list of types available."""
-        return [cls.SHELL, cls.MAYA, cls.NUKE, cls.BLENDER]
+        """return a list of available types."""
+        return list(cls.SETTINGS_MAP.keys())
+
+    @classmethod
+    def services(cls, jobType):
+        """return a list of services for a given jobType. (the "services" key in your yaml file)"""
+        return Constants.RENDER_CMDS[jobType].get('services', [])
+
+    @classmethod
+    def limits(cls, jobType):
+        """return a list of limits for a given jobType. (the "limits" key in your yaml file)"""
+        return Constants.RENDER_CMDS[jobType].get('limits', [])

cuesubmit/cuesubmit/Layer.py

@@ -41,6 +41,7 @@ def __init__(self):
         self.cmd = {}
         self.layerRange = ''
         self.chunk = '1'
+        self.overrideCores = False
         self.cores = '1'
         self.env = {}
         self.services = []
@@ -59,6 +60,7 @@ def toDict(self):
             'cmd': self.cmd,
             'layerRange': self.layerRange,
             'chunk': self.chunk,
+            'overrideCores': self.overrideCores,
             'cores': self.cores,
             'env': self.env,
             'services': self.services,
@@ -68,15 +70,19 @@ def toDict(self):
         }
 
     @staticmethod
-    def buildFactory(name=None, layerType=None, cmd=None, layerRange=None, chunk=None, cores=None,
-                     env=None, services=None, limits=None, dependType=None, dependsOn=None):
+    def buildFactory(name=None, layerType=None, cmd=None, layerRange=None, chunk=None,
+                     overrideCores=None, cores=None, env=None, services=None, limits=None,
+                     dependType=None, dependsOn=None):
         """Build a new LayerData object with the given settings."""
         layerData = LayerData()
-        layerData.update(name, layerType, cmd, layerRange, chunk, cores, env, services, limits,
-                         dependType, dependsOn)
+        layerData.update(name=name, layerType=layerType, cmd=cmd, layerRange=layerRange,
+                         chunk=chunk, overrideCores=overrideCores, cores=cores, env=env,
+                         services=services, limits=limits,
+                         dependType=dependType, dependsOn=dependsOn)
         return layerData
 
-    def update(self, name=None, layerType=None, cmd=None, layerRange=None, chunk=None, cores=None,
+    def update(self, name=None, layerType=None, cmd=None, layerRange=None, chunk=None,
+               overrideCores=None, cores=None,
                env=None, services=None, limits=None, dependType=None, dependsOn=None):
         """Update this Layer with the provided settings."""
         if name is not None:
@@ -91,6 +97,8 @@ def update(self, name=None, layerType=None, cmd=None, layerRange=None, chunk=Non
             self.chunk = chunk
         if cores is not None:
             self.cores = cores
+        if overrideCores is not None:
+            self.overrideCores = overrideCores
         if env is not None:
             self.env = env
         if services is not None:

cuesubmit/cuesubmit/Submission.py

@@ -29,6 +29,49 @@
 
 from cuesubmit import Constants
 from cuesubmit import JobTypes
+from cuesubmit import Util
+
+
+def isSoloFlag(flag):
+    """ Check if the flag is solo, meaning it has no associated value
+     solo flags are marked with a ~ (ex: --background~)
+     """
+    return re.match(r"^-+\w+~$", flag)
+
+
+def isFlag(flag):
+    """ Check if the provided string is a flag (starts with a -)"""
+    return re.match(r"^-+\w+$", flag)
+
+
+def formatValue(flag, value, isPath, isMandatory):
+    """ Adds quotes around file/folder path variables
+     and provide an error value to display for missing mandatory values.
+    """
+    if isPath and value:
+        value = f'"{value}"'
+    if isMandatory and value in ('', None):
+        value = f'!!missing value for {flag}!!'
+    return value
+
+
+def buildDynamicCmd(layerData):
+    """From a layer, builds a customized render command."""
+    renderCommand = Constants.RENDER_CMDS[layerData.layerType].get('command')
+    for (flag, isPath, isMandatory), value in layerData.cmd.items():
+        if isSoloFlag(flag):
+            renderCommand += f' {flag[:-1]}'
+            continue
+        value = formatValue(flag, value, isPath, isMandatory)
+        if isFlag(flag) and value not in ('', None):
+            # flag and value
+            renderCommand += f' {flag} {value}'
+            continue
+        # solo argument without flag
+        if value not in ('', None):
+            renderCommand += f' {value}'
+
+    return renderCommand
 
 
 def buildMayaCmd(layerData, silent=False):
@@ -97,10 +140,17 @@ def buildLayer(layerData, command, lastLayer=None):
     @type lastLayer: outline.layer.Layer
     @param lastLayer: layer that this new layer should be dependent on if dependType is set.
     """
-    threadable = float(layerData.cores) >= 2 or float(layerData.cores) <= 0
+    threadable = False
+    if layerData.overrideCores:
+        threadable = float(layerData.cores) >= 2 or float(layerData.cores) <= 0
+    elif layerData.services and layerData.services[0] in Util.getServices():
+        threadable = Util.getServiceOption(layerData.services[0], 'threadable')
+
+    cores = layerData.cores if layerData.overrideCores else None
     layer = outline.modules.shell.Shell(
         layerData.name, command=command.split(), chunk=layerData.chunk,
-        threads=float(layerData.cores), range=str(layerData.layerRange), threadable=threadable)
+        cores=cores,
+        range=str(layerData.layerRange), threadable=threadable)
     if layerData.services:
         layer.set_service(layerData.services[0])
     if layerData.limits:
@@ -112,9 +162,12 @@ def buildLayer(layerData, command, lastLayer=None):
             layer.depend_on(lastLayer)
     return layer
 
+
 def buildLayerCommand(layerData, silent=False):
     """Builds the command to be sent per jobType"""
-    if layerData.layerType == JobTypes.JobTypes.MAYA:
+    if layerData.layerType in JobTypes.JobTypes.FROM_CONFIG_FILE:
+        command = buildDynamicCmd(layerData)
+    elif layerData.layerType == JobTypes.JobTypes.MAYA:
         command = buildMayaCmd(layerData, silent)
     elif layerData.layerType == JobTypes.JobTypes.SHELL:
         command = layerData.cmd.get('commandTextBox') if silent else layerData.cmd['commandTextBox']
@@ -129,6 +182,7 @@ def buildLayerCommand(layerData, silent=False):
             raise ValueError('unrecognized layer type {}'.format(layerData.layerType))
     return command
 
+
 def submitJob(jobData):
     """Submits the job using the PyOutline API."""
     ol = outline.Outline(

cuesubmit/cuesubmit/Util.py

@@ -21,6 +21,7 @@
 from __future__ import absolute_import
 
 import os
+import re
 
 import opencue
 from cuesubmit import Constants
@@ -33,14 +34,37 @@ def getLimits():
 
 def getServices():
     """Returns a list of service names from cuebot."""
-    return [service.name() for service in opencue.api.getDefaultServices()]
+    try:
+        services = opencue.api.getDefaultServices()
+    except opencue.exception.ConnectionException:
+        return []
+    else:
+        return [service.name() for service in services]
+
+
+def getServiceOption(serviceName, option):
+    """Returns the value of a service property."""
+    service = next(iter(service for service in opencue.api.getDefaultServices()
+                        if service.name() == serviceName))
+    if service and hasattr(service, option):
+        return getattr(service, option)()
+    print(f'{service.name()} service has no {option} option.')
+    return None
 
 
 def getShows():
     """Returns a list of show names from cuebot."""
     return [show.name() for show in opencue.api.getShows()]
 
 
+def getDefaultShow():
+    """Returns the default show defined via environment variable or config file, if set."""
+    default_show = next(iter([show for show in getShows()
+                              if re.match(show, Constants.DEFAULT_SHOW, re.IGNORECASE)]),
+                        'no default show')
+    return default_show
+
+
 def getAllocations():
     """Returns a list of Allocations from cuebot."""
     return opencue.api.getAllocations()
@@ -60,3 +84,40 @@ def getFacilities(allocations):
     default_facilities = [Constants.DEFAULT_FACILITY_TEXT]
     facilities = set(alloc.data.facility for alloc in allocations)
     return default_facilities + list(facilities)
+
+
+def convertCommandOptions(options):
+    """ Parse command options from the config file
+    and return parameters to feed the UI (name, type, value)
+
+    :param options: All options for a given command (ex:{"-flag {Nice Name}": "default_value"})
+    :type options: dict
+    :return: list of dict of parameters
+    """
+    parameters = []
+    for option_line, value in options.items():
+        parse_option = re.search(Constants.REGEX_COMMAND_OPTIONS,
+                                 option_line)
+        options = {
+            'option_line': option_line,
+            'label': parse_option.group('label'),
+            'command_flag': parse_option.group('command_flag'),
+            'value': value,
+            'type': type(value),
+            'hidden': bool(parse_option.group('hidden'))
+                      or re.match(Constants.REGEX_CUETOKEN, str(value)),
+            'mandatory': bool(parse_option.group('mandatory')),
+            'browsable': parse_option.group('browsable'),
+            }
+        if isinstance(value, (tuple, list))\
+                and len(value) in (3, 4)\
+                and isinstance(value[0], (int, float)):
+            options.update({
+                'type': range,
+                'min': value[0],
+                'max': value[1],
+                'value': value[2],
+                'float_precision': value[3] if len(value)==4 else None
+                })
+        parameters.append(options)
+    return parameters