Skip to content

Commit

Permalink
Merge pull request #77 from LSSTDESC/rail-dev
Browse files Browse the repository at this point in the history 
Rail dev
  • Loading branch information
@joezuntz
joezuntz authored Aug 12, 2022
2 parents b32abad + 6303f56 commit 1f0198f
Show file tree
Hide file tree
Showing 10 changed files with 370 additions and 65 deletions.
54 changes: 1 addition & 53 deletions .pylintrc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,14 +30,6 @@ unsafe-load-any-extension=no
# run arbitrary code
extension-pkg-whitelist=

# Allow optimization of some AST trees. This will activate a peephole AST
# optimizer, which will apply various small optimizations. For instance, it can
# be used to obtain the result of joining multiple strings with the addition
# operator. Joining a lot of strings can lead to a maximum recursion error in
# Pylint and this flag can prevent that. It has one side effect, the resulting
# AST will be different than the one from reality.
optimize-ast=no


[MESSAGES CONTROL]

Expand All @@ -60,7 +52,7 @@ confidence=
# --enable=similarities". If you want to run only the classes checker, but have
# no Warning level messages displayed, use"--disable=all --enable=classes
# --disable=W"
disable=import-star-module-level,old-octal-literal,oct-method,print-statement,unpacking-in-except,parameter-unpacking,backtick,old-raise-syntax,old-ne-operator,long-suffix,dict-view-method,dict-iter-method,metaclass-assignment,next-method-called,raising-string,indexing-exception,raw_input-builtin,long-builtin,file-builtin,execfile-builtin,coerce-builtin,cmp-builtin,buffer-builtin,basestring-builtin,apply-builtin,filter-builtin-not-iterating,using-cmp-argument,useless-suppression,range-builtin-not-iterating,suppressed-message,no-absolute-import,old-division,cmp-method,reload-builtin,zip-builtin-not-iterating,intern-builtin,unichr-builtin,reduce-builtin,standarderror-builtin,unicode-builtin,xrange-builtin,coerce-method,delslice-method,getslice-method,setslice-method,input-builtin,round-builtin,hex-method,nonzero-method,map-builtin-not-iterating,abstract-method,broad-except,invalid-name,line-too-long,wrong-import-order,wrong-import-position,too-many-statements,super-with-arguments,import-outside-toplevel,too-many-arguments,too-many-instance-attributes,unspecified-encoding,no-else-return
disable=suppressed-message,abstract-method,broad-except,invalid-name,line-too-long,wrong-import-order,wrong-import-position,too-many-statements,super-with-arguments,import-outside-toplevel,too-many-arguments,too-many-instance-attributes,unspecified-encoding,no-else-return


[REPORTS]
Expand All @@ -70,11 +62,6 @@ disable=import-star-module-level,old-octal-literal,oct-method,print-statement,un
# mypackage.mymodule.MyReporterClass.
output-format=text

# Put messages in a separate file for each module / package specified on the
# command line instead of printing them on stdout. Reports (if any) will be
# written in a file name "pylint_global.[txt|html]".
files-output=no

# Tells whether to display a full report or only the messages
reports=yes

Expand All @@ -92,9 +79,6 @@ evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / stateme

[BASIC]

# List of builtins function names that should not be used, separated by a comma
bad-functions=map,filter,input

# Good variable names which should always be accepted, separated by a comma
good-names=i,j,k,ex,Run,_

Expand All @@ -111,63 +95,33 @@ include-naming-hint=no
# Regular expression matching correct function names
function-rgx=[a-z_][a-z0-9_]{2,30}$

# Naming hint for function names
function-name-hint=[a-z_][a-z0-9_]{2,30}$

# Regular expression matching correct variable names
variable-rgx=[a-z_][a-z0-9_]{2,30}$

# Naming hint for variable names
variable-name-hint=[a-z_][a-z0-9_]{2,30}$

# Regular expression matching correct constant names
const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$

# Naming hint for constant names
const-name-hint=(([A-Z_][A-Z0-9_]*)|(__.*__))$

# Regular expression matching correct attribute names
attr-rgx=[a-z_][a-z0-9_]{2,30}$

# Naming hint for attribute names
attr-name-hint=[a-z_][a-z0-9_]{2,30}$

# Regular expression matching correct argument names
argument-rgx=[a-z_][a-z0-9_]{2,30}$

# Naming hint for argument names
argument-name-hint=[a-z_][a-z0-9_]{2,30}$

# Regular expression matching correct class attribute names
class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$

# Naming hint for class attribute names
class-attribute-name-hint=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$

# Regular expression matching correct inline iteration names
inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$

# Naming hint for inline iteration names
inlinevar-name-hint=[A-Za-z_][A-Za-z0-9_]*$

# Regular expression matching correct class names
class-rgx=[A-Z_][a-zA-Z0-9]+$

# Naming hint for class names
class-name-hint=[A-Z_][a-zA-Z0-9]+$

# Regular expression matching correct module names
module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$

# Naming hint for module names
module-name-hint=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$

# Regular expression matching correct method names
method-rgx=[a-z_][a-z0-9_]{2,30}$

# Naming hint for method names
method-name-hint=[a-z_][a-z0-9_]{2,30}$

# Regular expression which should only match function or class names that do
# not require a docstring.
no-docstring-rgx=^_
Expand Down Expand Up @@ -282,12 +236,6 @@ ignore-long-lines=^\s*(# )?<?https?://\S+>?$
# else.
single-line-if-stmt=no

# List of optional constructs for which whitespace checking is disabled. `dict-
# separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}.
# `trailing-comma` allows a space between comma and closing bracket: (a, ).
# `empty-line` allows space-only lines.
no-space-check=trailing-comma,dict-separator

# Maximum number of lines in a module
max-module-lines=2000

Expand Down
5 changes: 5 additions & 0 deletions ceci/__main__.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
# This file must exist with these contents
from .stage import PipelineStage

if __name__ == "__main__":
PipelineStage.main()
12 changes: 8 additions & 4 deletions ceci/pipeline.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,8 @@ class StageExecutionConfig:
The name of the stage
class_name: str
The name of the class of the stage
module_name: str
The name of the module for the stage
site: Site object
(default the global default) The site this stage is run on
nprocess: int
Expand Down Expand Up @@ -88,6 +90,7 @@ def __init__(self, info):
# Core attributes - mandatory
self.name = info["name"]
self.class_name = info.get("classname", self.name)
self.module_name = info.get("module_name")
self.site = info.get("site", get_default_site())

# Parallelism attributes - optional
Expand Down Expand Up @@ -129,6 +132,7 @@ def create(cls, stage, **kwargs):
info = kwargs.copy()
info["name"] = stage.instance_name
info["classname"] = stage.name
info["module_name"] = stage.get_module()
sec = cls(info)
sec.set_stage_obj(stage)
return sec
Expand All @@ -146,7 +150,7 @@ def set_stage_obj(self, stage_obj):
TypeError : if stage_obj is not and instance of self.stage_class as
determined by the self.class_name attribute
"""
self.stage_class = PipelineStage.get_stage(self.class_name)
self.stage_class = PipelineStage.get_stage(self.class_name, self.module_name)
if not isinstance(stage_obj, self.stage_class): # pragma: no cover
raise TypeError(f"{str(stage_obj)} is not a {str(self.stage_class)}")
self.stage_obj = stage_obj
Expand All @@ -155,7 +159,7 @@ def build_stage_class(self):
"""Set the stage_class attribute by finding
self.class_name in the dictionary of classes from `Pipeline_stage`
"""
self.stage_class = PipelineStage.get_stage(self.class_name)
self.stage_class = PipelineStage.get_stage(self.class_name, self.module_name)
return self.stage_class

def build_stage_object(self, args):
Expand All @@ -173,7 +177,7 @@ def build_stage_object(self, args):
The newly constructed object
"""
if self.stage_class is None: # pragma: no cover
self.stage_class = PipelineStage.get_stage(self.class_name)
self.stage_class = PipelineStage.get_stage(self.class_name, self.module_name)
self.stage_obj = self.stage_class(args)
return self.stage_obj

Expand Down Expand Up @@ -881,7 +885,7 @@ def save(self, pipefile, stagefile=None, reduce_config=False):
if site is None:
site = val.site.config
pipe_stage_info = dict(
name=val.name, classname=val.class_name, nprocess=val.nprocess
name=val.name, classname=val.class_name, nprocess=val.nprocess, module_name=val.module_name,
)
if val.threads_per_process != 1:
pipe_stage_info["threads_per_process"] = val.threads_per_process
Expand Down
28 changes: 25 additions & 3 deletions ceci/stage.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -344,20 +344,27 @@ def __init_subclass__(cls, **kwargs):
#############################################

@classmethod
def get_stage(cls, name):
def get_stage(cls, name, module_name=None):
"""
Return the PipelineStage subclass with the given name.
This is used so that we do not need a new entry point __main__ function
for each new stage - instead we can just use a single one which can query
which class it should be using based on the name.
If module_name is provided, this will import that module
in order to load the required class.
Returns
-------
cls: class
The corresponding subclass
"""
stage = cls.pipeline_stages.get(name)
if stage is None:
if module_name:
__import__(module_name)
stage = cls.pipeline_stages.get(name)

# If not found, then check for incomplete stages
if stage is None:
Expand Down Expand Up @@ -450,7 +457,14 @@ def main(cls):
if stage_name in ["--help", "-h"] and len(sys.argv) == 2: # pragma: no cover
cls.usage()
return 1
stage = cls.get_stage(stage_name)
if stage_name.find('.') >= 0:
tokens = stage_name.split('.')
module_name = '.'.join(tokens[:-1])
stage_name = tokens[-1]
else:
module_name = None

stage = cls.get_stage(stage_name, module_name)
args = stage.parse_command_line()
stage.execute(args)
return 0
Expand Down Expand Up @@ -1284,7 +1298,15 @@ def generate_command(
module = cls.get_module()
module = module.split(".")[0]

flags = [cls.name]
if sys.modules[module].__file__:
# Regular module, stage will be imported with module
flags = [f"{cls.name}"]
else:
# Namescape module, use 'ceci' to the get main
# and specify the full path
flags = [f"{cls.get_module()}.{cls.name}"]
module = 'ceci'

aliases = aliases or {}

for tag, _ in cls.inputs_():
Expand Down
Loading

0 comments on commit 1f0198f

Please sign in to comment.