diff --git a/bin/fisbatch b/bin/fisbatch index 58de765..c042719 100755 --- a/bin/fisbatch +++ b/bin/fisbatch @@ -119,7 +119,8 @@ fi # Submit the job and get the job id MyExport=SLURM_CPUS_PER_TASK,SLURM_JOB_NAME,SLURM_NTASKS_PER_NODE,SLURM_PRIO_PROCESS,SLURM_SUBMIT_DIR,SLURM_SUBMIT_HOST #JOB=`sbatch --export=$MyExport --job-name=FISBATCH --output=/dev/null --error=/dev/null $@ $BS 2>&1 | egrep -o -e "\b[0-9]+$"` -JOB=`sbatch --export=$MyExport --job-name=FISBATCH --output=/dev/null --error=/dev/null $@ $BS 2>&1 | egrep -o -e "\b[0-9]+"` +#JOB=`sbatch --export=$MyExport --job-name=FISBATCH --output=/dev/null --error=/dev/null $@ $BS 2>&1 | egrep -o -e "\b[0-9]+"` +JOB=`sbatch --job-name=FISBATCH --output=/dev/null --error=/dev/null $@ $BS 2>&1 | egrep -o -e "\b[0-9]+"` JOB=`echo $JOB | awk '{ printf("%d", $1 + 0); }'` # Make sure the job is always canceled @@ -155,7 +156,8 @@ usr=`whoami` NODE=`squeue -h --clusters=${cluster} --partition=${partition} --jobs=${JOB} -o %N | grep -v "^CLUSTER"` NODE=`nodeset -e $NODE` for i in $NODE; do - if [ `ssh $i "ps -ef | grep $usr | grep [S]CREEN | wc -l"` != "0" ]; then + screenTest=`ssh $i "ps -ef | grep $usr | grep [S]CREEN | wc -l"` + if [ "$screenTest" != "0" ]; then HNODE=$i break fi diff --git a/bin/sjeff b/bin/sjeff index 98c07f3..cfeacd2 100755 --- a/bin/sjeff +++ b/bin/sjeff @@ -16,6 +16,10 @@ if [ "$1" == "" -o "$1" == "--help" ]; then echo " list of jobids or --all for all " echo " running jobs. " echo " " + echo " All other command line arguments are" + echo " passed along to SLURM. For example: " + echo " --partition and/or --clusters " + echo " " echo " Environment Variables: " echo " STUBL_SJEFF_PCPU=(ps|top) " echo " " @@ -32,16 +36,19 @@ if [ "$1" == "" -o "$1" == "--help" ]; then fi if [ "$1" == "--all" ]; then - jobList=`squeue -h --state='R' -o %A` + jobList=`squeue -h $@ --state='R' -o %A | grep -v CLUSTER` else jobList=`echo $1 | tr ',' ' '` fi +# pop first arg off list, all others passed on to SLURM commands +shift + #display header echo "Job_ID Username Efficiency Number_of_CPUs_In_Use" for job in $jobList; do - jobInfo=`squeue -h --job=$job --state='R' 2>/dev/null` + jobInfo=`squeue -h $@ --job=$job --state='R' 2>/dev/null | grep -v CLUSTER` if [ "$jobInfo" == "" -a "$1" != "--all" ]; then echo "Invalid job ID ($job)" @@ -51,7 +58,7 @@ for job in $jobList; do nodelist=`nodeset -e $nodelist | tr ' ' ','` # how many cpus were requested for the job - ncpus=`squeue -h --job=$job -o %C` + ncpus=`squeue -h $@ --job=$job -o %C | grep -v CLUSTER` # accumulate efficiency of each requested cpu bStackedJobs=no @@ -65,7 +72,7 @@ for job in $jobList; do if [ "$effsum" == "" ]; then effsum=0 fi - njobs=`squeue -h --nodes=$nodelist --user=$user -o %A | sort -u | wc -l` + njobs=`squeue -h $@ --nodes=$nodelist --user=$user -o %A | grep -v CLUSTER |sort -u | wc -l` if [ "$njobs" -gt "1" ]; then bStackedJobs=yes fi diff --git a/bin/slogs b/bin/slogs index e41f039..682b01b 100755 --- a/bin/slogs +++ b/bin/slogs @@ -38,13 +38,15 @@ if [ "$1" == "--help" ]; then echo " For each job run after the given start date, the " echo " following information is gathered from the SLURM " echo " accounting logs: " - echo " number of CPUS, start time, elapsed time, Amount " - echo " of RAM Requested, Avg RAM Used, and Max RAM Used " + echo " number of CPUS, wait time, start time, elapsed " + echo " time, Amount of RAM Requested, Avg RAM Used, and " + echo " Max RAM Used " echo "" - echo " Usage: slogs [start_date] [users]" + echo " Usage: slogs [start_date] [users] [sacct_args] " echo "" echo " Notes: start_date should be in MMDDYY format. " echo " Use a comma-separated list for multiple users." + echo " All sacct_args passed along to sacct command. " echo "" echo " Defaults: " echo " start_date : 010113 if no start date is given. " @@ -56,7 +58,7 @@ if [ "$1" == "--help" ]; then exit fi -# try to figure out is users are in first or second arg +# try to figure out if users are in first or second arg if [ "$1" != "" -a "`echo $1 | tr ',' '\n' | head -n1 | xargs id 2>/dev/null | wc -l`" != 0 ]; then dol1=$2 dol2=$1 @@ -65,35 +67,38 @@ else dol2=$2 fi -if [ "$dol1" == "" ]; then +dol1Test=`echo $dol1 | grep '\--' | wc -l` +if [ "$dol1" == "" -o "$dol1Test" == "1" ]; then StartDate=010113 else StartDate=$dol1 + shift fi if [ "$dol2" == "" ]; then TheGroup=`whoami` else TheGroup=$dol2 + shift fi TheList=`echo "$TheGroup" | tr ',' ' '` EndDate=`date +%m%d%y` -export SACCT_FORMAT="jobid,User,ncpus,start,elapsed,ReqMem,AveRSS,MaxRSS" +export SACCT_FORMAT="jobid%20,User,ncpus,start,elapsed,ReqMem,AveRSS,MaxRSS" TMPFILE=${TheGroup}_usage_logs_${StartDate}_to_${EndDate}.tmp OUTFILE=${TheGroup}_usage_logs_${StartDate}_to_${EndDate}.txt -echo "JobID User NCPUS Start Date Time Elapsed ReqMem AveRSS MaxRSS" > $TMPFILE +echo "JobID User NCPUS Start Date Time Elapsed ReqMem AveRSS MaxRSS" > $TMPFILE for i in $TheList; do echo "Retrieving accouning data for user $i ..." - sacct --start=${StartDate} --user=$i | \ - grep 'K' | \ - sed "s/\.bat+/ $i/g" | \ - sed "s/\.[0-9]*/& $i/g" | \ - sed "s/T/ /g" | \ - awk '{ printf("%-12s %-8s %5s %10s %8s %10s %8s %8s %8s\n", $1, $2, $3, $4, $5, $6, $7, $8, $9); }' \ + sacct --start=${StartDate} --user=$i $@ | \ + sed "s/$i//g" | + sed "s/T/ /g" | \ + awk -vuser=$i '{ + printf("%-20s %-8s %5s %10s %8s %10s %8s %9s %9s\n", $1, user, $2, $3, $4, $5, $6, $7, $8, $9); + }' \ >> $TMPFILE done diff --git a/bin/snodes b/bin/snodes index 6727b0a..f86f397 100755 --- a/bin/snodes +++ b/bin/snodes @@ -78,17 +78,17 @@ fi # so use grep filter instead.... if [ "$cluster" != "all" ]; then if [ "$3" == "mix" -o "$3" == "mixed" ]; then - sinfo $OPTIONS -o "%10n %8t %4c %8z %15C %8O %8m %8G %18P %f" | grep -v "^CLUSTER" | head -n1 - sinfo $OPTIONS -o "%10n %8t %4c %8z %15C %8O %8m %8G %18P %f" | grep -v "^CLUSTER" | grep 'mix' + sinfo $OPTIONS -o "%13n %8t %4c %8z %15C %8O %8m %8G %18P %f" | grep -v "^CLUSTER" | head -n1 + sinfo $OPTIONS -o "%13n %8t %4c %8z %15C %8O %8m %8G %18P %f" | grep -v "^CLUSTER" | grep 'mix' else - sinfo $OPTIONS -o "%10n %8t %4c %8z %15C %8O %8m %8G %18P %f" | grep -v "^CLUSTER" + sinfo $OPTIONS -o "%13n %8t %4c %8z %15C %8O %8m %8G %18P %f" | grep -v "^CLUSTER" fi else if [ "$3" == "mix" -o "$3" == "mixed" ]; then - sinfo $OPTIONS -o "%10n %8t %4c %8z %15C %8O %8m %8G %18P %f" | head -n1 - sinfo $OPTIONS -o "%10n %8t %4c %8z %15C %8O %8m %8G %18P %f" | grep 'mix' + sinfo $OPTIONS -o "%13n %8t %4c %8z %15C %8O %8m %8G %18P %f" | head -n1 + sinfo $OPTIONS -o "%13n %8t %4c %8z %15C %8O %8m %8G %18P %f" | grep 'mix' else - sinfo $OPTIONS -o "%10n %8t %4c %8z %15C %8O %8m %8G %18P %f" + sinfo $OPTIONS -o "%13n %8t %4c %8z %15C %8O %8m %8G %18P %f" fi fi diff --git a/bin/sueff b/bin/sueff index beb8d3d..aa7ed42 100755 --- a/bin/sueff +++ b/bin/sueff @@ -11,6 +11,10 @@ if [ "$1" == "" -o "$1" == "--help" ]; then echo " arg1 = userid (or comma-separated " echo " list of userids or --all for all " echo " users with currently active jobs. " + echo " " + echo " All other command line args will be " + echo " passed along to SLURM. " + echo " (e.g. --clusters, --partition, etc.)" echo "========================================" exit fi @@ -21,11 +25,13 @@ else userList=`echo $1 | tr ',' ' '` fi +shift + #display header echo "Username Efficiency Number_of_CPUs_In_Use" for user in $userList; do - nodelist=`squeue -h --state='R' --user=$user -o %N | sort -u | tr '\n' ' '` + nodelist=`squeue -h --state='R' --user=$user -o %N $@ | grep -v "CLUSTER" | sort -u | tr '\n' ' '` if [ "$nodelist" == "" -a "$1" != "--all" ]; then echo "$user has no running jobs!" @@ -33,7 +39,7 @@ for user in $userList; do nodelist=`nodeset -e $nodelist | tr ' ' ','` # how many cpus were requested for the users jobs - ncpus=`squeue -h --state='R' --user=$user -o %C | awk '{ SUM += $1;} END {print SUM }'` + ncpus=`squeue -h --state='R' --user=$user -o %C $@ | grep -v "CLUSTER" | awk '{ SUM += $1;} END {print SUM }'` # accumulate efficiency effsum=`clush -w $nodelist -N "ps -u $user -o pcpu= " 2>/dev/null | awk 'BEGIN {sum=0} {sum+=$1} END {print sum}'` diff --git a/docs/stubl_thumb.gif b/docs/stubl_thumb.gif new file mode 100644 index 0000000..f0caabd Binary files /dev/null and b/docs/stubl_thumb.gif differ