#!/bin/ksh
# $Id: mpirun.unicos.batch,v 1.6 2008/07/23 04:51:56 theurich Exp $
################################################################################
# This script abides to the ESMF script rules for executing the bundled test 
# and example applications.
#
# 1) <this_script> -np N prog
#    runs N copies of executable "prog" in parallel. The script must hide all
#    of the system specific details, such as going through a queueing system
#    and/or calling a system specific mpirun script with modified arguments.
#
# 2) The output of the script must be written to a file named "prog".stdout and
#    it must contain the combination of stdout and stderr output of the
#    execution.
#
# 3) The script must block, i.e. _not_ return, until the output described in #2
#    has become accessible.
#
# To access this script set environment variable ESMF_MPIRUN=<this_script> in
# your shell before executing any of the ESMF run targets or targets that 
# call run targets indirectly, such as the all_tests.
################################################################################

if [ "$1" != "-np" ]
then
        echo "Usage: mpirun -np # prog"
        exit 1
fi

export num_procs=$2
shift 2
export prog=$*
# Make the mppe value a  multiple of 4.
remainder=`expr $num_procs % 4`
if [[ $remainder != 0 ]]
then
        export mppe=`expr $num_procs + 4 - $remainder`
else
        export mppe=$num_procs
fi

export working_dir=`pwd`

# Extract batch run script
cat > $ESMF_DIR/scripts/esmf_script << THE_END_OF_BATCH_SCRIPT
#PBS $ESMF_MPIBATCHOPTIONS
#PBS -j oe
#PBS -l mppe=$mppe
cd $working_dir
aprun -n $num_procs $prog

THE_END_OF_BATCH_SCRIPT

chmod a+x $ESMF_DIR/scripts/esmf_script

echo " qsub -V $ESMF_DIR/scripts/esmf_script"

proc_id=`qsub -V $ESMF_DIR/scripts/esmf_script `

# Since this is not an interactive shell, we must grep for the proc_id until 
# it is gone, indicating the job is done or the walltime has expired.

# Strip off name and keep the number for grepping.
proc_id=`echo $proc_id | sed 's/\..*//'`

while qstat | grep $proc_id
do
        sleep 30
done

# move job output file to stdout.
mv -f *$proc_id*  $prog.stdout


# Clean up
#rm -f $ESMF_DIR/scripts/esmf_script