#!/bin/ksh
# $Id: mpirun.scali.batch,v 1.5 2009/01/08 22:48:17 svasquez Exp $
################################################################################
# This script abides to the ESMF script rules for executing the bundled test 
# and example applications.
#
# 1) <this_script> -np N prog
#    runs N copies of executable "prog" in parallel. The script must hide all
#    of the system specific details, such as going through a queueing system
#    and/or calling a system specific mpirun script with modified arguments.
#
# 2) The output of the application must arrive at the calling shell via 
#    stdout and stderr.
#
# To access this script set environment variable ESMF_MPIRUN=<this_script> in
# your shell before executing any of the ESMF run targets or targets that 
# call run targets indirectly, such as the all_tests.
################################################################################

if [ "$1" != "-np" ]
then
        echo "Usage: mpirun -np # prog"
        exit 1
fi

export num_procs=$2
shift 2
export prog=$*

export working_dir=`pwd`



# Extract batch run script
cat > $ESMF_DIR/scripts/esmf_script << THE_END_OF_BATCH_SCRIPT
#PBS $ESMF_MPIBATCHOPTIONS
#PBS -j oe
cd $working_dir
mpirun -np $num_procs $prog

THE_END_OF_BATCH_SCRIPT

chmod a+x $ESMF_DIR/scripts/esmf_script

echo " qsub -V $ESMF_DIR/scripts/esmf_script"

proc_id=`qsub -V $ESMF_DIR/scripts/esmf_script `

# Since this is not an interactive shell, we must grep for the proc_id until 
# it is gone, indicating the job is done or the walltime has expired.

# Strip off name and keep the number for grepping.
proc_id=`echo $proc_id | sed 's/\..*//'`

while qstat | grep $proc_id
do
        sleep 60
done

#wait for file to appear.
sleep 60
# move job output file to stdout.
mv -f *$proc_id*  $prog.stdout


# Clean up
#rm -f $ESMF_DIR/scripts/esmf_script