#!/bin/ksh 
# $Id: mpirun.sge.batch,v 1.8 2010/09/30 19:43:45 svasquez Exp $
################################################################################
# This script abides to the ESMF script rules for executing the bundled test 
# and example applications.
#
# 1) <this_script> -np N prog
#    runs N copies of executable "prog" in parallel. The script must hide all
#    of the system specific details, such as going through a queueing system
#    and/or calling a system specific mpirun script with modified arguments.
#
# 2) The output of the script must be written to a file named "prog".stdout and
#    it must contain the combination of stdout and stderr output of the
#    execution.
#
# 3) The script must block, i.e. _not_ return, until the output described in #2
#    has become accessible.
#
# To access this script set environment variable ESMF_MPIRUN=<this_script> in
# your shell before executing any of the ESMF run targets or targets that 
# call run targets indirectly, such as the all_tests.
################################################################################

if [ "$1" != "-np" ]
then
        echo "Usage: mpirun -np # prog"
        exit 1
fi

export np=$2
export TNAME=$3
shift 2
prog=$*

export working_dir=`pwd`


# Extract batch run script
cat > $ESMF_DIR/scripts/esmf_script << THE_END_OF_BATCH_SCRIPT
#!/bin/csh -f
#
#  Set the name of the job.
#$ -N esmf_test
#
#  Make sure that the .e and .o file arrive in the working directory
#$ -cwd
#
#$ $ESMF_MPIBATCHOPTIONS
#
#  My code is re-runnable
#$ -r n
#
#  The max walltime for this job is 30 minutes
#$ -l h_rt=00:30:00
#
# Environment variables exported to job
#$ -V 



#source /usr/local/bin/setup-mpi.csh

cd $working_dir

mpirun -np $np  $prog


THE_END_OF_BATCH_SCRIPT

#chmod a+x $ESMF_DIR/scripts/esmf_script

echo " qsub  $ESMF_DIR/scripts/esmf_script"

job_num=`qsub  $ESMF_DIR/scripts/esmf_script `

# Since this is not an interactive shell, we must grep for the proc_id until 
# it is gone, indicating the job is done or the walltime has expired.

# Strip off name and keep the number for grepping.
echo $job_num > proc_id_file
read word1 word2 proc_id word4 < proc_id_file
echo $proc_id > proc_id

while qstat | grep $proc_id > tmp
do
	# if the test can't be run delete it.
	if grep Eqw tmp
	then
		qdel $proc_id
	else
        	sleep 30
	fi
done

# move job output file to stdout.
mv -f esmf_test.o$proc_id  $TNAME.stdout


# Clean up
rm -f esmf_script
rm -f proc_id_file
rm -f tmp