#!/bin/ksh
# $Id: mpirun.rs6000_sp,v 1.9 2008/07/23 04:51:56 theurich Exp $
################################################################################
# This script abides to the ESMF script rules for executing the bundled test 
# and example applications.
#
# 1) <this_script> -np N prog
#    runs N copies of executable "prog" in parallel. The script must hide all
#    of the system specific details, such as going through a queueing system
#    and/or calling a system specific mpirun script with modified arguments.
#
# 2) The output of the application must arrive at the calling shell via 
#    stdout and stderr.
#
# To access this script set environment variable ESMF_MPIRUN=<this_script> in
# your shell before executing any of the ESMF run targets or targets that 
# call run targets indirectly, such as the all_tests.
################################################################################

if [ "$1" != "-np" ] 
then
	echo "Usage: mpirun -np # prog"
	exit 1
fi

if [ "$MP_RESD" != "no" ]; then
if [ x$MP_RMPOOL == x ]; then
if [ x$MP_HOSTFILE == x ]; then
  echo "***ERROR***"
  echo "It appears that ./scripts/mpirun.rs6000_sp has been called without"
  echo "resources. You must either execute from within a batch script or"
  echo "set environment variable MP_RMPOOL to a system-dependent shared pool,"
  echo "if available. Please see your system documentation for details on how"
  echo "to execute parallel applications."
  exit 1
fi
fi
fi

num_procs=$2
shift 2
prog=$*

# this may be different on different machines.
# TODO: make it an optional arg to mpirun?
MAX_TASKS_PER_NODE=4      # true on blackforest
#MAX_TASKS_PER_NODE=32    # true on bluesky

if [ $num_procs -le $MAX_TASKS_PER_NODE ]; then
  num_nodes=1
  num_tasks=$num_procs
else
  let num_nodes="($num_procs + $MAX_TASKS_PER_NODE - 1)/$MAX_TASKS_PER_NODE"
  let num_tasks="$num_procs / $num_nodes"
  let odd_procs="$num_procs - ($num_tasks * $num_nodes)"
  if [ $odd_procs -ne 0 ] ; then
     echo "!!WARNING, number of processes does not divide evenly. odd_procs =" $odd_procs
     echo "num_procs =" $num_procs " num_nodes =" $num_nodes " num_tasks=" $num_tasks
  fi
fi

MP_PROCS=$num_procs
MP_NODES=$num_nodes
MP_TASKS_PER_NODE=$num_tasks

# TODO: when we install the real error/logger handler, we may want to
#  unset this.  but for now, we can ask the default stdout be sorted by
#  processor.  it means that if the processes do not finish correctly you
#  may not see messages - so for debugging you generally do not want this
#  set.   nsc 4/2/2003

MP_STDOUTMODE=ordered
export MP_STDOUTMODE
echo "setting MP_STDOUTMODE=ordered, stdout to be sorted by proc"

export MP_NODES MP_TASKS_PER_NODE MP_PROCS

echo "env: MP_PROCS =" $MP_PROCS " MP_NODES =" $MP_NODES " MP_TASKS_PER_NODE =" $MP_TASKS_PER_NODE
echo "about to <exec poe $prog>"

exec poe $prog