#include "ESMFPIO.h"
#define __PIO_FILE__ "pionfread_mod.F90.in"
module pionfread_mod

  implicit none

  private
!> 
!! @private
!<
  public :: read_nf
  interface read_nf
     ! TYPE real,double,int
     module procedure read_nfdarray_{TYPE}
  end interface

  character(len=*), parameter :: modName='pionfread_mod'

contains

  ! TYPE real,double,int
!>
!! @private
!<
  integer function read_nfdarray_{TYPE} (File,IOBUF,varDesc,IODesc, start,count) result(ierr)
    use pio_types, only : file_desc_t, var_desc_t, io_desc_t, pio_real, pio_double, pio_int, &
	pio_noerr, pio_iotype_netcdf4p, pio_iotype_netcdf4c, pio_iotype_pnetcdf, pio_iotype_netcdf, &
	pio_max_var_dims
    use pio_kinds, only : pio_offset, i4, r4, r8
    use pio_utils, only : check_netcdf, bad_iotype 
    use pio_support, only : Debug, DebugIO, piodie, checkmpireturn
    use alloc_mod, only: alloc_check
#ifdef _NETCDF
    use netcdf, only : nf90_get_var  !_EXTERNAL
#endif
#ifdef TIMING
    use perf_mod, only : t_startf, t_stopf  !_EXTERNAL
#endif
    implicit none

    include 'mpif.h'   !_EXTERNAL
#ifdef _PNETCDF
#include <pnetcdf.inc>   /* _EXTERNAL */
#endif

    type (File_desc_t), intent(inout) :: File
    {VTYPE}, intent(inout)           :: IOBUF(:)
    type (Var_desc_t), intent(in) :: varDesc
    type (IO_desc_t), intent(in) :: IODesc
    integer(kind=pio_offset), intent(in) :: start(:), count(:)


    character(len=*), parameter :: subName=modName//'::read_nfdarray_{TYPE}'
    integer(kind=i4) :: iotype

    integer :: iobuf_size, max_iobuf_size
    integer :: status(MPI_STATUS_SIZE)
    integer, dimension(PIO_MAX_VAR_DIMS) :: temp_start, temp_count
    integer :: i, mpierr, ndims, sdims

#ifdef TIMING
    call t_startf("pio_read_nfdarray_{TYPE}")
#endif
    iotype = File%iotype
    ierr=PIO_noerr


    ndims = size(start)
    if (File%iosystem%IOproc) then
       select case (iotype) 
#ifdef _PNETCDF
       case(pio_iotype_pnetcdf)
          if(DebugIO) print *,__PIO_FILE__,__LINE__, &
               '  IAM: ',File%iosystem%io_rank

          if(Debug) print *,__PIO_FILE__,__LINE__, &
               '  IAM: ',File%iosystem%io_rank,' start: ', &
               start,' count: ', &
               count, ' iobuf size: ',size(iobuf)

          ierr=nfmpi_get_vara_all( File%fh,varDesc%varid, &
               start, &
               count, &
               IOBUF,int(iodesc%Read%n_ElemTYPE,kind=pio_offset), &
               iodesc%Read%ElemTYPE)
#endif

#ifdef _NETCDF
       case(pio_iotype_netcdf4p, pio_iotype_netcdf4c)	
! all reads can be parallel in netcdf4 format
          ierr= nf90_get_var(File%fh, vardesc%varid, iobuf, start=int(start),count=int(count))
       case(pio_iotype_netcdf)
          iobuf_size=size(IOBUF)
          call MPI_REDUCE( iobuf_size,max_iobuf_size, &
               1,MPI_INTEGER,MPI_MAX,0,File%iosystem%IO_comm,mpierr )
          call checkmpireturn(subName, mpierr)

          if (File%iosystem%io_rank==0) then
             if (max_iobuf_size > iobuf_size) then
                print *, 'IOBUF on root is not big enough', max_iobuf_size, iobuf_size
                call abort
             endif
          endif

          ! create temporaries of size int (netcdf limitation)
	 
	  temp_start=1
	  temp_count=1
          if (File%iosystem%io_rank>0) then
             temp_start(1:ndims)=start(1:ndims)
             temp_count(1:ndims)=count(1:ndims)

             if (Debug) print *, File%iosystem%comp_rank,': waiting to receive IOBUF', start, count

             call MPI_SEND( temp_start,ndims,MPI_INTEGER, &
                  0,File%iosystem%io_rank,File%iosystem%IO_comm,mpierr )
             call checkmpireturn(subName, mpierr)

             call MPI_SEND( temp_count,ndims,MPI_INTEGER, &
                  0,File%iosystem%io_rank,File%iosystem%IO_comm,mpierr )
             call checkmpireturn(subName, mpierr)

             call MPI_SEND( iobuf_size,1,MPI_INTEGER, &
                  0,File%iosystem%io_rank,File%iosystem%IO_comm,mpierr )
             call checkmpireturn(subName, mpierr)

             call MPI_RECV( IOBUF,size(IOBUF), &
                  {MPITYPE}, &
                  0,File%iosystem%io_rank,File%iosystem%IO_comm,status,mpierr )
             call checkmpireturn(subName, mpierr)

             if (Debug) print *, subName,':: comp_rank: ',File%iosystem%comp_rank, &
                  ': received IOBUF size=',size(IOBUF)
          endif

          ! Read rank>0 first then go back and read 0
          ! so that we can re-use the rank 0 IOBUF

          if (File%iosystem%io_rank==0) then
             do i=1,File%iosystem%num_iotasks-1
                if (Debug) print *, subName,': 0: reading netcdf for ',i

                call MPI_RECV( temp_start, ndims, MPI_INTEGER, &
                     i,i,File%iosystem%IO_comm,status,mpierr)
                call CheckMPIReturn('read_nfdarray_{TYPE}',mpierr)

                call MPI_RECV( temp_count, ndims, MPI_INTEGER, &
                     i,i,File%iosystem%IO_comm,status,mpierr)
                call CheckMPIReturn('read_nfdarray_{TYPE}',mpierr)

                call MPI_RECV( iobuf_size, 1, MPI_INTEGER,    &
                     i,i,File%iosystem%IO_comm,status,mpierr)
                call CheckMPIReturn('read_nfdarray_{TYPE}',mpierr)

                ierr=nf90_get_var( File%fh, varDesc%varid, &
                     IOBUF, temp_start(1:ndims), temp_count(1:ndims) )

                call MPI_SEND( IOBUF,iobuf_size, &
                     {MPITYPE}, &
                     i,i,File%iosystem%IO_comm,mpierr)
                call CheckMPIReturn('read_nfdarray_{TYPE}',mpierr)

                if (Debug) print *, subName,': 0: done reading netcdf for ',i
             end do ! i=1,File%iosystem%num_iotasks-1

             ! Read root data last

             if (Debug) print *, subName,': 0: reading netcdf for self', vardesc%varid, ndims, start, count

             temp_start(1:ndims)=start(1:ndims)
             temp_count(1:ndims)=count(1:ndims)

             ierr=nf90_get_var( File%fh, varDesc%varid, &
                  IOBUF, temp_start(1:ndims), temp_count(1:ndims) )

             if (Debug) print *, subName,': 0: done reading netcdf for self'

          endif ! File%iosystem%io_rank==0

#endif

       case default
          call bad_iotype(iotype,__PIO_FILE__,__LINE__)

       end select
    endif ! File%iosystem%IOproc
    call check_netcdf(File, ierr,__PIO_FILE__,__LINE__);
#ifdef TIMING
    call t_stopf("pio_read_nfdarray_{TYPE}")
#endif

  end function read_nfdarray_{TYPE}
end module pionfread_mod