#!/bin/bash
#SBATCH --job-name=sette_test
#SBATCH --time=0:20:0
#SBATCH --nodes=NODES
#SBATCH --ntasks=TOTAL_NPROCS
#SBATCH --account=n01
#SBATCH --partition=standard
#SBATCH --qos=standard
##BATCH --reservation=shortqos
##BATCH --qos=short
module -s restore /work/n01/shared/acc/n01_modules/ucx_env
#
  export XIO_HOME=/work/n01/shared/acc/xios-2.5
#
  export OMP_NUM_THREADS=1
  export OCORES=NPROCS
  export XCORES=NXIOPROCS
  export SETTE_DIR=DEF_SETTE_DIR
#
# load sette functions (only post_test_tidyup needed)
#
  . ${SETTE_DIR}/all_functions.sh
###############################################################
#
# set up mpp computing environment
#
#
# Local settings. These settings are for a particular machine (the MOBILIS ClusterVision 
# system at NOCS) at a particular stage of that machine's evolution. This template file
# is provided for illustration purposes only and will not work on any other machine. There
# should, however, be sufficient similarity with other MPP platforms and batch systems
# for this example to provide a useful guide for experienced users
#
# Don't remove neither change the following line
# BODY
#
# Test specific settings. Do not hand edit these lines; the fcm_job.sh script will set these
# (via sed operating on this template job file). Note that the number of compute nodes required
# is also set by the fcm_job.sh on the PBS select header line above.
#
# These variables are needed by post_test_tidyup function in all_functions.sh
#
  export INPUT_DIR=DEF_INPUT_DIR
  export CONFIG_DIR=DEF_CONFIG_DIR
  export TOOLS_DIR=DEF_TOOLS_DIR
  export NEMO_VALIDATION_DIR=DEF_NEMO_VALIDATION
  export NEW_CONF=DEF_NEW_CONF
  export CMP_NAM=DEF_CMP_NAM
  export TEST_NAME=DEF_TEST_NAME
  export EXE_DIR=DEF_EXE_DIR
#
# end of set up
###############################################################
#
# change to the working directory 
#
  cd $EXE_DIR
  echo Directory is `pwd`
  if [ $XCORES -eq 0 ]; then
#
# Run SPMD case
#
       echo srun --cpu-bind=v,rank_ldom -n $OCORES ./nemo
       srun --cpu-bind=v,rank_ldom -n $OCORES ./nemo
  else
       if [ ! -f ./xios_server.exe ] && [ -f ${XIO_HOME}/bin/xios_server.exe ]; then
          cp ${XIO_HOME}/bin/xios_server.exe .
       fi
       if [ ! -f ./xios_server.exe ]; then
          echo "./xios_server.exe not found"
          echo "run aborted"
          exit
       fi
#
#  Run MPMD case
#
cat > myscript_wrapper2.sh << EOFB
#!/bin/ksh
#
set -A map ./xios_server.exe ./nemo
exec_map=( 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 )
#
exec \${map[\${exec_map[\$SLURM_PROCID]}]}
##
EOFB
chmod u+x ./myscript_wrapper2.sh
#
srun --mem-bind=local --cpu-bind=v,map_cpu:00,0x4,0x8,0xc,0x10,0x12,0x14,0x16,0x18,0x1a,0x1c,0x1e,0x20,0x22,0x24,0x26,0x28,0x2a,0x2c,0x2e,0x30,0x32,0x34,0x36,0x38,0x3a,0x3c,0x3e,0x40,0x42,0x44,0x46,0x48,0x4a,0x4c,0x4e,0x50,0x52,0x54,0x56,0x58,0x5a,0x5c,0x5e,0x60,0x62,0x64,0x66,0x68,0x6a,0x6c,0x6e,0x70,0x72,0x74,0x76,0x78,0x7a,0x7c,0x7e, ./myscript_wrapper2.sh
#
  fi
#
  post_test_tidyup
# END_BODY
# Don't remove neither change the previous line
  exit