#!/bin/bash
#SBATCH -e err
#SBATCH -o out
#SBATCH --account=fv3-cpu
#SBATCH --qos=batch
#SBATCH --ntasks=600
### #SBATCH --nodes=15
### #SBATCH --ntasks-per-node=40
#SBATCH --time=30
#SBATCH --job-name="run_control_fv3wamphys_ugwpv1_p8_intel"
### #SBATCH --exclusive

set -eux
date_s_start=$(date +%s)
echo -n "${date_s_start}," > job_timestamp.txt

set +x
export MACHINE_ID=hera
source ./module-setup.sh
module use "${PWD}/modulefiles"
module load modules.fv3
module list
set -x

date_start=$(date)
echo "Model started: ${date_start}"

export MPI_TYPE_DEPTH=20
export OMP_STACKSIZE=512M
# shellcheck disable=SC2125
export OMP_NUM_THREADS=1
export ESMF_RUNTIME_COMPLIANCECHECK=OFF:depth=4
export ESMF_RUNTIME_PROFILE=ON
export ESMF_RUNTIME_PROFILE_OUTPUT="SUMMARY"
export PSM_RANKS_PER_CONTEXT=4
export PSM_SHAREDCONTEXTS=1

# Avoid job errors because of filesystem synchronization delays
sync && sleep 1

# This "if" block is part of the rt.sh self-tests in error-test.conf. It emulates the model failing to run.
if [ "${JOB_SHOULD_FAIL:-NO}" = WHEN_RUNNING ] ; then
    echo "The job should abort now, with exit status 1." 1>&2
    echo "If error checking is working, the metascheduler should mark the job as failed." 1>&2
    false
fi

# shellcheck disable=SC2102
srun --label -n 600 ./fv3.exe

date_end=$(date)
echo "Model ended: ${date_end}"
date_s_end=$(date +%s)
echo -n "${date_s_end}," >> job_timestamp.txt