Update Derecho Port (CICE-Consortium#501)

Update inteloneapi, cray, intel, nvhpc modules/compiler. Changes answers for inteloneapi and cray. Switch default Derecho queue from main to develop to support quicker turn around and lower costs. Develop is a shared queue. Error in the snow physics with the cray compiler needs further assessment, but suggest we merge this now as is. That error does not present itself in CICE with snow physics with the same cray compiler.
E3SM-Project · Sep 27, 2024 · ba0bc0b · ba0bc0b
1 parent 05ac0ec
commit ba0bc0b
Show file tree

Hide file tree

Showing 8 changed files with 26 additions and 25 deletions.
diff --git a/configuration/scripts/icepack.batch.csh b/configuration/scripts/icepack.batch.csh
@@ -55,7 +55,7 @@ cat >> ${jobfile} << EOFB
 #PBS -l job_priority=regular
 #PBS -N ${ICE_CASENAME}
 #PBS -A ${acct}
-#PBS -l select=${nnodes}:ncpus=${corespernode}:mpiprocs=${taskpernodelimit}:ompthreads=${nthrds}
+#PBS -l select=${nnodes}:ncpus=${corespernode}:mpiprocs=${taskpernodelimit}:ompthreads=${nthrds}:mem=5GB
 #PBS -l walltime=${ICE_RUNLENGTH}
 #PBS -j oe
 #PBS -W umask=022

diff --git a/configuration/scripts/machines/Macros.derecho_inteloneapi b/configuration/scripts/machines/Macros.derecho_inteloneapi
@@ -12,11 +12,12 @@ FFLAGS     := -fp-model precise -convert big_endian -assume byterecl -ftz -trace
 FFLAGS_NOOPT:= -O0
 
 ifeq ($(ICE_BLDDEBUG), true)
-  FFLAGS     += -O0 -g -check uninit -check bounds -check pointers -fpe0 -check noarg_temp_created -link_mpi=dbg
+# -check uninit is needed on the ld step but it still throws errors in 2023.* and 2024.0.*, likely compiler bug
+  FFLAGS     += -O0 -g  -check bounds -check pointers -fpe0 -check noarg_temp_created -link_mpi=dbg
 #  FFLAGS     += -O0 -g -check all -fpe0 -ftrapuv -fp-model except -check noarg_temp_created -link_mpi=dbg -stand f08
 #  FFLAGS     += -O0 -g -check all -fpe0 -ftrapuv -fp-model except -check noarg_temp_created -init=snan,arrays -link_mpi=dbg
 else
-  FFLAGS     += -O2
+  FFLAGS     += -O1
 endif
 
 SCC   := icx

diff --git a/configuration/scripts/machines/env.derecho_cray b/configuration/scripts/machines/env.derecho_cray
@@ -10,14 +10,14 @@ if ("$inp" != "-nomodules") then
 source ${MODULESHOME}/init/csh
 
 module --force purge
-module load ncarenv/23.06
+module load ncarenv/23.09
 module load craype
-module load cce/15.0.1
+module load cce/16.0.1
 module load ncarcompilers
 #module load cray-mpich/8.1.25
 #module load hdf5/1.12.2
 module load netcdf/4.9.2
-module load cray-libsci/23.02.1.1
+module load cray-libsci/23.09.1.1
 
 # For perftools with mpiexec
 # module load perftools-base
@@ -38,14 +38,14 @@ setenv OMP_STACKSIZE 64M
 setenv ICE_MACHINE_MACHNAME derecho
 setenv ICE_MACHINE_MACHINFO "HPE Cray EX Milan Slingshot 11"
 setenv ICE_MACHINE_ENVNAME cray
-setenv ICE_MACHINE_ENVINFO "cce 15.0.1, netcdf4.9.2"
+setenv ICE_MACHINE_ENVINFO "Cray clang/fortran cce 16.0.1, netcdf4.9.2"
 setenv ICE_MACHINE_MAKE gmake
 setenv ICE_MACHINE_WKDIR /glade/derecho/scratch/$user/ICEPACK_RUNS
 setenv ICE_MACHINE_INPUTDATA /glade/campaign/cesm/development/pcwg
 setenv ICE_MACHINE_BASELINE /glade/derecho/scratch/$user/ICEPACK_BASELINE
 setenv ICE_MACHINE_SUBMIT "qsub"
 setenv ICE_MACHINE_ACCT P00000000
-setenv ICE_MACHINE_QUEUE "main"
+setenv ICE_MACHINE_QUEUE "develop"
 setenv ICE_MACHINE_TPNODE 128
 setenv ICE_MACHINE_BLDTHRDS 1
 setenv ICE_MACHINE_QSTAT "qstat "
diff --git a/configuration/scripts/machines/env.derecho_gnu b/configuration/scripts/machines/env.derecho_gnu
@@ -45,7 +45,7 @@ setenv ICE_MACHINE_INPUTDATA /glade/campaign/cesm/development/pcwg
 setenv ICE_MACHINE_BASELINE /glade/derecho/scratch/$user/ICEPACK_BASELINE
 setenv ICE_MACHINE_SUBMIT "qsub"
 setenv ICE_MACHINE_ACCT P00000000
-setenv ICE_MACHINE_QUEUE "main"
+setenv ICE_MACHINE_QUEUE "develop"
 setenv ICE_MACHINE_TPNODE 128
 setenv ICE_MACHINE_BLDTHRDS 1
 setenv ICE_MACHINE_QSTAT "qstat "
diff --git a/configuration/scripts/machines/env.derecho_intel b/configuration/scripts/machines/env.derecho_intel
@@ -10,14 +10,14 @@ if ("$inp" != "-nomodules") then
 source ${MODULESHOME}/init/csh
 
 module --force purge
-module load ncarenv/23.06
+module load ncarenv/23.09
 module load craype
-module load intel/2023.0.0
+module load intel/2023.2.1
 module load ncarcompilers
 #module load cray-mpich/8.1.25
 #module load hdf5/1.12.2
 module load netcdf/4.9.2
-module load cray-libsci/23.02.1.1
+module load cray-libsci/23.09.1.1
 
 # For perftools with mpiexec
 # module load perftools-base
@@ -38,14 +38,14 @@ setenv OMP_STACKSIZE 64M
 setenv ICE_MACHINE_MACHNAME derecho
 setenv ICE_MACHINE_MACHINFO "HPE Cray EX Milan Slingshot 11"
 setenv ICE_MACHINE_ENVNAME intel
-setenv ICE_MACHINE_ENVINFO "ifort 2021.8.0 20221119, netcdf4.9.2"
+setenv ICE_MACHINE_ENVINFO "ifort 2021.10.0 20230609, netcdf4.9.2"
 setenv ICE_MACHINE_MAKE gmake
 setenv ICE_MACHINE_WKDIR /glade/derecho/scratch/$user/ICEPACK_RUNS
 setenv ICE_MACHINE_INPUTDATA /glade/campaign/cesm/development/pcwg
 setenv ICE_MACHINE_BASELINE /glade/derecho/scratch/$user/ICEPACK_BASELINE
 setenv ICE_MACHINE_SUBMIT "qsub"
 setenv ICE_MACHINE_ACCT P00000000
-setenv ICE_MACHINE_QUEUE "main"
+setenv ICE_MACHINE_QUEUE "develop"
 setenv ICE_MACHINE_TPNODE 128
 setenv ICE_MACHINE_BLDTHRDS 1
 setenv ICE_MACHINE_QSTAT "qstat "
diff --git a/configuration/scripts/machines/env.derecho_intelclassic b/configuration/scripts/machines/env.derecho_intelclassic
@@ -45,7 +45,7 @@ setenv ICE_MACHINE_INPUTDATA /glade/campaign/cesm/development/pcwg
 setenv ICE_MACHINE_BASELINE /glade/derecho/scratch/$user/ICEPACK_BASELINE
 setenv ICE_MACHINE_SUBMIT "qsub"
 setenv ICE_MACHINE_ACCT P00000000
-setenv ICE_MACHINE_QUEUE "main"
+setenv ICE_MACHINE_QUEUE "develop"
 setenv ICE_MACHINE_TPNODE 128
 setenv ICE_MACHINE_BLDTHRDS 1
 setenv ICE_MACHINE_QSTAT "qstat "
diff --git a/configuration/scripts/machines/env.derecho_inteloneapi b/configuration/scripts/machines/env.derecho_inteloneapi
@@ -10,14 +10,14 @@ if ("$inp" != "-nomodules") then
 source ${MODULESHOME}/init/csh
 
 module --force purge
-module load ncarenv/23.06
+module load ncarenv/23.09
 module load craype
-module load intel-oneapi/2023.0.0
+module load intel-oneapi/2023.2.1
 module load ncarcompilers
 #module load cray-mpich/8.1.25
 #module load hdf5/1.12.2
 module load netcdf/4.9.2
-module load cray-libsci/23.02.1.1
+module load cray-libsci/23.09.1.1
 
 # For perftools with mpiexec
 # module load perftools-base
@@ -38,14 +38,14 @@ setenv OMP_STACKSIZE 64M
 setenv ICE_MACHINE_MACHNAME derecho
 setenv ICE_MACHINE_MACHINFO "HPE Cray EX Milan Slingshot 11"
 setenv ICE_MACHINE_ENVNAME inteloneapi
-setenv ICE_MACHINE_ENVINFO "ifx 2023.0.0 20221201, netcdf4.9.2"
+setenv ICE_MACHINE_ENVINFO "ifx 2023.2.0 20230721, netcdf4.9.2"
 setenv ICE_MACHINE_MAKE gmake
 setenv ICE_MACHINE_WKDIR /glade/derecho/scratch/$user/ICEPACK_RUNS
 setenv ICE_MACHINE_INPUTDATA /glade/campaign/cesm/development/pcwg
 setenv ICE_MACHINE_BASELINE /glade/derecho/scratch/$user/ICEPACK_BASELINE
 setenv ICE_MACHINE_SUBMIT "qsub"
 setenv ICE_MACHINE_ACCT P00000000
-setenv ICE_MACHINE_QUEUE "main"
+setenv ICE_MACHINE_QUEUE "develop"
 setenv ICE_MACHINE_TPNODE 128
 setenv ICE_MACHINE_BLDTHRDS 1
 setenv ICE_MACHINE_QSTAT "qstat "
diff --git a/configuration/scripts/machines/env.derecho_nvhpc b/configuration/scripts/machines/env.derecho_nvhpc
@@ -10,14 +10,14 @@ if ("$inp" != "-nomodules") then
 source ${MODULESHOME}/init/csh
 
 module --force purge
-module load ncarenv/23.06
+module load ncarenv/23.09
 module load craype
-module load nvhpc/23.5
+module load nvhpc/23.7
 module load ncarcompilers
 #module load cray-mpich/8.1.25
 #module load hdf5/1.12.2
 module load netcdf/4.9.2
-module load cray-libsci/23.02.1.1
+module load cray-libsci/23.09.1.1
 
 # For perftools with mpiexec
 # module load perftools-base
@@ -38,14 +38,14 @@ setenv OMP_STACKSIZE 64M
 setenv ICE_MACHINE_MACHNAME derecho
 setenv ICE_MACHINE_MACHINFO "HPE Cray EX Milan Slingshot 11"
 setenv ICE_MACHINE_ENVNAME nvhpc
-setenv ICE_MACHINE_ENVINFO "nvc 23.5-0, netcdf4.9.2"
+setenv ICE_MACHINE_ENVINFO "nvfortran 23.7-0, netcdf4.9.2"
 setenv ICE_MACHINE_MAKE gmake
 setenv ICE_MACHINE_WKDIR /glade/derecho/scratch/$user/ICEPACK_RUNS
 setenv ICE_MACHINE_INPUTDATA /glade/campaign/cesm/development/pcwg
 setenv ICE_MACHINE_BASELINE /glade/derecho/scratch/$user/ICEPACK_BASELINE
 setenv ICE_MACHINE_SUBMIT "qsub"
 setenv ICE_MACHINE_ACCT P00000000
-setenv ICE_MACHINE_QUEUE "main"
+setenv ICE_MACHINE_QUEUE "develop"
 setenv ICE_MACHINE_TPNODE 128
 setenv ICE_MACHINE_BLDTHRDS 1
 setenv ICE_MACHINE_QSTAT "qstat "