From ba0bc0b8f76d965aa4186bc3e4ffc7255a91e7fb Mon Sep 17 00:00:00 2001 From: Tony Craig Date: Fri, 27 Sep 2024 10:43:13 -0700 Subject: [PATCH] Update Derecho Port (#501) Update inteloneapi, cray, intel, nvhpc modules/compiler. Changes answers for inteloneapi and cray. Switch default Derecho queue from main to develop to support quicker turn around and lower costs. Develop is a shared queue. Error in the snow physics with the cray compiler needs further assessment, but suggest we merge this now as is. That error does not present itself in CICE with snow physics with the same cray compiler. --- configuration/scripts/icepack.batch.csh | 2 +- .../scripts/machines/Macros.derecho_inteloneapi | 5 +++-- configuration/scripts/machines/env.derecho_cray | 10 +++++----- configuration/scripts/machines/env.derecho_gnu | 2 +- configuration/scripts/machines/env.derecho_intel | 10 +++++----- .../scripts/machines/env.derecho_intelclassic | 2 +- configuration/scripts/machines/env.derecho_inteloneapi | 10 +++++----- configuration/scripts/machines/env.derecho_nvhpc | 10 +++++----- 8 files changed, 26 insertions(+), 25 deletions(-) diff --git a/configuration/scripts/icepack.batch.csh b/configuration/scripts/icepack.batch.csh index ae49998c4..8a077969d 100755 --- a/configuration/scripts/icepack.batch.csh +++ b/configuration/scripts/icepack.batch.csh @@ -55,7 +55,7 @@ cat >> ${jobfile} << EOFB #PBS -l job_priority=regular #PBS -N ${ICE_CASENAME} #PBS -A ${acct} -#PBS -l select=${nnodes}:ncpus=${corespernode}:mpiprocs=${taskpernodelimit}:ompthreads=${nthrds} +#PBS -l select=${nnodes}:ncpus=${corespernode}:mpiprocs=${taskpernodelimit}:ompthreads=${nthrds}:mem=5GB #PBS -l walltime=${ICE_RUNLENGTH} #PBS -j oe #PBS -W umask=022 diff --git a/configuration/scripts/machines/Macros.derecho_inteloneapi b/configuration/scripts/machines/Macros.derecho_inteloneapi index df7d5b42f..e073f8c21 100644 --- a/configuration/scripts/machines/Macros.derecho_inteloneapi +++ b/configuration/scripts/machines/Macros.derecho_inteloneapi @@ -12,11 +12,12 @@ FFLAGS := -fp-model precise -convert big_endian -assume byterecl -ftz -trace FFLAGS_NOOPT:= -O0 ifeq ($(ICE_BLDDEBUG), true) - FFLAGS += -O0 -g -check uninit -check bounds -check pointers -fpe0 -check noarg_temp_created -link_mpi=dbg +# -check uninit is needed on the ld step but it still throws errors in 2023.* and 2024.0.*, likely compiler bug + FFLAGS += -O0 -g -check bounds -check pointers -fpe0 -check noarg_temp_created -link_mpi=dbg # FFLAGS += -O0 -g -check all -fpe0 -ftrapuv -fp-model except -check noarg_temp_created -link_mpi=dbg -stand f08 # FFLAGS += -O0 -g -check all -fpe0 -ftrapuv -fp-model except -check noarg_temp_created -init=snan,arrays -link_mpi=dbg else - FFLAGS += -O2 + FFLAGS += -O1 endif SCC := icx diff --git a/configuration/scripts/machines/env.derecho_cray b/configuration/scripts/machines/env.derecho_cray index 9faf34aa2..332c68e67 100644 --- a/configuration/scripts/machines/env.derecho_cray +++ b/configuration/scripts/machines/env.derecho_cray @@ -10,14 +10,14 @@ if ("$inp" != "-nomodules") then source ${MODULESHOME}/init/csh module --force purge -module load ncarenv/23.06 +module load ncarenv/23.09 module load craype -module load cce/15.0.1 +module load cce/16.0.1 module load ncarcompilers #module load cray-mpich/8.1.25 #module load hdf5/1.12.2 module load netcdf/4.9.2 -module load cray-libsci/23.02.1.1 +module load cray-libsci/23.09.1.1 # For perftools with mpiexec # module load perftools-base @@ -38,14 +38,14 @@ setenv OMP_STACKSIZE 64M setenv ICE_MACHINE_MACHNAME derecho setenv ICE_MACHINE_MACHINFO "HPE Cray EX Milan Slingshot 11" setenv ICE_MACHINE_ENVNAME cray -setenv ICE_MACHINE_ENVINFO "cce 15.0.1, netcdf4.9.2" +setenv ICE_MACHINE_ENVINFO "Cray clang/fortran cce 16.0.1, netcdf4.9.2" setenv ICE_MACHINE_MAKE gmake setenv ICE_MACHINE_WKDIR /glade/derecho/scratch/$user/ICEPACK_RUNS setenv ICE_MACHINE_INPUTDATA /glade/campaign/cesm/development/pcwg setenv ICE_MACHINE_BASELINE /glade/derecho/scratch/$user/ICEPACK_BASELINE setenv ICE_MACHINE_SUBMIT "qsub" setenv ICE_MACHINE_ACCT P00000000 -setenv ICE_MACHINE_QUEUE "main" +setenv ICE_MACHINE_QUEUE "develop" setenv ICE_MACHINE_TPNODE 128 setenv ICE_MACHINE_BLDTHRDS 1 setenv ICE_MACHINE_QSTAT "qstat " diff --git a/configuration/scripts/machines/env.derecho_gnu b/configuration/scripts/machines/env.derecho_gnu index 606dbbebc..0b5121c12 100644 --- a/configuration/scripts/machines/env.derecho_gnu +++ b/configuration/scripts/machines/env.derecho_gnu @@ -45,7 +45,7 @@ setenv ICE_MACHINE_INPUTDATA /glade/campaign/cesm/development/pcwg setenv ICE_MACHINE_BASELINE /glade/derecho/scratch/$user/ICEPACK_BASELINE setenv ICE_MACHINE_SUBMIT "qsub" setenv ICE_MACHINE_ACCT P00000000 -setenv ICE_MACHINE_QUEUE "main" +setenv ICE_MACHINE_QUEUE "develop" setenv ICE_MACHINE_TPNODE 128 setenv ICE_MACHINE_BLDTHRDS 1 setenv ICE_MACHINE_QSTAT "qstat " diff --git a/configuration/scripts/machines/env.derecho_intel b/configuration/scripts/machines/env.derecho_intel index fdf64e157..3c6bed2c0 100644 --- a/configuration/scripts/machines/env.derecho_intel +++ b/configuration/scripts/machines/env.derecho_intel @@ -10,14 +10,14 @@ if ("$inp" != "-nomodules") then source ${MODULESHOME}/init/csh module --force purge -module load ncarenv/23.06 +module load ncarenv/23.09 module load craype -module load intel/2023.0.0 +module load intel/2023.2.1 module load ncarcompilers #module load cray-mpich/8.1.25 #module load hdf5/1.12.2 module load netcdf/4.9.2 -module load cray-libsci/23.02.1.1 +module load cray-libsci/23.09.1.1 # For perftools with mpiexec # module load perftools-base @@ -38,14 +38,14 @@ setenv OMP_STACKSIZE 64M setenv ICE_MACHINE_MACHNAME derecho setenv ICE_MACHINE_MACHINFO "HPE Cray EX Milan Slingshot 11" setenv ICE_MACHINE_ENVNAME intel -setenv ICE_MACHINE_ENVINFO "ifort 2021.8.0 20221119, netcdf4.9.2" +setenv ICE_MACHINE_ENVINFO "ifort 2021.10.0 20230609, netcdf4.9.2" setenv ICE_MACHINE_MAKE gmake setenv ICE_MACHINE_WKDIR /glade/derecho/scratch/$user/ICEPACK_RUNS setenv ICE_MACHINE_INPUTDATA /glade/campaign/cesm/development/pcwg setenv ICE_MACHINE_BASELINE /glade/derecho/scratch/$user/ICEPACK_BASELINE setenv ICE_MACHINE_SUBMIT "qsub" setenv ICE_MACHINE_ACCT P00000000 -setenv ICE_MACHINE_QUEUE "main" +setenv ICE_MACHINE_QUEUE "develop" setenv ICE_MACHINE_TPNODE 128 setenv ICE_MACHINE_BLDTHRDS 1 setenv ICE_MACHINE_QSTAT "qstat " diff --git a/configuration/scripts/machines/env.derecho_intelclassic b/configuration/scripts/machines/env.derecho_intelclassic index a2d583733..b7ef39efe 100644 --- a/configuration/scripts/machines/env.derecho_intelclassic +++ b/configuration/scripts/machines/env.derecho_intelclassic @@ -45,7 +45,7 @@ setenv ICE_MACHINE_INPUTDATA /glade/campaign/cesm/development/pcwg setenv ICE_MACHINE_BASELINE /glade/derecho/scratch/$user/ICEPACK_BASELINE setenv ICE_MACHINE_SUBMIT "qsub" setenv ICE_MACHINE_ACCT P00000000 -setenv ICE_MACHINE_QUEUE "main" +setenv ICE_MACHINE_QUEUE "develop" setenv ICE_MACHINE_TPNODE 128 setenv ICE_MACHINE_BLDTHRDS 1 setenv ICE_MACHINE_QSTAT "qstat " diff --git a/configuration/scripts/machines/env.derecho_inteloneapi b/configuration/scripts/machines/env.derecho_inteloneapi index 358bd9834..70c98c68b 100644 --- a/configuration/scripts/machines/env.derecho_inteloneapi +++ b/configuration/scripts/machines/env.derecho_inteloneapi @@ -10,14 +10,14 @@ if ("$inp" != "-nomodules") then source ${MODULESHOME}/init/csh module --force purge -module load ncarenv/23.06 +module load ncarenv/23.09 module load craype -module load intel-oneapi/2023.0.0 +module load intel-oneapi/2023.2.1 module load ncarcompilers #module load cray-mpich/8.1.25 #module load hdf5/1.12.2 module load netcdf/4.9.2 -module load cray-libsci/23.02.1.1 +module load cray-libsci/23.09.1.1 # For perftools with mpiexec # module load perftools-base @@ -38,14 +38,14 @@ setenv OMP_STACKSIZE 64M setenv ICE_MACHINE_MACHNAME derecho setenv ICE_MACHINE_MACHINFO "HPE Cray EX Milan Slingshot 11" setenv ICE_MACHINE_ENVNAME inteloneapi -setenv ICE_MACHINE_ENVINFO "ifx 2023.0.0 20221201, netcdf4.9.2" +setenv ICE_MACHINE_ENVINFO "ifx 2023.2.0 20230721, netcdf4.9.2" setenv ICE_MACHINE_MAKE gmake setenv ICE_MACHINE_WKDIR /glade/derecho/scratch/$user/ICEPACK_RUNS setenv ICE_MACHINE_INPUTDATA /glade/campaign/cesm/development/pcwg setenv ICE_MACHINE_BASELINE /glade/derecho/scratch/$user/ICEPACK_BASELINE setenv ICE_MACHINE_SUBMIT "qsub" setenv ICE_MACHINE_ACCT P00000000 -setenv ICE_MACHINE_QUEUE "main" +setenv ICE_MACHINE_QUEUE "develop" setenv ICE_MACHINE_TPNODE 128 setenv ICE_MACHINE_BLDTHRDS 1 setenv ICE_MACHINE_QSTAT "qstat " diff --git a/configuration/scripts/machines/env.derecho_nvhpc b/configuration/scripts/machines/env.derecho_nvhpc index 4723c0da4..04c5ac456 100644 --- a/configuration/scripts/machines/env.derecho_nvhpc +++ b/configuration/scripts/machines/env.derecho_nvhpc @@ -10,14 +10,14 @@ if ("$inp" != "-nomodules") then source ${MODULESHOME}/init/csh module --force purge -module load ncarenv/23.06 +module load ncarenv/23.09 module load craype -module load nvhpc/23.5 +module load nvhpc/23.7 module load ncarcompilers #module load cray-mpich/8.1.25 #module load hdf5/1.12.2 module load netcdf/4.9.2 -module load cray-libsci/23.02.1.1 +module load cray-libsci/23.09.1.1 # For perftools with mpiexec # module load perftools-base @@ -38,14 +38,14 @@ setenv OMP_STACKSIZE 64M setenv ICE_MACHINE_MACHNAME derecho setenv ICE_MACHINE_MACHINFO "HPE Cray EX Milan Slingshot 11" setenv ICE_MACHINE_ENVNAME nvhpc -setenv ICE_MACHINE_ENVINFO "nvc 23.5-0, netcdf4.9.2" +setenv ICE_MACHINE_ENVINFO "nvfortran 23.7-0, netcdf4.9.2" setenv ICE_MACHINE_MAKE gmake setenv ICE_MACHINE_WKDIR /glade/derecho/scratch/$user/ICEPACK_RUNS setenv ICE_MACHINE_INPUTDATA /glade/campaign/cesm/development/pcwg setenv ICE_MACHINE_BASELINE /glade/derecho/scratch/$user/ICEPACK_BASELINE setenv ICE_MACHINE_SUBMIT "qsub" setenv ICE_MACHINE_ACCT P00000000 -setenv ICE_MACHINE_QUEUE "main" +setenv ICE_MACHINE_QUEUE "develop" setenv ICE_MACHINE_TPNODE 128 setenv ICE_MACHINE_BLDTHRDS 1 setenv ICE_MACHINE_QSTAT "qstat "