From c1fe8fb59e9a29be925190e2e7b08cf1bddb88ff Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Fri, 2 Oct 2026 18:27:48 -0600 Subject: [PATCH 01/37] I've itterated with Claude Opus 5.5 Medium all day on this script to get a monthly climatology of daily CPLHIST data for aerosols --- tools/unsupported/monthly_clim.sh | 306 ++++++++++++++++++++++++++++++ 1 file changed, 306 insertions(+) create mode 100755 tools/unsupported/monthly_clim.sh diff --git a/tools/unsupported/monthly_clim.sh b/tools/unsupported/monthly_clim.sh new file mode 100755 index 0000000000..69c6629546 --- /dev/null +++ b/tools/unsupported/monthly_clim.sh @@ -0,0 +1,306 @@ +#!/usr/bin/env bash +#PBS -N monthly_clim +#PBS -r n +#PBS -j oe +#PBS -S /bin/bash +#PBS -l select=1:ncpus=128:mpiprocs=128:ompthreads=1:mem=235GB +#PBS -l walltime=04:00:00 +#PBS -A P93300606 +#PBS -q main +# +# ============================================================================================== +# +# Build a 12-month climatology from daily CESM coupler history files: +# $ARCHDIR/$CASE/cpl/hist/$CASE.cpl.hx.atm.24h.avrg.YYYY-MM-DD-00000.nc +# for the years BEGYEAR to ENDYEAR +# Step 1: monthly mean for each year (ncra over the daily files of that month), +# for the default variable list, a list given with -v, or all variables with -a +# Step 2: average each calendar month over all years (ncra), then ncrcat to 12 records +# Step 3: reset the time axis to mid-month of YEAR (noleap calendar) +# +# ============================================================================================== + + +# NOTE: This will fail on errors, or if variables are used but not set, and pipefail fails on first exit in a pipe of commands +set -euo pipefail + +default_vars="atmImp_Faxa_bcph2,atmImp_Faxa_bcph3,atmImp_Faxa_ocph1,atmImp_Faxa_ocph2,atmImp_Faxa_ocph3,atmImp_Faxa_dstwet1,atmImp_Faxa_dstdry1,atmImp_Faxa_dstwet2,atmImp_Faxa_dstdry2,atmImp_Faxa_dstwet3,atmImp_Faxa_dstdry3,atmImp_Faxa_dstwet4,atmImp_Faxa_dstdry4" +default_year="1850" +default_outdir="ARCHDIR" +# Number of simultaneous ncra jobs: 16 inside a PBS batch job, 1 otherwise +if [[ -n ${PBS_JOBID:-} ]]; then + default_njobs=16 +else + default_njobs=1 +fi + +usage() { + cat >&2 < 0 && ! batch )); then + echo "-A is only used with -b" >&2 + exit 1 +fi + +# Error check input +for y in "$begyr" "$endyr"; do + if ! [[ $y =~ ^[0-9]{1,4}$ ]]; then + echo "BEGYEAR and ENDYEAR must be integers from 0 to 9999: $y" >&2 + exit 1 + fi +done +begyr=$((10#$begyr)); endyr=$((10#$endyr)) +if (( begyr > endyr )); then + echo "BEGYEAR ($begyr) must not be after ENDYEAR ($endyr)" >&2 + exit 1 +fi +if ! [[ $njobs =~ ^[1-9][0-9]*$ ]]; then + echo "NJOBS must be a positive integer: $njobs" >&2 + exit 1 +fi +if (( allvars && userlist )); then + echo "Use either -v or -a, not both" >&2 + exit 1 +fi +# Variable subset passed to ncra (coordinates and time_bnds are included automatically) +if (( allvars )); then + vopt=() + echo "Averaging all variables" +else + [[ -n $varlist ]] || { echo "Variable list given with -v is empty" >&2; exit 1; } + vopt=(-v "$varlist") + echo "Averaging variables: $varlist" +fi +if ! [[ $year =~ ^[0-9]{1,4}$ ]]; then + echo "YEAR must be an integer from 0 to 9999: $year" >&2 + exit 1 +fi +year=$(printf '%04d' "$((10#$year))") +if [ ! -d "$archdir" ]; then + echo "Input $archdir directory does not exist" >&2 + exit 1 +fi +indir="$archdir/$case/cpl/hist" +if [ "$outdir" = "$default_outdir" ]; then + outdir="$archdir/$case/proc" +elif [ ! -d "$outdir" ]; then + echo "Output $outdir directory does not exist" >&2 + exit 1 +else + outdir="$outdir/$case/proc" +fi +if [ ! -d "$indir" ]; then + echo "$indir directory does not exist" >&2 + exit 1 +fi +mkdir -p "$outdir/mon" || { echo "Can not create $outdir/mon" >&2; exit 1; } + +echo "Create a monthly climatology from a list of years from daily averaged CPLHIST files for DATM" + +if command -v module >/dev/null 2>&1; then + # Lmod's module function can reference unset variables, so relax set -u around it + set +u + module load nco + set -u +else + echo "module command not found, so expecting NCO to already be in the PATH" >&2 +fi +for cmd in ncra ncrcat ncap2 ncatted ncks; do + command -v $cmd >/dev/null || { echo "$cmd not found (module load nco)" >&2; exit 1; } +done +prefix="$case.cpl.hx.atm.24h.avrg" +noleap_days=(31 28 31 30 31 30 31 31 30 31 30 31) + +# Year-months to process, BEGYEAR-01 to ENDYEAR-12 (files are matched by name, not the time coordinate) +yms=() +for (( y = begyr; y <= endyr; y++ )); do + for mm in 01 02 03 04 05 06 07 08 09 10 11 12; do + yms+=("$(printf '%04d' "$y")-$mm") + done +done +echo "Processing ${#yms[@]} year-months: ${yms[0]} .. ${yms[-1]}" + +# Check that every requested variable is in the first file before doing any work +if (( ! allvars )); then + echo "Checking that all requested variables are present in the first file for ${yms[0]}" + first=$(find "$indir" -maxdepth 1 -name "${prefix}.${yms[0]}-??-00000.nc" | sort | sed -n 1p) + [[ -n $first ]] || { echo "ERROR: no files for ${yms[0]} matching ${prefix}.${yms[0]}-??-00000.nc in $indir" >&2; exit 1; } + missing=() + IFS=',' read -ra vars <<< "$varlist" + for v in "${vars[@]}"; do + ncks -m -v "$v" "$first" >/dev/null 2>&1 || missing+=("$v") + done + if (( ${#missing[@]} > 0 )); then + echo "ERROR: variables not found in $first: ${missing[*]}" >&2 + exit 1 + fi +fi + +# Print the year at January of BEGYEAR, BEGYEAR+10, BEGYEAR+20, ... (every 120 year-months) +progress() { + local y=$((10#${1%%-*})) + if [[ ${1#*-} == 01 ]] && (( (y - begyr) % 10 == 0 )); then + echo " at year $y" + fi +} + +# Run ncra in the background, with at most $njobs running at once +# Usage: bg_ncra OUTFILE INFILE... (uses the ncra_opts array for extra ncra options) +# The input list goes to ncra on stdin, so a long list never hits the shell argument limit. +# A failed job leaves the $failed marker file, which wait_ncra checks. +failed="$outdir/mon/.ncra_failed" +rm -f "$failed" +bg_ncra() { + local out=$1; shift + while (( $(jobs -rp | wc -l) >= njobs )); do wait -n || true; done + ( printf '%s\n' "$@" | ncra -O "${ncra_opts[@]}" -o "$out" \ + || { echo "ERROR: ncra failed creating $out" >&2; touch "$failed"; } ) & +} +# Wait for all background ncra jobs, and stop if any of them failed +wait_ncra() { + wait + if [[ -e $failed ]]; then + echo "ERROR: one or more ncra jobs failed, stopping" >&2 + exit 1 + fi +} + +# Check every month in the range has a full set of daily files before doing any averaging +bad=0 +echo "Checking that every month has the expected number of daily files" +for ym in "${yms[@]}"; do + mm=${ym#*-} + progress "$ym" + n=$(find "$indir" -maxdepth 1 -name "${prefix}.${ym}-??-00000.nc" | wc -l) + expect=${noleap_days[$((10#$mm - 1))]} + if (( n != expect )); then + echo "ERROR: $ym has $n daily files (expected $expect)" >&2 + bad=1 + fi +done +(( ! bad )) || exit 1 + +# With -b: the input has been checked, so submit this script to PBS and stop here. +# The arguments are saved (shell-quoted, so commas and spaces survive) to a file that the job reads back. +if (( batch )); then + command -v qsub >/dev/null || { echo "qsub not found, can not submit with -b" >&2; exit 1; } + argfile="$outdir/monthly_clim.args.$$" + printf '%q ' "${job_args[@]}" > "$argfile" + echo "Submitting to PBS with arguments: ${job_args[*]}" + if ! jobid=$(qsub "${qsub_opts[@]}" -v "MONTHLY_CLIM_ARGFILE=$argfile,MONTHLY_CLIM_WORKDIR=$PWD" "$0"); then + rm -f "$argfile" + echo "ERROR: qsub failed" >&2 + exit 1 + fi + echo "Successfully submitted job $jobid (log file will be monthly_clim.o${jobid%%.*} in $PWD)" + exit 0 +fi + +# ---- Step 1: monthly means for each year ---- +echo "Computing monthly means for each year ($njobs ncra jobs at a time)" +ncra_opts=("${vopt[@]}") +for ym in "${yms[@]}"; do + progress "$ym" + mapfile -t files < <(find "$indir" -maxdepth 1 -name "${prefix}.${ym}-??-00000.nc" | sort) + bg_ncra "$outdir/mon/mon_${ym}.nc" "${files[@]}" +done +wait_ncra + +# ---- Step 2: average each calendar month over the years ---- +echo "Averaging each calendar month over the years" +ncra_opts=() +for mm in 01 02 03 04 05 06 07 08 09 10 11 12; do + # Only handle this run's years, so leftover files from an earlier -k run are not included + mons=() + for (( y = begyr; y <= endyr; y++ )); do + mons+=("$outdir/mon/mon_$(printf '%04d' "$y")-${mm}.nc") + done + echo "Month $mm: averaging ${#mons[@]} years ($begyr-$endyr)" + bg_ncra "$outdir/mon/clim_${mm}.nc" "${mons[@]}" +done +wait_ncra + +echo "Put the climatology into one and file and finalize metadata on the file" +out="$outdir/${case}.cpl.hx.atm.monclim.$(printf '%04d-%04d' "$begyr" "$endyr")_c$(date +%Y%m%d).nc" +ncrcat -O "$outdir"/mon/clim_{01..12}.nc "$out" + +# ---- Step 3: reset time to mid-month of $year (noleap), days since $year-01-01 ---- +ncap2 -O -s 'time[$time]={15.5,45.0,74.5,105.0,135.5,166.0,196.5,227.5,258.0,288.5,319.0,349.5};' "$out" "$out" +ncatted -O -a units,time,o,c,"days since ${year}-01-01 00:00:00" \ + -a calendar,time,o,c,"noleap" "$out" +# If the file has time bounds, set them to the start and end of each month +if ncks --trd -m -v time_bnds "$out" >/dev/null 2>&1; then + ncap2 -O -s '*lb[$time]={0.,31.,59.,90.,120.,151.,181.,212.,243.,273.,304.,334.}; + *ub[$time]={31.,59.,90.,120.,151.,181.,212.,243.,273.,304.,334.,365.}; + time_bnds(:,0)=lb; time_bnds(:,1)=ub;' "$out" "$out" + ncatted -O -a units,time_bnds,o,c,"days since ${year}-01-01 00:00:00" "$out" +fi +echo "Wrote $out (time = mid-month of year $year, noleap)" + +# Remove the temporary monthly files unless -k was given +if (( keep )); then + echo "Kept intermediate files in $outdir/mon" +else + rm -r "$outdir/mon" +fi + +echo "" +echo "Successfully created monthly climatology: $out" From 2b8dc0d8e681ce9fa6a379dfaccb94d88f6fa625 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Mon, 5 Oct 2026 12:29:08 -0600 Subject: [PATCH 02/37] Always output the lat/lon variables, and have Claude bring in some general changes from looking at our other bash scripts, one of which is handling abort on either killing the batch job, or hitting Ctrl-C when running interactive --- tools/unsupported/monthly_clim.sh | 37 ++++++++++++++++++++++++++----- 1 file changed, 32 insertions(+), 5 deletions(-) diff --git a/tools/unsupported/monthly_clim.sh b/tools/unsupported/monthly_clim.sh index 69c6629546..c14b1834cc 100755 --- a/tools/unsupported/monthly_clim.sh +++ b/tools/unsupported/monthly_clim.sh @@ -1,7 +1,8 @@ #!/usr/bin/env bash #PBS -N monthly_clim #PBS -r n -#PBS -j oe +#PBS -j oe +#PBS -k eod #PBS -S /bin/bash #PBS -l select=1:ncpus=128:mpiprocs=128:ompthreads=1:mem=235GB #PBS -l walltime=04:00:00 @@ -25,6 +26,8 @@ set -euo pipefail default_vars="atmImp_Faxa_bcph2,atmImp_Faxa_bcph3,atmImp_Faxa_ocph1,atmImp_Faxa_ocph2,atmImp_Faxa_ocph3,atmImp_Faxa_dstwet1,atmImp_Faxa_dstdry1,atmImp_Faxa_dstwet2,atmImp_Faxa_dstdry2,atmImp_Faxa_dstwet3,atmImp_Faxa_dstdry3,atmImp_Faxa_dstwet4,atmImp_Faxa_dstdry4" +# Grid fields that are always written to the output, whatever variable list is used +always_vars="atmImp_lon,atmImp_lat" default_year="1850" default_outdir="ARCHDIR" # Number of simultaneous ncra jobs: 16 inside a PBS batch job, 1 otherwise @@ -34,13 +37,16 @@ else default_njobs=1 fi +# Print the help: to stdout and exit 0 for -h, to stderr and exit 1 for bad arguments usage() { - cat >&2 <&1; else exec 3>&2; fi + cat >&3 <&2; exit 1; } + # Add the grid fields that are always output, dropping any duplicates + IFS=',' read -ra vl <<< "$varlist,$always_vars" + varlist=$(printf '%s\n' "${vl[@]}" | awk 'NF && !seen[$0]++' | paste -sd, -) vopt=(-v "$varlist") echo "Averaging variables: $varlist" fi @@ -223,6 +233,23 @@ wait_ncra() { fi } +# On Ctrl+C (INT), or when PBS kills the job with qdel or at the walltime limit (TERM), +# also stop the background ncra jobs. Background jobs in a script ignore Ctrl+C, +# so without this they would keep running after the script exits. +stop_on_signal() { + local sig=$1 pid + trap '' INT TERM + echo "Caught SIG$sig, stopping background ncra jobs" >&2 + for pid in $(jobs -p); do + pkill -TERM -P "$pid" 2>/dev/null || true # the ncra started by this background job + kill -TERM "$pid" 2>/dev/null || true + done + wait 2>/dev/null || true + if [[ $sig == INT ]]; then exit 130; else exit 143; fi +} +trap 'stop_on_signal INT' INT +trap 'stop_on_signal TERM' TERM + # Check every month in the range has a full set of daily files before doing any averaging bad=0 echo "Checking that every month has the expected number of daily files" From f99f9f1f6b0bba217e90a9427233320887dd1a88 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Tue, 6 Oct 2026 13:39:35 -0600 Subject: [PATCH 03/37] Rename script --- tools/unsupported/{monthly_clim.sh => daily_to_monthly_clim.sh} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tools/unsupported/{monthly_clim.sh => daily_to_monthly_clim.sh} (100%) diff --git a/tools/unsupported/monthly_clim.sh b/tools/unsupported/daily_to_monthly_clim.sh similarity index 100% rename from tools/unsupported/monthly_clim.sh rename to tools/unsupported/daily_to_monthly_clim.sh From 92d7bfd6c7320a689413c078161e2d15d645bea0 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Wed, 7 Oct 2026 13:48:44 -0600 Subject: [PATCH 04/37] Asked Claude to modularize and put into smaller functions --- tools/unsupported/daily_to_monthly_clim.sh | 511 ++++++++++++--------- 1 file changed, 300 insertions(+), 211 deletions(-) diff --git a/tools/unsupported/daily_to_monthly_clim.sh b/tools/unsupported/daily_to_monthly_clim.sh index c14b1834cc..3f05ebec54 100755 --- a/tools/unsupported/daily_to_monthly_clim.sh +++ b/tools/unsupported/daily_to_monthly_clim.sh @@ -1,9 +1,9 @@ #!/usr/bin/env bash -#PBS -N monthly_clim -#PBS -r n +#PBS -N daily_to_monthly_clim +#PBS -r n #PBS -j oe #PBS -k eod -#PBS -S /bin/bash +#PBS -S /bin/bash #PBS -l select=1:ncpus=128:mpiprocs=128:ompthreads=1:mem=235GB #PBS -l walltime=04:00:00 #PBS -A P93300606 @@ -19,6 +19,8 @@ # Step 2: average each calendar month over all years (ncra), then ncrcat to 12 records # Step 3: reset the time axis to mid-month of YEAR (noleap calendar) # +# The work is done in the functions below, called in order from main at the bottom of the file. +# # ============================================================================================== @@ -36,6 +38,13 @@ if [[ -n ${PBS_JOBID:-} ]]; then else default_njobs=1 fi +# PBS job name (same as the #PBS -N line above), used to name the job log file +pbs_jobname="daily_to_monthly_clim" +noleap_days=(31 28 31 30 31 30 31 31 30 31 30 31) + +# ============================================================================================== +# Command line handling +# ============================================================================================== # Print the help: to stdout and exit 0 for -h, to stderr and exit 1 for bad arguments usage() { @@ -64,145 +73,138 @@ EOF exit "$status" } -keep=0 -varlist=$default_vars -allvars=0 -userlist=0 -year=$default_year -outdir=$default_outdir -njobs=$default_njobs -batch=0 -qsub_opts=() - -# Inside a job submitted with -b: restore the original arguments saved by the submitting script -# (qsub can't pass arguments to a job script, so they go through a file named in the environment) -if [[ -n ${MONTHLY_CLIM_ARGFILE:-} ]]; then - cd "${MONTHLY_CLIM_WORKDIR:-.}" - eval "set -- $(cat "$MONTHLY_CLIM_ARGFILE")" - rm -f "$MONTHLY_CLIM_ARGFILE" -fi - -# Arguments to pass on to the batch job: everything except -b and -A -job_args=() -while getopts "bA:kj:v:o:y:ah" opt; do - case $opt in - b) batch=1 ;; - A) qsub_opts+=(-A "$OPTARG") ;; - k) keep=1; job_args+=(-k) ;; - j) njobs=$OPTARG; job_args+=(-j "$OPTARG") ;; - v) varlist=$OPTARG; userlist=1; job_args+=(-v "$OPTARG") ;; - o) outdir=$OPTARG; job_args+=(-o "$OPTARG") ;; - y) year=$OPTARG; job_args+=(-y "$OPTARG") ;; - a) allvars=1; job_args+=(-a) ;; - h) usage 0 ;; - *) usage ;; - esac -done -shift $((OPTIND-1)) -[[ $# -eq 4 ]] || usage -archdir=$1; case=$2; begyr=$3; endyr=$4 -job_args+=("$@") -if (( ${#qsub_opts[@]} > 0 && ! batch )); then - echo "-A is only used with -b" >&2 - exit 1 -fi +# Parse the command line options and arguments into global settings +# Sets: keep varlist allvars userlist year outdir njobs batch qsub_opts job_args archdir case begyr endyr +parse_args() { + local opt OPTIND=1 + keep=0 + varlist=$default_vars + allvars=0 + userlist=0 + year=$default_year + outdir=$default_outdir + njobs=$default_njobs + batch=0 + qsub_opts=() + # Arguments to pass on to the batch job: everything except -b and -A + job_args=() + while getopts "bA:kj:v:o:y:ah" opt; do + case $opt in + b) batch=1 ;; + A) qsub_opts+=(-A "$OPTARG") ;; + k) keep=1; job_args+=(-k) ;; + j) njobs=$OPTARG; job_args+=(-j "$OPTARG") ;; + v) varlist=$OPTARG; userlist=1; job_args+=(-v "$OPTARG") ;; + o) outdir=$OPTARG; job_args+=(-o "$OPTARG") ;; + y) year=$OPTARG; job_args+=(-y "$OPTARG") ;; + a) allvars=1; job_args+=(-a) ;; + h) usage 0 ;; + *) usage ;; + esac + done + shift $((OPTIND-1)) + [[ $# -eq 4 ]] || usage + archdir=$1; case=$2; begyr=$3; endyr=$4 + job_args+=("$@") +} -# Error check input -for y in "$begyr" "$endyr"; do - if ! [[ $y =~ ^[0-9]{1,4}$ ]]; then - echo "BEGYEAR and ENDYEAR must be integers from 0 to 9999: $y" >&2 +# Check the settings from parse_args, and set up the variable list and the input and output directories +# Sets: begyr endyr year (normalized), varlist vopt, indir outdir prefix +check_args() { + local y vl + if (( ${#qsub_opts[@]} > 0 && ! batch )); then + echo "-A is only used with -b" >&2 exit 1 fi -done -begyr=$((10#$begyr)); endyr=$((10#$endyr)) -if (( begyr > endyr )); then - echo "BEGYEAR ($begyr) must not be after ENDYEAR ($endyr)" >&2 - exit 1 -fi -if ! [[ $njobs =~ ^[1-9][0-9]*$ ]]; then - echo "NJOBS must be a positive integer: $njobs" >&2 - exit 1 -fi -if (( allvars && userlist )); then - echo "Use either -v or -a, not both" >&2 - exit 1 -fi -# Variable subset passed to ncra (coordinates and time_bnds are included automatically) -if (( allvars )); then - vopt=() - echo "Averaging all variables" -else - [[ -n $varlist ]] || { echo "Variable list given with -v is empty" >&2; exit 1; } - # Add the grid fields that are always output, dropping any duplicates - IFS=',' read -ra vl <<< "$varlist,$always_vars" - varlist=$(printf '%s\n' "${vl[@]}" | awk 'NF && !seen[$0]++' | paste -sd, -) - vopt=(-v "$varlist") - echo "Averaging variables: $varlist" -fi -if ! [[ $year =~ ^[0-9]{1,4}$ ]]; then - echo "YEAR must be an integer from 0 to 9999: $year" >&2 - exit 1 -fi -year=$(printf '%04d' "$((10#$year))") -if [ ! -d "$archdir" ]; then - echo "Input $archdir directory does not exist" >&2 - exit 1 -fi -indir="$archdir/$case/cpl/hist" -if [ "$outdir" = "$default_outdir" ]; then - outdir="$archdir/$case/proc" -elif [ ! -d "$outdir" ]; then - echo "Output $outdir directory does not exist" >&2 - exit 1 -else - outdir="$outdir/$case/proc" -fi -if [ ! -d "$indir" ]; then - echo "$indir directory does not exist" >&2 - exit 1 -fi -mkdir -p "$outdir/mon" || { echo "Can not create $outdir/mon" >&2; exit 1; } + for y in "$begyr" "$endyr"; do + if ! [[ $y =~ ^[0-9]{1,4}$ ]]; then + echo "BEGYEAR and ENDYEAR must be integers from 0 to 9999: $y" >&2 + exit 1 + fi + done + begyr=$((10#$begyr)); endyr=$((10#$endyr)) + if (( begyr > endyr )); then + echo "BEGYEAR ($begyr) must not be after ENDYEAR ($endyr)" >&2 + exit 1 + fi + if ! [[ $njobs =~ ^[1-9][0-9]*$ ]]; then + echo "NJOBS must be a positive integer: $njobs" >&2 + exit 1 + fi + if (( allvars && userlist )); then + echo "Use either -v or -a, not both" >&2 + exit 1 + fi + # Variable subset passed to ncra (coordinates and time_bnds are included automatically) + if (( allvars )); then + vopt=() + echo "Averaging all variables" + else + [[ -n $varlist ]] || { echo "Variable list given with -v is empty" >&2; exit 1; } + # Add the grid fields that are always output, dropping any duplicates + IFS=',' read -ra vl <<< "$varlist,$always_vars" + varlist=$(printf '%s\n' "${vl[@]}" | awk 'NF && !seen[$0]++' | paste -sd, -) + vopt=(-v "$varlist") + echo "Averaging variables: $varlist" + fi + if ! [[ $year =~ ^[0-9]{1,4}$ ]]; then + echo "YEAR must be an integer from 0 to 9999: $year" >&2 + exit 1 + fi + year=$(printf '%04d' "$((10#$year))") + if [ ! -d "$archdir" ]; then + echo "Input $archdir directory does not exist" >&2 + exit 1 + fi + indir="$archdir/$case/cpl/hist" + if [ "$outdir" = "$default_outdir" ]; then + outdir="$archdir/$case/proc" + elif [ ! -d "$outdir" ]; then + echo "Output $outdir directory does not exist" >&2 + exit 1 + else + outdir="$outdir/$case/proc" + fi + if [ ! -d "$indir" ]; then + echo "$indir directory does not exist" >&2 + exit 1 + fi + mkdir -p "$outdir/mon" || { echo "Can not create $outdir/mon" >&2; exit 1; } + prefix="$case.cpl.hx.atm.24h.avrg" +} -echo "Create a monthly climatology from a list of years from daily averaged CPLHIST files for DATM" +# ============================================================================================== +# Environment and input checks +# ============================================================================================== -if command -v module >/dev/null 2>&1; then - # Lmod's module function can reference unset variables, so relax set -u around it - set +u - module load nco - set -u -else - echo "module command not found, so expecting NCO to already be in the PATH" >&2 -fi -for cmd in ncra ncrcat ncap2 ncatted ncks; do - command -v $cmd >/dev/null || { echo "$cmd not found (module load nco)" >&2; exit 1; } -done -prefix="$case.cpl.hx.atm.24h.avrg" -noleap_days=(31 28 31 30 31 30 31 31 30 31 30 31) +# Load NCO and make sure the NCO commands are available +setup_nco() { + local cmd + if command -v module >/dev/null 2>&1; then + # Lmod's module function can reference unset variables, so relax set -u around it + set +u + module load nco + set -u + else + echo "module command not found, so expecting NCO to already be in the PATH" >&2 + fi + for cmd in ncra ncrcat ncap2 ncatted ncks; do + command -v $cmd >/dev/null || { echo "$cmd not found (module load nco)" >&2; exit 1; } + done +} # Year-months to process, BEGYEAR-01 to ENDYEAR-12 (files are matched by name, not the time coordinate) -yms=() -for (( y = begyr; y <= endyr; y++ )); do - for mm in 01 02 03 04 05 06 07 08 09 10 11 12; do - yms+=("$(printf '%04d' "$y")-$mm") - done -done -echo "Processing ${#yms[@]} year-months: ${yms[0]} .. ${yms[-1]}" - -# Check that every requested variable is in the first file before doing any work -if (( ! allvars )); then - echo "Checking that all requested variables are present in the first file for ${yms[0]}" - first=$(find "$indir" -maxdepth 1 -name "${prefix}.${yms[0]}-??-00000.nc" | sort | sed -n 1p) - [[ -n $first ]] || { echo "ERROR: no files for ${yms[0]} matching ${prefix}.${yms[0]}-??-00000.nc in $indir" >&2; exit 1; } - missing=() - IFS=',' read -ra vars <<< "$varlist" - for v in "${vars[@]}"; do - ncks -m -v "$v" "$first" >/dev/null 2>&1 || missing+=("$v") +# Sets: yms +set_year_months() { + local y mm + yms=() + for (( y = begyr; y <= endyr; y++ )); do + for mm in 01 02 03 04 05 06 07 08 09 10 11 12; do + yms+=("$(printf '%04d' "$y")-$mm") + done done - if (( ${#missing[@]} > 0 )); then - echo "ERROR: variables not found in $first: ${missing[*]}" >&2 - exit 1 - fi -fi + echo "Processing ${#yms[@]} year-months: ${yms[0]} .. ${yms[-1]}" +} # Print the year at January of BEGYEAR, BEGYEAR+10, BEGYEAR+20, ... (every 120 year-months) progress() { @@ -212,12 +214,77 @@ progress() { fi } +# Check the input files before doing any work: every requested variable is in the first file, +# and every month in the range has a full set of daily files +check_input_files() { + local first v ym mm n expect bad=0 + local -a vars missing=() + if (( ! allvars )); then + echo "Checking that all requested variables are present in the first file for ${yms[0]}" + first=$(find "$indir" -maxdepth 1 -name "${prefix}.${yms[0]}-??-00000.nc" | sort | sed -n 1p) + [[ -n $first ]] || { echo "ERROR: no files for ${yms[0]} matching ${prefix}.${yms[0]}-??-00000.nc in $indir" >&2; exit 1; } + IFS=',' read -ra vars <<< "$varlist" + for v in "${vars[@]}"; do + ncks -m -v "$v" "$first" >/dev/null 2>&1 || missing+=("$v") + done + if (( ${#missing[@]} > 0 )); then + echo "ERROR: variables not found in $first: ${missing[*]}" >&2 + exit 1 + fi + fi + + echo "Checking that every month has the expected number of daily files" + for ym in "${yms[@]}"; do + mm=${ym#*-} + progress "$ym" + n=$(find "$indir" -maxdepth 1 -name "${prefix}.${ym}-??-00000.nc" | wc -l) + expect=${noleap_days[$((10#$mm - 1))]} + if (( n != expect )); then + echo "ERROR: $ym has $n daily files (expected $expect)" >&2 + bad=1 + fi + done + (( ! bad )) || exit 1 +} + +# ============================================================================================== +# PBS batch submission +# ============================================================================================== + +# Inside a job submitted with -b: go back to the submit directory and read the original +# arguments saved by the submitting script (qsub can't pass arguments to a job script, +# so they go through a file named in the environment) +# Sets: saved_args +read_batch_args() { + cd "${DAILY_TO_MONTHLY_CLIM_WORKDIR:-.}" + eval "saved_args=($(cat "$DAILY_TO_MONTHLY_CLIM_ARGFILE"))" + rm -f "$DAILY_TO_MONTHLY_CLIM_ARGFILE" +} + +# With -b: submit this script to PBS with the same arguments (minus -b and -A). +# The arguments are saved (shell-quoted, so commas and spaces survive) to a file that the job reads back. +submit_batch() { + local argfile jobid + command -v qsub >/dev/null || { echo "qsub not found, can not submit with -b" >&2; exit 1; } + argfile="$outdir/$pbs_jobname.args.$$" + printf '%q ' "${job_args[@]}" > "$argfile" + echo "Submitting to PBS with arguments: ${job_args[*]}" + if ! jobid=$(qsub "${qsub_opts[@]}" -v "DAILY_TO_MONTHLY_CLIM_ARGFILE=$argfile,DAILY_TO_MONTHLY_CLIM_WORKDIR=$PWD" "$0"); then + rm -f "$argfile" + echo "ERROR: qsub failed" >&2 + exit 1 + fi + echo "Successfully submitted job $jobid (log file will be $pbs_jobname.o${jobid%%.*} in $PWD)" +} + +# ============================================================================================== +# Running ncra jobs in the background +# ============================================================================================== + # Run ncra in the background, with at most $njobs running at once # Usage: bg_ncra OUTFILE INFILE... (uses the ncra_opts array for extra ncra options) # The input list goes to ncra on stdin, so a long list never hits the shell argument limit. # A failed job leaves the $failed marker file, which wait_ncra checks. -failed="$outdir/mon/.ncra_failed" -rm -f "$failed" bg_ncra() { local out=$1; shift while (( $(jobs -rp | wc -l) >= njobs )); do wait -n || true; done @@ -247,87 +314,109 @@ stop_on_signal() { wait 2>/dev/null || true if [[ $sig == INT ]]; then exit 130; else exit 143; fi } -trap 'stop_on_signal INT' INT -trap 'stop_on_signal TERM' TERM - -# Check every month in the range has a full set of daily files before doing any averaging -bad=0 -echo "Checking that every month has the expected number of daily files" -for ym in "${yms[@]}"; do - mm=${ym#*-} - progress "$ym" - n=$(find "$indir" -maxdepth 1 -name "${prefix}.${ym}-??-00000.nc" | wc -l) - expect=${noleap_days[$((10#$mm - 1))]} - if (( n != expect )); then - echo "ERROR: $ym has $n daily files (expected $expect)" >&2 - bad=1 - fi -done -(( ! bad )) || exit 1 -# With -b: the input has been checked, so submit this script to PBS and stop here. -# The arguments are saved (shell-quoted, so commas and spaces survive) to a file that the job reads back. -if (( batch )); then - command -v qsub >/dev/null || { echo "qsub not found, can not submit with -b" >&2; exit 1; } - argfile="$outdir/monthly_clim.args.$$" - printf '%q ' "${job_args[@]}" > "$argfile" - echo "Submitting to PBS with arguments: ${job_args[*]}" - if ! jobid=$(qsub "${qsub_opts[@]}" -v "MONTHLY_CLIM_ARGFILE=$argfile,MONTHLY_CLIM_WORKDIR=$PWD" "$0"); then - rm -f "$argfile" - echo "ERROR: qsub failed" >&2 - exit 1 - fi - echo "Successfully submitted job $jobid (log file will be monthly_clim.o${jobid%%.*} in $PWD)" - exit 0 -fi +# ============================================================================================== +# The climatology steps +# ============================================================================================== # ---- Step 1: monthly means for each year ---- -echo "Computing monthly means for each year ($njobs ncra jobs at a time)" -ncra_opts=("${vopt[@]}") -for ym in "${yms[@]}"; do - progress "$ym" - mapfile -t files < <(find "$indir" -maxdepth 1 -name "${prefix}.${ym}-??-00000.nc" | sort) - bg_ncra "$outdir/mon/mon_${ym}.nc" "${files[@]}" -done -wait_ncra +monthly_means() { + local ym + local -a files + echo "Computing monthly means for each year ($njobs ncra jobs at a time)" + ncra_opts=("${vopt[@]}") + for ym in "${yms[@]}"; do + progress "$ym" + mapfile -t files < <(find "$indir" -maxdepth 1 -name "${prefix}.${ym}-??-00000.nc" | sort) + bg_ncra "$outdir/mon/mon_${ym}.nc" "${files[@]}" + done + wait_ncra +} # ---- Step 2: average each calendar month over the years ---- -echo "Averaging each calendar month over the years" -ncra_opts=() -for mm in 01 02 03 04 05 06 07 08 09 10 11 12; do - # Only handle this run's years, so leftover files from an earlier -k run are not included - mons=() - for (( y = begyr; y <= endyr; y++ )); do - mons+=("$outdir/mon/mon_$(printf '%04d' "$y")-${mm}.nc") +monthly_climatology() { + local mm y + local -a mons + echo "Averaging each calendar month over the years" + ncra_opts=() + for mm in 01 02 03 04 05 06 07 08 09 10 11 12; do + # Only handle this run's years, so leftover files from an earlier -k run are not included + mons=() + for (( y = begyr; y <= endyr; y++ )); do + mons+=("$outdir/mon/mon_$(printf '%04d' "$y")-${mm}.nc") + done + echo "Month $mm: averaging ${#mons[@]} years ($begyr-$endyr)" + bg_ncra "$outdir/mon/clim_${mm}.nc" "${mons[@]}" done - echo "Month $mm: averaging ${#mons[@]} years ($begyr-$endyr)" - bg_ncra "$outdir/mon/clim_${mm}.nc" "${mons[@]}" -done -wait_ncra - -echo "Put the climatology into one and file and finalize metadata on the file" -out="$outdir/${case}.cpl.hx.atm.monclim.$(printf '%04d-%04d' "$begyr" "$endyr")_c$(date +%Y%m%d).nc" -ncrcat -O "$outdir"/mon/clim_{01..12}.nc "$out" - -# ---- Step 3: reset time to mid-month of $year (noleap), days since $year-01-01 ---- -ncap2 -O -s 'time[$time]={15.5,45.0,74.5,105.0,135.5,166.0,196.5,227.5,258.0,288.5,319.0,349.5};' "$out" "$out" -ncatted -O -a units,time,o,c,"days since ${year}-01-01 00:00:00" \ - -a calendar,time,o,c,"noleap" "$out" -# If the file has time bounds, set them to the start and end of each month -if ncks --trd -m -v time_bnds "$out" >/dev/null 2>&1; then - ncap2 -O -s '*lb[$time]={0.,31.,59.,90.,120.,151.,181.,212.,243.,273.,304.,334.}; - *ub[$time]={31.,59.,90.,120.,151.,181.,212.,243.,273.,304.,334.,365.}; - time_bnds(:,0)=lb; time_bnds(:,1)=ub;' "$out" "$out" - ncatted -O -a units,time_bnds,o,c,"days since ${year}-01-01 00:00:00" "$out" -fi -echo "Wrote $out (time = mid-month of year $year, noleap)" + wait_ncra +} + +# ---- Step 3: put the 12 months into one file and reset time to mid-month of $year (noleap) ---- +# Sets: out +write_output() { + echo "Put the climatology into one file and finalize metadata on the file" + out="$outdir/${case}.cpl.hx.atm.monclim.$(printf '%04d-%04d' "$begyr" "$endyr")_c$(date +%Y%m%d).nc" + ncrcat -O "$outdir"/mon/clim_{01..12}.nc "$out" + + # Time is days since $year-01-01 + ncap2 -O -s 'time[$time]={15.5,45.0,74.5,105.0,135.5,166.0,196.5,227.5,258.0,288.5,319.0,349.5};' "$out" "$out" + ncatted -O -a units,time,o,c,"days since ${year}-01-01 00:00:00" \ + -a calendar,time,o,c,"noleap" "$out" + # If the file has time bounds, set them to the start and end of each month + if ncks --trd -m -v time_bnds "$out" >/dev/null 2>&1; then + ncap2 -O -s '*lb[$time]={0.,31.,59.,90.,120.,151.,181.,212.,243.,273.,304.,334.}; + *ub[$time]={31.,59.,90.,120.,151.,181.,212.,243.,273.,304.,334.,365.}; + time_bnds(:,0)=lb; time_bnds(:,1)=ub;' "$out" "$out" + ncatted -O -a units,time_bnds,o,c,"days since ${year}-01-01 00:00:00" "$out" + fi + echo "Wrote $out (time = mid-month of year $year, noleap)" +} # Remove the temporary monthly files unless -k was given -if (( keep )); then - echo "Kept intermediate files in $outdir/mon" -else - rm -r "$outdir/mon" -fi +clean_up() { + if (( keep )); then + echo "Kept intermediate files in $outdir/mon" + else + rm -r "$outdir/mon" + fi +} + +# ============================================================================================== +# Main program +# ============================================================================================== + +main() { + # Inside a job submitted with -b, the arguments come from the file saved at submission + if [[ -n ${DAILY_TO_MONTHLY_CLIM_ARGFILE:-} ]]; then + read_batch_args + set -- "${saved_args[@]}" + fi + parse_args "$@" + check_args + + echo "Create a monthly climatology from a list of years from daily averaged CPLHIST files for DATM" + setup_nco + set_year_months + check_input_files + + # With -b: the input has been checked, so submit to PBS and stop here + if (( batch )); then + submit_batch + exit 0 + fi + + failed="$outdir/mon/.ncra_failed" + rm -f "$failed" + trap 'stop_on_signal INT' INT + trap 'stop_on_signal TERM' TERM + + monthly_means + monthly_climatology + write_output + clean_up + + echo "" + echo "Successfully created monthly climatology: $out" +} -echo "" -echo "Successfully created monthly climatology: $out" +main "$@" From b1ac65d10d1ea37f493abbfcddb68ac98a82ed7f Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Wed, 7 Oct 2026 13:50:17 -0600 Subject: [PATCH 05/37] Add a comment option so that write_part2 can be used in other contexts --- .../ctsm/toolchain/gen_mksurfdata_jobscript_single.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/python/ctsm/toolchain/gen_mksurfdata_jobscript_single.py b/python/ctsm/toolchain/gen_mksurfdata_jobscript_single.py index 1413e49158..7d53607287 100755 --- a/python/ctsm/toolchain/gen_mksurfdata_jobscript_single.py +++ b/python/ctsm/toolchain/gen_mksurfdata_jobscript_single.py @@ -152,9 +152,12 @@ def write_runscript_part1( runfile, descrip="input namelist", name="mksurfdata", + comment=None, ): """ Write run script (part 1) Batch headers + + comment: if given, replaces the default description line written after the batch headers """ runfile.write("#!/bin/bash\n") runfile.write("# Edit the batch directives for your batch system\n") @@ -196,9 +199,11 @@ def write_runscript_part1( runfile.write(f"cd {tool_path}\n") runfile.write("\n") - runfile.write( - f"# This is a batch script to run a set of resolutions for mksurfdata_esmf {descrip}\n" - ) + if comment is None: + comment = ( + f"This is a batch script to run a set of resolutions for mksurfdata_esmf {descrip}" + ) + runfile.write(f"# {comment}\n") runfile.write( "# NOTE: THIS SCRIPT IS AUTOMATICALLY GENERATED " + "SO IN GENERAL YOU SHOULD NOT EDIT it!!\n\n" From 9614c10a1d8c2520e246162ee54a516bfcbe38a9 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Wed, 7 Oct 2026 14:50:41 -0600 Subject: [PATCH 06/37] Remove the old tester status files, add a general match for batch files, and remove the specific one for mksurfdata_esmf --- .gitignore | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index a335b7107e..96349188fc 100644 --- a/.gitignore +++ b/.gitignore @@ -61,11 +61,6 @@ buildnmlc /bld/unit_testers/user_nl_ctsm_real_parameters* /bld/unit_testers/env_run.xml -# tools testing output -td.*.status -td.*.log -td.*.status.xFail - # mksurfdata output surfdata_*.log *.namelist @@ -76,7 +71,6 @@ ctsm.input_data_list ctsm.input_data_list.previous *.stdout.txt.o* /tools/mksurfdata_esmf/PET* -/tools/mksurfdata_esmf/job_name.o* /tools/mksurfdata_esmf/mksurfdata_in /tools/mksurfdata_esmf/surfdata_*.nc /tools/mksurfdata_esmf/landuse.timeseries_*.nc @@ -116,6 +110,9 @@ core.* *.pyc Depends +# Batch output files (both Derecho and Izumi) +*.o[0-9]+ + # Docs build and testing output _build*/ _publish*/ From 1dbb07d67d7b5cba25680971a8f6cf51734c684d Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Wed, 7 Oct 2026 16:04:31 -0600 Subject: [PATCH 07/37] Have Claude convert the bash script to python in our existing python directory structure and checking, I ran the new script for b.e30_alpha09e_m.B1850C_MTso_Gris_Marbl years 100-130 and the answers were identical to the shell script --- python/ctsm/cplhist/__init__.py | 0 python/ctsm/cplhist/daily_to_monthly_clim.py | 608 +++++++++++++++++++ tools/cplhist/daily_to_monthly_clim | 40 ++ 3 files changed, 648 insertions(+) create mode 100644 python/ctsm/cplhist/__init__.py create mode 100644 python/ctsm/cplhist/daily_to_monthly_clim.py create mode 100755 tools/cplhist/daily_to_monthly_clim diff --git a/python/ctsm/cplhist/__init__.py b/python/ctsm/cplhist/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/python/ctsm/cplhist/daily_to_monthly_clim.py b/python/ctsm/cplhist/daily_to_monthly_clim.py new file mode 100644 index 0000000000..0a69b06425 --- /dev/null +++ b/python/ctsm/cplhist/daily_to_monthly_clim.py @@ -0,0 +1,608 @@ +""" +Create a 12-month climatology from daily CMEPS coupler history (CPLHIST) files, for use by DATM. + +The input files are the daily averaged atmosphere coupler history files: + + ARCHDIR/CASE/cpl/hist/CASE.cpl.hx.atm.24h.avrg.YYYY-MM-DD-00000.nc + +for the years BEGYEAR to ENDYEAR. The steps are: + + 1. Average the daily files of each month of each year (ncra) + 2. Average each calendar month over all the years (ncra), and put the 12 months in one + file (ncrcat) + 3. Set the time axis to the middle of each month of YEAR (noleap calendar) + +The output file is: + + OUTDIR/CASE/proc/CASE.cpl.hx.atm.monclim.BEGYEAR-ENDYEAR_cYYYYMMDD.nc + +This needs the NCO commands ncra, ncrcat and ncatted (on Derecho: module load nco). +With --batch, the input is checked and then this tool is submitted to the PBS batch queue. +""" + +import argparse +import concurrent.futures +import datetime +import getpass +import io +import logging +import os +import re +import shlex +import shutil +import subprocess +import sys + +from netCDF4 import Dataset # pylint: disable=no-name-in-module + +from ctsm.args_utils import comma_separated_list +from ctsm.ctsm_logging import ( + setup_logging_pre_config, + add_logging_args, + process_logging_args, + log, +) +from ctsm.git_utils import get_ctsm_git_short_hash +from ctsm.os_utils import run_cmd_output_on_error +from ctsm.path_utils import path_to_ctsm_root +from ctsm.toolchain.gen_mksurfdata_jobscript_single import write_runscript_part1 +from ctsm.utils import abort + +logger = logging.getLogger(__name__) + +# Variables averaged by default +DEFAULT_VARS = [ + "atmImp_Faxa_bcph2", + "atmImp_Faxa_bcph3", + "atmImp_Faxa_ocph1", + "atmImp_Faxa_ocph2", + "atmImp_Faxa_ocph3", + "atmImp_Faxa_dstwet1", + "atmImp_Faxa_dstdry1", + "atmImp_Faxa_dstwet2", + "atmImp_Faxa_dstdry2", + "atmImp_Faxa_dstwet3", + "atmImp_Faxa_dstdry3", + "atmImp_Faxa_dstwet4", + "atmImp_Faxa_dstdry4", +] +# Grid fields that are always written to the output, whatever variable list is used +ALWAYS_VARS = ["atmImp_lon", "atmImp_lat"] +DEFAULT_YEAR = 1850 +# Days in each month of the noleap calendar +NOLEAP_DAYS = (31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31) +# File name tag of the daily averaged atmosphere coupler history files +HIST_TAG = "cpl.hx.atm.24h.avrg" +# NCO commands that are needed +NCO_COMMANDS = ("ncra", "ncrcat", "ncatted") +# Number of simultaneous ncra jobs inside a PBS batch job +PBS_NJOBS = 16 +# PBS batch settings +PBS_JOBNAME = "daily_to_monthly_clim" +DEFAULT_ACCOUNT = "P93300606" +DEFAULT_QUEUE = "main" +DEFAULT_WALLTIME = "04:00:00" +# The executable wrapper for this module, run by the batch job +WRAPPER = os.path.join("tools", "cplhist", "daily_to_monthly_clim") + + +def get_parser(): + """ + Get the parser object for daily_to_monthly_clim + """ + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument( + "archdir", help="Archive directory with the case directories (ARCHDIR/CASE/cpl/hist)" + ) + parser.add_argument("case", help="Case name") + parser.add_argument( + "begyear", type=int, help="First year of the daily files to average (inclusive)" + ) + parser.add_argument( + "endyear", type=int, help="Last year of the daily files to average (inclusive)" + ) + + vars_group = parser.add_mutually_exclusive_group() + vars_group.add_argument( + "--vars", + help=f"""Comma-separated list of variables to average + ({",".join(ALWAYS_VARS)} are always included) + (default: {",".join(DEFAULT_VARS)})""", + action="store", + dest="vars", + type=comma_separated_list, + default=DEFAULT_VARS, + ) + vars_group.add_argument( + "-a", + "--all-vars", + help="Average all variables in the files", + action="store_true", + dest="all_vars", + ) + parser.add_argument( + "-o", + "--outdir", + help="""Output directory if different from ARCHDIR. Files go under OUTDIR/CASE/proc + (default: ARCHDIR)""", + action="store", + dest="outdir", + default=None, + ) + parser.add_argument( + "-y", + "--year", + help="Year to set the climatology time axis to (default: %(default)s)", + action="store", + dest="year", + type=int, + default=DEFAULT_YEAR, + ) + parser.add_argument( + "-k", + "--keep", + help="Keep the intermediate files in the OUTDIR/CASE/proc/mon directory", + action="store_true", + dest="keep", + ) + parser.add_argument( + "-j", + "--njobs", + help=f"""Number of ncra jobs to run at the same time + (default: {PBS_NJOBS} in a PBS batch job, 1 otherwise)""", + action="store", + dest="njobs", + type=int, + default=None, + ) + parser.add_argument( + "-b", + "--batch", + help="Check the input, then submit this tool to the PBS batch queue with the same options", + action="store_true", + dest="batch", + ) + parser.add_argument( + "-A", + "--account", + help=f"""PBS project account to charge with --batch + (default: $ACCOUNT if set, otherwise {DEFAULT_ACCOUNT})""", + action="store", + dest="account", + default=None, + ) + parser.add_argument( + "--queue", + help="PBS queue to use with --batch (default: %(default)s)", + action="store", + dest="queue", + default=DEFAULT_QUEUE, + ) + parser.add_argument( + "--walltime", + help="PBS wall clock time limit to use with --batch (default: %(default)s)", + action="store", + dest="walltime", + default=DEFAULT_WALLTIME, + ) + add_logging_args(parser) + return parser + + +def default_njobs(): + """ + Number of simultaneous ncra jobs when not given: more inside a PBS batch job + """ + if os.environ.get("PBS_JOBID"): + return PBS_NJOBS + return 1 + + +def input_dir(archdir, case): + """ + Directory with the daily coupler history files + """ + return os.path.join(archdir, case, "cpl", "hist") + + +def output_dir(archdir, outdir, case): + """ + Directory for the output (OUTDIR/CASE/proc, where OUTDIR defaults to ARCHDIR) + """ + if outdir is None: + outdir = archdir + return os.path.join(outdir, case, "proc") + + +def check_args(args): + """ + Check the command line arguments, aborting if there is a problem + """ + for name, year in (("BEGYEAR", args.begyear), ("ENDYEAR", args.endyear), ("YEAR", args.year)): + if not 0 <= year <= 9999: + abort(f"{name} must be an integer from 0 to 9999: {year}") + if args.begyear > args.endyear: + abort(f"BEGYEAR ({args.begyear}) must not be after ENDYEAR ({args.endyear})") + if args.njobs is not None and args.njobs < 1: + abort(f"NJOBS must be a positive integer: {args.njobs}") + if args.account is not None and not args.batch: + abort("--account is only used with --batch") + if not args.all_vars and not build_var_list(args.vars, []): + abort("Variable list given with --vars is empty") + if not os.path.isdir(args.archdir): + abort(f"Input {args.archdir} directory does not exist") + if args.outdir is not None and not os.path.isdir(args.outdir): + abort(f"Output {args.outdir} directory does not exist") + indir = input_dir(args.archdir, args.case) + if not os.path.isdir(indir): + abort(f"{indir} directory does not exist") + + +def build_var_list(variables, always_vars): + """ + Combine the variables to average with the ones that are always output + + Keeps the order given, drops duplicates and empty names. Returns None when variables is None, + meaning all variables. + """ + if variables is None: + return None + var_list = [] + for var in list(variables) + list(always_vars): + if var and var not in var_list: + var_list.append(var) + return var_list + + +def year_months(begyr, endyr): + """ + List of (year, month) to process, from BEGYEAR-01 to ENDYEAR-12 + """ + return [(year, month) for year in range(begyr, endyr + 1) for month in range(1, 13)] + + +def index_daily_files(indir, case): + """ + Find the daily coupler history files for this case, matching them by name (not by the time + coordinate in the files) + + Returns a dictionary of sorted file path lists, with (year, month) keys + """ + pattern = re.compile( + rf"^{re.escape(case)}\.{re.escape(HIST_TAG)}\.(\d{{4}})-(\d{{2}})-(\d{{2}})-00000\.nc$" + ) + files_by_month = {} + for filename in sorted(os.listdir(indir)): + match = pattern.match(filename) + if match: + key = (int(match.group(1)), int(match.group(2))) + files_by_month.setdefault(key, []).append(os.path.join(indir, filename)) + return files_by_month + + +def check_day_counts(files_by_month, yms): + """ + Check that every month has a full set of daily files (noleap calendar) + + Returns a list of error messages, empty if all the months are complete + """ + errors = [] + for year, month in yms: + nfiles = len(files_by_month.get((year, month), [])) + expect = NOLEAP_DAYS[month - 1] + if nfiles != expect: + errors.append(f"{year:04d}-{month:02d} has {nfiles} daily files (expected {expect})") + return errors + + +def missing_vars(filename, var_list): + """ + Return the variables in var_list that are not in the netCDF file + """ + with Dataset(filename) as ncfile: + file_vars = set(ncfile.variables) + return [var for var in var_list if var not in file_vars] + + +def check_nco(): + """ + Abort if any of the needed NCO commands can't be found + """ + missing = [cmd for cmd in NCO_COMMANDS if shutil.which(cmd) is None] + if missing: + abort(f"NCO commands not found: {' '.join(missing)} (on Derecho: module load nco)") + + +def check_input(args, var_list): + """ + Check the input files before doing any work: every requested variable is in the first file, + and every month in the range has a full set of daily files + + Returns the dictionary of daily files from index_daily_files + """ + indir = input_dir(args.archdir, args.case) + yms = year_months(args.begyear, args.endyear) + log( + logger, + f"Processing {len(yms)} year-months: {args.begyear:04d}-01 .. {args.endyear:04d}-12", + ) + files_by_month = index_daily_files(indir, args.case) + + if var_list is not None: + first_files = files_by_month.get(yms[0], []) + if not first_files: + abort(f"No files for {yms[0][0]:04d}-{yms[0][1]:02d} in {indir}") + log(logger, f"Checking the requested variables are in {first_files[0]}") + missing = missing_vars(first_files[0], var_list) + if missing: + abort(f"Variables not found in {first_files[0]}: {' '.join(missing)}") + + log(logger, "Checking that every month has the expected number of daily files") + errors = check_day_counts(files_by_month, yms) + if errors: + abort("\n".join(errors)) + return files_by_month + + +def ncra_command(infiles, outfile, var_list=None): + """ + The ncra command (as a list) to average the input files into outfile + """ + cmd = ["ncra", "-O"] + if var_list is not None: + cmd += ["-v", ",".join(var_list)] + return cmd + list(infiles) + ["-o", outfile] + + +def run_jobs(commands, njobs, *, progress_every=None, runner=run_cmd_output_on_error): + """ + Run the commands (each a list, with the output file last), with at most njobs at once + + Stops at the first failure: the jobs that have not started are cancelled and the runner's + error (normally an abort) is raised. Logs progress every progress_every completed jobs. + """ + ndone = 0 + with concurrent.futures.ThreadPoolExecutor(max_workers=njobs) as executor: + futures = [executor.submit(runner, cmd, f"Failed creating {cmd[-1]}") for cmd in commands] + try: + for future in concurrent.futures.as_completed(futures): + future.result() + ndone += 1 + if progress_every and ndone % progress_every == 0: + log(logger, f" completed {ndone} of {len(commands)}") + except BaseException: + # Includes KeyboardInterrupt and the SystemExit from abort + for future in futures: + future.cancel() + raise + + +def mon_file(mondir, year, month): + """ + Intermediate file with the monthly mean for one year and month + """ + return os.path.join(mondir, f"mon_{year:04d}-{month:02d}.nc") + + +def clim_file(mondir, month): + """ + Intermediate file with the climatology for one calendar month + """ + return os.path.join(mondir, f"clim_{month:02d}.nc") + + +def monthly_means(files_by_month, yms, mondir, var_list, njobs): + """ + Step 1: average the daily files of each month of each year + """ + log(logger, f"Computing monthly means for each year ({njobs} ncra jobs at a time)") + commands = [ + ncra_command(files_by_month[(year, month)], mon_file(mondir, year, month), var_list) + for year, month in yms + ] + # Progress every 10 years + run_jobs(commands, njobs, progress_every=120) + + +def monthly_climatology(mondir, begyr, endyr, njobs): + """ + Step 2: average each calendar month over the years + + Only this run's years are used, so leftover files from an earlier --keep run are not included + """ + log(logger, f"Averaging each calendar month over the years {begyr}-{endyr}") + commands = [ + ncra_command( + [mon_file(mondir, year, month) for year in range(begyr, endyr + 1)], + clim_file(mondir, month), + ) + for month in range(1, 13) + ] + run_jobs(commands, njobs) + + +def output_filename(outdir, case, begyr, endyr, date): + """ + Name of the output climatology file, with the year range and creation date + """ + return os.path.join( + outdir, f"{case}.cpl.hx.atm.monclim.{begyr:04d}-{endyr:04d}_c{date:%Y%m%d}.nc" + ) + + +def month_bounds(): + """ + Start and end of each month, in days since the start of a noleap year + """ + bounds = [] + start = 0 + for ndays in NOLEAP_DAYS: + bounds.append((start, start + ndays)) + start += ndays + return bounds + + +def mid_month_times(): + """ + Middle of each month, in days since the start of a noleap year + """ + return [(start + end) / 2.0 for start, end in month_bounds()] + + +def set_time_axis(filename, year): + """ + Set the time (and time_bnds, if present) of the 12-month file to the middle (and bounds) of + each month of year, in the noleap calendar + """ + units = f"days since {year:04d}-01-01 00:00:00" + with Dataset(filename, "a") as ncfile: + time = ncfile.variables["time"] + time[:] = mid_month_times() + time.setncattr("units", units) + time.setncattr("calendar", "noleap") + if "time_bnds" in ncfile.variables: + time_bnds = ncfile.variables["time_bnds"] + time_bnds[:] = month_bounds() + time_bnds.setncattr("units", units) + + +def write_provenance(filename, date): + """ + Add global attributes saying when, by whom and with what the file was created (like + update_metadata in ctsm/site_and_regional/base_case.py, but using ncatted) + """ + created_with = f"./{os.path.basename(WRAPPER)} -- {get_ctsm_git_short_hash()}" + cmd = ["ncatted", "-O"] + for name, value in ( + ("Created_on", f"{date:%Y-%m-%d}"), + ("Created_by", getpass.getuser()), + ("Created_with", created_with), + ): + cmd += ["-a", f"{name},global,o,c,{value}"] + run_cmd_output_on_error(cmd + [filename], f"Failed adding provenance to {filename}") + + +def write_output(mondir, out, year, date): + """ + Step 3: put the 12 monthly climatologies in one file, reset its time axis to the middle of + each month of year, and add provenance attributes + """ + log(logger, "Putting the climatology into one file and finalizing its metadata") + clim_files = [clim_file(mondir, month) for month in range(1, 13)] + run_cmd_output_on_error(["ncrcat", "-O"] + clim_files + [out], f"Failed creating {out}") + set_time_axis(out, year) + write_provenance(out, date) + log(logger, f"Wrote {out} (time = mid-month of year {year:04d}, noleap)") + + +def job_args(args): + """ + The command line arguments for the batch job: the same as given, without the batch options + """ + argv = [args.archdir, args.case, str(args.begyear), str(args.endyear)] + if args.all_vars: + argv.append("--all-vars") + elif args.vars != DEFAULT_VARS: + argv += ["--vars", ",".join(args.vars)] + if args.outdir is not None: + argv += ["--outdir", args.outdir] + if args.year != DEFAULT_YEAR: + argv += ["--year", str(args.year)] + if args.keep: + argv.append("--keep") + if args.njobs is not None: + argv += ["--njobs", str(args.njobs)] + for option in ("verbose", "silent", "debug"): + if getattr(args, option): + argv.append(f"--{option}") + return argv + + +def batch_script(argv, *, account, walltime, workdir): + """ + Text of the PBS batch job script that runs this tool with the arguments argv + """ + runfile = io.StringIO() + write_runscript_part1( + number_of_nodes=1, + tasks_per_node=1, + machine="derecho", + account=account, + walltime=walltime, + runfile=runfile, + name=PBS_JOBNAME, + comment="This is a batch script to create a monthly climatology from daily CPLHIST files", + ) + runfile.write("module load nco\n") + runfile.write(f"cd {shlex.quote(workdir)}\n") + wrapper = os.path.join(path_to_ctsm_root(), WRAPPER) + runfile.write(shlex.join([sys.executable, wrapper] + argv) + "\n") + return runfile.getvalue() + + +def submit_batch(script, queue): + """ + Submit the batch job script to PBS (qsub reads it from stdin), and return the job id + """ + if shutil.which("qsub") is None: + abort("qsub not found, can not submit with --batch") + result = subprocess.run( + ["qsub", "-q", queue], input=script, capture_output=True, text=True, check=False + ) + if result.returncode != 0: + abort(f"qsub failed:\n{result.stdout}{result.stderr}") + return result.stdout.strip() + + +def main(): + """ + Main program for daily_to_monthly_clim + """ + setup_logging_pre_config() + args = get_parser().parse_args() + process_logging_args(args) + check_args(args) + + njobs = args.njobs if args.njobs is not None else default_njobs() + var_list = None if args.all_vars else build_var_list(args.vars, ALWAYS_VARS) + if var_list is None: + log(logger, "Averaging all variables") + else: + log(logger, f"Averaging variables: {','.join(var_list)}") + log(logger, "Create a monthly climatology from daily averaged CPLHIST files for DATM") + + check_nco() + files_by_month = check_input(args, var_list) + + # With --batch: the input has been checked, so submit to PBS and stop here + if args.batch: + account = args.account or os.environ.get("ACCOUNT") or DEFAULT_ACCOUNT + argv = job_args(args) + script = batch_script(argv, account=account, walltime=args.walltime, workdir=os.getcwd()) + logger.debug("Batch job script:\n%s", script) + jobid = submit_batch(script, args.queue) + log( + logger, + f"Successfully submitted job {jobid} with arguments: {' '.join(argv)}" + f" (log file will be {PBS_JOBNAME}.o{jobid.split('.')[0]} in {os.getcwd()})", + ) + return + + outdir = output_dir(args.archdir, args.outdir, args.case) + mondir = os.path.join(outdir, "mon") + os.makedirs(mondir, exist_ok=True) + date = datetime.date.today() + + monthly_means(files_by_month, year_months(args.begyear, args.endyear), mondir, var_list, njobs) + monthly_climatology(mondir, args.begyear, args.endyear, njobs) + out = output_filename(outdir, args.case, args.begyear, args.endyear, date) + write_output(mondir, out, args.year, date) + + if args.keep: + log(logger, f"Kept intermediate files in {mondir}") + else: + shutil.rmtree(mondir) + + log(logger, f"Successfully created monthly climatology: {out}") diff --git a/tools/cplhist/daily_to_monthly_clim b/tools/cplhist/daily_to_monthly_clim new file mode 100755 index 0000000000..c975f3d0a8 --- /dev/null +++ b/tools/cplhist/daily_to_monthly_clim @@ -0,0 +1,40 @@ +#!/usr/bin/env python3 +""" +This is a just top-level skeleton script that calls +daily_to_monthly_clim.py. +The original code (daily_to_monthly_clim.py) is located under the +python/ctsm/cplhist folder. + +It creates a 12-month climatology from daily averaged CMEPS coupler +history (CPLHIST) files, for use by DATM. + +For full instructions on how to run the code and different options, +please check the python/ctsm/cplhist/daily_to_monthly_clim.py file. + +To run this script the following packages are required: + - netCDF4 + - the NCO commands ncra, ncrcat and ncatted +---------------------------------------------------------------- +To see all available options for daily_to_monthly_clim + ./daily_to_monthly_clim --help +---------------------------------------------------------------- +Instructions for running using the conda python environments +and NCO on Derecho: +module load conda nco +../../py_env_create +conda activate ctsm_pylib +""" + +import os +import sys + +# -- add python/ctsm to path +_CTSM_PYTHON = os.path.join( + os.path.dirname(os.path.realpath(__file__)), os.pardir, os.pardir, "python" +) +sys.path.insert(1, _CTSM_PYTHON) + +from ctsm.cplhist.daily_to_monthly_clim import main + +if __name__ == "__main__": + main() From 366ae4e2d678fa0ab2bc29ce9166e509e9bd4227 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Wed, 7 Oct 2026 16:05:07 -0600 Subject: [PATCH 08/37] Remove the bash script now that I trust the new python script --- tools/unsupported/daily_to_monthly_clim.sh | 422 --------------------- 1 file changed, 422 deletions(-) delete mode 100755 tools/unsupported/daily_to_monthly_clim.sh diff --git a/tools/unsupported/daily_to_monthly_clim.sh b/tools/unsupported/daily_to_monthly_clim.sh deleted file mode 100755 index 3f05ebec54..0000000000 --- a/tools/unsupported/daily_to_monthly_clim.sh +++ /dev/null @@ -1,422 +0,0 @@ -#!/usr/bin/env bash -#PBS -N daily_to_monthly_clim -#PBS -r n -#PBS -j oe -#PBS -k eod -#PBS -S /bin/bash -#PBS -l select=1:ncpus=128:mpiprocs=128:ompthreads=1:mem=235GB -#PBS -l walltime=04:00:00 -#PBS -A P93300606 -#PBS -q main -# -# ============================================================================================== -# -# Build a 12-month climatology from daily CESM coupler history files: -# $ARCHDIR/$CASE/cpl/hist/$CASE.cpl.hx.atm.24h.avrg.YYYY-MM-DD-00000.nc -# for the years BEGYEAR to ENDYEAR -# Step 1: monthly mean for each year (ncra over the daily files of that month), -# for the default variable list, a list given with -v, or all variables with -a -# Step 2: average each calendar month over all years (ncra), then ncrcat to 12 records -# Step 3: reset the time axis to mid-month of YEAR (noleap calendar) -# -# The work is done in the functions below, called in order from main at the bottom of the file. -# -# ============================================================================================== - - -# NOTE: This will fail on errors, or if variables are used but not set, and pipefail fails on first exit in a pipe of commands -set -euo pipefail - -default_vars="atmImp_Faxa_bcph2,atmImp_Faxa_bcph3,atmImp_Faxa_ocph1,atmImp_Faxa_ocph2,atmImp_Faxa_ocph3,atmImp_Faxa_dstwet1,atmImp_Faxa_dstdry1,atmImp_Faxa_dstwet2,atmImp_Faxa_dstdry2,atmImp_Faxa_dstwet3,atmImp_Faxa_dstdry3,atmImp_Faxa_dstwet4,atmImp_Faxa_dstdry4" -# Grid fields that are always written to the output, whatever variable list is used -always_vars="atmImp_lon,atmImp_lat" -default_year="1850" -default_outdir="ARCHDIR" -# Number of simultaneous ncra jobs: 16 inside a PBS batch job, 1 otherwise -if [[ -n ${PBS_JOBID:-} ]]; then - default_njobs=16 -else - default_njobs=1 -fi -# PBS job name (same as the #PBS -N line above), used to name the job log file -pbs_jobname="daily_to_monthly_clim" -noleap_days=(31 28 31 30 31 30 31 31 30 31 30 31) - -# ============================================================================================== -# Command line handling -# ============================================================================================== - -# Print the help: to stdout and exit 0 for -h, to stderr and exit 1 for bad arguments -usage() { - local status=${1:-1} - if (( status == 0 )); then exec 3>&1; else exec 3>&2; fi - cat >&3 < 0 && ! batch )); then - echo "-A is only used with -b" >&2 - exit 1 - fi - for y in "$begyr" "$endyr"; do - if ! [[ $y =~ ^[0-9]{1,4}$ ]]; then - echo "BEGYEAR and ENDYEAR must be integers from 0 to 9999: $y" >&2 - exit 1 - fi - done - begyr=$((10#$begyr)); endyr=$((10#$endyr)) - if (( begyr > endyr )); then - echo "BEGYEAR ($begyr) must not be after ENDYEAR ($endyr)" >&2 - exit 1 - fi - if ! [[ $njobs =~ ^[1-9][0-9]*$ ]]; then - echo "NJOBS must be a positive integer: $njobs" >&2 - exit 1 - fi - if (( allvars && userlist )); then - echo "Use either -v or -a, not both" >&2 - exit 1 - fi - # Variable subset passed to ncra (coordinates and time_bnds are included automatically) - if (( allvars )); then - vopt=() - echo "Averaging all variables" - else - [[ -n $varlist ]] || { echo "Variable list given with -v is empty" >&2; exit 1; } - # Add the grid fields that are always output, dropping any duplicates - IFS=',' read -ra vl <<< "$varlist,$always_vars" - varlist=$(printf '%s\n' "${vl[@]}" | awk 'NF && !seen[$0]++' | paste -sd, -) - vopt=(-v "$varlist") - echo "Averaging variables: $varlist" - fi - if ! [[ $year =~ ^[0-9]{1,4}$ ]]; then - echo "YEAR must be an integer from 0 to 9999: $year" >&2 - exit 1 - fi - year=$(printf '%04d' "$((10#$year))") - if [ ! -d "$archdir" ]; then - echo "Input $archdir directory does not exist" >&2 - exit 1 - fi - indir="$archdir/$case/cpl/hist" - if [ "$outdir" = "$default_outdir" ]; then - outdir="$archdir/$case/proc" - elif [ ! -d "$outdir" ]; then - echo "Output $outdir directory does not exist" >&2 - exit 1 - else - outdir="$outdir/$case/proc" - fi - if [ ! -d "$indir" ]; then - echo "$indir directory does not exist" >&2 - exit 1 - fi - mkdir -p "$outdir/mon" || { echo "Can not create $outdir/mon" >&2; exit 1; } - prefix="$case.cpl.hx.atm.24h.avrg" -} - -# ============================================================================================== -# Environment and input checks -# ============================================================================================== - -# Load NCO and make sure the NCO commands are available -setup_nco() { - local cmd - if command -v module >/dev/null 2>&1; then - # Lmod's module function can reference unset variables, so relax set -u around it - set +u - module load nco - set -u - else - echo "module command not found, so expecting NCO to already be in the PATH" >&2 - fi - for cmd in ncra ncrcat ncap2 ncatted ncks; do - command -v $cmd >/dev/null || { echo "$cmd not found (module load nco)" >&2; exit 1; } - done -} - -# Year-months to process, BEGYEAR-01 to ENDYEAR-12 (files are matched by name, not the time coordinate) -# Sets: yms -set_year_months() { - local y mm - yms=() - for (( y = begyr; y <= endyr; y++ )); do - for mm in 01 02 03 04 05 06 07 08 09 10 11 12; do - yms+=("$(printf '%04d' "$y")-$mm") - done - done - echo "Processing ${#yms[@]} year-months: ${yms[0]} .. ${yms[-1]}" -} - -# Print the year at January of BEGYEAR, BEGYEAR+10, BEGYEAR+20, ... (every 120 year-months) -progress() { - local y=$((10#${1%%-*})) - if [[ ${1#*-} == 01 ]] && (( (y - begyr) % 10 == 0 )); then - echo " at year $y" - fi -} - -# Check the input files before doing any work: every requested variable is in the first file, -# and every month in the range has a full set of daily files -check_input_files() { - local first v ym mm n expect bad=0 - local -a vars missing=() - if (( ! allvars )); then - echo "Checking that all requested variables are present in the first file for ${yms[0]}" - first=$(find "$indir" -maxdepth 1 -name "${prefix}.${yms[0]}-??-00000.nc" | sort | sed -n 1p) - [[ -n $first ]] || { echo "ERROR: no files for ${yms[0]} matching ${prefix}.${yms[0]}-??-00000.nc in $indir" >&2; exit 1; } - IFS=',' read -ra vars <<< "$varlist" - for v in "${vars[@]}"; do - ncks -m -v "$v" "$first" >/dev/null 2>&1 || missing+=("$v") - done - if (( ${#missing[@]} > 0 )); then - echo "ERROR: variables not found in $first: ${missing[*]}" >&2 - exit 1 - fi - fi - - echo "Checking that every month has the expected number of daily files" - for ym in "${yms[@]}"; do - mm=${ym#*-} - progress "$ym" - n=$(find "$indir" -maxdepth 1 -name "${prefix}.${ym}-??-00000.nc" | wc -l) - expect=${noleap_days[$((10#$mm - 1))]} - if (( n != expect )); then - echo "ERROR: $ym has $n daily files (expected $expect)" >&2 - bad=1 - fi - done - (( ! bad )) || exit 1 -} - -# ============================================================================================== -# PBS batch submission -# ============================================================================================== - -# Inside a job submitted with -b: go back to the submit directory and read the original -# arguments saved by the submitting script (qsub can't pass arguments to a job script, -# so they go through a file named in the environment) -# Sets: saved_args -read_batch_args() { - cd "${DAILY_TO_MONTHLY_CLIM_WORKDIR:-.}" - eval "saved_args=($(cat "$DAILY_TO_MONTHLY_CLIM_ARGFILE"))" - rm -f "$DAILY_TO_MONTHLY_CLIM_ARGFILE" -} - -# With -b: submit this script to PBS with the same arguments (minus -b and -A). -# The arguments are saved (shell-quoted, so commas and spaces survive) to a file that the job reads back. -submit_batch() { - local argfile jobid - command -v qsub >/dev/null || { echo "qsub not found, can not submit with -b" >&2; exit 1; } - argfile="$outdir/$pbs_jobname.args.$$" - printf '%q ' "${job_args[@]}" > "$argfile" - echo "Submitting to PBS with arguments: ${job_args[*]}" - if ! jobid=$(qsub "${qsub_opts[@]}" -v "DAILY_TO_MONTHLY_CLIM_ARGFILE=$argfile,DAILY_TO_MONTHLY_CLIM_WORKDIR=$PWD" "$0"); then - rm -f "$argfile" - echo "ERROR: qsub failed" >&2 - exit 1 - fi - echo "Successfully submitted job $jobid (log file will be $pbs_jobname.o${jobid%%.*} in $PWD)" -} - -# ============================================================================================== -# Running ncra jobs in the background -# ============================================================================================== - -# Run ncra in the background, with at most $njobs running at once -# Usage: bg_ncra OUTFILE INFILE... (uses the ncra_opts array for extra ncra options) -# The input list goes to ncra on stdin, so a long list never hits the shell argument limit. -# A failed job leaves the $failed marker file, which wait_ncra checks. -bg_ncra() { - local out=$1; shift - while (( $(jobs -rp | wc -l) >= njobs )); do wait -n || true; done - ( printf '%s\n' "$@" | ncra -O "${ncra_opts[@]}" -o "$out" \ - || { echo "ERROR: ncra failed creating $out" >&2; touch "$failed"; } ) & -} -# Wait for all background ncra jobs, and stop if any of them failed -wait_ncra() { - wait - if [[ -e $failed ]]; then - echo "ERROR: one or more ncra jobs failed, stopping" >&2 - exit 1 - fi -} - -# On Ctrl+C (INT), or when PBS kills the job with qdel or at the walltime limit (TERM), -# also stop the background ncra jobs. Background jobs in a script ignore Ctrl+C, -# so without this they would keep running after the script exits. -stop_on_signal() { - local sig=$1 pid - trap '' INT TERM - echo "Caught SIG$sig, stopping background ncra jobs" >&2 - for pid in $(jobs -p); do - pkill -TERM -P "$pid" 2>/dev/null || true # the ncra started by this background job - kill -TERM "$pid" 2>/dev/null || true - done - wait 2>/dev/null || true - if [[ $sig == INT ]]; then exit 130; else exit 143; fi -} - -# ============================================================================================== -# The climatology steps -# ============================================================================================== - -# ---- Step 1: monthly means for each year ---- -monthly_means() { - local ym - local -a files - echo "Computing monthly means for each year ($njobs ncra jobs at a time)" - ncra_opts=("${vopt[@]}") - for ym in "${yms[@]}"; do - progress "$ym" - mapfile -t files < <(find "$indir" -maxdepth 1 -name "${prefix}.${ym}-??-00000.nc" | sort) - bg_ncra "$outdir/mon/mon_${ym}.nc" "${files[@]}" - done - wait_ncra -} - -# ---- Step 2: average each calendar month over the years ---- -monthly_climatology() { - local mm y - local -a mons - echo "Averaging each calendar month over the years" - ncra_opts=() - for mm in 01 02 03 04 05 06 07 08 09 10 11 12; do - # Only handle this run's years, so leftover files from an earlier -k run are not included - mons=() - for (( y = begyr; y <= endyr; y++ )); do - mons+=("$outdir/mon/mon_$(printf '%04d' "$y")-${mm}.nc") - done - echo "Month $mm: averaging ${#mons[@]} years ($begyr-$endyr)" - bg_ncra "$outdir/mon/clim_${mm}.nc" "${mons[@]}" - done - wait_ncra -} - -# ---- Step 3: put the 12 months into one file and reset time to mid-month of $year (noleap) ---- -# Sets: out -write_output() { - echo "Put the climatology into one file and finalize metadata on the file" - out="$outdir/${case}.cpl.hx.atm.monclim.$(printf '%04d-%04d' "$begyr" "$endyr")_c$(date +%Y%m%d).nc" - ncrcat -O "$outdir"/mon/clim_{01..12}.nc "$out" - - # Time is days since $year-01-01 - ncap2 -O -s 'time[$time]={15.5,45.0,74.5,105.0,135.5,166.0,196.5,227.5,258.0,288.5,319.0,349.5};' "$out" "$out" - ncatted -O -a units,time,o,c,"days since ${year}-01-01 00:00:00" \ - -a calendar,time,o,c,"noleap" "$out" - # If the file has time bounds, set them to the start and end of each month - if ncks --trd -m -v time_bnds "$out" >/dev/null 2>&1; then - ncap2 -O -s '*lb[$time]={0.,31.,59.,90.,120.,151.,181.,212.,243.,273.,304.,334.}; - *ub[$time]={31.,59.,90.,120.,151.,181.,212.,243.,273.,304.,334.,365.}; - time_bnds(:,0)=lb; time_bnds(:,1)=ub;' "$out" "$out" - ncatted -O -a units,time_bnds,o,c,"days since ${year}-01-01 00:00:00" "$out" - fi - echo "Wrote $out (time = mid-month of year $year, noleap)" -} - -# Remove the temporary monthly files unless -k was given -clean_up() { - if (( keep )); then - echo "Kept intermediate files in $outdir/mon" - else - rm -r "$outdir/mon" - fi -} - -# ============================================================================================== -# Main program -# ============================================================================================== - -main() { - # Inside a job submitted with -b, the arguments come from the file saved at submission - if [[ -n ${DAILY_TO_MONTHLY_CLIM_ARGFILE:-} ]]; then - read_batch_args - set -- "${saved_args[@]}" - fi - parse_args "$@" - check_args - - echo "Create a monthly climatology from a list of years from daily averaged CPLHIST files for DATM" - setup_nco - set_year_months - check_input_files - - # With -b: the input has been checked, so submit to PBS and stop here - if (( batch )); then - submit_batch - exit 0 - fi - - failed="$outdir/mon/.ncra_failed" - rm -f "$failed" - trap 'stop_on_signal INT' INT - trap 'stop_on_signal TERM' TERM - - monthly_means - monthly_climatology - write_output - clean_up - - echo "" - echo "Successfully created monthly climatology: $out" -} - -main "$@" From 29efae6dc2f14e69c3404c8ced96c9ae8b1e7c2d Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Sat, 10 Oct 2026 12:49:51 -0600 Subject: [PATCH 09/37] Add more comments and metadata, also abort if the outfile already exists, and if monthly files already exist use them rather than overwritting --- python/ctsm/cplhist/daily_to_monthly_clim.py | 107 +++++++++++++++---- 1 file changed, 84 insertions(+), 23 deletions(-) diff --git a/python/ctsm/cplhist/daily_to_monthly_clim.py b/python/ctsm/cplhist/daily_to_monthly_clim.py index 0a69b06425..53e3d1fec5 100644 --- a/python/ctsm/cplhist/daily_to_monthly_clim.py +++ b/python/ctsm/cplhist/daily_to_monthly_clim.py @@ -18,10 +18,25 @@ This needs the NCO commands ncra, ncrcat and ncatted (on Derecho: module load nco). With --batch, the input is checked and then this tool is submitted to the PBS batch queue. + +A previous version of this for CESM2 is in NCL and here on Derecho by Keith Oleson: + /glade/u/home/oleson/misc_programs/CLM5Dev/CMIP6/CreateAeroDepFile_Task5B.ncl +It did 5 year averages over the historical period and then did a linear interpolation +between the 5-year averages. It also used CPLHIST output from several ensemble members and +averged them together with ncea. This is something that will have to be done in the future +in this tool as well. + +Another example for Nitrogen deposition is here from Simone Tilmes and Mike Mills: + https://svn.code.sf.net/p/codescripts/code/trunk/ncl/cam_forcing/CreateDepositionFile.ncl + +Neither were quite right for what's needed here. And both are in NCL which is well after end of life now. + +Also there is code in LDF to create monthly climatologies, but it's not setup to bring in ensemble +averages and not specialized to create datafiles for DATM aerosol forcing. """ import argparse -import concurrent.futures +import concurrent.futures # Handles parallel multi-processing by sending ncra jobs on different threads import datetime import getpass import io @@ -238,6 +253,14 @@ def check_args(args): indir = input_dir(args.archdir, args.case) if not os.path.isdir(indir): abort(f"{indir} directory does not exist") + # + # Abort if the output file already exists + # + todaysdate = datetime.date.today() + outdir = output_dir(args.archdir, args.outdir, args.case) + outfile = output_filename(outdir, args.case, args.begyear, args.endyear, todaysdate) + if path.exists(outfile): + abort(f"Output file {outfile} already exists, remove it or use a different output directory") def build_var_list(variables, always_vars): @@ -349,8 +372,13 @@ def check_input(args, var_list): def ncra_command(infiles, outfile, var_list=None): """ The ncra command (as a list) to average the input files into outfile + If file already exists, returns an empty list and logs a message instead of running ncra """ - cmd = ["ncra", "-O"] + if path.exists(outfile): + log(logger, f"File already exists:{outfile} so skipping ncra command") + return [] + # Use --hst option so that history attribute isn't appended to as it's too long + cmd = ["ncra", "--hst"] if var_list is not None: cmd += ["-v", ",".join(var_list)] return cmd + list(infiles) + ["-o", outfile] @@ -374,6 +402,7 @@ def run_jobs(commands, njobs, *, progress_every=None, runner=run_cmd_output_on_e log(logger, f" completed {ndone} of {len(commands)}") except BaseException: # Includes KeyboardInterrupt and the SystemExit from abort + # Will die if the user hits Ctrl-C or if the batch job is killed for future in futures: future.cancel() raise @@ -402,7 +431,7 @@ def monthly_means(files_by_month, yms, mondir, var_list, njobs): ncra_command(files_by_month[(year, month)], mon_file(mondir, year, month), var_list) for year, month in yms ] - # Progress every 10 years + # Show progress every 10 years run_jobs(commands, njobs, progress_every=120) @@ -423,12 +452,12 @@ def monthly_climatology(mondir, begyr, endyr, njobs): run_jobs(commands, njobs) -def output_filename(outdir, case, begyr, endyr, date): +def output_filename(outdir, case, begyr, endyr, todaysdate): """ Name of the output climatology file, with the year range and creation date """ return os.path.join( - outdir, f"{case}.cpl.hx.atm.monclim.{begyr:04d}-{endyr:04d}_c{date:%Y%m%d}.nc" + outdir, f"{case}.cpl.hx.atm.monclim.{begyr:04d}-{endyr:04d}_c{todaysdate:%Y%m%d}.nc" ) @@ -453,48 +482,79 @@ def mid_month_times(): def set_time_axis(filename, year): """ - Set the time (and time_bnds, if present) of the 12-month file to the middle (and bounds) of + Set the time and time_bnds of the 12-month file to the middle (and bounds) of each month of year, in the noleap calendar """ units = f"days since {year:04d}-01-01 00:00:00" with Dataset(filename, "a") as ncfile: time = ncfile.variables["time"] time[:] = mid_month_times() + time.setncattr("climatology_bounds", "climatology_bounds") time.setncattr("units", units) time.setncattr("calendar", "noleap") - if "time_bnds" in ncfile.variables: - time_bnds = ncfile.variables["time_bnds"] - time_bnds[:] = month_bounds() - time_bnds.setncattr("units", units) - + # + # Change time_bounds to climatology_bounds or create it if it doesn't exist + # + if "time_bounds" in ncfile.variables: + ncfile.renameVariable("time_bounds", "climatology_bounds") + if "climatology_bounds" not in ncfile.variables: + if "ntb" not in ncfile.dimensions: + ncfile.createDimension("ntb", 2) + ncfile.createVariable("climatology_bounds", "f8", ("time", "ntb")) + climatology_bounds = ncfile.variables["climatology_bounds"] + climatology_bounds[:] = month_bounds() + climatology_bounds.setncattr("units", units) + climatology_bounds.setncattr("calendar", "noleap") + +def add_var_attributes(filename): + """ + Add variable attributes to the file + """ + with Dataset(filename, "a") as ncfile: + for var in ncfile.variables: + if var != "time" and var != "climatology_bounds": + ncfile.variables[var].setncattr("cell_methods", "time: mean") + ncfile.variables[var].setncattr("coordinates", "time atmImp_lon atmImp_lat") -def write_provenance(filename, date): +def write_provenance(case, filename, begyear, endyear, year, todaysdate): """ Add global attributes saying when, by whom and with what the file was created (like update_metadata in ctsm/site_and_regional/base_case.py, but using ncatted) """ created_with = f"./{os.path.basename(WRAPPER)} -- {get_ctsm_git_short_hash()}" + # Use Overwrite option since the file already exists and we want to add attributes to it cmd = ["ncatted", "-O"] + comment = "Monthly climatology created from daily averaged CPLHIST files for DATM" + title = "Monthly Climatology of Daily Averaged Atmosphere Coupler History Files for cyclical year {year}" + source = f"Created from daily averaged CPLHIST files for case {case} for years {begyear:04d}-{endyear:04d}" + institution = "National Science Foundation (NSF) - National Center for Atmospheric Research (NCAR) Community Earth System Model (CESM) project" for name, value in ( - ("Created_on", f"{date:%Y-%m-%d}"), + ("Created_on", f"{todaysdate}"), ("Created_by", getpass.getuser()), ("Created_with", created_with), + ("comment", comment), + ("title", title), + ("source", source), + ("institution", institution), + ("Conventions", "CF-1.13"), + ("case", case), ): cmd += ["-a", f"{name},global,o,c,{value}"] run_cmd_output_on_error(cmd + [filename], f"Failed adding provenance to {filename}") -def write_output(mondir, out, year, date): +def write_output(case, mondir, outfile, begyear, endyear, year, todaysdate): """ Step 3: put the 12 monthly climatologies in one file, reset its time axis to the middle of each month of year, and add provenance attributes """ log(logger, "Putting the climatology into one file and finalizing its metadata") clim_files = [clim_file(mondir, month) for month in range(1, 13)] - run_cmd_output_on_error(["ncrcat", "-O"] + clim_files + [out], f"Failed creating {out}") - set_time_axis(out, year) - write_provenance(out, date) - log(logger, f"Wrote {out} (time = mid-month of year {year:04d}, noleap)") + run_cmd_output_on_error(["ncrcat"] + clim_files + [outfile], f"Failed creating {outfile}") + set_time_axis(outfile, year) + add_var_attributes(outfile) + write_provenance(case, outfile, begyear, endyear, year, todaysdate) + log(logger, f"Wrote {outfile} (time = mid-month of year {year:04d}, noleap)") def job_args(args): @@ -523,11 +583,12 @@ def job_args(args): def batch_script(argv, *, account, walltime, workdir): """ Text of the PBS batch job script that runs this tool with the arguments argv + Written out to a variable so it can be submitted to qsub from stdin, instead of writing a temporary file """ runfile = io.StringIO() write_runscript_part1( number_of_nodes=1, - tasks_per_node=1, + tasks_per_node=argv.njobs, machine="derecho", account=account, walltime=walltime, @@ -593,16 +654,16 @@ def main(): outdir = output_dir(args.archdir, args.outdir, args.case) mondir = os.path.join(outdir, "mon") os.makedirs(mondir, exist_ok=True) - date = datetime.date.today() + todaysdate = datetime.date.today() monthly_means(files_by_month, year_months(args.begyear, args.endyear), mondir, var_list, njobs) monthly_climatology(mondir, args.begyear, args.endyear, njobs) - out = output_filename(outdir, args.case, args.begyear, args.endyear, date) - write_output(mondir, out, args.year, date) + outfile = output_filename(outdir, args.case, args.begyear, args.endyear, todaysdate) + write_output(args.case, mondir, outfile, args.begyear, args.endyear, args.year, todaysdate) if args.keep: log(logger, f"Kept intermediate files in {mondir}") else: shutil.rmtree(mondir) - log(logger, f"Successfully created monthly climatology: {out}") + log(logger, f"Successfully created monthly climatology: {outfile}") From 6f51a69bb32c55919e112320ae24b106615702f8 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Sat, 10 Oct 2026 13:09:05 -0600 Subject: [PATCH 10/37] Run black and get it to pass through pylint --- python/ctsm/cplhist/daily_to_monthly_clim.py | 45 +++++++++++++++----- 1 file changed, 35 insertions(+), 10 deletions(-) diff --git a/python/ctsm/cplhist/daily_to_monthly_clim.py b/python/ctsm/cplhist/daily_to_monthly_clim.py index 53e3d1fec5..1afe5893a2 100644 --- a/python/ctsm/cplhist/daily_to_monthly_clim.py +++ b/python/ctsm/cplhist/daily_to_monthly_clim.py @@ -29,14 +29,15 @@ Another example for Nitrogen deposition is here from Simone Tilmes and Mike Mills: https://svn.code.sf.net/p/codescripts/code/trunk/ncl/cam_forcing/CreateDepositionFile.ncl -Neither were quite right for what's needed here. And both are in NCL which is well after end of life now. +Neither were quite right for what's needed here. And both are in NCL which is well after +end of life now. Also there is code in LDF to create monthly climatologies, but it's not setup to bring in ensemble averages and not specialized to create datafiles for DATM aerosol forcing. """ import argparse -import concurrent.futures # Handles parallel multi-processing by sending ncra jobs on different threads +import concurrent.futures # Handles parallel multi-processing with ncra jobs on different threads import datetime import getpass import io @@ -259,8 +260,10 @@ def check_args(args): todaysdate = datetime.date.today() outdir = output_dir(args.archdir, args.outdir, args.case) outfile = output_filename(outdir, args.case, args.begyear, args.endyear, todaysdate) - if path.exists(outfile): - abort(f"Output file {outfile} already exists, remove it or use a different output directory") + if os.path.exists(outfile): + abort( + f"Output file {outfile} already exists, remove it or use a different output directory" + ) def build_var_list(variables, always_vars): @@ -374,7 +377,7 @@ def ncra_command(infiles, outfile, var_list=None): The ncra command (as a list) to average the input files into outfile If file already exists, returns an empty list and logs a message instead of running ncra """ - if path.exists(outfile): + if os.path.exists(outfile): log(logger, f"File already exists:{outfile} so skipping ncra command") return [] # Use --hst option so that history attribute isn't appended to as it's too long @@ -506,28 +509,48 @@ def set_time_axis(filename, year): climatology_bounds.setncattr("units", units) climatology_bounds.setncattr("calendar", "noleap") + def add_var_attributes(filename): """ Add variable attributes to the file """ with Dataset(filename, "a") as ncfile: for var in ncfile.variables: - if var != "time" and var != "climatology_bounds": + if var not in ("time", "climatology_bounds"): ncfile.variables[var].setncattr("cell_methods", "time: mean") ncfile.variables[var].setncattr("coordinates", "time atmImp_lon atmImp_lat") + +# pylint: disable=too-many-positional-arguments def write_provenance(case, filename, begyear, endyear, year, todaysdate): """ Add global attributes saying when, by whom and with what the file was created (like update_metadata in ctsm/site_and_regional/base_case.py, but using ncatted) """ + logger.debug( + "Write provenance information out to %s for year=%s case=%s years=%s-%s", + filename, + year, + case, + begyear, + endyear, + ) created_with = f"./{os.path.basename(WRAPPER)} -- {get_ctsm_git_short_hash()}" # Use Overwrite option since the file already exists and we want to add attributes to it cmd = ["ncatted", "-O"] comment = "Monthly climatology created from daily averaged CPLHIST files for DATM" - title = "Monthly Climatology of Daily Averaged Atmosphere Coupler History Files for cyclical year {year}" - source = f"Created from daily averaged CPLHIST files for case {case} for years {begyear:04d}-{endyear:04d}" - institution = "National Science Foundation (NSF) - National Center for Atmospheric Research (NCAR) Community Earth System Model (CESM) project" + title = ( + "Monthly Climatology of Daily Averaged Atmosphere Coupler History Files" + + " for cyclical year {year}" + ) + source = ( + f"Created from daily averaged CPLHIST files for case {case} for " + + f"years {begyear:04d}-{endyear:04d}" + ) + institution = ( + "National Science Foundation (NSF) - National Center for Atmospheric Research" + + " (NCAR) Community Earth System Model (CESM) project" + ) for name, value in ( ("Created_on", f"{todaysdate}"), ("Created_by", getpass.getuser()), @@ -543,6 +566,7 @@ def write_provenance(case, filename, begyear, endyear, year, todaysdate): run_cmd_output_on_error(cmd + [filename], f"Failed adding provenance to {filename}") +# pylint: disable=too-many-positional-arguments def write_output(case, mondir, outfile, begyear, endyear, year, todaysdate): """ Step 3: put the 12 monthly climatologies in one file, reset its time axis to the middle of @@ -583,7 +607,8 @@ def job_args(args): def batch_script(argv, *, account, walltime, workdir): """ Text of the PBS batch job script that runs this tool with the arguments argv - Written out to a variable so it can be submitted to qsub from stdin, instead of writing a temporary file + Written out to a variable so it can be submitted to qsub from stdin, + instead of writing a temporary file """ runfile = io.StringIO() write_runscript_part1( From 7fe2d46039b2027f9499af70355c5862b6db0f49 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Sat, 10 Oct 2026 13:44:43 -0600 Subject: [PATCH 11/37] OK these are the changes from Claude to fix problems I introduced, I carefully examined it's description of what it did and why, as well as the final code changes --- python/ctsm/cplhist/daily_to_monthly_clim.py | 157 ++++++++++++++----- 1 file changed, 121 insertions(+), 36 deletions(-) diff --git a/python/ctsm/cplhist/daily_to_monthly_clim.py b/python/ctsm/cplhist/daily_to_monthly_clim.py index 1afe5893a2..3dc6dbe5fc 100644 --- a/python/ctsm/cplhist/daily_to_monthly_clim.py +++ b/python/ctsm/cplhist/daily_to_monthly_clim.py @@ -323,13 +323,20 @@ def check_day_counts(files_by_month, yms): return errors +def file_vars(filename): + """ + Return the set of variable names in the netCDF file + """ + with Dataset(filename) as ncfile: + return set(ncfile.variables) + + def missing_vars(filename, var_list): """ Return the variables in var_list that are not in the netCDF file """ - with Dataset(filename) as ncfile: - file_vars = set(ncfile.variables) - return [var for var in var_list if var not in file_vars] + in_file = file_vars(filename) + return [var for var in var_list if var not in in_file] def check_nco(): @@ -372,16 +379,20 @@ def check_input(args, var_list): return files_by_month -def ncra_command(infiles, outfile, var_list=None): +def ncra_command(infiles, outfile, var_list=None, *, skip_existing=False): """ The ncra command (as a list) to average the input files into outfile - If file already exists, returns an empty list and logs a message instead of running ncra + + With skip_existing, if outfile already exists, returns an empty list (no command to run) + and logs a message """ - if os.path.exists(outfile): - log(logger, f"File already exists:{outfile} so skipping ncra command") + if skip_existing and os.path.exists(outfile): + log(logger, f"File already exists: {outfile} so skipping ncra command") return [] # Use --hst option so that history attribute isn't appended to as it's too long - cmd = ["ncra", "--hst"] + # Use -O to overwrite the output file without asking (existing files are skipped above + # if they should be kept) + cmd = ["ncra", "-O", "--hst"] if var_list is not None: cmd += ["-v", ",".join(var_list)] return cmd + list(infiles) + ["-o", outfile] @@ -393,7 +404,11 @@ def run_jobs(commands, njobs, *, progress_every=None, runner=run_cmd_output_on_e Stops at the first failure: the jobs that have not started are cancelled and the runner's error (normally an abort) is raised. Logs progress every progress_every completed jobs. + Empty commands (for output files that already exist) are skipped. """ + commands = [cmd for cmd in commands if cmd] + if not commands: + return ndone = 0 with concurrent.futures.ThreadPoolExecutor(max_workers=njobs) as executor: futures = [executor.submit(runner, cmd, f"Failed creating {cmd[-1]}") for cmd in commands] @@ -425,15 +440,39 @@ def clim_file(mondir, month): return os.path.join(mondir, f"clim_{month:02d}.nc") +def check_reused_mon_file(monfile, daily_files, var_list): + """ + Abort if a monthly file left from an earlier run doesn't have all the variables this run + needs (for example if it was made with a different variable list) + + With all variables (var_list is None), the monthly file must have all the variables in the + first daily file + """ + if var_list is None: + var_list = sorted(file_vars(daily_files[0])) + missing = missing_vars(monfile, var_list) + if missing: + abort( + f"Existing monthly file {monfile} is missing variables: {' '.join(missing)}" + " (it was probably made with a different variable list, remove it and rerun)" + ) + + def monthly_means(files_by_month, yms, mondir, var_list, njobs): """ Step 1: average the daily files of each month of each year + + Monthly files left from an earlier run (for example with --keep) are reused, after checking + that they have the variables needed """ log(logger, f"Computing monthly means for each year ({njobs} ncra jobs at a time)") - commands = [ - ncra_command(files_by_month[(year, month)], mon_file(mondir, year, month), var_list) - for year, month in yms - ] + commands = [] + for year, month in yms: + monfile = mon_file(mondir, year, month) + daily_files = files_by_month[(year, month)] + if os.path.exists(monfile): + check_reused_mon_file(monfile, daily_files, var_list) + commands.append(ncra_command(daily_files, monfile, var_list, skip_existing=True)) # Show progress every 10 years run_jobs(commands, njobs, progress_every=120) @@ -442,7 +481,9 @@ def monthly_climatology(mondir, begyr, endyr, njobs): """ Step 2: average each calendar month over the years - Only this run's years are used, so leftover files from an earlier --keep run are not included + Only this run's years are used, so leftover files from an earlier --keep run are not included. + The climatology files are always recreated, so ones left from an earlier run with a + different year range are not reused. """ log(logger, f"Averaging each calendar month over the years {begyr}-{endyr}") commands = [ @@ -483,46 +524,77 @@ def mid_month_times(): return [(start + end) / 2.0 for start, end in month_bounds()] -def set_time_axis(filename, year): +def climatology_bounds_values(begyr, endyr, year): """ - Set the time and time_bnds of the 12-month file to the middle (and bounds) of - each month of year, in the noleap calendar + CF climatology bounds for each calendar month, in days since year-01-01 (noleap calendar) + + Each month's bounds span all the averaged years: from the start of the month in begyr + to the end of the month in endyr (for January: begyr-01-01 to endyr-02-01). The values + are negative for years before year. + """ + days_per_year = sum(NOLEAP_DAYS) + return [ + ((begyr - year) * days_per_year + start, (endyr - year) * days_per_year + end) + for start, end in month_bounds() + ] + + +def set_time_axis(filename, year, begyr, endyr): + """ + Set the time of the 12-month file to the middle of each month of year, and its CF + climatology bounds to span the averaged years begyr to endyr, in the noleap calendar """ units = f"days since {year:04d}-01-01 00:00:00" with Dataset(filename, "a") as ncfile: time = ncfile.variables["time"] time[:] = mid_month_times() - time.setncattr("climatology_bounds", "climatology_bounds") time.setncattr("units", units) time.setncattr("calendar", "noleap") + # For a climatology, CF uses the climatology attribute instead of bounds, and time + # itself has no cell_methods (ncra adds one) + for attr in ("bounds", "cell_methods"): + if attr in time.ncattrs(): + time.delncattr(attr) + time.setncattr("climatology", "climatology_bounds") # - # Change time_bounds to climatology_bounds or create it if it doesn't exist + # Change time_bnds (from CMEPS) to climatology_bounds or create it if it doesn't exist # - if "time_bounds" in ncfile.variables: - ncfile.renameVariable("time_bounds", "climatology_bounds") + if "time_bnds" in ncfile.variables: + ncfile.renameVariable("time_bnds", "climatology_bounds") if "climatology_bounds" not in ncfile.variables: if "ntb" not in ncfile.dimensions: ncfile.createDimension("ntb", 2) ncfile.createVariable("climatology_bounds", "f8", ("time", "ntb")) climatology_bounds = ncfile.variables["climatology_bounds"] - climatology_bounds[:] = month_bounds() + climatology_bounds[:] = climatology_bounds_values(begyr, endyr, year) + # Remove attributes from the renamed time_bnds that don't apply to bounds + for attr in ("bounds", "cell_methods", "coordinates"): + if attr in climatology_bounds.ncattrs(): + climatology_bounds.delncattr(attr) climatology_bounds.setncattr("units", units) climatology_bounds.setncattr("calendar", "noleap") def add_var_attributes(filename): """ - Add variable attributes to the file + Add the climatology cell_methods and the lon/lat coordinates attributes to the averaged data + variables (the variables on the time dimension, other than time, its bounds and lon/lat) """ with Dataset(filename, "a") as ncfile: - for var in ncfile.variables: - if var not in ("time", "climatology_bounds"): - ncfile.variables[var].setncattr("cell_methods", "time: mean") - ncfile.variables[var].setncattr("coordinates", "time atmImp_lon atmImp_lat") + coords = " ".join(var for var in ALWAYS_VARS if var in ncfile.variables) + for name, var in ncfile.variables.items(): + if name in ("time", "climatology_bounds") or name in ALWAYS_VARS: + continue + if "time" not in var.dimensions: + continue + var.setncattr("cell_methods", "time: mean within years time: mean over years") + if coords: + var.setncattr("coordinates", coords) -# pylint: disable=too-many-positional-arguments -def write_provenance(case, filename, begyear, endyear, year, todaysdate): +def write_provenance( # pylint: disable=too-many-positional-arguments + case, filename, begyear, endyear, year, todaysdate +): """ Add global attributes saying when, by whom and with what the file was created (like update_metadata in ctsm/site_and_regional/base_case.py, but using ncatted) @@ -541,7 +613,7 @@ def write_provenance(case, filename, begyear, endyear, year, todaysdate): comment = "Monthly climatology created from daily averaged CPLHIST files for DATM" title = ( "Monthly Climatology of Daily Averaged Atmosphere Coupler History Files" - + " for cyclical year {year}" + + f" for cyclical year {year}" ) source = ( f"Created from daily averaged CPLHIST files for case {case} for " @@ -566,16 +638,19 @@ def write_provenance(case, filename, begyear, endyear, year, todaysdate): run_cmd_output_on_error(cmd + [filename], f"Failed adding provenance to {filename}") -# pylint: disable=too-many-positional-arguments -def write_output(case, mondir, outfile, begyear, endyear, year, todaysdate): +def write_output( # pylint: disable=too-many-positional-arguments + case, mondir, outfile, begyear, endyear, year, todaysdate +): """ Step 3: put the 12 monthly climatologies in one file, reset its time axis to the middle of each month of year, and add provenance attributes """ log(logger, "Putting the climatology into one file and finalizing its metadata") clim_files = [clim_file(mondir, month) for month in range(1, 13)] - run_cmd_output_on_error(["ncrcat"] + clim_files + [outfile], f"Failed creating {outfile}") - set_time_axis(outfile, year) + # -O so ncrcat never stops to ask about overwriting (check_args already made sure the + # output file didn't exist when the run started) + run_cmd_output_on_error(["ncrcat", "-O"] + clim_files + [outfile], f"Failed creating {outfile}") + set_time_axis(outfile, year, begyear, endyear) add_var_attributes(outfile) write_provenance(case, outfile, begyear, endyear, year, todaysdate) log(logger, f"Wrote {outfile} (time = mid-month of year {year:04d}, noleap)") @@ -604,16 +679,18 @@ def job_args(args): return argv -def batch_script(argv, *, account, walltime, workdir): +def batch_script(argv, *, account, walltime, workdir, njobs): """ Text of the PBS batch job script that runs this tool with the arguments argv Written out to a variable so it can be submitted to qsub from stdin, instead of writing a temporary file + + njobs is the number of simultaneous ncra jobs the batch job will run """ runfile = io.StringIO() write_runscript_part1( number_of_nodes=1, - tasks_per_node=argv.njobs, + tasks_per_node=njobs, machine="derecho", account=account, walltime=walltime, @@ -666,7 +743,15 @@ def main(): if args.batch: account = args.account or os.environ.get("ACCOUNT") or DEFAULT_ACCOUNT argv = job_args(args) - script = batch_script(argv, account=account, walltime=args.walltime, workdir=os.getcwd()) + # The batch job runs with the njobs given, or the default inside a PBS job + job_njobs = args.njobs if args.njobs is not None else PBS_NJOBS + script = batch_script( + argv, + account=account, + walltime=args.walltime, + workdir=os.getcwd(), + njobs=job_njobs, + ) logger.debug("Batch job script:\n%s", script) jobid = submit_batch(script, args.queue) log( From a12719460eccbb4c704cd219d249b023744f4db0 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Sat, 10 Oct 2026 20:45:42 -0600 Subject: [PATCH 12/37] Add long_name, standard_name and axis, as well as host and tweak some comments --- python/ctsm/cplhist/daily_to_monthly_clim.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/python/ctsm/cplhist/daily_to_monthly_clim.py b/python/ctsm/cplhist/daily_to_monthly_clim.py index 3dc6dbe5fc..492846dcf1 100644 --- a/python/ctsm/cplhist/daily_to_monthly_clim.py +++ b/python/ctsm/cplhist/daily_to_monthly_clim.py @@ -549,7 +549,10 @@ def set_time_axis(filename, year, begyr, endyr): time = ncfile.variables["time"] time[:] = mid_month_times() time.setncattr("units", units) + time.setncattr("long_name", "time") + time.setncattr("standard_name", "time") time.setncattr("calendar", "noleap") + time.setncattr("cell_methods", "time: mean") # For a climatology, CF uses the climatology attribute instead of bounds, and time # itself has no cell_methods (ncra adds one) for attr in ("bounds", "cell_methods"): @@ -572,6 +575,9 @@ def set_time_axis(filename, year, begyr, endyr): if attr in climatology_bounds.ncattrs(): climatology_bounds.delncattr(attr) climatology_bounds.setncattr("units", units) + climatology_bounds.setncattr("long_name", "time") + climatology_bounds.setncattr("standard_name", "time") + climatology_bounds.setncattr("axis", "T") climatology_bounds.setncattr("calendar", "noleap") @@ -587,7 +593,7 @@ def add_var_attributes(filename): continue if "time" not in var.dimensions: continue - var.setncattr("cell_methods", "time: mean within years time: mean over years") + var.setncattr("cell_methods", "time: mean") if coords: var.setncattr("coordinates", coords) @@ -616,13 +622,14 @@ def write_provenance( # pylint: disable=too-many-positional-arguments + f" for cyclical year {year}" ) source = ( - f"Created from daily averaged CPLHIST files for case {case} for " + f"Created from daily averaged CESM CPLHIST files for case {case} for " + f"years {begyear:04d}-{endyear:04d}" ) institution = ( "National Science Foundation (NSF) - National Center for Atmospheric Research" + " (NCAR) Community Earth System Model (CESM) project" ) + hostname = os.getenv("HOSTNAME") for name, value in ( ("Created_on", f"{todaysdate}"), ("Created_by", getpass.getuser()), @@ -632,6 +639,7 @@ def write_provenance( # pylint: disable=too-many-positional-arguments ("source", source), ("institution", institution), ("Conventions", "CF-1.13"), + ("host", hostname), ("case", case), ): cmd += ["-a", f"{name},global,o,c,{value}"] From cb0f6fafd370d488eec0f5efd3a6dcdc48b554a5 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Sat, 10 Oct 2026 22:28:33 -0600 Subject: [PATCH 13/37] Some adjustments from Claude to get it closer to CF-Convention compliance --- python/ctsm/cplhist/daily_to_monthly_clim.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/python/ctsm/cplhist/daily_to_monthly_clim.py b/python/ctsm/cplhist/daily_to_monthly_clim.py index 492846dcf1..4b05733c5c 100644 --- a/python/ctsm/cplhist/daily_to_monthly_clim.py +++ b/python/ctsm/cplhist/daily_to_monthly_clim.py @@ -577,8 +577,11 @@ def set_time_axis(filename, year, begyr, endyr): climatology_bounds.setncattr("units", units) climatology_bounds.setncattr("long_name", "time") climatology_bounds.setncattr("standard_name", "time") - climatology_bounds.setncattr("axis", "T") - climatology_bounds.setncattr("calendar", "noleap") + # Bounds take axis and calendar from their coordinate (time), so CF checkers report + # these as non-standard on a bounds variable; remove them (time_bnds from CMEPS has them) + for attr in ("axis", "calendar"): + if attr in climatology_bounds.ncattrs(): + climatology_bounds.delncattr(attr) def add_var_attributes(filename): @@ -589,7 +592,12 @@ def add_var_attributes(filename): with Dataset(filename, "a") as ncfile: coords = " ".join(var for var in ALWAYS_VARS if var in ncfile.variables) for name, var in ncfile.variables.items(): - if name in ("time", "climatology_bounds") or name in ALWAYS_VARS: + if name in ALWAYS_VARS: + # cell_methods is for data variables, not coordinates like lon/lat (ncra adds it) + if "cell_methods" in var.ncattrs(): + var.delncattr("cell_methods") + continue + if name in ("time", "climatology_bounds"): continue if "time" not in var.dimensions: continue From bd1eb5f3acf2a69bccb04bc9bbe5f125aecf600e Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Mon, 12 Oct 2026 02:11:31 -0600 Subject: [PATCH 14/37] Get the compliance-checker to pass for CF-1.11, tweak some of the metadata around for clarity and ordering --- python/ctsm/cplhist/daily_to_monthly_clim.py | 34 +++++++++++++------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/python/ctsm/cplhist/daily_to_monthly_clim.py b/python/ctsm/cplhist/daily_to_monthly_clim.py index 4b05733c5c..6430b76bef 100644 --- a/python/ctsm/cplhist/daily_to_monthly_clim.py +++ b/python/ctsm/cplhist/daily_to_monthly_clim.py @@ -549,7 +549,7 @@ def set_time_axis(filename, year, begyr, endyr): time = ncfile.variables["time"] time[:] = mid_month_times() time.setncattr("units", units) - time.setncattr("long_name", "time") + time.setncattr("long_name", "climatological time at exact middle of each month of climatology_bounds") time.setncattr("standard_name", "time") time.setncattr("calendar", "noleap") time.setncattr("cell_methods", "time: mean") @@ -575,7 +575,7 @@ def set_time_axis(filename, year, begyr, endyr): if attr in climatology_bounds.ncattrs(): climatology_bounds.delncattr(attr) climatology_bounds.setncattr("units", units) - climatology_bounds.setncattr("long_name", "time") + climatology_bounds.setncattr("long_name", "climatology time interval endpoints") climatology_bounds.setncattr("standard_name", "time") # Bounds take axis and calendar from their coordinate (time), so CF checkers report # these as non-standard on a bounds variable; remove them (time_bnds from CMEPS has them) @@ -601,7 +601,9 @@ def add_var_attributes(filename): continue if "time" not in var.dimensions: continue - var.setncattr("cell_methods", "time: mean") + var.setncattr("cell_methods", "time: mean within years time: mean over years") + var.delncattr("standard_name") # Remove standard_name as they don't match CF-Convention standard names + var.setncattr("long_name", name) if coords: var.setncattr("coordinates", coords) @@ -623,7 +625,8 @@ def write_provenance( # pylint: disable=too-many-positional-arguments ) created_with = f"./{os.path.basename(WRAPPER)} -- {get_ctsm_git_short_hash()}" # Use Overwrite option since the file already exists and we want to add attributes to it - cmd = ["ncatted", "-O"] + # USe --hst option so that history attribute isn't appended as it's not helpful + cmd = ["ncatted", "-O", "--hst"] comment = "Monthly climatology created from daily averaged CPLHIST files for DATM" title = ( "Monthly Climatology of Daily Averaged Atmosphere Coupler History Files" @@ -639,19 +642,28 @@ def write_provenance( # pylint: disable=too-many-positional-arguments ) hostname = os.getenv("HOSTNAME") for name, value in ( - ("Created_on", f"{todaysdate}"), - ("Created_by", getpass.getuser()), - ("Created_with", created_with), - ("comment", comment), ("title", title), + ("comment", comment), ("source", source), ("institution", institution), - ("Conventions", "CF-1.13"), - ("host", hostname), - ("case", case), + ("Conventions", "CF-1.11"), # Use CF-11 as the compliance-checker doens't support CF-12 yet + ("Created_on", f"{todaysdate}"), + ("Created_by", getpass.getuser()), + ("Created_with", created_with), + ("time_period_freq", "month_1"), + ("hostname", hostname), + ("model_doi_url", "https://doi.org/10.5065/D67H1H0V"), + ("made_from_cesm_case", case), ): cmd += ["-a", f"{name},global,o,c,{value}"] run_cmd_output_on_error(cmd + [filename], f"Failed adding provenance to {filename}") + # + # Delete a few global attributes + # + name = "file_version" + cmd = ["ncatted", "-O"] + cmd += ["-a", f"{name},global,d,,"] + run_cmd_output_on_error(cmd + [filename], f"Failed removing attribute {name} from {filename}") def write_output( # pylint: disable=too-many-positional-arguments From 5cc51c7a1fb74c41a5642d485583bbce50e0bf32 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Mon, 12 Oct 2026 02:16:49 -0600 Subject: [PATCH 15/37] Run black and make sure passes pylint --- python/ctsm/cplhist/daily_to_monthly_clim.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/python/ctsm/cplhist/daily_to_monthly_clim.py b/python/ctsm/cplhist/daily_to_monthly_clim.py index 6430b76bef..55f6455ed2 100644 --- a/python/ctsm/cplhist/daily_to_monthly_clim.py +++ b/python/ctsm/cplhist/daily_to_monthly_clim.py @@ -549,7 +549,9 @@ def set_time_axis(filename, year, begyr, endyr): time = ncfile.variables["time"] time[:] = mid_month_times() time.setncattr("units", units) - time.setncattr("long_name", "climatological time at exact middle of each month of climatology_bounds") + time.setncattr( + "long_name", "climatological time at exact middle of each month of climatology_bounds" + ) time.setncattr("standard_name", "time") time.setncattr("calendar", "noleap") time.setncattr("cell_methods", "time: mean") @@ -602,7 +604,9 @@ def add_var_attributes(filename): if "time" not in var.dimensions: continue var.setncattr("cell_methods", "time: mean within years time: mean over years") - var.delncattr("standard_name") # Remove standard_name as they don't match CF-Convention standard names + var.delncattr( + "standard_name" + ) # Remove standard_name as they don't match CF-Convention standard names var.setncattr("long_name", name) if coords: var.setncattr("coordinates", coords) @@ -646,7 +650,7 @@ def write_provenance( # pylint: disable=too-many-positional-arguments ("comment", comment), ("source", source), ("institution", institution), - ("Conventions", "CF-1.11"), # Use CF-11 as the compliance-checker doens't support CF-12 yet + ("Conventions", "CF-1.11"), # Use CF-11 as the compliance-checker doens't support CF-12 yet ("Created_on", f"{todaysdate}"), ("Created_by", getpass.getuser()), ("Created_with", created_with), From f0b6c26afbae9b537a244cf7698101436fa8cdeb Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Mon, 12 Oct 2026 02:21:20 -0600 Subject: [PATCH 16/37] Add a comment that the doi is the CESM2 one and should be updated when the CESM3 one is available --- python/ctsm/cplhist/daily_to_monthly_clim.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/ctsm/cplhist/daily_to_monthly_clim.py b/python/ctsm/cplhist/daily_to_monthly_clim.py index 55f6455ed2..e48f0069a2 100644 --- a/python/ctsm/cplhist/daily_to_monthly_clim.py +++ b/python/ctsm/cplhist/daily_to_monthly_clim.py @@ -656,7 +656,7 @@ def write_provenance( # pylint: disable=too-many-positional-arguments ("Created_with", created_with), ("time_period_freq", "month_1"), ("hostname", hostname), - ("model_doi_url", "https://doi.org/10.5065/D67H1H0V"), + ("model_doi_url", "https://doi.org/10.5065/D67H1H0V"), # NOTE: this is the CESM2 DOI, and should be updated when a CESM3 is available ("made_from_cesm_case", case), ): cmd += ["-a", f"{name},global,o,c,{value}"] From 724d94e9590e92bd2c719df09ceb98c1885228fe Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Mon, 12 Oct 2026 02:27:31 -0600 Subject: [PATCH 17/37] Recommendations from Claude --- python/ctsm/cplhist/daily_to_monthly_clim.py | 34 ++++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/python/ctsm/cplhist/daily_to_monthly_clim.py b/python/ctsm/cplhist/daily_to_monthly_clim.py index e48f0069a2..a071e47b31 100644 --- a/python/ctsm/cplhist/daily_to_monthly_clim.py +++ b/python/ctsm/cplhist/daily_to_monthly_clim.py @@ -46,6 +46,7 @@ import re import shlex import shutil +import socket import subprocess import sys @@ -554,7 +555,6 @@ def set_time_axis(filename, year, begyr, endyr): ) time.setncattr("standard_name", "time") time.setncattr("calendar", "noleap") - time.setncattr("cell_methods", "time: mean") # For a climatology, CF uses the climatology attribute instead of bounds, and time # itself has no cell_methods (ncra adds one) for attr in ("bounds", "cell_methods"): @@ -604,10 +604,12 @@ def add_var_attributes(filename): if "time" not in var.dimensions: continue var.setncattr("cell_methods", "time: mean within years time: mean over years") - var.delncattr( - "standard_name" - ) # Remove standard_name as they don't match CF-Convention standard names - var.setncattr("long_name", name) + # Remove standard_name, as the CMEPS field names aren't CF-Convention standard names + if "standard_name" in var.ncattrs(): + var.delncattr("standard_name") + # Use the variable name as the long_name, unless the file already has one + if "long_name" not in var.ncattrs(): + var.setncattr("long_name", name) if coords: var.setncattr("coordinates", coords) @@ -629,7 +631,7 @@ def write_provenance( # pylint: disable=too-many-positional-arguments ) created_with = f"./{os.path.basename(WRAPPER)} -- {get_ctsm_git_short_hash()}" # Use Overwrite option since the file already exists and we want to add attributes to it - # USe --hst option so that history attribute isn't appended as it's not helpful + # Use --hst option so that history attribute isn't appended as it's not helpful cmd = ["ncatted", "-O", "--hst"] comment = "Monthly climatology created from daily averaged CPLHIST files for DATM" title = ( @@ -644,30 +646,28 @@ def write_provenance( # pylint: disable=too-many-positional-arguments "National Science Foundation (NSF) - National Center for Atmospheric Research" + " (NCAR) Community Earth System Model (CESM) project" ) - hostname = os.getenv("HOSTNAME") for name, value in ( ("title", title), ("comment", comment), ("source", source), ("institution", institution), - ("Conventions", "CF-1.11"), # Use CF-11 as the compliance-checker doens't support CF-12 yet + # Use CF-1.11 as the compliance-checker doesn't support CF-1.12 yet + ("Conventions", "CF-1.11"), ("Created_on", f"{todaysdate}"), ("Created_by", getpass.getuser()), ("Created_with", created_with), ("time_period_freq", "month_1"), - ("hostname", hostname), - ("model_doi_url", "https://doi.org/10.5065/D67H1H0V"), # NOTE: this is the CESM2 DOI, and should be updated when a CESM3 is available + ("hostname", socket.gethostname()), + # NOTE: this is the CESM2 DOI, and should be updated when a CESM3 DOI is available + ("model_doi_url", "https://doi.org/10.5065/D67H1H0V"), ("made_from_cesm_case", case), ): cmd += ["-a", f"{name},global,o,c,{value}"] + # Delete global attributes that don't apply to the climatology (in the same ncatted call, + # so --hst covers it too) + for name in ("file_version",): + cmd += ["-a", f"{name},global,d,,"] run_cmd_output_on_error(cmd + [filename], f"Failed adding provenance to {filename}") - # - # Delete a few global attributes - # - name = "file_version" - cmd = ["ncatted", "-O"] - cmd += ["-a", f"{name},global,d,,"] - run_cmd_output_on_error(cmd + [filename], f"Failed removing attribute {name} from {filename}") def write_output( # pylint: disable=too-many-positional-arguments From 0aec3e7d30bd48246e3f1f4a132436fc5bdb5d58 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Mon, 12 Oct 2026 03:20:12 -0600 Subject: [PATCH 18/37] Updates from Claude so that the final lat/lon variables on the file do NOT have the time dimension as that can't be done on the final file, and it's a mistake that CPLHIST files have it to begin with --- python/ctsm/cplhist/daily_to_monthly_clim.py | 125 +++++++++++++++---- 1 file changed, 98 insertions(+), 27 deletions(-) diff --git a/python/ctsm/cplhist/daily_to_monthly_clim.py b/python/ctsm/cplhist/daily_to_monthly_clim.py index a071e47b31..b860ff77e4 100644 --- a/python/ctsm/cplhist/daily_to_monthly_clim.py +++ b/python/ctsm/cplhist/daily_to_monthly_clim.py @@ -125,7 +125,8 @@ def get_parser(): vars_group.add_argument( "--vars", help=f"""Comma-separated list of variables to average - ({",".join(ALWAYS_VARS)} are always included) + ({",".join(ALWAYS_VARS)} are always included, copied from the first daily file + without the time dimension) (default: {",".join(DEFAULT_VARS)})""", action="store", dest="vars", @@ -246,8 +247,8 @@ def check_args(args): abort(f"NJOBS must be a positive integer: {args.njobs}") if args.account is not None and not args.batch: abort("--account is only used with --batch") - if not args.all_vars and not build_var_list(args.vars, []): - abort("Variable list given with --vars is empty") + if not args.all_vars and not averaged_vars(args.vars): + abort(f"No variables to average given with --vars (besides {','.join(ALWAYS_VARS)})") if not os.path.isdir(args.archdir): abort(f"Input {args.archdir} directory does not exist") if args.outdir is not None and not os.path.isdir(args.outdir): @@ -283,6 +284,17 @@ def build_var_list(variables, always_vars): return var_list +def averaged_vars(variables): + """ + The variables for ncra to average: the ones given, without duplicates, empty names or the + ALWAYS_VARS (which are copied to the output without the time dimension instead). Returns + None when variables is None, meaning all variables. + """ + if variables is None: + return None + return [var for var in build_var_list(variables, []) if var not in ALWAYS_VARS] + + def year_months(begyr, endyr): """ List of (year, month) to process, from BEGYEAR-01 to ENDYEAR-12 @@ -351,8 +363,9 @@ def check_nco(): def check_input(args, var_list): """ - Check the input files before doing any work: every requested variable is in the first file, - and every month in the range has a full set of daily files + Check the input files before doing any work: every variable in var_list (the requested + variables and the ALWAYS_VARS) is in the first file, and every month in the range has a full + set of daily files Returns the dictionary of daily files from index_daily_files """ @@ -364,14 +377,13 @@ def check_input(args, var_list): ) files_by_month = index_daily_files(indir, args.case) - if var_list is not None: - first_files = files_by_month.get(yms[0], []) - if not first_files: - abort(f"No files for {yms[0][0]:04d}-{yms[0][1]:02d} in {indir}") - log(logger, f"Checking the requested variables are in {first_files[0]}") - missing = missing_vars(first_files[0], var_list) - if missing: - abort(f"Variables not found in {first_files[0]}: {' '.join(missing)}") + first_files = files_by_month.get(yms[0], []) + if not first_files: + abort(f"No files for {yms[0][0]:04d}-{yms[0][1]:02d} in {indir}") + log(logger, f"Checking the requested variables are in {first_files[0]}") + missing = missing_vars(first_files[0], var_list) + if missing: + abort(f"Variables not found in {first_files[0]}: {' '.join(missing)}") log(logger, "Checking that every month has the expected number of daily files") errors = check_day_counts(files_by_month, yms) @@ -380,10 +392,13 @@ def check_input(args, var_list): return files_by_month -def ncra_command(infiles, outfile, var_list=None, *, skip_existing=False): +def ncra_command(infiles, outfile, var_list=None, *, exclude=None, skip_existing=False): """ The ncra command (as a list) to average the input files into outfile + var_list is the variables to average (None for all). When var_list is None, the variables + in exclude (if given) are left out. + With skip_existing, if outfile already exists, returns an empty list (no command to run) and logs a message """ @@ -396,6 +411,8 @@ def ncra_command(infiles, outfile, var_list=None, *, skip_existing=False): cmd = ["ncra", "-O", "--hst"] if var_list is not None: cmd += ["-v", ",".join(var_list)] + elif exclude: + cmd += ["-x", "-v", ",".join(exclude)] return cmd + list(infiles) + ["-o", outfile] @@ -447,11 +464,19 @@ def check_reused_mon_file(monfile, daily_files, var_list): needs (for example if it was made with a different variable list) With all variables (var_list is None), the monthly file must have all the variables in the - first daily file + first daily file, except the ALWAYS_VARS. The monthly file must not have the ALWAYS_VARS + (older versions of this tool averaged them with the time dimension). """ + in_monfile = file_vars(monfile) + stale = [var for var in ALWAYS_VARS if var in in_monfile] + if stale: + abort( + f"Existing monthly file {monfile} has {' '.join(stale)}, so it was made by an older" + " version of this tool, remove it and rerun" + ) if var_list is None: - var_list = sorted(file_vars(daily_files[0])) - missing = missing_vars(monfile, var_list) + var_list = sorted(file_vars(daily_files[0]) - set(ALWAYS_VARS)) + missing = [var for var in var_list if var not in in_monfile] if missing: abort( f"Existing monthly file {monfile} is missing variables: {' '.join(missing)}" @@ -464,7 +489,8 @@ def monthly_means(files_by_month, yms, mondir, var_list, njobs): Step 1: average the daily files of each month of each year Monthly files left from an earlier run (for example with --keep) are reused, after checking - that they have the variables needed + that they have the variables needed. The ALWAYS_VARS are never averaged (with all + variables they are excluded), as they are copied to the output at the end without time. """ log(logger, f"Computing monthly means for each year ({njobs} ncra jobs at a time)") commands = [] @@ -473,7 +499,9 @@ def monthly_means(files_by_month, yms, mondir, var_list, njobs): daily_files = files_by_month[(year, month)] if os.path.exists(monfile): check_reused_mon_file(monfile, daily_files, var_list) - commands.append(ncra_command(daily_files, monfile, var_list, skip_existing=True)) + commands.append( + ncra_command(daily_files, monfile, var_list, exclude=ALWAYS_VARS, skip_existing=True) + ) # Show progress every 10 years run_jobs(commands, njobs, progress_every=120) @@ -586,6 +614,33 @@ def set_time_axis(filename, year, begyr, endyr): climatology_bounds.delncattr(attr) +def copy_grid_vars(src_file, dst_file, names): + """ + Copy the grid variables (names) from src_file (a daily file) to dst_file, without the time + dimension, using the first time slice + + The grid doesn't change in time, so the first time slice is the same as all the others (this + isn't checked). Missing dimensions are created in dst_file, and the attributes are copied + (without cell_methods, which is for data variables). + """ + with Dataset(src_file) as src, Dataset(dst_file, "a") as dst: + for name in names: + src_var = src.variables[name] + dims = tuple(dim for dim in src_var.dimensions if dim != "time") + for dim in dims: + if dim not in dst.dimensions: + dst.createDimension(dim, len(src.dimensions[dim])) + attrs = {attr: src_var.getncattr(attr) for attr in src_var.ncattrs()} + # _FillValue can only be set when the variable is created + fill_value = attrs.pop("_FillValue", None) + attrs.pop("cell_methods", None) + dst_var = dst.createVariable(name, src_var.dtype, dims, fill_value=fill_value) + dst_var.setncatts(attrs) + # Index 0 for time, all of every other dimension + first_time = tuple(0 if dim == "time" else slice(None) for dim in src_var.dimensions) + dst_var[...] = src_var[first_time] + + def add_var_attributes(filename): """ Add the climatology cell_methods and the lon/lat coordinates attributes to the averaged data @@ -595,7 +650,8 @@ def add_var_attributes(filename): coords = " ".join(var for var in ALWAYS_VARS if var in ncfile.variables) for name, var in ncfile.variables.items(): if name in ALWAYS_VARS: - # cell_methods is for data variables, not coordinates like lon/lat (ncra adds it) + # cell_methods is for data variables, not coordinates like lon/lat (copy_grid_vars + # already leaves it out, this is just in case) if "cell_methods" in var.ncattrs(): var.delncattr("cell_methods") continue @@ -671,10 +727,11 @@ def write_provenance( # pylint: disable=too-many-positional-arguments def write_output( # pylint: disable=too-many-positional-arguments - case, mondir, outfile, begyear, endyear, year, todaysdate + case, mondir, outfile, begyear, endyear, year, todaysdate, *, grid_file ): """ - Step 3: put the 12 monthly climatologies in one file, reset its time axis to the middle of + Step 3: put the 12 monthly climatologies in one file, copy the grid variables (ALWAYS_VARS) + from grid_file (the first daily file) without time, reset the time axis to the middle of each month of year, and add provenance attributes """ log(logger, "Putting the climatology into one file and finalizing its metadata") @@ -682,6 +739,7 @@ def write_output( # pylint: disable=too-many-positional-arguments # -O so ncrcat never stops to ask about overwriting (check_args already made sure the # output file didn't exist when the run started) run_cmd_output_on_error(["ncrcat", "-O"] + clim_files + [outfile], f"Failed creating {outfile}") + copy_grid_vars(grid_file, outfile, ALWAYS_VARS) set_time_axis(outfile, year, begyear, endyear) add_var_attributes(outfile) write_provenance(case, outfile, begyear, endyear, year, todaysdate) @@ -761,15 +819,18 @@ def main(): check_args(args) njobs = args.njobs if args.njobs is not None else default_njobs() - var_list = None if args.all_vars else build_var_list(args.vars, ALWAYS_VARS) + # Variables for ncra to average (None for all), and the variables the input must have + var_list = None if args.all_vars else averaged_vars(args.vars) + check_list = build_var_list(var_list or [], ALWAYS_VARS) if var_list is None: - log(logger, "Averaging all variables") + log(logger, f"Averaging all variables except {','.join(ALWAYS_VARS)}") else: log(logger, f"Averaging variables: {','.join(var_list)}") + log(logger, f"Copying without time from the first daily file: {','.join(ALWAYS_VARS)}") log(logger, "Create a monthly climatology from daily averaged CPLHIST files for DATM") check_nco() - files_by_month = check_input(args, var_list) + files_by_month = check_input(args, check_list) # With --batch: the input has been checked, so submit to PBS and stop here if args.batch: @@ -798,10 +859,20 @@ def main(): os.makedirs(mondir, exist_ok=True) todaysdate = datetime.date.today() - monthly_means(files_by_month, year_months(args.begyear, args.endyear), mondir, var_list, njobs) + yms = year_months(args.begyear, args.endyear) + monthly_means(files_by_month, yms, mondir, var_list, njobs) monthly_climatology(mondir, args.begyear, args.endyear, njobs) outfile = output_filename(outdir, args.case, args.begyear, args.endyear, todaysdate) - write_output(args.case, mondir, outfile, args.begyear, args.endyear, args.year, todaysdate) + write_output( + args.case, + mondir, + outfile, + args.begyear, + args.endyear, + args.year, + todaysdate, + grid_file=files_by_month[yms[0]][0], + ) if args.keep: log(logger, f"Kept intermediate files in {mondir}") From b1426faf1c1c45dd9c159d6ec9ba335bfdc12347 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Mon, 12 Oct 2026 03:23:29 -0600 Subject: [PATCH 19/37] Up the number of jobs for batch to 64 and 16 for interactive --- python/ctsm/cplhist/daily_to_monthly_clim.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python/ctsm/cplhist/daily_to_monthly_clim.py b/python/ctsm/cplhist/daily_to_monthly_clim.py index b860ff77e4..14448e14be 100644 --- a/python/ctsm/cplhist/daily_to_monthly_clim.py +++ b/python/ctsm/cplhist/daily_to_monthly_clim.py @@ -93,7 +93,7 @@ # NCO commands that are needed NCO_COMMANDS = ("ncra", "ncrcat", "ncatted") # Number of simultaneous ncra jobs inside a PBS batch job -PBS_NJOBS = 16 +PBS_NJOBS = 64 # PBS batch settings PBS_JOBNAME = "daily_to_monthly_clim" DEFAULT_ACCOUNT = "P93300606" @@ -169,7 +169,7 @@ def get_parser(): "-j", "--njobs", help=f"""Number of ncra jobs to run at the same time - (default: {PBS_NJOBS} in a PBS batch job, 1 otherwise)""", + (default: {PBS_NJOBS} in a PBS batch job, 16 otherwise)""", action="store", dest="njobs", type=int, @@ -215,7 +215,7 @@ def default_njobs(): """ if os.environ.get("PBS_JOBID"): return PBS_NJOBS - return 1 + return 16 def input_dir(archdir, case): From 5b1e4a68bbcc830765d3bd4da22cafe3db57333b Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Mon, 12 Oct 2026 03:39:18 -0600 Subject: [PATCH 20/37] Have Claude add INTERACTIVE_JOBS and set it to 12 as that's the number for the last step of adding the 12 climatological months together --- python/ctsm/cplhist/daily_to_monthly_clim.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/python/ctsm/cplhist/daily_to_monthly_clim.py b/python/ctsm/cplhist/daily_to_monthly_clim.py index 14448e14be..0bf56b688e 100644 --- a/python/ctsm/cplhist/daily_to_monthly_clim.py +++ b/python/ctsm/cplhist/daily_to_monthly_clim.py @@ -92,8 +92,9 @@ HIST_TAG = "cpl.hx.atm.24h.avrg" # NCO commands that are needed NCO_COMMANDS = ("ncra", "ncrcat", "ncatted") -# Number of simultaneous ncra jobs inside a PBS batch job +# Number of simultaneous ncra jobs inside a PBS batch job, and otherwise (on a login node) PBS_NJOBS = 64 +INTERACTIVE_NJOBS = 12 # PBS batch settings PBS_JOBNAME = "daily_to_monthly_clim" DEFAULT_ACCOUNT = "P93300606" @@ -169,7 +170,7 @@ def get_parser(): "-j", "--njobs", help=f"""Number of ncra jobs to run at the same time - (default: {PBS_NJOBS} in a PBS batch job, 16 otherwise)""", + (default: {PBS_NJOBS} in a PBS batch job, {INTERACTIVE_NJOBS} otherwise)""", action="store", dest="njobs", type=int, @@ -215,7 +216,7 @@ def default_njobs(): """ if os.environ.get("PBS_JOBID"): return PBS_NJOBS - return 16 + return INTERACTIVE_NJOBS def input_dir(archdir, case): From 58b88c838265ed82903cd548d9c71502895a6d0b Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Mon, 12 Oct 2026 14:05:25 -0600 Subject: [PATCH 21/37] Update python/ctsm/cplhist/daily_to_monthly_clim.py --- python/ctsm/cplhist/daily_to_monthly_clim.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/ctsm/cplhist/daily_to_monthly_clim.py b/python/ctsm/cplhist/daily_to_monthly_clim.py index 0bf56b688e..8835d855ca 100644 --- a/python/ctsm/cplhist/daily_to_monthly_clim.py +++ b/python/ctsm/cplhist/daily_to_monthly_clim.py @@ -716,7 +716,8 @@ def write_provenance( # pylint: disable=too-many-positional-arguments ("time_period_freq", "month_1"), ("hostname", socket.gethostname()), # NOTE: this is the CESM2 DOI, and should be updated when a CESM3 DOI is available - ("model_doi_url", "https://doi.org/10.5065/D67H1H0V"), + # ("model_doi_url", "https://doi.org/10.5065/D67H1H0V"), + # TODO: Uncomment the above when we have a DOI for CESM3.0 ("made_from_cesm_case", case), ): cmd += ["-a", f"{name},global,o,c,{value}"] From 8b615772610129066a8f7899c3222cb4f7bfab00 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Mon, 12 Oct 2026 14:06:03 -0600 Subject: [PATCH 22/37] Update python/ctsm/cplhist/daily_to_monthly_clim.py --- python/ctsm/cplhist/daily_to_monthly_clim.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/ctsm/cplhist/daily_to_monthly_clim.py b/python/ctsm/cplhist/daily_to_monthly_clim.py index 8835d855ca..38b03b2d85 100644 --- a/python/ctsm/cplhist/daily_to_monthly_clim.py +++ b/python/ctsm/cplhist/daily_to_monthly_clim.py @@ -696,7 +696,7 @@ def write_provenance( # pylint: disable=too-many-positional-arguments + f" for cyclical year {year}" ) source = ( - f"Created from daily averaged CESM CPLHIST files for case {case} for " + f"Created from daily averaged CESM3.0 CPLHIST files for case {case} for " + f"years {begyear:04d}-{endyear:04d}" ) institution = ( From 9f77e5934d1f964bc8880ac33e733ecc4249a117 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Fri, 16 Oct 2026 12:46:40 -0600 Subject: [PATCH 23/37] There was one field that was missing and I also added an attribute to give the full command and command line options --- python/ctsm/cplhist/daily_to_monthly_clim.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python/ctsm/cplhist/daily_to_monthly_clim.py b/python/ctsm/cplhist/daily_to_monthly_clim.py index 38b03b2d85..0375204430 100644 --- a/python/ctsm/cplhist/daily_to_monthly_clim.py +++ b/python/ctsm/cplhist/daily_to_monthly_clim.py @@ -69,6 +69,7 @@ # Variables averaged by default DEFAULT_VARS = [ + "atmImp_Faxa_bcph1", "atmImp_Faxa_bcph2", "atmImp_Faxa_bcph3", "atmImp_Faxa_ocph1", @@ -713,6 +714,7 @@ def write_provenance( # pylint: disable=too-many-positional-arguments ("Created_on", f"{todaysdate}"), ("Created_by", getpass.getuser()), ("Created_with", created_with), + ("command_used", shlex.join(sys.argv)), ("time_period_freq", "month_1"), ("hostname", socket.gethostname()), # NOTE: this is the CESM2 DOI, and should be updated when a CESM3 DOI is available From 879760762227f40bd82b36e33f9285f387ec7e50 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Fri, 16 Oct 2026 13:04:06 -0600 Subject: [PATCH 24/37] Add compliance-checker to the ctsm_pylib env so that a CF compliance checker will be available to check compliance of our files we make with CTSM tools --- python/conda_env_ctsm_py.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/python/conda_env_ctsm_py.yml b/python/conda_env_ctsm_py.yml index 96656696db..13dec57601 100644 --- a/python/conda_env_ctsm_py.yml +++ b/python/conda_env_ctsm_py.yml @@ -17,6 +17,7 @@ dependencies: - black=25.1.0 - cartopy=0.24.0 - matplotlib=3.10.1 + - compliance-checker=6.1.0 - pip=25.0.1 # For VS code From b22f86e174eb981be87c8a4edff0746b7b632b8e Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Sat, 24 Oct 2026 01:15:40 -0600 Subject: [PATCH 25/37] unit and sys tests from Claude Opus 5.5 medium for the new tool, this passes all python tests. It adds about a minute to the sys test time, which seems acceptable, passes black and pylint checks --- .../test/test_sys_daily_to_monthly_clim.py | 218 ++++++ .../test/test_unit_daily_to_monthly_clim.py | 705 ++++++++++++++++++ 2 files changed, 923 insertions(+) create mode 100644 python/ctsm/test/test_sys_daily_to_monthly_clim.py create mode 100644 python/ctsm/test/test_unit_daily_to_monthly_clim.py diff --git a/python/ctsm/test/test_sys_daily_to_monthly_clim.py b/python/ctsm/test/test_sys_daily_to_monthly_clim.py new file mode 100644 index 0000000000..e310f323c6 --- /dev/null +++ b/python/ctsm/test/test_sys_daily_to_monthly_clim.py @@ -0,0 +1,218 @@ +#!/usr/bin/env python3 + +"""System tests for daily_to_monthly_clim (needs the NCO commands, on Derecho: module load nco)""" + +import contextlib +import datetime +import io +import os +import shlex +import shutil +import sys +import tempfile +import unittest +from unittest import mock + +import numpy as np +from netCDF4 import Dataset # pylint: disable=no-name-in-module + +# -- add python/ctsm to path (needed if we want to run test stand-alone) +_CTSM_PYTHON = os.path.join(os.path.dirname(os.path.realpath(__file__)), os.pardir, os.pardir) +sys.path.insert(1, _CTSM_PYTHON) +# pylint: disable=wrong-import-position +from ctsm import unit_testing +from ctsm.cplhist import daily_to_monthly_clim as dtm + +# Allow test names that pylint doesn't like; otherwise hard to make them +# readable +# pylint: disable=invalid-name + +CASE = "TC" +BEGYEAR = 2001 +ENDYEAR = 2002 +# A variable that isn't in the default list +EXTRA_VAR = "atmImp_Sa_extra" +NX = 3 +LON = 10.0 + np.arange(NX, dtype="f4") +LAT = -60.0 + np.arange(NX, dtype="f4") + + +def _value(year, month): + """Value of the default variables in the daily files for a year and month""" + return (year - 2000) * 100 + month + + +def _write_daily_file(filename, year, month, day_of_year): + """ + Write a daily file like the CMEPS 24h average coupler history files: 2 time samples, time_bnds, + the default variables and EXTRA_VAR on (time, atmImp_ny, atmImp_nx), and lon/lat with time too + """ + with Dataset(filename, "w") as ncfile: + ncfile.createDimension("time", None) + ncfile.createDimension("ntb", 2) + ncfile.createDimension("atmImp_ny", 1) + ncfile.createDimension("atmImp_nx", NX) + ncfile.file_version = "cmeps0" + start = (year - 1) * 365 + day_of_year + time = ncfile.createVariable("time", "f8", ("time",)) + time.setncatts({"units": "days since 0001-01-01 00:00:00", "calendar": "noleap"}) + time[:] = [start + 0.25, start + 0.75] + time_bnds = ncfile.createVariable("time_bnds", "f8", ("time", "ntb")) + time_bnds.setncatts({"bounds": "time_bnds", "axis": "T", "calendar": "noleap"}) + time_bnds[:] = [[start, start + 0.5], [start + 0.5, start + 1.0]] + dims = ("time", "atmImp_ny", "atmImp_nx") + value = _value(year, month) + for name in dtm.DEFAULT_VARS + [EXTRA_VAR]: + var = ncfile.createVariable(name, "f4", dims, fill_value=1.0e30) + var.setncatts({"units": "kg m-2 s-1", "standard_name": name}) + var[:] = (2.0 * value if name == EXTRA_VAR else value) * np.ones((2, 1, NX)) + for name, values, units in ( + ("atmImp_lon", LON, "degrees_E"), + ("atmImp_lat", LAT, "degrees_N"), + ): + var = ncfile.createVariable(name, "f4", dims, fill_value=1.0e30) + var.setncatts({"long_name": name, "units": units}) + var[:] = values * np.ones((2, 1, NX)) + + +@unittest.skipUnless(shutil.which("ncra"), "NCO not found (on Derecho: module load nco)") +class TestSysDailyToMonthlyClim(unittest.TestCase): + """System tests for daily_to_monthly_clim, running the NCO commands""" + + @classmethod + def setUpClass(cls): + """Write BEGYEAR to ENDYEAR of daily files, once for all the tests""" + cls._indir_root = tempfile.mkdtemp() + cls._archdir = os.path.join(cls._indir_root, "archive") + indir = dtm.input_dir(cls._archdir, CASE) + os.makedirs(indir) + cls._first_daily = None + for year in range(BEGYEAR, ENDYEAR + 1): + day_of_year = 0 + for month, ndays in enumerate(dtm.NOLEAP_DAYS, start=1): + for day in range(1, ndays + 1): + filename = os.path.join( + indir, f"{CASE}.{dtm.HIST_TAG}.{year:04d}-{month:02d}-{day:02d}-00000.nc" + ) + _write_daily_file(filename, year, month, day_of_year) + day_of_year += 1 + if cls._first_daily is None: + cls._first_daily = filename + + @classmethod + def tearDownClass(cls): + shutil.rmtree(cls._indir_root, ignore_errors=True) + + def setUp(self): + self._previous_dir = os.getcwd() + self._tempdir = tempfile.mkdtemp() + os.chdir(self._tempdir) + + def tearDown(self): + os.chdir(self._previous_dir) + shutil.rmtree(self._tempdir, ignore_errors=True) + + def _run(self, *options, outdir=None): + """ + Run main with the options (and 4 ncra jobs), writing under outdir (a new directory under + the test directory by default). Returns the output file name and what was printed. + """ + if outdir is None: + outdir = tempfile.mkdtemp(dir=self._tempdir) + argv = ["daily_to_monthly_clim", "-j", "4", "-o", outdir] + list(options) + argv += [self._archdir, CASE, str(BEGYEAR), str(ENDYEAR)] + output = io.StringIO() + with mock.patch.object(sys, "argv", argv), contextlib.redirect_stdout(output): + dtm.main() + outfile = dtm.output_filename( + dtm.output_dir(self._archdir, outdir, CASE), + CASE, + BEGYEAR, + ENDYEAR, + datetime.date.today(), + ) + self.assertTrue(os.path.exists(outfile), msg=f"{outfile} not created") + return outfile, output.getvalue(), argv + + def _check_output(self, outfile, data_vars, year=dtm.DEFAULT_YEAR): + """Check the variables, values, grid, time axis and metadata of the output file""" + units = f"days since {year:04d}-01-01 00:00:00" + with Dataset(outfile) as ncfile: + expected = set(data_vars) | set(dtm.ALWAYS_VARS) | {"time", "climatology_bounds"} + self.assertEqual(set(ncfile.variables), expected) + # Each month is the average over the years of that month + for name in data_vars: + var = ncfile.variables[name] + self.assertEqual(var.dimensions, ("time", "atmImp_ny", "atmImp_nx")) + for month in range(1, 13): + expect = np.mean([_value(yr, month) for yr in range(BEGYEAR, ENDYEAR + 1)]) + if name == EXTRA_VAR: + expect *= 2.0 + np.testing.assert_allclose(var[month - 1], expect, rtol=1.0e-6) + self.assertEqual(var.cell_methods, "time: mean within years time: mean over years") + self.assertEqual(var.coordinates, "atmImp_lon atmImp_lat") + self.assertNotIn("standard_name", var.ncattrs()) + # The grid without the time dimension + for name, values in (("atmImp_lon", LON), ("atmImp_lat", LAT)): + var = ncfile.variables[name] + self.assertEqual(var.dimensions, ("atmImp_ny", "atmImp_nx")) + np.testing.assert_array_equal(var[0], values) + self.assertNotIn("cell_methods", var.ncattrs()) + # The time axis and the climatology bounds + time = ncfile.variables["time"] + np.testing.assert_array_equal(time[:], dtm.mid_month_times()) + self.assertEqual(time.units, units) + self.assertEqual(time.calendar, "noleap") + self.assertEqual(time.climatology, "climatology_bounds") + np.testing.assert_array_equal( + ncfile.variables["climatology_bounds"][:], + dtm.climatology_bounds_values(BEGYEAR, ENDYEAR, year), + ) + self.assertEqual(ncfile.Conventions, "CF-1.11") + self.assertEqual(ncfile.made_from_cesm_case, CASE) + self.assertNotIn("file_version", ncfile.ncattrs()) + return ncfile.command_used + + def test_defaultVars(self): + """The default variables, with the intermediate files removed at the end""" + outfile, _, argv = self._run() + command_used = self._check_output(outfile, dtm.DEFAULT_VARS) + self.assertEqual(command_used, shlex.join(argv)) + self.assertFalse(os.path.exists(os.path.join(os.path.dirname(outfile), "mon"))) + + def test_varsAndYear(self): + """A variable list (including a grid variable) and the time axis set to another year""" + outfile, _, argv = self._run("--vars", f"{EXTRA_VAR},atmImp_lat", "--year", "2001") + command_used = self._check_output(outfile, [EXTRA_VAR], year=2001) + # The comma in the --vars list makes it through ncatted + self.assertEqual(command_used, shlex.join(argv)) + + def test_allVars(self): + """All variables: time_bnds is renamed to climatology_bounds, lon/lat aren't averaged""" + outfile, _, _ = self._run("--all-vars") + self._check_output(outfile, dtm.DEFAULT_VARS + [EXTRA_VAR]) + + def test_keepAndReuse(self): + """With --keep the monthly files are kept, and reused on a rerun""" + outdir = tempfile.mkdtemp(dir=self._tempdir) + outfile, _, _ = self._run("--keep", outdir=outdir) + mondir = os.path.join(os.path.dirname(outfile), "mon") + nyears = ENDYEAR - BEGYEAR + 1 + self.assertEqual(len(os.listdir(mondir)), 12 * nyears + 12) + os.rename(outfile, os.path.join(self._tempdir, "first.nc")) + outfile, printed, _ = self._run("--keep", outdir=outdir) + self.assertEqual(printed.count("skipping ncra command"), 12 * nyears) + self._check_output(outfile, dtm.DEFAULT_VARS) + + def test_reuseWithDifferentVars_aborts(self): + """Monthly files kept from a run with fewer variables abort a run with all variables""" + outdir = tempfile.mkdtemp(dir=self._tempdir) + outfile, _, _ = self._run("--keep", outdir=outdir) + os.remove(outfile) + with self.assertRaisesRegex(SystemExit, f"is missing variables: {EXTRA_VAR}"): + self._run("--all-vars", outdir=outdir) + + +if __name__ == "__main__": + unit_testing.setup_for_tests() + unittest.main() diff --git a/python/ctsm/test/test_unit_daily_to_monthly_clim.py b/python/ctsm/test/test_unit_daily_to_monthly_clim.py new file mode 100644 index 0000000000..6ab53f9f36 --- /dev/null +++ b/python/ctsm/test/test_unit_daily_to_monthly_clim.py @@ -0,0 +1,705 @@ +#!/usr/bin/env python3 + +"""Unit tests for daily_to_monthly_clim""" + +import datetime +import io +import os +import shlex +import shutil +import subprocess +import sys +import tempfile +import unittest +from unittest import mock + +import numpy as np +from netCDF4 import Dataset # pylint: disable=no-name-in-module + +from ctsm import unit_testing +from ctsm.cplhist import daily_to_monthly_clim as dtm + +# Allow names that pylint doesn't like, because otherwise I find it hard +# to make readable unit test names +# pylint: disable=invalid-name + +CASE = "TC" + + +def _daily_name(case, year, month, day): + """Name of a daily coupler history file""" + return f"{case}.{dtm.HIST_TAG}.{year:04d}-{month:02d}-{day:02d}-00000.nc" + + +def _write_nc(filename, variables, ntime=1): + """ + Write a small netCDF file with the dimensions time (unlimited, ntime records) and n (2), + and the given variables: a dict of name -> (dimensions, attributes) + """ + with Dataset(filename, "w") as ncfile: + ncfile.createDimension("time", None) + ncfile.createDimension("n", 2) + for name, (dims, attrs) in variables.items(): + var = ncfile.createVariable(name, "f8", dims) + var.setncatts(attrs) + shape = tuple(ntime if dim == "time" else 2 for dim in dims) + var[...] = np.ones(shape) + + +class TestGetParser(unittest.TestCase): + """Tests of get_parser""" + + def test_defaults(self): + """The defaults when only the positional arguments are given""" + args = dtm.get_parser().parse_args(["arch", CASE, "2001", "2002"]) + self.assertEqual(args.archdir, "arch") + self.assertEqual(args.case, CASE) + self.assertEqual((args.begyear, args.endyear), (2001, 2002)) + self.assertEqual(args.vars, dtm.DEFAULT_VARS) + self.assertFalse(args.all_vars) + self.assertIsNone(args.outdir) + self.assertEqual(args.year, dtm.DEFAULT_YEAR) + self.assertFalse(args.keep) + self.assertIsNone(args.njobs) + self.assertFalse(args.batch) + self.assertIsNone(args.account) + self.assertEqual(args.queue, dtm.DEFAULT_QUEUE) + self.assertEqual(args.walltime, dtm.DEFAULT_WALLTIME) + + def test_varsSplitsCommaList(self): + """--vars is split on commas, with spaces stripped""" + args = dtm.get_parser().parse_args(["--vars", "a, b,c", "arch", CASE, "1", "2"]) + self.assertEqual(args.vars, ["a", "b", "c"]) + + def test_allVarsWithVars_exits(self): + """--all-vars and --vars can't be used together""" + with mock.patch("sys.stderr", new_callable=io.StringIO): + with self.assertRaises(SystemExit): + dtm.get_parser().parse_args(["-a", "--vars", "a", "arch", CASE, "1", "2"]) + + +class TestCheckArgs(unittest.TestCase): + """Tests of check_args""" + + def setUp(self): + self._testdir = tempfile.mkdtemp() + self._archdir = os.path.join(self._testdir, "archive") + os.makedirs(dtm.input_dir(self._archdir, CASE)) + + def tearDown(self): + shutil.rmtree(self._testdir, ignore_errors=True) + + def _args(self, *options, archdir=None, years=("2001", "2002")): + """Parse the options, with the archive directory, case and years""" + if archdir is None: + archdir = self._archdir + return dtm.get_parser().parse_args(list(options) + [archdir, CASE] + list(years)) + + def test_validArgs(self): + """Valid arguments pass""" + dtm.check_args(self._args()) + + def test_yearOutOfRange(self): + """BEGYEAR, ENDYEAR and YEAR must be from 0 to 9999""" + with self.assertRaisesRegex(SystemExit, "ENDYEAR must be an integer from 0 to 9999"): + dtm.check_args(self._args(years=("2001", "10000"))) + with self.assertRaisesRegex(SystemExit, "YEAR must be an integer from 0 to 9999"): + dtm.check_args(self._args("--year", "10000")) + + def test_begyearAfterEndyear(self): + """BEGYEAR can't be after ENDYEAR""" + with self.assertRaisesRegex(SystemExit, "must not be after ENDYEAR"): + dtm.check_args(self._args(years=("2002", "2001"))) + + def test_njobsZero(self): + """--njobs must be positive""" + with self.assertRaisesRegex(SystemExit, "NJOBS must be a positive integer"): + dtm.check_args(self._args("--njobs", "0")) + + def test_accountWithoutBatch(self): + """--account is only allowed with --batch""" + with self.assertRaisesRegex(SystemExit, "--account is only used with --batch"): + dtm.check_args(self._args("--account", "P1")) + dtm.check_args(self._args("--batch", "--account", "P1")) + + def test_onlyAlwaysVars(self): + """--vars with only the ALWAYS_VARS leaves nothing to average""" + with self.assertRaisesRegex(SystemExit, "No variables to average"): + dtm.check_args(self._args("--vars", ",".join(dtm.ALWAYS_VARS))) + + def test_missingDirectories(self): + """The archive, output and input directories must exist""" + with self.assertRaisesRegex(SystemExit, "Input .* directory does not exist"): + dtm.check_args(self._args(archdir=os.path.join(self._testdir, "nothere"))) + with self.assertRaisesRegex(SystemExit, "Output .* directory does not exist"): + dtm.check_args(self._args("--outdir", os.path.join(self._testdir, "nothere"))) + shutil.rmtree(dtm.input_dir(self._archdir, CASE)) + with self.assertRaisesRegex(SystemExit, "hist directory does not exist"): + dtm.check_args(self._args()) + + def test_outputExists(self): + """Abort if today's output file already exists""" + outdir = dtm.output_dir(self._archdir, None, CASE) + os.makedirs(outdir) + outfile = dtm.output_filename(outdir, CASE, 2001, 2002, datetime.date.today()) + with open(outfile, "w", encoding="utf-8"): + pass + with self.assertRaisesRegex(SystemExit, "already exists"): + dtm.check_args(self._args()) + + +class TestDefaultNjobs(unittest.TestCase): + """Tests of default_njobs""" + + def test_inBatchJob(self): + """Inside a PBS job (PBS_JOBID set) use PBS_NJOBS""" + with mock.patch.dict(os.environ, {"PBS_JOBID": "1234.desched1"}): + self.assertEqual(dtm.default_njobs(), dtm.PBS_NJOBS) + + def test_notInBatchJob(self): + """Outside a PBS job use INTERACTIVE_NJOBS""" + with mock.patch.dict(os.environ): + os.environ.pop("PBS_JOBID", None) + self.assertEqual(dtm.default_njobs(), dtm.INTERACTIVE_NJOBS) + + +class TestVarLists(unittest.TestCase): + """Tests of build_var_list and averaged_vars""" + + def test_buildVarList(self): + """Keeps the order, drops duplicates and empty names, adds the extra variables at the end""" + self.assertEqual(dtm.build_var_list(["b", "a", "", "b"], ["c", "a"]), ["b", "a", "c"]) + + def test_buildVarList_none(self): + """None (all variables) stays None""" + self.assertIsNone(dtm.build_var_list(None, ["c"])) + + def test_averagedVars(self): + """The ALWAYS_VARS are removed from the list to average""" + variables = ["a", dtm.ALWAYS_VARS[0], "", "a", "b"] + self.assertEqual(dtm.averaged_vars(variables), ["a", "b"]) + + def test_averagedVars_none(self): + """None (all variables) stays None""" + self.assertIsNone(dtm.averaged_vars(None)) + + +class TestYearMonths(unittest.TestCase): + """Tests of year_months""" + + def test_twoYears(self): + """Two years in order""" + yms = dtm.year_months(2001, 2002) + self.assertEqual(len(yms), 24) + self.assertEqual(yms[0], (2001, 1)) + self.assertEqual(yms[11], (2001, 12)) + self.assertEqual(yms[12], (2002, 1)) + self.assertEqual(yms[-1], (2002, 12)) + + def test_yearZero(self): + """A single year, including year 0""" + self.assertEqual(dtm.year_months(0, 0), [(0, month) for month in range(1, 13)]) + + +class TestDailyFiles(unittest.TestCase): + """Tests of index_daily_files and check_day_counts""" + + def setUp(self): + self._testdir = tempfile.mkdtemp() + # Empty files are enough, the files are found by name + for month, ndays in enumerate(dtm.NOLEAP_DAYS, start=1): + for day in range(1, ndays + 1): + self._touch(_daily_name(CASE, 2001, month, day)) + # Files that should be ignored: another case, and other kinds of files + self._touch(_daily_name("OTHER", 2001, 1, 1)) + self._touch(f"{CASE}.cpl.hi.2001-01-01-00000.nc") + self._touch(_daily_name(CASE, 2001, 1, 1) + ".bak") + + def tearDown(self): + shutil.rmtree(self._testdir, ignore_errors=True) + + def _touch(self, filename): + """Create an empty file in the test directory""" + with open(os.path.join(self._testdir, filename), "w", encoding="utf-8"): + pass + + def test_indexDailyFiles(self): + """The files are grouped by year and month, sorted, and other files are ignored""" + files_by_month = dtm.index_daily_files(self._testdir, CASE) + self.assertEqual(sorted(files_by_month), dtm.year_months(2001, 2001)) + for (_, month), files in files_by_month.items(): + self.assertEqual(len(files), dtm.NOLEAP_DAYS[month - 1]) + self.assertEqual(files, sorted(files)) + self.assertEqual( + files_by_month[(2001, 1)][0], os.path.join(self._testdir, _daily_name(CASE, 2001, 1, 1)) + ) + + def test_checkDayCounts_complete(self): + """A complete year has no errors""" + files_by_month = dtm.index_daily_files(self._testdir, CASE) + self.assertEqual(dtm.check_day_counts(files_by_month, dtm.year_months(2001, 2001)), []) + + def test_checkDayCounts_missing(self): + """A missing day and a missing year are all reported""" + os.remove(os.path.join(self._testdir, _daily_name(CASE, 2001, 2, 14))) + files_by_month = dtm.index_daily_files(self._testdir, CASE) + errors = dtm.check_day_counts(files_by_month, dtm.year_months(2001, 2002)) + self.assertEqual(len(errors), 13) + self.assertEqual(errors[0], "2001-02 has 27 daily files (expected 28)") + self.assertEqual(errors[1], "2002-01 has 0 daily files (expected 31)") + + +class TestNetcdfVars(unittest.TestCase): + """Tests of file_vars, missing_vars and check_reused_mon_file""" + + def setUp(self): + self._testdir = tempfile.mkdtemp() + self._daily = os.path.join(self._testdir, "daily.nc") + _write_nc( + self._daily, + { + "a": (("time", "n"), {}), + "b": (("time", "n"), {}), + dtm.ALWAYS_VARS[0]: (("time", "n"), {}), + }, + ) + + def tearDown(self): + shutil.rmtree(self._testdir, ignore_errors=True) + + def _monfile(self, names): + """Write a monthly file with the given variables (on time and n)""" + monfile = os.path.join(self._testdir, "mon.nc") + _write_nc(monfile, {name: (("time", "n"), {}) for name in names}) + return monfile + + def test_fileVars(self): + """The set of variable names in the file""" + self.assertEqual(dtm.file_vars(self._daily), {"a", "b", dtm.ALWAYS_VARS[0]}) + + def test_missingVars(self): + """Only the variables not in the file are returned, in order""" + self.assertEqual(dtm.missing_vars(self._daily, ["c", "a", "d"]), ["c", "d"]) + + def test_checkReusedMonFile_ok(self): + """A monthly file with the needed variables passes, with a list or all variables""" + monfile = self._monfile(["a", "b"]) + dtm.check_reused_mon_file(monfile, [self._daily], ["a"]) + dtm.check_reused_mon_file(monfile, [self._daily], None) + + def test_checkReusedMonFile_missing(self): + """A monthly file missing a requested variable aborts""" + monfile = self._monfile(["a"]) + with self.assertRaisesRegex(SystemExit, "is missing variables: c"): + dtm.check_reused_mon_file(monfile, [self._daily], ["a", "c"]) + with self.assertRaisesRegex(SystemExit, "is missing variables: b"): + dtm.check_reused_mon_file(monfile, [self._daily], None) + + def test_checkReusedMonFile_stale(self): + """A monthly file with the ALWAYS_VARS was made by an older version, so aborts""" + monfile = self._monfile(["a", "b", dtm.ALWAYS_VARS[0]]) + with self.assertRaisesRegex(SystemExit, "older version"): + dtm.check_reused_mon_file(monfile, [self._daily], ["a"]) + + +class TestCheckNco(unittest.TestCase): + """Tests of check_nco""" + + def test_missingCommand(self): + """Abort naming the NCO commands that aren't found""" + with mock.patch.object( + dtm.shutil, "which", side_effect=lambda cmd: None if cmd == "ncrcat" else "/bin/x" + ): + with self.assertRaisesRegex(SystemExit, "NCO commands not found: ncrcat "): + dtm.check_nco() + + def test_allFound(self): + """No abort when all the commands are found""" + with mock.patch.object(dtm.shutil, "which", return_value="/bin/x"): + dtm.check_nco() + + +class TestNcraCommand(unittest.TestCase): + """Tests of ncra_command""" + + def setUp(self): + self._testdir = tempfile.mkdtemp() + self._out = os.path.join(self._testdir, "out.nc") + + def tearDown(self): + shutil.rmtree(self._testdir, ignore_errors=True) + + def test_varList(self): + """A variable list gives -v, and exclude is ignored""" + cmd = dtm.ncra_command(["f1", "f2"], self._out, ["a", "b"], exclude=["c"]) + self.assertEqual(cmd, ["ncra", "-O", "--hst", "-v", "a,b", "f1", "f2", "-o", self._out]) + + def test_allVarsExclude(self): + """All variables with exclude gives -x -v""" + cmd = dtm.ncra_command(["f1"], self._out, None, exclude=["c", "d"]) + self.assertEqual(cmd, ["ncra", "-O", "--hst", "-x", "-v", "c,d", "f1", "-o", self._out]) + + def test_allVars(self): + """All variables without exclude averages everything""" + cmd = dtm.ncra_command(["f1"], self._out) + self.assertEqual(cmd, ["ncra", "-O", "--hst", "f1", "-o", self._out]) + + def test_skipExisting(self): + """With skip_existing, an existing output file gives no command""" + expected = ["ncra", "-O", "--hst", "f1", "-o", self._out] + self.assertEqual(dtm.ncra_command(["f1"], self._out, skip_existing=True), expected) + with open(self._out, "w", encoding="utf-8"): + pass + self.assertEqual(dtm.ncra_command(["f1"], self._out, skip_existing=True), []) + self.assertEqual(dtm.ncra_command(["f1"], self._out), expected) + + +class TestRunJobs(unittest.TestCase): + """Tests of run_jobs (with a mock runner instead of running the commands)""" + + def test_allRun(self): + """Every non-empty command is run, with an error message naming its output file""" + runner = mock.MagicMock() + dtm.run_jobs([["a", "o1"], [], ["b", "o2"]], 2, runner=runner) + runner.assert_has_calls( + [ + mock.call(["a", "o1"], "Failed creating o1"), + mock.call(["b", "o2"], "Failed creating o2"), + ], + any_order=True, + ) + self.assertEqual(runner.call_count, 2) + + def test_oneJobInOrder(self): + """With one job at a time the commands run in order""" + runner = mock.MagicMock() + commands = [[f"c{i}", f"o{i}"] for i in range(5)] + dtm.run_jobs(commands, 1, runner=runner) + self.assertEqual([call.args[0] for call in runner.call_args_list], commands) + + def test_noCommands(self): + """Only empty commands runs nothing""" + runner = mock.MagicMock() + dtm.run_jobs([[], []], 4, runner=runner) + runner.assert_not_called() + + def test_failure(self): + """A failing command (the runner aborts) stops with SystemExit""" + + def runner(cmd, errmsg): + if cmd[0] == "bad": + sys.exit(errmsg) + + with self.assertRaisesRegex(SystemExit, "Failed creating o2"): + dtm.run_jobs([["good", "o1"], ["bad", "o2"], ["good", "o3"]], 2, runner=runner) + + +class TestFilenames(unittest.TestCase): + """Tests of output_filename, mon_file and clim_file""" + + def test_outputFilename(self): + """Zero-padded years and the creation date""" + self.assertEqual( + dtm.output_filename("/out", CASE, 100, 130, datetime.date(2026, 10, 3)), + os.path.join("/out", f"{CASE}.cpl.hx.atm.monclim.0100-0130_c20261003.nc"), + ) + + def test_intermediateFiles(self): + """Names of the monthly and climatology files""" + self.assertEqual(dtm.mon_file("/m", 100, 2), os.path.join("/m", "mon_0100-02.nc")) + self.assertEqual(dtm.clim_file("/m", 2), os.path.join("/m", "clim_02.nc")) + + +class TestTimeValues(unittest.TestCase): + """Tests of month_bounds, mid_month_times and climatology_bounds_values""" + + def test_monthBounds(self): + """Start and end of each month of a noleap year""" + bounds = dtm.month_bounds() + self.assertEqual(len(bounds), 12) + self.assertEqual(bounds[0], (0, 31)) + self.assertEqual(bounds[1], (31, 59)) + self.assertEqual(bounds[-1], (334, 365)) + + def test_midMonthTimes(self): + """Middle of each month""" + times = dtm.mid_month_times() + self.assertEqual(times[0], 15.5) + self.assertEqual(times[1], 45.0) + self.assertEqual(times[-1], 349.5) + + def test_climatologyBounds_singleYear(self): + """Averaging only YEAR gives the bounds of each month of YEAR""" + self.assertEqual(dtm.climatology_bounds_values(1850, 1850, 1850), dtm.month_bounds()) + + def test_climatologyBounds_span(self): + """The bounds go from the month in BEGYEAR to the month in ENDYEAR, negative before YEAR""" + bounds = dtm.climatology_bounds_values(100, 130, 1850) + self.assertEqual(bounds[0], ((100 - 1850) * 365, (130 - 1850) * 365 + 31)) + # December ends at the start of ENDYEAR+1 + self.assertEqual(bounds[-1][1], (131 - 1850) * 365) + self.assertTrue(all(start < 0 and end < 0 for start, end in bounds)) + + +class TestSetTimeAxis(unittest.TestCase): + """Tests of set_time_axis""" + + def setUp(self): + self._testdir = tempfile.mkdtemp() + self._file = os.path.join(self._testdir, "clim.nc") + + def tearDown(self): + shutil.rmtree(self._testdir, ignore_errors=True) + + def _write_clim(self, with_time_bnds): + """Write a 12-record file like the one from ncrcat, optionally with time_bnds""" + with Dataset(self._file, "w") as ncfile: + ncfile.createDimension("time", None) + time = ncfile.createVariable("time", "f8", ("time",)) + time.setncatts({"units": "days since 0001-01-01", "bounds": "time_bnds"}) + time.setncattr("cell_methods", "time: mean") + time[:] = np.arange(12.0) + if with_time_bnds: + ncfile.createDimension("ntb", 2) + time_bnds = ncfile.createVariable("time_bnds", "f8", ("time", "ntb")) + time_bnds.setncatts({"axis": "T", "calendar": "noleap", "bounds": "time_bnds"}) + time_bnds.setncattr("cell_methods", "time: mean") + time_bnds[:] = np.zeros((12, 2)) + + def _check(self): + """Check the time axis and climatology bounds after set_time_axis""" + dtm.set_time_axis(self._file, 1850, 2001, 2002) + units = "days since 1850-01-01 00:00:00" + with Dataset(self._file) as ncfile: + self.assertNotIn("time_bnds", ncfile.variables) + time = ncfile.variables["time"] + np.testing.assert_array_equal(time[:], dtm.mid_month_times()) + self.assertEqual(time.units, units) + self.assertEqual(time.calendar, "noleap") + self.assertEqual(time.climatology, "climatology_bounds") + self.assertNotIn("bounds", time.ncattrs()) + self.assertNotIn("cell_methods", time.ncattrs()) + bounds = ncfile.variables["climatology_bounds"] + self.assertEqual(bounds.dimensions, ("time", "ntb")) + np.testing.assert_array_equal( + bounds[:], dtm.climatology_bounds_values(2001, 2002, 1850) + ) + self.assertEqual(bounds.units, units) + for attr in ("axis", "calendar", "bounds", "cell_methods"): + self.assertNotIn(attr, bounds.ncattrs()) + + def test_withTimeBnds(self): + """time_bnds is renamed to climatology_bounds""" + self._write_clim(with_time_bnds=True) + self._check() + + def test_withoutTimeBnds(self): + """climatology_bounds is created when there is no time_bnds""" + self._write_clim(with_time_bnds=False) + self._check() + + +class TestCopyGridVars(unittest.TestCase): + """Tests of copy_grid_vars""" + + def setUp(self): + self._testdir = tempfile.mkdtemp() + self._src = os.path.join(self._testdir, "daily.nc") + self._dst = os.path.join(self._testdir, "clim.nc") + # Like CMEPS: float lon/lat on (time, ny, nx) with a fill value and attributes + with Dataset(self._src, "w") as ncfile: + ncfile.createDimension("time", None) + ncfile.createDimension("atmImp_ny", 1) + ncfile.createDimension("atmImp_nx", 3) + for name, base in (("atmImp_lon", 10.0), ("atmImp_lat", -60.0)): + var = ncfile.createVariable( + name, "f4", ("time", "atmImp_ny", "atmImp_nx"), fill_value=1.0e30 + ) + var.setncatts({"long_name": name, "units": "degrees"}) + var.setncattr("cell_methods", "time: mean") + # Different values for the second time, so the first time slice can be checked + var[0] = base + np.arange(3.0).reshape(1, 3) + var[1] = -999.0 + # The destination has atmImp_ny but not atmImp_nx + with Dataset(self._dst, "w") as ncfile: + ncfile.createDimension("time", None) + ncfile.createDimension("atmImp_ny", 1) + + def tearDown(self): + shutil.rmtree(self._testdir, ignore_errors=True) + + def test_copyGridVars(self): + """Copied without time, with the first time slice, attributes and fill value""" + dtm.copy_grid_vars(self._src, self._dst, ["atmImp_lon", "atmImp_lat"]) + with Dataset(self._dst) as ncfile: + self.assertEqual(len(ncfile.dimensions["atmImp_nx"]), 3) + for name, base in (("atmImp_lon", 10.0), ("atmImp_lat", -60.0)): + var = ncfile.variables[name] + self.assertEqual(var.dimensions, ("atmImp_ny", "atmImp_nx")) + self.assertEqual(var.dtype, np.float32) + self.assertAlmostEqual(float(var.getncattr("_FillValue")), 1.0e30, delta=1.0e24) + self.assertEqual(var.long_name, name) + self.assertEqual(var.units, "degrees") + self.assertNotIn("cell_methods", var.ncattrs()) + np.testing.assert_array_equal(var[:], base + np.arange(3.0).reshape(1, 3)) + + +class TestAddVarAttributes(unittest.TestCase): + """Tests of add_var_attributes""" + + def setUp(self): + self._testdir = tempfile.mkdtemp() + self._file = os.path.join(self._testdir, "clim.nc") + lon, lat = dtm.ALWAYS_VARS + _write_nc( + self._file, + { + "time": (("time",), {}), + "climatology_bounds": (("time", "n"), {}), + lon: (("n",), {"cell_methods": "time: mean"}), + lat: (("n",), {}), + "a": (("time", "n"), {"standard_name": "a", "cell_methods": "time: mean"}), + "b": (("time", "n"), {"long_name": "Keep this name"}), + "c": (("n",), {}), + }, + ) + + def tearDown(self): + shutil.rmtree(self._testdir, ignore_errors=True) + + def test_addVarAttributes(self): + """Only the data variables on time get the climatology attributes""" + dtm.add_var_attributes(self._file) + cell_methods = "time: mean within years time: mean over years" + with Dataset(self._file) as ncfile: + for name in ("a", "b"): + var = ncfile.variables[name] + self.assertEqual(var.cell_methods, cell_methods) + self.assertEqual(var.coordinates, " ".join(dtm.ALWAYS_VARS)) + self.assertNotIn("standard_name", var.ncattrs()) + self.assertEqual(ncfile.variables["a"].long_name, "a") + self.assertEqual(ncfile.variables["b"].long_name, "Keep this name") + for name in ["time", "climatology_bounds", "c"] + dtm.ALWAYS_VARS: + self.assertNotIn("cell_methods", ncfile.variables[name].ncattrs()) + self.assertNotIn("coordinates", ncfile.variables[name].ncattrs()) + + +class TestWriteProvenance(unittest.TestCase): + """Tests of write_provenance (with the ncatted command mocked)""" + + def test_oneNcattedCommand(self): + """One ncatted --hst command sets the global attributes and deletes file_version""" + with mock.patch.object(dtm, "run_cmd_output_on_error") as run_cmd, mock.patch.object( + dtm, "get_ctsm_git_short_hash", return_value="abc1234" + ), mock.patch.object(sys, "argv", ["daily_to_monthly_clim", "--vars", "a,b", "x"]): + dtm.write_provenance(CASE, "out.nc", 100, 130, 1850, datetime.date(2026, 10, 3)) + run_cmd.assert_called_once() + cmd = run_cmd.call_args.args[0] + self.assertEqual(cmd[:3], ["ncatted", "-O", "--hst"]) + self.assertEqual(cmd[-1], "out.nc") + attrs = cmd[4:-1:2] + self.assertEqual(cmd[3:-1:2], ["-a"] * len(attrs)) + self.assertIn("Conventions,global,o,c,CF-1.11", attrs) + self.assertIn("Created_on,global,o,c,2026-10-03", attrs) + self.assertIn("Created_with,global,o,c,./daily_to_monthly_clim -- abc1234", attrs) + self.assertIn("command_used,global,o,c,daily_to_monthly_clim --vars a,b x", attrs) + self.assertIn(f"made_from_cesm_case,global,o,c,{CASE}", attrs) + self.assertIn("file_version,global,d,,", attrs) + title = [attr for attr in attrs if attr.startswith("title,")] + self.assertTrue(title[0].endswith("for cyclical year 1850")) + + +class TestBatch(unittest.TestCase): + """Tests of job_args, batch_script and submit_batch""" + + def test_jobArgs_default(self): + """Only the positional arguments when the defaults are used""" + args = dtm.get_parser().parse_args(["arch", CASE, "2001", "2002"]) + self.assertEqual(dtm.job_args(args), ["arch", CASE, "2001", "2002"]) + + def test_jobArgs_options(self): + """The options given are passed on, but not the batch options""" + args = dtm.get_parser().parse_args( + [ + "--batch", + "--account", + "P1", + "--queue", + "develop", + "--walltime", + "01:00:00", + "--vars", + "a,b", + "-o", + "out", + "-y", + "2000", + "-k", + "-j", + "8", + "-v", + "arch", + CASE, + "2001", + "2002", + ] + ) + self.assertEqual( + dtm.job_args(args), + ["arch", CASE, "2001", "2002", "--vars", "a,b", "--outdir", "out", "--year", "2000"] + + ["--keep", "--njobs", "8", "--verbose"], + ) + + def test_jobArgs_allVars(self): + """--all-vars is passed on""" + args = dtm.get_parser().parse_args(["-a", "arch", CASE, "2001", "2002"]) + self.assertEqual(dtm.job_args(args), ["arch", CASE, "2001", "2002", "--all-vars"]) + + def test_batchScript(self): + """The PBS settings, NCO, the directory and the quoted command line""" + argv = ["arch", CASE, "2001", "2002", "--vars", "a,b", "--outdir", "my dir"] + script = dtm.batch_script( + argv, account="P1", walltime="01:00:00", workdir="/work dir", njobs=8 + ) + lines = script.splitlines() + self.assertEqual(lines[0], "#!/bin/bash") + self.assertIn(f"#PBS -N {dtm.PBS_JOBNAME}", lines) + self.assertIn("#PBS -A P1", lines) + self.assertIn("#PBS -l walltime=01:00:00", lines) + self.assertIn("mpiprocs=8", script) + self.assertIn("module load nco", lines) + self.assertIn("cd '/work dir'", lines) + command = shlex.split(lines[-1]) + self.assertEqual(command[0], sys.executable) + self.assertTrue(command[1].endswith(dtm.WRAPPER)) + self.assertEqual(command[2:], argv) + + def test_submitBatch(self): + """qsub gets the queue and the script on stdin, and the job id is returned""" + result = subprocess.CompletedProcess([], 0, stdout="1234.desched1\n", stderr="") + with mock.patch.object(dtm.shutil, "which", return_value="/bin/qsub"), mock.patch.object( + dtm.subprocess, "run", return_value=result + ) as run: + self.assertEqual(dtm.submit_batch("the script", "develop"), "1234.desched1") + self.assertEqual(run.call_args.args[0], ["qsub", "-q", "develop"]) + self.assertEqual(run.call_args.kwargs["input"], "the script") + + def test_submitBatch_fails(self): + """A qsub error aborts""" + result = subprocess.CompletedProcess([], 1, stdout="", stderr="qsub: bad account") + with mock.patch.object(dtm.shutil, "which", return_value="/bin/qsub"), mock.patch.object( + dtm.subprocess, "run", return_value=result + ): + with self.assertRaisesRegex(SystemExit, "qsub failed:\nqsub: bad account"): + dtm.submit_batch("the script", "main") + + def test_submitBatch_noQsub(self): + """Abort if qsub isn't found""" + with mock.patch.object(dtm.shutil, "which", return_value=None): + with self.assertRaisesRegex(SystemExit, "qsub not found"): + dtm.submit_batch("the script", "main") + + +if __name__ == "__main__": + unit_testing.setup_for_tests() + unittest.main() From 6e5ecf464ac391fb90c5ead6b8c6b9e1dc8c18f5 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Sun, 25 Oct 2026 13:06:01 -0600 Subject: [PATCH 26/37] Change short local name from dtm to d2mc for daily_to_monthly_climatology --- .../test/test_unit_daily_to_monthly_clim.py | 202 +++++++++--------- 1 file changed, 101 insertions(+), 101 deletions(-) diff --git a/python/ctsm/test/test_unit_daily_to_monthly_clim.py b/python/ctsm/test/test_unit_daily_to_monthly_clim.py index 6ab53f9f36..c5e5941403 100644 --- a/python/ctsm/test/test_unit_daily_to_monthly_clim.py +++ b/python/ctsm/test/test_unit_daily_to_monthly_clim.py @@ -17,7 +17,7 @@ from netCDF4 import Dataset # pylint: disable=no-name-in-module from ctsm import unit_testing -from ctsm.cplhist import daily_to_monthly_clim as dtm +from ctsm.cplhist import daily_to_monthly_clim as d2mc # Allow names that pylint doesn't like, because otherwise I find it hard # to make readable unit test names @@ -28,7 +28,7 @@ def _daily_name(case, year, month, day): """Name of a daily coupler history file""" - return f"{case}.{dtm.HIST_TAG}.{year:04d}-{month:02d}-{day:02d}-00000.nc" + return f"{case}.{d2mc.HIST_TAG}.{year:04d}-{month:02d}-{day:02d}-00000.nc" def _write_nc(filename, variables, ntime=1): @@ -51,31 +51,31 @@ class TestGetParser(unittest.TestCase): def test_defaults(self): """The defaults when only the positional arguments are given""" - args = dtm.get_parser().parse_args(["arch", CASE, "2001", "2002"]) + args = d2mc.get_parser().parse_args(["arch", CASE, "2001", "2002"]) self.assertEqual(args.archdir, "arch") self.assertEqual(args.case, CASE) self.assertEqual((args.begyear, args.endyear), (2001, 2002)) - self.assertEqual(args.vars, dtm.DEFAULT_VARS) + self.assertEqual(args.vars, d2mc.DEFAULT_VARS) self.assertFalse(args.all_vars) self.assertIsNone(args.outdir) - self.assertEqual(args.year, dtm.DEFAULT_YEAR) + self.assertEqual(args.year, d2mc.DEFAULT_YEAR) self.assertFalse(args.keep) self.assertIsNone(args.njobs) self.assertFalse(args.batch) self.assertIsNone(args.account) - self.assertEqual(args.queue, dtm.DEFAULT_QUEUE) - self.assertEqual(args.walltime, dtm.DEFAULT_WALLTIME) + self.assertEqual(args.queue, d2mc.DEFAULT_QUEUE) + self.assertEqual(args.walltime, d2mc.DEFAULT_WALLTIME) def test_varsSplitsCommaList(self): """--vars is split on commas, with spaces stripped""" - args = dtm.get_parser().parse_args(["--vars", "a, b,c", "arch", CASE, "1", "2"]) + args = d2mc.get_parser().parse_args(["--vars", "a, b,c", "arch", CASE, "1", "2"]) self.assertEqual(args.vars, ["a", "b", "c"]) def test_allVarsWithVars_exits(self): """--all-vars and --vars can't be used together""" with mock.patch("sys.stderr", new_callable=io.StringIO): with self.assertRaises(SystemExit): - dtm.get_parser().parse_args(["-a", "--vars", "a", "arch", CASE, "1", "2"]) + d2mc.get_parser().parse_args(["-a", "--vars", "a", "arch", CASE, "1", "2"]) class TestCheckArgs(unittest.TestCase): @@ -84,7 +84,7 @@ class TestCheckArgs(unittest.TestCase): def setUp(self): self._testdir = tempfile.mkdtemp() self._archdir = os.path.join(self._testdir, "archive") - os.makedirs(dtm.input_dir(self._archdir, CASE)) + os.makedirs(d2mc.input_dir(self._archdir, CASE)) def tearDown(self): shutil.rmtree(self._testdir, ignore_errors=True) @@ -93,59 +93,59 @@ def _args(self, *options, archdir=None, years=("2001", "2002")): """Parse the options, with the archive directory, case and years""" if archdir is None: archdir = self._archdir - return dtm.get_parser().parse_args(list(options) + [archdir, CASE] + list(years)) + return d2mc.get_parser().parse_args(list(options) + [archdir, CASE] + list(years)) def test_validArgs(self): """Valid arguments pass""" - dtm.check_args(self._args()) + d2mc.check_args(self._args()) def test_yearOutOfRange(self): """BEGYEAR, ENDYEAR and YEAR must be from 0 to 9999""" with self.assertRaisesRegex(SystemExit, "ENDYEAR must be an integer from 0 to 9999"): - dtm.check_args(self._args(years=("2001", "10000"))) + d2mc.check_args(self._args(years=("2001", "10000"))) with self.assertRaisesRegex(SystemExit, "YEAR must be an integer from 0 to 9999"): - dtm.check_args(self._args("--year", "10000")) + d2mc.check_args(self._args("--year", "10000")) def test_begyearAfterEndyear(self): """BEGYEAR can't be after ENDYEAR""" with self.assertRaisesRegex(SystemExit, "must not be after ENDYEAR"): - dtm.check_args(self._args(years=("2002", "2001"))) + d2mc.check_args(self._args(years=("2002", "2001"))) def test_njobsZero(self): """--njobs must be positive""" with self.assertRaisesRegex(SystemExit, "NJOBS must be a positive integer"): - dtm.check_args(self._args("--njobs", "0")) + d2mc.check_args(self._args("--njobs", "0")) def test_accountWithoutBatch(self): """--account is only allowed with --batch""" with self.assertRaisesRegex(SystemExit, "--account is only used with --batch"): - dtm.check_args(self._args("--account", "P1")) - dtm.check_args(self._args("--batch", "--account", "P1")) + d2mc.check_args(self._args("--account", "P1")) + d2mc.check_args(self._args("--batch", "--account", "P1")) def test_onlyAlwaysVars(self): """--vars with only the ALWAYS_VARS leaves nothing to average""" with self.assertRaisesRegex(SystemExit, "No variables to average"): - dtm.check_args(self._args("--vars", ",".join(dtm.ALWAYS_VARS))) + d2mc.check_args(self._args("--vars", ",".join(d2mc.ALWAYS_VARS))) def test_missingDirectories(self): """The archive, output and input directories must exist""" with self.assertRaisesRegex(SystemExit, "Input .* directory does not exist"): - dtm.check_args(self._args(archdir=os.path.join(self._testdir, "nothere"))) + d2mc.check_args(self._args(archdir=os.path.join(self._testdir, "nothere"))) with self.assertRaisesRegex(SystemExit, "Output .* directory does not exist"): - dtm.check_args(self._args("--outdir", os.path.join(self._testdir, "nothere"))) - shutil.rmtree(dtm.input_dir(self._archdir, CASE)) + d2mc.check_args(self._args("--outdir", os.path.join(self._testdir, "nothere"))) + shutil.rmtree(d2mc.input_dir(self._archdir, CASE)) with self.assertRaisesRegex(SystemExit, "hist directory does not exist"): - dtm.check_args(self._args()) + d2mc.check_args(self._args()) def test_outputExists(self): """Abort if today's output file already exists""" - outdir = dtm.output_dir(self._archdir, None, CASE) + outdir = d2mc.output_dir(self._archdir, None, CASE) os.makedirs(outdir) - outfile = dtm.output_filename(outdir, CASE, 2001, 2002, datetime.date.today()) + outfile = d2mc.output_filename(outdir, CASE, 2001, 2002, datetime.date.today()) with open(outfile, "w", encoding="utf-8"): pass with self.assertRaisesRegex(SystemExit, "already exists"): - dtm.check_args(self._args()) + d2mc.check_args(self._args()) class TestDefaultNjobs(unittest.TestCase): @@ -154,13 +154,13 @@ class TestDefaultNjobs(unittest.TestCase): def test_inBatchJob(self): """Inside a PBS job (PBS_JOBID set) use PBS_NJOBS""" with mock.patch.dict(os.environ, {"PBS_JOBID": "1234.desched1"}): - self.assertEqual(dtm.default_njobs(), dtm.PBS_NJOBS) + self.assertEqual(d2mc.default_njobs(), d2mc.PBS_NJOBS) def test_notInBatchJob(self): """Outside a PBS job use INTERACTIVE_NJOBS""" with mock.patch.dict(os.environ): os.environ.pop("PBS_JOBID", None) - self.assertEqual(dtm.default_njobs(), dtm.INTERACTIVE_NJOBS) + self.assertEqual(d2mc.default_njobs(), d2mc.INTERACTIVE_NJOBS) class TestVarLists(unittest.TestCase): @@ -168,20 +168,20 @@ class TestVarLists(unittest.TestCase): def test_buildVarList(self): """Keeps the order, drops duplicates and empty names, adds the extra variables at the end""" - self.assertEqual(dtm.build_var_list(["b", "a", "", "b"], ["c", "a"]), ["b", "a", "c"]) + self.assertEqual(d2mc.build_var_list(["b", "a", "", "b"], ["c", "a"]), ["b", "a", "c"]) def test_buildVarList_none(self): """None (all variables) stays None""" - self.assertIsNone(dtm.build_var_list(None, ["c"])) + self.assertIsNone(d2mc.build_var_list(None, ["c"])) def test_averagedVars(self): """The ALWAYS_VARS are removed from the list to average""" - variables = ["a", dtm.ALWAYS_VARS[0], "", "a", "b"] - self.assertEqual(dtm.averaged_vars(variables), ["a", "b"]) + variables = ["a", d2mc.ALWAYS_VARS[0], "", "a", "b"] + self.assertEqual(d2mc.averaged_vars(variables), ["a", "b"]) def test_averagedVars_none(self): """None (all variables) stays None""" - self.assertIsNone(dtm.averaged_vars(None)) + self.assertIsNone(d2mc.averaged_vars(None)) class TestYearMonths(unittest.TestCase): @@ -189,7 +189,7 @@ class TestYearMonths(unittest.TestCase): def test_twoYears(self): """Two years in order""" - yms = dtm.year_months(2001, 2002) + yms = d2mc.year_months(2001, 2002) self.assertEqual(len(yms), 24) self.assertEqual(yms[0], (2001, 1)) self.assertEqual(yms[11], (2001, 12)) @@ -198,7 +198,7 @@ def test_twoYears(self): def test_yearZero(self): """A single year, including year 0""" - self.assertEqual(dtm.year_months(0, 0), [(0, month) for month in range(1, 13)]) + self.assertEqual(d2mc.year_months(0, 0), [(0, month) for month in range(1, 13)]) class TestDailyFiles(unittest.TestCase): @@ -207,7 +207,7 @@ class TestDailyFiles(unittest.TestCase): def setUp(self): self._testdir = tempfile.mkdtemp() # Empty files are enough, the files are found by name - for month, ndays in enumerate(dtm.NOLEAP_DAYS, start=1): + for month, ndays in enumerate(d2mc.NOLEAP_DAYS, start=1): for day in range(1, ndays + 1): self._touch(_daily_name(CASE, 2001, month, day)) # Files that should be ignored: another case, and other kinds of files @@ -225,10 +225,10 @@ def _touch(self, filename): def test_indexDailyFiles(self): """The files are grouped by year and month, sorted, and other files are ignored""" - files_by_month = dtm.index_daily_files(self._testdir, CASE) - self.assertEqual(sorted(files_by_month), dtm.year_months(2001, 2001)) + files_by_month = d2mc.index_daily_files(self._testdir, CASE) + self.assertEqual(sorted(files_by_month), d2mc.year_months(2001, 2001)) for (_, month), files in files_by_month.items(): - self.assertEqual(len(files), dtm.NOLEAP_DAYS[month - 1]) + self.assertEqual(len(files), d2mc.NOLEAP_DAYS[month - 1]) self.assertEqual(files, sorted(files)) self.assertEqual( files_by_month[(2001, 1)][0], os.path.join(self._testdir, _daily_name(CASE, 2001, 1, 1)) @@ -236,14 +236,14 @@ def test_indexDailyFiles(self): def test_checkDayCounts_complete(self): """A complete year has no errors""" - files_by_month = dtm.index_daily_files(self._testdir, CASE) - self.assertEqual(dtm.check_day_counts(files_by_month, dtm.year_months(2001, 2001)), []) + files_by_month = d2mc.index_daily_files(self._testdir, CASE) + self.assertEqual(d2mc.check_day_counts(files_by_month, d2mc.year_months(2001, 2001)), []) def test_checkDayCounts_missing(self): """A missing day and a missing year are all reported""" os.remove(os.path.join(self._testdir, _daily_name(CASE, 2001, 2, 14))) - files_by_month = dtm.index_daily_files(self._testdir, CASE) - errors = dtm.check_day_counts(files_by_month, dtm.year_months(2001, 2002)) + files_by_month = d2mc.index_daily_files(self._testdir, CASE) + errors = d2mc.check_day_counts(files_by_month, d2mc.year_months(2001, 2002)) self.assertEqual(len(errors), 13) self.assertEqual(errors[0], "2001-02 has 27 daily files (expected 28)") self.assertEqual(errors[1], "2002-01 has 0 daily files (expected 31)") @@ -260,7 +260,7 @@ def setUp(self): { "a": (("time", "n"), {}), "b": (("time", "n"), {}), - dtm.ALWAYS_VARS[0]: (("time", "n"), {}), + d2mc.ALWAYS_VARS[0]: (("time", "n"), {}), }, ) @@ -275,31 +275,31 @@ def _monfile(self, names): def test_fileVars(self): """The set of variable names in the file""" - self.assertEqual(dtm.file_vars(self._daily), {"a", "b", dtm.ALWAYS_VARS[0]}) + self.assertEqual(d2mc.file_vars(self._daily), {"a", "b", d2mc.ALWAYS_VARS[0]}) def test_missingVars(self): """Only the variables not in the file are returned, in order""" - self.assertEqual(dtm.missing_vars(self._daily, ["c", "a", "d"]), ["c", "d"]) + self.assertEqual(d2mc.missing_vars(self._daily, ["c", "a", "d"]), ["c", "d"]) def test_checkReusedMonFile_ok(self): """A monthly file with the needed variables passes, with a list or all variables""" monfile = self._monfile(["a", "b"]) - dtm.check_reused_mon_file(monfile, [self._daily], ["a"]) - dtm.check_reused_mon_file(monfile, [self._daily], None) + d2mc.check_reused_mon_file(monfile, [self._daily], ["a"]) + d2mc.check_reused_mon_file(monfile, [self._daily], None) def test_checkReusedMonFile_missing(self): """A monthly file missing a requested variable aborts""" monfile = self._monfile(["a"]) with self.assertRaisesRegex(SystemExit, "is missing variables: c"): - dtm.check_reused_mon_file(monfile, [self._daily], ["a", "c"]) + d2mc.check_reused_mon_file(monfile, [self._daily], ["a", "c"]) with self.assertRaisesRegex(SystemExit, "is missing variables: b"): - dtm.check_reused_mon_file(monfile, [self._daily], None) + d2mc.check_reused_mon_file(monfile, [self._daily], None) def test_checkReusedMonFile_stale(self): """A monthly file with the ALWAYS_VARS was made by an older version, so aborts""" - monfile = self._monfile(["a", "b", dtm.ALWAYS_VARS[0]]) + monfile = self._monfile(["a", "b", d2mc.ALWAYS_VARS[0]]) with self.assertRaisesRegex(SystemExit, "older version"): - dtm.check_reused_mon_file(monfile, [self._daily], ["a"]) + d2mc.check_reused_mon_file(monfile, [self._daily], ["a"]) class TestCheckNco(unittest.TestCase): @@ -308,15 +308,15 @@ class TestCheckNco(unittest.TestCase): def test_missingCommand(self): """Abort naming the NCO commands that aren't found""" with mock.patch.object( - dtm.shutil, "which", side_effect=lambda cmd: None if cmd == "ncrcat" else "/bin/x" + d2mc.shutil, "which", side_effect=lambda cmd: None if cmd == "ncrcat" else "/bin/x" ): with self.assertRaisesRegex(SystemExit, "NCO commands not found: ncrcat "): - dtm.check_nco() + d2mc.check_nco() def test_allFound(self): """No abort when all the commands are found""" - with mock.patch.object(dtm.shutil, "which", return_value="/bin/x"): - dtm.check_nco() + with mock.patch.object(d2mc.shutil, "which", return_value="/bin/x"): + d2mc.check_nco() class TestNcraCommand(unittest.TestCase): @@ -331,27 +331,27 @@ def tearDown(self): def test_varList(self): """A variable list gives -v, and exclude is ignored""" - cmd = dtm.ncra_command(["f1", "f2"], self._out, ["a", "b"], exclude=["c"]) + cmd = d2mc.ncra_command(["f1", "f2"], self._out, ["a", "b"], exclude=["c"]) self.assertEqual(cmd, ["ncra", "-O", "--hst", "-v", "a,b", "f1", "f2", "-o", self._out]) def test_allVarsExclude(self): """All variables with exclude gives -x -v""" - cmd = dtm.ncra_command(["f1"], self._out, None, exclude=["c", "d"]) + cmd = d2mc.ncra_command(["f1"], self._out, None, exclude=["c", "d"]) self.assertEqual(cmd, ["ncra", "-O", "--hst", "-x", "-v", "c,d", "f1", "-o", self._out]) def test_allVars(self): """All variables without exclude averages everything""" - cmd = dtm.ncra_command(["f1"], self._out) + cmd = d2mc.ncra_command(["f1"], self._out) self.assertEqual(cmd, ["ncra", "-O", "--hst", "f1", "-o", self._out]) def test_skipExisting(self): """With skip_existing, an existing output file gives no command""" expected = ["ncra", "-O", "--hst", "f1", "-o", self._out] - self.assertEqual(dtm.ncra_command(["f1"], self._out, skip_existing=True), expected) + self.assertEqual(d2mc.ncra_command(["f1"], self._out, skip_existing=True), expected) with open(self._out, "w", encoding="utf-8"): pass - self.assertEqual(dtm.ncra_command(["f1"], self._out, skip_existing=True), []) - self.assertEqual(dtm.ncra_command(["f1"], self._out), expected) + self.assertEqual(d2mc.ncra_command(["f1"], self._out, skip_existing=True), []) + self.assertEqual(d2mc.ncra_command(["f1"], self._out), expected) class TestRunJobs(unittest.TestCase): @@ -360,7 +360,7 @@ class TestRunJobs(unittest.TestCase): def test_allRun(self): """Every non-empty command is run, with an error message naming its output file""" runner = mock.MagicMock() - dtm.run_jobs([["a", "o1"], [], ["b", "o2"]], 2, runner=runner) + d2mc.run_jobs([["a", "o1"], [], ["b", "o2"]], 2, runner=runner) runner.assert_has_calls( [ mock.call(["a", "o1"], "Failed creating o1"), @@ -374,13 +374,13 @@ def test_oneJobInOrder(self): """With one job at a time the commands run in order""" runner = mock.MagicMock() commands = [[f"c{i}", f"o{i}"] for i in range(5)] - dtm.run_jobs(commands, 1, runner=runner) + d2mc.run_jobs(commands, 1, runner=runner) self.assertEqual([call.args[0] for call in runner.call_args_list], commands) def test_noCommands(self): """Only empty commands runs nothing""" runner = mock.MagicMock() - dtm.run_jobs([[], []], 4, runner=runner) + d2mc.run_jobs([[], []], 4, runner=runner) runner.assert_not_called() def test_failure(self): @@ -391,7 +391,7 @@ def runner(cmd, errmsg): sys.exit(errmsg) with self.assertRaisesRegex(SystemExit, "Failed creating o2"): - dtm.run_jobs([["good", "o1"], ["bad", "o2"], ["good", "o3"]], 2, runner=runner) + d2mc.run_jobs([["good", "o1"], ["bad", "o2"], ["good", "o3"]], 2, runner=runner) class TestFilenames(unittest.TestCase): @@ -400,14 +400,14 @@ class TestFilenames(unittest.TestCase): def test_outputFilename(self): """Zero-padded years and the creation date""" self.assertEqual( - dtm.output_filename("/out", CASE, 100, 130, datetime.date(2026, 10, 3)), + d2mc.output_filename("/out", CASE, 100, 130, datetime.date(2026, 10, 3)), os.path.join("/out", f"{CASE}.cpl.hx.atm.monclim.0100-0130_c20261003.nc"), ) def test_intermediateFiles(self): """Names of the monthly and climatology files""" - self.assertEqual(dtm.mon_file("/m", 100, 2), os.path.join("/m", "mon_0100-02.nc")) - self.assertEqual(dtm.clim_file("/m", 2), os.path.join("/m", "clim_02.nc")) + self.assertEqual(d2mc.mon_file("/m", 100, 2), os.path.join("/m", "mon_0100-02.nc")) + self.assertEqual(d2mc.clim_file("/m", 2), os.path.join("/m", "clim_02.nc")) class TestTimeValues(unittest.TestCase): @@ -415,7 +415,7 @@ class TestTimeValues(unittest.TestCase): def test_monthBounds(self): """Start and end of each month of a noleap year""" - bounds = dtm.month_bounds() + bounds = d2mc.month_bounds() self.assertEqual(len(bounds), 12) self.assertEqual(bounds[0], (0, 31)) self.assertEqual(bounds[1], (31, 59)) @@ -423,18 +423,18 @@ def test_monthBounds(self): def test_midMonthTimes(self): """Middle of each month""" - times = dtm.mid_month_times() + times = d2mc.mid_month_times() self.assertEqual(times[0], 15.5) self.assertEqual(times[1], 45.0) self.assertEqual(times[-1], 349.5) def test_climatologyBounds_singleYear(self): """Averaging only YEAR gives the bounds of each month of YEAR""" - self.assertEqual(dtm.climatology_bounds_values(1850, 1850, 1850), dtm.month_bounds()) + self.assertEqual(d2mc.climatology_bounds_values(1850, 1850, 1850), d2mc.month_bounds()) def test_climatologyBounds_span(self): """The bounds go from the month in BEGYEAR to the month in ENDYEAR, negative before YEAR""" - bounds = dtm.climatology_bounds_values(100, 130, 1850) + bounds = d2mc.climatology_bounds_values(100, 130, 1850) self.assertEqual(bounds[0], ((100 - 1850) * 365, (130 - 1850) * 365 + 31)) # December ends at the start of ENDYEAR+1 self.assertEqual(bounds[-1][1], (131 - 1850) * 365) @@ -468,12 +468,12 @@ def _write_clim(self, with_time_bnds): def _check(self): """Check the time axis and climatology bounds after set_time_axis""" - dtm.set_time_axis(self._file, 1850, 2001, 2002) + d2mc.set_time_axis(self._file, 1850, 2001, 2002) units = "days since 1850-01-01 00:00:00" with Dataset(self._file) as ncfile: self.assertNotIn("time_bnds", ncfile.variables) time = ncfile.variables["time"] - np.testing.assert_array_equal(time[:], dtm.mid_month_times()) + np.testing.assert_array_equal(time[:], d2mc.mid_month_times()) self.assertEqual(time.units, units) self.assertEqual(time.calendar, "noleap") self.assertEqual(time.climatology, "climatology_bounds") @@ -482,7 +482,7 @@ def _check(self): bounds = ncfile.variables["climatology_bounds"] self.assertEqual(bounds.dimensions, ("time", "ntb")) np.testing.assert_array_equal( - bounds[:], dtm.climatology_bounds_values(2001, 2002, 1850) + bounds[:], d2mc.climatology_bounds_values(2001, 2002, 1850) ) self.assertEqual(bounds.units, units) for attr in ("axis", "calendar", "bounds", "cell_methods"): @@ -530,7 +530,7 @@ def tearDown(self): def test_copyGridVars(self): """Copied without time, with the first time slice, attributes and fill value""" - dtm.copy_grid_vars(self._src, self._dst, ["atmImp_lon", "atmImp_lat"]) + d2mc.copy_grid_vars(self._src, self._dst, ["atmImp_lon", "atmImp_lat"]) with Dataset(self._dst) as ncfile: self.assertEqual(len(ncfile.dimensions["atmImp_nx"]), 3) for name, base in (("atmImp_lon", 10.0), ("atmImp_lat", -60.0)): @@ -550,7 +550,7 @@ class TestAddVarAttributes(unittest.TestCase): def setUp(self): self._testdir = tempfile.mkdtemp() self._file = os.path.join(self._testdir, "clim.nc") - lon, lat = dtm.ALWAYS_VARS + lon, lat = d2mc.ALWAYS_VARS _write_nc( self._file, { @@ -569,17 +569,17 @@ def tearDown(self): def test_addVarAttributes(self): """Only the data variables on time get the climatology attributes""" - dtm.add_var_attributes(self._file) + d2mc.add_var_attributes(self._file) cell_methods = "time: mean within years time: mean over years" with Dataset(self._file) as ncfile: for name in ("a", "b"): var = ncfile.variables[name] self.assertEqual(var.cell_methods, cell_methods) - self.assertEqual(var.coordinates, " ".join(dtm.ALWAYS_VARS)) + self.assertEqual(var.coordinates, " ".join(d2mc.ALWAYS_VARS)) self.assertNotIn("standard_name", var.ncattrs()) self.assertEqual(ncfile.variables["a"].long_name, "a") self.assertEqual(ncfile.variables["b"].long_name, "Keep this name") - for name in ["time", "climatology_bounds", "c"] + dtm.ALWAYS_VARS: + for name in ["time", "climatology_bounds", "c"] + d2mc.ALWAYS_VARS: self.assertNotIn("cell_methods", ncfile.variables[name].ncattrs()) self.assertNotIn("coordinates", ncfile.variables[name].ncattrs()) @@ -589,10 +589,10 @@ class TestWriteProvenance(unittest.TestCase): def test_oneNcattedCommand(self): """One ncatted --hst command sets the global attributes and deletes file_version""" - with mock.patch.object(dtm, "run_cmd_output_on_error") as run_cmd, mock.patch.object( - dtm, "get_ctsm_git_short_hash", return_value="abc1234" + with mock.patch.object(d2mc, "run_cmd_output_on_error") as run_cmd, mock.patch.object( + d2mc, "get_ctsm_git_short_hash", return_value="abc1234" ), mock.patch.object(sys, "argv", ["daily_to_monthly_clim", "--vars", "a,b", "x"]): - dtm.write_provenance(CASE, "out.nc", 100, 130, 1850, datetime.date(2026, 10, 3)) + d2mc.write_provenance(CASE, "out.nc", 100, 130, 1850, datetime.date(2026, 10, 3)) run_cmd.assert_called_once() cmd = run_cmd.call_args.args[0] self.assertEqual(cmd[:3], ["ncatted", "-O", "--hst"]) @@ -614,12 +614,12 @@ class TestBatch(unittest.TestCase): def test_jobArgs_default(self): """Only the positional arguments when the defaults are used""" - args = dtm.get_parser().parse_args(["arch", CASE, "2001", "2002"]) - self.assertEqual(dtm.job_args(args), ["arch", CASE, "2001", "2002"]) + args = d2mc.get_parser().parse_args(["arch", CASE, "2001", "2002"]) + self.assertEqual(d2mc.job_args(args), ["arch", CASE, "2001", "2002"]) def test_jobArgs_options(self): """The options given are passed on, but not the batch options""" - args = dtm.get_parser().parse_args( + args = d2mc.get_parser().parse_args( [ "--batch", "--account", @@ -645,25 +645,25 @@ def test_jobArgs_options(self): ] ) self.assertEqual( - dtm.job_args(args), + d2mc.job_args(args), ["arch", CASE, "2001", "2002", "--vars", "a,b", "--outdir", "out", "--year", "2000"] + ["--keep", "--njobs", "8", "--verbose"], ) def test_jobArgs_allVars(self): """--all-vars is passed on""" - args = dtm.get_parser().parse_args(["-a", "arch", CASE, "2001", "2002"]) - self.assertEqual(dtm.job_args(args), ["arch", CASE, "2001", "2002", "--all-vars"]) + args = d2mc.get_parser().parse_args(["-a", "arch", CASE, "2001", "2002"]) + self.assertEqual(d2mc.job_args(args), ["arch", CASE, "2001", "2002", "--all-vars"]) def test_batchScript(self): """The PBS settings, NCO, the directory and the quoted command line""" argv = ["arch", CASE, "2001", "2002", "--vars", "a,b", "--outdir", "my dir"] - script = dtm.batch_script( + script = d2mc.batch_script( argv, account="P1", walltime="01:00:00", workdir="/work dir", njobs=8 ) lines = script.splitlines() self.assertEqual(lines[0], "#!/bin/bash") - self.assertIn(f"#PBS -N {dtm.PBS_JOBNAME}", lines) + self.assertIn(f"#PBS -N {d2mc.PBS_JOBNAME}", lines) self.assertIn("#PBS -A P1", lines) self.assertIn("#PBS -l walltime=01:00:00", lines) self.assertIn("mpiprocs=8", script) @@ -671,33 +671,33 @@ def test_batchScript(self): self.assertIn("cd '/work dir'", lines) command = shlex.split(lines[-1]) self.assertEqual(command[0], sys.executable) - self.assertTrue(command[1].endswith(dtm.WRAPPER)) + self.assertTrue(command[1].endswith(d2mc.WRAPPER)) self.assertEqual(command[2:], argv) def test_submitBatch(self): """qsub gets the queue and the script on stdin, and the job id is returned""" result = subprocess.CompletedProcess([], 0, stdout="1234.desched1\n", stderr="") - with mock.patch.object(dtm.shutil, "which", return_value="/bin/qsub"), mock.patch.object( - dtm.subprocess, "run", return_value=result + with mock.patch.object(d2mc.shutil, "which", return_value="/bin/qsub"), mock.patch.object( + d2mc.subprocess, "run", return_value=result ) as run: - self.assertEqual(dtm.submit_batch("the script", "develop"), "1234.desched1") + self.assertEqual(d2mc.submit_batch("the script", "develop"), "1234.desched1") self.assertEqual(run.call_args.args[0], ["qsub", "-q", "develop"]) self.assertEqual(run.call_args.kwargs["input"], "the script") def test_submitBatch_fails(self): """A qsub error aborts""" result = subprocess.CompletedProcess([], 1, stdout="", stderr="qsub: bad account") - with mock.patch.object(dtm.shutil, "which", return_value="/bin/qsub"), mock.patch.object( - dtm.subprocess, "run", return_value=result + with mock.patch.object(d2mc.shutil, "which", return_value="/bin/qsub"), mock.patch.object( + d2mc.subprocess, "run", return_value=result ): with self.assertRaisesRegex(SystemExit, "qsub failed:\nqsub: bad account"): - dtm.submit_batch("the script", "main") + d2mc.submit_batch("the script", "main") def test_submitBatch_noQsub(self): """Abort if qsub isn't found""" - with mock.patch.object(dtm.shutil, "which", return_value=None): + with mock.patch.object(d2mc.shutil, "which", return_value=None): with self.assertRaisesRegex(SystemExit, "qsub not found"): - dtm.submit_batch("the script", "main") + d2mc.submit_batch("the script", "main") if __name__ == "__main__": From 1903f6ab78378e7edd0697284c0e4732951cc52c Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Sun, 25 Oct 2026 16:47:44 -0600 Subject: [PATCH 27/37] Clarify some error messages, and add some comments to explain what's happening --- python/ctsm/cplhist/daily_to_monthly_clim.py | 22 ++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/python/ctsm/cplhist/daily_to_monthly_clim.py b/python/ctsm/cplhist/daily_to_monthly_clim.py index 0375204430..ff07ee1bc3 100644 --- a/python/ctsm/cplhist/daily_to_monthly_clim.py +++ b/python/ctsm/cplhist/daily_to_monthly_clim.py @@ -252,12 +252,12 @@ def check_args(args): if not args.all_vars and not averaged_vars(args.vars): abort(f"No variables to average given with --vars (besides {','.join(ALWAYS_VARS)})") if not os.path.isdir(args.archdir): - abort(f"Input {args.archdir} directory does not exist") + abort(f"Input {args.archdir} archive directory does not exist") if args.outdir is not None and not os.path.isdir(args.outdir): - abort(f"Output {args.outdir} directory does not exist") + abort(f"Output {args.outdir} output archive directory does not exist") indir = input_dir(args.archdir, args.case) if not os.path.isdir(indir): - abort(f"{indir} directory does not exist") + abort(f"Input {indir} case directory does not exist") # # Abort if the output file already exists # @@ -425,22 +425,36 @@ def run_jobs(commands, njobs, *, progress_every=None, runner=run_cmd_output_on_e Stops at the first failure: the jobs that have not started are cancelled and the runner's error (normally an abort) is raised. Logs progress every progress_every completed jobs. Empty commands (for output files that already exist) are skipped. + Args: + commands (list): Command to execute and it's arguments + njobs (int): Number of jobs to run simultaneously + process_every (int): Report on the progress by this count + runner (function): python function to call to run the commands + + Returns: + Nothing if commands aren't valid or if it was successful + Raises exception if something failed or the job was killed """ commands = [cmd for cmd in commands if cmd] if not commands: return - ndone = 0 + ndone = 0 # number of jobs that have completed + # concurrent.futures executes up to njobs at the same time, but prepares all of them with concurrent.futures.ThreadPoolExecutor(max_workers=njobs) as executor: futures = [executor.submit(runner, cmd, f"Failed creating {cmd[-1]}") for cmd in commands] try: + # Check which jobs are completed for future in concurrent.futures.as_completed(futures): future.result() ndone += 1 + # Report on the progress if progress_every and ndone % progress_every == 0: log(logger, f" completed {ndone} of {len(commands)}") except BaseException: # Includes KeyboardInterrupt and the SystemExit from abort # Will die if the user hits Ctrl-C or if the batch job is killed + + # Cancel all the future jobs for future in futures: future.cancel() raise From 1e99820f282f990799b73b297629f226a174eba0 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Sun, 25 Oct 2026 18:08:04 -0600 Subject: [PATCH 28/37] Small black commit from last edit --- python/ctsm/cplhist/daily_to_monthly_clim.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/ctsm/cplhist/daily_to_monthly_clim.py b/python/ctsm/cplhist/daily_to_monthly_clim.py index ff07ee1bc3..239434949a 100644 --- a/python/ctsm/cplhist/daily_to_monthly_clim.py +++ b/python/ctsm/cplhist/daily_to_monthly_clim.py @@ -438,7 +438,7 @@ def run_jobs(commands, njobs, *, progress_every=None, runner=run_cmd_output_on_e commands = [cmd for cmd in commands if cmd] if not commands: return - ndone = 0 # number of jobs that have completed + ndone = 0 # number of jobs that have completed # concurrent.futures executes up to njobs at the same time, but prepares all of them with concurrent.futures.ThreadPoolExecutor(max_workers=njobs) as executor: futures = [executor.submit(runner, cmd, f"Failed creating {cmd[-1]}") for cmd in commands] From cb0f4af771aa5314b20d78880f80dd98fa29f544 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Sun, 25 Oct 2026 18:14:28 -0600 Subject: [PATCH 29/37] Got through the new unit test and add comments to explain things better, change some names to be more descriptive, also tweak some error messages, and also disallow year 0, which means the unit test doesn't run --- .../test/test_unit_daily_to_monthly_clim.py | 186 ++++++++++++++---- 1 file changed, 149 insertions(+), 37 deletions(-) diff --git a/python/ctsm/test/test_unit_daily_to_monthly_clim.py b/python/ctsm/test/test_unit_daily_to_monthly_clim.py index c5e5941403..225162b643 100644 --- a/python/ctsm/test/test_unit_daily_to_monthly_clim.py +++ b/python/ctsm/test/test_unit_daily_to_monthly_clim.py @@ -23,9 +23,10 @@ # to make readable unit test names # pylint: disable=invalid-name -CASE = "TC" +CASE = "TESTCASE" +# A couple utility functions for the tests def _daily_name(case, year, month, day): """Name of a daily coupler history file""" return f"{case}.{d2mc.HIST_TAG}.{year:04d}-{month:02d}-{day:02d}-00000.nc" @@ -46,6 +47,9 @@ def _write_nc(filename, variables, ntime=1): var[...] = np.ones(shape) +# +# Tests for argument parsing +# class TestGetParser(unittest.TestCase): """Tests of get_parser""" @@ -78,6 +82,9 @@ def test_allVarsWithVars_exits(self): d2mc.get_parser().parse_args(["-a", "--vars", "a", "arch", CASE, "1", "2"]) +# +# Tests for the argument error checking +# class TestCheckArgs(unittest.TestCase): """Tests of check_args""" @@ -100,11 +107,15 @@ def test_validArgs(self): d2mc.check_args(self._args()) def test_yearOutOfRange(self): - """BEGYEAR, ENDYEAR and YEAR must be from 0 to 9999""" - with self.assertRaisesRegex(SystemExit, "ENDYEAR must be an integer from 0 to 9999"): + """BEGYEAR, ENDYEAR and YEAR must be from 1 to 9999""" + with self.assertRaisesRegex(SystemExit, "ENDYEAR must be an integer from 1 to 9999"): d2mc.check_args(self._args(years=("2001", "10000"))) - with self.assertRaisesRegex(SystemExit, "YEAR must be an integer from 0 to 9999"): + with self.assertRaisesRegex(SystemExit, "BEGYEAR must be an integer from 1 to 9999"): + d2mc.check_args(self._args(years=("0", "2001"))) + with self.assertRaisesRegex(SystemExit, "YEAR must be an integer from 1 to 9999"): d2mc.check_args(self._args("--year", "10000")) + with self.assertRaisesRegex(SystemExit, "YEAR must be an integer from 1 to 9999"): + d2mc.check_args(self._args("--year", "0")) def test_begyearAfterEndyear(self): """BEGYEAR can't be after ENDYEAR""" @@ -113,7 +124,7 @@ def test_begyearAfterEndyear(self): def test_njobsZero(self): """--njobs must be positive""" - with self.assertRaisesRegex(SystemExit, "NJOBS must be a positive integer"): + with self.assertRaisesRegex(SystemExit, "NJOBS must be a positive integer more than one"): d2mc.check_args(self._args("--njobs", "0")) def test_accountWithoutBatch(self): @@ -130,9 +141,13 @@ def test_onlyAlwaysVars(self): def test_missingDirectories(self): """The archive, output and input directories must exist""" with self.assertRaisesRegex(SystemExit, "Input .* directory does not exist"): - d2mc.check_args(self._args(archdir=os.path.join(self._testdir, "nothere"))) + d2mc.check_args( + self._args(archdir=os.path.join(self._testdir, "dir_that_does_not_exist")) + ) with self.assertRaisesRegex(SystemExit, "Output .* directory does not exist"): - d2mc.check_args(self._args("--outdir", os.path.join(self._testdir, "nothere"))) + d2mc.check_args( + self._args("--outdir", os.path.join(self._testdir, "dir_that_does_not_exist")) + ) shutil.rmtree(d2mc.input_dir(self._archdir, CASE)) with self.assertRaisesRegex(SystemExit, "hist directory does not exist"): d2mc.check_args(self._args()) @@ -148,6 +163,9 @@ def test_outputExists(self): d2mc.check_args(self._args()) +# +# Tests for batch and number of jobs +# class TestDefaultNjobs(unittest.TestCase): """Tests of default_njobs""" @@ -163,6 +181,9 @@ def test_notInBatchJob(self): self.assertEqual(d2mc.default_njobs(), d2mc.INTERACTIVE_NJOBS) +# +# Tests for the variable lists +# class TestVarLists(unittest.TestCase): """Tests of build_var_list and averaged_vars""" @@ -184,6 +205,9 @@ def test_averagedVars_none(self): self.assertIsNone(d2mc.averaged_vars(None)) +# +# Tests for the year/month list and the daily file indexing +# class TestYearMonths(unittest.TestCase): """Tests of year_months""" @@ -196,11 +220,17 @@ def test_twoYears(self): self.assertEqual(yms[12], (2002, 1)) self.assertEqual(yms[-1], (2002, 12)) - def test_yearZero(self): - """A single year, including year 0""" - self.assertEqual(d2mc.year_months(0, 0), [(0, month) for month in range(1, 13)]) + def test_singleYear(self): + """A single year, including year 1""" + year = 1 + self.assertEqual(d2mc.year_months(year, year), [(0, month) for month in range(1, 13)]) + year = d2mc.DEFAULT_YEAR + self.assertEqual(d2mc.year_months(year, year), [(year, month) for month in range(1, 13)]) +# +# Test the list of daily files and the file count +# class TestDailyFiles(unittest.TestCase): """Tests of index_daily_files and check_day_counts""" @@ -209,22 +239,30 @@ def setUp(self): # Empty files are enough, the files are found by name for month, ndays in enumerate(d2mc.NOLEAP_DAYS, start=1): for day in range(1, ndays + 1): - self._touch(_daily_name(CASE, 2001, month, day)) + self._create_empty_file(_daily_name(CASE, 2001, month, day)) # Files that should be ignored: another case, and other kinds of files - self._touch(_daily_name("OTHER", 2001, 1, 1)) - self._touch(f"{CASE}.cpl.hi.2001-01-01-00000.nc") - self._touch(_daily_name(CASE, 2001, 1, 1) + ".bak") + self._create_empty_file(_daily_name("DIFFERENT_CASE", 2001, 1, 1)) + self._create_empty_file(f"{CASE}.cpl.hi.2001-01-01-00000.nc") + self._create_empty_file(_daily_name(CASE, 2001, 1, 1) + ".base") def tearDown(self): shutil.rmtree(self._testdir, ignore_errors=True) - def _touch(self, filename): + def _create_empty_file(self, filename): """Create an empty file in the test directory""" with open(os.path.join(self._testdir, filename), "w", encoding="utf-8"): pass def test_indexDailyFiles(self): - """The files are grouped by year and month, sorted, and other files are ignored""" + """ + Check that index_daily_files properly groups and sorts by year and month + and ignores any other files + """ + # Files that should be ignored: another case, and other kinds of files + self._create_empty_file(_daily_name("DIFFERENT_CASE", 2001, 1, 1)) + self._create_empty_file(f"{CASE}.cpl.hi.2001-01-01-00000.nc") + self._create_empty_file(_daily_name(CASE, 2001, 1, 1) + ".base") + # files_by_month = d2mc.index_daily_files(self._testdir, CASE) self.assertEqual(sorted(files_by_month), d2mc.year_months(2001, 2001)) for (_, month), files in files_by_month.items(): @@ -235,20 +273,28 @@ def test_indexDailyFiles(self): ) def test_checkDayCounts_complete(self): - """A complete year has no errors""" + """Test that checkDayCounts for a complete year is correct""" files_by_month = d2mc.index_daily_files(self._testdir, CASE) self.assertEqual(d2mc.check_day_counts(files_by_month, d2mc.year_months(2001, 2001)), []) def test_checkDayCounts_missing(self): - """A missing day and a missing year are all reported""" + """Test that checkDayCounts reports missing days and years correctly""" + # Remove one day from the first year os.remove(os.path.join(self._testdir, _daily_name(CASE, 2001, 2, 14))) + # NOTE: Setup only created files for the first year, so the second year has no files files_by_month = d2mc.index_daily_files(self._testdir, CASE) errors = d2mc.check_day_counts(files_by_month, d2mc.year_months(2001, 2002)) self.assertEqual(len(errors), 13) + # There will be an error reported for the missing day in the first year + # and each month of the second year self.assertEqual(errors[0], "2001-02 has 27 daily files (expected 28)") self.assertEqual(errors[1], "2002-01 has 0 daily files (expected 31)") +# +# Test the netCDF variable handling functions for the monthly files that were kept +# previously -- to make sure they have the needed variables +# class TestNetcdfVars(unittest.TestCase): """Tests of file_vars, missing_vars and check_reused_mon_file""" @@ -295,6 +341,8 @@ def test_checkReusedMonFile_missing(self): with self.assertRaisesRegex(SystemExit, "is missing variables: b"): d2mc.check_reused_mon_file(monfile, [self._daily], None) + # This checks against an original version that had the ALWAYS_VARS on the monthly files + # TODO: Remove this since it's not really needed def test_checkReusedMonFile_stale(self): """A monthly file with the ALWAYS_VARS was made by an older version, so aborts""" monfile = self._monfile(["a", "b", d2mc.ALWAYS_VARS[0]]) @@ -302,23 +350,34 @@ def test_checkReusedMonFile_stale(self): d2mc.check_reused_mon_file(monfile, [self._daily], ["a"]) +# +# Check some of the NCO (NetCDF Climate Operators) functions +# class TestCheckNco(unittest.TestCase): """Tests of check_nco""" def test_missingCommand(self): """Abort naming the NCO commands that aren't found""" + # mock d2mc.shutil with the input arguments below, and the side effects listed with mock.patch.object( - d2mc.shutil, "which", side_effect=lambda cmd: None if cmd == "ncrcat" else "/bin/x" + # lambda is used here to have a small inline function to define the side effects + d2mc.shutil, + "which", + side_effect=lambda cmd: None if cmd == "ncrcat" else "/bin/x", ): with self.assertRaisesRegex(SystemExit, "NCO commands not found: ncrcat "): d2mc.check_nco() def test_allFound(self): """No abort when all the commands are found""" + # mock d2mc.shutil with the input arguments below, and the side effects as listed with mock.patch.object(d2mc.shutil, "which", return_value="/bin/x"): d2mc.check_nco() +# +# Tests of the ncra command for reacord averaging +# class TestNcraCommand(unittest.TestCase): """Tests of ncra_command""" @@ -354,31 +413,45 @@ def test_skipExisting(self): self.assertEqual(d2mc.ncra_command(["f1"], self._out), expected) +# +# Tests of running jobs +# class TestRunJobs(unittest.TestCase): """Tests of run_jobs (with a mock runner instead of running the commands)""" def test_allRun(self): """Every non-empty command is run, with an error message naming its output file""" + # Create runner as a MagicMock so that it does nothing, but records each call runner = mock.MagicMock() - d2mc.run_jobs([["a", "o1"], [], ["b", "o2"]], 2, runner=runner) + # Run a job in one call sending it three different commands to run at the same time + commands = [ + ["a", "o1"], # run "a" creating file o1 + [], # empty command (output already exists) is skipped + ["b", "o2"], # run "b" creating file o2 + ] + d2mc.run_jobs(commands, 2, runner=runner) + # Check that runner had recorded these two calls, and the appropriate error runner.assert_has_calls( [ mock.call(["a", "o1"], "Failed creating o1"), mock.call(["b", "o2"], "Failed creating o2"), ], - any_order=True, + any_order=True, # running jobs simultaneously we don't know which one will be first ) self.assertEqual(runner.call_count, 2) def test_oneJobInOrder(self): """With one job at a time the commands run in order""" + # Create runner as a MagicMock so that it does nothing, but records each call runner = mock.MagicMock() commands = [[f"c{i}", f"o{i}"] for i in range(5)] d2mc.run_jobs(commands, 1, runner=runner) + # Check that the calls ran from runner match what's expected self.assertEqual([call.args[0] for call in runner.call_args_list], commands) def test_noCommands(self): """Only empty commands runs nothing""" + # Create runner as a MagicMock so that it does nothing, but records each call runner = mock.MagicMock() d2mc.run_jobs([[], []], 4, runner=runner) runner.assert_not_called() @@ -386,6 +459,7 @@ def test_noCommands(self): def test_failure(self): """A failing command (the runner aborts) stops with SystemExit""" + # Define a runner that aborts if the input command is "bad" def runner(cmd, errmsg): if cmd[0] == "bad": sys.exit(errmsg) @@ -394,6 +468,9 @@ def runner(cmd, errmsg): d2mc.run_jobs([["good", "o1"], ["bad", "o2"], ["good", "o3"]], 2, runner=runner) +# +# Tests of the different output files +# class TestFilenames(unittest.TestCase): """Tests of output_filename, mon_file and clim_file""" @@ -410,23 +487,26 @@ def test_intermediateFiles(self): self.assertEqual(d2mc.clim_file("/m", 2), os.path.join("/m", "clim_02.nc")) +# +# Test file output for time and time and climaatology bounds +# class TestTimeValues(unittest.TestCase): """Tests of month_bounds, mid_month_times and climatology_bounds_values""" def test_monthBounds(self): """Start and end of each month of a noleap year""" bounds = d2mc.month_bounds() - self.assertEqual(len(bounds), 12) - self.assertEqual(bounds[0], (0, 31)) - self.assertEqual(bounds[1], (31, 59)) - self.assertEqual(bounds[-1], (334, 365)) + self.assertEqual(len(bounds), 12) # Check that there is just 12 bounds for each month + self.assertEqual(bounds[0], (0, 31)) # January + self.assertEqual(bounds[1], (31, 59)) # February + self.assertEqual(bounds[-1], (334, 365)) # December (last month) def test_midMonthTimes(self): """Middle of each month""" times = d2mc.mid_month_times() - self.assertEqual(times[0], 15.5) - self.assertEqual(times[1], 45.0) - self.assertEqual(times[-1], 349.5) + self.assertEqual(times[0], 15.5) # January + self.assertEqual(times[1], 45.0) # Febreary + self.assertEqual(times[-1], 349.5) # December (last month) def test_climatologyBounds_singleYear(self): """Averaging only YEAR gives the bounds of each month of YEAR""" @@ -441,6 +521,9 @@ def test_climatologyBounds_span(self): self.assertTrue(all(start < 0 and end < 0 for start, end in bounds)) +# +# Test setting of the time axis +# class TestSetTimeAxis(unittest.TestCase): """Tests of set_time_axis""" @@ -451,8 +534,11 @@ def setUp(self): def tearDown(self): shutil.rmtree(self._testdir, ignore_errors=True) - def _write_clim(self, with_time_bnds): - """Write a 12-record file like the one from ncrcat, optionally with time_bnds""" + def _write_clim(self, with_time_bnds=False): + """ + Write a 12-record file like the one from ncrcat, optionally with time_bnds + This is a utility function called both ways in later tests + """ with Dataset(self._file, "w") as ncfile: ncfile.createDimension("time", None) time = ncfile.createVariable("time", "f8", ("time",)) @@ -499,6 +585,9 @@ def test_withoutTimeBnds(self): self._check() +# +# Test the copy of the grid varaibles (the ALWAYS_VARS) +# class TestCopyGridVars(unittest.TestCase): """Tests of copy_grid_vars""" @@ -506,7 +595,8 @@ def setUp(self): self._testdir = tempfile.mkdtemp() self._src = os.path.join(self._testdir, "daily.nc") self._dst = os.path.join(self._testdir, "clim.nc") - # Like CMEPS: float lon/lat on (time, ny, nx) with a fill value and attributes + # Create a CPLHIST history file like from CMEPS: + # float lon/lat on (time, ny, nx) with a fill value and attributes with Dataset(self._src, "w") as ncfile: ncfile.createDimension("time", None) ncfile.createDimension("atmImp_ny", 1) @@ -544,6 +634,9 @@ def test_copyGridVars(self): np.testing.assert_array_equal(var[:], base + np.arange(3.0).reshape(1, 3)) +# +# Test that adding variable attributes works correctly +# class TestAddVarAttributes(unittest.TestCase): """Tests of add_var_attributes""" @@ -584,11 +677,16 @@ def test_addVarAttributes(self): self.assertNotIn("coordinates", ncfile.variables[name].ncattrs()) +# +# Tests of writing prvenance information is correct +# class TestWriteProvenance(unittest.TestCase): """Tests of write_provenance (with the ncatted command mocked)""" def test_oneNcattedCommand(self): """One ncatted --hst command sets the global attributes and deletes file_version""" + # mock run_cmnd_output_on_error function as run_cmd that does nothing, but returns as set + # mock sys.argv to work as given next with mock.patch.object(d2mc, "run_cmd_output_on_error") as run_cmd, mock.patch.object( d2mc, "get_ctsm_git_short_hash", return_value="abc1234" ), mock.patch.object(sys, "argv", ["daily_to_monthly_clim", "--vars", "a,b", "x"]): @@ -609,6 +707,9 @@ def test_oneNcattedCommand(self): self.assertTrue(title[0].endswith("for cyclical year 1850")) +# +# Test the batch options +# class TestBatch(unittest.TestCase): """Tests of job_args, batch_script and submit_batch""" @@ -617,8 +718,8 @@ def test_jobArgs_default(self): args = d2mc.get_parser().parse_args(["arch", CASE, "2001", "2002"]) self.assertEqual(d2mc.job_args(args), ["arch", CASE, "2001", "2002"]) - def test_jobArgs_options(self): - """The options given are passed on, but not the batch options""" + def test_jobArgs_removes_batch_options(self): + """Set all options, but show that job_args returns only the non-batch options""" args = d2mc.get_parser().parse_args( [ "--batch", @@ -651,12 +752,14 @@ def test_jobArgs_options(self): ) def test_jobArgs_allVars(self): - """--all-vars is passed on""" + """Test that --all-vars is passed on when it's called as '-a'""" args = d2mc.get_parser().parse_args(["-a", "arch", CASE, "2001", "2002"]) self.assertEqual(d2mc.job_args(args), ["arch", CASE, "2001", "2002", "--all-vars"]) def test_batchScript(self): - """The PBS settings, NCO, the directory and the quoted command line""" + """ + Test that batch_script creates: PBS settings, NCO, the directory and the quoted command + """ argv = ["arch", CASE, "2001", "2002", "--vars", "a,b", "--outdir", "my dir"] script = d2mc.batch_script( argv, account="P1", walltime="01:00:00", workdir="/work dir", njobs=8 @@ -675,8 +778,13 @@ def test_batchScript(self): self.assertEqual(command[2:], argv) def test_submitBatch(self): - """qsub gets the queue and the script on stdin, and the job id is returned""" + """ + Test submitBatch that qsub gets the queue and the script on stdin, + and the job id is returned + """ result = subprocess.CompletedProcess([], 0, stdout="1234.desched1\n", stderr="") + # mock d2mc.shutil for which to just return qsub path + # Then mock d2mc.subprocess as run to return the result with mock.patch.object(d2mc.shutil, "which", return_value="/bin/qsub"), mock.patch.object( d2mc.subprocess, "run", return_value=result ) as run: @@ -685,8 +793,11 @@ def test_submitBatch(self): self.assertEqual(run.call_args.kwargs["input"], "the script") def test_submitBatch_fails(self): - """A qsub error aborts""" + """test that submitBatch fails if the qsub aborts""" + # Create a result that mimics a qsub error result = subprocess.CompletedProcess([], 1, stdout="", stderr="qsub: bad account") + # mock d2mc.shutil on which to return qsub path + # Then mock d2mc.subprocess run to return the result with mock.patch.object(d2mc.shutil, "which", return_value="/bin/qsub"), mock.patch.object( d2mc.subprocess, "run", return_value=result ): @@ -695,6 +806,7 @@ def test_submitBatch_fails(self): def test_submitBatch_noQsub(self): """Abort if qsub isn't found""" + # mock d2mc.shutil for which, to return nothing as it would if the command wasn't found with mock.patch.object(d2mc.shutil, "which", return_value=None): with self.assertRaisesRegex(SystemExit, "qsub not found"): d2mc.submit_batch("the script", "main") From 3633b0793df3258a62b94d9b290ba8eb7c729bb1 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Sun, 25 Oct 2026 18:26:28 -0600 Subject: [PATCH 30/37] Fix the error messages so the unit test now passes, passes both black and lint --- python/ctsm/cplhist/daily_to_monthly_clim.py | 8 ++++---- python/ctsm/test/test_unit_daily_to_monthly_clim.py | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/python/ctsm/cplhist/daily_to_monthly_clim.py b/python/ctsm/cplhist/daily_to_monthly_clim.py index 239434949a..7ade3dc2b7 100644 --- a/python/ctsm/cplhist/daily_to_monthly_clim.py +++ b/python/ctsm/cplhist/daily_to_monthly_clim.py @@ -241,12 +241,12 @@ def check_args(args): Check the command line arguments, aborting if there is a problem """ for name, year in (("BEGYEAR", args.begyear), ("ENDYEAR", args.endyear), ("YEAR", args.year)): - if not 0 <= year <= 9999: - abort(f"{name} must be an integer from 0 to 9999: {year}") + if not 1 <= year <= 9999: + abort(f"{name} must be an integer from 1 to 9999: {year}") if args.begyear > args.endyear: abort(f"BEGYEAR ({args.begyear}) must not be after ENDYEAR ({args.endyear})") if args.njobs is not None and args.njobs < 1: - abort(f"NJOBS must be a positive integer: {args.njobs}") + abort(f"NJOBS must be a positive integer more than one: {args.njobs}") if args.account is not None and not args.batch: abort("--account is only used with --batch") if not args.all_vars and not averaged_vars(args.vars): @@ -257,7 +257,7 @@ def check_args(args): abort(f"Output {args.outdir} output archive directory does not exist") indir = input_dir(args.archdir, args.case) if not os.path.isdir(indir): - abort(f"Input {indir} case directory does not exist") + abort(f"Input {indir} case archive directory does not exist") # # Abort if the output file already exists # diff --git a/python/ctsm/test/test_unit_daily_to_monthly_clim.py b/python/ctsm/test/test_unit_daily_to_monthly_clim.py index 225162b643..e685b4f3e0 100644 --- a/python/ctsm/test/test_unit_daily_to_monthly_clim.py +++ b/python/ctsm/test/test_unit_daily_to_monthly_clim.py @@ -149,7 +149,7 @@ def test_missingDirectories(self): self._args("--outdir", os.path.join(self._testdir, "dir_that_does_not_exist")) ) shutil.rmtree(d2mc.input_dir(self._archdir, CASE)) - with self.assertRaisesRegex(SystemExit, "hist directory does not exist"): + with self.assertRaisesRegex(SystemExit, "case archive directory does not exist"): d2mc.check_args(self._args()) def test_outputExists(self): @@ -221,9 +221,9 @@ def test_twoYears(self): self.assertEqual(yms[-1], (2002, 12)) def test_singleYear(self): - """A single year, including year 1""" + """A single year, including year 1 and the default year""" year = 1 - self.assertEqual(d2mc.year_months(year, year), [(0, month) for month in range(1, 13)]) + self.assertEqual(d2mc.year_months(year, year), [(1, month) for month in range(1, 13)]) year = d2mc.DEFAULT_YEAR self.assertEqual(d2mc.year_months(year, year), [(year, month) for month in range(1, 13)]) From c308ef3d74ca5cd040d6eeae7b03169bef992c46 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Sun, 25 Oct 2026 18:38:20 -0600 Subject: [PATCH 31/37] Add a comment that this unit test does I/O --- python/ctsm/test/test_unit_daily_to_monthly_clim.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/python/ctsm/test/test_unit_daily_to_monthly_clim.py b/python/ctsm/test/test_unit_daily_to_monthly_clim.py index e685b4f3e0..f875bc21b6 100644 --- a/python/ctsm/test/test_unit_daily_to_monthly_clim.py +++ b/python/ctsm/test/test_unit_daily_to_monthly_clim.py @@ -1,6 +1,13 @@ #!/usr/bin/env python3 -"""Unit tests for daily_to_monthly_clim""" +""" +Unit tests for daily_to_monthly_clim + +NOTE: SOME OF THESE UNIT TESTS HANDLE NetCDF FILES. + Normally we want tests that do I/O in the system tests, because I/O is slow + and has random glitches. Other unit tests use xarray with in memory objects + that don't create actual files. However, right now this tests only takes about 10 seconds. +""" import datetime import io From d31b1c232d363efffc726a98c1f167e25cd870c4 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Sun, 25 Oct 2026 20:34:58 -0600 Subject: [PATCH 32/37] Fixes from Claude on the changes I made, fixing typos and other clarifying things --- python/ctsm/cplhist/daily_to_monthly_clim.py | 23 ++++--- .../test/test_unit_daily_to_monthly_clim.py | 62 +++++++++---------- 2 files changed, 44 insertions(+), 41 deletions(-) diff --git a/python/ctsm/cplhist/daily_to_monthly_clim.py b/python/ctsm/cplhist/daily_to_monthly_clim.py index 7ade3dc2b7..19581e532b 100644 --- a/python/ctsm/cplhist/daily_to_monthly_clim.py +++ b/python/ctsm/cplhist/daily_to_monthly_clim.py @@ -246,18 +246,18 @@ def check_args(args): if args.begyear > args.endyear: abort(f"BEGYEAR ({args.begyear}) must not be after ENDYEAR ({args.endyear})") if args.njobs is not None and args.njobs < 1: - abort(f"NJOBS must be a positive integer more than one: {args.njobs}") + abort(f"NJOBS must be a positive integer: {args.njobs}") if args.account is not None and not args.batch: abort("--account is only used with --batch") if not args.all_vars and not averaged_vars(args.vars): abort(f"No variables to average given with --vars (besides {','.join(ALWAYS_VARS)})") if not os.path.isdir(args.archdir): - abort(f"Input {args.archdir} archive directory does not exist") + abort(f"Input archive directory {args.archdir} does not exist") if args.outdir is not None and not os.path.isdir(args.outdir): - abort(f"Output {args.outdir} output archive directory does not exist") + abort(f"Output archive directory {args.outdir} does not exist") indir = input_dir(args.archdir, args.case) if not os.path.isdir(indir): - abort(f"Input {indir} case archive directory does not exist") + abort(f"Input case archive directory {indir} does not exist") # # Abort if the output file already exists # @@ -425,15 +425,20 @@ def run_jobs(commands, njobs, *, progress_every=None, runner=run_cmd_output_on_e Stops at the first failure: the jobs that have not started are cancelled and the runner's error (normally an abort) is raised. Logs progress every progress_every completed jobs. Empty commands (for output files that already exist) are skipped. + Args: - commands (list): Command to execute and it's arguments + commands (list): The commands to run, each a list of the command and its arguments, + with the output file last (an empty list is skipped) njobs (int): Number of jobs to run simultaneously - process_every (int): Report on the progress by this count - runner (function): python function to call to run the commands + progress_every (int): Log the progress every time this many jobs have completed + runner (function): Python function to call to run each command Returns: - Nothing if commands aren't valid or if it was successful - Raises exception if something failed or the job was killed + None + + Raises: + SystemExit: From the runner's abort if a command fails + KeyboardInterrupt: If the user hits Ctrl-C """ commands = [cmd for cmd in commands if cmd] if not commands: diff --git a/python/ctsm/test/test_unit_daily_to_monthly_clim.py b/python/ctsm/test/test_unit_daily_to_monthly_clim.py index f875bc21b6..667e18dd23 100644 --- a/python/ctsm/test/test_unit_daily_to_monthly_clim.py +++ b/python/ctsm/test/test_unit_daily_to_monthly_clim.py @@ -6,7 +6,7 @@ NOTE: SOME OF THESE UNIT TESTS HANDLE NetCDF FILES. Normally we want tests that do I/O in the system tests, because I/O is slow and has random glitches. Other unit tests use xarray with in memory objects - that don't create actual files. However, right now this tests only takes about 10 seconds. + that don't create actual files. However, right now these tests only take about 10 seconds. """ import datetime @@ -131,7 +131,7 @@ def test_begyearAfterEndyear(self): def test_njobsZero(self): """--njobs must be positive""" - with self.assertRaisesRegex(SystemExit, "NJOBS must be a positive integer more than one"): + with self.assertRaisesRegex(SystemExit, "NJOBS must be a positive integer"): d2mc.check_args(self._args("--njobs", "0")) def test_accountWithoutBatch(self): @@ -147,16 +147,16 @@ def test_onlyAlwaysVars(self): def test_missingDirectories(self): """The archive, output and input directories must exist""" - with self.assertRaisesRegex(SystemExit, "Input .* directory does not exist"): + with self.assertRaisesRegex(SystemExit, "Input archive directory .* does not exist"): d2mc.check_args( self._args(archdir=os.path.join(self._testdir, "dir_that_does_not_exist")) ) - with self.assertRaisesRegex(SystemExit, "Output .* directory does not exist"): + with self.assertRaisesRegex(SystemExit, "Output archive directory .* does not exist"): d2mc.check_args( self._args("--outdir", os.path.join(self._testdir, "dir_that_does_not_exist")) ) shutil.rmtree(d2mc.input_dir(self._archdir, CASE)) - with self.assertRaisesRegex(SystemExit, "case archive directory does not exist"): + with self.assertRaisesRegex(SystemExit, "Input case archive directory .* does not exist"): d2mc.check_args(self._args()) def test_outputExists(self): @@ -229,10 +229,11 @@ def test_twoYears(self): def test_singleYear(self): """A single year, including year 1 and the default year""" - year = 1 - self.assertEqual(d2mc.year_months(year, year), [(1, month) for month in range(1, 13)]) - year = d2mc.DEFAULT_YEAR - self.assertEqual(d2mc.year_months(year, year), [(year, month) for month in range(1, 13)]) + for year in (1, d2mc.DEFAULT_YEAR): + with self.subTest(year=year): + self.assertEqual( + d2mc.year_months(year, year), [(year, month) for month in range(1, 13)] + ) # @@ -247,10 +248,6 @@ def setUp(self): for month, ndays in enumerate(d2mc.NOLEAP_DAYS, start=1): for day in range(1, ndays + 1): self._create_empty_file(_daily_name(CASE, 2001, month, day)) - # Files that should be ignored: another case, and other kinds of files - self._create_empty_file(_daily_name("DIFFERENT_CASE", 2001, 1, 1)) - self._create_empty_file(f"{CASE}.cpl.hi.2001-01-01-00000.nc") - self._create_empty_file(_daily_name(CASE, 2001, 1, 1) + ".base") def tearDown(self): shutil.rmtree(self._testdir, ignore_errors=True) @@ -280,12 +277,12 @@ def test_indexDailyFiles(self): ) def test_checkDayCounts_complete(self): - """Test that checkDayCounts for a complete year is correct""" + """Test that check_day_counts for a complete year is correct""" files_by_month = d2mc.index_daily_files(self._testdir, CASE) self.assertEqual(d2mc.check_day_counts(files_by_month, d2mc.year_months(2001, 2001)), []) def test_checkDayCounts_missing(self): - """Test that checkDayCounts reports missing days and years correctly""" + """Test that check_day_counts reports missing days and years correctly""" # Remove one day from the first year os.remove(os.path.join(self._testdir, _daily_name(CASE, 2001, 2, 14))) # NOTE: Setup only created files for the first year, so the second year has no files @@ -365,11 +362,11 @@ class TestCheckNco(unittest.TestCase): def test_missingCommand(self): """Abort naming the NCO commands that aren't found""" - # mock d2mc.shutil with the input arguments below, and the side effects listed + # Replace shutil.which so that only ncrcat is "not found" with mock.patch.object( - # lambda is used here to have a small inline function to define the side effects d2mc.shutil, "which", + # lambda is used here to have a small inline function to define the side effects side_effect=lambda cmd: None if cmd == "ncrcat" else "/bin/x", ): with self.assertRaisesRegex(SystemExit, "NCO commands not found: ncrcat "): @@ -377,13 +374,13 @@ def test_missingCommand(self): def test_allFound(self): """No abort when all the commands are found""" - # mock d2mc.shutil with the input arguments below, and the side effects as listed + # Replace shutil.which so that all the commands are found with mock.patch.object(d2mc.shutil, "which", return_value="/bin/x"): d2mc.check_nco() # -# Tests of the ncra command for reacord averaging +# Tests of the ncra command for record averaging # class TestNcraCommand(unittest.TestCase): """Tests of ncra_command""" @@ -495,7 +492,7 @@ def test_intermediateFiles(self): # -# Test file output for time and time and climaatology bounds +# Test file output for time and time and climatology bounds # class TestTimeValues(unittest.TestCase): """Tests of month_bounds, mid_month_times and climatology_bounds_values""" @@ -512,7 +509,7 @@ def test_midMonthTimes(self): """Middle of each month""" times = d2mc.mid_month_times() self.assertEqual(times[0], 15.5) # January - self.assertEqual(times[1], 45.0) # Febreary + self.assertEqual(times[1], 45.0) # February self.assertEqual(times[-1], 349.5) # December (last month) def test_climatologyBounds_singleYear(self): @@ -593,7 +590,7 @@ def test_withoutTimeBnds(self): # -# Test the copy of the grid varaibles (the ALWAYS_VARS) +# Test the copy of the grid variables (the ALWAYS_VARS) # class TestCopyGridVars(unittest.TestCase): """Tests of copy_grid_vars""" @@ -685,15 +682,16 @@ def test_addVarAttributes(self): # -# Tests of writing prvenance information is correct +# Tests of writing provenance information is correct # class TestWriteProvenance(unittest.TestCase): """Tests of write_provenance (with the ncatted command mocked)""" def test_oneNcattedCommand(self): """One ncatted --hst command sets the global attributes and deletes file_version""" - # mock run_cmnd_output_on_error function as run_cmd that does nothing, but returns as set - # mock sys.argv to work as given next + # Replace run_cmd_output_on_error with a mock (run_cmd) that does nothing but record + # the call, get_ctsm_git_short_hash to return a fixed hash, and sys.argv with the + # command line given with mock.patch.object(d2mc, "run_cmd_output_on_error") as run_cmd, mock.patch.object( d2mc, "get_ctsm_git_short_hash", return_value="abc1234" ), mock.patch.object(sys, "argv", ["daily_to_monthly_clim", "--vars", "a,b", "x"]): @@ -786,12 +784,12 @@ def test_batchScript(self): def test_submitBatch(self): """ - Test submitBatch that qsub gets the queue and the script on stdin, + Test submit_batch that qsub gets the queue and the script on stdin, and the job id is returned """ result = subprocess.CompletedProcess([], 0, stdout="1234.desched1\n", stderr="") - # mock d2mc.shutil for which to just return qsub path - # Then mock d2mc.subprocess as run to return the result + # Replace shutil.which to return the qsub path + # and subprocess.run to return the result without running qsub with mock.patch.object(d2mc.shutil, "which", return_value="/bin/qsub"), mock.patch.object( d2mc.subprocess, "run", return_value=result ) as run: @@ -800,11 +798,11 @@ def test_submitBatch(self): self.assertEqual(run.call_args.kwargs["input"], "the script") def test_submitBatch_fails(self): - """test that submitBatch fails if the qsub aborts""" + """Test that submit_batch aborts if qsub fails""" # Create a result that mimics a qsub error result = subprocess.CompletedProcess([], 1, stdout="", stderr="qsub: bad account") - # mock d2mc.shutil on which to return qsub path - # Then mock d2mc.subprocess run to return the result + # Replace shutil.which to return the qsub path + # and subprocess.run to return the result without running qsub with mock.patch.object(d2mc.shutil, "which", return_value="/bin/qsub"), mock.patch.object( d2mc.subprocess, "run", return_value=result ): @@ -813,7 +811,7 @@ def test_submitBatch_fails(self): def test_submitBatch_noQsub(self): """Abort if qsub isn't found""" - # mock d2mc.shutil for which, to return nothing as it would if the command wasn't found + # Replace shutil.which to return None, as it does when the command isn't found with mock.patch.object(d2mc.shutil, "which", return_value=None): with self.assertRaisesRegex(SystemExit, "qsub not found"): d2mc.submit_batch("the script", "main") From fe8bd71e54c100f2499ac8aae46ca64fc7f0bf15 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Sun, 25 Oct 2026 20:42:54 -0600 Subject: [PATCH 33/37] Two other things commented on from Claude, remove the backwards compatiblity stale check and test for it, and run _write_clim with default for with_time_bnds as well as both explicit ways --- python/ctsm/cplhist/daily_to_monthly_clim.py | 6 ------ python/ctsm/test/test_unit_daily_to_monthly_clim.py | 11 ++--------- 2 files changed, 2 insertions(+), 15 deletions(-) diff --git a/python/ctsm/cplhist/daily_to_monthly_clim.py b/python/ctsm/cplhist/daily_to_monthly_clim.py index 19581e532b..56c83053a0 100644 --- a/python/ctsm/cplhist/daily_to_monthly_clim.py +++ b/python/ctsm/cplhist/daily_to_monthly_clim.py @@ -489,12 +489,6 @@ def check_reused_mon_file(monfile, daily_files, var_list): (older versions of this tool averaged them with the time dimension). """ in_monfile = file_vars(monfile) - stale = [var for var in ALWAYS_VARS if var in in_monfile] - if stale: - abort( - f"Existing monthly file {monfile} has {' '.join(stale)}, so it was made by an older" - " version of this tool, remove it and rerun" - ) if var_list is None: var_list = sorted(file_vars(daily_files[0]) - set(ALWAYS_VARS)) missing = [var for var in var_list if var not in in_monfile] diff --git a/python/ctsm/test/test_unit_daily_to_monthly_clim.py b/python/ctsm/test/test_unit_daily_to_monthly_clim.py index 667e18dd23..496285480d 100644 --- a/python/ctsm/test/test_unit_daily_to_monthly_clim.py +++ b/python/ctsm/test/test_unit_daily_to_monthly_clim.py @@ -345,15 +345,6 @@ def test_checkReusedMonFile_missing(self): with self.assertRaisesRegex(SystemExit, "is missing variables: b"): d2mc.check_reused_mon_file(monfile, [self._daily], None) - # This checks against an original version that had the ALWAYS_VARS on the monthly files - # TODO: Remove this since it's not really needed - def test_checkReusedMonFile_stale(self): - """A monthly file with the ALWAYS_VARS was made by an older version, so aborts""" - monfile = self._monfile(["a", "b", d2mc.ALWAYS_VARS[0]]) - with self.assertRaisesRegex(SystemExit, "older version"): - d2mc.check_reused_mon_file(monfile, [self._daily], ["a"]) - - # # Check some of the NCO (NetCDF Climate Operators) functions # @@ -585,6 +576,8 @@ def test_withTimeBnds(self): def test_withoutTimeBnds(self): """climatology_bounds is created when there is no time_bnds""" + self._write_clim() + self._check() self._write_clim(with_time_bnds=False) self._check() From 63b0b1013c80464042414c3460ebdf889222b977 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Sun, 25 Oct 2026 20:45:18 -0600 Subject: [PATCH 34/37] Run black --- python/ctsm/test/test_unit_daily_to_monthly_clim.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python/ctsm/test/test_unit_daily_to_monthly_clim.py b/python/ctsm/test/test_unit_daily_to_monthly_clim.py index 496285480d..55d44d2af6 100644 --- a/python/ctsm/test/test_unit_daily_to_monthly_clim.py +++ b/python/ctsm/test/test_unit_daily_to_monthly_clim.py @@ -345,6 +345,7 @@ def test_checkReusedMonFile_missing(self): with self.assertRaisesRegex(SystemExit, "is missing variables: b"): d2mc.check_reused_mon_file(monfile, [self._daily], None) + # # Check some of the NCO (NetCDF Climate Operators) functions # From 0d114b429ff607fa364cf33a985abc452b1ad4a1 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Sun, 25 Oct 2026 20:47:17 -0600 Subject: [PATCH 35/37] Change local shortname from dtm to d2mc --- .../test/test_sys_daily_to_monthly_clim.py | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/python/ctsm/test/test_sys_daily_to_monthly_clim.py b/python/ctsm/test/test_sys_daily_to_monthly_clim.py index e310f323c6..3752c0bdc9 100644 --- a/python/ctsm/test/test_sys_daily_to_monthly_clim.py +++ b/python/ctsm/test/test_sys_daily_to_monthly_clim.py @@ -21,7 +21,7 @@ sys.path.insert(1, _CTSM_PYTHON) # pylint: disable=wrong-import-position from ctsm import unit_testing -from ctsm.cplhist import daily_to_monthly_clim as dtm +from ctsm.cplhist import daily_to_monthly_clim as d2mc # Allow test names that pylint doesn't like; otherwise hard to make them # readable @@ -62,7 +62,7 @@ def _write_daily_file(filename, year, month, day_of_year): time_bnds[:] = [[start, start + 0.5], [start + 0.5, start + 1.0]] dims = ("time", "atmImp_ny", "atmImp_nx") value = _value(year, month) - for name in dtm.DEFAULT_VARS + [EXTRA_VAR]: + for name in d2mc.DEFAULT_VARS + [EXTRA_VAR]: var = ncfile.createVariable(name, "f4", dims, fill_value=1.0e30) var.setncatts({"units": "kg m-2 s-1", "standard_name": name}) var[:] = (2.0 * value if name == EXTRA_VAR else value) * np.ones((2, 1, NX)) @@ -84,15 +84,15 @@ def setUpClass(cls): """Write BEGYEAR to ENDYEAR of daily files, once for all the tests""" cls._indir_root = tempfile.mkdtemp() cls._archdir = os.path.join(cls._indir_root, "archive") - indir = dtm.input_dir(cls._archdir, CASE) + indir = d2mc.input_dir(cls._archdir, CASE) os.makedirs(indir) cls._first_daily = None for year in range(BEGYEAR, ENDYEAR + 1): day_of_year = 0 - for month, ndays in enumerate(dtm.NOLEAP_DAYS, start=1): + for month, ndays in enumerate(d2mc.NOLEAP_DAYS, start=1): for day in range(1, ndays + 1): filename = os.path.join( - indir, f"{CASE}.{dtm.HIST_TAG}.{year:04d}-{month:02d}-{day:02d}-00000.nc" + indir, f"{CASE}.{d2mc.HIST_TAG}.{year:04d}-{month:02d}-{day:02d}-00000.nc" ) _write_daily_file(filename, year, month, day_of_year) day_of_year += 1 @@ -123,9 +123,9 @@ def _run(self, *options, outdir=None): argv += [self._archdir, CASE, str(BEGYEAR), str(ENDYEAR)] output = io.StringIO() with mock.patch.object(sys, "argv", argv), contextlib.redirect_stdout(output): - dtm.main() - outfile = dtm.output_filename( - dtm.output_dir(self._archdir, outdir, CASE), + d2mc.main() + outfile = d2mc.output_filename( + d2mc.output_dir(self._archdir, outdir, CASE), CASE, BEGYEAR, ENDYEAR, @@ -134,11 +134,11 @@ def _run(self, *options, outdir=None): self.assertTrue(os.path.exists(outfile), msg=f"{outfile} not created") return outfile, output.getvalue(), argv - def _check_output(self, outfile, data_vars, year=dtm.DEFAULT_YEAR): + def _check_output(self, outfile, data_vars, year=d2mc.DEFAULT_YEAR): """Check the variables, values, grid, time axis and metadata of the output file""" units = f"days since {year:04d}-01-01 00:00:00" with Dataset(outfile) as ncfile: - expected = set(data_vars) | set(dtm.ALWAYS_VARS) | {"time", "climatology_bounds"} + expected = set(data_vars) | set(d2mc.ALWAYS_VARS) | {"time", "climatology_bounds"} self.assertEqual(set(ncfile.variables), expected) # Each month is the average over the years of that month for name in data_vars: @@ -160,13 +160,13 @@ def _check_output(self, outfile, data_vars, year=dtm.DEFAULT_YEAR): self.assertNotIn("cell_methods", var.ncattrs()) # The time axis and the climatology bounds time = ncfile.variables["time"] - np.testing.assert_array_equal(time[:], dtm.mid_month_times()) + np.testing.assert_array_equal(time[:], d2mc.mid_month_times()) self.assertEqual(time.units, units) self.assertEqual(time.calendar, "noleap") self.assertEqual(time.climatology, "climatology_bounds") np.testing.assert_array_equal( ncfile.variables["climatology_bounds"][:], - dtm.climatology_bounds_values(BEGYEAR, ENDYEAR, year), + d2mc.climatology_bounds_values(BEGYEAR, ENDYEAR, year), ) self.assertEqual(ncfile.Conventions, "CF-1.11") self.assertEqual(ncfile.made_from_cesm_case, CASE) @@ -176,7 +176,7 @@ def _check_output(self, outfile, data_vars, year=dtm.DEFAULT_YEAR): def test_defaultVars(self): """The default variables, with the intermediate files removed at the end""" outfile, _, argv = self._run() - command_used = self._check_output(outfile, dtm.DEFAULT_VARS) + command_used = self._check_output(outfile, d2mc.DEFAULT_VARS) self.assertEqual(command_used, shlex.join(argv)) self.assertFalse(os.path.exists(os.path.join(os.path.dirname(outfile), "mon"))) @@ -190,7 +190,7 @@ def test_varsAndYear(self): def test_allVars(self): """All variables: time_bnds is renamed to climatology_bounds, lon/lat aren't averaged""" outfile, _, _ = self._run("--all-vars") - self._check_output(outfile, dtm.DEFAULT_VARS + [EXTRA_VAR]) + self._check_output(outfile, d2mc.DEFAULT_VARS + [EXTRA_VAR]) def test_keepAndReuse(self): """With --keep the monthly files are kept, and reused on a rerun""" @@ -202,7 +202,7 @@ def test_keepAndReuse(self): os.rename(outfile, os.path.join(self._tempdir, "first.nc")) outfile, printed, _ = self._run("--keep", outdir=outdir) self.assertEqual(printed.count("skipping ncra command"), 12 * nyears) - self._check_output(outfile, dtm.DEFAULT_VARS) + self._check_output(outfile, d2mc.DEFAULT_VARS) def test_reuseWithDifferentVars_aborts(self): """Monthly files kept from a run with fewer variables abort a run with all variables""" From 2515276d1e5ccd718a3dcf8633eb9b28d3277931 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Mon, 26 Oct 2026 11:43:33 -0600 Subject: [PATCH 36/37] Add the compliance checker and get the tests to pass with it, which required an update to the fake files, Claude helped me to get it working --- .../test/test_sys_daily_to_monthly_clim.py | 26 ++++++++++++++----- 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/python/ctsm/test/test_sys_daily_to_monthly_clim.py b/python/ctsm/test/test_sys_daily_to_monthly_clim.py index 3752c0bdc9..863a00f77b 100644 --- a/python/ctsm/test/test_sys_daily_to_monthly_clim.py +++ b/python/ctsm/test/test_sys_daily_to_monthly_clim.py @@ -22,6 +22,7 @@ # pylint: disable=wrong-import-position from ctsm import unit_testing from ctsm.cplhist import daily_to_monthly_clim as d2mc +from ctsm.os_utils import run_cmd_output_on_error # Allow test names that pylint doesn't like; otherwise hard to make them # readable @@ -39,13 +40,16 @@ def _value(year, month): """Value of the default variables in the daily files for a year and month""" + # This is arbitrary, but having it be a function of year and month makes it predictable + # but also different for files of different years and months return (year - 2000) * 100 + month def _write_daily_file(filename, year, month, day_of_year): """ - Write a daily file like the CMEPS 24h average coupler history files: 2 time samples, time_bnds, - the default variables and EXTRA_VAR on (time, atmImp_ny, atmImp_nx), and lon/lat with time too + Write a daily file like the CMEPS CPLHIST 24h average coupler history files: + 2 time samples, time_bnds, the default variables and EXTRA_VAR on + (time, atmImp_ny, atmImp_nx), and lon/lat with time too """ with Dataset(filename, "w") as ncfile: ncfile.createDimension("time", None) @@ -66,12 +70,13 @@ def _write_daily_file(filename, year, month, day_of_year): var = ncfile.createVariable(name, "f4", dims, fill_value=1.0e30) var.setncatts({"units": "kg m-2 s-1", "standard_name": name}) var[:] = (2.0 * value if name == EXTRA_VAR else value) * np.ones((2, 1, NX)) - for name, values, units in ( - ("atmImp_lon", LON, "degrees_E"), - ("atmImp_lat", LAT, "degrees_N"), + for name, values, coord, units in ( + ("atmImp_lon", LON, "longitude", "degrees_E"), + ("atmImp_lat", LAT, "latitude", "degrees_N"), ): var = ncfile.createVariable(name, "f4", dims, fill_value=1.0e30) - var.setncatts({"long_name": name, "units": units}) + # Same attributes as CMEPS, the standard_name is needed for CF compliance + var.setncatts({"long_name": coord, "units": units, "standard_name": coord}) var[:] = values * np.ones((2, 1, NX)) @@ -134,6 +139,11 @@ def _run(self, *options, outdir=None): self.assertTrue(os.path.exists(outfile), msg=f"{outfile} not created") return outfile, output.getvalue(), argv + def _check_cf_compliance(self, outfile): + """Check the output file for CF compliance with the compliance-checker tool""" + cmd = ["compliance-checker", "--test", "cf", outfile] + run_cmd_output_on_error(cmd, "The output file did not pass the CF compliance checker") + def _check_output(self, outfile, data_vars, year=d2mc.DEFAULT_YEAR): """Check the variables, values, grid, time axis and metadata of the output file""" units = f"days since {year:04d}-01-01 00:00:00" @@ -177,6 +187,7 @@ def test_defaultVars(self): """The default variables, with the intermediate files removed at the end""" outfile, _, argv = self._run() command_used = self._check_output(outfile, d2mc.DEFAULT_VARS) + self._check_cf_compliance(outfile) self.assertEqual(command_used, shlex.join(argv)) self.assertFalse(os.path.exists(os.path.join(os.path.dirname(outfile), "mon"))) @@ -184,6 +195,7 @@ def test_varsAndYear(self): """A variable list (including a grid variable) and the time axis set to another year""" outfile, _, argv = self._run("--vars", f"{EXTRA_VAR},atmImp_lat", "--year", "2001") command_used = self._check_output(outfile, [EXTRA_VAR], year=2001) + self._check_cf_compliance(outfile) # The comma in the --vars list makes it through ncatted self.assertEqual(command_used, shlex.join(argv)) @@ -191,6 +203,7 @@ def test_allVars(self): """All variables: time_bnds is renamed to climatology_bounds, lon/lat aren't averaged""" outfile, _, _ = self._run("--all-vars") self._check_output(outfile, d2mc.DEFAULT_VARS + [EXTRA_VAR]) + self._check_cf_compliance(outfile) def test_keepAndReuse(self): """With --keep the monthly files are kept, and reused on a rerun""" @@ -203,6 +216,7 @@ def test_keepAndReuse(self): outfile, printed, _ = self._run("--keep", outdir=outdir) self.assertEqual(printed.count("skipping ncra command"), 12 * nyears) self._check_output(outfile, d2mc.DEFAULT_VARS) + self._check_cf_compliance(outfile) def test_reuseWithDifferentVars_aborts(self): """Monthly files kept from a run with fewer variables abort a run with all variables""" From 8eef6a88f577478452afddd7ae1fd61792d67bf3 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Tue, 27 Oct 2026 21:12:13 -0600 Subject: [PATCH 37/37] I asked Claude to change it to an Object Oriented programming paradigm with a more generic base class, and the specific application for aerosols on top as a seperate wrapper, this is what it gave me, the tests all work, and black and pylint is clean --- python/ctsm/cplhist/aero_1850_monthly_clim.py | 79 ++ python/ctsm/cplhist/cplhist_monthly_clim.py | 959 ++++++++++++++++++ python/ctsm/cplhist/daily_to_monthly_clim.py | 899 ---------------- ....py => test_sys_aero_1850_monthly_clim.py} | 37 +- .../test/test_unit_aero_1850_monthly_clim.py | 338 ++++++ ...m.py => test_unit_cplhist_monthly_clim.py} | 349 ++----- ...to_monthly_clim => aero_1850_monthly_clim} | 16 +- 7 files changed, 1471 insertions(+), 1206 deletions(-) create mode 100644 python/ctsm/cplhist/aero_1850_monthly_clim.py create mode 100644 python/ctsm/cplhist/cplhist_monthly_clim.py delete mode 100644 python/ctsm/cplhist/daily_to_monthly_clim.py rename python/ctsm/test/{test_sys_daily_to_monthly_clim.py => test_sys_aero_1850_monthly_clim.py} (87%) create mode 100644 python/ctsm/test/test_unit_aero_1850_monthly_clim.py rename python/ctsm/test/{test_unit_daily_to_monthly_clim.py => test_unit_cplhist_monthly_clim.py} (59%) rename tools/cplhist/{daily_to_monthly_clim => aero_1850_monthly_clim} (62%) diff --git a/python/ctsm/cplhist/aero_1850_monthly_clim.py b/python/ctsm/cplhist/aero_1850_monthly_clim.py new file mode 100644 index 0000000000..9f2eff8e01 --- /dev/null +++ b/python/ctsm/cplhist/aero_1850_monthly_clim.py @@ -0,0 +1,79 @@ +""" +Create a 12-month climatology of the aerosol deposition fields from daily CMEPS coupler history +(CPLHIST) files, for use by DATM. + +The input files are the daily averaged atmosphere coupler history files: + + ARCHDIR/CASE/cpl/hist/CASE.cpl.hx.atm.24h.avrg.YYYY-MM-DD-00000.nc + +for the years BEGYEAR to ENDYEAR. The steps are: + + 1. Average the daily files of each month of each year (ncra) + 2. Average each calendar month over all the years (ncra), and put the 12 months in one + file (ncrcat) + 3. Set the time axis to the middle of each month of YEAR (noleap calendar) + +The output file is: + + OUTDIR/CASE/proc/CASE.cpl.hx.atm.monclim.BEGYEAR-ENDYEAR_cYYYYMMDD.nc + +This needs the NCO commands ncra, ncrcat and ncatted (on Derecho: module load nco). +With --batch, the input is checked and then this tool is submitted to the PBS batch queue. + +The general code is in cplhist_monthly_clim.py, this sets what's specific to aerosols for 1850. + +A previous version of this for CESM2 is in NCL and here on Derecho by Keith Oleson: + /glade/u/home/oleson/misc_programs/CLM5Dev/CMIP6/CreateAeroDepFile_Task5B.ncl +It did 5 year averages over the historical period and then did a linear interpolation +between the 5-year averages. It also used CPLHIST output from several ensemble members and +averged them together with ncea. This is something that will have to be done in the future +in this tool as well. + +Another example for Nitrogen deposition is here from Simone Tilmes and Mike Mills: + https://svn.code.sf.net/p/codescripts/code/trunk/ncl/cam_forcing/CreateDepositionFile.ncl + +Neither were quite right for what's needed here. And both are in NCL which is well after +end of life now. + +Also there is code in LDF to create monthly climatologies, but it's not setup to bring in ensemble +averages and not specialized to create datafiles for DATM aerosol forcing. +""" + +import os + +from ctsm.cplhist.cplhist_monthly_clim import CplhistMonthlyClim + + +class Aero1850MonthlyClim(CplhistMonthlyClim): + """ + Monthly climatology of the aerosol deposition fields for 1850 + """ + + DEFAULT_VARS = [ + "atmImp_Faxa_bcph1", + "atmImp_Faxa_bcph2", + "atmImp_Faxa_bcph3", + "atmImp_Faxa_ocph1", + "atmImp_Faxa_ocph2", + "atmImp_Faxa_ocph3", + "atmImp_Faxa_dstwet1", + "atmImp_Faxa_dstdry1", + "atmImp_Faxa_dstwet2", + "atmImp_Faxa_dstdry2", + "atmImp_Faxa_dstwet3", + "atmImp_Faxa_dstdry3", + "atmImp_Faxa_dstwet4", + "atmImp_Faxa_dstdry4", + ] + DEFAULT_YEAR = 1850 + PBS_JOBNAME = "aero_1850_monthly_clim" + WRAPPER = os.path.join("tools", "cplhist", "aero_1850_monthly_clim") + TITLE = "Monthly Climatology of Daily Averaged Atmosphere Coupler History Files" + COMMENT = "Monthly climatology created from daily averaged CPLHIST files for DATM" + + +def main(): + """ + Main program for aero_1850_monthly_clim + """ + Aero1850MonthlyClim.main() diff --git a/python/ctsm/cplhist/cplhist_monthly_clim.py b/python/ctsm/cplhist/cplhist_monthly_clim.py new file mode 100644 index 0000000000..4e88bfca54 --- /dev/null +++ b/python/ctsm/cplhist/cplhist_monthly_clim.py @@ -0,0 +1,959 @@ +""" +Generic code to create a 12-month climatology from daily CMEPS coupler history (CPLHIST) files, +for use by DATM. + +A specific product (for example aerosol deposition for 1850, see aero_1850_monthly_clim.py) is a +subclass of CplhistMonthlyClim that sets the class attributes for its variables, metadata and +tool name, and can override the methods for the steps that are different for it. + +The input files are the daily averaged atmosphere coupler history files: + + ARCHDIR/CASE/cpl/hist/CASE.cpl.hx.atm.24h.avrg.YYYY-MM-DD-00000.nc + +for the years BEGYEAR to ENDYEAR. The steps are: + + 1. Average the daily files of each month of each year (ncra) + 2. Average each calendar month over all the years (ncra), and put the 12 months in one + file (ncrcat) + 3. Set the time axis to the middle of each month of YEAR (noleap calendar) + +The output file is: + + OUTDIR/CASE/proc/CASE.cpl.hx.atm.OUTPUT_TAG.BEGYEAR-ENDYEAR_cYYYYMMDD.nc + +This needs the NCO commands ncra, ncrcat and ncatted (on Derecho: module load nco). +""" + +import argparse +import concurrent.futures # Handles parallel multi-processing with ncra jobs on different threads +import datetime +import getpass +import io +import logging +import os +import re +import shlex +import shutil +import socket +import subprocess +import sys + +from netCDF4 import Dataset # pylint: disable=no-name-in-module + +from ctsm.args_utils import comma_separated_list +from ctsm.ctsm_logging import ( + setup_logging_pre_config, + add_logging_args, + process_logging_args, + log, +) +from ctsm.git_utils import get_ctsm_git_short_hash +from ctsm.os_utils import run_cmd_output_on_error +from ctsm.path_utils import path_to_ctsm_root +from ctsm.toolchain.gen_mksurfdata_jobscript_single import write_runscript_part1 +from ctsm.utils import abort + +logger = logging.getLogger(__name__) + +# Grid fields that are always written to the output, whatever variable list is used +ALWAYS_VARS = ["atmImp_lon", "atmImp_lat"] +# Days in each month of the noleap calendar +NOLEAP_DAYS = (31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31) +# File name tag of the daily averaged atmosphere coupler history files +HIST_TAG = "cpl.hx.atm.24h.avrg" +# NCO commands that are needed +NCO_COMMANDS = ("ncra", "ncrcat", "ncatted") +# Number of simultaneous ncra jobs inside a PBS batch job, and otherwise (on a login node) +PBS_NJOBS = 64 +INTERACTIVE_NJOBS = 12 +# PBS batch settings +DEFAULT_ACCOUNT = "P93300606" +DEFAULT_QUEUE = "main" + + +def default_njobs(): + """ + Number of simultaneous ncra jobs when not given: more inside a PBS batch job + """ + if os.environ.get("PBS_JOBID"): + return PBS_NJOBS + return INTERACTIVE_NJOBS + + +def input_dir(archdir, case): + """ + Directory with the daily coupler history files + """ + return os.path.join(archdir, case, "cpl", "hist") + + +def output_dir(archdir, outdir, case): + """ + Directory for the output (OUTDIR/CASE/proc, where OUTDIR defaults to ARCHDIR) + """ + if outdir is None: + outdir = archdir + return os.path.join(outdir, case, "proc") + + +def build_var_list(variables, always_vars): + """ + Combine the variables to average with the ones that are always output + + Keeps the order given, drops duplicates and empty names. Returns None when variables is None, + meaning all variables. + """ + if variables is None: + return None + var_list = [] + for var in list(variables) + list(always_vars): + if var and var not in var_list: + var_list.append(var) + return var_list + + +def averaged_vars(variables): + """ + The variables for ncra to average: the ones given, without duplicates, empty names or the + ALWAYS_VARS (which are copied to the output without the time dimension instead). Returns + None when variables is None, meaning all variables. + """ + if variables is None: + return None + return [var for var in build_var_list(variables, []) if var not in ALWAYS_VARS] + + +def year_months(begyr, endyr): + """ + List of (year, month) to process, from BEGYEAR-01 to ENDYEAR-12 + """ + return [(year, month) for year in range(begyr, endyr + 1) for month in range(1, 13)] + + +def index_daily_files(indir, case): + """ + Find the daily coupler history files for this case, matching them by name (not by the time + coordinate in the files) + + Returns a dictionary of sorted file path lists, with (year, month) keys + """ + pattern = re.compile( + rf"^{re.escape(case)}\.{re.escape(HIST_TAG)}\.(\d{{4}})-(\d{{2}})-(\d{{2}})-00000\.nc$" + ) + files_by_month = {} + for filename in sorted(os.listdir(indir)): + match = pattern.match(filename) + if match: + key = (int(match.group(1)), int(match.group(2))) + files_by_month.setdefault(key, []).append(os.path.join(indir, filename)) + return files_by_month + + +def check_day_counts(files_by_month, yms): + """ + Check that every month has a full set of daily files (noleap calendar) + + Returns a list of error messages, empty if all the months are complete + """ + errors = [] + for year, month in yms: + nfiles = len(files_by_month.get((year, month), [])) + expect = NOLEAP_DAYS[month - 1] + if nfiles != expect: + errors.append(f"{year:04d}-{month:02d} has {nfiles} daily files (expected {expect})") + return errors + + +def file_vars(filename): + """ + Return the set of variable names in the netCDF file + """ + with Dataset(filename) as ncfile: + return set(ncfile.variables) + + +def missing_vars(filename, var_list): + """ + Return the variables in var_list that are not in the netCDF file + """ + in_file = file_vars(filename) + return [var for var in var_list if var not in in_file] + + +def check_nco(): + """ + Abort if any of the needed NCO commands can't be found + """ + missing = [cmd for cmd in NCO_COMMANDS if shutil.which(cmd) is None] + if missing: + abort(f"NCO commands not found: {' '.join(missing)} (on Derecho: module load nco)") + + +def ncra_command(infiles, outfile, var_list=None, *, exclude=None, skip_existing=False): + """ + The ncra command (as a list) to average the input files into outfile + + var_list is the variables to average (None for all). When var_list is None, the variables + in exclude (if given) are left out. + + With skip_existing, if outfile already exists, returns an empty list (no command to run) + and logs a message + """ + if skip_existing and os.path.exists(outfile): + log(logger, f"File already exists: {outfile} so skipping ncra command") + return [] + # Use --hst option so that history attribute isn't appended to as it's too long + # Use -O to overwrite the output file without asking (existing files are skipped above + # if they should be kept) + cmd = ["ncra", "-O", "--hst"] + if var_list is not None: + cmd += ["-v", ",".join(var_list)] + elif exclude: + cmd += ["-x", "-v", ",".join(exclude)] + return cmd + list(infiles) + ["-o", outfile] + + +def run_jobs(commands, njobs, *, progress_every=None, runner=run_cmd_output_on_error): + """ + Run the commands (each a list, with the output file last), with at most njobs at once + + Stops at the first failure: the jobs that have not started are cancelled and the runner's + error (normally an abort) is raised. Logs progress every progress_every completed jobs. + Empty commands (for output files that already exist) are skipped. + + Args: + commands (list): The commands to run, each a list of the command and its arguments, + with the output file last (an empty list is skipped) + njobs (int): Number of jobs to run simultaneously + progress_every (int): Log the progress every time this many jobs have completed + runner (function): Python function to call to run each command + + Returns: + None + + Raises: + SystemExit: From the runner's abort if a command fails + KeyboardInterrupt: If the user hits Ctrl-C + """ + commands = [cmd for cmd in commands if cmd] + if not commands: + return + ndone = 0 # number of jobs that have completed + # concurrent.futures executes up to njobs at the same time, but prepares all of them + with concurrent.futures.ThreadPoolExecutor(max_workers=njobs) as executor: + futures = [executor.submit(runner, cmd, f"Failed creating {cmd[-1]}") for cmd in commands] + try: + # Check which jobs are completed + for future in concurrent.futures.as_completed(futures): + future.result() + ndone += 1 + # Report on the progress + if progress_every and ndone % progress_every == 0: + log(logger, f" completed {ndone} of {len(commands)}") + except BaseException: + # Includes KeyboardInterrupt and the SystemExit from abort + # Will die if the user hits Ctrl-C or if the batch job is killed + + # Cancel all the future jobs + for future in futures: + future.cancel() + raise + + +def mon_file(mondir, year, month): + """ + Intermediate file with the monthly mean for one year and month + """ + return os.path.join(mondir, f"mon_{year:04d}-{month:02d}.nc") + + +def clim_file(mondir, month): + """ + Intermediate file with the climatology for one calendar month + """ + return os.path.join(mondir, f"clim_{month:02d}.nc") + + +def check_reused_mon_file(monfile, daily_files, var_list): + """ + Abort if a monthly file left from an earlier run doesn't have all the variables this run + needs (for example if it was made with a different variable list) + + With all variables (var_list is None), the monthly file must have all the variables in the + first daily file, except the ALWAYS_VARS. + """ + in_monfile = file_vars(monfile) + if var_list is None: + var_list = sorted(file_vars(daily_files[0]) - set(ALWAYS_VARS)) + missing = [var for var in var_list if var not in in_monfile] + if missing: + abort( + f"Existing monthly file {monfile} is missing variables: {' '.join(missing)}" + " (it was probably made with a different variable list, remove it and rerun)" + ) + + +def output_filename( # pylint: disable=too-many-positional-arguments + outdir, case, begyr, endyr, todaysdate, tag="monclim" +): + """ + Name of the output climatology file, with the year range and creation date + """ + return os.path.join( + outdir, f"{case}.cpl.hx.atm.{tag}.{begyr:04d}-{endyr:04d}_c{todaysdate:%Y%m%d}.nc" + ) + + +def month_bounds(): + """ + Start and end of each month, in days since the start of a noleap year + """ + bounds = [] + start = 0 + for ndays in NOLEAP_DAYS: + bounds.append((start, start + ndays)) + start += ndays + return bounds + + +def mid_month_times(): + """ + Middle of each month, in days since the start of a noleap year + """ + return [(start + end) / 2.0 for start, end in month_bounds()] + + +def climatology_bounds_values(begyr, endyr, year): + """ + CF climatology bounds for each calendar month, in days since year-01-01 (noleap calendar) + + Each month's bounds span all the averaged years: from the start of the month in begyr + to the end of the month in endyr (for January: begyr-01-01 to endyr-02-01). The values + are negative for years before year. + """ + days_per_year = sum(NOLEAP_DAYS) + return [ + ((begyr - year) * days_per_year + start, (endyr - year) * days_per_year + end) + for start, end in month_bounds() + ] + + +def set_time_axis(filename, year, begyr, endyr): + """ + Set the time of the 12-month file to the middle of each month of year, and its CF + climatology bounds to span the averaged years begyr to endyr, in the noleap calendar + """ + units = f"days since {year:04d}-01-01 00:00:00" + with Dataset(filename, "a") as ncfile: + time = ncfile.variables["time"] + time[:] = mid_month_times() + time.setncattr("units", units) + time.setncattr( + "long_name", "climatological time at exact middle of each month of climatology_bounds" + ) + time.setncattr("standard_name", "time") + time.setncattr("calendar", "noleap") + # For a climatology, CF uses the climatology attribute instead of bounds, and time + # itself has no cell_methods (ncra adds one) + for attr in ("bounds", "cell_methods"): + if attr in time.ncattrs(): + time.delncattr(attr) + time.setncattr("climatology", "climatology_bounds") + # + # Change time_bnds (from CMEPS) to climatology_bounds or create it if it doesn't exist + # + if "time_bnds" in ncfile.variables: + ncfile.renameVariable("time_bnds", "climatology_bounds") + if "climatology_bounds" not in ncfile.variables: + if "ntb" not in ncfile.dimensions: + ncfile.createDimension("ntb", 2) + ncfile.createVariable("climatology_bounds", "f8", ("time", "ntb")) + climatology_bounds = ncfile.variables["climatology_bounds"] + climatology_bounds[:] = climatology_bounds_values(begyr, endyr, year) + # Remove attributes from the renamed time_bnds that don't apply to bounds + for attr in ("bounds", "cell_methods", "coordinates"): + if attr in climatology_bounds.ncattrs(): + climatology_bounds.delncattr(attr) + climatology_bounds.setncattr("units", units) + climatology_bounds.setncattr("long_name", "climatology time interval endpoints") + climatology_bounds.setncattr("standard_name", "time") + # Bounds take axis and calendar from their coordinate (time), so CF checkers report + # these as non-standard on a bounds variable; remove them (time_bnds from CMEPS has them) + for attr in ("axis", "calendar"): + if attr in climatology_bounds.ncattrs(): + climatology_bounds.delncattr(attr) + + +def copy_grid_vars(src_file, dst_file, names): + """ + Copy the grid variables (names) from src_file (a daily file) to dst_file, without the time + dimension, using the first time slice + + The grid doesn't change in time, so the first time slice is the same as all the others (this + isn't checked). Missing dimensions are created in dst_file, and the attributes are copied + (without cell_methods, which is for data variables). + """ + with Dataset(src_file) as src, Dataset(dst_file, "a") as dst: + for name in names: + src_var = src.variables[name] + dims = tuple(dim for dim in src_var.dimensions if dim != "time") + for dim in dims: + if dim not in dst.dimensions: + dst.createDimension(dim, len(src.dimensions[dim])) + attrs = {attr: src_var.getncattr(attr) for attr in src_var.ncattrs()} + # _FillValue can only be set when the variable is created + fill_value = attrs.pop("_FillValue", None) + attrs.pop("cell_methods", None) + dst_var = dst.createVariable(name, src_var.dtype, dims, fill_value=fill_value) + dst_var.setncatts(attrs) + # Index 0 for time, all of every other dimension + first_time = tuple(0 if dim == "time" else slice(None) for dim in src_var.dimensions) + dst_var[...] = src_var[first_time] + + +def add_var_attributes(filename): + """ + Add the climatology cell_methods and the lon/lat coordinates attributes to the averaged data + variables (the variables on the time dimension, other than time, its bounds and lon/lat) + """ + with Dataset(filename, "a") as ncfile: + coords = " ".join(var for var in ALWAYS_VARS if var in ncfile.variables) + for name, var in ncfile.variables.items(): + if name in ALWAYS_VARS: + # cell_methods is for data variables, not coordinates like lon/lat (copy_grid_vars + # already leaves it out, this is just in case) + if "cell_methods" in var.ncattrs(): + var.delncattr("cell_methods") + continue + if name in ("time", "climatology_bounds"): + continue + if "time" not in var.dimensions: + continue + var.setncattr("cell_methods", "time: mean within years time: mean over years") + # Remove standard_name, as the CMEPS field names aren't CF-Convention standard names + if "standard_name" in var.ncattrs(): + var.delncattr("standard_name") + # Use the variable name as the long_name, unless the file already has one + if "long_name" not in var.ncattrs(): + var.setncattr("long_name", name) + if coords: + var.setncattr("coordinates", coords) + + +def submit_batch(script, queue): + """ + Submit the batch job script to PBS (qsub reads it from stdin), and return the job id + """ + if shutil.which("qsub") is None: + abort("qsub not found, can not submit with --batch") + result = subprocess.run( + ["qsub", "-q", queue], input=script, capture_output=True, text=True, check=False + ) + if result.returncode != 0: + abort(f"qsub failed:\n{result.stdout}{result.stderr}") + return result.stdout.strip() + + +class CplhistMonthlyClim: # pylint: disable=too-many-instance-attributes + """ + Monthly climatology from daily CPLHIST files + + This is the base class, a product subclasses it and sets the class attributes that are None + here. The methods are the steps of the work, which a product can override when it needs to do + something different (for example a different averaging, or files from several cases). + """ + + # Variables averaged by default (required) + DEFAULT_VARS = None + # Year to set the climatology time axis to + DEFAULT_YEAR = 1850 + # Tag in the output file name: CASE.cpl.hx.atm.OUTPUT_TAG.BEGYEAR-ENDYEAR_cYYYYMMDD.nc + OUTPUT_TAG = "monclim" + # PBS batch job name (required) + PBS_JOBNAME = None + # The executable wrapper for the product, run by the batch job, e.g. + # os.path.join("tools", "cplhist", "tool_name") (required) + WRAPPER = None + DEFAULT_WALLTIME = "04:00:00" + # Global title and comment attributes of the output file (required), + # " for cyclical year YEAR" is added to the title + TITLE = None + COMMENT = None + + _REQUIRED = ("DEFAULT_VARS", "PBS_JOBNAME", "WRAPPER", "TITLE", "COMMENT") + + def __init_subclass__(cls, **kwargs): + """ + Make sure a product subclass sets the required class attributes + (so a mistake shows up when the module is imported, not part way through a run) + """ + super().__init_subclass__(**kwargs) + missing = [name for name in cls._REQUIRED if getattr(cls, name) is None] + if missing: + raise TypeError(f"{cls.__name__} must set the class attributes: {', '.join(missing)}") + + @classmethod + def get_parser(cls): + """ + Get the parser object, with the description from the product's module docstring + """ + parser = argparse.ArgumentParser( + description=sys.modules[cls.__module__].__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument( + "archdir", help="Archive directory with the case directories (ARCHDIR/CASE/cpl/hist)" + ) + parser.add_argument("case", help="Case name") + parser.add_argument( + "begyear", type=int, help="First year of the daily files to average (inclusive)" + ) + parser.add_argument( + "endyear", type=int, help="Last year of the daily files to average (inclusive)" + ) + + vars_group = parser.add_mutually_exclusive_group() + vars_group.add_argument( + "--vars", + help=f"""Comma-separated list of variables to average + ({",".join(ALWAYS_VARS)} are always included, copied from the first daily file + without the time dimension) + (default: {",".join(cls.DEFAULT_VARS)})""", + action="store", + dest="vars", + type=comma_separated_list, + default=cls.DEFAULT_VARS, + ) + vars_group.add_argument( + "-a", + "--all-vars", + help="Average all variables in the files", + action="store_true", + dest="all_vars", + ) + parser.add_argument( + "-o", + "--outdir", + help="""Output directory if different from ARCHDIR. Files go under OUTDIR/CASE/proc + (default: ARCHDIR)""", + action="store", + dest="outdir", + default=None, + ) + parser.add_argument( + "-y", + "--year", + help="Year to set the climatology time axis to (default: %(default)s)", + action="store", + dest="year", + type=int, + default=cls.DEFAULT_YEAR, + ) + parser.add_argument( + "-k", + "--keep", + help="Keep the intermediate files in the OUTDIR/CASE/proc/mon directory", + action="store_true", + dest="keep", + ) + parser.add_argument( + "-j", + "--njobs", + help=f"""Number of ncra jobs to run at the same time + (default: {PBS_NJOBS} in a PBS batch job, {INTERACTIVE_NJOBS} otherwise)""", + action="store", + dest="njobs", + type=int, + default=None, + ) + parser.add_argument( + "-b", + "--batch", + help="Check the input, then submit this tool to the PBS batch queue with the same " + "options", + action="store_true", + dest="batch", + ) + parser.add_argument( + "-A", + "--account", + help=f"""PBS project account to charge with --batch + (default: $ACCOUNT if set, otherwise {DEFAULT_ACCOUNT})""", + action="store", + dest="account", + default=None, + ) + parser.add_argument( + "--queue", + help="PBS queue to use with --batch (default: %(default)s)", + action="store", + dest="queue", + default=DEFAULT_QUEUE, + ) + parser.add_argument( + "--walltime", + help="PBS wall clock time limit to use with --batch (default: %(default)s)", + action="store", + dest="walltime", + default=cls.DEFAULT_WALLTIME, + ) + cls.add_product_args(parser) + add_logging_args(parser) + return parser + + @classmethod + def add_product_args(cls, parser): + """ + Add command line arguments specific to a product (none in the base class) + """ + + def __init__(self, args): + """ + Set up from the parsed command line arguments (check_args checks them) + """ + self.args = args + self.njobs = args.njobs if args.njobs is not None else default_njobs() + # Variables for ncra to average (None for all), and the variables the input must have + self.var_list = None if args.all_vars else averaged_vars(args.vars) + self.check_list = build_var_list(self.var_list or [], ALWAYS_VARS) + self.indir = input_dir(args.archdir, args.case) + self.outdir = output_dir(args.archdir, args.outdir, args.case) + self.mondir = os.path.join(self.outdir, "mon") + self.todaysdate = datetime.date.today() + self.yms = year_months(args.begyear, args.endyear) + # Set by check_input + self.files_by_month = None + + def check_args(self): + """ + Check the command line arguments, aborting if there is a problem + """ + args = self.args + for name, year in ( + ("BEGYEAR", args.begyear), + ("ENDYEAR", args.endyear), + ("YEAR", args.year), + ): + if not 1 <= year <= 9999: + abort(f"{name} must be an integer from 1 to 9999: {year}") + if args.begyear > args.endyear: + abort(f"BEGYEAR ({args.begyear}) must not be after ENDYEAR ({args.endyear})") + if args.njobs is not None and args.njobs < 1: + abort(f"NJOBS must be a positive integer: {args.njobs}") + if args.account is not None and not args.batch: + abort("--account is only used with --batch") + if not args.all_vars and not averaged_vars(args.vars): + abort(f"No variables to average given with --vars (besides {','.join(ALWAYS_VARS)})") + if not os.path.isdir(args.archdir): + abort(f"Input archive directory {args.archdir} does not exist") + if args.outdir is not None and not os.path.isdir(args.outdir): + abort(f"Output archive directory {args.outdir} does not exist") + if not os.path.isdir(self.indir): + abort(f"Input case archive directory {self.indir} does not exist") + # + # Abort if the output file already exists + # + outfile = self.output_file() + if os.path.exists(outfile): + abort( + f"Output file {outfile} already exists, remove it or use a different output " + "directory" + ) + + def output_file(self): + """ + Name of the output file for this run + """ + return output_filename( + self.outdir, + self.args.case, + self.args.begyear, + self.args.endyear, + self.todaysdate, + tag=self.OUTPUT_TAG, + ) + + def input_files_by_month(self): + """ + The daily files to average for each month: a dictionary of sorted file path lists, + with (year, month) keys + """ + return index_daily_files(self.indir, self.args.case) + + def check_input(self): + """ + Check the input files before doing any work: every variable in check_list (the requested + variables and the ALWAYS_VARS) is in the first file, and every month in the range has a + full set of daily files + + Sets files_by_month to the daily files for each month + """ + yms = self.yms + log( + logger, + f"Processing {len(yms)} year-months: {self.args.begyear:04d}-01 .." + f" {self.args.endyear:04d}-12", + ) + files_by_month = self.input_files_by_month() + + first_files = files_by_month.get(yms[0], []) + if not first_files: + abort(f"No files for {yms[0][0]:04d}-{yms[0][1]:02d} in {self.indir}") + log(logger, f"Checking the requested variables are in {first_files[0]}") + missing = missing_vars(first_files[0], self.check_list) + if missing: + abort(f"Variables not found in {first_files[0]}: {' '.join(missing)}") + + log(logger, "Checking that every month has the expected number of daily files") + errors = check_day_counts(files_by_month, yms) + if errors: + abort("\n".join(errors)) + self.files_by_month = files_by_month + + def monthly_means(self): + """ + Step 1: average the daily files of each month of each year + + Monthly files left from an earlier run (for example with --keep) are reused, after checking + that they have the variables needed. The ALWAYS_VARS are never averaged (with all + variables they are excluded), as they are copied to the output at the end without time. + """ + log(logger, f"Computing monthly means for each year ({self.njobs} ncra jobs at a time)") + commands = [] + for year, month in self.yms: + monfile = mon_file(self.mondir, year, month) + daily_files = self.files_by_month[(year, month)] + if os.path.exists(monfile): + check_reused_mon_file(monfile, daily_files, self.var_list) + commands.append( + ncra_command( + daily_files, monfile, self.var_list, exclude=ALWAYS_VARS, skip_existing=True + ) + ) + # Show progress every 10 years + run_jobs(commands, self.njobs, progress_every=120) + + def monthly_climatology(self): + """ + Step 2: average each calendar month over the years + + Only this run's years are used, so leftover files from an earlier --keep run are not + included. The climatology files are always recreated, so ones left from an earlier run + with a different year range are not reused. + """ + begyr, endyr = self.args.begyear, self.args.endyear + log(logger, f"Averaging each calendar month over the years {begyr}-{endyr}") + commands = [ + ncra_command( + [mon_file(self.mondir, year, month) for year in range(begyr, endyr + 1)], + clim_file(self.mondir, month), + ) + for month in range(1, 13) + ] + run_jobs(commands, self.njobs) + + def average(self): + """ + Do the averaging steps, ending with the 12 climatology files in mondir + """ + self.monthly_means() + self.monthly_climatology() + + def set_time_axis(self, outfile): + """ + Set the time axis and its bounds on the output file + """ + set_time_axis(outfile, self.args.year, self.args.begyear, self.args.endyear) + + def provenance_attrs(self): + """ + Global attributes saying what the file is, and when, by whom and with what it was created, + as a list of (name, value) + """ + args = self.args + created_with = f"./{os.path.basename(self.WRAPPER)} -- {get_ctsm_git_short_hash()}" + source = ( + f"Created from daily averaged CESM3.0 CPLHIST files for case {args.case} for " + + f"years {args.begyear:04d}-{args.endyear:04d}" + ) + institution = ( + "National Science Foundation (NSF) - National Center for Atmospheric Research" + + " (NCAR) Community Earth System Model (CESM) project" + ) + return [ + ("title", f"{self.TITLE} for cyclical year {args.year}"), + ("comment", self.COMMENT), + ("source", source), + ("institution", institution), + # Use CF-1.11 as the compliance-checker doesn't support CF-1.12 yet + ("Conventions", "CF-1.11"), + ("Created_on", f"{self.todaysdate}"), + ("Created_by", getpass.getuser()), + ("Created_with", created_with), + ("command_used", shlex.join(sys.argv)), + ("time_period_freq", "month_1"), + ("hostname", socket.gethostname()), + # NOTE: this is the CESM2 DOI, and should be updated when a CESM3 DOI is available + # ("model_doi_url", "https://doi.org/10.5065/D67H1H0V"), + # TODO: Uncomment the above when we have a DOI for CESM3.0 + ("made_from_cesm_case", args.case), + ] + + def write_provenance(self, filename): + """ + Add the provenance_attrs global attributes to the file (like update_metadata in + ctsm/site_and_regional/base_case.py, but using ncatted) + """ + logger.debug( + "Write provenance information out to %s for year=%s case=%s years=%s-%s", + filename, + self.args.year, + self.args.case, + self.args.begyear, + self.args.endyear, + ) + # Use Overwrite option since the file already exists and we want to add attributes to it + # Use --hst option so that history attribute isn't appended as it's not helpful + cmd = ["ncatted", "-O", "--hst"] + for name, value in self.provenance_attrs(): + cmd += ["-a", f"{name},global,o,c,{value}"] + # Delete global attributes that don't apply to the climatology (in the same ncatted call, + # so --hst covers it too) + for name in ("file_version",): + cmd += ["-a", f"{name},global,d,,"] + run_cmd_output_on_error(cmd + [filename], f"Failed adding provenance to {filename}") + + def write_output(self, outfile): + """ + Step 3: put the 12 monthly climatologies in one file, copy the grid variables (ALWAYS_VARS) + from the first daily file without time, reset the time axis, and add provenance + attributes + """ + log(logger, "Putting the climatology into one file and finalizing its metadata") + clim_files = [clim_file(self.mondir, month) for month in range(1, 13)] + # -O so ncrcat never stops to ask about overwriting (check_args already made sure the + # output file didn't exist when the run started) + run_cmd_output_on_error( + ["ncrcat", "-O"] + clim_files + [outfile], f"Failed creating {outfile}" + ) + copy_grid_vars(self.files_by_month[self.yms[0]][0], outfile, ALWAYS_VARS) + self.set_time_axis(outfile) + add_var_attributes(outfile) + self.write_provenance(outfile) + log(logger, f"Wrote {outfile} (time = mid-month of year {self.args.year:04d}, noleap)") + + def job_args(self): + """ + The command line arguments for the batch job: the same as given, without the batch options + """ + args = self.args + argv = [args.archdir, args.case, str(args.begyear), str(args.endyear)] + if args.all_vars: + argv.append("--all-vars") + elif args.vars != self.DEFAULT_VARS: + argv += ["--vars", ",".join(args.vars)] + if args.outdir is not None: + argv += ["--outdir", args.outdir] + if args.year != self.DEFAULT_YEAR: + argv += ["--year", str(args.year)] + if args.keep: + argv.append("--keep") + if args.njobs is not None: + argv += ["--njobs", str(args.njobs)] + for option in ("verbose", "silent", "debug"): + if getattr(args, option): + argv.append(f"--{option}") + return argv + + def batch_script(self, argv, *, account, walltime, workdir, njobs): + """ + Text of the PBS batch job script that runs this tool with the arguments argv + Written out to a variable so it can be submitted to qsub from stdin, + instead of writing a temporary file + + njobs is the number of simultaneous ncra jobs the batch job will run + """ + runfile = io.StringIO() + write_runscript_part1( + number_of_nodes=1, + tasks_per_node=njobs, + machine="derecho", + account=account, + walltime=walltime, + runfile=runfile, + name=self.PBS_JOBNAME, + comment="This is a batch script to create a monthly climatology from daily CPLHIST " + "files", + ) + runfile.write("module load nco\n") + runfile.write(f"cd {shlex.quote(workdir)}\n") + wrapper = os.path.join(path_to_ctsm_root(), self.WRAPPER) + runfile.write(shlex.join([sys.executable, wrapper] + argv) + "\n") + return runfile.getvalue() + + def submit(self): + """ + Submit this tool with the same arguments (except the batch ones) to the PBS batch queue + """ + args = self.args + account = args.account or os.environ.get("ACCOUNT") or DEFAULT_ACCOUNT + argv = self.job_args() + # The batch job runs with the njobs given, or the default inside a PBS job + job_njobs = args.njobs if args.njobs is not None else PBS_NJOBS + script = self.batch_script( + argv, + account=account, + walltime=args.walltime, + workdir=os.getcwd(), + njobs=job_njobs, + ) + logger.debug("Batch job script:\n%s", script) + jobid = submit_batch(script, args.queue) + log( + logger, + f"Successfully submitted job {jobid} with arguments: {' '.join(argv)}" + f" (log file will be {self.PBS_JOBNAME}.o{jobid.split('.')[0]} in {os.getcwd()})", + ) + + def run(self): + """ + Check the arguments and the input, then either submit to the batch queue (--batch) + or create the climatology + """ + self.check_args() + if self.var_list is None: + log(logger, f"Averaging all variables except {','.join(ALWAYS_VARS)}") + else: + log(logger, f"Averaging variables: {','.join(self.var_list)}") + log(logger, f"Copying without time from the first daily file: {','.join(ALWAYS_VARS)}") + log(logger, "Create a monthly climatology from daily averaged CPLHIST files for DATM") + + check_nco() + self.check_input() + + # With --batch: the input has been checked, so submit to PBS and stop here + if self.args.batch: + self.submit() + return + + os.makedirs(self.mondir, exist_ok=True) + self.average() + outfile = self.output_file() + self.write_output(outfile) + + if self.args.keep: + log(logger, f"Kept intermediate files in {self.mondir}") + else: + shutil.rmtree(self.mondir) + + log(logger, f"Successfully created monthly climatology: {outfile}") + + @classmethod + def main(cls): + """ + Main program for a product: cls.main() runs it with the command line arguments + """ + setup_logging_pre_config() + args = cls.get_parser().parse_args() + process_logging_args(args) + cls(args).run() diff --git a/python/ctsm/cplhist/daily_to_monthly_clim.py b/python/ctsm/cplhist/daily_to_monthly_clim.py deleted file mode 100644 index 56c83053a0..0000000000 --- a/python/ctsm/cplhist/daily_to_monthly_clim.py +++ /dev/null @@ -1,899 +0,0 @@ -""" -Create a 12-month climatology from daily CMEPS coupler history (CPLHIST) files, for use by DATM. - -The input files are the daily averaged atmosphere coupler history files: - - ARCHDIR/CASE/cpl/hist/CASE.cpl.hx.atm.24h.avrg.YYYY-MM-DD-00000.nc - -for the years BEGYEAR to ENDYEAR. The steps are: - - 1. Average the daily files of each month of each year (ncra) - 2. Average each calendar month over all the years (ncra), and put the 12 months in one - file (ncrcat) - 3. Set the time axis to the middle of each month of YEAR (noleap calendar) - -The output file is: - - OUTDIR/CASE/proc/CASE.cpl.hx.atm.monclim.BEGYEAR-ENDYEAR_cYYYYMMDD.nc - -This needs the NCO commands ncra, ncrcat and ncatted (on Derecho: module load nco). -With --batch, the input is checked and then this tool is submitted to the PBS batch queue. - -A previous version of this for CESM2 is in NCL and here on Derecho by Keith Oleson: - /glade/u/home/oleson/misc_programs/CLM5Dev/CMIP6/CreateAeroDepFile_Task5B.ncl -It did 5 year averages over the historical period and then did a linear interpolation -between the 5-year averages. It also used CPLHIST output from several ensemble members and -averged them together with ncea. This is something that will have to be done in the future -in this tool as well. - -Another example for Nitrogen deposition is here from Simone Tilmes and Mike Mills: - https://svn.code.sf.net/p/codescripts/code/trunk/ncl/cam_forcing/CreateDepositionFile.ncl - -Neither were quite right for what's needed here. And both are in NCL which is well after -end of life now. - -Also there is code in LDF to create monthly climatologies, but it's not setup to bring in ensemble -averages and not specialized to create datafiles for DATM aerosol forcing. -""" - -import argparse -import concurrent.futures # Handles parallel multi-processing with ncra jobs on different threads -import datetime -import getpass -import io -import logging -import os -import re -import shlex -import shutil -import socket -import subprocess -import sys - -from netCDF4 import Dataset # pylint: disable=no-name-in-module - -from ctsm.args_utils import comma_separated_list -from ctsm.ctsm_logging import ( - setup_logging_pre_config, - add_logging_args, - process_logging_args, - log, -) -from ctsm.git_utils import get_ctsm_git_short_hash -from ctsm.os_utils import run_cmd_output_on_error -from ctsm.path_utils import path_to_ctsm_root -from ctsm.toolchain.gen_mksurfdata_jobscript_single import write_runscript_part1 -from ctsm.utils import abort - -logger = logging.getLogger(__name__) - -# Variables averaged by default -DEFAULT_VARS = [ - "atmImp_Faxa_bcph1", - "atmImp_Faxa_bcph2", - "atmImp_Faxa_bcph3", - "atmImp_Faxa_ocph1", - "atmImp_Faxa_ocph2", - "atmImp_Faxa_ocph3", - "atmImp_Faxa_dstwet1", - "atmImp_Faxa_dstdry1", - "atmImp_Faxa_dstwet2", - "atmImp_Faxa_dstdry2", - "atmImp_Faxa_dstwet3", - "atmImp_Faxa_dstdry3", - "atmImp_Faxa_dstwet4", - "atmImp_Faxa_dstdry4", -] -# Grid fields that are always written to the output, whatever variable list is used -ALWAYS_VARS = ["atmImp_lon", "atmImp_lat"] -DEFAULT_YEAR = 1850 -# Days in each month of the noleap calendar -NOLEAP_DAYS = (31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31) -# File name tag of the daily averaged atmosphere coupler history files -HIST_TAG = "cpl.hx.atm.24h.avrg" -# NCO commands that are needed -NCO_COMMANDS = ("ncra", "ncrcat", "ncatted") -# Number of simultaneous ncra jobs inside a PBS batch job, and otherwise (on a login node) -PBS_NJOBS = 64 -INTERACTIVE_NJOBS = 12 -# PBS batch settings -PBS_JOBNAME = "daily_to_monthly_clim" -DEFAULT_ACCOUNT = "P93300606" -DEFAULT_QUEUE = "main" -DEFAULT_WALLTIME = "04:00:00" -# The executable wrapper for this module, run by the batch job -WRAPPER = os.path.join("tools", "cplhist", "daily_to_monthly_clim") - - -def get_parser(): - """ - Get the parser object for daily_to_monthly_clim - """ - parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter - ) - parser.add_argument( - "archdir", help="Archive directory with the case directories (ARCHDIR/CASE/cpl/hist)" - ) - parser.add_argument("case", help="Case name") - parser.add_argument( - "begyear", type=int, help="First year of the daily files to average (inclusive)" - ) - parser.add_argument( - "endyear", type=int, help="Last year of the daily files to average (inclusive)" - ) - - vars_group = parser.add_mutually_exclusive_group() - vars_group.add_argument( - "--vars", - help=f"""Comma-separated list of variables to average - ({",".join(ALWAYS_VARS)} are always included, copied from the first daily file - without the time dimension) - (default: {",".join(DEFAULT_VARS)})""", - action="store", - dest="vars", - type=comma_separated_list, - default=DEFAULT_VARS, - ) - vars_group.add_argument( - "-a", - "--all-vars", - help="Average all variables in the files", - action="store_true", - dest="all_vars", - ) - parser.add_argument( - "-o", - "--outdir", - help="""Output directory if different from ARCHDIR. Files go under OUTDIR/CASE/proc - (default: ARCHDIR)""", - action="store", - dest="outdir", - default=None, - ) - parser.add_argument( - "-y", - "--year", - help="Year to set the climatology time axis to (default: %(default)s)", - action="store", - dest="year", - type=int, - default=DEFAULT_YEAR, - ) - parser.add_argument( - "-k", - "--keep", - help="Keep the intermediate files in the OUTDIR/CASE/proc/mon directory", - action="store_true", - dest="keep", - ) - parser.add_argument( - "-j", - "--njobs", - help=f"""Number of ncra jobs to run at the same time - (default: {PBS_NJOBS} in a PBS batch job, {INTERACTIVE_NJOBS} otherwise)""", - action="store", - dest="njobs", - type=int, - default=None, - ) - parser.add_argument( - "-b", - "--batch", - help="Check the input, then submit this tool to the PBS batch queue with the same options", - action="store_true", - dest="batch", - ) - parser.add_argument( - "-A", - "--account", - help=f"""PBS project account to charge with --batch - (default: $ACCOUNT if set, otherwise {DEFAULT_ACCOUNT})""", - action="store", - dest="account", - default=None, - ) - parser.add_argument( - "--queue", - help="PBS queue to use with --batch (default: %(default)s)", - action="store", - dest="queue", - default=DEFAULT_QUEUE, - ) - parser.add_argument( - "--walltime", - help="PBS wall clock time limit to use with --batch (default: %(default)s)", - action="store", - dest="walltime", - default=DEFAULT_WALLTIME, - ) - add_logging_args(parser) - return parser - - -def default_njobs(): - """ - Number of simultaneous ncra jobs when not given: more inside a PBS batch job - """ - if os.environ.get("PBS_JOBID"): - return PBS_NJOBS - return INTERACTIVE_NJOBS - - -def input_dir(archdir, case): - """ - Directory with the daily coupler history files - """ - return os.path.join(archdir, case, "cpl", "hist") - - -def output_dir(archdir, outdir, case): - """ - Directory for the output (OUTDIR/CASE/proc, where OUTDIR defaults to ARCHDIR) - """ - if outdir is None: - outdir = archdir - return os.path.join(outdir, case, "proc") - - -def check_args(args): - """ - Check the command line arguments, aborting if there is a problem - """ - for name, year in (("BEGYEAR", args.begyear), ("ENDYEAR", args.endyear), ("YEAR", args.year)): - if not 1 <= year <= 9999: - abort(f"{name} must be an integer from 1 to 9999: {year}") - if args.begyear > args.endyear: - abort(f"BEGYEAR ({args.begyear}) must not be after ENDYEAR ({args.endyear})") - if args.njobs is not None and args.njobs < 1: - abort(f"NJOBS must be a positive integer: {args.njobs}") - if args.account is not None and not args.batch: - abort("--account is only used with --batch") - if not args.all_vars and not averaged_vars(args.vars): - abort(f"No variables to average given with --vars (besides {','.join(ALWAYS_VARS)})") - if not os.path.isdir(args.archdir): - abort(f"Input archive directory {args.archdir} does not exist") - if args.outdir is not None and not os.path.isdir(args.outdir): - abort(f"Output archive directory {args.outdir} does not exist") - indir = input_dir(args.archdir, args.case) - if not os.path.isdir(indir): - abort(f"Input case archive directory {indir} does not exist") - # - # Abort if the output file already exists - # - todaysdate = datetime.date.today() - outdir = output_dir(args.archdir, args.outdir, args.case) - outfile = output_filename(outdir, args.case, args.begyear, args.endyear, todaysdate) - if os.path.exists(outfile): - abort( - f"Output file {outfile} already exists, remove it or use a different output directory" - ) - - -def build_var_list(variables, always_vars): - """ - Combine the variables to average with the ones that are always output - - Keeps the order given, drops duplicates and empty names. Returns None when variables is None, - meaning all variables. - """ - if variables is None: - return None - var_list = [] - for var in list(variables) + list(always_vars): - if var and var not in var_list: - var_list.append(var) - return var_list - - -def averaged_vars(variables): - """ - The variables for ncra to average: the ones given, without duplicates, empty names or the - ALWAYS_VARS (which are copied to the output without the time dimension instead). Returns - None when variables is None, meaning all variables. - """ - if variables is None: - return None - return [var for var in build_var_list(variables, []) if var not in ALWAYS_VARS] - - -def year_months(begyr, endyr): - """ - List of (year, month) to process, from BEGYEAR-01 to ENDYEAR-12 - """ - return [(year, month) for year in range(begyr, endyr + 1) for month in range(1, 13)] - - -def index_daily_files(indir, case): - """ - Find the daily coupler history files for this case, matching them by name (not by the time - coordinate in the files) - - Returns a dictionary of sorted file path lists, with (year, month) keys - """ - pattern = re.compile( - rf"^{re.escape(case)}\.{re.escape(HIST_TAG)}\.(\d{{4}})-(\d{{2}})-(\d{{2}})-00000\.nc$" - ) - files_by_month = {} - for filename in sorted(os.listdir(indir)): - match = pattern.match(filename) - if match: - key = (int(match.group(1)), int(match.group(2))) - files_by_month.setdefault(key, []).append(os.path.join(indir, filename)) - return files_by_month - - -def check_day_counts(files_by_month, yms): - """ - Check that every month has a full set of daily files (noleap calendar) - - Returns a list of error messages, empty if all the months are complete - """ - errors = [] - for year, month in yms: - nfiles = len(files_by_month.get((year, month), [])) - expect = NOLEAP_DAYS[month - 1] - if nfiles != expect: - errors.append(f"{year:04d}-{month:02d} has {nfiles} daily files (expected {expect})") - return errors - - -def file_vars(filename): - """ - Return the set of variable names in the netCDF file - """ - with Dataset(filename) as ncfile: - return set(ncfile.variables) - - -def missing_vars(filename, var_list): - """ - Return the variables in var_list that are not in the netCDF file - """ - in_file = file_vars(filename) - return [var for var in var_list if var not in in_file] - - -def check_nco(): - """ - Abort if any of the needed NCO commands can't be found - """ - missing = [cmd for cmd in NCO_COMMANDS if shutil.which(cmd) is None] - if missing: - abort(f"NCO commands not found: {' '.join(missing)} (on Derecho: module load nco)") - - -def check_input(args, var_list): - """ - Check the input files before doing any work: every variable in var_list (the requested - variables and the ALWAYS_VARS) is in the first file, and every month in the range has a full - set of daily files - - Returns the dictionary of daily files from index_daily_files - """ - indir = input_dir(args.archdir, args.case) - yms = year_months(args.begyear, args.endyear) - log( - logger, - f"Processing {len(yms)} year-months: {args.begyear:04d}-01 .. {args.endyear:04d}-12", - ) - files_by_month = index_daily_files(indir, args.case) - - first_files = files_by_month.get(yms[0], []) - if not first_files: - abort(f"No files for {yms[0][0]:04d}-{yms[0][1]:02d} in {indir}") - log(logger, f"Checking the requested variables are in {first_files[0]}") - missing = missing_vars(first_files[0], var_list) - if missing: - abort(f"Variables not found in {first_files[0]}: {' '.join(missing)}") - - log(logger, "Checking that every month has the expected number of daily files") - errors = check_day_counts(files_by_month, yms) - if errors: - abort("\n".join(errors)) - return files_by_month - - -def ncra_command(infiles, outfile, var_list=None, *, exclude=None, skip_existing=False): - """ - The ncra command (as a list) to average the input files into outfile - - var_list is the variables to average (None for all). When var_list is None, the variables - in exclude (if given) are left out. - - With skip_existing, if outfile already exists, returns an empty list (no command to run) - and logs a message - """ - if skip_existing and os.path.exists(outfile): - log(logger, f"File already exists: {outfile} so skipping ncra command") - return [] - # Use --hst option so that history attribute isn't appended to as it's too long - # Use -O to overwrite the output file without asking (existing files are skipped above - # if they should be kept) - cmd = ["ncra", "-O", "--hst"] - if var_list is not None: - cmd += ["-v", ",".join(var_list)] - elif exclude: - cmd += ["-x", "-v", ",".join(exclude)] - return cmd + list(infiles) + ["-o", outfile] - - -def run_jobs(commands, njobs, *, progress_every=None, runner=run_cmd_output_on_error): - """ - Run the commands (each a list, with the output file last), with at most njobs at once - - Stops at the first failure: the jobs that have not started are cancelled and the runner's - error (normally an abort) is raised. Logs progress every progress_every completed jobs. - Empty commands (for output files that already exist) are skipped. - - Args: - commands (list): The commands to run, each a list of the command and its arguments, - with the output file last (an empty list is skipped) - njobs (int): Number of jobs to run simultaneously - progress_every (int): Log the progress every time this many jobs have completed - runner (function): Python function to call to run each command - - Returns: - None - - Raises: - SystemExit: From the runner's abort if a command fails - KeyboardInterrupt: If the user hits Ctrl-C - """ - commands = [cmd for cmd in commands if cmd] - if not commands: - return - ndone = 0 # number of jobs that have completed - # concurrent.futures executes up to njobs at the same time, but prepares all of them - with concurrent.futures.ThreadPoolExecutor(max_workers=njobs) as executor: - futures = [executor.submit(runner, cmd, f"Failed creating {cmd[-1]}") for cmd in commands] - try: - # Check which jobs are completed - for future in concurrent.futures.as_completed(futures): - future.result() - ndone += 1 - # Report on the progress - if progress_every and ndone % progress_every == 0: - log(logger, f" completed {ndone} of {len(commands)}") - except BaseException: - # Includes KeyboardInterrupt and the SystemExit from abort - # Will die if the user hits Ctrl-C or if the batch job is killed - - # Cancel all the future jobs - for future in futures: - future.cancel() - raise - - -def mon_file(mondir, year, month): - """ - Intermediate file with the monthly mean for one year and month - """ - return os.path.join(mondir, f"mon_{year:04d}-{month:02d}.nc") - - -def clim_file(mondir, month): - """ - Intermediate file with the climatology for one calendar month - """ - return os.path.join(mondir, f"clim_{month:02d}.nc") - - -def check_reused_mon_file(monfile, daily_files, var_list): - """ - Abort if a monthly file left from an earlier run doesn't have all the variables this run - needs (for example if it was made with a different variable list) - - With all variables (var_list is None), the monthly file must have all the variables in the - first daily file, except the ALWAYS_VARS. The monthly file must not have the ALWAYS_VARS - (older versions of this tool averaged them with the time dimension). - """ - in_monfile = file_vars(monfile) - if var_list is None: - var_list = sorted(file_vars(daily_files[0]) - set(ALWAYS_VARS)) - missing = [var for var in var_list if var not in in_monfile] - if missing: - abort( - f"Existing monthly file {monfile} is missing variables: {' '.join(missing)}" - " (it was probably made with a different variable list, remove it and rerun)" - ) - - -def monthly_means(files_by_month, yms, mondir, var_list, njobs): - """ - Step 1: average the daily files of each month of each year - - Monthly files left from an earlier run (for example with --keep) are reused, after checking - that they have the variables needed. The ALWAYS_VARS are never averaged (with all - variables they are excluded), as they are copied to the output at the end without time. - """ - log(logger, f"Computing monthly means for each year ({njobs} ncra jobs at a time)") - commands = [] - for year, month in yms: - monfile = mon_file(mondir, year, month) - daily_files = files_by_month[(year, month)] - if os.path.exists(monfile): - check_reused_mon_file(monfile, daily_files, var_list) - commands.append( - ncra_command(daily_files, monfile, var_list, exclude=ALWAYS_VARS, skip_existing=True) - ) - # Show progress every 10 years - run_jobs(commands, njobs, progress_every=120) - - -def monthly_climatology(mondir, begyr, endyr, njobs): - """ - Step 2: average each calendar month over the years - - Only this run's years are used, so leftover files from an earlier --keep run are not included. - The climatology files are always recreated, so ones left from an earlier run with a - different year range are not reused. - """ - log(logger, f"Averaging each calendar month over the years {begyr}-{endyr}") - commands = [ - ncra_command( - [mon_file(mondir, year, month) for year in range(begyr, endyr + 1)], - clim_file(mondir, month), - ) - for month in range(1, 13) - ] - run_jobs(commands, njobs) - - -def output_filename(outdir, case, begyr, endyr, todaysdate): - """ - Name of the output climatology file, with the year range and creation date - """ - return os.path.join( - outdir, f"{case}.cpl.hx.atm.monclim.{begyr:04d}-{endyr:04d}_c{todaysdate:%Y%m%d}.nc" - ) - - -def month_bounds(): - """ - Start and end of each month, in days since the start of a noleap year - """ - bounds = [] - start = 0 - for ndays in NOLEAP_DAYS: - bounds.append((start, start + ndays)) - start += ndays - return bounds - - -def mid_month_times(): - """ - Middle of each month, in days since the start of a noleap year - """ - return [(start + end) / 2.0 for start, end in month_bounds()] - - -def climatology_bounds_values(begyr, endyr, year): - """ - CF climatology bounds for each calendar month, in days since year-01-01 (noleap calendar) - - Each month's bounds span all the averaged years: from the start of the month in begyr - to the end of the month in endyr (for January: begyr-01-01 to endyr-02-01). The values - are negative for years before year. - """ - days_per_year = sum(NOLEAP_DAYS) - return [ - ((begyr - year) * days_per_year + start, (endyr - year) * days_per_year + end) - for start, end in month_bounds() - ] - - -def set_time_axis(filename, year, begyr, endyr): - """ - Set the time of the 12-month file to the middle of each month of year, and its CF - climatology bounds to span the averaged years begyr to endyr, in the noleap calendar - """ - units = f"days since {year:04d}-01-01 00:00:00" - with Dataset(filename, "a") as ncfile: - time = ncfile.variables["time"] - time[:] = mid_month_times() - time.setncattr("units", units) - time.setncattr( - "long_name", "climatological time at exact middle of each month of climatology_bounds" - ) - time.setncattr("standard_name", "time") - time.setncattr("calendar", "noleap") - # For a climatology, CF uses the climatology attribute instead of bounds, and time - # itself has no cell_methods (ncra adds one) - for attr in ("bounds", "cell_methods"): - if attr in time.ncattrs(): - time.delncattr(attr) - time.setncattr("climatology", "climatology_bounds") - # - # Change time_bnds (from CMEPS) to climatology_bounds or create it if it doesn't exist - # - if "time_bnds" in ncfile.variables: - ncfile.renameVariable("time_bnds", "climatology_bounds") - if "climatology_bounds" not in ncfile.variables: - if "ntb" not in ncfile.dimensions: - ncfile.createDimension("ntb", 2) - ncfile.createVariable("climatology_bounds", "f8", ("time", "ntb")) - climatology_bounds = ncfile.variables["climatology_bounds"] - climatology_bounds[:] = climatology_bounds_values(begyr, endyr, year) - # Remove attributes from the renamed time_bnds that don't apply to bounds - for attr in ("bounds", "cell_methods", "coordinates"): - if attr in climatology_bounds.ncattrs(): - climatology_bounds.delncattr(attr) - climatology_bounds.setncattr("units", units) - climatology_bounds.setncattr("long_name", "climatology time interval endpoints") - climatology_bounds.setncattr("standard_name", "time") - # Bounds take axis and calendar from their coordinate (time), so CF checkers report - # these as non-standard on a bounds variable; remove them (time_bnds from CMEPS has them) - for attr in ("axis", "calendar"): - if attr in climatology_bounds.ncattrs(): - climatology_bounds.delncattr(attr) - - -def copy_grid_vars(src_file, dst_file, names): - """ - Copy the grid variables (names) from src_file (a daily file) to dst_file, without the time - dimension, using the first time slice - - The grid doesn't change in time, so the first time slice is the same as all the others (this - isn't checked). Missing dimensions are created in dst_file, and the attributes are copied - (without cell_methods, which is for data variables). - """ - with Dataset(src_file) as src, Dataset(dst_file, "a") as dst: - for name in names: - src_var = src.variables[name] - dims = tuple(dim for dim in src_var.dimensions if dim != "time") - for dim in dims: - if dim not in dst.dimensions: - dst.createDimension(dim, len(src.dimensions[dim])) - attrs = {attr: src_var.getncattr(attr) for attr in src_var.ncattrs()} - # _FillValue can only be set when the variable is created - fill_value = attrs.pop("_FillValue", None) - attrs.pop("cell_methods", None) - dst_var = dst.createVariable(name, src_var.dtype, dims, fill_value=fill_value) - dst_var.setncatts(attrs) - # Index 0 for time, all of every other dimension - first_time = tuple(0 if dim == "time" else slice(None) for dim in src_var.dimensions) - dst_var[...] = src_var[first_time] - - -def add_var_attributes(filename): - """ - Add the climatology cell_methods and the lon/lat coordinates attributes to the averaged data - variables (the variables on the time dimension, other than time, its bounds and lon/lat) - """ - with Dataset(filename, "a") as ncfile: - coords = " ".join(var for var in ALWAYS_VARS if var in ncfile.variables) - for name, var in ncfile.variables.items(): - if name in ALWAYS_VARS: - # cell_methods is for data variables, not coordinates like lon/lat (copy_grid_vars - # already leaves it out, this is just in case) - if "cell_methods" in var.ncattrs(): - var.delncattr("cell_methods") - continue - if name in ("time", "climatology_bounds"): - continue - if "time" not in var.dimensions: - continue - var.setncattr("cell_methods", "time: mean within years time: mean over years") - # Remove standard_name, as the CMEPS field names aren't CF-Convention standard names - if "standard_name" in var.ncattrs(): - var.delncattr("standard_name") - # Use the variable name as the long_name, unless the file already has one - if "long_name" not in var.ncattrs(): - var.setncattr("long_name", name) - if coords: - var.setncattr("coordinates", coords) - - -def write_provenance( # pylint: disable=too-many-positional-arguments - case, filename, begyear, endyear, year, todaysdate -): - """ - Add global attributes saying when, by whom and with what the file was created (like - update_metadata in ctsm/site_and_regional/base_case.py, but using ncatted) - """ - logger.debug( - "Write provenance information out to %s for year=%s case=%s years=%s-%s", - filename, - year, - case, - begyear, - endyear, - ) - created_with = f"./{os.path.basename(WRAPPER)} -- {get_ctsm_git_short_hash()}" - # Use Overwrite option since the file already exists and we want to add attributes to it - # Use --hst option so that history attribute isn't appended as it's not helpful - cmd = ["ncatted", "-O", "--hst"] - comment = "Monthly climatology created from daily averaged CPLHIST files for DATM" - title = ( - "Monthly Climatology of Daily Averaged Atmosphere Coupler History Files" - + f" for cyclical year {year}" - ) - source = ( - f"Created from daily averaged CESM3.0 CPLHIST files for case {case} for " - + f"years {begyear:04d}-{endyear:04d}" - ) - institution = ( - "National Science Foundation (NSF) - National Center for Atmospheric Research" - + " (NCAR) Community Earth System Model (CESM) project" - ) - for name, value in ( - ("title", title), - ("comment", comment), - ("source", source), - ("institution", institution), - # Use CF-1.11 as the compliance-checker doesn't support CF-1.12 yet - ("Conventions", "CF-1.11"), - ("Created_on", f"{todaysdate}"), - ("Created_by", getpass.getuser()), - ("Created_with", created_with), - ("command_used", shlex.join(sys.argv)), - ("time_period_freq", "month_1"), - ("hostname", socket.gethostname()), - # NOTE: this is the CESM2 DOI, and should be updated when a CESM3 DOI is available - # ("model_doi_url", "https://doi.org/10.5065/D67H1H0V"), - # TODO: Uncomment the above when we have a DOI for CESM3.0 - ("made_from_cesm_case", case), - ): - cmd += ["-a", f"{name},global,o,c,{value}"] - # Delete global attributes that don't apply to the climatology (in the same ncatted call, - # so --hst covers it too) - for name in ("file_version",): - cmd += ["-a", f"{name},global,d,,"] - run_cmd_output_on_error(cmd + [filename], f"Failed adding provenance to {filename}") - - -def write_output( # pylint: disable=too-many-positional-arguments - case, mondir, outfile, begyear, endyear, year, todaysdate, *, grid_file -): - """ - Step 3: put the 12 monthly climatologies in one file, copy the grid variables (ALWAYS_VARS) - from grid_file (the first daily file) without time, reset the time axis to the middle of - each month of year, and add provenance attributes - """ - log(logger, "Putting the climatology into one file and finalizing its metadata") - clim_files = [clim_file(mondir, month) for month in range(1, 13)] - # -O so ncrcat never stops to ask about overwriting (check_args already made sure the - # output file didn't exist when the run started) - run_cmd_output_on_error(["ncrcat", "-O"] + clim_files + [outfile], f"Failed creating {outfile}") - copy_grid_vars(grid_file, outfile, ALWAYS_VARS) - set_time_axis(outfile, year, begyear, endyear) - add_var_attributes(outfile) - write_provenance(case, outfile, begyear, endyear, year, todaysdate) - log(logger, f"Wrote {outfile} (time = mid-month of year {year:04d}, noleap)") - - -def job_args(args): - """ - The command line arguments for the batch job: the same as given, without the batch options - """ - argv = [args.archdir, args.case, str(args.begyear), str(args.endyear)] - if args.all_vars: - argv.append("--all-vars") - elif args.vars != DEFAULT_VARS: - argv += ["--vars", ",".join(args.vars)] - if args.outdir is not None: - argv += ["--outdir", args.outdir] - if args.year != DEFAULT_YEAR: - argv += ["--year", str(args.year)] - if args.keep: - argv.append("--keep") - if args.njobs is not None: - argv += ["--njobs", str(args.njobs)] - for option in ("verbose", "silent", "debug"): - if getattr(args, option): - argv.append(f"--{option}") - return argv - - -def batch_script(argv, *, account, walltime, workdir, njobs): - """ - Text of the PBS batch job script that runs this tool with the arguments argv - Written out to a variable so it can be submitted to qsub from stdin, - instead of writing a temporary file - - njobs is the number of simultaneous ncra jobs the batch job will run - """ - runfile = io.StringIO() - write_runscript_part1( - number_of_nodes=1, - tasks_per_node=njobs, - machine="derecho", - account=account, - walltime=walltime, - runfile=runfile, - name=PBS_JOBNAME, - comment="This is a batch script to create a monthly climatology from daily CPLHIST files", - ) - runfile.write("module load nco\n") - runfile.write(f"cd {shlex.quote(workdir)}\n") - wrapper = os.path.join(path_to_ctsm_root(), WRAPPER) - runfile.write(shlex.join([sys.executable, wrapper] + argv) + "\n") - return runfile.getvalue() - - -def submit_batch(script, queue): - """ - Submit the batch job script to PBS (qsub reads it from stdin), and return the job id - """ - if shutil.which("qsub") is None: - abort("qsub not found, can not submit with --batch") - result = subprocess.run( - ["qsub", "-q", queue], input=script, capture_output=True, text=True, check=False - ) - if result.returncode != 0: - abort(f"qsub failed:\n{result.stdout}{result.stderr}") - return result.stdout.strip() - - -def main(): - """ - Main program for daily_to_monthly_clim - """ - setup_logging_pre_config() - args = get_parser().parse_args() - process_logging_args(args) - check_args(args) - - njobs = args.njobs if args.njobs is not None else default_njobs() - # Variables for ncra to average (None for all), and the variables the input must have - var_list = None if args.all_vars else averaged_vars(args.vars) - check_list = build_var_list(var_list or [], ALWAYS_VARS) - if var_list is None: - log(logger, f"Averaging all variables except {','.join(ALWAYS_VARS)}") - else: - log(logger, f"Averaging variables: {','.join(var_list)}") - log(logger, f"Copying without time from the first daily file: {','.join(ALWAYS_VARS)}") - log(logger, "Create a monthly climatology from daily averaged CPLHIST files for DATM") - - check_nco() - files_by_month = check_input(args, check_list) - - # With --batch: the input has been checked, so submit to PBS and stop here - if args.batch: - account = args.account or os.environ.get("ACCOUNT") or DEFAULT_ACCOUNT - argv = job_args(args) - # The batch job runs with the njobs given, or the default inside a PBS job - job_njobs = args.njobs if args.njobs is not None else PBS_NJOBS - script = batch_script( - argv, - account=account, - walltime=args.walltime, - workdir=os.getcwd(), - njobs=job_njobs, - ) - logger.debug("Batch job script:\n%s", script) - jobid = submit_batch(script, args.queue) - log( - logger, - f"Successfully submitted job {jobid} with arguments: {' '.join(argv)}" - f" (log file will be {PBS_JOBNAME}.o{jobid.split('.')[0]} in {os.getcwd()})", - ) - return - - outdir = output_dir(args.archdir, args.outdir, args.case) - mondir = os.path.join(outdir, "mon") - os.makedirs(mondir, exist_ok=True) - todaysdate = datetime.date.today() - - yms = year_months(args.begyear, args.endyear) - monthly_means(files_by_month, yms, mondir, var_list, njobs) - monthly_climatology(mondir, args.begyear, args.endyear, njobs) - outfile = output_filename(outdir, args.case, args.begyear, args.endyear, todaysdate) - write_output( - args.case, - mondir, - outfile, - args.begyear, - args.endyear, - args.year, - todaysdate, - grid_file=files_by_month[yms[0]][0], - ) - - if args.keep: - log(logger, f"Kept intermediate files in {mondir}") - else: - shutil.rmtree(mondir) - - log(logger, f"Successfully created monthly climatology: {outfile}") diff --git a/python/ctsm/test/test_sys_daily_to_monthly_clim.py b/python/ctsm/test/test_sys_aero_1850_monthly_clim.py similarity index 87% rename from python/ctsm/test/test_sys_daily_to_monthly_clim.py rename to python/ctsm/test/test_sys_aero_1850_monthly_clim.py index 863a00f77b..10cff77f35 100644 --- a/python/ctsm/test/test_sys_daily_to_monthly_clim.py +++ b/python/ctsm/test/test_sys_aero_1850_monthly_clim.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -"""System tests for daily_to_monthly_clim (needs the NCO commands, on Derecho: module load nco)""" +"""System tests for aero_1850_monthly_clim (needs the NCO commands, on Derecho: module load nco)""" import contextlib import datetime @@ -21,7 +21,8 @@ sys.path.insert(1, _CTSM_PYTHON) # pylint: disable=wrong-import-position from ctsm import unit_testing -from ctsm.cplhist import daily_to_monthly_clim as d2mc +from ctsm.cplhist import aero_1850_monthly_clim as aero +from ctsm.cplhist import cplhist_monthly_clim as mclim from ctsm.os_utils import run_cmd_output_on_error # Allow test names that pylint doesn't like; otherwise hard to make them @@ -66,7 +67,7 @@ def _write_daily_file(filename, year, month, day_of_year): time_bnds[:] = [[start, start + 0.5], [start + 0.5, start + 1.0]] dims = ("time", "atmImp_ny", "atmImp_nx") value = _value(year, month) - for name in d2mc.DEFAULT_VARS + [EXTRA_VAR]: + for name in aero.Aero1850MonthlyClim.DEFAULT_VARS + [EXTRA_VAR]: var = ncfile.createVariable(name, "f4", dims, fill_value=1.0e30) var.setncatts({"units": "kg m-2 s-1", "standard_name": name}) var[:] = (2.0 * value if name == EXTRA_VAR else value) * np.ones((2, 1, NX)) @@ -82,22 +83,22 @@ def _write_daily_file(filename, year, month, day_of_year): @unittest.skipUnless(shutil.which("ncra"), "NCO not found (on Derecho: module load nco)") class TestSysDailyToMonthlyClim(unittest.TestCase): - """System tests for daily_to_monthly_clim, running the NCO commands""" + """System tests for aero_1850_monthly_clim, running the NCO commands""" @classmethod def setUpClass(cls): """Write BEGYEAR to ENDYEAR of daily files, once for all the tests""" cls._indir_root = tempfile.mkdtemp() cls._archdir = os.path.join(cls._indir_root, "archive") - indir = d2mc.input_dir(cls._archdir, CASE) + indir = mclim.input_dir(cls._archdir, CASE) os.makedirs(indir) cls._first_daily = None for year in range(BEGYEAR, ENDYEAR + 1): day_of_year = 0 - for month, ndays in enumerate(d2mc.NOLEAP_DAYS, start=1): + for month, ndays in enumerate(mclim.NOLEAP_DAYS, start=1): for day in range(1, ndays + 1): filename = os.path.join( - indir, f"{CASE}.{d2mc.HIST_TAG}.{year:04d}-{month:02d}-{day:02d}-00000.nc" + indir, f"{CASE}.{mclim.HIST_TAG}.{year:04d}-{month:02d}-{day:02d}-00000.nc" ) _write_daily_file(filename, year, month, day_of_year) day_of_year += 1 @@ -124,13 +125,13 @@ def _run(self, *options, outdir=None): """ if outdir is None: outdir = tempfile.mkdtemp(dir=self._tempdir) - argv = ["daily_to_monthly_clim", "-j", "4", "-o", outdir] + list(options) + argv = ["aero_1850_monthly_clim", "-j", "4", "-o", outdir] + list(options) argv += [self._archdir, CASE, str(BEGYEAR), str(ENDYEAR)] output = io.StringIO() with mock.patch.object(sys, "argv", argv), contextlib.redirect_stdout(output): - d2mc.main() - outfile = d2mc.output_filename( - d2mc.output_dir(self._archdir, outdir, CASE), + aero.main() + outfile = mclim.output_filename( + mclim.output_dir(self._archdir, outdir, CASE), CASE, BEGYEAR, ENDYEAR, @@ -144,11 +145,11 @@ def _check_cf_compliance(self, outfile): cmd = ["compliance-checker", "--test", "cf", outfile] run_cmd_output_on_error(cmd, "The output file did not pass the CF compliance checker") - def _check_output(self, outfile, data_vars, year=d2mc.DEFAULT_YEAR): + def _check_output(self, outfile, data_vars, year=aero.Aero1850MonthlyClim.DEFAULT_YEAR): """Check the variables, values, grid, time axis and metadata of the output file""" units = f"days since {year:04d}-01-01 00:00:00" with Dataset(outfile) as ncfile: - expected = set(data_vars) | set(d2mc.ALWAYS_VARS) | {"time", "climatology_bounds"} + expected = set(data_vars) | set(mclim.ALWAYS_VARS) | {"time", "climatology_bounds"} self.assertEqual(set(ncfile.variables), expected) # Each month is the average over the years of that month for name in data_vars: @@ -170,13 +171,13 @@ def _check_output(self, outfile, data_vars, year=d2mc.DEFAULT_YEAR): self.assertNotIn("cell_methods", var.ncattrs()) # The time axis and the climatology bounds time = ncfile.variables["time"] - np.testing.assert_array_equal(time[:], d2mc.mid_month_times()) + np.testing.assert_array_equal(time[:], mclim.mid_month_times()) self.assertEqual(time.units, units) self.assertEqual(time.calendar, "noleap") self.assertEqual(time.climatology, "climatology_bounds") np.testing.assert_array_equal( ncfile.variables["climatology_bounds"][:], - d2mc.climatology_bounds_values(BEGYEAR, ENDYEAR, year), + mclim.climatology_bounds_values(BEGYEAR, ENDYEAR, year), ) self.assertEqual(ncfile.Conventions, "CF-1.11") self.assertEqual(ncfile.made_from_cesm_case, CASE) @@ -186,7 +187,7 @@ def _check_output(self, outfile, data_vars, year=d2mc.DEFAULT_YEAR): def test_defaultVars(self): """The default variables, with the intermediate files removed at the end""" outfile, _, argv = self._run() - command_used = self._check_output(outfile, d2mc.DEFAULT_VARS) + command_used = self._check_output(outfile, aero.Aero1850MonthlyClim.DEFAULT_VARS) self._check_cf_compliance(outfile) self.assertEqual(command_used, shlex.join(argv)) self.assertFalse(os.path.exists(os.path.join(os.path.dirname(outfile), "mon"))) @@ -202,7 +203,7 @@ def test_varsAndYear(self): def test_allVars(self): """All variables: time_bnds is renamed to climatology_bounds, lon/lat aren't averaged""" outfile, _, _ = self._run("--all-vars") - self._check_output(outfile, d2mc.DEFAULT_VARS + [EXTRA_VAR]) + self._check_output(outfile, aero.Aero1850MonthlyClim.DEFAULT_VARS + [EXTRA_VAR]) self._check_cf_compliance(outfile) def test_keepAndReuse(self): @@ -215,7 +216,7 @@ def test_keepAndReuse(self): os.rename(outfile, os.path.join(self._tempdir, "first.nc")) outfile, printed, _ = self._run("--keep", outdir=outdir) self.assertEqual(printed.count("skipping ncra command"), 12 * nyears) - self._check_output(outfile, d2mc.DEFAULT_VARS) + self._check_output(outfile, aero.Aero1850MonthlyClim.DEFAULT_VARS) self._check_cf_compliance(outfile) def test_reuseWithDifferentVars_aborts(self): diff --git a/python/ctsm/test/test_unit_aero_1850_monthly_clim.py b/python/ctsm/test/test_unit_aero_1850_monthly_clim.py new file mode 100644 index 0000000000..aaa6e9be60 --- /dev/null +++ b/python/ctsm/test/test_unit_aero_1850_monthly_clim.py @@ -0,0 +1,338 @@ +#!/usr/bin/env python3 + +""" +Unit tests for aero_1850_monthly_clim, and the CplhistMonthlyClim class methods it uses + +The module-level functions of cplhist_monthly_clim are tested in test_unit_cplhist_monthly_clim.py +""" + +import datetime +import io +import os +import shlex +import shutil +import sys +import tempfile +import unittest +from unittest import mock + +from ctsm import unit_testing +from ctsm.cplhist import aero_1850_monthly_clim as aero +from ctsm.cplhist import cplhist_monthly_clim as mclim + +# Allow names that pylint doesn't like, because otherwise I find it hard +# to make readable unit test names +# pylint: disable=invalid-name + +CASE = "TESTCASE" +# The product class tested here +CLIM = aero.Aero1850MonthlyClim + + +def _check_args(args): + """Run check_args for the parsed arguments""" + CLIM(args).check_args() + + +# +# Tests for argument parsing +# +class TestGetParser(unittest.TestCase): + """Tests of get_parser""" + + def test_defaults(self): + """The defaults when only the positional arguments are given""" + args = CLIM.get_parser().parse_args(["arch", CASE, "2001", "2002"]) + self.assertEqual(args.archdir, "arch") + self.assertEqual(args.case, CASE) + self.assertEqual((args.begyear, args.endyear), (2001, 2002)) + self.assertEqual(args.vars, CLIM.DEFAULT_VARS) + self.assertFalse(args.all_vars) + self.assertIsNone(args.outdir) + self.assertEqual(args.year, CLIM.DEFAULT_YEAR) + self.assertFalse(args.keep) + self.assertIsNone(args.njobs) + self.assertFalse(args.batch) + self.assertIsNone(args.account) + self.assertEqual(args.queue, mclim.DEFAULT_QUEUE) + self.assertEqual(args.walltime, CLIM.DEFAULT_WALLTIME) + + def test_varsSplitsCommaList(self): + """--vars is split on commas, with spaces stripped""" + args = CLIM.get_parser().parse_args(["--vars", "a, b,c", "arch", CASE, "1", "2"]) + self.assertEqual(args.vars, ["a", "b", "c"]) + + def test_allVarsWithVars_exits(self): + """--all-vars and --vars can't be used together""" + with mock.patch("sys.stderr", new_callable=io.StringIO): + with self.assertRaises(SystemExit): + CLIM.get_parser().parse_args(["-a", "--vars", "a", "arch", CASE, "1", "2"]) + + +# +# Tests for the argument error checking +# +class TestCheckArgs(unittest.TestCase): + """Tests of check_args""" + + def setUp(self): + self._testdir = tempfile.mkdtemp() + self._archdir = os.path.join(self._testdir, "archive") + os.makedirs(mclim.input_dir(self._archdir, CASE)) + + def tearDown(self): + shutil.rmtree(self._testdir, ignore_errors=True) + + def _args(self, *options, archdir=None, years=("2001", "2002")): + """Parse the options, with the archive directory, case and years""" + if archdir is None: + archdir = self._archdir + return CLIM.get_parser().parse_args(list(options) + [archdir, CASE] + list(years)) + + def test_validArgs(self): + """Valid arguments pass""" + _check_args(self._args()) + + def test_yearOutOfRange(self): + """BEGYEAR, ENDYEAR and YEAR must be from 1 to 9999""" + with self.assertRaisesRegex(SystemExit, "ENDYEAR must be an integer from 1 to 9999"): + _check_args(self._args(years=("2001", "10000"))) + with self.assertRaisesRegex(SystemExit, "BEGYEAR must be an integer from 1 to 9999"): + _check_args(self._args(years=("0", "2001"))) + with self.assertRaisesRegex(SystemExit, "YEAR must be an integer from 1 to 9999"): + _check_args(self._args("--year", "10000")) + with self.assertRaisesRegex(SystemExit, "YEAR must be an integer from 1 to 9999"): + _check_args(self._args("--year", "0")) + + def test_begyearAfterEndyear(self): + """BEGYEAR can't be after ENDYEAR""" + with self.assertRaisesRegex(SystemExit, "must not be after ENDYEAR"): + _check_args(self._args(years=("2002", "2001"))) + + def test_njobsZero(self): + """--njobs must be positive""" + with self.assertRaisesRegex(SystemExit, "NJOBS must be a positive integer"): + _check_args(self._args("--njobs", "0")) + + def test_accountWithoutBatch(self): + """--account is only allowed with --batch""" + with self.assertRaisesRegex(SystemExit, "--account is only used with --batch"): + _check_args(self._args("--account", "P1")) + _check_args(self._args("--batch", "--account", "P1")) + + def test_onlyAlwaysVars(self): + """--vars with only the ALWAYS_VARS leaves nothing to average""" + with self.assertRaisesRegex(SystemExit, "No variables to average"): + _check_args(self._args("--vars", ",".join(mclim.ALWAYS_VARS))) + + def test_missingDirectories(self): + """The archive, output and input directories must exist""" + with self.assertRaisesRegex(SystemExit, "Input archive directory .* does not exist"): + _check_args(self._args(archdir=os.path.join(self._testdir, "dir_that_does_not_exist"))) + with self.assertRaisesRegex(SystemExit, "Output archive directory .* does not exist"): + _check_args( + self._args("--outdir", os.path.join(self._testdir, "dir_that_does_not_exist")) + ) + shutil.rmtree(mclim.input_dir(self._archdir, CASE)) + with self.assertRaisesRegex(SystemExit, "Input case archive directory .* does not exist"): + _check_args(self._args()) + + def test_outputExists(self): + """Abort if today's output file already exists""" + outdir = mclim.output_dir(self._archdir, None, CASE) + os.makedirs(outdir) + outfile = mclim.output_filename(outdir, CASE, 2001, 2002, datetime.date.today()) + with open(outfile, "w", encoding="utf-8"): + pass + with self.assertRaisesRegex(SystemExit, "already exists"): + _check_args(self._args()) + + +# +# Tests of writing provenance information is correct +# +class TestWriteProvenance(unittest.TestCase): + """Tests of write_provenance (with the ncatted command mocked)""" + + def test_oneNcattedCommand(self): + """One ncatted --hst command sets the global attributes and deletes file_version""" + # Replace run_cmd_output_on_error with a mock (run_cmd) that does nothing but record + # the call, get_ctsm_git_short_hash to return a fixed hash, and sys.argv with the + # command line given + clim = CLIM(CLIM.get_parser().parse_args(["arch", CASE, "100", "130"])) + clim.todaysdate = datetime.date(2026, 10, 3) + with mock.patch.object(mclim, "run_cmd_output_on_error") as run_cmd, mock.patch.object( + mclim, "get_ctsm_git_short_hash", return_value="abc1234" + ), mock.patch.object(sys, "argv", ["aero_1850_monthly_clim", "--vars", "a,b", "x"]): + clim.write_provenance("out.nc") + run_cmd.assert_called_once() + cmd = run_cmd.call_args.args[0] + self.assertEqual(cmd[:3], ["ncatted", "-O", "--hst"]) + self.assertEqual(cmd[-1], "out.nc") + attrs = cmd[4:-1:2] + self.assertEqual(cmd[3:-1:2], ["-a"] * len(attrs)) + self.assertIn("Conventions,global,o,c,CF-1.11", attrs) + self.assertIn("Created_on,global,o,c,2026-10-03", attrs) + self.assertIn("Created_with,global,o,c,./aero_1850_monthly_clim -- abc1234", attrs) + self.assertIn("command_used,global,o,c,aero_1850_monthly_clim --vars a,b x", attrs) + self.assertIn(f"made_from_cesm_case,global,o,c,{CASE}", attrs) + self.assertIn("file_version,global,d,,", attrs) + title = [attr for attr in attrs if attr.startswith("title,")] + self.assertTrue(title[0].endswith("for cyclical year 1850")) + + +# +# Test the batch options +# +class TestBatch(unittest.TestCase): + """Tests of job_args and batch_script""" + + def test_jobArgs_default(self): + """Only the positional arguments when the defaults are used""" + args = CLIM.get_parser().parse_args(["arch", CASE, "2001", "2002"]) + self.assertEqual(CLIM(args).job_args(), ["arch", CASE, "2001", "2002"]) + + def test_jobArgs_removes_batch_options(self): + """Set all options, but show that job_args returns only the non-batch options""" + args = CLIM.get_parser().parse_args( + [ + "--batch", + "--account", + "P1", + "--queue", + "develop", + "--walltime", + "01:00:00", + "--vars", + "a,b", + "-o", + "out", + "-y", + "2000", + "-k", + "-j", + "8", + "-v", + "arch", + CASE, + "2001", + "2002", + ] + ) + self.assertEqual( + CLIM(args).job_args(), + ["arch", CASE, "2001", "2002", "--vars", "a,b", "--outdir", "out", "--year", "2000"] + + ["--keep", "--njobs", "8", "--verbose"], + ) + + def test_jobArgs_allVars(self): + """Test that --all-vars is passed on when it's called as '-a'""" + args = CLIM.get_parser().parse_args(["-a", "arch", CASE, "2001", "2002"]) + self.assertEqual(CLIM(args).job_args(), ["arch", CASE, "2001", "2002", "--all-vars"]) + + def test_batchScript(self): + """ + Test that batch_script creates: PBS settings, NCO, the directory and the quoted command + """ + argv = ["arch", CASE, "2001", "2002", "--vars", "a,b", "--outdir", "my dir"] + clim = CLIM(CLIM.get_parser().parse_args(["arch", CASE, "2001", "2002"])) + script = clim.batch_script( + argv, account="P1", walltime="01:00:00", workdir="/work dir", njobs=8 + ) + lines = script.splitlines() + self.assertEqual(lines[0], "#!/bin/bash") + self.assertIn(f"#PBS -N {CLIM.PBS_JOBNAME}", lines) + self.assertIn("#PBS -A P1", lines) + self.assertIn("#PBS -l walltime=01:00:00", lines) + self.assertIn("mpiprocs=8", script) + self.assertIn("module load nco", lines) + self.assertIn("cd '/work dir'", lines) + command = shlex.split(lines[-1]) + self.assertEqual(command[0], sys.executable) + self.assertTrue(command[1].endswith(CLIM.WRAPPER)) + self.assertEqual(command[2:], argv) + + +# +# Test that the class attributes of a product subclass are used (this is what a new product, +# for example ozone, will set) +# +class _TestProductClim(mclim.CplhistMonthlyClim): + """A product with different settings than the aerosol one""" + + DEFAULT_VARS = ["atmImp_Sa_o3"] + DEFAULT_YEAR = 2000 + OUTPUT_TAG = "testclim" + PBS_JOBNAME = "test_product_clim" + WRAPPER = os.path.join("tools", "cplhist", "test_product_clim") + DEFAULT_WALLTIME = "01:00:00" + TITLE = "Test product title" + COMMENT = "Test product comment" + + +class TestProductSubclass(unittest.TestCase): + """Tests that a product subclass's class attributes flow through the base class""" + + def setUp(self): + self._args = _TestProductClim.get_parser().parse_args(["arch", CASE, "2001", "2002"]) + self._clim = _TestProductClim(self._args) + + def test_parserDefaults(self): + """The parser defaults come from the subclass""" + self.assertEqual(self._args.vars, ["atmImp_Sa_o3"]) + self.assertEqual(self._args.year, 2000) + self.assertEqual(self._args.walltime, "01:00:00") + + def test_jobArgs(self): + """The subclass defaults aren't passed on to the batch job, the aerosol ones are""" + self.assertEqual(self._clim.job_args(), ["arch", CASE, "2001", "2002"]) + args = _TestProductClim.get_parser().parse_args( + ["--vars", ",".join(CLIM.DEFAULT_VARS), "-y", "1850", "arch", CASE, "2001", "2002"] + ) + self.assertEqual( + _TestProductClim(args).job_args(), + ["arch", CASE, "2001", "2002", "--vars", ",".join(CLIM.DEFAULT_VARS)] + + ["--year", "1850"], + ) + + def test_outputFile(self): + """The output file name uses OUTPUT_TAG""" + self._clim.todaysdate = datetime.date(2026, 10, 3) + self.assertEqual( + self._clim.output_file(), + os.path.join( + "arch", CASE, "proc", f"{CASE}.cpl.hx.atm.testclim.2001-2002_c20261003.nc" + ), + ) + + def test_batchScript(self): + """The batch script uses the subclass's job name and wrapper""" + script = self._clim.batch_script( + ["arch"], account="P1", walltime="01:00:00", workdir="/work", njobs=8 + ) + self.assertIn("#PBS -N test_product_clim", script.splitlines()) + self.assertTrue(shlex.split(script.splitlines()[-1])[1].endswith(_TestProductClim.WRAPPER)) + + def test_provenanceAttrs(self): + """The title, comment and Created_with come from the subclass""" + with mock.patch.object(mclim, "get_ctsm_git_short_hash", return_value="abc1234"): + attrs = dict(self._clim.provenance_attrs()) + self.assertEqual(attrs["title"], "Test product title for cyclical year 2000") + self.assertEqual(attrs["comment"], "Test product comment") + self.assertEqual(attrs["Created_with"], "./test_product_clim -- abc1234") + + def test_missingRequiredAttribute(self): + """A subclass that doesn't set a required class attribute is rejected when it's defined""" + with self.assertRaisesRegex(TypeError, "must set the class attributes: TITLE, COMMENT"): + + class _IncompleteClim(mclim.CplhistMonthlyClim): # pylint: disable=unused-variable + """A product missing TITLE and COMMENT""" + + DEFAULT_VARS = ["a"] + PBS_JOBNAME = "incomplete" + WRAPPER = "incomplete" + + +if __name__ == "__main__": + unit_testing.setup_for_tests() + unittest.main() diff --git a/python/ctsm/test/test_unit_daily_to_monthly_clim.py b/python/ctsm/test/test_unit_cplhist_monthly_clim.py similarity index 59% rename from python/ctsm/test/test_unit_daily_to_monthly_clim.py rename to python/ctsm/test/test_unit_cplhist_monthly_clim.py index 55d44d2af6..12a9b7661e 100644 --- a/python/ctsm/test/test_unit_daily_to_monthly_clim.py +++ b/python/ctsm/test/test_unit_cplhist_monthly_clim.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 """ -Unit tests for daily_to_monthly_clim +Unit tests for the module-level functions in cplhist_monthly_clim NOTE: SOME OF THESE UNIT TESTS HANDLE NetCDF FILES. Normally we want tests that do I/O in the system tests, because I/O is slow @@ -10,9 +10,7 @@ """ import datetime -import io import os -import shlex import shutil import subprocess import sys @@ -24,7 +22,7 @@ from netCDF4 import Dataset # pylint: disable=no-name-in-module from ctsm import unit_testing -from ctsm.cplhist import daily_to_monthly_clim as d2mc +from ctsm.cplhist import cplhist_monthly_clim as mclim # Allow names that pylint doesn't like, because otherwise I find it hard # to make readable unit test names @@ -36,7 +34,7 @@ # A couple utility functions for the tests def _daily_name(case, year, month, day): """Name of a daily coupler history file""" - return f"{case}.{d2mc.HIST_TAG}.{year:04d}-{month:02d}-{day:02d}-00000.nc" + return f"{case}.{mclim.HIST_TAG}.{year:04d}-{month:02d}-{day:02d}-00000.nc" def _write_nc(filename, variables, ntime=1): @@ -54,122 +52,6 @@ def _write_nc(filename, variables, ntime=1): var[...] = np.ones(shape) -# -# Tests for argument parsing -# -class TestGetParser(unittest.TestCase): - """Tests of get_parser""" - - def test_defaults(self): - """The defaults when only the positional arguments are given""" - args = d2mc.get_parser().parse_args(["arch", CASE, "2001", "2002"]) - self.assertEqual(args.archdir, "arch") - self.assertEqual(args.case, CASE) - self.assertEqual((args.begyear, args.endyear), (2001, 2002)) - self.assertEqual(args.vars, d2mc.DEFAULT_VARS) - self.assertFalse(args.all_vars) - self.assertIsNone(args.outdir) - self.assertEqual(args.year, d2mc.DEFAULT_YEAR) - self.assertFalse(args.keep) - self.assertIsNone(args.njobs) - self.assertFalse(args.batch) - self.assertIsNone(args.account) - self.assertEqual(args.queue, d2mc.DEFAULT_QUEUE) - self.assertEqual(args.walltime, d2mc.DEFAULT_WALLTIME) - - def test_varsSplitsCommaList(self): - """--vars is split on commas, with spaces stripped""" - args = d2mc.get_parser().parse_args(["--vars", "a, b,c", "arch", CASE, "1", "2"]) - self.assertEqual(args.vars, ["a", "b", "c"]) - - def test_allVarsWithVars_exits(self): - """--all-vars and --vars can't be used together""" - with mock.patch("sys.stderr", new_callable=io.StringIO): - with self.assertRaises(SystemExit): - d2mc.get_parser().parse_args(["-a", "--vars", "a", "arch", CASE, "1", "2"]) - - -# -# Tests for the argument error checking -# -class TestCheckArgs(unittest.TestCase): - """Tests of check_args""" - - def setUp(self): - self._testdir = tempfile.mkdtemp() - self._archdir = os.path.join(self._testdir, "archive") - os.makedirs(d2mc.input_dir(self._archdir, CASE)) - - def tearDown(self): - shutil.rmtree(self._testdir, ignore_errors=True) - - def _args(self, *options, archdir=None, years=("2001", "2002")): - """Parse the options, with the archive directory, case and years""" - if archdir is None: - archdir = self._archdir - return d2mc.get_parser().parse_args(list(options) + [archdir, CASE] + list(years)) - - def test_validArgs(self): - """Valid arguments pass""" - d2mc.check_args(self._args()) - - def test_yearOutOfRange(self): - """BEGYEAR, ENDYEAR and YEAR must be from 1 to 9999""" - with self.assertRaisesRegex(SystemExit, "ENDYEAR must be an integer from 1 to 9999"): - d2mc.check_args(self._args(years=("2001", "10000"))) - with self.assertRaisesRegex(SystemExit, "BEGYEAR must be an integer from 1 to 9999"): - d2mc.check_args(self._args(years=("0", "2001"))) - with self.assertRaisesRegex(SystemExit, "YEAR must be an integer from 1 to 9999"): - d2mc.check_args(self._args("--year", "10000")) - with self.assertRaisesRegex(SystemExit, "YEAR must be an integer from 1 to 9999"): - d2mc.check_args(self._args("--year", "0")) - - def test_begyearAfterEndyear(self): - """BEGYEAR can't be after ENDYEAR""" - with self.assertRaisesRegex(SystemExit, "must not be after ENDYEAR"): - d2mc.check_args(self._args(years=("2002", "2001"))) - - def test_njobsZero(self): - """--njobs must be positive""" - with self.assertRaisesRegex(SystemExit, "NJOBS must be a positive integer"): - d2mc.check_args(self._args("--njobs", "0")) - - def test_accountWithoutBatch(self): - """--account is only allowed with --batch""" - with self.assertRaisesRegex(SystemExit, "--account is only used with --batch"): - d2mc.check_args(self._args("--account", "P1")) - d2mc.check_args(self._args("--batch", "--account", "P1")) - - def test_onlyAlwaysVars(self): - """--vars with only the ALWAYS_VARS leaves nothing to average""" - with self.assertRaisesRegex(SystemExit, "No variables to average"): - d2mc.check_args(self._args("--vars", ",".join(d2mc.ALWAYS_VARS))) - - def test_missingDirectories(self): - """The archive, output and input directories must exist""" - with self.assertRaisesRegex(SystemExit, "Input archive directory .* does not exist"): - d2mc.check_args( - self._args(archdir=os.path.join(self._testdir, "dir_that_does_not_exist")) - ) - with self.assertRaisesRegex(SystemExit, "Output archive directory .* does not exist"): - d2mc.check_args( - self._args("--outdir", os.path.join(self._testdir, "dir_that_does_not_exist")) - ) - shutil.rmtree(d2mc.input_dir(self._archdir, CASE)) - with self.assertRaisesRegex(SystemExit, "Input case archive directory .* does not exist"): - d2mc.check_args(self._args()) - - def test_outputExists(self): - """Abort if today's output file already exists""" - outdir = d2mc.output_dir(self._archdir, None, CASE) - os.makedirs(outdir) - outfile = d2mc.output_filename(outdir, CASE, 2001, 2002, datetime.date.today()) - with open(outfile, "w", encoding="utf-8"): - pass - with self.assertRaisesRegex(SystemExit, "already exists"): - d2mc.check_args(self._args()) - - # # Tests for batch and number of jobs # @@ -179,13 +61,13 @@ class TestDefaultNjobs(unittest.TestCase): def test_inBatchJob(self): """Inside a PBS job (PBS_JOBID set) use PBS_NJOBS""" with mock.patch.dict(os.environ, {"PBS_JOBID": "1234.desched1"}): - self.assertEqual(d2mc.default_njobs(), d2mc.PBS_NJOBS) + self.assertEqual(mclim.default_njobs(), mclim.PBS_NJOBS) def test_notInBatchJob(self): """Outside a PBS job use INTERACTIVE_NJOBS""" with mock.patch.dict(os.environ): os.environ.pop("PBS_JOBID", None) - self.assertEqual(d2mc.default_njobs(), d2mc.INTERACTIVE_NJOBS) + self.assertEqual(mclim.default_njobs(), mclim.INTERACTIVE_NJOBS) # @@ -196,20 +78,20 @@ class TestVarLists(unittest.TestCase): def test_buildVarList(self): """Keeps the order, drops duplicates and empty names, adds the extra variables at the end""" - self.assertEqual(d2mc.build_var_list(["b", "a", "", "b"], ["c", "a"]), ["b", "a", "c"]) + self.assertEqual(mclim.build_var_list(["b", "a", "", "b"], ["c", "a"]), ["b", "a", "c"]) def test_buildVarList_none(self): """None (all variables) stays None""" - self.assertIsNone(d2mc.build_var_list(None, ["c"])) + self.assertIsNone(mclim.build_var_list(None, ["c"])) def test_averagedVars(self): """The ALWAYS_VARS are removed from the list to average""" - variables = ["a", d2mc.ALWAYS_VARS[0], "", "a", "b"] - self.assertEqual(d2mc.averaged_vars(variables), ["a", "b"]) + variables = ["a", mclim.ALWAYS_VARS[0], "", "a", "b"] + self.assertEqual(mclim.averaged_vars(variables), ["a", "b"]) def test_averagedVars_none(self): """None (all variables) stays None""" - self.assertIsNone(d2mc.averaged_vars(None)) + self.assertIsNone(mclim.averaged_vars(None)) # @@ -220,7 +102,7 @@ class TestYearMonths(unittest.TestCase): def test_twoYears(self): """Two years in order""" - yms = d2mc.year_months(2001, 2002) + yms = mclim.year_months(2001, 2002) self.assertEqual(len(yms), 24) self.assertEqual(yms[0], (2001, 1)) self.assertEqual(yms[11], (2001, 12)) @@ -229,10 +111,10 @@ def test_twoYears(self): def test_singleYear(self): """A single year, including year 1 and the default year""" - for year in (1, d2mc.DEFAULT_YEAR): + for year in (1, mclim.CplhistMonthlyClim.DEFAULT_YEAR): with self.subTest(year=year): self.assertEqual( - d2mc.year_months(year, year), [(year, month) for month in range(1, 13)] + mclim.year_months(year, year), [(year, month) for month in range(1, 13)] ) @@ -245,7 +127,7 @@ class TestDailyFiles(unittest.TestCase): def setUp(self): self._testdir = tempfile.mkdtemp() # Empty files are enough, the files are found by name - for month, ndays in enumerate(d2mc.NOLEAP_DAYS, start=1): + for month, ndays in enumerate(mclim.NOLEAP_DAYS, start=1): for day in range(1, ndays + 1): self._create_empty_file(_daily_name(CASE, 2001, month, day)) @@ -267,10 +149,10 @@ def test_indexDailyFiles(self): self._create_empty_file(f"{CASE}.cpl.hi.2001-01-01-00000.nc") self._create_empty_file(_daily_name(CASE, 2001, 1, 1) + ".base") # - files_by_month = d2mc.index_daily_files(self._testdir, CASE) - self.assertEqual(sorted(files_by_month), d2mc.year_months(2001, 2001)) + files_by_month = mclim.index_daily_files(self._testdir, CASE) + self.assertEqual(sorted(files_by_month), mclim.year_months(2001, 2001)) for (_, month), files in files_by_month.items(): - self.assertEqual(len(files), d2mc.NOLEAP_DAYS[month - 1]) + self.assertEqual(len(files), mclim.NOLEAP_DAYS[month - 1]) self.assertEqual(files, sorted(files)) self.assertEqual( files_by_month[(2001, 1)][0], os.path.join(self._testdir, _daily_name(CASE, 2001, 1, 1)) @@ -278,16 +160,16 @@ def test_indexDailyFiles(self): def test_checkDayCounts_complete(self): """Test that check_day_counts for a complete year is correct""" - files_by_month = d2mc.index_daily_files(self._testdir, CASE) - self.assertEqual(d2mc.check_day_counts(files_by_month, d2mc.year_months(2001, 2001)), []) + files_by_month = mclim.index_daily_files(self._testdir, CASE) + self.assertEqual(mclim.check_day_counts(files_by_month, mclim.year_months(2001, 2001)), []) def test_checkDayCounts_missing(self): """Test that check_day_counts reports missing days and years correctly""" # Remove one day from the first year os.remove(os.path.join(self._testdir, _daily_name(CASE, 2001, 2, 14))) # NOTE: Setup only created files for the first year, so the second year has no files - files_by_month = d2mc.index_daily_files(self._testdir, CASE) - errors = d2mc.check_day_counts(files_by_month, d2mc.year_months(2001, 2002)) + files_by_month = mclim.index_daily_files(self._testdir, CASE) + errors = mclim.check_day_counts(files_by_month, mclim.year_months(2001, 2002)) self.assertEqual(len(errors), 13) # There will be an error reported for the missing day in the first year # and each month of the second year @@ -310,7 +192,7 @@ def setUp(self): { "a": (("time", "n"), {}), "b": (("time", "n"), {}), - d2mc.ALWAYS_VARS[0]: (("time", "n"), {}), + mclim.ALWAYS_VARS[0]: (("time", "n"), {}), }, ) @@ -325,25 +207,25 @@ def _monfile(self, names): def test_fileVars(self): """The set of variable names in the file""" - self.assertEqual(d2mc.file_vars(self._daily), {"a", "b", d2mc.ALWAYS_VARS[0]}) + self.assertEqual(mclim.file_vars(self._daily), {"a", "b", mclim.ALWAYS_VARS[0]}) def test_missingVars(self): """Only the variables not in the file are returned, in order""" - self.assertEqual(d2mc.missing_vars(self._daily, ["c", "a", "d"]), ["c", "d"]) + self.assertEqual(mclim.missing_vars(self._daily, ["c", "a", "d"]), ["c", "d"]) def test_checkReusedMonFile_ok(self): """A monthly file with the needed variables passes, with a list or all variables""" monfile = self._monfile(["a", "b"]) - d2mc.check_reused_mon_file(monfile, [self._daily], ["a"]) - d2mc.check_reused_mon_file(monfile, [self._daily], None) + mclim.check_reused_mon_file(monfile, [self._daily], ["a"]) + mclim.check_reused_mon_file(monfile, [self._daily], None) def test_checkReusedMonFile_missing(self): """A monthly file missing a requested variable aborts""" monfile = self._monfile(["a"]) with self.assertRaisesRegex(SystemExit, "is missing variables: c"): - d2mc.check_reused_mon_file(monfile, [self._daily], ["a", "c"]) + mclim.check_reused_mon_file(monfile, [self._daily], ["a", "c"]) with self.assertRaisesRegex(SystemExit, "is missing variables: b"): - d2mc.check_reused_mon_file(monfile, [self._daily], None) + mclim.check_reused_mon_file(monfile, [self._daily], None) # @@ -356,19 +238,19 @@ def test_missingCommand(self): """Abort naming the NCO commands that aren't found""" # Replace shutil.which so that only ncrcat is "not found" with mock.patch.object( - d2mc.shutil, + mclim.shutil, "which", # lambda is used here to have a small inline function to define the side effects side_effect=lambda cmd: None if cmd == "ncrcat" else "/bin/x", ): with self.assertRaisesRegex(SystemExit, "NCO commands not found: ncrcat "): - d2mc.check_nco() + mclim.check_nco() def test_allFound(self): """No abort when all the commands are found""" # Replace shutil.which so that all the commands are found - with mock.patch.object(d2mc.shutil, "which", return_value="/bin/x"): - d2mc.check_nco() + with mock.patch.object(mclim.shutil, "which", return_value="/bin/x"): + mclim.check_nco() # @@ -386,27 +268,27 @@ def tearDown(self): def test_varList(self): """A variable list gives -v, and exclude is ignored""" - cmd = d2mc.ncra_command(["f1", "f2"], self._out, ["a", "b"], exclude=["c"]) + cmd = mclim.ncra_command(["f1", "f2"], self._out, ["a", "b"], exclude=["c"]) self.assertEqual(cmd, ["ncra", "-O", "--hst", "-v", "a,b", "f1", "f2", "-o", self._out]) def test_allVarsExclude(self): """All variables with exclude gives -x -v""" - cmd = d2mc.ncra_command(["f1"], self._out, None, exclude=["c", "d"]) + cmd = mclim.ncra_command(["f1"], self._out, None, exclude=["c", "d"]) self.assertEqual(cmd, ["ncra", "-O", "--hst", "-x", "-v", "c,d", "f1", "-o", self._out]) def test_allVars(self): """All variables without exclude averages everything""" - cmd = d2mc.ncra_command(["f1"], self._out) + cmd = mclim.ncra_command(["f1"], self._out) self.assertEqual(cmd, ["ncra", "-O", "--hst", "f1", "-o", self._out]) def test_skipExisting(self): """With skip_existing, an existing output file gives no command""" expected = ["ncra", "-O", "--hst", "f1", "-o", self._out] - self.assertEqual(d2mc.ncra_command(["f1"], self._out, skip_existing=True), expected) + self.assertEqual(mclim.ncra_command(["f1"], self._out, skip_existing=True), expected) with open(self._out, "w", encoding="utf-8"): pass - self.assertEqual(d2mc.ncra_command(["f1"], self._out, skip_existing=True), []) - self.assertEqual(d2mc.ncra_command(["f1"], self._out), expected) + self.assertEqual(mclim.ncra_command(["f1"], self._out, skip_existing=True), []) + self.assertEqual(mclim.ncra_command(["f1"], self._out), expected) # @@ -425,7 +307,7 @@ def test_allRun(self): [], # empty command (output already exists) is skipped ["b", "o2"], # run "b" creating file o2 ] - d2mc.run_jobs(commands, 2, runner=runner) + mclim.run_jobs(commands, 2, runner=runner) # Check that runner had recorded these two calls, and the appropriate error runner.assert_has_calls( [ @@ -441,7 +323,7 @@ def test_oneJobInOrder(self): # Create runner as a MagicMock so that it does nothing, but records each call runner = mock.MagicMock() commands = [[f"c{i}", f"o{i}"] for i in range(5)] - d2mc.run_jobs(commands, 1, runner=runner) + mclim.run_jobs(commands, 1, runner=runner) # Check that the calls ran from runner match what's expected self.assertEqual([call.args[0] for call in runner.call_args_list], commands) @@ -449,7 +331,7 @@ def test_noCommands(self): """Only empty commands runs nothing""" # Create runner as a MagicMock so that it does nothing, but records each call runner = mock.MagicMock() - d2mc.run_jobs([[], []], 4, runner=runner) + mclim.run_jobs([[], []], 4, runner=runner) runner.assert_not_called() def test_failure(self): @@ -461,7 +343,7 @@ def runner(cmd, errmsg): sys.exit(errmsg) with self.assertRaisesRegex(SystemExit, "Failed creating o2"): - d2mc.run_jobs([["good", "o1"], ["bad", "o2"], ["good", "o3"]], 2, runner=runner) + mclim.run_jobs([["good", "o1"], ["bad", "o2"], ["good", "o3"]], 2, runner=runner) # @@ -473,14 +355,14 @@ class TestFilenames(unittest.TestCase): def test_outputFilename(self): """Zero-padded years and the creation date""" self.assertEqual( - d2mc.output_filename("/out", CASE, 100, 130, datetime.date(2026, 10, 3)), + mclim.output_filename("/out", CASE, 100, 130, datetime.date(2026, 10, 3)), os.path.join("/out", f"{CASE}.cpl.hx.atm.monclim.0100-0130_c20261003.nc"), ) def test_intermediateFiles(self): """Names of the monthly and climatology files""" - self.assertEqual(d2mc.mon_file("/m", 100, 2), os.path.join("/m", "mon_0100-02.nc")) - self.assertEqual(d2mc.clim_file("/m", 2), os.path.join("/m", "clim_02.nc")) + self.assertEqual(mclim.mon_file("/m", 100, 2), os.path.join("/m", "mon_0100-02.nc")) + self.assertEqual(mclim.clim_file("/m", 2), os.path.join("/m", "clim_02.nc")) # @@ -491,7 +373,7 @@ class TestTimeValues(unittest.TestCase): def test_monthBounds(self): """Start and end of each month of a noleap year""" - bounds = d2mc.month_bounds() + bounds = mclim.month_bounds() self.assertEqual(len(bounds), 12) # Check that there is just 12 bounds for each month self.assertEqual(bounds[0], (0, 31)) # January self.assertEqual(bounds[1], (31, 59)) # February @@ -499,18 +381,18 @@ def test_monthBounds(self): def test_midMonthTimes(self): """Middle of each month""" - times = d2mc.mid_month_times() + times = mclim.mid_month_times() self.assertEqual(times[0], 15.5) # January self.assertEqual(times[1], 45.0) # February self.assertEqual(times[-1], 349.5) # December (last month) def test_climatologyBounds_singleYear(self): """Averaging only YEAR gives the bounds of each month of YEAR""" - self.assertEqual(d2mc.climatology_bounds_values(1850, 1850, 1850), d2mc.month_bounds()) + self.assertEqual(mclim.climatology_bounds_values(1850, 1850, 1850), mclim.month_bounds()) def test_climatologyBounds_span(self): """The bounds go from the month in BEGYEAR to the month in ENDYEAR, negative before YEAR""" - bounds = d2mc.climatology_bounds_values(100, 130, 1850) + bounds = mclim.climatology_bounds_values(100, 130, 1850) self.assertEqual(bounds[0], ((100 - 1850) * 365, (130 - 1850) * 365 + 31)) # December ends at the start of ENDYEAR+1 self.assertEqual(bounds[-1][1], (131 - 1850) * 365) @@ -550,12 +432,12 @@ def _write_clim(self, with_time_bnds=False): def _check(self): """Check the time axis and climatology bounds after set_time_axis""" - d2mc.set_time_axis(self._file, 1850, 2001, 2002) + mclim.set_time_axis(self._file, 1850, 2001, 2002) units = "days since 1850-01-01 00:00:00" with Dataset(self._file) as ncfile: self.assertNotIn("time_bnds", ncfile.variables) time = ncfile.variables["time"] - np.testing.assert_array_equal(time[:], d2mc.mid_month_times()) + np.testing.assert_array_equal(time[:], mclim.mid_month_times()) self.assertEqual(time.units, units) self.assertEqual(time.calendar, "noleap") self.assertEqual(time.climatology, "climatology_bounds") @@ -564,7 +446,7 @@ def _check(self): bounds = ncfile.variables["climatology_bounds"] self.assertEqual(bounds.dimensions, ("time", "ntb")) np.testing.assert_array_equal( - bounds[:], d2mc.climatology_bounds_values(2001, 2002, 1850) + bounds[:], mclim.climatology_bounds_values(2001, 2002, 1850) ) self.assertEqual(bounds.units, units) for attr in ("axis", "calendar", "bounds", "cell_methods"): @@ -618,7 +500,7 @@ def tearDown(self): def test_copyGridVars(self): """Copied without time, with the first time slice, attributes and fill value""" - d2mc.copy_grid_vars(self._src, self._dst, ["atmImp_lon", "atmImp_lat"]) + mclim.copy_grid_vars(self._src, self._dst, ["atmImp_lon", "atmImp_lat"]) with Dataset(self._dst) as ncfile: self.assertEqual(len(ncfile.dimensions["atmImp_nx"]), 3) for name, base in (("atmImp_lon", 10.0), ("atmImp_lat", -60.0)): @@ -641,7 +523,7 @@ class TestAddVarAttributes(unittest.TestCase): def setUp(self): self._testdir = tempfile.mkdtemp() self._file = os.path.join(self._testdir, "clim.nc") - lon, lat = d2mc.ALWAYS_VARS + lon, lat = mclim.ALWAYS_VARS _write_nc( self._file, { @@ -660,121 +542,26 @@ def tearDown(self): def test_addVarAttributes(self): """Only the data variables on time get the climatology attributes""" - d2mc.add_var_attributes(self._file) + mclim.add_var_attributes(self._file) cell_methods = "time: mean within years time: mean over years" with Dataset(self._file) as ncfile: for name in ("a", "b"): var = ncfile.variables[name] self.assertEqual(var.cell_methods, cell_methods) - self.assertEqual(var.coordinates, " ".join(d2mc.ALWAYS_VARS)) + self.assertEqual(var.coordinates, " ".join(mclim.ALWAYS_VARS)) self.assertNotIn("standard_name", var.ncattrs()) self.assertEqual(ncfile.variables["a"].long_name, "a") self.assertEqual(ncfile.variables["b"].long_name, "Keep this name") - for name in ["time", "climatology_bounds", "c"] + d2mc.ALWAYS_VARS: + for name in ["time", "climatology_bounds", "c"] + mclim.ALWAYS_VARS: self.assertNotIn("cell_methods", ncfile.variables[name].ncattrs()) self.assertNotIn("coordinates", ncfile.variables[name].ncattrs()) # -# Tests of writing provenance information is correct -# -class TestWriteProvenance(unittest.TestCase): - """Tests of write_provenance (with the ncatted command mocked)""" - - def test_oneNcattedCommand(self): - """One ncatted --hst command sets the global attributes and deletes file_version""" - # Replace run_cmd_output_on_error with a mock (run_cmd) that does nothing but record - # the call, get_ctsm_git_short_hash to return a fixed hash, and sys.argv with the - # command line given - with mock.patch.object(d2mc, "run_cmd_output_on_error") as run_cmd, mock.patch.object( - d2mc, "get_ctsm_git_short_hash", return_value="abc1234" - ), mock.patch.object(sys, "argv", ["daily_to_monthly_clim", "--vars", "a,b", "x"]): - d2mc.write_provenance(CASE, "out.nc", 100, 130, 1850, datetime.date(2026, 10, 3)) - run_cmd.assert_called_once() - cmd = run_cmd.call_args.args[0] - self.assertEqual(cmd[:3], ["ncatted", "-O", "--hst"]) - self.assertEqual(cmd[-1], "out.nc") - attrs = cmd[4:-1:2] - self.assertEqual(cmd[3:-1:2], ["-a"] * len(attrs)) - self.assertIn("Conventions,global,o,c,CF-1.11", attrs) - self.assertIn("Created_on,global,o,c,2026-10-03", attrs) - self.assertIn("Created_with,global,o,c,./daily_to_monthly_clim -- abc1234", attrs) - self.assertIn("command_used,global,o,c,daily_to_monthly_clim --vars a,b x", attrs) - self.assertIn(f"made_from_cesm_case,global,o,c,{CASE}", attrs) - self.assertIn("file_version,global,d,,", attrs) - title = [attr for attr in attrs if attr.startswith("title,")] - self.assertTrue(title[0].endswith("for cyclical year 1850")) - - -# -# Test the batch options +# Test the submission to the batch queue # -class TestBatch(unittest.TestCase): - """Tests of job_args, batch_script and submit_batch""" - - def test_jobArgs_default(self): - """Only the positional arguments when the defaults are used""" - args = d2mc.get_parser().parse_args(["arch", CASE, "2001", "2002"]) - self.assertEqual(d2mc.job_args(args), ["arch", CASE, "2001", "2002"]) - - def test_jobArgs_removes_batch_options(self): - """Set all options, but show that job_args returns only the non-batch options""" - args = d2mc.get_parser().parse_args( - [ - "--batch", - "--account", - "P1", - "--queue", - "develop", - "--walltime", - "01:00:00", - "--vars", - "a,b", - "-o", - "out", - "-y", - "2000", - "-k", - "-j", - "8", - "-v", - "arch", - CASE, - "2001", - "2002", - ] - ) - self.assertEqual( - d2mc.job_args(args), - ["arch", CASE, "2001", "2002", "--vars", "a,b", "--outdir", "out", "--year", "2000"] - + ["--keep", "--njobs", "8", "--verbose"], - ) - - def test_jobArgs_allVars(self): - """Test that --all-vars is passed on when it's called as '-a'""" - args = d2mc.get_parser().parse_args(["-a", "arch", CASE, "2001", "2002"]) - self.assertEqual(d2mc.job_args(args), ["arch", CASE, "2001", "2002", "--all-vars"]) - - def test_batchScript(self): - """ - Test that batch_script creates: PBS settings, NCO, the directory and the quoted command - """ - argv = ["arch", CASE, "2001", "2002", "--vars", "a,b", "--outdir", "my dir"] - script = d2mc.batch_script( - argv, account="P1", walltime="01:00:00", workdir="/work dir", njobs=8 - ) - lines = script.splitlines() - self.assertEqual(lines[0], "#!/bin/bash") - self.assertIn(f"#PBS -N {d2mc.PBS_JOBNAME}", lines) - self.assertIn("#PBS -A P1", lines) - self.assertIn("#PBS -l walltime=01:00:00", lines) - self.assertIn("mpiprocs=8", script) - self.assertIn("module load nco", lines) - self.assertIn("cd '/work dir'", lines) - command = shlex.split(lines[-1]) - self.assertEqual(command[0], sys.executable) - self.assertTrue(command[1].endswith(d2mc.WRAPPER)) - self.assertEqual(command[2:], argv) +class TestSubmitBatch(unittest.TestCase): + """Tests of submit_batch""" def test_submitBatch(self): """ @@ -784,10 +571,10 @@ def test_submitBatch(self): result = subprocess.CompletedProcess([], 0, stdout="1234.desched1\n", stderr="") # Replace shutil.which to return the qsub path # and subprocess.run to return the result without running qsub - with mock.patch.object(d2mc.shutil, "which", return_value="/bin/qsub"), mock.patch.object( - d2mc.subprocess, "run", return_value=result + with mock.patch.object(mclim.shutil, "which", return_value="/bin/qsub"), mock.patch.object( + mclim.subprocess, "run", return_value=result ) as run: - self.assertEqual(d2mc.submit_batch("the script", "develop"), "1234.desched1") + self.assertEqual(mclim.submit_batch("the script", "develop"), "1234.desched1") self.assertEqual(run.call_args.args[0], ["qsub", "-q", "develop"]) self.assertEqual(run.call_args.kwargs["input"], "the script") @@ -797,18 +584,18 @@ def test_submitBatch_fails(self): result = subprocess.CompletedProcess([], 1, stdout="", stderr="qsub: bad account") # Replace shutil.which to return the qsub path # and subprocess.run to return the result without running qsub - with mock.patch.object(d2mc.shutil, "which", return_value="/bin/qsub"), mock.patch.object( - d2mc.subprocess, "run", return_value=result + with mock.patch.object(mclim.shutil, "which", return_value="/bin/qsub"), mock.patch.object( + mclim.subprocess, "run", return_value=result ): with self.assertRaisesRegex(SystemExit, "qsub failed:\nqsub: bad account"): - d2mc.submit_batch("the script", "main") + mclim.submit_batch("the script", "main") def test_submitBatch_noQsub(self): """Abort if qsub isn't found""" # Replace shutil.which to return None, as it does when the command isn't found - with mock.patch.object(d2mc.shutil, "which", return_value=None): + with mock.patch.object(mclim.shutil, "which", return_value=None): with self.assertRaisesRegex(SystemExit, "qsub not found"): - d2mc.submit_batch("the script", "main") + mclim.submit_batch("the script", "main") if __name__ == "__main__": diff --git a/tools/cplhist/daily_to_monthly_clim b/tools/cplhist/aero_1850_monthly_clim similarity index 62% rename from tools/cplhist/daily_to_monthly_clim rename to tools/cplhist/aero_1850_monthly_clim index c975f3d0a8..c7ecab1a97 100755 --- a/tools/cplhist/daily_to_monthly_clim +++ b/tools/cplhist/aero_1850_monthly_clim @@ -1,22 +1,22 @@ #!/usr/bin/env python3 """ This is a just top-level skeleton script that calls -daily_to_monthly_clim.py. -The original code (daily_to_monthly_clim.py) is located under the +aero_1850_monthly_clim.py. +The original code (aero_1850_monthly_clim.py, using cplhist_monthly_clim.py) is located under the python/ctsm/cplhist folder. -It creates a 12-month climatology from daily averaged CMEPS coupler -history (CPLHIST) files, for use by DATM. +It creates a 12-month climatology of the aerosol deposition fields from +daily averaged CMEPS coupler history (CPLHIST) files, for use by DATM. For full instructions on how to run the code and different options, -please check the python/ctsm/cplhist/daily_to_monthly_clim.py file. +please check the python/ctsm/cplhist/aero_1850_monthly_clim.py file. To run this script the following packages are required: - netCDF4 - the NCO commands ncra, ncrcat and ncatted ---------------------------------------------------------------- -To see all available options for daily_to_monthly_clim - ./daily_to_monthly_clim --help +To see all available options for aero_1850_monthly_clim + ./aero_1850_monthly_clim --help ---------------------------------------------------------------- Instructions for running using the conda python environments and NCO on Derecho: @@ -34,7 +34,7 @@ _CTSM_PYTHON = os.path.join( ) sys.path.insert(1, _CTSM_PYTHON) -from ctsm.cplhist.daily_to_monthly_clim import main +from ctsm.cplhist.aero_1850_monthly_clim import main if __name__ == "__main__": main()