# shellcheck shell=bash
# -*-sh-*-

#-------------------------------------
# OHPC common test functions
#-------------------------------------

# Error message with exit
ERROR() {
	echo "[OHPC-TEST:ERROR]: $1" >&2
	exit 1
}

# Error message with return
ERROR_RETURN() {
	echo "[OHPC-TEST:ERROR]: $1" >&2
	return 1
}

# Check if specific RPM package is installed
check_if_rpm_installed() {
	rpm -q --quiet "$1" || ERROR "RPM $1 is not installed locally"
}

# The Flux system instance reads every *.toml in its conf.d, whatever the
# files are called (the install guide writes a single system.toml, the CI
# setup writes bootstrap.toml and others), so look for any non-empty one.
flux_system_configured() {
	local conf
	for conf in /etc/flux/system/conf.d/*.toml; do
		[ -s "${conf}" ] && return 0
	done
	return 1
}

# Check if we have a known/supported resource manager
check_rms() {

	if [ -s /etc/pbs.conf ]; then
		export RESOURCE_MANAGER=openpbs
	elif [ -s /etc/slurm/slurm.conf ]; then
		export RESOURCE_MANAGER=slurm
	elif flux_system_configured; then
		export RESOURCE_MANAGER=flux
	else
		ERROR "Unsupported or unknown resource manager"
	fi
}

# The cores of the smallest node the resource manager schedules, for sizing a
# job's ranks per node to the nodes it will run on. Cores rather than CPUs:
# Slurm counts the threads per core its configuration claims, and the example
# slurm.conf claims two on hardware that has one, so a node it reports as 32
# CPUs gives a job only the 16 cores behind them. One rank per claimed CPU then
# puts two ranks on every core, and the shared-memory collectives of MPICH and
# Open MPI busy-wait rather than yield, so the job stalls instead of merely
# running at half speed. 8 when the manager does not say (openpbs, or a query
# that fails).
node_cores() {
	local cores=""
	if [ "${RESOURCE_MANAGER}" = "slurm" ]; then
		# %X is the sockets per node and %Y the cores per socket; %c would
		# be those multiplied by the configured threads per core.
		cores=$(sinfo -h -N -o '%X %Y' 2>/dev/null |
			awk '$1 ~ /^[0-9]+$/ && $2 ~ /^[0-9]+$/ {print $1 * $2}' |
			sort -n | head -1)
	elif [ "${RESOURCE_MANAGER}" = "flux" ]; then
		cores=$(flux resource list -s up -no '{nnodes} {ncores}' 2>/dev/null | awk '$1 > 0 {print int($2 / $1)}')
	fi
	case "${cores}" in
	'' | *[!0-9]* | 0)
		cores=8
		;;
	esac
	echo "${cores}"
}

# Every run_*_binary caller hardcodes its -t timeout as a PBS/Slurm-style
# walltime (HH:MM:SS, MM:SS, or a bare number), since that is what those two
# resource managers expect. flux batch's own -t/--time-limit instead wants a
# Flux standard duration, whose unitless form is a plain number of minutes,
# not that walltime string; convert here rather than at every call site.
flux_timeout_minutes() {
	local t="$1"
	case "${t}" in
	*:*:*)
		local h m s
		IFS=: read -r h m s <<<"${t}"
		echo $(((10#${h} * 60) + 10#${m} + (10#${s} + 59) / 60))
		;;
	*:*)
		local m s
		IFS=: read -r m s <<<"${t}"
		echo $((10#${m} + (10#${s} + 59) / 60))
		;;
	*)
		echo "${t}"
		;;
	esac
}

# Check for valid executable
check_exe() {
	if ! type "$1" >&/dev/null; then
		ERROR_RETURN "$1 is not available for execution"
	fi
}

# export python family variables
get_python_vars() {
	local __PYTHON_FAMILY
	__PYTHON_FAMILY="$1"
	if [ "${__PYTHON_FAMILY}" == "python3" ]; then
		export _python=python3
		export python_module_prefix=py3
		export python_package_prefix=python3
	fi
}

save_logs_compiler_family() {
	if [ $# -lt 2 ]; then
		ERROR "insufficient input provided to save_logs_compiler_family()"
	fi

	local __TEST_DIR
	local __COMPILER
	local __SAVR_DIR
	__TEST_DIR="$1"
	__COMPILER="$2"
	__SAVE_DIR="family-${__COMPILER}"

	cd "${__TEST_DIR}" || ERROR "unable to cd to ${__TEST_DIR}"

	if [ -d "${__SAVE_DIR}" ]; then
		rm -rf "${__SAVE_DIR}"
	fi

	mkdir "${__SAVE_DIR}"

	shopt -s nullglob

	# Cache .log files

	for i in *.log; do
		mv "${i}" "${__SAVE_DIR}" || ERROR "Unable to move file -> ${i}"
	done

	# Also cache log.xml files (for JUnit parsing)

	for i in *.log.xml; do
		mv "${i}" "${__SAVE_DIR}" || ERROR "Unable to move file -> ${i}"
	done

	cd - >/dev/null || ERROR "Changing directory failed"

} # end of save_logs_compiler_family()

save_logs_mpi_family() {
	if [ $# -lt 3 ]; then
		ERROR "insufficient input provided to save_logs_mpi_family()"
	fi

	local __TEST_DIR
	local __COMPILER
	local __MPI
	local __SAVE_DIR
	__TEST_DIR="$1"
	__COMPILER="$2"
	__MPI="$3"
	__SAVE_DIR="family-${__COMPILER}-${__MPI}"

	cd "${__TEST_DIR}" || ERROR "unable to cd to ${__TEST_DIR}"

	if [ -d "${__SAVE_DIR}" ]; then
		rm -rf "${__SAVE_DIR}"
	fi

	mkdir "${__SAVE_DIR}"

	shopt -s nullglob

	# Cache .log files
	for i in *.log; do
		mv "${i}" "${__SAVE_DIR}" || ERROR "Unable to move file -> ${i}"
	done

	# Also cache log.xml files (for JUnit parsing)

	for i in *.log.xml; do
		mv "${i}" "${__SAVE_DIR}" || ERROR "Unable to move file -> ${i}"
	done
	cd - >/dev/null || ERROR "Changing directory failed"

} # end of save_logs_mpi_family()

# Check if we have a known/supported compiler family
# and set relevant compiler variables
check_compiler_family() {
	local __CC
	local __CXX
	local __FC
	__CC="$1"
	__CXX="$2"
	__FC="$3"

	local myCC
	local myCXX
	local myFC
	myCC=""
	myCXX=""
	myFC=""

	if [ "${LMOD_FAMILY_COMPILER}" == "intel" ]; then
		myCC="icx"
		myCXX="icpx"
		myFC="ifx"
	elif [[ "${LMOD_FAMILY_COMPILER}" =~ "arm" ]]; then
		myCC="armclang"
		myCXX="armclang++"
		myFC="armflang"
	elif [[ "${LMOD_FAMILY_COMPILER}" =~ "acfl" ]]; then
		myCC="armclang"
		myCXX="armclang++"
		myFC="armflang"
	elif [ "${LMOD_FAMILY_COMPILER}" == "gnu" ]; then
		myCC="gcc"
		myCXX="g++"
		myFC="gfortran"
	elif [ "${LMOD_FAMILY_COMPILER}" == "gnu14" ]; then
		myCC="gcc"
		myCXX="g++"
		myFC="gfortran"
	elif [ "${LMOD_FAMILY_COMPILER}" == "gnu15" ]; then
		myCC="gcc"
		myCXX="g++"
		myFC="gfortran"
	elif [ "${LMOD_FAMILY_COMPILER}" == "gnu16" ]; then
		myCC="gcc"
		myCXX="g++"
		myFC="gfortran"
	elif [ "${LMOD_FAMILY_COMPILER}" == "gnu13" ]; then
		myCC="gcc"
		myCXX="g++"
		myFC="gfortran"
	elif [ "${LMOD_FAMILY_COMPILER}" == "gnu12" ]; then
		myCC="gcc"
		myCXX="g++"
		myFC="gfortran"
	elif [ "${LMOD_FAMILY_COMPILER}" == "gnu9" ]; then
		myCC="gcc"
		myCXX="g++"
		myFC="gfortran"
	elif [[ "${LMOD_FAMILY_COMPILER}" == "llvm9" ]]; then
		myCC="clang"
		myCXX="clang++"
		myFC="gfortran"
	else
		ERROR "Unknown compiler family"
	fi

	eval "${__CC}"="'${myCC}'"
	eval "${__CXX}"="'${myCXX}'"
	eval "${__FC}"="'${myFC}'"
}

# Run serial binary through resource manager
#
# The executable is checked for on the node running the tests before it is
# handed to the resource manager. Pass --skip-local-check for binaries that
# only exist on the compute nodes (e.g. nvidia-smi on a GPU-less head node).
run_serial_binary() {

	# Parse optional arguments
	output_file=""
	timeout=1 # default job timeout (in minutes)
	local skip_local_check=0

	local OPTIND=1
	# "-:" lets getopts hand a "--long-option" over as "-" with OPTARG=long-option
	while getopts "o:s:t:-:" OPTION; do
		case "${OPTION}" in
		o)
			output_file="${OPTARG}"
			;;
		s)
			scalasca="${OPTARG}"
			;;
		t)
			timeout="${OPTARG}"
			;;
		-)
			case "${OPTARG}" in
			skip-local-check)
				skip_local_check=1
				;;
			*)
				echo "Unknown option --${OPTARG} given to run_serial_binary" >&2
				exit 1
				;;
			esac
			;;
		'?')
			echo "Unknown option given to run_serial_binary" >&2
			exit 1
			;;
		esac
	done

	shift "$((OPTIND - 1))" # Shift off the options and optional --.

	EXE="$1"
	shift

	if [ "${skip_local_check}" -eq 0 ]; then
		# shellcheck disable=SC2086
		check_exe ${EXE}
	fi

	if [ -z "${scalasca}" ]; then
		exec_prefix=""
	else
		exec_prefix="scalasca -analyze -t"
	fi

	if [ -z "${RESOURCE_MANAGER}" ]; then
		ERROR_RETURN "OHPC resource manager is not defined - please set via RESOURCE_MANAGER variable"
	fi

	if [ "${RESOURCE_MANAGER}" = "slurm" ]; then
		if [ -n "${output_file}" ]; then
			eval "${exec_prefix}" srun -n 1 -N 1 -t "${timeout}" "${EXE}" "$*" >&"${output_file}"
		else
			# srun -n 1 -N 1 -t $timeout $EXE "$@"
			eval "${exec_prefix}" srun -n 1 -N 1 -t "${timeout}" "${EXE}" "$*"
		fi
		return "$?"
	elif [ "${RESOURCE_MANAGER}" = "openpbs" ]; then

		echo "EXE = ${EXE}"

		# Generate script with commands so we can execute out of submission directory
		jobScript=".job.${RANDOM}"
		{
			echo "#!/bin/bash"
			echo "cd \${PBS_O_WORKDIR}"
			echo "${exec_prefix}" "${EXE}" "$@"
		} >"${jobScript}"
		chmod 700 "${jobScript}"

		if [ -n "${output_file}" ]; then
			QSUB_OUT=$(qsub -k oe -W block=true -l select=1 -l walltime="${timeout}" -- "$(readlink -f "${jobScript}")")
			JOB=$(echo "${QSUB_OUT}" | awk -F . '{print $1}')
			[[ -e ${HOME}/STDIN.o"${JOB}" ]] || exit 1
			mv "${HOME}/STDIN.o${JOB}" "${output_file}"
			cat "${output_file}"
		else
			qsub -k n -W block=true -l select=1 -l walltime="${timeout}" -- "$(readlink -f "${jobScript}")"

			RC="$?"
			rm -f "${jobScript}"
			return "${RC}"
		fi
	elif [ "${RESOURCE_MANAGER}" = "flux" ]; then
		JOB_SCRIPT=".job.${RANDOM}"
		{
			echo "#!/bin/bash"
			echo "${exec_prefix}" "${EXE}" "$@"
		} >"${JOB_SCRIPT}"
		chmod 700 "${JOB_SCRIPT}"

		# Same reasoning as run_mpi_binary's flux branch: flux batch's
		# own default output template doesn't match the jobid actually
		# printed to stdout, so name it explicitly up front, in the
		# shared submission directory rather than the compute node's /tmp.
		JOB_OUT="${PWD}/.job.${RANDOM}.out"

		jobId=$(flux batch -N 1 -n 1 -t "$(flux_timeout_minutes "${timeout}")" --output="${JOB_OUT}" "${JOB_SCRIPT}") ||
			ERROR "Unable to submit batch job"

		echo "Batch job ${jobId} submitted"

		# Capture the status without tripping errexit, so a failed
		# job's output still gets shown below.
		RC=0
		flux job status "${jobId}" || RC="$?"

		if [ -n "${output_file}" ]; then
			mv "${JOB_OUT}" "${output_file}"
			cat "${output_file}"
		else
			cat "${JOB_OUT}" 2>/dev/null || true
		fi

		rm -f "${JOB_SCRIPT}" "${JOB_OUT}"
		return "${RC}"
	else
		ERROR_RETURN "Unsupported resource manager"
	fi
}

# Run MPI binary through resource manager
run_mpi_binary() {

	# Parse optional arguments
	input_file=""
	output_file=""
	if [ "${RESOURCE_MANAGER}" = "slurm" ]; then
		if [ -n "${SIMPLE_CI}" ]; then
			timeout=5 # default job timeout (in minutes)
			# Do not use pmix in the SIMPLE_CI case (GitHub Actions)
			unset OHPC_MPI_LAUNCHERS
		else
			timeout=2 # default job timeout (in minutes)
		fi
	elif [ "${RESOURCE_MANAGER}" = "flux" ]; then
		# flux's -t/--time-limit takes bare minutes (or a Flux Standard
		# Duration), not openpbs' HH:MM:SS walltime format below.
		timeout=2 # default job timeout (in minutes)
	else
		timeout="00:02:00"
	fi

	local OPTIND
	OPTIND=1
	while getopts "i:o:s:t:" OPTION; do
		case "${OPTION}" in
		i)
			input_file="${OPTARG}"
			;;
		o)
			output_file="${OPTARG}"
			;;
		s)
			scalasca="${OPTARG}"
			;;
		t)
			timeout="${OPTARG}"
			;;
		'?')
			echo "Unknown option given to run_mpi_binary" >&2
			exit 1
			;;
		esac
	done

	if [ -z "${scalasca}" ]; then
		mpi_launcher="prun"
	else
		mpi_launcher="scalasca -analyze -t prun"
	fi

	shift "$((OPTIND - 1))" # Shift off the options and optional --.

	EXE="$1"
	ARGS="$2"
	NNODES="$3"
	NTASKS="$4"

	# shellcheck disable=SC2086
	check_exe ${EXE}

	if [ -z "${RESOURCE_MANAGER}" ]; then
		ERROR_RETURN "Resource manager is not defined - please set via RESOURCE_MANAGER variable"
	fi

	if [ -z "${LMOD_FAMILY_MPI}" ]; then
		ERROR_RETURN "MPI toolchain s not loaded - please load MPI stack first"
	fi

	if [ "${RESOURCE_MANAGER}" = "slurm" ]; then
		if [ "${LMOD_FAMILY_MPI}" != "impi" ] && [ "${LMOD_FAMILY_MPI}" != "mvapich2" ] && [ "${LMOD_FAMILY_MPI}" != "openmpi5" ] && [ "${LMOD_FAMILY_MPI}" != "mpich" ]; then
			ERROR_RETURN "Unsupported MPI family"
		fi

		JOB_SCRIPT="/tmp/job.${USER}.${RANDOM}"

		{
			echo "#!/bin/bash"
			echo "#SBATCH -J OpenHPC-test"
			echo "#SBATCH -N ${NNODES}"
			echo "#SBATCH -n ${NTASKS}"
			echo "#SBATCH -t ${timeout}"
			echo "#SBATCH -o job.%j.out"
		} >"${JOB_SCRIPT}"

		if [ -n "${input_file}" ]; then
			echo "${mpi_launcher} ${EXE} ${ARGS} < ${input_file}" >>"${JOB_SCRIPT}"
		else
			echo "${mpi_launcher} ${EXE} ${ARGS}" >>"${JOB_SCRIPT}"
		fi

		# Submit batch job

		tmpState="/tmp/submitId.${RANDOM}"

		echo "job script = ${JOB_SCRIPT}"
		# bats runs the test under set -e, so a refused sbatch ended the test
		# right here with no message: check it, and show what Slurm said
		if ! sbatch "${JOB_SCRIPT}" >&"${tmpState}"; then
			cat "${tmpState}"
			rm -f "${tmpState}"
			ERROR "Unable to submit batch job"
		fi

		head -1 "${tmpState}" | grep -q "Submitted batch job" || ERROR "Unable to submit batch job"
		jobId=$(head -1 "${tmpState}" | awk '{print $4}')
		if [ "${jobId}" -le 0 ]; then
			ERROR "Invalid jobID"
		fi
		rm "${tmpState}"

		echo "Batch job ${jobId} submitted"
		RC=1

		for i in $(seq 1 3000); do
			if ! tmpState=$(scontrol show job "${jobId}" | grep JobState); then
				ERROR_RETURN "Error querying job"
			fi

			if echo "${tmpState}" | grep -E -q "JobState=COMPLETED"; then
				echo "Job completed..."
				RC=0
				break

			elif echo "${tmpState}" | grep -E -q "JobState=FAILED"; then
				local TMP_REASON
				TMP_REASON=$(scontrol show job "${jobId}" | grep Reason | awk '{print $2}')
				echo " "
				echo "Job ${jobId} failed..."
				echo "${TMP_REASON}"
				break

			elif echo "${tmpState}" | grep -E -q "JobState=TIMEOUT"; then
				local TMP_REASON
				TMP_REASON=$(scontrol show job "${jobId}" | grep Reason | awk '{print $2}')
				echo " "
				echo "Job ${jobId} encountered timeout..."
				echo "${TMP_REASON}"
				break

			elif echo "${tmpState}" | grep -E -q "JobState=CANCELLED"; then
				echo " "
				echo "Job ${jobId} cancelled..."
				break

			elif echo "${tmpState}" | grep -E -q "JobState=PENDING"; then
				# A job pending for a reason that cannot clear on its own will
				# never run: no node configuration in the partition fits its
				# request (PartitionConfig, what Slurm gives -N 2 -n 16 on two
				# 4-CPU nodes), its node count or time limit is outside the
				# partition's (PartitionNodeLimit, PartitionTimeLimit), no node
				# satisfies its constraints (BadConstraints), or its account or
				# QOS is invalid (squeue(1), JOB REASON CODES). Cancel it and
				# fail now, with Slurm's reason, rather than poll for 3000 s and
				# fail on the missing output file. Resources, Priority and the
				# like clear as other jobs finish, so those keep waiting.
				local TMP_REASON
				TMP_REASON=$(scontrol show job "${jobId}" | grep Reason | awk '{print $2}')
				case "${TMP_REASON}" in
				Reason=PartitionConfig | Reason=PartitionNodeLimit | Reason=PartitionTimeLimit | Reason=BadConstraints | Reason=InvalidAccount | Reason=InvalidQOS)
					echo " "
					echo "Job ${jobId} can never start: ${TMP_REASON}"
					scancel "${jobId}"
					rm -f "${JOB_SCRIPT}"
					return 1
					;;
				esac
				sleep 1

			else
				sleep 1
			fi
		done

		# Look for evidence of failure in job output

		echo " "
		cat job."${jobId}".out

		if grep -E -q "${jobId} FAILED|${jobId} CANCELLED|exited on signal|command not found|failed to start|Unable to access executable|Error in init phase" job."${jobId}".out; then
			RC=1
		fi

		if [ -n "${output_file}" ]; then
			mv job."${jobId}".out "${output_file}"
		fi

		rm "${JOB_SCRIPT}"
		return "${RC}"

	elif [ "${RESOURCE_MANAGER}" = "openpbs" ]; then
		JOB_SCRIPT="/tmp/job.${USER}.${RANDOM}"

		# infer tasks/node from total tasks
		((TASKS_PER_NODE = NTASKS / NNODES))

		{
			echo "#!/bin/bash"
			echo "#PBS -N OpenHPC-test"
			echo "#PBS -l select=${NNODES}:mpiprocs=${TASKS_PER_NODE} -l place=scatter"
			echo "#PBS -l walltime=${timeout}"
			echo "#PBS -l place=excl"
			echo "#PBS -o job.out"
			echo "cd \$PBS_O_WORKDIR"
		} >>"${JOB_SCRIPT}"

		if [ -n "${input_file}" ]; then
			echo "${mpi_launcher} ${EXE} ${ARGS} < ${input_file}" >>"${JOB_SCRIPT}"
		else
			echo "${mpi_launcher} ${EXE} ${ARGS}" >>"${JOB_SCRIPT}"
		fi

		# Submit batch job
		qsub -W block=true "${JOB_SCRIPT}"
		RC="$?"

		if [ -n "${output_file}" ]; then
			if [ "${output_file}" != "job.out" ]; then
				mv job.out "${output_file}"
			fi
		fi
		rm "${JOB_SCRIPT}"
		return "${RC}"
	elif [ "${RESOURCE_MANAGER}" = "flux" ]; then
		JOB_SCRIPT="/tmp/job.${USER}.${RANDOM}"

		# Unlike srun inside an salloc/sbatch job, "flux run" with no
		# sizing flags does not inherit this job's node/task count;
		# prun's own flux branch reads these to size it explicitly.
		# See prun's launch_flux() for the full explanation.
		{
			echo "#!/bin/bash"
			echo "export FLUX_JOB_NNODES=${NNODES}"
			echo "export FLUX_JOB_NTASKS=${NTASKS}"
		} >"${JOB_SCRIPT}"

		if [ -n "${input_file}" ]; then
			echo "${mpi_launcher} ${EXE} ${ARGS} < ${input_file}" >>"${JOB_SCRIPT}"
		else
			echo "${mpi_launcher} ${EXE} ${ARGS}" >>"${JOB_SCRIPT}"
		fi
		chmod +x "${JOB_SCRIPT}"

		# Set an explicit, predictable output path rather than relying
		# on flux batch's default "flux-{{id}}.out" template: the
		# jobid it substitutes there is a different, ASCII-only
		# encoding than the one "flux batch" prints to stdout and
		# "flux job status"/"flux job attach" accept, so a filename
		# built from the printed jobid does not actually match the
		# file flux itself creates. The job shell writes it on the first
		# compute node, so keep it in the shared submission directory
		# rather than that node's /tmp.
		JOB_OUT="${PWD}/.job.${RANDOM}.out"

		# Every batch job is a separate Flux sub-instance, and
		# sub-instances started at the same time can give their first
		# job the same ID. Flux passes that ID to MPI as the PMI
		# kvsname, and MPICH names its shared memory segments after
		# it, so two such jobs on the same host corrupt each other's
		# communication. Tests that run in parallel (bats -j) submit
		# their batch jobs at the same time, and on a cluster with two
		# compute nodes they all land on the same two nodes, so
		# allocate the nodes exclusively, like the Slurm partition
		# does with Oversubscribe=EXCLUSIVE, and the jobs run one
		# after another.
		#
		# In SIMPLE_CI (GitHub Actions) all Flux nodes run on the same
		# host, where exclusive nodes still share /dev/shm, so run one
		# MPI batch job at a time there with a lock instead.
		local flux_lock_fd=""
		if [ -n "${SIMPLE_CI}" ]; then
			exec {flux_lock_fd}>"/tmp/ohpc-flux-mpi.${USER}.lock"
			flock "${flux_lock_fd}"
		fi

		# Submit batch job
		jobId=$(flux batch --exclusive -N "${NNODES}" -n "${NTASKS}" -t "$(flux_timeout_minutes "${timeout}")" --output="${JOB_OUT}" "${JOB_SCRIPT}") ||
			ERROR "Unable to submit batch job"

		echo "Batch job ${jobId} submitted"

		# "flux job status" blocks until the job completes and exits
		# with the job's own largest exit code; unlike slurm's
		# scontrol-based polling loop above, this needs no separate
		# wait/poll step or job-state text parsing.
		RC=0
		flux job status "${jobId}" || RC="$?"

		# Let the next MPI test submit its batch job
		if [ -n "${flux_lock_fd}" ]; then
			exec {flux_lock_fd}>&-
		fi

		echo " "
		cat "${JOB_OUT}" 2>/dev/null || true

		if [ -n "${output_file}" ]; then
			mv "${JOB_OUT}" "${output_file}"
		fi

		rm -f "${JOB_SCRIPT}" "${JOB_OUT}"
		return "${RC}"
	else
		ERROR_RETURN "Unsupported resource manager"
	fi

} # end run_mpi_binary()

# Run a command with retry logic for flaky tests
# Usage: run_with_retry [-n max_attempts] [-d delay_seconds] command [args...]
# Default: 3 attempts with 2 second delay between retries
run_with_retry() {
	local max_attempts=3
	local delay=2

	local OPTIND=1
	while getopts "n:d:" OPTION; do
		case "${OPTION}" in
		n)
			max_attempts="${OPTARG}"
			;;
		d)
			delay="${OPTARG}"
			;;
		'?')
			echo "Unknown option given to run_with_retry" >&2
			return 1
			;;
		esac
	done

	shift "$((OPTIND - 1))"

	local attempt=1
	local rc=1

	while [ "${attempt}" -le "${max_attempts}" ]; do
		if "$@"; then
			return 0
		fi
		rc=$?

		if [ "${attempt}" -lt "${max_attempts}" ]; then
			echo "Attempt ${attempt}/${max_attempts} failed, retrying in ${delay}s..." >&2
			sleep "${delay}"
		fi
		((attempt++))
	done

	echo "All ${max_attempts} attempts failed" >&2
	return "${rc}"
}

get_rpm_name() {
	local family
	if [[ "${LMOD_FAMILY_COMPILER}" =~ "acfl" ]]; then
		# The arm compiler package set "acfl".
		# OpenHPC expects "arm1".
		family="arm1"
	else
		family="${LMOD_FAMILY_COMPILER}"
	fi

	if [ -n "${LMOD_FAMILY_MPI}" ]; then
		echo "${1}-${family}-${LMOD_FAMILY_MPI}${DELIM}"
	else
		echo "${1}-${family}${DELIM}"
	fi
}
