#!/bin/bash
#SBATCH --job-name="verify"
#SBATCH --partition=gpuA40x4
#SBATCH --mem=12G
#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --cpus-per-task=1
#SBATCH --constraint="projects,perf,nvperf"
#SBATCH --gpus-per-node=1
#SBATCH --gpu-bind=closest
#SBATCH --account=bche-delta-gpu
#SBATCH -t 00:15:00
#SBATCH --exclude=gpub029,gpub034,gpub038,gpub043,gpub074,gpub054,gpub012

module reset
module load cuda

SUFFIX=${1:-op9}
BIN=./build/test_gpt2_${SUFFIX}

# -------------------------
# output dirs
# -------------------------
OUTDIR=output/perf/nsys
REPORTDIR=output/perf/reports

mkdir -p $OUTDIR
mkdir -p $REPORTDIR

OUTFILE=$OUTDIR/verify_${SUFFIX}.out
ERRFILE=$OUTDIR/verify_${SUFFIX}.err

REPORT=$REPORTDIR/report_${SUFFIX}

echo "Binary: $BIN"
echo "NSYS report: $REPORT"

# redirect only SLURM logs
exec > $OUTFILE 2> $ERRFILE

echo -e "job $SLURM_JOBID starting on $(hostname)\n\n"

# -------------------------
# NSYS profiling
# -------------------------
srun nsys profile \
    --stats=true \
    --output=$REPORT \
    $BIN