Monitor performance of target GPU simulations

Merge pull request #3450 from CliMA/he/test-default-config-help

Failed in 15m 42s
buildkite-agent pipeline upload .buildkite/gpu_...
init :GPU:
dry baroclinic wavemkdir -p target_gpu_implicit_baroclinic_wave && nsys profile --delay 100 --trace=nvtx,mpi,cuda,osrt --output=target_gpu_implicit_baroclinic_wave/output_active/report julia --threads=3 --color=yes --project=examples examples/hybrid/driver.jl --config_file config/gpu_configs/target_gpu_implicit_baroclinic_wave.yml --job_id target_gpu_implicit_baroclinic_wave
Canceled
Waited 30s
moist Held-Suarezmkdir -p gpu_hs_rhoe_equil_55km_nz63_0M && nsys profile --delay 100 --trace=nvtx,mpi,cuda,osrt --output=gpu_hs_rhoe_equil_55km_nz63_0M/output_active/report julia --threads=3 --color=yes --project=examples examples/hybrid/driver.jl --config_file config/gpu_configs/gpu_hs_rhoe_equil_0M.yml --job_id gpu_hs_rhoe_equil_55km_nz63_0M
Canceled
Waited 32s
·
Ran in 13s
dry baroclinic wave - 4 gpusmkdir -p target_gpu_implicit_baroclinic_wave_4process && srun --cpu-bind=threads --cpus-per-task=4 nsys profile --delay 100 --trace=osrt,nvtx,cuda,mpi,ucx --output=target_gpu_implicit_baroclinic_wave_4process/output_active/report-%q{PMI_RANK} julia --threads=3 --color=yes --project=examples examples/hybrid/driver.jl --config_file config/gpu_configs/target_gpu_implicit_baroclinic_wave.yml --job_id target_gpu_implicit_baroclinic_wave_4process
Canceled
Waited 42s
gpu_aquaplanet_dyamond with diagnostics - strong scaling - 1 GPUmkdir -p gpu_aquaplanet_dyamond_diag_1process && srun --cpu-bind=threads --cpus-per-task=4 nsys profile --delay 100 --trace=nvtx,mpi,cuda,osrt --output=gpu_aquaplanet_dyamond_diag_1process/output_active/report julia --threads=3 --color=yes --project=examples examples/hybrid/driver.jl --config_file config/gpu_configs/gpu_aquaplanet_dyamond_diag_1process.yml --job_id gpu_aquaplanet_dyamond_diag_1process
Canceled
Waited 46s
gpu_aquaplanet_dyamond - strong scaling - 1 GPUmkdir -p gpu_aquaplanet_dyamond_ss_1process && srun --cpu-bind=threads --cpus-per-task=4 nsys profile --delay 100 --trace=nvtx,mpi,cuda,osrt --output=gpu_aquaplanet_dyamond_ss_1process/output_active/report julia --threads=3 --color=yes --project=examples examples/hybrid/driver.jl --config_file config/gpu_configs/gpu_aquaplanet_dyamond_ss.yml --job_id gpu_aquaplanet_dyamond_ss_1process
Canceled
Waited 50s
gpu_aquaplanet_dyamond - strong scaling - 2 GPUsmkdir -p gpu_aquaplanet_dyamond_ss_2process && srun --cpu-bind=threads --cpus-per-task=4 julia --threads=3 --color=yes --project=examples examples/hybrid/driver.jl --config_file config/gpu_configs/gpu_aquaplanet_dyamond_ss.yml --job_id gpu_aquaplanet_dyamond_ss_2process
Canceled
Waited 55s
gpu_aquaplanet_dyamond - strong scaling - 4 GPUsmkdir -p gpu_aquaplanet_dyamond_ss_4process && srun --cpu-bind=threads --cpus-per-task=4 julia --threads=3 --color=yes --project=examples examples/hybrid/driver.jl --config_file config/gpu_configs/gpu_aquaplanet_dyamond_ss.yml --job_id gpu_aquaplanet_dyamond_ss_4process
Canceled
Waited 58s
gpu_aquaplanet_dyamond - weak scaling - 1 GPUmkdir -p gpu_aquaplanet_dyamond_ws_1process && srun --cpu-bind=threads --cpus-per-task=4 julia --threads=3 --color=yes --project=examples examples/hybrid/driver.jl --config_file config/gpu_configs/gpu_aquaplanet_dyamond_ws_1process.yml --job_id gpu_aquaplanet_dyamond_ws_1process
Canceled
Waited 1m 6s
gpu_aquaplanet_dyamond - weak scaling - 2 GPUsmkdir -p gpu_aquaplanet_dyamond_ws_2process && srun --cpu-bind=threads --cpus-per-task=4 julia --threads=3 --color=yes --project=examples examples/hybrid/driver.jl --config_file config/gpu_configs/gpu_aquaplanet_dyamond_ws_2process.yml --job_id gpu_aquaplanet_dyamond_ws_2process
Canceled
Waited 1m 10s
gpu_aquaplanet_dyamond - weak scaling - 4 GPUsmkdir -p gpu_aquaplanet_dyamond_ws_4process && srun --cpu-bind=threads --cpus-per-task=4 julia --threads=3 --color=yes --project=examples examples/hybrid/driver.jl --config_file config/gpu_configs/gpu_aquaplanet_dyamond_ws_4process.yml --job_id gpu_aquaplanet_dyamond_ws_4process
Canceled
Waited 1m 13s
gpu_aquaplanet_diagedmf - 1 GPUmkdir -p gpu_aquaplanet_diagedmf && nsys profile --delay 200 --trace=nvtx,mpi,cuda,osrt --output=gpu_aquaplanet_diagedmf/output_active/report julia --threads=3 --color=yes --project=examples examples/hybrid/driver.jl --config_file config/model_configs/aquaplanet_diagedmf.yml --job_id gpu_aquaplanet_diagedmf
Canceled
Waited 1m 19s
gpu_aquaplanet_diagedmf_benchmarkjulia --color=yes --project=perf perf/benchmark.jl --config_file config/model_configs/aquaplanet_diagedmf.yml --job_id gpu_aquaplanet_diagedmf_benchmark
Canceled
Waited 1m 23s
gpu_aquaplanet_progedmf - 1 GPUmkdir -p gpu_aquaplanet_progedmf && nsys profile --delay 100 --trace=nvtx,mpi,cuda,osrt --output=gpu_aquaplanet_progedmf/output_active/report julia --threads=3 --color=yes --project=examples examples/hybrid/driver.jl --config_file config/model_configs/aquaplanet_progedmf.yml --job_id gpu_aquaplanet_progedmf
Canceled
Waited 1m 27s
gpu_aquaplanet_progedmf_benchmarkjulia --color=yes --project=perf perf/benchmark.jl --config_file config/model_configs/aquaplanet_progedmf.yml --job_id gpu_aquaplanet_progedmf_benchmark
Canceled
Waited 1m 30s
Total Job Run Time: 14m 17s