Monitor performance of target GPU simulations
Triggered from Web
dry baroclinic wavemkdir -p target_gpu_implicit_baroclinic_wave && nsys profile --delay 100 --trace=nvtx,mpi,cuda,osrt --output=target_gpu_implicit_baroclinic_wave/output_active/report julia --threads=3 --color=yes --project=examples examples/hybrid/driver.jl --config_file config/gpu_configs/target_gpu_implicit_baroclinic_wave.yml --job_id target_gpu_implicit_baroclinic_wave
Waited 10s
Ran in 4m 58s
moist Held-Suarezmkdir -p gpu_hs_rhoe_equil_55km_nz63_0M && nsys profile --delay 100 --trace=nvtx,mpi,cuda,osrt --output=gpu_hs_rhoe_equil_55km_nz63_0M/output_active/report julia --threads=3 --color=yes --project=examples examples/hybrid/driver.jl --config_file config/gpu_configs/gpu_hs_rhoe_equil_0M.yml --job_id gpu_hs_rhoe_equil_55km_nz63_0M
Waited 5m 11s
Ran in 6m 25s
moist Held-Suarez - 4 gpusmkdir -p gpu_hs_rhoe_equil_55km_nz63_0M_4process && srun --cpu-bind=threads --cpus-per-task=4 nsys profile --delay 100 --trace=nvtx,mpi,cuda,osrt --output=gpu_hs_rhoe_equil_55km_nz63_0M_4process/output_active/report-%q{PMI_RANK} julia --threads=3 --color=yes --project=examples examples/hybrid/driver.jl --config_file config/gpu_configs/gpu_hs_rhoe_equil_0M.yml --job_id gpu_hs_rhoe_equil_55km_nz63_0M_4process
Waited 39m 42s
Ran in 5m 47s
dry baroclinic wave - 4 gpusmkdir -p target_gpu_implicit_baroclinic_wave_4process && srun --cpu-bind=threads --cpus-per-task=4 nsys profile --delay 100 --trace=osrt,nvtx,cuda,mpi,ucx --output=target_gpu_implicit_baroclinic_wave_4process/output_active/report-%q{PMI_RANK} julia --threads=3 --color=yes --project=examples examples/hybrid/driver.jl --config_file config/gpu_configs/target_gpu_implicit_baroclinic_wave.yml --job_id target_gpu_implicit_baroclinic_wave_4process
Waited 45m 31s
Ran in 4m 56s
gpu_aquaplanet_dyamond with diagnostics - strong scaling - 1 GPUmkdir -p gpu_aquaplanet_dyamond_diag_1process && srun --cpu-bind=threads --cpus-per-task=4 nsys profile --delay 100 --trace=nvtx,mpi,cuda,osrt --output=gpu_aquaplanet_dyamond_diag_1process/output_active/report julia --threads=3 --color=yes --project=examples examples/hybrid/driver.jl --config_file config/gpu_configs/gpu_aquaplanet_dyamond_diag_1process.yml --job_id gpu_aquaplanet_dyamond_diag_1process
Waited 49m 50s
Ran in 9m 35s
gpu_aquaplanet_dyamond - strong scaling - 1 GPUmkdir -p gpu_aquaplanet_dyamond_ss_1process && srun --cpu-bind=threads --cpus-per-task=4 nsys profile --delay 100 --trace=nvtx,mpi,cuda,osrt --output=gpu_aquaplanet_dyamond_ss_1process/output_active/report julia --threads=3 --color=yes --project=examples examples/hybrid/driver.jl --config_file config/gpu_configs/gpu_aquaplanet_dyamond_ss.yml --job_id gpu_aquaplanet_dyamond_ss_1process
Waited 50m 28s
Ran in 8m 22s
gpu_aquaplanet_dyamond - strong scaling - 2 GPUsmkdir -p gpu_aquaplanet_dyamond_ss_2process && srun --cpu-bind=threads --cpus-per-task=4 julia --threads=3 --color=yes --project=examples examples/hybrid/driver.jl --config_file config/gpu_configs/gpu_aquaplanet_dyamond_ss.yml --job_id gpu_aquaplanet_dyamond_ss_2process
Waited 50m 29s
Ran in 9m 55s
gpu_aquaplanet_dyamond - strong scaling - 4 GPUsmkdir -p gpu_aquaplanet_dyamond_ss_4process && srun --cpu-bind=threads --cpus-per-task=4 julia --threads=3 --color=yes --project=examples examples/hybrid/driver.jl --config_file config/gpu_configs/gpu_aquaplanet_dyamond_ss.yml --job_id gpu_aquaplanet_dyamond_ss_4process
Waited 1h 0m
Ran in 8m 31s
gpu_aquaplanet_dyamond - weak scaling - 1 GPUmkdir -p gpu_aquaplanet_dyamond_ws_1process && srun --cpu-bind=threads --cpus-per-task=4 julia --threads=3 --color=yes --project=examples examples/hybrid/driver.jl --config_file config/gpu_configs/gpu_aquaplanet_dyamond_ws_1process.yml --job_id gpu_aquaplanet_dyamond_ws_1process
Waited 1h 0m
Ran in 11m 56s
gpu_aquaplanet_dyamond - weak scaling - 2 GPUsmkdir -p gpu_aquaplanet_dyamond_ws_2process && srun --cpu-bind=threads --cpus-per-task=4 julia --threads=3 --color=yes --project=examples examples/hybrid/driver.jl --config_file config/gpu_configs/gpu_aquaplanet_dyamond_ws_2process.yml --job_id gpu_aquaplanet_dyamond_ws_2process
Waited 1h 8m
Ran in 11m 42s
gpu_aquaplanet_dyamond - weak scaling - 2 GPUsmkdir -p gpu_aquaplanet_dyamond_ws_2process && srun --cpu-bind=threads --cpus-per-task=4 julia --threads=3 --color=yes --project=examples examples/hybrid/driver.jl --config_file config/gpu_configs/gpu_aquaplanet_dyamond_ws_2process.yml --job_id gpu_aquaplanet_dyamond_ws_2process
Canceled
Waited 10s
gpu_aquaplanet_dyamond - weak scaling - 2 GPUsmkdir -p gpu_aquaplanet_dyamond_ws_2process && srun --cpu-bind=threads --cpus-per-task=4 julia --threads=3 --color=yes --project=examples examples/hybrid/driver.jl --config_file config/gpu_configs/gpu_aquaplanet_dyamond_ws_2process.yml --job_id gpu_aquaplanet_dyamond_ws_2process
Canceled
Waited 11s
gpu_aquaplanet_dyamond - weak scaling - 4 GPUsmkdir -p gpu_aquaplanet_dyamond_ws_4process && srun --cpu-bind=threads --cpus-per-task=4 julia --threads=3 --color=yes --project=examples examples/hybrid/driver.jl --config_file config/gpu_configs/gpu_aquaplanet_dyamond_ws_4process.yml --job_id gpu_aquaplanet_dyamond_ws_4process
Waited 1h 16m
Ran in 12m 5s
gpu_aquaplanet_diagedmf - 1 GPUmkdir -p gpu_aquaplanet_diagedmf && nsys profile --delay 200 --trace=nvtx,mpi,cuda,osrt --output=gpu_aquaplanet_diagedmf/output_active/report julia --threads=3 --color=yes --project=examples examples/hybrid/driver.jl --config_file config/model_configs/aquaplanet_diagedmf.yml --job_id gpu_aquaplanet_diagedmf
Waited 9m 47s
Ran in 9m 21s
gpu_aquaplanet_progedmf - 1 GPUmkdir -p gpu_aquaplanet_progedmf && nsys profile --delay 100 --trace=nvtx,mpi,cuda,osrt --output=gpu_aquaplanet_progedmf/output_active/report julia --threads=3 --color=yes --project=examples examples/hybrid/driver.jl --config_file config/model_configs/aquaplanet_progedmf.yml --job_id gpu_aquaplanet_progedmf
Waited 19m 18s
Ran in 12m 43s
Total Job Run Time: 2h 35m