From ab3c41ce7971556d69ababf96f4dc30274dc3d7a Mon Sep 17 00:00:00 2001 From: Charles Kawczynski Date: Sun, 7 Nov 2021 12:17:45 -0800 Subject: [PATCH] Update perf/allocations analysis tools --- .buildkite/pipeline.yml | 11 +++++ perf/alloc_per_case.jl | 19 ++++++++ perf/allocs.jl | 97 +++++++++++++++++++++++++++++++++++++++++ perf/common.jl | 41 +++++++++++++++++ perf/perf.jl | 6 +++ perf/profile.jl | 10 +++++ test/Project.toml | 3 ++ test/perf.jl | 20 --------- test/profile.jl | 22 ---------- 9 files changed, 187 insertions(+), 42 deletions(-) create mode 100644 perf/alloc_per_case.jl create mode 100644 perf/allocs.jl create mode 100644 perf/common.jl create mode 100644 perf/perf.jl create mode 100644 perf/profile.jl delete mode 100644 test/perf.jl delete mode 100644 test/profile.jl diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index eae2e0401..c3cf2e818 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -156,6 +156,17 @@ steps: queue: central slurm_ntasks: 1 + - label: ":rocket: Allocations analysis" + key: "cpu_allocations" + command: + - "julia --color=yes --project=test --track-allocation=user perf/allocs.jl" + artifact_paths: + - "perf/allocations_output/*" + agents: + config: cpu + queue: central + slurm_ntasks: 1 + - wait: ~ continue_on_failure: true diff --git a/perf/alloc_per_case.jl b/perf/alloc_per_case.jl new file mode 100644 index 000000000..0224c620d --- /dev/null +++ b/perf/alloc_per_case.jl @@ -0,0 +1,19 @@ +# Launch with `julia --project --track-allocation=user` +if !("." in LOAD_PATH) + push!(LOAD_PATH, ".") +end +import Profile + +include("common.jl") +case_name = ENV["ALLOCATION_CASE_NAME"] +@info "Recording allocations for $case_name" +sim = init_sim(case_name) +update_n(sim, 1) # compile first +Profile.clear_malloc_data() +update_n(sim, 1) + +# Quit julia (which generates .mem files), then call +#= +import Coverage +allocs = Coverage.analyze_malloc("src") +=# diff --git a/perf/allocs.jl b/perf/allocs.jl new file mode 100644 index 000000000..c013116a4 --- /dev/null +++ b/perf/allocs.jl @@ -0,0 +1,97 @@ +if !("." in LOAD_PATH) + push!(LOAD_PATH, ".") +end +import Coverage +import Plots + +# https://github.com/jheinen/GR.jl/issues/278#issuecomment-587090846 +ENV["GKSwstype"] = "nul" + +all_cases = [ + # "ARM_SGP", + # "Bomex", + # "DryBubble", + # "DYCOMS_RF01", + # "GABLS", + # "GATE_III", + # "life_cycle_Tan2018", + # "Nieuwstadt", + "Rico", + # "Soares", + # "SP", + "TRMM_LBA", + "LES_driven_SCM", +] + +filter!(x -> x ≠ "GATE_III", all_cases) # no mse tables for GATE_III +filter!(x -> x ≠ "SP", all_cases) # not currently running SP +allocs = Dict() +for case in all_cases + ENV["ALLOCATION_CASE_NAME"] = case + run(`julia --project=test/ --track-allocation=user perf/alloc_per_case.jl`) + allocs[case] = Coverage.analyze_malloc(".") + + # Clean up files + all_files = [joinpath(root, f) for (root, dirs, files) in Base.Filesystem.walkdir(".") for f in files] + all_mem_files = filter(x -> endswith(x, ".mem"), all_files) + for f in all_mem_files + rm(f) + end +end + +@info "Post-processing allocations" + +function plot_allocs(case_name, allocs_per_case, n_unique_bytes) + p = Plots.plot() + case_bytes = getproperty.(allocs_per_case, :bytes)[end:-1:1] + case_filename = getproperty.(allocs_per_case, :filename)[end:-1:1] + case_linenumber = getproperty.(allocs_per_case, :linenumber)[end:-1:1] + all_bytes = Int[] + filenames = String[] + linenumbers = Int[] + loc_ids = String[] + for (bytes, filename, linenumber) in zip(case_bytes, case_filename, case_linenumber) + filename_only = first(split(filename, ".jl")) * ".jl" + if endswith(filename_only, "TurbulenceConvection.jl") && linenumber == 1 + continue # Skip loading module + end + loc_id = "$filename_only" * "$linenumber" + if !(bytes in all_bytes) && !(loc_id in loc_ids) + push!(all_bytes, bytes) + push!(filenames, filename) + push!(linenumbers, linenumber) + push!(loc_ids, loc_id) + if length(all_bytes) ≥ n_unique_bytes + break + end + end + end + + all_bytes = all_bytes ./ 10^3 + max_bytes = maximum(all_bytes) + @info "$case_name: $all_bytes" + xtick_name(filename, linenumber) = "$filename, line number: $linenumber" + markershape = (:circle, :star, :square, :hexagon) + for (bytes, filename, linenumber) in zip(all_bytes, filenames, linenumbers) + markershape = (markershape[end], markershape[1:(end - 1)]...) + filename_only = first(split(filename, ".jl")) * ".jl" + Plots.plot!( + [0], + [bytes]; + seriestype = :scatter, + label = xtick_name(filename_only, linenumber), + markershape = markershape[1], + markersize = 1 + bytes / max_bytes * 10, + ) + end + Plots.plot!(ylabel = "Number of allocations (KB)") + Plots.savefig(joinpath(folder, "allocations_$case_name.png")) +end + +folder = "perf/allocations_output" +mkpath(folder) + +@info "Allocated bytes for single tendency per case:" +for case in all_cases + plot_allocs(case, allocs[case], 10) +end diff --git a/perf/common.jl b/perf/common.jl new file mode 100644 index 000000000..c8a2fd632 --- /dev/null +++ b/perf/common.jl @@ -0,0 +1,41 @@ +import TurbulenceConvection + +tc_dir = dirname(dirname(pathof(TurbulenceConvection))) +include(joinpath(tc_dir, "integration_tests", "utils", "generate_namelist.jl")) +include(joinpath(tc_dir, "integration_tests", "utils", "Cases.jl")) +include(joinpath(tc_dir, "integration_tests", "utils", "parameter_set.jl")) +include(joinpath(tc_dir, "integration_tests", "utils", "main.jl")) +import .NameList + +TurbulenceConvection.initialize_io(sim::Simulation1d) = nothing +TurbulenceConvection.io(sim::Simulation1d) = nothing + +update_n(sim, N::Int) = update_n(sim, Val(N)) + +function update_n(sim, ::Val{N}) where {N} + for i in 1:N + TC.update(sim.Turb, sim.grid, sim.state, sim.GMV, sim.Case, sim.TS) + end + return nothing +end + +function init_sim(case_name) + @info "Initializing $case_name for single timestep, with no IO." + @info "call update_n(sim, n) to run update n-times" + namelist = NameList.default_namelist(case_name) + namelist["time_stepping"]["t_max"] = namelist["time_stepping"]["dt"] + namelist["stats_io"]["frequency"] = 10.0e10 + namelist["stats_io"]["skip"] = true + namelist["meta"]["uuid"] = "01" + sim = Simulation1d(namelist) + + Cases.initialize_profiles(sim.Case, sim.grid, sim.GMV, sim.state) + TC.satadjust(sim.GMV, sim.grid, sim.state) + + Cases.initialize_surface(sim.Case, sim.grid, sim.state, sim.param_set) + Cases.initialize_forcing(sim.Case, sim.grid, sim.state, sim.GMV, sim.param_set) + Cases.initialize_radiation(sim.Case, sim.grid, sim.state, sim.GMV, sim.param_set) + + initialize_edmf(sim.Turb, sim.grid, sim.state, sim.Case, sim.GMV, sim.TS) + return sim +end diff --git a/perf/perf.jl b/perf/perf.jl new file mode 100644 index 000000000..9806c8501 --- /dev/null +++ b/perf/perf.jl @@ -0,0 +1,6 @@ +include("common.jl") +import BenchmarkTools + +sim = init_sim("Bomex") +update_n(sim, 1) # compile first +BenchmarkTools.@benchmark update_n($sim, 1) diff --git a/perf/profile.jl b/perf/profile.jl new file mode 100644 index 000000000..debadc7fa --- /dev/null +++ b/perf/profile.jl @@ -0,0 +1,10 @@ +include("common.jl") +import ProfileView +import Profile + +Profile.@profile update_n(sim, 100) +Profile.print() +# Profile.print(; format = :flat, sortedby = :count) +ProfileView.@profview update_n(sim, 1000) # compile first +ProfileView.@profview update_n(sim, 1000) +Profile.clear_malloc_data() diff --git a/test/Project.toml b/test/Project.toml index 1422798dd..241bcbb95 100644 --- a/test/Project.toml +++ b/test/Project.toml @@ -1,9 +1,11 @@ [deps] ArgParse = "c7e460c6-2fb9-53a9-8c5b-16f535851c63" ArtifactWrappers = "a14bc488-3040-4b00-9dc1-f6467924858a" +BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf" CLIMAParameters = "6eacf6c3-8458-43b9-ae03-caf5306d3d53" ClimaCore = "d414da3d-4745-48bb-8d80-42e94e092884" CloudMicrophysics = "6a9e3e04-43cd-43ba-94b9-e8782df3c71b" +Coverage = "a2441757-f6aa-5fb2-8edb-039e3f45d037" Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" Debugger = "31a5f54b-26ea-5ae9-a837-f05ce5417438" Dierckx = "39dd38d3-220a-591b-8e3c-4c3a8c710a94" @@ -23,6 +25,7 @@ Plots = "91a5bcdd-55d7-5caf-9e0b-520d859cae80" PoissonRandom = "e409e4f3-bfea-5376-8464-e040bb5c01ab" PrettyTables = "08abe8d2-0d0c-5749-adfa-8a2ac140af0d" Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7" +Profile = "9abbd945-dff8-562f-b5e8-e1ebf5ef1b79" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" RootSolvers = "7181ea78-2dcb-4de3-ab41-2b8ab5a31e74" StaticArrays = "90137ffa-7385-5640-81b9-e52037218182" diff --git a/test/perf.jl b/test/perf.jl deleted file mode 100644 index 4765a6bef..000000000 --- a/test/perf.jl +++ /dev/null @@ -1,20 +0,0 @@ -import TurbulenceConvection -using Test - -using Profile - -include(joinpath("integration_tests", "utils", "Cases.jl")) -include(joinpath("integration_tests", "utils", "generate_namelist.jl")) -using .Cases -import .NameList - -include(joinpath("integration_tests", "utils", "main.jl")) - -function run_main(; time_run = false) - namelist = default_namelist("Bomex") - namelist["meta"]["uuid"] = "01" - main(namelist; time_run = time_run) -end - -run_main() # run first to compile -# run_main(;time_run=true) diff --git a/test/profile.jl b/test/profile.jl deleted file mode 100644 index 8e8596658..000000000 --- a/test/profile.jl +++ /dev/null @@ -1,22 +0,0 @@ -import TurbulenceConvection -using Test - -using Profile - -include(joinpath("integration_tests", "utils", "Cases.jl")) -include(joinpath("integration_tests", "utils", "generate_namelist.jl")) -using .Cases -import .NameList - -include(joinpath("integration_tests", "utils", "main.jl")) - -function run_main() - namelist = default_namelist("GATE_III") - namelist["meta"]["uuid"] = "01" - main(namelist) -end - -run_main() # run first to compile -@profile run_main() -# Profile.print() -Profile.print(; format = :flat, sortedby = :count)