Introduce heatmap shader (#4317)

* add resampling heatmap shader

* add fast heatmap datashader

* fix bugs and improve implementation

* bugfixies

* add figure

* fix 1.6

* fix channel for 1.6

* add zoombutton for axis

* fix events picking from async events coming from closed session

* switch from heatmapshader(data) to heatmap(Resampler(data))

* update reference test

* clean up

* really dont update when scroll

* 1.6 compat and hide while updating

* fix 1.6

* fix issues

* add docs

* changelog + smal change

* final clean up

* Fix datainspector slowing down WGLMakie

* fix nothing bug

* use 3.2 for async inbox

* fix datainspector for heatmapshader

CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ## [Unreleased]
 
+- Introduce `heatmap(Resampler(large_matrix))`, allowing to show big images interactively [#4317](https://github.com/MakieOrg/Makie.jl/pull/4317).
 - Make sure we wait for the screen session [#4316](https://github.com/MakieOrg/Makie.jl/pull/4316).
 - Fix for absrect [#4312](https://github.com/MakieOrg/Makie.jl/pull/4312).
 - Fix attribute updates for SpecApi and SpecPlots (e.g. ecdfplot) [#4265](https://github.com/MakieOrg/Makie.jl/pull/4265).

CairoMakie/src/primitives.jl

@@ -33,7 +33,7 @@ function draw_atomic(scene::Scene, screen::Screen, @nospecialize(primitive::Unio
     # avoid them inverting.
     # TODO: If we have neither perspective projection not clip_planes we can
     #       use the normal projection_position() here
-    projected_positions, color, linewidth = 
+    projected_positions, color, linewidth =
         project_line_points(scene, primitive, positions, color, linewidth)
 
     # The linestyle can be set globally, as we do here.
@@ -798,7 +798,7 @@ function draw_atomic(scene::Scene, screen::Screen{RT}, @nospecialize(primitive::
 
         weird_cairo_limit = (2^15) - 23
         if s.width > weird_cairo_limit || s.height > weird_cairo_limit
-            error("Cairo stops rendering images bigger than $(weird_cairo_limit), which is likely a bug in Cairo. Please resample your image/heatmap with e.g. `ImageTransformations.imresize`")
+            error("Cairo stops rendering images bigger than $(weird_cairo_limit), which is likely a bug in Cairo. Please resample your image/heatmap with heatmap(Resampler(data)).")
         end
         Cairo.rectangle(ctx, xy..., w, h)
         Cairo.save(ctx)

GLMakie/src/drawing_primitives.jl

@@ -891,7 +891,7 @@ function draw_atomic(screen::Screen, scene::Scene, plot::Volume)
             # model/modelinv has no perspective projection so we should be fine
             # with just applying it to the plane origin and transpose(inv(modelinv))
             # to plane.normal
-            @assert (length(planes) == 0) || isapprox(modelinv[4, 4], 1, atol = 1e-6) 
+            @assert (length(planes) == 0) || isapprox(modelinv[4, 4], 1, atol = 1e-6)
 
             output = Vector{Vec4f}(undef, 8)
             for i in 1:min(length(planes), 8)
@@ -965,7 +965,7 @@ function draw_atomic(screen::Screen, scene::Scene, plot::Voxels)
             # with just applying it to the plane origin and transpose(inv(modelinv))
             # to plane.normal
             modelinv = inv(model)
-            @assert (length(planes) == 0) || isapprox(modelinv[4, 4], 1, atol = 1e-6) 
+            @assert (length(planes) == 0) || isapprox(modelinv[4, 4], 1, atol = 1e-6)
 
             output = Vector{Vec4f}(undef, 8)
             for i in 1:min(length(planes), 8)

MakieCore/src/basic_plots.jl

@@ -78,9 +78,9 @@ function mixin_generic_plot_attributes()
         "Sets a callback function `(inspector, plot, index) -> ...` which replaces the default `show_data` methods."
         inspector_hover = automatic
         """
-        Clip planes offer a way to do clipping in 3D space. You can set a Vector of up to 8 `Plane3f` planes here, 
-        behind which plots will be clipped (i.e. become invisible). By default clip planes are inherited from the 
-        parent plot or scene. You can remove parent `clip_planes` by passing `Plane3f[]`. 
+        Clip planes offer a way to do clipping in 3D space. You can set a Vector of up to 8 `Plane3f` planes here,
+        behind which plots will be clipped (i.e. become invisible). By default clip planes are inherited from the
+        parent plot or scene. You can remove parent `clip_planes` by passing `Plane3f[]`.
         """
         clip_planes = automatic
     end

Project.toml

@@ -26,8 +26,10 @@ FreeType = "b38be410-82b0-50bf-ab77-7b57e271db43"
 FreeTypeAbstraction = "663a7486-cb36-511b-a19d-713bb74d65c9"
 GeometryBasics = "5c1252a2-5f33-56bf-86c9-59e7332b4326"
 GridLayoutBase = "3955a311-db13-416c-9275-1d80ed98e5e9"
+ImageBase = "c817782e-172a-44cc-b673-b171935fbb9e"
 ImageIO = "82e4d734-157c-48bb-816b-45c225c6df19"
 InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240"
+Interpolations = "a98d9a8b-a2ab-59e6-89dd-64a1c18fca59"
 IntervalSets = "8197267c-284f-5f27-9208-e0e47529a953"
 Isoband = "f1662d9f-8043-43de-a69a-05efc1cc6ff4"
 KernelDensity = "5ab0869b-81aa-558d-bb23-cbf5423bbe9b"
@@ -82,8 +84,10 @@ FreeType = "3.0, 4.0"
 FreeTypeAbstraction = "0.10.3"
 GeometryBasics = "0.4.11"
 GridLayoutBase = "0.11"
+ImageBase = "0.1.7"
 ImageIO = "0.2, 0.3, 0.4, 0.5, 0.6"
 InteractiveUtils = "1.0, 1.6"
+Interpolations = "0.15.1"
 IntervalSets = "0.3, 0.4, 0.5, 0.6, 0.7"
 Isoband = "0.1"
 KernelDensity = "0.5, 0.6"

ReferenceTests/src/tests/examples2d.jl

@@ -1375,7 +1375,7 @@ end
 @reference_test "Triplot with nonlinear transformation" begin
     f = Figure()
     ax = PolarAxis(f[1, 1])
-    points = Point2f[(phi, r) for r in 1:10 for phi in range(0, 2pi, length=36)[1:35]] 
+    points = Point2f[(phi, r) for r in 1:10 for phi in range(0, 2pi, length=36)[1:35]]
     noise = i -> 1f-4 * (isodd(i) ? 1 : -1) * i/sqrt(50) # should have small discrepancy
     points = points .+ [Point2f(noise(i), noise(i)) for i in eachindex(points)]
     # The noise forces the triangulation to be unique. Not using RNG to not disrupt the RNG stream later
@@ -1605,3 +1605,20 @@ end
 @reference_test "Lines with OffsetArrays" begin
     lines(Makie.OffsetArrays.Origin(-50)(1:100))
 end
+
+@reference_test "Heatmap Shader" begin
+    data = Makie.peaks(10_000)
+    data2 = map(data) do x
+        Float32(round(x))
+    end
+    f = Figure()
+    ax1, pl1 = heatmap(f[1, 1], Resampler(data))
+    ax2, pl2 = heatmap(f[1, 2], Resampler(data))
+    limits!(ax2, 2800, 4800, 2800, 5000)
+    ax3, pl3 = heatmap(f[2, 1], Resampler(data2))
+    ax4, pl4 = heatmap(f[2, 2], Resampler(data2))
+    limits!(ax4, 3000, 3090, 3460, 3500)
+    Colorbar(f[:, 3], pl1)
+    sleep(1) # give the async operations some time
+    f
+end

WGLMakie/Project.toml

@@ -20,7 +20,7 @@ ShaderAbstractions = "65257c39-d410-5151-9873-9b3e5be5013e"
 StaticArrays = "90137ffa-7385-5640-81b9-e52037218182"
 
 [compat]
-Bonito = "3.1.2"
+Bonito = "3.2.0"
 Colors = "0.11, 0.12"
 FileIO = "1.1"
 FreeTypeAbstraction = "0.10"

WGLMakie/src/Serialization.js

@@ -265,7 +265,7 @@ function connect_uniforms(mesh, updater) {
 
 function convert_RGB_to_RGBA(rgbArray) {
     const length = rgbArray.length;
-    const rgbaArray = new Float32Array((length / 3) * 4);
+    const rgbaArray = new rgbArray.constructor((length / 3) * 4);
 
     for (let i = 0, j = 0; i < length; i += 3, j += 4) {
         rgbaArray[j] = rgbArray[i]; // R

WGLMakie/src/display.jl

@@ -47,7 +47,7 @@ $(Base.doc(ScreenConfig))
 $(Base.doc(MakieScreen))
 """
 mutable struct Screen <: Makie.MakieScreen
-    plot_initialized::Channel{Bool}
+    plot_initialized::Channel{Any}
     session::Union{Nothing,Session}
     scene::Union{Nothing,Scene}
     displayed_scenes::Set{String}
@@ -56,7 +56,7 @@ mutable struct Screen <: Makie.MakieScreen
     tick_clock::Makie.BudgetedTimer
     function Screen(scene::Union{Nothing,Scene}, config::ScreenConfig)
         timer = Makie.BudgetedTimer(1.0 / 30.0)
-        screen = new(Channel{Bool}(1), nothing, scene, Set{String}(), config, nothing, timer)
+        screen = new(Channel{Any}(1), nothing, scene, Set{String}(), config, nothing, timer)
 
         finalizer(screen) do screen
             close(screen.tick_clock)
@@ -78,17 +78,23 @@ end
 function render_with_init(screen::Screen, session::Session, scene::Scene)
     # Reference to three object which gets set once we serve this to a browser
     # Make sure it's a new Channel, since we may re-use the screen.
-    screen.plot_initialized = Channel{Bool}(1)
+    screen.plot_initialized = Channel{Any}(1)
     screen.session = session
     Makie.push_screen!(scene, screen)
     canvas, on_init = three_display(screen, session, scene)
     screen.canvas = canvas
     on(session, on_init) do initialized
-        if !isready(screen.plot_initialized) && initialized
+        if isready(screen.plot_initialized)
+            # plot_initialized contains already an item
+            # This should not happen, but lets check anyways, so it errors and doesn't hang forever
+            error("Plot inititalized multiple times?")
+        end
+        if initialized == true
             put!(screen.plot_initialized, true)
             mark_as_displayed!(screen, scene)
         else
-            error("Three object should be ready after init, but isn't - connection interrupted? Session: $(session), initialized: $(initialized)")
+            # Will be an eror from WGLMakie.js
+            put!(screen.plot_initialized, initialized)
         end
         return
     end
@@ -203,6 +209,8 @@ function get_screen_session(screen::Screen; timeout=100,
         if !isnothing(error)
             message = "Can't get three: $(status)\n$(error)"
             Base.error(message)
+        else
+            # @warn "Can't get three: $(status)\n$(error)"
         end
     end
     if isnothing(screen.session)
@@ -211,7 +219,7 @@ function get_screen_session(screen::Screen; timeout=100,
     end
     session = screen.session
     if !(session.status in (Bonito.RENDERED, Bonito.DISPLAYED, Bonito.OPEN))
-        throw_error("Screen Session uninitialized. Not yet displayed? Session status: $(screen.session.status)")
+        throw_error("Screen Session uninitialized. Not yet displayed? Session status: $(screen.session.status), id: $(session.id)")
         return nothing
     end
     success = Bonito.wait_for_ready(session; timeout=timeout)
@@ -223,6 +231,12 @@ function get_screen_session(screen::Screen; timeout=100,
     # Throw error if error message specified
     if success !== :success
         throw_error("Timed out waiting $(timeout)s for session to get initilize")
+        return nothing
+    end
+    value = fetch(screen.plot_initialized)
+    if value !== true
+        throw_error("Error initializing plot: $(value)")
+        return nothing
     end
     # At this point we should have a fully initialized plot + session
     return session
@@ -243,7 +257,7 @@ Screen(scene::Scene, config::ScreenConfig, ::Makie.ImageStorageFormat) = Screen(
 
 function Base.empty!(screen::Screen)
     screen.scene = nothing
-    screen.plot_initialized = Channel{Bool}(1)
+    screen.plot_initialized = Channel{Any}(1)
     return
     # TODO, empty state in JS, to be able to reuse screen
 end

WGLMakie/src/meshes.jl

@@ -94,7 +94,6 @@ function draw_mesh(mscene::Scene, per_vertex, plot, uniforms; permute_tex=true)
     pos = pop!(per_vertex, :positions)
     faces = pop!(per_vertex, :faces)
     mesh = GeometryBasics.Mesh(meta(pos; per_vertex...), faces)
-
     return Program(WebGL(), lasset("mesh.vert"), lasset("mesh.frag"), mesh, uniforms)
 end
 

WGLMakie/src/particles.jl

@@ -163,6 +163,7 @@ function scatter_shader(scene::Scene, attributes, plot)
         font = get(attributes, :font, Observable(Makie.defaultfont()))
         marker = lift(plot, attributes[:marker]) do marker
             marker isa Makie.FastPixel && return Rect # FastPixel not supported, but same as Rect just slower
+            marker isa AbstractMatrix{<:Colorant} && return to_color(marker)
             return Makie.to_spritemarker(marker)
         end
 

WGLMakie/src/picking.jl

@@ -2,12 +2,13 @@
 function pick_native(screen::Screen, rect::Rect2i)
     (x, y) = minimum(rect)
     (w, h) = widths(rect)
-    session = get_screen_session(screen; error="Can't do picking!")
+    session = get_screen_session(screen)
+    empty = Matrix{Tuple{Union{Nothing,AbstractPlot},Int}}(undef, 0, 0)
+    isnothing(session) && return empty
     scene = screen.scene
     picking_data = Bonito.evaljs_value(session, js"""
         Promise.all([$(WGL), $(scene)]).then(([WGL, scene]) => WGL.pick_native_matrix(scene, $x, $y, $w, $h))
     """)
-    empty = Matrix{Tuple{Union{Nothing, AbstractPlot}, Int}}(undef, 0, 0)
     if isnothing(picking_data)
         return empty
     end
@@ -34,7 +35,9 @@ function Makie.pick_closest(scene::Scene, screen::Screen, xy, range::Integer)
     # isopen(screen) || return (nothing, 0)
     xy_vec = Cint[round.(Cint, xy)...]
     range = round(Int, range)
-    session = get_screen_session(screen; error="Can't do picking!")
+    session = get_screen_session(screen)
+    # E.g. if websocket got closed
+    isnothing(session) && return (nothing, 0)
     selection = Bonito.evaljs_value(session, js"""
         Promise.all([$(WGL), $(scene)]).then(([WGL, scene]) => WGL.pick_closest(scene, $(xy_vec), $(range)))
     """)
@@ -45,19 +48,18 @@ end
 
 # Skips some allocations
 function Makie.pick_sorted(scene::Scene, screen::Screen, xy, range)
+
     xy_vec = Cint[round.(Cint, xy)...]
     range = round(Int, range)
-
-    session = get_screen_session(screen; error="Can't do picking!")
+    session = get_screen_session(screen)
+    # E.g. if websocket got closed
+    isnothing(session) && return Tuple{Plot,Int}[]
     selection = Bonito.evaljs_value(session, js"""
         Promise.all([$(WGL), $(scene)]).then(([WGL, scene]) => WGL.pick_sorted(scene, $(xy_vec), $(range)))
     """)
-    isnothing(selection) && return Tuple{Union{Nothing,AbstractPlot},Int}[]
+    isnothing(selection) && return Tuple{Plot,Int}[]
     lookup = plot_lookup(scene)
-    return map(selection) do (plot_id, index)
-        !haskey(lookup, plot_id) && return (nothing, 0)
-        return (lookup[plot_id], index + 1)
-    end
+    return [(lookup[plot_id], index + 1) for (plot_id, index) in selection if haskey(lookup, plot_id)]
 end
 
 function Makie.pick(::Scene, screen::Screen, xy)

WGLMakie/src/serialization.jl

@@ -145,7 +145,7 @@ function flatten_buffer(array::Buffer)
 end
 
 function flatten_buffer(array::AbstractArray{T}) where {T<:N0f8}
-    return reinterpret(UInt8, array)
+    return collect(reinterpret(UInt8, array))
 end
 
 function flatten_buffer(array::AbstractArray{T}) where {T}
@@ -171,8 +171,6 @@ function ShaderAbstractions.convert_uniform(::ShaderAbstractions.AbstractContext
     return convert(Quaternion, t)
 end
 
-
-
 function wgl_convert(value, key1, key2...)
     val = Makie.convert_attribute(value, key1, key2...)
     return if val isa AbstractArray{<:Float64}
@@ -200,6 +198,7 @@ function register_geometry_updates(@nospecialize(plot), update_buffer::Observabl
     for (name, buffer) in _pairs(named_buffers)
         if buffer isa Buffer
             on(plot, ShaderAbstractions.updater(buffer).update) do (f, args)
+
                 # update to replace the whole buffer!
                 if f === ShaderAbstractions.update!
                     new_array = args[1]

WGLMakie/src/three_plot.jl

@@ -43,7 +43,7 @@ function three_display(screen::Screen, session::Session, scene::Scene)
     )
     wrapper = DOM.div(canvas; style="width: 100%; height: 100%")
     comm = Observable(Dict{String,Any}())
-    done_init = Observable(false)
+    done_init = Observable{Any}(nothing)
     # Keep texture atlas in parent session, so we don't need to send it over and over again
     ta = Bonito.Retain(TEXTURE_ATLAS)
     evaljs(session, js"""
@@ -61,7 +61,7 @@ function three_display(screen::Screen, session::Session, scene::Scene)
             $(done_init).notify(true)
         } catch (e) {
             Bonito.Connection.send_error("error initializing scene", e)
-            $(done_init).notify(false)
+            $(done_init).notify(e)
             return
         }
     })

WGLMakie/src/wglmakie.bundled.js

@@ -22500,7 +22500,7 @@ function connect_uniforms(mesh, updater) {
 }
 function convert_RGB_to_RGBA(rgbArray) {
     const length = rgbArray.length;
-    const rgbaArray = new Float32Array(length / 3 * 4);
+    const rgbaArray = new rgbArray.constructor(length / 3 * 4);
     for(let i = 0, j = 0; i < length; i += 3, j += 4){
         rgbaArray[j] = rgbArray[i];
         rgbaArray[j + 1] = rgbArray[i + 1];
@@ -22518,10 +22518,13 @@ function create_texture_from_data(data) {
         return tex;
     } else {
         let format = mod[data.three_format];
+        console.log(buffer);
+        console.log(data);
         if (data.three_format == "RGBFormat") {
             buffer = convert_RGB_to_RGBA(buffer);
             format = mod.RGBAFormat;
         }
+        console.log(format);
         return new mod.DataTexture(buffer, data.size[0], data.size[1], format, mod[data.three_type]);
     }
 }