[gtk/matthiasc/color-profiles] wip: Do texture conversions in shaders



commit ff1b0a2820451942fa6ca2ac0bb2c7a259dab435
Author: Matthias Clasen <mclasen redhat com>
Date:   Sat Oct 2 11:45:26 2021 -0400

    wip: Do texture conversions in shaders
    
    Do linearization and premultiplication of textures
    in a shader after uploading the unconverted data.
    This moves move work from the cpu to the gpu.
    
    Conversions between other colorspaces are still
    done with lcms on the cpu.

 gsk/meson.build                              |   2 +
 gsk/ngl/gsknglcommandqueue.c                 |  68 ++++++++++++----
 gsk/ngl/gsknglcommandqueueprivate.h          |   9 ++-
 gsk/ngl/gskngldriver.c                       | 111 +++++++++++++++++++++------
 gsk/ngl/gsknglprograms.defs                  |   8 ++
 gsk/ngl/resources/linearize_premultiply.glsl |  21 +++++
 gsk/ngl/resources/premultiply.glsl           |  20 +++++
 7 files changed, 196 insertions(+), 43 deletions(-)
---
diff --git a/gsk/meson.build b/gsk/meson.build
index 7c1442cfd6..49655afbb0 100644
--- a/gsk/meson.build
+++ b/gsk/meson.build
@@ -21,6 +21,8 @@ gsk_private_ngl_shaders = [
   'ngl/resources/filled_border.glsl',
   'ngl/resources/postprocessing.glsl',
   'ngl/resources/linearize.glsl',
+  'ngl/resources/premultiply.glsl',
+  'ngl/resources/linearize_premultiply.glsl',
 ]
 
 gsk_public_sources = files([
diff --git a/gsk/ngl/gsknglcommandqueue.c b/gsk/ngl/gsknglcommandqueue.c
index dbcbbf9ca2..1c299e9542 100644
--- a/gsk/ngl/gsknglcommandqueue.c
+++ b/gsk/ngl/gsknglcommandqueue.c
@@ -530,7 +530,7 @@ gsk_ngl_command_queue_begin_draw (GskNglCommandQueue   *self,
   GskNglCommandBatch *batch;
 
   g_assert (GSK_IS_NGL_COMMAND_QUEUE (self));
-  g_assert (self->in_draw == FALSE);
+  g_assert (!self->in_draw);
   g_assert (width <= G_MAXUINT16);
   g_assert (height <= G_MAXUINT16);
 
@@ -568,6 +568,7 @@ gsk_ngl_command_queue_end_draw (GskNglCommandQueue *self)
   GskNglCommandBatch *batch;
 
   g_assert (GSK_IS_NGL_COMMAND_QUEUE (self));
+  g_assert (self->in_draw);
   g_assert (self->batches.len > 0);
 
   if (will_ignore_batch (self))
@@ -1331,20 +1332,31 @@ gsk_ngl_command_queue_do_upload_texture (GdkGLContext    *context,
                                          int              y,
                                          int              width,
                                          int              height,
-                                         guint            texture_target)
+                                         guint            texture_target,
+                                         GskConversion   *conversion)
 {
   GdkMemoryTexture *memory_texture;
   GdkMemoryFormat data_format;
+  GdkColorProfile *data_profile;
   GLenum gl_internalformat;
   GLenum gl_format;
   GLenum gl_type;
   gsize bpp, stride;
   const guchar *data;
+  gboolean convert_locally = FALSE;
 
   g_return_if_fail (GDK_IS_GL_CONTEXT (context));
 
   memory_texture = GDK_MEMORY_TEXTURE (gdk_texture_download_texture (texture));
   data_format = gdk_memory_texture_get_format (memory_texture);
+  data_profile = gdk_texture_get_color_profile (GDK_TEXTURE (memory_texture));
+
+  if (data_profile == gdk_color_profile_get_srgb ())
+    *conversion = GSK_CONVERSION_LINEARIZE;
+  else if (data_profile == gdk_color_profile_get_srgb_linear ())
+    *conversion = 0;
+  else
+    convert_locally = TRUE;
 
   if (!gdk_memory_format_gl_format (data_format,
                                     gdk_gl_context_get_use_es (context),
@@ -1352,21 +1364,33 @@ gsk_ngl_command_queue_do_upload_texture (GdkGLContext    *context,
                                     &gl_format,
                                     &gl_type))
     {
-      data_format = GDK_MEMORY_R8G8B8A8_PREMULTIPLIED;
-      if (!gdk_memory_format_gl_format (GDK_MEMORY_R8G8B8A8_PREMULTIPLIED,
-                                        gdk_gl_context_get_use_es (context),
-                                        &gl_internalformat,
-                                        &gl_format,
-                                        &gl_type))
+      if (!gdk_gl_context_get_use_es (context))
         {
-          g_assert_not_reached ();
+          *conversion |= GSK_CONVERSION_PREMULTIPLY;
+        }
+      else
+        {
+          convert_locally = TRUE;
+          data_format = GDK_MEMORY_R8G8B8A8_PREMULTIPLIED;
+          if (!gdk_memory_format_gl_format (GDK_MEMORY_R8G8B8A8_PREMULTIPLIED,
+                                            gdk_gl_context_get_use_es (context),
+                                            &gl_internalformat,
+                                            &gl_format,
+                                            &gl_type))
+            {
+              g_assert_not_reached ();
+            }
         }
     }
 
-  memory_texture = gdk_memory_texture_convert (memory_texture,
-                                               data_format,
-                                               gdk_color_profile_get_srgb_linear (),
-                                               &(GdkRectangle) { x, y, width, height });
+  if (convert_locally)
+    {
+      memory_texture = gdk_memory_texture_convert (memory_texture,
+                                                   data_format,
+                                                   gdk_color_profile_get_srgb_linear (),
+                                                   &(GdkRectangle) { x, y, width, height });
+      *conversion = 0;
+    }
 
   bpp = gdk_memory_format_bytes_per_pixel (data_format);
   stride = gdk_memory_texture_get_stride (memory_texture);
@@ -1412,9 +1436,11 @@ gsk_ngl_command_queue_upload_texture (GskNglCommandQueue *self,
                                       guint               width,
                                       guint               height,
                                       int                 min_filter,
-                                      int                 mag_filter)
+                                      int                 mag_filter,
+                                      GskConversion      *conversion)
 {
   G_GNUC_UNUSED gint64 start_time = GDK_PROFILER_CURRENT_TIME;
+  int format;
   int texture_id;
 
   g_assert (GSK_IS_NGL_COMMAND_QUEUE (self));
@@ -1424,6 +1450,8 @@ gsk_ngl_command_queue_upload_texture (GskNglCommandQueue *self,
   g_assert (min_filter == GL_LINEAR || min_filter == GL_NEAREST);
   g_assert (mag_filter == GL_LINEAR || min_filter == GL_NEAREST);
 
+  *conversion = 0;
+
   if (width > self->max_texture_size || height > self->max_texture_size)
     {
       g_warning ("Attempt to create texture of size %ux%u but max size is %d. "
@@ -1433,7 +1461,12 @@ gsk_ngl_command_queue_upload_texture (GskNglCommandQueue *self,
       height = MAX (height, self->max_texture_size);
     }
 
-  texture_id = gsk_ngl_command_queue_create_texture (self, width, height, GL_RGBA8, min_filter, mag_filter);
+  format = gdk_texture_is_hdr (texture) ? GL_RGBA16F : GL_RGBA8;
+
+  texture_id = gsk_ngl_command_queue_create_texture (self,
+                                                     width, height,
+                                                     format,
+                                                     min_filter, mag_filter);
   if (texture_id == -1)
     return texture_id;
 
@@ -1443,11 +1476,12 @@ gsk_ngl_command_queue_upload_texture (GskNglCommandQueue *self,
   glActiveTexture (GL_TEXTURE0);
   glBindTexture (GL_TEXTURE_2D, texture_id);
 
-  gsk_ngl_command_queue_do_upload_texture (gdk_gl_context_get_current (),
+  gsk_ngl_command_queue_do_upload_texture (self->context,
                                            texture,
                                            x_offset, y_offset,
                                            width, height,
-                                           GL_TEXTURE_2D);
+                                           GL_TEXTURE_2D,
+                                           conversion);
 
   /* Restore previous texture state if any */
   if (self->attachments->textures[0].id > 0)
diff --git a/gsk/ngl/gsknglcommandqueueprivate.h b/gsk/ngl/gsknglcommandqueueprivate.h
index 40cca01162..4e7ac594ad 100644
--- a/gsk/ngl/gsknglcommandqueueprivate.h
+++ b/gsk/ngl/gsknglcommandqueueprivate.h
@@ -279,6 +279,12 @@ void                gsk_ngl_command_queue_execute              (GskNglCommandQue
                                                                 guint                  surface_height,
                                                                 guint                  scale_factor,
                                                                 const cairo_region_t  *scissor);
+
+typedef enum {
+  GSK_CONVERSION_LINEARIZE   = 1 << 0,
+  GSK_CONVERSION_PREMULTIPLY = 1 << 1,
+} GskConversion;
+
 int                 gsk_ngl_command_queue_upload_texture       (GskNglCommandQueue    *self,
                                                                 GdkTexture            *texture,
                                                                 guint                  x_offset,
@@ -286,7 +292,8 @@ int                 gsk_ngl_command_queue_upload_texture       (GskNglCommandQue
                                                                 guint                  width,
                                                                 guint                  height,
                                                                 int                    min_filter,
-                                                                int                    mag_filter);
+                                                                int                    mag_filter,
+                                                                GskConversion         *remaining);
 int                 gsk_ngl_command_queue_create_texture       (GskNglCommandQueue    *self,
                                                                 int                    width,
                                                                 int                    height,
diff --git a/gsk/ngl/gskngldriver.c b/gsk/ngl/gskngldriver.c
index 993dad11a8..a3e3e6b040 100644
--- a/gsk/ngl/gskngldriver.c
+++ b/gsk/ngl/gskngldriver.c
@@ -730,9 +730,10 @@ draw_offscreen (GskNglCommandQueue *command_queue,
 }
 
 static void
-set_viewport_for_size (GskNglProgram *program,
-                         float         width,
-                         float         height)
+set_viewport_for_size (GskNglDriver  *self,
+                       GskNglProgram *program,
+                       float          width,
+                       float          height)
 {
   float viewport[4] = { 0, 0, width, height };
 
@@ -741,15 +742,17 @@ set_viewport_for_size (GskNglProgram *program,
                                 UNIFORM_SHARED_VIEWPORT, 0,
                                 1,
                                 (const float *)&viewport);
+  self->stamps[UNIFORM_SHARED_VIEWPORT]++;
 }
 
 #define ORTHO_NEAR_PLANE   -10000
 #define ORTHO_FAR_PLANE     10000
 
 static void
-set_projection_for_size (GskNglProgram *program,
-                       float            width,
-                       float            height)
+set_projection_for_size (GskNglDriver  *self,
+                         GskNglProgram *program,
+                         float          width,
+                         float          height)
 {
   graphene_matrix_t projection;
 
@@ -760,10 +763,12 @@ set_projection_for_size (GskNglProgram *program,
                                     program->program_info,
                                     UNIFORM_SHARED_PROJECTION, 0,
                                     &projection);
+  self->stamps[UNIFORM_SHARED_PROJECTION]++;
 }
 
 static void
-reset_modelview (GskNglProgram *program)
+reset_modelview (GskNglDriver  *self,
+                 GskNglProgram *program)
 {
   graphene_matrix_t modelview;
 
@@ -773,25 +778,38 @@ reset_modelview (GskNglProgram *program)
                                     program->program_info,
                                     UNIFORM_SHARED_MODELVIEW, 0,
                                     &modelview);
+  self->stamps[UNIFORM_SHARED_MODELVIEW]++;
 }
 
 static GskNglTexture *
-gsk_ngl_driver_convert_texture (GskNglDriver *self,
-                                int           texture_id,
-                                int           width,
-                                int           height,
-                                int           format)
+gsk_ngl_driver_convert_texture (GskNglDriver  *self,
+                                int            texture_id,
+                                int            width,
+                                int            height,
+                                int            format,
+                                int            min_filter,
+                                int            mag_filter,
+                                GskConversion  conversion)
 {
   GskNglRenderTarget *target;
   int prev_fbo;
-  GskNglProgram *program = self->linearize_no_clip;
+  GskNglProgram *program;
+
+  if (conversion == (GSK_CONVERSION_LINEARIZE | GSK_CONVERSION_PREMULTIPLY))
+    program = self->linearize_premultiply_no_clip;
+  else if (conversion & GSK_CONVERSION_LINEARIZE)
+    program = self->linearize_no_clip;
+  else if (conversion & GSK_CONVERSION_PREMULTIPLY)
+    program = self->premultiply_no_clip;
+  else
+    g_assert_not_reached ();
 
   gdk_gl_context_make_current (self->command_queue->context);
 
   gsk_ngl_driver_create_render_target (self,
                                        width, height,
                                        format,
-                                       GL_LINEAR, GL_LINEAR,
+                                       min_filter, mag_filter,
                                        &target);
 
   prev_fbo = gsk_ngl_command_queue_bind_framebuffer (self->command_queue, target->framebuffer_id);
@@ -801,14 +819,13 @@ gsk_ngl_driver_convert_texture (GskNglDriver *self,
                                     program->program_info,
                                     width, height);
 
-  set_projection_for_size (program, width, height);
-  set_viewport_for_size (program, width, height);
-  reset_modelview (program);
+  set_projection_for_size (self, program, width, height);
+  set_viewport_for_size (self, program, width, height);
+  reset_modelview (self, program);
 
   gsk_ngl_program_set_uniform_texture (program,
                                        UNIFORM_SHARED_SOURCE, 0,
                                        GL_TEXTURE_2D, GL_TEXTURE0, texture_id);
-
   draw_offscreen (self->command_queue, 0, 0, width, height);
 
   gsk_ngl_command_queue_end_draw (self->command_queue);
@@ -856,6 +873,7 @@ gsk_ngl_driver_load_texture (GskNglDriver *self,
   int height;
   int width;
   int format;
+  GskConversion conversion;
 
   g_return_val_if_fail (GSK_IS_NGL_DRIVER (self), 0);
   g_return_val_if_fail (GDK_IS_TEXTURE (texture), 0);
@@ -895,10 +913,14 @@ gsk_ngl_driver_load_texture (GskNglDriver *self,
                * is nonlinear sRGB. Eventually, we should figure out how
                * to convert from other color spaces to linear sRGB
                */
+              t = gsk_ngl_driver_convert_texture (self,
+                                                  gl_texture_id,
+                                                  width, height, format,
+                                                  min_filter, mag_filter,
+                                                  GSK_CONVERSION_LINEARIZE);
 
-              t = gsk_ngl_driver_convert_texture (self, gl_texture_id, width, height, format);
 
-              if (gdk_texture_set_render_data (texture, self, t, (GDestroyNotify)gsk_ngl_texture_destroyed))
+              if (gdk_texture_set_render_data (texture, self, t, gsk_ngl_texture_destroyed))
                 t->user = texture;
 
               return t->texture_id;
@@ -909,7 +931,8 @@ gsk_ngl_driver_load_texture (GskNglDriver *self,
   downloaded_texture = gdk_texture_download_texture (texture);
 
   /* The download_texture() call may have switched the GL context. Make sure
-   * the right context is at work again. */
+   * the right context is at work again.
+   */
   gdk_gl_context_make_current (context);
 
   texture_id = gsk_ngl_command_queue_upload_texture (self->command_queue,
@@ -919,9 +942,8 @@ gsk_ngl_driver_load_texture (GskNglDriver *self,
                                                      width,
                                                      height,
                                                      min_filter,
-                                                     mag_filter);
-
-  g_clear_object (&downloaded_texture);
+                                                     mag_filter,
+                                                     &conversion);
 
   t = gsk_ngl_texture_new (texture_id,
                            width, height, format, min_filter, mag_filter,
@@ -929,6 +951,15 @@ gsk_ngl_driver_load_texture (GskNglDriver *self,
 
   g_hash_table_insert (self->textures, GUINT_TO_POINTER (texture_id), t);
 
+  g_clear_object (&downloaded_texture);
+
+  if (conversion)
+    t = gsk_ngl_driver_convert_texture (self,
+                                        texture_id,
+                                        width, height, format,
+                                        min_filter, mag_filter,
+                                        conversion);
+
   if (gdk_texture_set_render_data (texture, self, t, gsk_ngl_texture_destroyed))
     t->user = texture;
 
@@ -976,6 +1007,7 @@ gsk_ngl_driver_create_texture (GskNglDriver *self,
   g_hash_table_insert (self->textures,
                        GUINT_TO_POINTER (texture->texture_id),
                        texture);
+
   texture->last_used_in_frame = self->current_frame_id;
   return texture;
 }
@@ -1335,6 +1367,7 @@ gsk_ngl_driver_add_texture_slices (GskNglDriver        *self,
   int tex_width;
   int tex_height;
   int x = 0, y = 0;
+  int format;
 
   g_assert (GSK_IS_NGL_DRIVER (self));
   g_assert (GDK_IS_TEXTURE (texture));
@@ -1346,6 +1379,9 @@ gsk_ngl_driver_add_texture_slices (GskNglDriver        *self,
 
   tex_width = texture->width;
   tex_height = texture->height;
+
+  format = gdk_texture_is_hdr (texture) ? GL_RGBA16F : GL_RGBA8;
+
   cols = (texture->width / max_texture_size) + 1;
   rows = (texture->height / max_texture_size) + 1;
 
@@ -1368,12 +1404,37 @@ gsk_ngl_driver_add_texture_slices (GskNglDriver        *self,
           int slice_height = MIN (max_texture_size, texture->height - y);
           int slice_index = (col * rows) + row;
           guint texture_id;
+          GskConversion conversion;
 
           texture_id = gsk_ngl_command_queue_upload_texture (self->command_queue,
                                                              texture,
                                                              x, y,
                                                              slice_width, slice_height,
-                                                             GL_NEAREST, GL_NEAREST);
+                                                             GL_NEAREST, GL_NEAREST,
+                                                             &conversion);
+
+
+          if (conversion)
+            {
+              t = gsk_ngl_texture_new (texture_id,
+                                       slice_width, slice_height,
+                                       format,
+                                       GL_NEAREST, GL_NEAREST,
+                                       0);
+              g_hash_table_insert (self->textures, GUINT_TO_POINTER (texture_id), t);
+
+              t = gsk_ngl_driver_convert_texture (self,
+                                                  texture_id,
+                                                  slice_width, slice_height,
+                                                  format,
+                                                  GL_NEAREST, GL_NEAREST,
+                                                  conversion);
+
+              texture_id = t->texture_id;
+              t->texture_id = 0;
+              g_hash_table_steal (self->textures, GUINT_TO_POINTER (texture_id));
+              gsk_ngl_texture_free (t);
+            }
 
           slices[slice_index].rect.x = x;
           slices[slice_index].rect.y = y;
diff --git a/gsk/ngl/gsknglprograms.defs b/gsk/ngl/gsknglprograms.defs
index 1454e56770..974f4512e5 100644
--- a/gsk/ngl/gsknglprograms.defs
+++ b/gsk/ngl/gsknglprograms.defs
@@ -90,3 +90,11 @@ GSK_NGL_DEFINE_PROGRAM (postprocessing,
 GSK_NGL_DEFINE_PROGRAM (linearize,
                         "/org/gtk/libgsk/ngl/linearize.glsl",
                         GSK_NGL_NO_UNIFORMS)
+
+GSK_NGL_DEFINE_PROGRAM (premultiply,
+                        "/org/gtk/libgsk/ngl/premultiply.glsl",
+                        GSK_NGL_NO_UNIFORMS)
+
+GSK_NGL_DEFINE_PROGRAM (linearize_premultiply,
+                        "/org/gtk/libgsk/ngl/linearize_premultiply.glsl",
+                        GSK_NGL_NO_UNIFORMS)
diff --git a/gsk/ngl/resources/linearize_premultiply.glsl b/gsk/ngl/resources/linearize_premultiply.glsl
new file mode 100644
index 0000000000..85a37c4ca3
--- /dev/null
+++ b/gsk/ngl/resources/linearize_premultiply.glsl
@@ -0,0 +1,21 @@
+
+// VERTEX_SHADER:
+// linearize_premultiply.glsl
+
+void main() {
+  gl_Position = u_projection * u_modelview * vec4(aPosition, 0.0, 1.0);
+
+  vUv = vec2(aUv.x, aUv.y);
+}
+
+// FRAGMENT_SHADER:
+// linearize_premultiply.glsl
+
+void main() {
+  vec4 diffuse = GskTexture(u_source, vUv);
+
+  diffuse = gsk_srgb_to_linear(diffuse);
+  diffuse = gsk_premultiply(diffuse);
+
+  gskSetOutputColor(diffuse);
+}
diff --git a/gsk/ngl/resources/premultiply.glsl b/gsk/ngl/resources/premultiply.glsl
new file mode 100644
index 0000000000..b83ce9e90e
--- /dev/null
+++ b/gsk/ngl/resources/premultiply.glsl
@@ -0,0 +1,20 @@
+
+// VERTEX_SHADER:
+// premultiply.glsl
+
+void main() {
+  gl_Position = u_projection * u_modelview * vec4(aPosition, 0.0, 1.0);
+
+  vUv = vec2(aUv.x, aUv.y);
+}
+
+// FRAGMENT_SHADER:
+// premultiply.glsl
+
+void main() {
+  vec4 diffuse = GskTexture(u_source, vUv);
+
+  diffuse = gsk_premultiply(diffuse);
+
+  gskSetOutputColor(diffuse);
+}


[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]