JBR-8555 Vulkan: Do not flush the surface on transform change

src/java.desktop/share/native/common/java2d/vulkan/VKRenderQueue.c

@@ -526,16 +526,10 @@ JNIEXPORT void JNICALL Java_sun_java2d_vulkan_VKRenderQueue_flushBuffer
                     .m11 = m11, .m02 = m02, .m12 = m12
                 };
 
-                if (VKRenderer_GetContext()->surface != NULL &&
-                    VK_IS_NEQ_TRANSFORM(&VKRenderer_GetContext()->transform, &transform))
-                {
-                  // TODO: Consider replacing this with setting a new transform
-                  //  in VKRenderingContext and increment transformModCount,
-                  //  then check for changed transform modification counter in
-                  //  VKRenderer_Validate, do VKRenderer_FlushDraw and push
-                  //  new constants into the shader
-                  VKRenderer_FlushSurface(VKRenderer_GetContext()->surface);
-                  VKRenderer_GetContext()->transform = transform;
+                VKRenderingContext* context = VKRenderer_GetContext();
+                if (VK_IS_NEQ_TRANSFORM(&context->transform, &transform)) {
+                    context->transform = transform;
+                    context->transformModCount++;
                 }
             }
             break;
@@ -543,11 +537,10 @@ JNIEXPORT void JNICALL Java_sun_java2d_vulkan_VKRenderQueue_flushBuffer
             {
                 J2dRlsTraceLn(J2D_TRACE_VERBOSE,
                     "VKRenderQueue_flushBuffer: RESET_TRANSFORM");
-                if (VKRenderer_GetContext()->surface != NULL &&
-                    VK_IS_NEQ_TRANSFORM(&VKRenderer_GetContext()->transform, &VK_ID_TRANSFORM))
-                {
-                  VKRenderer_FlushSurface(VKRenderer_GetContext()->surface);
-                  VKRenderer_GetContext()->transform = VK_ID_TRANSFORM;
+                VKRenderingContext* context = VKRenderer_GetContext();
+                if (VK_IS_NEQ_TRANSFORM(&context->transform, &VK_ID_TRANSFORM)) {
+                    context->transform = VK_ID_TRANSFORM;
+                    context->transformModCount++;
                 }
             }
             break;

src/java.desktop/share/native/common/java2d/vulkan/VKRenderer.c

@@ -152,6 +152,7 @@ struct VKRenderPass {
 
     VKPipelineDescriptor state;
     uint64_t             lastTimestamp; // When was this surface last used?
+    uint64_t             transformModCount; // Just a tag to detect when transform was changed.
     uint64_t             clipModCount; // Just a tag to detect when clip was changed.
     VkBool32             pendingFlush    : 1;
     VkBool32             pendingCommands : 1;
@@ -163,7 +164,8 @@ struct VKRenderPass {
 // which is only called from queue flusher thread, no need for synchronization.
 static VKRenderingContext context = {
         .surface = NULL,
-        .transform = {1.0, 0.0, 0.0,0.0, 1.0, 0.0},
+        .transform = VK_ID_TRANSFORM,
+        .transformModCount = 1,
         .color = {},
         .renderColor = {},
         .composite = ALPHA_COMPOSITE_SRC_OVER,
@@ -605,6 +607,7 @@ static void VKRenderer_ResetDrawing(VKSDOps* surface) {
     assert(surface != NULL && surface->renderPass != NULL);
     surface->renderPass->state.composite = NO_COMPOSITE;
     surface->renderPass->state.shader = NO_SHADER;
+    surface->renderPass->transformModCount = 0;
     surface->renderPass->firstVertex = 0;
     surface->renderPass->vertexCount = 0;
     surface->renderPass->vertexBufferWriting = (BufferWritingState) {NULL, 0, VK_FALSE};
@@ -689,6 +692,7 @@ static VkBool32 VKRenderer_InitRenderPass(VKSDOps* surface) {
                 .shader = NO_SHADER
             },
             .lastTimestamp = 0,
+            .transformModCount = 0,
             .clipModCount = 0,
             .pendingFlush = VK_FALSE,
             .pendingCommands = VK_FALSE,
@@ -817,24 +821,6 @@ static void VKRenderer_BeginRenderPass(VKSDOps* surface) {
     };
     device->vkCmdSetViewport(commandBuffer, 0, 1, &viewport);
 
-    // Calculate user to device transform
-    VKTransform transform = {
-        2.0f/viewport.width, 0.0f,                 -1.0f,
-        0.0f,                2.0f/viewport.height, -1.0f
-    };
-
-    // Combine it with user transform
-    VKUtil_ConcatenateTransform(&transform, &context.transform);
-
-    device->vkCmdPushConstants(
-            commandBuffer,
-            renderer->pipelineContext->colorPipelineLayout, // TODO what if our pipeline layout differs?
-            VK_SHADER_STAGE_VERTEX_BIT,
-            0,
-            sizeof(VKTransform),
-            &transform
-    );
-
     surface->renderPass->pendingCommands = VK_TRUE;
     J2dRlsTraceLn1(J2D_TRACE_VERBOSE, "VKRenderer_BeginRenderPass(%p)", surface);
 }
@@ -1112,6 +1098,30 @@ static BufferWritingState VKRenderer_AllocateMaskFillBytes(const VKRenderingCont
     return state;
 }
 
+static void VKRenderer_ValidateTransform() {
+    assert(context.surface != NULL);
+    VKSDOps* surface = context.surface;
+    VKRenderPass* renderPass = surface->renderPass;
+    if (renderPass->transformModCount != context.transformModCount) {
+        J2dTraceLn(J2D_TRACE_VERBOSE, "VKRenderer_ValidateTransform: updating transform");
+        VKRenderer_FlushDraw(surface);
+        renderPass->transformModCount = context.transformModCount;
+        // Calculate user to device transform.
+        VKTransform transform = {
+            2.0f / (float) surface->image->extent.width, 0.0f, -1.0f,
+            0.0f, 2.0f / (float) surface->image->extent.height, -1.0f
+        };
+        // Combine it with user transform.
+        VKUtil_ConcatenateTransform(&transform, &context.transform);
+        // Push the transform into shader.
+        surface->device->vkCmdPushConstants(
+                renderPass->commandBuffer,
+                surface->device->renderer->pipelineContext->colorPipelineLayout, // TODO what if our pipeline layout differs?
+                VK_SHADER_STAGE_VERTEX_BIT, 0, sizeof(VKTransform), &transform
+        );
+    }
+}
+
 /**
  * Setup stencil attachment according to the context clip state.
  * If there is a clip shape, attachment is cleared with "fail" value and then
@@ -1124,6 +1134,7 @@ static void VKRenderer_SetupStencil(const VKRenderingContext* context) {
     VKRenderPass* renderPass = surface->renderPass;
     VkCommandBuffer cb = renderPass->commandBuffer;
     VKRenderer_FlushDraw(surface);
+    VKRenderer_ValidateTransform();
 
     // Clear stencil attachment.
     VkClearAttachment clearAttachment = {
@@ -1221,6 +1232,9 @@ VkBool32 VKRenderer_Validate(VKShader shader, VkPrimitiveTopology topology, Alph
         }
     }
 
+    // Validate current transform.
+    VKRenderer_ValidateTransform();
+
     // Validate current pipeline.
     if (renderPass->state.shader != shader ||
         renderPass->state.topology != topology ||

src/java.desktop/share/native/common/java2d/vulkan/VKRenderer.h

@@ -34,7 +34,9 @@
 
 struct VKRenderingContext {
     VKSDOps*        surface;
+
     VKTransform     transform;
+    uint64_t        transformModCount;
 
     // We keep this color separately from renderColor,
     // because we need consistent state when switching between XOR and alpha