JBR-6193 Impossible to resize snapped IDE when native header is turned off

Drop the maximized state right before the resize operation for
undecorated windows.

Also fixes setExtendedState() to work when changing snapped window's state
(MAXIMIZED_HORIZ or MAXIMIZED_VERT) to NORMAL.

src/java.desktop/unix/classes/sun/awt/X11/XDecoratedPeer.java

@@ -636,6 +636,27 @@ abstract class XDecoratedPeer extends XWindowPeer {
             updateSizeHints(dimensions);
             Rectangle client = dimensions.getClientRect();
             checkShellRect(client);
+
+            if (isTargetUndecorated() && this instanceof XFramePeer framePeer) {
+                if (isMaximized()) {
+                    // Under Xorg, if an undecorated (read: client-side decorated)
+                    // window has been maximized (either vertically or horizontally),
+                    // it cannot change its size programmatically until both maximized
+                    // states have been cleared. And since the window is undecorated, it also
+                    // cannot change its size with the user's help because
+                    // "undecorated" means "no borders" and that, perhaps incorrectly, implies
+                    // that there's nothing to grab on in order to resize interactively.
+                    // To exit this viscous circle, drop the maximized state. This does have
+                    // unpleasant side effects (such as an animation played by the WM), but
+                    // those seem to be a lesser evil than the total inability to resize.
+                    int state = framePeer.getState();
+                    if ((state & Frame.MAXIMIZED_BOTH) != Frame.MAXIMIZED_BOTH) {
+                        state &= ~Frame.MAXIMIZED_BOTH;
+                        framePeer.setExtendedState(state);
+                    }
+                }
+            }
+
             setShellBounds(client);
             if (content != null &&
                 !content.getSize().equals(newDimensions.getSize()))

src/java.desktop/unix/classes/sun/awt/X11/XNETProtocol.java

@@ -80,15 +80,11 @@ final class XNETProtocol extends XProtocol implements XStateProtocol, XLayerProt
     }
 
     private void requestState(XWindowPeer window, int state) {
-        /*
-         * We have to use toggle for maximization because of transitions
-         * from maximization in one direction only to maximization in the
-         * other direction only.
-         */
         int old_net_state = getState(window);
         int max_changed = (state ^ old_net_state) & (Frame.MAXIMIZED_BOTH);
 
         XClientMessageEvent req = new XClientMessageEvent();
+        int action = 0;
         try {
             switch(max_changed) {
               case 0:
@@ -96,14 +92,22 @@ final class XNETProtocol extends XProtocol implements XStateProtocol, XLayerProt
               case Frame.MAXIMIZED_HORIZ:
                   req.set_data(1, XA_NET_WM_STATE_MAXIMIZED_HORZ.getAtom());
                   req.set_data(2, 0);
+                  action = ((state & Frame.MAXIMIZED_HORIZ) == Frame.MAXIMIZED_HORIZ)
+                          ? _NET_WM_STATE_ADD
+                          : _NET_WM_STATE_REMOVE;
                   break;
               case Frame.MAXIMIZED_VERT:
                   req.set_data(1, XA_NET_WM_STATE_MAXIMIZED_VERT.getAtom());
                   req.set_data(2, 0);
+                  action = ((state & Frame.MAXIMIZED_VERT) == Frame.MAXIMIZED_VERT)
+                          ? _NET_WM_STATE_ADD
+                          : _NET_WM_STATE_REMOVE;
                   break;
               case Frame.MAXIMIZED_BOTH:
+                  // Somehow this doesn't work when changing HORZ->VERT, but works VERT->HORZ
                   req.set_data(1, XA_NET_WM_STATE_MAXIMIZED_HORZ.getAtom());
                   req.set_data(2, XA_NET_WM_STATE_MAXIMIZED_VERT.getAtom());
+                  action = _NET_WM_STATE_TOGGLE;
                   break;
               default:
                   return;
@@ -115,7 +119,7 @@ final class XNETProtocol extends XProtocol implements XStateProtocol, XLayerProt
             req.set_window(window.getWindow());
             req.set_message_type(XA_NET_WM_STATE.getAtom());
             req.set_format(32);
-            req.set_data(0, _NET_WM_STATE_TOGGLE);
+            req.set_data(0, action);
             XToolkit.awtLock();
             try {
                 XlibWrapper.XSendEvent(XToolkit.getDisplay(),

test/jdk/jb/javax/swing/Frame/ExtendedFrameState.java

@@ -0,0 +1,159 @@
+/*
+ * Copyright 2023 JetBrains s.r.o.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/* @test
+ * @summary Verifies transitions between extended frame states
+ * @requires os.family == "linux"
+ * @run main ExtendedFrameState
+ */
+
+import javax.swing.JFrame;
+import javax.swing.JLabel;
+import javax.swing.JPanel;
+import javax.swing.SwingUtilities;
+import javax.swing.WindowConstants;
+import java.awt.Frame;
+import java.awt.Robot;
+import java.awt.Toolkit;
+import java.lang.reflect.InvocationTargetException;
+
+public class ExtendedFrameState {
+    static final int DELAY_MS = 1000;
+    JFrame frame;
+    Robot robot = new Robot();
+
+    public static void main(String[] args) throws Exception {
+        var toolkit = Toolkit.getDefaultToolkit();
+        if (toolkit.isFrameStateSupported(Frame.MAXIMIZED_HORIZ)
+                && toolkit.isFrameStateSupported(Frame.MAXIMIZED_VERT)
+                && toolkit.isFrameStateSupported(Frame.MAXIMIZED_BOTH)) {
+            var test = new ExtendedFrameState();
+
+            try {
+                test.prepare();
+                test.execute();
+            } finally {
+                test.dispose();
+            }
+        }
+    }
+
+    ExtendedFrameState() throws Exception {
+    }
+
+    void prepare() {
+        runSwing(() -> {
+            frame = new JFrame("ExtendedFrameState");
+            frame.setUndecorated(true);
+            frame.setDefaultCloseOperation(WindowConstants.EXIT_ON_CLOSE);
+            JPanel panel = new JPanel();
+            panel.add(new JLabel("label text"));
+            frame.setContentPane(panel);
+            frame.pack();
+            frame.setVisible(true);
+        });
+    }
+
+    void execute() {
+        setState(Frame.NORMAL);
+        assertStateIs(Frame.NORMAL);
+
+        // NORMAL -> MAXIMIZED_VERT
+        setState(Frame.MAXIMIZED_VERT);
+        assertStateIs(Frame.MAXIMIZED_VERT);
+
+        // MAXIMIZED_VERT -> NORMAL
+        setState(Frame.NORMAL);
+        assertStateIs(Frame.NORMAL);
+
+        // NORMAL -> MAXIMIZED_HORIZ
+        setState(Frame.MAXIMIZED_HORIZ);
+        assertStateIs(Frame.MAXIMIZED_HORIZ);
+
+        // MAXIMIZED_HORIZ -> MAXIMIZED_VERT
+        // This transition is not supported by the WM for some obscure reason
+        //setState(Frame.MAXIMIZED_VERT);
+        //assertStateIs(Frame.MAXIMIZED_VERT);
+
+        // MAXIMIZED_VERT -> MAXIMIZED_HORIZ
+        setState(Frame.MAXIMIZED_HORIZ);
+        assertStateIs(Frame.MAXIMIZED_HORIZ);
+
+        // MAXIMIZED_HORIZ -> NORMAL
+        setState(Frame.NORMAL);
+        assertStateIs(Frame.NORMAL);
+
+        // NORMAL -> MAXIMIZED_BOTH
+        setState(Frame.MAXIMIZED_BOTH);
+        assertStateIs(Frame.MAXIMIZED_BOTH);
+
+        // MAXIMIZED_BOTH -> MAXIMIZED_HORIZ
+        setState(Frame.MAXIMIZED_HORIZ);
+        assertStateIs(Frame.MAXIMIZED_HORIZ);
+
+        // MAXIMIZED_HORIZ -> MAXIMIZED_BOTH
+        setState(Frame.MAXIMIZED_BOTH);
+        assertStateIs(Frame.MAXIMIZED_BOTH);
+
+        // MAXIMIZED_BOTH -> MAXIMIZED_VERT
+        setState(Frame.MAXIMIZED_VERT);
+        assertStateIs(Frame.MAXIMIZED_VERT);
+
+        // MAXIMIZED_VERT -> NORMAL
+        setState(Frame.NORMAL);
+        assertStateIs(Frame.NORMAL);
+    }
+
+    void setState(int state) {
+        runSwing(() -> {
+            frame.setExtendedState(state);
+        });
+        robot.waitForIdle();
+        robot.delay(DELAY_MS);
+    }
+
+    void assertStateIs(int idealState) {
+        runSwing(() -> {
+            int realState = frame.getExtendedState();
+            if (idealState != realState) {
+                throw new RuntimeException(
+                        String.format("Expected frame state %d, got %d", idealState, realState));
+            }
+        });
+    }
+
+    void dispose() {
+        runSwing(() -> {
+            frame.dispose();
+        });
+    }
+
+    private static void runSwing(Runnable r) {
+        try {
+            SwingUtilities.invokeAndWait(r);
+        } catch (InterruptedException e) {
+        } catch (InvocationTargetException e) {
+            throw new RuntimeException(e);
+        }
+    }
+}

test/jdk/jb/javax/swing/Frame/ResizeUndecorated.java

@@ -0,0 +1,156 @@
+/*
+ * Copyright 2023 JetBrains s.r.o.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/* @test
+ * @summary Verifies transitions between extended frame states
+ * @requires os.family == "linux"
+ * @run main ResizeUndecorated
+ */
+
+import javax.swing.JFrame;
+import javax.swing.JLabel;
+import javax.swing.JPanel;
+import javax.swing.SwingUtilities;
+import javax.swing.WindowConstants;
+import java.awt.Frame;
+import java.awt.Robot;
+import java.awt.Toolkit;
+import java.lang.reflect.InvocationTargetException;
+
+public class ResizeUndecorated {
+    static final int DELAY_MS = 1000;
+    JFrame frame;
+    Robot robot = new Robot();
+
+    public static void main(String[] args) throws Exception {
+        var toolkit = Toolkit.getDefaultToolkit();
+        if (toolkit.isFrameStateSupported(Frame.MAXIMIZED_HORIZ)
+                && toolkit.isFrameStateSupported(Frame.MAXIMIZED_VERT)
+                && toolkit.isFrameStateSupported(Frame.MAXIMIZED_BOTH)) {
+            var test = new ResizeUndecorated();
+
+            try {
+                test.prepare();
+                test.execute();
+            } finally {
+                test.dispose();
+            }
+        }
+    }
+
+    ResizeUndecorated() throws Exception {
+    }
+
+    void prepare() {
+        runSwing(() -> {
+            frame = new JFrame("Resize Undecorated Test");
+            frame.setUndecorated(true);
+            frame.setDefaultCloseOperation(WindowConstants.EXIT_ON_CLOSE);
+            JPanel panel = new JPanel();
+            panel.add(new JLabel("label text"));
+            frame.setContentPane(panel);
+            frame.pack();
+            frame.setVisible(true);
+        });
+    }
+
+    void execute() {
+        setState(Frame.NORMAL);
+        assertStateIs(Frame.NORMAL);
+
+        setState(Frame.MAXIMIZED_VERT);
+        assertStateIs(Frame.MAXIMIZED_VERT);
+        resizeFrame(10, 0);
+
+        setState(Frame.NORMAL);
+        assertStateIs(Frame.NORMAL);
+        runSwing(() -> {
+            frame.setBounds(100, 100, 200, 300);
+        });
+
+        robot.waitForIdle();
+        robot.delay(DELAY_MS);
+
+        setState(Frame.MAXIMIZED_HORIZ);
+        assertStateIs(Frame.MAXIMIZED_HORIZ);
+        resizeFrame(0, 20);
+    }
+
+    void resizeFrame(int dx, int dy) {
+        var bounds = frame.getBounds();
+        runSwing(() -> {
+            frame.setBounds(bounds.x, bounds.y, bounds.width + dx, bounds.height + dy);
+        });
+        robot.waitForIdle();
+        robot.delay(DELAY_MS);
+        runSwing(() -> {
+            var newBounds = frame.getBounds();
+            if (bounds.width + dx != newBounds.width) {
+                throw new RuntimeException(
+                        String.format("Frame width after resize is %d, should be %d",
+                                newBounds.width,
+                                bounds.width + dx));
+            }
+
+            if (bounds.height + dy != newBounds.height) {
+                throw new RuntimeException(
+                        String.format("Frame height after resize is %d, should be %d",
+                                newBounds.height,
+                                bounds.height + dy));
+            }
+        });
+    }
+
+    void setState(int state) {
+        runSwing(() -> {
+            frame.setExtendedState(state);
+        });
+        robot.waitForIdle();
+        robot.delay(DELAY_MS);
+    }
+
+    void assertStateIs(int idealState) {
+        runSwing(() -> {
+            int realState = frame.getExtendedState();
+            if (idealState != realState) {
+                throw new RuntimeException(
+                        String.format("Expected frame state %d, got %d", idealState, realState));
+            }
+        });
+    }
+
+    void dispose() {
+        runSwing(() -> {
+            frame.dispose();
+        });
+    }
+
+    private static void runSwing(Runnable r) {
+        try {
+            SwingUtilities.invokeAndWait(r);
+        } catch (InterruptedException e) {
+        } catch (InvocationTargetException e) {
+            throw new RuntimeException(e);
+        }
+    }
+}