update exclude list on results of 21_b243.1 test runs

- Implements the optional method [NSTextInputClient windowLevel] to tell the macOS IM subsystem correct level of the window;
- Adds a regression test ImWindowIsPlacedUnderPopup5984.java.

(cherry picked from commit 5a91aae9c2)

.github/actions/get-jtreg/action.yml

@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved.
 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 #
 # This code is free software; you can redistribute it and/or modify it
@@ -57,7 +57,7 @@ runs:
     - name: 'Build JTReg'
       run: |
         # Build JTReg and move files to the proper locations
-        bash make/build.sh --jdk "$JAVA_HOME_11_X64"
+        bash make/build.sh --jdk "$JAVA_HOME_17_X64"
         mkdir ../installed
         mv build/images/jtreg/* ../installed
       working-directory: jtreg/src

.github/actions/get-msys2/action.yml

@@ -30,7 +30,8 @@ runs:
   using: composite
   steps:
     - name: 'Install MSYS2'
-      uses: msys2/setup-msys2@v2
+      # use a specific release of msys2/setup-msys2 to prevent jtreg build failures on newer release
+      uses: msys2/setup-msys2@7efe20baefed56359985e327d329042cde2434ff
       with:
         install: 'autoconf tar unzip zip make'
         path-type: minimal

.github/workflows/main.yml

@@ -145,7 +145,7 @@ jobs:
       apt-architecture: 'i386'
       # Some multilib libraries do not have proper inter-dependencies, so we have to
       # install their dependencies manually.
-      apt-extra-packages: 'libfreetype6-dev:i386 libtiff-dev:i386 libcupsimage2-dev:i386 libc6-i386'
+      apt-extra-packages: 'libfreetype6-dev:i386 libtiff-dev:i386 libcupsimage2-dev:i386 libc6-i386 libgcc-s1:i386 libstdc++6:i386'
       extra-conf-options: '--with-target-bits=32'
       configure-arguments: ${{ github.event.inputs.configure-arguments }}
       make-arguments: ${{ github.event.inputs.make-arguments }}

doc/building.html

@@ -80,6 +80,7 @@ id="toc-getting-jdk-binaries">Getting JDK binaries</a></li>
 id="toc-external-library-requirements">External Library Requirements</a>
 <ul>
 <li><a href="#freetype" id="toc-freetype">FreeType</a></li>
+<li><a href="#fontconfig" id="toc-fontconfig">Fontconfig</a></li>
 <li><a href="#cups" id="toc-cups">CUPS</a></li>
 <li><a href="#x11" id="toc-x11">X11</a></li>
 <li><a href="#alsa" id="toc-alsa">ALSA</a></li>
@@ -323,6 +324,7 @@ GB of free disk space is required.</p>
 <p>Even for 32-bit builds, it is recommended to use a 64-bit build
 machine, and instead create a 32-bit target using
 <code>--with-target-bits=32</code>.</p>
+<p>Note: The Windows 32-bit x86 port is deprecated and may be removed in a future release.</p>
 <h3 id="building-on-aarch64">Building on aarch64</h3>
 <p>At a minimum, a machine with 8 cores is advisable, as well as 8 GB of
 RAM. (The more cores to use, the more memory you need.) At least 6 GB of
@@ -353,22 +355,22 @@ to date at the time of writing.</p>
 <table>
 <thead>
 <tr class="header">
-<th style="text-align: left;">Operating system</th>
-<th style="text-align: left;">Vendor/version used</th>
+<th>Operating system</th>
+<th>Vendor/version used</th>
 </tr>
 </thead>
 <tbody>
 <tr class="odd">
-<td style="text-align: left;">Linux</td>
-<td style="text-align: left;">Oracle Enterprise Linux 6.4 / 7.6</td>
+<td>Linux</td>
+<td>Oracle Enterprise Linux 6.4 / 7.6</td>
 </tr>
 <tr class="even">
-<td style="text-align: left;">macOS</td>
-<td style="text-align: left;">Mac OS X 10.13 (High Sierra)</td>
+<td>macOS</td>
+<td>Mac OS X 10.13 (High Sierra)</td>
 </tr>
 <tr class="odd">
-<td style="text-align: left;">Windows</td>
-<td style="text-align: left;">Windows Server 2012 R2</td>
+<td>Windows</td>
+<td>Windows Server 2012 R2</td>
 </tr>
 </tbody>
 </table>
@@ -399,6 +401,7 @@ to the build system, e.g. in arguments to <code>configure</code>. So,
 use <code>--with-msvcr-dll=/cygdrive/c/msvcr100.dll</code> rather than
 <code>--with-msvcr-dll=c:\msvcr100.dll</code>. For details on this
 conversion, see the section on <a href="#fixpath">Fixpath</a>.</p>
+<p>Note: The Windows 32-bit x86 port is deprecated and may be removed in a future release.</p>
 <h4 id="cygwin">Cygwin</h4>
 <p>A functioning <a href="http://www.cygwin.com/">Cygwin</a> environment
 is required for building the JDK on Windows. If you have a 64-bit OS, we
@@ -677,6 +680,19 @@ copy.</p>
 <p>Use <code>--with-freetype-include=&lt;path&gt;</code> and
 <code>--with-freetype-lib=&lt;path&gt;</code> if <code>configure</code>
 does not automatically locate the platform FreeType files.</p>
+<h3 id="fontconfig">Fontconfig</h3>
+<p>Fontconfig from <a href="http://fontconfig.org">freedesktop.org
+Fontconfig</a> is required on all platforms except Windows and
+macOS.</p>
+<ul>
+<li>To install on an apt-based Linux, try running
+<code>sudo apt-get install     libfontconfig-dev</code>.</li>
+<li>To install on an rpm-based Linux, try running
+<code>sudo yum install     fontconfig-devel</code>.</li>
+</ul>
+<p>Use <code>--with-fontconfig-include=&lt;path&gt;</code> and
+<code>--with-fontconfig=&lt;path&gt;</code> if <code>configure</code>
+does not automatically locate the platform Fontconfig files.</p>
 <h3 id="cups">CUPS</h3>
 <p>CUPS, <a href="http://www.cups.org">Common UNIX Printing System</a>
 header files are required on all platforms, except Windows. Often these
@@ -1195,27 +1211,27 @@ following targets are known to work:</p>
 <table>
 <thead>
 <tr class="header">
-<th style="text-align: left;">Supported devkit targets</th>
+<th>Supported devkit targets</th>
 </tr>
 </thead>
 <tbody>
 <tr class="odd">
-<td style="text-align: left;">x86_64-linux-gnu</td>
+<td>x86_64-linux-gnu</td>
 </tr>
 <tr class="even">
-<td style="text-align: left;">aarch64-linux-gnu</td>
+<td>aarch64-linux-gnu</td>
 </tr>
 <tr class="odd">
-<td style="text-align: left;">arm-linux-gnueabihf</td>
+<td>arm-linux-gnueabihf</td>
 </tr>
 <tr class="even">
-<td style="text-align: left;">ppc64-linux-gnu</td>
+<td>ppc64-linux-gnu</td>
 </tr>
 <tr class="odd">
-<td style="text-align: left;">ppc64le-linux-gnu</td>
+<td>ppc64le-linux-gnu</td>
 </tr>
 <tr class="even">
-<td style="text-align: left;">s390x-linux-gnu</td>
+<td>s390x-linux-gnu</td>
 </tr>
 </tbody>
 </table>
@@ -1417,112 +1433,119 @@ example <code>${sysroot}/usr/lib/${target}/</code></p></li>
 <p>Architectures that are known to successfully cross-compile like this
 are:</p>
 <table>
+<colgroup>
+<col style="width: 13%" />
+<col style="width: 13%" />
+<col style="width: 15%" />
+<col style="width: 27%" />
+<col style="width: 29%" />
+</colgroup>
 <thead>
 <tr class="header">
-<th style="text-align: left;">Target</th>
-<th style="text-align: left;">Debian tree</th>
-<th style="text-align: left;">Debian arch</th>
-<th style="text-align: left;"><code>--openjdk-target=...</code></th>
+<th>Target</th>
+<th>Debian tree</th>
+<th>Debian arch</th>
+<th><code>--openjdk-target=...</code></th>
 <th><code>--with-jvm-variants=...</code></th>
 </tr>
 </thead>
 <tbody>
 <tr class="odd">
-<td style="text-align: left;">x86</td>
-<td style="text-align: left;">buster</td>
-<td style="text-align: left;">i386</td>
-<td style="text-align: left;">i386-linux-gnu</td>
+<td>x86</td>
+<td>buster</td>
+<td>i386</td>
+<td>i386-linux-gnu</td>
 <td>(all)</td>
 </tr>
 <tr class="even">
-<td style="text-align: left;">arm</td>
-<td style="text-align: left;">buster</td>
-<td style="text-align: left;">armhf</td>
-<td style="text-align: left;">arm-linux-gnueabihf</td>
+<td>arm</td>
+<td>buster</td>
+<td>armhf</td>
+<td>arm-linux-gnueabihf</td>
 <td>(all)</td>
 </tr>
 <tr class="odd">
-<td style="text-align: left;">aarch64</td>
-<td style="text-align: left;">buster</td>
-<td style="text-align: left;">arm64</td>
-<td style="text-align: left;">aarch64-linux-gnu</td>
+<td>aarch64</td>
+<td>buster</td>
+<td>arm64</td>
+<td>aarch64-linux-gnu</td>
 <td>(all)</td>
 </tr>
 <tr class="even">
-<td style="text-align: left;">ppc64le</td>
-<td style="text-align: left;">buster</td>
-<td style="text-align: left;">ppc64el</td>
-<td style="text-align: left;">powerpc64le-linux-gnu</td>
+<td>ppc64le</td>
+<td>buster</td>
+<td>ppc64el</td>
+<td>powerpc64le-linux-gnu</td>
 <td>(all)</td>
 </tr>
 <tr class="odd">
-<td style="text-align: left;">s390x</td>
-<td style="text-align: left;">buster</td>
-<td style="text-align: left;">s390x</td>
-<td style="text-align: left;">s390x-linux-gnu</td>
+<td>s390x</td>
+<td>buster</td>
+<td>s390x</td>
+<td>s390x-linux-gnu</td>
 <td>(all)</td>
 </tr>
 <tr class="even">
-<td style="text-align: left;">mipsle</td>
-<td style="text-align: left;">buster</td>
-<td style="text-align: left;">mipsel</td>
-<td style="text-align: left;">mipsel-linux-gnu</td>
+<td>mipsle</td>
+<td>buster</td>
+<td>mipsel</td>
+<td>mipsel-linux-gnu</td>
 <td>zero</td>
 </tr>
 <tr class="odd">
-<td style="text-align: left;">mips64le</td>
-<td style="text-align: left;">buster</td>
-<td style="text-align: left;">mips64el</td>
-<td style="text-align: left;">mips64el-linux-gnueabi64</td>
+<td>mips64le</td>
+<td>buster</td>
+<td>mips64el</td>
+<td>mips64el-linux-gnueabi64</td>
 <td>zero</td>
 </tr>
 <tr class="even">
-<td style="text-align: left;">armel</td>
-<td style="text-align: left;">buster</td>
-<td style="text-align: left;">arm</td>
-<td style="text-align: left;">arm-linux-gnueabi</td>
+<td>armel</td>
+<td>buster</td>
+<td>arm</td>
+<td>arm-linux-gnueabi</td>
 <td>zero</td>
 </tr>
 <tr class="odd">
-<td style="text-align: left;">ppc</td>
-<td style="text-align: left;">sid</td>
-<td style="text-align: left;">powerpc</td>
-<td style="text-align: left;">powerpc-linux-gnu</td>
+<td>ppc</td>
+<td>sid</td>
+<td>powerpc</td>
+<td>powerpc-linux-gnu</td>
 <td>zero</td>
 </tr>
 <tr class="even">
-<td style="text-align: left;">ppc64be</td>
-<td style="text-align: left;">sid</td>
-<td style="text-align: left;">ppc64</td>
-<td style="text-align: left;">powerpc64-linux-gnu</td>
+<td>ppc64be</td>
+<td>sid</td>
+<td>ppc64</td>
+<td>powerpc64-linux-gnu</td>
 <td>(all)</td>
 </tr>
 <tr class="odd">
-<td style="text-align: left;">m68k</td>
-<td style="text-align: left;">sid</td>
-<td style="text-align: left;">m68k</td>
-<td style="text-align: left;">m68k-linux-gnu</td>
+<td>m68k</td>
+<td>sid</td>
+<td>m68k</td>
+<td>m68k-linux-gnu</td>
 <td>zero</td>
 </tr>
 <tr class="even">
-<td style="text-align: left;">alpha</td>
-<td style="text-align: left;">sid</td>
-<td style="text-align: left;">alpha</td>
-<td style="text-align: left;">alpha-linux-gnu</td>
+<td>alpha</td>
+<td>sid</td>
+<td>alpha</td>
+<td>alpha-linux-gnu</td>
 <td>zero</td>
 </tr>
 <tr class="odd">
-<td style="text-align: left;">sh4</td>
-<td style="text-align: left;">sid</td>
-<td style="text-align: left;">sh4</td>
-<td style="text-align: left;">sh4-linux-gnu</td>
+<td>sh4</td>
+<td>sid</td>
+<td>sh4</td>
+<td>sh4-linux-gnu</td>
 <td>zero</td>
 </tr>
 <tr class="even">
-<td style="text-align: left;">riscv64</td>
-<td style="text-align: left;">sid</td>
-<td style="text-align: left;">riscv64</td>
-<td style="text-align: left;">riscv64-linux-gnu</td>
+<td>riscv64</td>
+<td>sid</td>
+<td>riscv64</td>
+<td>riscv64-linux-gnu</td>
 <td>(all)</td>
 </tr>
 </tbody>

doc/building.md

@@ -126,6 +126,8 @@ space is required.
 Even for 32-bit builds, it is recommended to use a 64-bit build machine, and
 instead create a 32-bit target using `--with-target-bits=32`.
 
+Note: The Windows 32-bit x86 port is deprecated and may be removed in a future release.
+
 ### Building on aarch64
 
 At a minimum, a machine with 8 cores is advisable, as well as 8 GB of RAM.
@@ -162,11 +164,11 @@ This table lists the OS versions used by Oracle when building the JDK. Such
 information is always subject to change, but this table is up to date at the
 time of writing.
 
- Operating system   Vendor/version used
- -----------------  -------------------------------------------------------
- Linux              Oracle Enterprise Linux 6.4 / 7.6
- macOS              Mac OS X 10.13 (High Sierra)
- Windows            Windows Server 2012 R2
+| Operating system  | Vendor/version used                |
+| ----------------- | ---------------------------------- |
+| Linux             | Oracle Enterprise Linux 6.4 / 7.6  |
+| macOS             | Mac OS X 10.13 (High Sierra)       |
+| Windows           | Windows Server 2012 R2             |
 
 The double version numbers for Linux are due to the hybrid model
 used at Oracle, where header files and external libraries from an older version
@@ -199,6 +201,8 @@ rule also applies to input to the build system, e.g. in arguments to
 `--with-msvcr-dll=c:\msvcr100.dll`. For details on this conversion, see the section
 on [Fixpath](#fixpath).
 
+Note: The Windows 32-bit x86 port is deprecated and may be removed in a future release.
+
 #### Cygwin
 
 A functioning [Cygwin](http://www.cygwin.com/) environment is required for
@@ -471,6 +475,19 @@ rather than bundling the JDK's own copy.
 Use `--with-freetype-include=<path>` and `--with-freetype-lib=<path>`
 if `configure` does not automatically locate the platform FreeType files.
 
+### Fontconfig
+
+Fontconfig from [freedesktop.org Fontconfig](http://fontconfig.org) is required
+on all platforms except Windows and macOS.
+
+  * To install on an apt-based Linux, try running `sudo apt-get install
+    libfontconfig-dev`.
+  * To install on an rpm-based Linux, try running `sudo yum install
+    fontconfig-devel`.
+
+Use `--with-fontconfig-include=<path>` and `--with-fontconfig=<path>`
+if `configure` does not automatically locate the platform Fontconfig files.
+
 ### CUPS
 
 CUPS, [Common UNIX Printing System](http://www.cups.org) header files are
@@ -970,14 +987,14 @@ https://sourceware.org/autobook/autobook/autobook_17.html). If no
 targets are given, a native toolchain for the current platform will be
 created. Currently, at least the following targets are known to work:
 
- Supported devkit targets
- -------------------------
- x86_64-linux-gnu
- aarch64-linux-gnu
- arm-linux-gnueabihf
- ppc64-linux-gnu
- ppc64le-linux-gnu
- s390x-linux-gnu
+| Supported devkit targets |
+| ------------------------ |
+| x86_64-linux-gnu         |
+| aarch64-linux-gnu        |
+| arm-linux-gnueabihf      |
+| ppc64-linux-gnu          |
+| ppc64le-linux-gnu        |
+| s390x-linux-gnu          |
 
 `BASE_OS` must be one of "OEL6" for Oracle Enterprise Linux 6 or
 "Fedora" (if not specified "OEL6" will be the default). If the base OS
@@ -1204,22 +1221,22 @@ it might require a little nudge with:
 
 Architectures that are known to successfully cross-compile like this are:
 
-  Target        Debian tree  Debian arch   `--openjdk-target=...`   `--with-jvm-variants=...`
-  ------------  ------------ ------------- ------------------------ --------------
-  x86           buster       i386          i386-linux-gnu           (all)
-  arm           buster       armhf         arm-linux-gnueabihf      (all)
-  aarch64       buster       arm64         aarch64-linux-gnu        (all)
-  ppc64le       buster       ppc64el       powerpc64le-linux-gnu    (all)
-  s390x         buster       s390x         s390x-linux-gnu          (all)
-  mipsle        buster       mipsel        mipsel-linux-gnu         zero
-  mips64le      buster       mips64el      mips64el-linux-gnueabi64 zero
-  armel         buster       arm           arm-linux-gnueabi        zero
-  ppc           sid          powerpc       powerpc-linux-gnu        zero
-  ppc64be       sid          ppc64         powerpc64-linux-gnu      (all)
-  m68k          sid          m68k          m68k-linux-gnu           zero
-  alpha         sid          alpha         alpha-linux-gnu          zero
-  sh4           sid          sh4           sh4-linux-gnu            zero
-  riscv64       sid          riscv64       riscv64-linux-gnu        (all)
+| Target       | Debian tree  | Debian arch   | `--openjdk-target=...`   | `--with-jvm-variants=...` |
+| ------------ | ------------ | ------------- | ------------------------ | ------------------------- |
+| x86          | buster       | i386          | i386-linux-gnu           | (all)                     |
+| arm          | buster       | armhf         | arm-linux-gnueabihf      | (all)                     |
+| aarch64      | buster       | arm64         | aarch64-linux-gnu        | (all)                     |
+| ppc64le      | buster       | ppc64el       | powerpc64le-linux-gnu    | (all)                     |
+| s390x        | buster       | s390x         | s390x-linux-gnu          | (all)                     |
+| mipsle       | buster       | mipsel        | mipsel-linux-gnu         | zero                      |
+| mips64le     | buster       | mips64el      | mips64el-linux-gnueabi64 | zero                      |
+| armel        | buster       | arm           | arm-linux-gnueabi        | zero                      |
+| ppc          | sid          | powerpc       | powerpc-linux-gnu        | zero                      |
+| ppc64be      | sid          | ppc64         | powerpc64-linux-gnu      | (all)                     |
+| m68k         | sid          | m68k          | m68k-linux-gnu           | zero                      |
+| alpha        | sid          | alpha         | alpha-linux-gnu          | zero                      |
+| sh4          | sid          | sh4           | sh4-linux-gnu            | zero                      |
+| riscv64      | sid          | riscv64       | riscv64-linux-gnu        | (all)                     |
 
 ### Building for ARM/aarch64
 

jb/generate-wakefield.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+if [[ -z "$1" ]]; then
+	SCANNER=wayland-scanner
+else
+	SCANNER="$1"
+fi
+
+set -ex
+
+"$SCANNER" client-header src/java.desktop/share/native/libwakefield/protocol/wakefield.xml src/java.desktop/unix/native/libawt_wlawt/wakefield-client-protocol.h
+"$SCANNER" private-code src/java.desktop/share/native/libwakefield/protocol/wakefield.xml src/java.desktop/unix/native/libawt_wlawt/wakefield-client-protocol.c

jb/project/docker/Dockerfile

@@ -1,13 +0,0 @@
-# jetbrains/runtime:jbr15env
-FROM centos:7
-RUN yum -y install centos-release-scl
-RUN yum -y install devtoolset-8
-RUN yum -y install zip bzip2 unzip tar wget make autoconf automake libtool gcc gcc-c++ libstdc++-devel alsa-devel cups-devel xorg-x11-devel libjpeg62-devel giflib-devel freetype-devel file which libXtst-devel libXt-devel libXrender-devel alsa-lib-devel fontconfig-devel libXrandr-devel libXi-devel git
-# Install Java 16
-RUN wget https://cdn.azul.com/zulu/bin/zulu17.28.13-ca-jdk17.0.0-linux_x64.tar.gz \
-  -O - | tar xz -C /
-RUN mv /zulu17.28.13-ca-jdk17.0.0-linux_x64 /jdk17.0.0
-ENV PATH /opt/rh/devtoolset-8/root/usr/bin:$PATH
-RUN mkdir .git
-RUN git config user.email "teamcity@jetbrains.com"
-RUN git config user.name "builduser"

jb/project/docker/Dockerfile.musl_aarch64

@@ -8,15 +8,36 @@
 FROM arm64v8/alpine:3.12
 
 # Install the necessary build tools
-RUN     apk --no-cache add --update bash grep tar zip bzip2 rsync fontconfig build-base \
-	    git libx11-dev libxext-dev libxrandr-dev libxrender-dev libxt-dev \
-	    libxtst-dev autoconf freetype-dev cups-dev alsa-lib-dev file \
-	    fontconfig fontconfig-dev linux-headers
+RUN apk --no-cache add --update \
+      alsa-lib-dev=1.2.2-r0 \
+      autoconf=2.69-r2 \
+      bash=5.0.17-r0 \
+      build-base=0.5-r2 \
+      bzip2=1.0.8-r1 \
+      cups-dev=2.3.3-r0 \
+      file=5.38-r0 \
+      fontconfig=2.13.1-r2 \
+      fontconfig-dev=2.13.1-r2 \
+      freetype-dev=2.10.4-r2 \
+      git=2.26.3-r1 \
+      grep=3.4-r0 \
+      libx11-dev=1.6.12-r1 \
+      libxext-dev=1.3.4-r0 \
+      libxrandr-dev=1.5.2-r0 \
+      libxrender-dev=0.9.10-r3 \
+      libxt-dev=1.2.0-r0 \
+      libxtst-dev=1.2.3-r3 \
+      linux-headers=5.4.5-r1 \
+      rsync=3.1.3-r3 \
+      tar=1.32-r2 \
+      wayland-dev=1.18.0-r4 \
+      zip=3.0-r8
+
 
 # Set up boot JDK for building
-COPY boot_jdk_musl_aarch64.tar.gz /jdk17/
-RUN cd /jdk17 && tar --strip-components=1 -xzf boot_jdk_musl_aarch64.tar.gz && rm /jdk17/boot_jdk_musl_aarch64.tar.gz
-ENV BOOT_JDK=/jdk17
+COPY boot_jdk_musl_aarch64.tar.gz /jdk20/
+RUN cd /jdk20 && tar --strip-components=1 -xzf boot_jdk_musl_aarch64.tar.gz && rm /jdk20/boot_jdk_musl_aarch64.tar.gz
+ENV BOOT_JDK=/jdk20
 
 RUN git config --global user.email "teamcity@jetbrains.com" && \
     git config --global user.name "builduser"

jb/project/docker/Dockerfile.musl_x64

@@ -8,15 +8,35 @@
 FROM alpine:3.14
 
 # Install the necessary build tools
-RUN     apk --no-cache add --update bash grep tar zip bzip2 rsync fontconfig build-base \
-	    git libx11-dev libxext-dev libxrandr-dev libxrender-dev libxt-dev \
-	    libxtst-dev autoconf freetype-dev cups-dev alsa-lib-dev file \
-	    fontconfig fontconfig-dev linux-headers
+RUN apk --no-cache add --update \
+      alsa-lib-dev=1.2.5-r2 \
+      autoconf=2.71-r0 \
+      bash=5.1.16-r0 \
+      build-base=0.5-r3 \
+      bzip2=1.0.8-r1 \
+      cups-dev=2.3.3-r3 \
+      file=5.40-r1 \
+      fontconfig=2.13.1-r4 \
+      fontconfig-dev=2.13.1-r4 \
+      freetype-dev=2.10.4-r3 \
+      git=2.32.7-r0 \
+      grep=3.7-r0 \
+      libx11-dev=1.7.3.1-r0 \
+      libxext-dev=1.3.4-r0 \
+      libxrandr-dev=1.5.2-r1 \
+      libxrender-dev=0.9.10-r3 \
+      libxt-dev=1.2.1-r0 \
+      libxtst-dev=1.2.3-r3 \
+      linux-headers=5.10.41-r0 \
+      rsync=3.2.5-r0 \
+      tar=1.34-r1 \
+      wayland-dev=1.19.0-r0 \
+      zip=3.0-r9
 
 # Set up boot JDK for building
-COPY boot_jdk_musl_amd64.tar.gz /jdk17/
-RUN cd /jdk17 && tar --strip-components=1 -xzf boot_jdk_musl_amd64.tar.gz && rm /jdk17/boot_jdk_musl_amd64.tar.gz
-ENV BOOT_JDK=/jdk17
+COPY boot_jdk_musl_amd64.tar.gz /jdk20/
+RUN cd /jdk20 && tar --strip-components=1 -xzf boot_jdk_musl_amd64.tar.gz && rm /jdk20/boot_jdk_musl_amd64.tar.gz
+ENV BOOT_JDK=/jdk20
 
 RUN git config --global user.email "teamcity@jetbrains.com" && \
     git config --global user.name "builduser"

jb/project/docker/Dockerfile.x86_64

@@ -0,0 +1,57 @@
+# jetbrains/runtime:jbr17env_x86_64
+FROM centos:7
+RUN yum -y install centos-release-scl; \
+    yum -y install devtoolset-10-10.1-0.el7; \
+    yum -y install \
+        alsa-lib-devel-1.1.8-1.el7 \
+        autoconf-2.69-11.el7 \
+        automake-1.13.4-3.el7 \
+        bzip2-1.0.6-13.el7 \
+        cups-devel-1.6.3-51.el7 \
+        file-5.11-37.el7 \
+        fontconfig-devel-2.13.0-4.3.el7 \
+        freetype-devel-2.8-14.el7_9.1 \
+        giflib-devel-4.1.6-9.el7 \
+        git-1.8.3.1-24.el7_9 \
+        libtool-2.4.2-22.el7_3 \
+        libXi-devel-1.7.9-1.el7 \
+        libXrandr-devel-1.5.1-2.el7 \
+        libXrender-devel-0.9.10-1.el7 \
+        libXt-devel-1.1.5-3.el7 \
+        libXtst-devel-1.2.3-1.el7 \
+        make-3.82-24.el7 \
+        tar-1.26-35.el7 \
+        unzip-6.0-24.el7_9 \
+        wayland-devel-1.15.0-1.el7 \
+        wget-1.14-18.el7_6.1 \
+        which-2.20-7.el7 \
+        zip-3.0-11.el7 \
+        python3-3.6.8-17.el7
+
+RUN mkdir .git && \
+    git config user.email "teamcity@jetbrains.com" && \
+    git config user.name "builduser"
+
+ENV LD_LIBRARY_PATH="/opt/rh/devtoolset-10/root/usr/lib64:/opt/rh/devtoolset-10/root/usr/lib:/opt/rh/devtoolset-10/root/usr/lib64/dyninst:/opt/rh/devtoolset-10/root/usr/lib/dyninst:/opt/rh/devtoolset-10/root/usr/lib64:/opt/rh/devtoolset-10/root/usr/lib"
+ENV PATH="/opt/rh/devtoolset-10/root/usr/bin::${PATH}"
+ENV PKG_CONFIG_PATH="/opt/rh/devtoolset-10/root/usr/lib64/pkgconfig"
+
+# Build GLSLC
+RUN curl -OL https://github.com/Kitware/CMake/releases/download/v3.27.5/cmake-3.27.5-linux-x86_64.tar.gz \
+  && echo 138c68addae825b16ed78d792dafef5e0960194833f48bd77e7e0429c6bc081c *cmake-3.27.5-linux-x86_64.tar.gz | sha256sum -c - \
+  && tar -xzf cmake-3.27.5-linux-x86_64.tar.gz \
+  && rm cmake-3.27.5-linux-x86_64.tar.gz \
+  && git clone https://github.com/google/shaderc --branch v2023.6 \
+  && cd shaderc \
+  && ./utils/git-sync-deps \
+  && mkdir build \
+  && cd build \
+  && /cmake-3.27.5-linux-x86_64/bin/cmake \
+    -DCMAKE_BUILD_TYPE=Release \
+    -DCMAKE_INSTALL_PREFIX=/usr/local \
+    -DSHADERC_SKIP_TESTS=ON \
+    -DSHADERC_SKIP_EXAMPLES=ON \
+    -DSHADERC_SKIP_COPYRIGHT_CHECK=ON \
+    .. \
+  && make install
+ENV PATH="/cmake-3.27.5-linux-x86_64/bin::${PATH}"

jb/project/docker/mkdocker_musl_aarch64.sh

@@ -4,10 +4,10 @@ set -euo pipefail
 set -x
 
 # This script creates a Docker image suitable for building musl AArch64 variant
-# of the JetBrains Runtime version 17.
+# of the JetBrains Runtime version 21.
 
-BOOT_JDK_REMOTE_FILE=zulu17.32.13-ca-jdk17.0.2-linux_musl_aarch64.tar.gz
-BOOT_JDK_SHA=6b920559abafbe9bdef386a20ecf3a2f318bc1f0d8359eb1f95aee26606bbc70
+BOOT_JDK_REMOTE_FILE=zulu20.32.11-ca-jdk20.0.2-linux_musl_aarch64.tar.gz
+BOOT_JDK_SHA=eec57cf744c2438f695221f041d4804de3033ad33b6dba769d3359813ba3f90d
 BOOT_JDK_LOCAL_FILE=boot_jdk_musl_aarch64.tar.gz
 
 if [ ! -f $BOOT_JDK_LOCAL_FILE ]; then
@@ -22,7 +22,7 @@ sha256sum -c - <<EOF
 $BOOT_JDK_SHA *$BOOT_JDK_LOCAL_FILE
 EOF
 
-docker build -t jbr17buildenv -f Dockerfile.musl_aarch64 .
+docker build -t jetbrains/runtime:jbr21env_musl_aarch64 -f Dockerfile.musl_aarch64 .
 
 # NB: the resulting container can (and should) be used without the network
 # connection (--network none) during build in order to reduce the chance

jb/project/docker/mkdocker_musl_x64.sh

@@ -4,10 +4,10 @@ set -euo pipefail
 set -x
 
 # This script creates a Docker image suitable for building musl-x64 variant
-# of the JetBrains Runtime version 17.
+# of the JetBrains Runtime version 21.
 
-BOOT_JDK_REMOTE_FILE=zulu17.32.13-ca-jdk17.0.2-linux_musl_x64.tar.gz
-BOOT_JDK_SHA=bcc5342011bd9f3643372aadbdfa68d47463ff0d8621668a0bdf2910614d95c6
+BOOT_JDK_REMOTE_FILE=zulu20.32.11-ca-jdk20.0.2-linux_musl_x64.tar.gz
+BOOT_JDK_SHA=fca5081dd6da847fcd06f5b755e58edae22d6784f21b81bf73da2b538f842c07
 BOOT_JDK_LOCAL_FILE=boot_jdk_musl_amd64.tar.gz
 
 if [ ! -f $BOOT_JDK_LOCAL_FILE ]; then
@@ -22,7 +22,7 @@ sha256sum -c - <<EOF
 $BOOT_JDK_SHA *$BOOT_JDK_LOCAL_FILE
 EOF
 
-docker build -t jbr17buildenv -f Dockerfile.musl_x64 .
+docker build -t jetbrains/runtime:jbr21env_musl_x64 -f Dockerfile.musl_x64 .
 
 # NB: the resulting container can (and should) be used without the network
 # connection (--network none) during build in order to reduce the chance

jb/project/tools/common/scripts/common.sh

@@ -94,8 +94,7 @@ esac
 
 WITH_ZIPPED_NATIVE_DEBUG_SYMBOLS="--with-native-debug-symbols=zipped"
 
-REPRODUCIBLE_BUILD_OPTS="--enable-reproducible-build
-  --with-source-date=$SOURCE_DATE_EPOCH
+REPRODUCIBLE_BUILD_OPTS="--with-source-date=$SOURCE_DATE_EPOCH
   --with-hotspot-build-time=$BUILD_TIME
   --with-copyright-year=$COPYRIGHT_YEAR
   --disable-absolute-paths-in-output

jb/project/tools/linux/scripts/mkimages_aarch64.sh

@@ -135,6 +135,7 @@ if [ "$bundle_type" == "jcef" ] || [ "$bundle_type" == "fd" ]; then
   cp $JCEF_PATH/jmods/* $JSDK_MODS_DIR # $JSDK/jmods is not changed
 
   jbr_name_postfix="_${bundle_type}"
+  cat $JCEF_PATH/jcef.version >> $JSDK/release
 else
   jbr_name_postfix=""
 fi

jb/project/tools/linux/scripts/mkimages_x64.sh

@@ -93,7 +93,7 @@ function create_image_bundle {
   [ -f "$JBR".tar.gz ] && rm "$JBR.tar.gz"
   touch -c -d "@$SOURCE_DATE_EPOCH" "$JBR".tar
   gzip "$JBR".tar || do_exit $?
-  rm -rf "${IMAGES_DIR:?}"/"$__root_dir"
+  #rm -rf "${IMAGES_DIR:?}"/"$__root_dir"
 }
 
 WITH_DEBUG_LEVEL="--with-debug-level=release"
@@ -134,7 +134,7 @@ if [ "$bundle_type" == "jcef" ] || [ "$bundle_type" == "fd" ]; then
   cp $JCEF_PATH/jmods/* $JSDK_MODS_DIR # $JSDK/jmods is not changed
 
   jbr_name_postfix="_${bundle_type}"
-  [ "$bundle_type" != "fd" ] && jbrsdk_name_postfix="_${bundle_type}"
+  cat $JCEF_PATH/jcef.version >> $JSDK/release
 else
   jbr_name_postfix=""
 fi

jb/project/tools/mac/scripts/codesign.sh

@@ -0,0 +1,47 @@
+#!/bin/bash
+SCRIPT_DIR="$(cd "$(dirname "$0")" >/dev/null && pwd)"
+
+source "$SCRIPT_DIR/jetsign-common.sh" || exit 1
+
+function isMacOsBinary() {
+  file "$1" | grep -q 'Mach-O'
+}
+
+function isSigned() {
+  codesign --verify "$1" >/dev/null 2>&1 && ! grep -q Signature=adhoc < <(codesign --display --verbose "$1" 2>&1)
+}
+
+# last argument is a path to be signed
+pathToBeSigned="$(pwd)/${*: -1}"
+jetSignArgs=("${@:1:$#-1}")
+if [[ ! -f "$pathToBeSigned" ]]; then
+  echo "$pathToBeSigned is missing or not a file"
+  exit 1
+elif isSigned "$pathToBeSigned" && ! isForced "${jetSignArgs[@]}" ; then
+  echo "Already signed: $pathToBeSigned"
+elif [[ "$JETSIGN_CLIENT" == "null" ]]; then
+  echo "JetSign client is missing, cannot proceed with signing"
+  exit 1
+elif ! isMacOsBinary "$pathToBeSigned" && [[ "$pathToBeSigned" != *.sit ]] && [[ "$pathToBeSigned" != *.tar.gz ]]; then
+  echo "$pathToBeSigned won't be signed, assumed not to be a macOS executable"
+else
+  if isMacOsBinary "$pathToBeSigned" && ! isSigned "$pathToBeSigned" ; then
+    echo "Unsigned macOS binary: $pathToBeSigned"
+  fi
+  workDir=$(dirname "$pathToBeSigned")
+  pathSigned="$workDir/signed/${pathToBeSigned##*/}"
+  jetSignExtensions=$(jetSignExtensions "${jetSignArgs[@]}")
+  contentType=$(jetSignContentType "$pathToBeSigned")
+  (
+    cd "$workDir" || exit 1
+    "$JETSIGN_CLIENT" -log-format text -denoted-content-type "$contentType" -extensions "$jetSignExtensions" "$pathToBeSigned"
+    # SRE-1223 (Codesign removes execute bits in executable files) workaround
+    chmod "$(stat -f %A "$pathToBeSigned")" "$pathSigned"
+    if isMacOsBinary "$pathSigned"; then
+      isSigned "$pathSigned"
+    fi
+    rm "$pathToBeSigned"
+    mv "$pathSigned" "$pathToBeSigned"
+    rm -rf "$workDir/signed"
+  )
+fi

jb/project/tools/mac/scripts/jetsign-common.sh

@@ -0,0 +1,63 @@
+#!/bin/bash
+set -euo pipefail
+
+function isForced() {
+  for arg in "$@"; do
+    if [[ "$arg" == --force ]]; then
+      return 0
+    fi
+  done
+  return 1
+}
+
+function jetSignExtensions() {
+  args=("$@")
+  ((lastElementIndex=${#args[@]}-1))
+  for index in "${!args[@]}"; do
+    arg=${args[$index]}
+    case "$arg" in
+    --sign | -s)
+      echo -n 'mac_codesign_identity='
+      continue
+      ;;
+    --entitlements)
+      echo -n 'mac_codesign_entitlements='
+      continue
+      ;;
+    --options=runtime)
+      echo -n 'mac_codesign_options=runtime'
+      ;;
+    --force)
+      echo -n 'mac_codesign_force=true'
+      ;;
+    --timestamp | --verbose | -v)
+      continue
+      ;;
+    *)
+      echo -n "$arg"
+      ;;
+    esac
+    if [[ $index != "$lastElementIndex" ]]; then
+      echo -n ","
+    fi
+  done
+}
+
+# See jetbrains.sign.util.FileUtil.contentType
+function jetSignContentType() {
+  case "${1##*/}" in
+  *.sit)
+    echo -n 'application/x-mac-app-zip'
+    ;;
+  *.tar.gz)
+    echo -n 'application/x-mac-app-targz'
+    ;;
+  *.pkg)
+    echo -n 'application/x-mac-pkg'
+    ;;
+  *)
+    echo -n 'application/x-mac-app-bin'
+    ;;
+  esac
+}
+

jb/project/tools/mac/scripts/mkimages.sh

@@ -81,6 +81,10 @@ function create_image_bundle {
     zip_native_debug_symbols $IMAGES_DIR/jdk-bundle/jdk-$JBSDK_VERSION.jdk "${JBR}_diz"
   fi
 
+  if [ "$bundle_type" == "jcef" ]; then
+    cat $JCEF_PATH/jcef.version >> "$JRE_CONTENTS/Home/release"
+  fi
+
   cp -R "$JSDK"/../MacOS "$JRE_CONTENTS"
   cp "$JSDK"/../Info.plist "$JRE_CONTENTS"
 

jb/project/tools/mac/scripts/notarize.sh

@@ -1,21 +1,17 @@
 #!/bin/bash
 
+#immediately exit script with an error if a command fails
 set -euo pipefail
-set -x
+[[ "${SCRIPT_VERBOSE:-}" == "1" ]] && set -x
 
-APP_DIRECTORY=$1
-APPL_USER=$2
-APPL_PASSWORD=$3
-APP_NAME=$4
-BUNDLE_ID=$5
-FAKE_ROOT="${6:-fake-root}"
+APP_PATH=$1
 
-if [[ -z "$APP_DIRECTORY" ]] || [[ -z "$APPL_USER" ]] || [[ -z "$APPL_PASSWORD" ]]; then
-  echo "Usage: $0 AppDirectory Username Password"
+if [[ -z "$APP_PATH" ]]; then
+  echo "Usage: $0 AppPath"
   exit 1
 fi
-if [[ ! -d "$APP_DIRECTORY" ]]; then
-  echo "AppDirectory '$APP_DIRECTORY' does not exist or not a directory"
+if [[ ! -f "$APP_PATH" ]]; then
+  echo "AppName '$APP_PATH' does not exist or not a file"
   exit 1
 fi
 
@@ -23,99 +19,22 @@ function log() {
   echo "$(date '+[%H:%M:%S]') $*"
 }
 
-function publish-log() {
-  id=$1
-  file=$2
-  curl -T "$file" "$ARTIFACTORY_URL/$id" || true
-}
 
-function altool-upload() {
-  # Since altool uses same file for upload token we have to trick it into using different folders for token file location
-  # Also it copies zip into TMPDIR so we override it too, to simplify cleanup
-  OLD_HOME="$HOME"
-  export HOME="$FAKE_ROOT/home"
-  export TMPDIR="$FAKE_ROOT/tmp"
-  mkdir -p "$HOME"
-  mkdir -p "$TMPDIR"
-  export _JAVA_OPTIONS="-Duser.home=$HOME -Djava.io.tmpdir=$TMPDIR"
-  # Reduce amount of downloads, cache transporter libraries
-  shared_itmstransporter="$OLD_HOME/shared-itmstransporter"
-  if [[ -f "$shared_itmstransporter" ]]; then
-    cp -r "$shared_itmstransporter" "$HOME/.itmstransporter"
-  fi
-  # For some reason altool prints everything to stderr, not stdout
-  set +e
-  xcrun altool --notarize-app \
-    --username "$APPL_USER" --password "$APPL_PASSWORD" \
-    --primary-bundle-id "$BUNDLE_ID" \
-    --asc-provider JetBrainssro --file "$1" 2>&1 | tee "altool.init.out"
-  unset TMPDIR
-  export HOME="$OLD_HOME"
-  set -e
-}
+# check required parameters
+: "${APPLE_ISSUER_ID}"
+: "${APPLE_KEY_ID}"
+: "${APPLE_PRIVATE_KEY}"
 
-#immediately exit script with an error if a command fails
-set -euo pipefail
+# shellcheck disable=SC2064
+trap "rm -f \"$PWD/tmp_key\"" INT EXIT RETURN
+echo -n "${APPLE_PRIVATE_KEY}" > tmp_key
 
-#file="$APP_NAME.zip"
+log "Notarizing $APP_PATH..."
+xcrun notarytool submit --key tmp_key --key-id "${APPLE_KEY_ID}" --issuer "${APPLE_ISSUER_ID}" "$APP_PATH" 2>&1 | tee "notarytool.submit.out"
+REQUEST_ID="$(grep -e " id: " "notarytool.submit.out" | grep -oE '([0-9a-f-]{36})'| head -n1)"
 
-#log "Zipping $file..."
-#rm -rf "$file"
-#ditto -c -k --sequesterRsrc --keepParent "$APP_DIRECTORY" "$file"
+xcrun notarytool wait "$REQUEST_ID" --key tmp_key --key-id "${APPLE_KEY_ID}" --issuer "${APPLE_ISSUER_ID}" --timeout 6h ||:
+xcrun notarytool log  "$REQUEST_ID" --key tmp_key --key-id "${APPLE_KEY_ID}" --issuer "${APPLE_ISSUER_ID}" developer_log.json ||:
+xcrun notarytool info "$REQUEST_ID" --key tmp_key --key-id "${APPLE_KEY_ID}" --issuer "${APPLE_ISSUER_ID}"
 
-log "Notarizing $APP_NAME..."
-rm -rf "altool.init.out" "altool.check.out"
-altool-upload "$APP_NAME"
-
-notarization_info="$(grep -e "RequestUUID" "altool.init.out" | grep -oE '([0-9a-f-]{36})')"
-
-if [ -z "$notarization_info" ]; then
-  log "Faile to read RequestUUID from altool.init.out"
-  exit 10
-fi
-
-PATH="$PATH:/usr/local/bin/"
-
-log "Notarization request sent, awaiting response"
-spent=0
-
-while true; do
-  # For some reason altool prints everything to stderr, not stdout
-  xcrun altool --username "$APPL_USER" --notarization-info "$notarization_info" --password "$APPL_PASSWORD" >"altool.check.out" 2>&1 || true
-  status="$(grep -oe 'Status: .*' "altool.check.out" | cut -c 9- || true)"
-  log "Current status: $status"
-  if [ "$status" = "invalid" ]; then
-    log "Notarization failed"
-    ec=1
-  elif [ "$status" = "success" ]; then
-    log "Notarization succeeded"
-    ec=0
-  else
-    if [ "$status" != "in progress" ]; then
-      log "Unknown notarization status, waiting more, altool output:"
-      cat "altool.check.out"
-    fi
-    if [[ $spent -gt 60 ]]; then
-      log "Waiting time out (apx 60 minutes)"
-      ec=2
-      break
-    fi
-    sleep 60
-    ((spent += 1))
-    continue
-  fi
-  developer_log="developer_log.json"
-  log "Fetching $developer_log"
-  # TODO: Replace cut with trim or something better
-  url="$(grep -oe 'LogFileURL: .*' "altool.check.out" | cut -c 13-)"
-  wget "$url" -O "$developer_log" && cat "$developer_log" || true
-  if [ $ec != 0 ]; then
-    log "Publishing $developer_log"
-    publish-log "$notarization_info" "$developer_log"
-  fi
-  break
-done
-cat "altool.check.out"
-
-rm -rf "altool.init.out" "altool.check.out"
-exit $ec
+log "Notarizing finished"

jb/project/tools/mac/scripts/productsign.sh

@@ -0,0 +1,41 @@
+#!/bin/bash
+SCRIPT_DIR="$(cd "$(dirname "$0")" >/dev/null && pwd)"
+
+source "$SCRIPT_DIR/jetsign-common.sh" || exit 1
+
+function isSigned() {
+  pkgutil --check-signature "$1" >/dev/null 2>&1 && grep -q "signed by a developer certificate" < <(pkgutil --check-signature "$1" 2>&1)
+}
+
+# second last argument is a path to be signed
+pathToBeSigned="$(pwd)/${*:(-2):1}"
+# last argument is a path to signed file
+pathOut="$(pwd)/${*:(-1)}"
+jetSignArgs=("${@:1:$#-2}")
+if [[ ! -f "$pathToBeSigned" ]]; then
+  echo "$pathToBeSigned is missing or not a file"
+  exit 1
+elif isSigned "$pathToBeSigned" && ! isForced "${jetSignArgs[@]}" ; then
+  echo "Already signed: $pathToBeSigned"
+elif [[ "$JETSIGN_CLIENT" == "null" ]]; then
+  echo "JetSign client is missing, cannot proceed with signing"
+  exit 1
+elif [[ "$pathToBeSigned" != *.pkg ]]; then
+  echo "$pathToBeSigned won't be signed, assumed not to be a macOS package"
+else
+  if ! isSigned "$pathToBeSigned" ; then
+    echo "Unsigned macOS package: $pathToBeSigned"
+  fi
+  workDir=$(dirname "$pathToBeSigned")
+  pathSigned="$workDir/signed/${pathToBeSigned##*/}"
+  jetSignExtensions=$(jetSignExtensions "${jetSignArgs[@]}")
+  contentType=$(jetSignContentType "$pathToBeSigned")
+  (
+    cd "$workDir" || exit 1
+    "$JETSIGN_CLIENT" -log-format text -denoted-content-type "$contentType" -extensions "$jetSignExtensions" "$pathToBeSigned"
+    isSigned "$pathSigned"
+    rm -f "$pathOut"
+    mv "$pathSigned" "$pathOut"
+    rm -rf "$workDir/signed"
+  )
+fi

jb/project/tools/mac/scripts/sign.sh

@@ -1,18 +1,33 @@
 #!/bin/bash
 
+#immediately exit script with an error if a command fails
 set -euo pipefail
-set -x
+[[ "${SCRIPT_VERBOSE:-}" == "1" ]] && set -x
+
+if [[ $# -lt 5  ]]; then
+  echo "Usage: $0 AppDirectory AppName BundleId CertificateID InstallerCertificateID"
+  exit 1
+fi
 
 APPLICATION_PATH=$1
-APP_NAME=$2
+PKG_NAME=$2
 BUNDLE_ID=$3
 JB_DEVELOPER_CERT=$4
 JB_INSTALLER_CERT=$5
 
-if [[ -z "$APPLICATION_PATH" ]] || [[ -z "$JB_DEVELOPER_CERT" ]]; then
-  echo "Usage: $0 AppDirectory CertificateID"
-  exit 1
+SCRIPT_DIR="$(cd "$(dirname "$0")" >/dev/null && pwd)"
+
+# Use JetBrains sign utility if it's available
+if [[ "${JETSIGN_CLIENT:=}" == "null" ]] || [[ "$JETSIGN_CLIENT" == "" ]]; then
+  JB_SIGN=false
+  SIGN_UTILITY="codesign"
+  PRODUCTSIGN_UTILITY="productsign"
+else
+  JB_SIGN=true
+  SIGN_UTILITY="$SCRIPT_DIR/codesign.sh"
+  PRODUCTSIGN_UTILITY="$SCRIPT_DIR/productsign.sh"
 fi
+
 if [[ ! -d "$APPLICATION_PATH" ]]; then
   echo "AppDirectory '$APPLICATION_PATH' does not exist or not a directory"
   exit 1
@@ -22,9 +37,6 @@ function log() {
   echo "$(date '+[%H:%M:%S]') $*"
 }
 
-#immediately exit script with an error if a command fails
-set -euo pipefail
-
 # Cleanup files left from previous sign attempt (if any)
 find "$APPLICATION_PATH" -name '*.cstemp' -exec rm '{}' \;
 
@@ -37,13 +49,13 @@ for f in \
   if [ -d "$APPLICATION_PATH/$f" ]; then
     find "$APPLICATION_PATH/$f" \
       -type f \( -name "*.jnilib" -o -name "*.dylib" -o -name "*.so" -o -name "*.tbd" -o -name "*.node" -o -perm +111 \) \
-      -exec codesign --timestamp \
+      -exec "$SIGN_UTILITY" --timestamp \
       -v -s "$JB_DEVELOPER_CERT" --options=runtime --force \
-      --entitlements entitlements.xml {} \;
+      --entitlements "$SCRIPT_DIR/entitlements.xml" {} \;
   fi
 done
 
-log "Signing libraries in jars in $PWD"
+log "Signing libraries in jars in $APPLICATION_PATH"
 
 # todo: add set -euo pipefail; into the inner sh -c
 # `-e` prevents `grep -q && printf` loginc
@@ -61,10 +73,10 @@ find "$APPLICATION_PATH" -name '*.jar' \
 
     find jarfolder \
       -type f \( -name "*.jnilib" -o -name "*.dylib" -o -name "*.so" -o -name "*.tbd" -o -name "jattach" \) \
-      -exec codesign --timestamp \
+      -exec "$SIGN_UTILITY" --timestamp \
       --force \
       -v -s "$JB_DEVELOPER_CERT" --options=runtime \
-      --entitlements entitlements.xml {} \;
+      --entitlements "$SCRIPT_DIR/entitlements.xml" {} \;
 
     (cd jarfolder; zip -q -r -o -0 ../jar.jar .)
     mv jar.jar "$file"
@@ -73,44 +85,77 @@ find "$APPLICATION_PATH" -name '*.jar' \
 rm -rf jarfolder jar.jar
 
 log "Signing other files..."
+# shellcheck disable=SC2043
 for f in \
   "Contents/Home/bin"; do
   if [ -d "$APPLICATION_PATH/$f" ]; then
     find "$APPLICATION_PATH/$f" \
       -type f \( -name "*.jnilib" -o -name "*.dylib" -o -name "*.so" -o -name "*.tbd" -o -perm +111 \) \
-      -exec codesign --timestamp \
+      -exec "$SIGN_UTILITY" --timestamp \
       -v -s "$JB_DEVELOPER_CERT" --options=runtime --force \
-      --entitlements entitlements.xml {} \;
+      --entitlements "$SCRIPT_DIR/entitlements.xml" {} \;
   fi
 done
 
-#log "Signing executable..."
-#codesign --timestamp \
-#    -v -s "$JB_DEVELOPER_CERT" --options=runtime \
-#    --force \
-#    --entitlements entitlements.xml "$APPLICATION_PATH/Contents/MacOS/idea"
+log "Signing whole frameworks..."
+# shellcheck disable=SC2043
+if [ "$JB_SIGN" = true ]; then for f in \
+  "Contents/Home/Frameworks" "Contents/Frameworks"; do
+  if [ -d "$APPLICATION_PATH/$f" ]; then
+    find "$APPLICATION_PATH/$f" \( -name '*.framework' -o -name '*.app' \) -maxdepth 1 | while read -r line
+      do
+        log "Signing '$line':"
+        tar -pczf tmp-to-sign.tar.gz -C "$(dirname "$line")" "$(basename "$line")"
+        "$SIGN_UTILITY" --timestamp \
+            -v -s "$JB_DEVELOPER_CERT" --options=runtime \
+            --force \
+            --entitlements "$SCRIPT_DIR/entitlements.xml" tmp-to-sign.tar.gz
+        rm -rf "$line"
+        tar -xzf tmp-to-sign.tar.gz --directory "$(dirname "$line")"
+        rm -f tmp-to-sign.tar.gz
+      done
+  fi
+done; fi
+
+log "Checking framework signatures..."
+for f in \
+  "Contents/Home/Frameworks" "Contents/Frameworks"; do
+  if [ -d "$APPLICATION_PATH/$f" ]; then
+    find "$APPLICATION_PATH/$f" -name '*.framework'  -maxdepth 1 | while read -r line
+      do
+        log "Checking '$line':"
+        codesign --verify --deep --strict --verbose=4 "$line"
+      done
+  fi
+done
 
 log "Signing whole app..."
-codesign --timestamp \
-  -v -s "$JB_DEVELOPER_CERT" --options=runtime \
-  --force \
-  --entitlements entitlements.xml "$APPLICATION_PATH"
+if [ "$JB_SIGN" = true ]; then
+  tar -pczf tmp-to-sign.tar.gz --exclude='man' -C "$(dirname "$APPLICATION_PATH")" "$(basename "$APPLICATION_PATH")"
+  "$SIGN_UTILITY" --timestamp \
+    -v -s "$JB_DEVELOPER_CERT" --options=runtime \
+    --force \
+    --entitlements "$SCRIPT_DIR/entitlements.xml" tmp-to-sign.tar.gz
+  rm -rf "$APPLICATION_PATH"
+  tar -xzf tmp-to-sign.tar.gz --directory "$(dirname "$APPLICATION_PATH")"
+  rm -f tmp-to-sign.tar.gz
+else
+  "$SIGN_UTILITY" --timestamp \
+    -v -s "$JB_DEVELOPER_CERT" --options=runtime \
+    --force \
+    --entitlements "$SCRIPT_DIR/entitlements.xml" "$APPLICATION_PATH"
+fi
 
-BUILD_NAME=$(echo $APPLICATION_PATH | awk -F"/" '{ print $2 }')
+BUILD_NAME="$(basename "$APPLICATION_PATH")"
 
-log "Creating $APP_NAME.pkg..."
-rm -rf "$APP_NAME.pkg"
+log "Creating $PKG_NAME..."
+rm -rf "$PKG_NAME"
 
 mkdir -p unsigned
 pkgbuild --identifier $BUNDLE_ID --root $APPLICATION_PATH \
-    --install-location /Library/Java/JavaVirtualMachines/${BUILD_NAME} unsigned/${APP_NAME}.pkg
-productsign --timestamp --sign "$JB_INSTALLER_CERT" unsigned/${APP_NAME}.pkg ${APP_NAME}.pkg
-
-#log "Signing whole app..."
-#codesign --timestamp \
-#  -v -s "$JB_DEVELOPER_CERT" --options=runtime \
-#  --force \
-#  --entitlements entitlements.xml $APP_NAME.pkg
+    --install-location /Library/Java/JavaVirtualMachines/${BUILD_NAME} unsigned/${PKG_NAME}
+log "Signing $PKG_NAME..."
+"$PRODUCTSIGN_UTILITY" --timestamp --sign "$JB_INSTALLER_CERT" unsigned/${PKG_NAME} ${PKG_NAME}
 
 log "Verifying java is not broken"
 find "$APPLICATION_PATH" \

jb/project/tools/mac/scripts/signapp.sh

@@ -2,7 +2,7 @@
 
 #immediately exit script with an error if a command fails
 set -euo pipefail
-set -x
+[[ "${SCRIPT_VERBOSE:-}" == "1" ]] && set -x
 
 export COPY_EXTENDED_ATTRIBUTES_DISABLE=true
 export COPYFILE_DISABLE=true
@@ -17,7 +17,7 @@ JB_INSTALLER_CERT=$6
 NOTARIZE=$7
 BUNDLE_ID=$8
 
-cd "$(dirname "$0")"
+SCRIPT_DIR="$(cd "$(dirname "$0")" >/dev/null && pwd)"
 
 function log() {
   echo "$(date '+[%H:%M:%S]') $*"
@@ -44,7 +44,8 @@ fi
 
 log "$INPUT_FILE extracted and removed"
 
-APP_NAME=$(echo ${INPUT_FILE} | awk -F".tar" '{ print $1 }')
+APP_NAME=$(basename "$INPUT_FILE" | awk -F".tar" '{ print $1 }')
+PKG_NAME="$APP_NAME.pkg"
 APPLICATION_PATH=$EXPLODED/$(ls $EXPLODED)
 
 find "$APPLICATION_PATH/Contents/Home/bin" \
@@ -73,16 +74,18 @@ if [[ $non_plist -gt 0 ]]; then
   exit 1
 fi
 
-log "Unlocking keychain..."
-# Make sure *.p12 is imported into local KeyChain
-security unlock-keychain -p "$PASSWORD" "/Users/$USERNAME/Library/Keychains/login.keychain"
+if [[ "${JETSIGN_CLIENT:=}" == "null" ]] || [[ "$JETSIGN_CLIENT" == "" ]]; then
+  log "Unlocking keychain..."
+  # Make sure *.p12 is imported into local KeyChain
+  security unlock-keychain -p "$PASSWORD" "/Users/$USERNAME/Library/Keychains/login.keychain"
+fi
 
 attempt=1
 limit=3
 set +e
 while [[ $attempt -le $limit ]]; do
   log "Signing (attempt $attempt) $APPLICATION_PATH ..."
-  ./sign.sh "$APPLICATION_PATH" "$APP_NAME" "$BUNDLE_ID" "$CODESIGN_STRING" "$JB_INSTALLER_CERT"
+  "$SCRIPT_DIR/sign.sh" "$APPLICATION_PATH" "$PKG_NAME" "$BUNDLE_ID" "$CODESIGN_STRING" "$JB_INSTALLER_CERT"
   ec=$?
   if [[ $ec -ne 0 ]]; then
     ((attempt += 1))
@@ -104,19 +107,10 @@ set -e
 
 if [ "$NOTARIZE" = "yes" ]; then
   log "Notarizing..."
-  # shellcheck disable=SC1090
-  source "$HOME/.notarize_token"
-  # Since notarization tool uses same file for upload token we have to trick it into using different folders, hence fake root
-  # Also it leaves copy of zip file in TMPDIR, so notarize.sh overrides it and uses FAKE_ROOT as location for temp TMPDIR
-  FAKE_ROOT="$(pwd)/fake-root"
-  mkdir -p "$FAKE_ROOT"
-  echo "Notarization will use fake root: $FAKE_ROOT"
-  ./notarize.sh "$APPLICATION_PATH" "$APPLE_USERNAME" "$APPLE_PASSWORD" "$APP_NAME.pkg" "$BUNDLE_ID" "$FAKE_ROOT"
-  rm -rf "$FAKE_ROOT"
-
-  set +e
+  "$SCRIPT_DIR/notarize.sh" "$PKG_NAME"
   log "Stapling..."
-  xcrun stapler staple "$APPLICATION_PATH"
+  xcrun stapler staple "$APPLICATION_PATH" ||:
+  xcrun stapler staple "$PKG_NAME" ||:
 else
   log "Notarization disabled"
   log "Stapling disabled"
@@ -129,7 +123,11 @@ log "Zipping $BUILD_NAME to $INPUT_FILE ..."
   if test -d $BACKUP_JMODS/jmods; then
     mv $BACKUP_JMODS/jmods $APPLICATION_PATH/Contents/Home
   fi
-  mv $APPLICATION_PATH $EXPLODED/$BUILD_NAME
+  if [[ "$APPLICATION_PATH" != "$EXPLODED/$BUILD_NAME" ]]; then
+    mv $APPLICATION_PATH $EXPLODED/$BUILD_NAME
+  else
+    echo "No move, source == destination: $APPLICATION_PATH"
+  fi
 
   tar -pczvf $INPUT_FILE --exclude='man' -C $EXPLODED $BUILD_NAME
   log "Finished zipping"

jb/project/tools/windows/scripts/mkimages_aarch64.sh

@@ -34,6 +34,7 @@ NVDA_PATH=${NVDA_PATH:=$WORK_DIR/nvda_controllerClient}
 
 function do_configure {
   sh ./configure \
+    --enable-option-checking=fatal \
     --openjdk-target=aarch64-unknown-cygwin \
     $WITH_DEBUG_LEVEL \
     --with-vendor-name="$VENDOR_NAME" \
@@ -47,9 +48,10 @@ function do_configure {
     --with-build-jdk=$BUILD_JDK \
     --with-nvdacontrollerclient=$NVDA_PATH \
     --disable-ccache \
+    --enable-cds=yes \
     $STATIC_CONF_ARGS \
     $REPRODUCIBLE_BUILD_OPTS \
-    --enable-cds=yes || do_exit $?
+    || do_exit $?
 }
 
 function create_image_bundle {
@@ -68,7 +70,7 @@ function create_image_bundle {
     --module-path $__modules_path --no-man-pages --compress=2 \
     --add-modules $__modules --output $__root_dir || do_exit $?
 
-  grep -v "^JAVA_VERSION" "$JSDK"/release | grep -v "^MODULES" >> $__arch_name/release
+  grep -v "^JAVA_VERSION" "$JSDK"/release | grep -v "^MODULES" >> $__root_dir/release
   if [ "$__arch_name" == "$JBRSDK_BUNDLE" ]; then
     sed 's/JBR/JBRSDK/g' $__root_dir/release > release
     mv release $__root_dir/release
@@ -129,6 +131,7 @@ if [ "$bundle_type" == "jcef" ] || [ "$bundle_type" == "fd" ]; then
   cp $JCEF_PATH/jmods/* $JSDK_MODS_DIR # $JSDK/jmods is not unchanged
 
   jbr_name_postfix="_${bundle_type}"
+  cat $JCEF_PATH/jcef.version >> $JSDK/release
 else
   jbr_name_postfix=""
 fi

jb/project/tools/windows/scripts/mkimages_x64.sh

@@ -122,6 +122,7 @@ if [ "$bundle_type" == "jcef" ] || [ "$bundle_type" == "fd" ]; then
   cp $JCEF_PATH/jmods/* ${JSDK_MODS_DIR} # $JSDK/jmods is not unchanged
 
   jbr_name_postfix="_${bundle_type}"
+  cat $JCEF_PATH/jcef.version >> $JSDK/release
 else
   jbr_name_postfix=""
 fi

make/CompileInterimLangtools.gmk

@@ -98,6 +98,7 @@ define SetupInterimModule
       EXCLUDES := sun javax/tools/snippet-files, \
       EXCLUDE_FILES := $(TOPDIR)/src/$1/share/classes/module-info.java \
           $(TOPDIR)/src/$1/share/classes/javax/tools/ToolProvider.java \
+          $(TOPDIR)/src/$1/share/classes/com/sun/tools/javac/launcher/Main.java \
           Standard.java, \
       EXTRA_FILES := $(BUILDTOOLS_OUTPUTDIR)/gensrc/$1.interim/module-info.java \
           $($1.interim_EXTRA_FILES), \

make/Docs.gmk

@@ -110,7 +110,6 @@ JAVA_WARNINGS_ARE_ERRORS ?= -Werror
 JAVADOC_OPTIONS := -use -keywords -notimestamp \
     -encoding ISO-8859-1 -docencoding UTF-8 -breakiterator \
     -splitIndex --system none -javafx --expand-requires transitive \
-    --enable-preview -source $(JDK_SOURCE_TARGET_VERSION) \
     --override-methods=summary \
     --no-external-specs-page
 
@@ -118,7 +117,6 @@ JAVADOC_OPTIONS := -use -keywords -notimestamp \
 # development cycle.
 REFERENCE_OPTIONS := -XDignore.symbol.file=true -use -keywords -notimestamp \
     -encoding ISO-8859-1 -breakiterator -splitIndex --system none \
-    --enable-preview -source $(JDK_SOURCE_TARGET_VERSION) \
     -html5 -javafx --expand-requires transitive \
     --no-external-specs-page
 

make/Global.gmk

@@ -125,6 +125,11 @@ test-prebuilt:
 	    $(MAKE) --no-print-directory -r -R -I make/common/ -f make/RunTestsPrebuilt.gmk \
 	    test-prebuilt CUSTOM_MAKE_DIR=$(CUSTOM_MAKE_DIR) TEST="$(TEST)" )
 
+test-prebuilt-with-exit-code:
+	@( cd $(topdir) && \
+	    $(MAKE) --no-print-directory -r -R -I make/common/ -f make/RunTestsPrebuilt.gmk \
+	    test-prebuilt-with-exit-code CUSTOM_MAKE_DIR=$(CUSTOM_MAKE_DIR) TEST="$(TEST)" )
+
 # Alias for backwards compatibility
 run-test-prebuilt: test-prebuilt
 

make/RunTests.gmk

@@ -595,16 +595,7 @@ define SetupRunMicroTestBody
   endif
 
   # Set library path for native dependencies
-  $1_JMH_JVM_ARGS := -Djava.library.path=$$(TEST_IMAGE_DIR)/micro/native \
-        --add-exports java.base/jdk.internal.org.objectweb.asm=ALL-UNNAMED \
-        --add-exports java.base/jdk.internal.org.objectweb.asm.tree=ALL-UNNAMED \
-        --add-exports java.base/jdk.internal.classfile=ALL-UNNAMED \
-        --add-exports java.base/jdk.internal.classfile.attribute=ALL-UNNAMED \
-        --add-exports java.base/jdk.internal.classfile.constantpool=ALL-UNNAMED \
-        --add-exports java.base/jdk.internal.classfile.instruction=ALL-UNNAMED \
-        --add-exports java.base/jdk.internal.classfile.java.lang.constant=ALL-UNNAMED \
-        --add-exports java.base/jdk.internal.classfile.components=ALL-UNNAMED \
-        --add-exports java.base/jdk.internal.classfile.impl=ALL-UNNAMED
+  $1_JMH_JVM_ARGS := -Djava.library.path=$$(TEST_IMAGE_DIR)/micro/native
 
   ifneq ($$(MICRO_VM_OPTIONS)$$(MICRO_JAVA_OPTIONS), )
     $1_JMH_JVM_ARGS += $$(MICRO_VM_OPTIONS) $$(MICRO_JAVA_OPTIONS)

make/RunTestsPrebuilt.gmk

@@ -295,6 +295,11 @@ test-prebuilt:
 	@cd $(TOPDIR) && $(MAKE) $(MAKE_ARGS) -f make/RunTests.gmk run-test \
 	    TEST="$(TEST)"
 
+test-prebuilt-with-exit-code: test-prebuilt
+	@if test -f $(MAKESUPPORT_OUTPUTDIR)/exit-with-error ; then \
+	  exit 1 ; \
+	fi
+
 all: test-prebuilt
 
 .PHONY: default all test-prebuilt

make/autoconf/flags-cflags.m4

@@ -51,6 +51,14 @@ AC_DEFUN([FLAGS_SETUP_SHARED_LIBS],
       SET_SHARED_LIBRARY_NAME='-Wl,-install_name,@rpath/[$]1'
       SET_SHARED_LIBRARY_MAPFILE='-Wl,-exported_symbols_list,[$]1'
 
+    elif test "x$OPENJDK_TARGET_OS" = xaix; then
+      # Linking is different on aix
+      SHARED_LIBRARY_FLAGS="-shared -Wl,-bM:SRE -Wl,-bnoentry"
+      SET_EXECUTABLE_ORIGIN=""
+      SET_SHARED_LIBRARY_ORIGIN=''
+      SET_SHARED_LIBRARY_NAME=''
+      SET_SHARED_LIBRARY_MAPFILE=''
+
     else
       # Default works for linux, might work on other platforms as well.
       SHARED_LIBRARY_FLAGS='-shared'
@@ -206,7 +214,6 @@ AC_DEFUN([FLAGS_SETUP_WARNINGS],
       WARNINGS_ENABLE_ALL="-Wall -Wextra -Wformat=2 $WARNINGS_ENABLE_ADDITIONAL"
 
       DISABLED_WARNINGS="unknown-warning-option unused-parameter unused"
-
       ;;
 
     xlc)
@@ -284,9 +291,15 @@ AC_DEFUN([FLAGS_SETUP_OPTIMIZATION],
       C_O_FLAG_NONE="${C_O_FLAG_NONE} ${DISABLE_FORTIFY_CFLAGS}"
     fi
   elif test "x$TOOLCHAIN_TYPE" = xclang; then
-    C_O_FLAG_HIGHEST_JVM="-O3"
-    C_O_FLAG_HIGHEST="-O3"
-    C_O_FLAG_HI="-O3"
+    if test "x$OPENJDK_TARGET_OS" = xaix; then
+      C_O_FLAG_HIGHEST_JVM="-O3 -finline-functions"
+      C_O_FLAG_HIGHEST="-O3 -finline-functions"
+      C_O_FLAG_HI="-O3 -finline-functions"
+    else
+      C_O_FLAG_HIGHEST_JVM="-O3"
+      C_O_FLAG_HIGHEST="-O3"
+      C_O_FLAG_HI="-O3"
+    fi
     C_O_FLAG_NORM="-O2"
     C_O_FLAG_DEBUG_JVM="-O0"
     C_O_FLAG_SIZE="-Os"
@@ -458,6 +471,9 @@ AC_DEFUN([FLAGS_SETUP_CFLAGS_HELPER],
       # so for debug we build with '-qpic=large -bbigtoc'.
       DEBUG_CFLAGS_JVM="-qpic=large"
     fi
+    if test "x$TOOLCHAIN_TYPE" = xclang && test "x$OPENJDK_TARGET_OS" = xaix; then
+      DEBUG_CFLAGS_JVM="-fpic -mcmodel=large"
+    fi
   fi
 
   if test "x$DEBUG_LEVEL" != xrelease; then
@@ -493,6 +509,12 @@ AC_DEFUN([FLAGS_SETUP_CFLAGS_HELPER],
         -fvisibility=hidden -fno-strict-aliasing -fno-omit-frame-pointer"
   fi
 
+  if test "x$TOOLCHAIN_TYPE" = xclang && test "x$OPENJDK_TARGET_OS" = xaix; then
+    # clang compiler on aix needs -ffunction-sections
+    TOOLCHAIN_CFLAGS_JVM="$TOOLCHAIN_CFLAGS_JVM -ffunction-sections -ftls-model -fno-math-errno -fstack-protector"
+    TOOLCHAIN_CFLAGS_JDK="-ffunction-sections -fsigned-char -fstack-protector"
+  fi
+
   if test "x$TOOLCHAIN_TYPE" = xgcc; then
     TOOLCHAIN_CFLAGS_JVM="$TOOLCHAIN_CFLAGS_JVM -fstack-protector"
     TOOLCHAIN_CFLAGS_JDK="-pipe -fstack-protector"
@@ -601,6 +623,9 @@ AC_DEFUN([FLAGS_SETUP_CFLAGS_HELPER],
   if test "x$TOOLCHAIN_TYPE" = xgcc || test "x$TOOLCHAIN_TYPE" = xclang; then
     PICFLAG="-fPIC"
     PIEFLAG="-fPIE"
+  elif test "x$TOOLCHAIN_TYPE" = xclang && test "x$OPENJDK_TARGET_OS" = xaix; then
+    JVM_PICFLAG="-fpic -mcmodel=large -Wl,-bbigtoc
+    JDK_PICFLAG="-fpic
   elif test "x$TOOLCHAIN_TYPE" = xxlc; then
     # '-qpic' defaults to 'qpic=small'. This means that the compiler generates only
     # one instruction for accessing the TOC. If the TOC grows larger than 64K, the linker
@@ -746,6 +771,9 @@ AC_DEFUN([FLAGS_SETUP_CFLAGS_CPU_DEP],
         $1_CFLAGS_CPU_JDK="${$1_CFLAGS_CPU_JDK} -fno-omit-frame-pointer"
       fi
     fi
+    if test "x$OPENJDK_TARGET_OS" = xaix; then
+      $1_CFLAGS_CPU="-mcpu=pwr8"
+    fi
 
   elif test "x$TOOLCHAIN_TYPE" = xxlc; then
     if test "x$FLAGS_CPU" = xppc64; then

make/autoconf/flags-ldflags.m4

@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2011, 2022, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2011, 2023, Oracle and/or its affiliates. All rights reserved.
 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 #
 # This code is free software; you can redistribute it and/or modify it
@@ -75,7 +75,11 @@ AC_DEFUN([FLAGS_SETUP_LDFLAGS_HELPER],
   elif test "x$TOOLCHAIN_TYPE" = xclang; then
     BASIC_LDFLAGS_JVM_ONLY="-mno-omit-leaf-frame-pointer -mstack-alignment=16 \
         -fPIC"
-
+    if test "x$OPENJDK_TARGET_OS" = xaix; then
+      BASIC_LDFLAGS="-Wl,-b64 -Wl,-brtl -Wl,-bnorwexec -Wl,-bnolibpath -Wl,-bnoexpall \
+        -Wl,-bernotok -Wl,-bdatapsize:64k -Wl,-btextpsize:64k -Wl,-bstackpsize:64k"
+      BASIC_LDFLAGS_JVM_ONLY="$BASIC_LDFLAGS_JVM_ONLY -Wl,-lC_r -Wl,-bbigtoc"
+    fi
   elif test "x$TOOLCHAIN_TYPE" = xxlc; then
     BASIC_LDFLAGS="-b64 -brtl -bnorwexec -bnolibpath -bnoexpall -bernotok -btextpsize:64K \
         -bdatapsize:64K -bstackpsize:64K"
@@ -88,7 +92,8 @@ AC_DEFUN([FLAGS_SETUP_LDFLAGS_HELPER],
     BASIC_LDFLAGS_JVM_ONLY="-opt:icf,8 -subsystem:windows"
   fi
 
-  if test "x$TOOLCHAIN_TYPE" = xgcc || test "x$TOOLCHAIN_TYPE" = xclang; then
+  if (test "x$TOOLCHAIN_TYPE" = xgcc || test "x$TOOLCHAIN_TYPE" = xclang) \
+      && test "x$OPENJDK_TARGET_OS" != xaix; then
     if test -n "$HAS_NOEXECSTACK"; then
       BASIC_LDFLAGS="$BASIC_LDFLAGS -Wl,-z,noexecstack"
     fi
@@ -116,6 +121,14 @@ AC_DEFUN([FLAGS_SETUP_LDFLAGS_HELPER],
     if test "x$DEBUG_LEVEL" != xrelease; then
       DEBUGLEVEL_LDFLAGS_JVM_ONLY="$DEBUGLEVEL_LDFLAGS_JVM_ONLY -bbigtoc"
     fi
+
+  elif test "x$TOOLCHAIN_TYPE" = xclang && test "x$OPENJDK_TARGET_OS" = xaix; then
+    # We need '-fpic' or '-fpic -mcmodel=large -Wl,-bbigtoc' if the TOC overflows.
+    # Hotspot now overflows its 64K TOC (currently only for debug),
+    # so we build with '-fpic -mcmodel=large -Wl,-bbigtoc'.
+    if test "x$DEBUG_LEVEL" != xrelease; then
+      DEBUGLEVEL_LDFLAGS_JVM_ONLY="$DEBUGLEVEL_LDFLAGS_JVM_ONLY -Wl,-bbigtoc"
+    fi
   fi
 
   # Setup LDFLAGS for linking executables

make/autoconf/lib-wayland.m4

@@ -99,6 +99,9 @@ AC_DEFUN_ONCE([LIB_SETUP_WAYLAND],
     AC_ARG_WITH(vulkan-include, [AS_HELP_STRING([--with-vulkan-include],
       [specify directory for the vulkan include files])])
 
+    AC_ARG_WITH(vulkan-shader-compiler, [AS_HELP_STRING([--with-vulkan-shader-compiler],
+      [specify which shader compiler to use: glslc/glslangValidator])])
+
     if test "x$SUPPORTS_LIB_VULKAN" = xfalse; then
 
       if (test "x${with_vulkan}" != x && test "x${with_vulkan}" != xno) || \
@@ -120,7 +123,6 @@ AC_DEFUN_ONCE([LIB_SETUP_WAYLAND],
           AC_CHECK_HEADERS([${with_vulkan_include}/include/vulkan/vulkan.h],
             [ VULKAN_FOUND=yes
               VULKAN_FLAGS="-DVK_USE_PLATFORM_WAYLAND_KHR -I${with_vulkan_include}/include -DVULKAN_ENABLED"
-              VULKAN_ENABLED=true
             ],
             [ AC_MSG_ERROR([Can't find 'vulkan/vulkan.h' under '${with_vulkan_include}']) ]
           )
@@ -131,7 +133,6 @@ AC_DEFUN_ONCE([LIB_SETUP_WAYLAND],
           AC_CHECK_HEADERS([$VULKAN_SDK/include/vulkan/vulkan.h],
             [ VULKAN_FOUND=yes
               VULKAN_FLAGS="-DVK_USE_PLATFORM_WAYLAND_KHR -I${VULKAN_SDK}/include -DVULKAN_ENABLED"
-              VULKAN_ENABLED=true
             ],
             [ VULKAN_FOUND=no; break ]
           )
@@ -142,7 +143,6 @@ AC_DEFUN_ONCE([LIB_SETUP_WAYLAND],
           AC_CHECK_HEADERS([vulkan/vulkan.h],
             [ VULKAN_FOUND=yes
               VULKAN_FLAGS="-DVK_USE_PLATFORM_WAYLAND_KHR -DVULKAN_ENABLED"
-              VULKAN_ENABLED=true
             ],
             [ VULKAN_FOUND=no; break ]
           )
@@ -151,11 +151,32 @@ AC_DEFUN_ONCE([LIB_SETUP_WAYLAND],
         if test "x$VULKAN_FOUND" = xno; then
           HELP_MSG_MISSING_DEPENDENCY([vulkan])
           AC_MSG_ERROR([Could not find vulkan! $HELP_MSG ])
+        else
+          # Find shader compiler - glslc or glslangValidator
+          if (test "x${with_vulkan_shader_compiler}" = x || test "x${with_vulkan_shader_compiler}" = xglslc); then
+            UTIL_LOOKUP_PROGS(GLSLC, glslc)
+            SHADER_COMPILER="$GLSLC"
+            VULKAN_SHADER_COMPILER="glslc --target-env=vulkan1.2 -mfmt=num -o"
+          fi
+
+          if (test "x${with_vulkan_shader_compiler}" = x || test "x${with_vulkan_shader_compiler}" = xglslangValidator) && \
+              test "x$SHADER_COMPILER" = x; then
+            UTIL_LOOKUP_PROGS(GLSLANG, glslangValidator)
+            SHADER_COMPILER="$GLSLANG"
+            VULKAN_SHADER_COMPILER="glslangValidator --target-env vulkan1.2 -x -o"
+          fi
+
+          if test "x$SHADER_COMPILER" != x; then
+            VULKAN_ENABLED=true
+          else
+            AC_MSG_ERROR([Can't find shader compiler])
+          fi
         fi
       fi
     fi
   fi
   AC_SUBST(VULKAN_FLAGS)
+  AC_SUBST(VULKAN_SHADER_COMPILER)
   AC_SUBST(VULKAN_ENABLED)
   AC_SUBST(WAYLAND_CFLAGS)
   AC_SUBST(WAYLAND_LIBS)

make/autoconf/platform.m4

@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2011, 2022, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2011, 2023, Oracle and/or its affiliates. All rights reserved.
 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 #
 # This code is free software; you can redistribute it and/or modify it
@@ -657,6 +657,21 @@ AC_DEFUN_ONCE([PLATFORM_SETUP_OPENJDK_BUILD_AND_TARGET],
   PLATFORM_SET_MODULE_TARGET_OS_VALUES
   PLATFORM_SET_RELEASE_FILE_OS_VALUES
   PLATFORM_SETUP_LEGACY_VARS
+  PLATFORM_CHECK_DEPRECATION
+])
+
+AC_DEFUN([PLATFORM_CHECK_DEPRECATION],
+[
+  AC_ARG_ENABLE(deprecated-ports, [AS_HELP_STRING([--enable-deprecated-ports@<:@=yes/no@:>@],
+       [Suppress the error when configuring for a deprecated port @<:@no@:>@])])
+  if test "x$OPENJDK_TARGET_OS" = xwindows && test "x$OPENJDK_TARGET_CPU" = xx86; then
+    if test "x$enable_deprecated_ports" = "xyes"; then
+      AC_MSG_WARN([The Windows 32-bit x86 port is deprecated and may be removed in a future release.])
+    else
+      AC_MSG_ERROR(m4_normalize([The Windows 32-bit x86 port is deprecated and may be removed in a future release.
+        Use --enable-deprecated-ports=yes to suppress this error.]))
+    fi
+  fi
 ])
 
 AC_DEFUN_ONCE([PLATFORM_SETUP_OPENJDK_BUILD_OS_VERSION],

make/autoconf/spec.gmk.in

@@ -482,6 +482,7 @@ A11Y_JAWS_ANNOUNCING_ENABLED:=@A11Y_JAWS_ANNOUNCING_ENABLED@
 WAYLAND_CFLAGS:=@WAYLAND_CFLAGS@
 WAYLAND_LIBS:=@WAYLAND_LIBS@
 VULKAN_FLAGS:=@VULKAN_FLAGS@
+VULKAN_SHADER_COMPILER:=@VULKAN_SHADER_COMPILER@
 VULKAN_ENABLED:=@VULKAN_ENABLED@
 
 # The lowest required version of macosx

make/autoconf/toolchain.m4

@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2011, 2022, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2011, 2023, Oracle and/or its affiliates. All rights reserved.
 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 #
 # This code is free software; you can redistribute it and/or modify it
@@ -40,7 +40,7 @@ VALID_TOOLCHAINS_all="gcc clang xlc microsoft"
 # These toolchains are valid on different platforms
 VALID_TOOLCHAINS_linux="gcc clang"
 VALID_TOOLCHAINS_macosx="clang"
-VALID_TOOLCHAINS_aix="xlc"
+VALID_TOOLCHAINS_aix="xlc clang"
 VALID_TOOLCHAINS_windows="microsoft"
 
 # Toolchain descriptions
@@ -53,7 +53,7 @@ TOOLCHAIN_DESCRIPTION_xlc="IBM XL C/C++"
 TOOLCHAIN_MINIMUM_VERSION_clang="3.5"
 TOOLCHAIN_MINIMUM_VERSION_gcc="6.0"
 TOOLCHAIN_MINIMUM_VERSION_microsoft="19.28.0.0" # VS2019 16.8, aka MSVC 14.28
-TOOLCHAIN_MINIMUM_VERSION_xlc=""
+TOOLCHAIN_MINIMUM_VERSION_xlc="16.1.0.0011"
 
 # Minimum supported linker versions, empty means unspecified
 TOOLCHAIN_MINIMUM_LD_VERSION_gcc="2.18"
@@ -234,6 +234,25 @@ AC_DEFUN_ONCE([TOOLCHAIN_DETERMINE_TOOLCHAIN_TYPE],
   # First toolchain type in the list is the default
   DEFAULT_TOOLCHAIN=${VALID_TOOLCHAINS%% *}
 
+  # On AIX the default toolchain depends on the installed (found) compiler
+  #   xlclang++     -> xlc toolchain
+  #   ibm-clang++_r -> clang toolchain
+  # The compiler is searched on the PATH and TOOLCHAIN_PATH
+  # xlclang++ has precedence over ibm-clang++_r if both are installed
+  if test "x$OPENJDK_TARGET_OS" = xaix; then
+    DEFAULT_TOOLCHAIN="clang"
+    if test "x$TOOLCHAIN_PATH" != x; then
+      if test -e ${TOOLCHAIN_PATH}/xlclang++; then
+        DEFAULT_TOOLCHAIN="xlc"
+      fi
+    else
+      UTIL_LOOKUP_PROGS(XLCLANG_TEST_PATH, xlclang++)
+      if test "x$XLCLANG_TEST_PATH" != x; then
+        DEFAULT_TOOLCHAIN="xlc"
+      fi
+    fi
+  fi
+
   if test "x$with_toolchain_type" = xlist; then
     # List all toolchains
     AC_MSG_NOTICE([The following toolchains are valid on this platform:])
@@ -263,22 +282,40 @@ AC_DEFUN_ONCE([TOOLCHAIN_DETERMINE_TOOLCHAIN_TYPE],
     if test "x$TOOLCHAIN_PATH" != x; then
       XLC_TEST_PATH=${TOOLCHAIN_PATH}/
     fi
-
-    XLCLANG_VERSION_OUTPUT=`${XLC_TEST_PATH}xlclang++ -qversion 2>&1 | $HEAD -n 1`
-    $ECHO "$XLCLANG_VERSION_OUTPUT" | $GREP "IBM XL C/C++ for AIX" > /dev/null
-    if test $? -eq 0; then
-      AC_MSG_NOTICE([xlclang++ output: $XLCLANG_VERSION_OUTPUT])
+    if test "x$TOOLCHAIN_TYPE" = xclang; then
+      TOOLCHAIN_DESCRIPTION_clang="IBM Open XL C/C++"
+      XLCLANG_VERSION_OUTPUT=`${XLC_TEST_PATH}ibm-clang++_r --version 2>&1 | $HEAD -n 1`
+      $ECHO "$XLCLANG_VERSION_OUTPUT" | $GREP "IBM Open XL C/C++ for AIX" > /dev/null
+      if test $? -eq 0; then
+        AC_MSG_NOTICE([ibm-clang++_r output: $XLCLANG_VERSION_OUTPUT])
+      else
+        AC_MSG_ERROR([ibm-clang++_r version output check failed, output: $XLCLANG_VERSION_OUTPUT])
+      fi
     else
-      AC_MSG_ERROR([xlclang++ version output check failed, output: $XLCLANG_VERSION_OUTPUT])
+      XLCLANG_VERSION_OUTPUT=`${XLC_TEST_PATH}xlclang++ -qversion 2>&1 | $HEAD -n 1`
+      $ECHO "$XLCLANG_VERSION_OUTPUT" | $GREP "IBM XL C/C++ for AIX" > /dev/null
+      if test $? -eq 0; then
+        AC_MSG_NOTICE([xlclang++ output: $XLCLANG_VERSION_OUTPUT])
+      else
+        AC_MSG_ERROR([xlclang++ version output check failed, output: $XLCLANG_VERSION_OUTPUT])
+      fi
     fi
   fi
 
-  TOOLCHAIN_CC_BINARY_clang="clang"
+  if test "x$OPENJDK_TARGET_OS" = xaix; then
+    TOOLCHAIN_CC_BINARY_clang="ibm-clang_r"
+  else
+    TOOLCHAIN_CC_BINARY_clang="clang"
+  fi
   TOOLCHAIN_CC_BINARY_gcc="gcc"
   TOOLCHAIN_CC_BINARY_microsoft="cl"
   TOOLCHAIN_CC_BINARY_xlc="xlclang"
 
-  TOOLCHAIN_CXX_BINARY_clang="clang++"
+  if test "x$OPENJDK_TARGET_OS" = xaix; then
+    TOOLCHAIN_CXX_BINARY_clang="ibm-clang++_r"
+  else
+    TOOLCHAIN_CXX_BINARY_clang="clang++"
+  fi
   TOOLCHAIN_CXX_BINARY_gcc="g++"
   TOOLCHAIN_CXX_BINARY_microsoft="cl"
   TOOLCHAIN_CXX_BINARY_xlc="xlclang++"
@@ -384,7 +421,7 @@ AC_DEFUN([TOOLCHAIN_EXTRACT_COMPILER_VERSION],
     # Collapse compiler output into a single line
     COMPILER_VERSION_STRING=`$ECHO $COMPILER_VERSION_OUTPUT`
     COMPILER_VERSION_NUMBER=`$ECHO $COMPILER_VERSION_OUTPUT | \
-        $SED -e 's/^.*, V\(@<:@1-9@:>@@<:@0-9.@:>@*\).*$/\1/'`
+        $SED -e 's/^.*Version: \(@<:@1-9@:>@@<:@0-9.@:>@*\).*$/\1/'`
   elif test  "x$TOOLCHAIN_TYPE" = xmicrosoft; then
     # There is no specific version flag, but all output starts with a version string.
     # First line typically looks something like:
@@ -632,7 +669,7 @@ AC_DEFUN_ONCE([TOOLCHAIN_DETECT_TOOLCHAIN_CORE],
   if test "x$TOOLCHAIN_MINIMUM_VERSION" != x; then
     TOOLCHAIN_CHECK_COMPILER_VERSION(VERSION: $TOOLCHAIN_MINIMUM_VERSION,
         IF_OLDER_THAN: [
-          AC_MSG_WARN([You are using $TOOLCHAIN_TYPE older than $TOOLCHAIN_MINIMUM_VERSION. This is not a supported configuration.])
+          AC_MSG_WARN([You are using $TOOLCHAIN_TYPE $CC_VERSION_NUMBER which is older than $TOOLCHAIN_MINIMUM_VERSION. This is not a supported configuration.])
         ]
     )
   fi
@@ -966,7 +1003,11 @@ AC_DEFUN_ONCE([TOOLCHAIN_MISC_CHECKS],
   # Setup hotspot lecagy names for toolchains
   HOTSPOT_TOOLCHAIN_TYPE=$TOOLCHAIN_TYPE
   if test "x$TOOLCHAIN_TYPE" = xclang; then
-    HOTSPOT_TOOLCHAIN_TYPE=gcc
+    if test "x$OPENJDK_TARGET_OS" = xaix; then
+      HOTSPOT_TOOLCHAIN_TYPE=xlc
+    else
+      HOTSPOT_TOOLCHAIN_TYPE=gcc
+    fi
   elif test "x$TOOLCHAIN_TYPE" = xmicrosoft; then
     HOTSPOT_TOOLCHAIN_TYPE=visCPP
   fi

make/conf/docs-modules.conf

@@ -42,7 +42,6 @@ DOCS_MODULES= \
     jdk.hotspot.agent \
     jdk.httpserver \
     jdk.jpackage \
-    jdk.incubator.concurrent \
     jdk.incubator.vector \
     jdk.jartool \
     jdk.javadoc \

make/conf/jib-profiles.js

@@ -463,7 +463,8 @@ var getJibProfilesProfiles = function (input, common, data) {
             target_cpu: "x86",
             build_cpu: "x64",
             dependencies: ["devkit", "gtest"],
-            configure_args: concat(common.configure_args_32bit),
+            configure_args: concat(common.configure_args_32bit,
+                "--enable-deprecated-ports"),
         },
 
         "windows-aarch64": {

make/conf/module-loader-map.conf

@@ -43,7 +43,6 @@ BOOT_MODULES= \
     java.rmi \
     java.security.sasl \
     java.xml \
-    jdk.incubator.concurrent \
     jdk.incubator.vector \
     jdk.internal.vm.ci \
     jdk.jfr \

make/conf/version-numbers.conf

@@ -39,4 +39,4 @@ DEFAULT_VERSION_CLASSFILE_MINOR=0
 DEFAULT_VERSION_DOCS_API_SINCE=11
 DEFAULT_ACCEPTABLE_BOOT_VERSIONS="20 21"
 DEFAULT_JDK_SOURCE_TARGET_VERSION=21
-DEFAULT_PROMOTED_VERSION_PRE=ea
+DEFAULT_PROMOTED_VERSION_PRE=

make/data/charsetmapping/DoubleByte-X.java.template

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2009, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2009, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -56,10 +56,10 @@ public class $NAME_CLZ$ extends Charset
         return new DoubleByte.Encoder$ENCTYPE$(this, $ENC_REPLACEMENT$ EncodeHolder.c2b, EncodeHolder.c2bIndex, $ASCIICOMPATIBLE$);
     }
 
-    static class DecodeHolder {
+    public static class DecodeHolder {
         $B2C$
-        static final char[][] b2c = new char[b2cStr.length][];
-        static final char[] b2cSB;
+        public static final char[][] b2c = new char[b2cStr.length][];
+        public static final char[] b2cSB;
 
         static {
             for (int i = 0; i < b2cStr.length; i++) {
@@ -72,9 +72,9 @@ public class $NAME_CLZ$ extends Charset
         }
     }
 
-    static class EncodeHolder {
-        static final char[] c2b = new char[$C2BLENGTH$];
-        static final char[] c2bIndex = new char[0x100];
+    public static class EncodeHolder {
+        public static final char[] c2b = new char[$C2BLENGTH$];
+        public static final char[] c2bIndex = new char[0x100];
 
         static {
             $NONROUNDTRIP_B2C$

make/data/charsetmapping/stdcs-linux

@@ -2,13 +2,11 @@
 #   generate these charsets into sun.nio.cs
 #
 Big5
-Big5_Solaris
 Big5_HKSCS
 EUC_CN
 EUC_KR
 EUC_JP
 EUC_JP_LINUX
-EUC_JP_Open
 EUC_TW
 GBK
 ISO_8859_11
@@ -16,12 +14,9 @@ ISO_8859_3
 ISO_8859_6
 ISO_8859_8
 Johab
-PCK
 TIS_620
 JIS_X_0201
 JIS_X_0208
 JIS_X_0212
-JIS_X_0208_Solaris
-JIS_X_0212_Solaris
 MS932
 SJIS

make/data/hotspot-symbols/symbols-unix

@@ -181,6 +181,7 @@ JVM_NewArray
 JVM_NewInstanceFromConstructor
 JVM_NewMultiArray
 JVM_PhantomReferenceRefersTo
+JVM_PrintWarningAtDynamicAgentLoad
 JVM_RaiseSignal
 JVM_RawMonitorCreate
 JVM_RawMonitorDestroy

make/hotspot/gensrc/GensrcAdlc.gmk

@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2013, 2022, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2013, 2023, Oracle and/or its affiliates. All rights reserved.
 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 #
 # This code is free software; you can redistribute it and/or modify it
@@ -37,8 +37,13 @@ ifeq ($(call check-jvm-feature, compiler2), true)
   ifeq ($(call isBuildOs, linux), true)
     ADLC_CFLAGS := -fno-exceptions -DLINUX
   else ifeq ($(call isBuildOs, aix), true)
-    ADLC_LDFLAGS += -q64
-    ADLC_CFLAGS := -qnortti -qeh -q64 -DAIX
+    ifeq ($(TOOLCHAIN_TYPE), clang)
+      ADLC_LDFLAGS += -m64
+      ADLC_CFLAGS := -fno-rtti -fexceptions -ffunction-sections -m64 -DAIX -mcpu=pwr8
+    else
+      ADLC_LDFLAGS += -q64
+      ADLC_CFLAGS := -qnortti -qeh -q64 -DAIX
+    endif
   else ifeq ($(call isBuildOs, windows), true)
     ADLC_LDFLAGS += -nologo
     ADLC_CFLAGS := -nologo -EHsc
@@ -167,6 +172,8 @@ ifeq ($(call check-jvm-feature, compiler2), true)
 
   ifeq ($(call check-jvm-feature, zgc), true)
     AD_SRC_FILES += $(call uniq, $(wildcard $(foreach d, $(AD_SRC_ROOTS), \
+        $d/cpu/$(HOTSPOT_TARGET_CPU_ARCH)/gc/x/x_$(HOTSPOT_TARGET_CPU).ad \
+        $d/cpu/$(HOTSPOT_TARGET_CPU_ARCH)/gc/x/x_$(HOTSPOT_TARGET_CPU_ARCH).ad \
         $d/cpu/$(HOTSPOT_TARGET_CPU_ARCH)/gc/z/z_$(HOTSPOT_TARGET_CPU).ad \
         $d/cpu/$(HOTSPOT_TARGET_CPU_ARCH)/gc/z/z_$(HOTSPOT_TARGET_CPU_ARCH).ad \
       )))

make/hotspot/lib/CompileJvm.gmk

@@ -168,10 +168,14 @@ $(eval $(call SetupJdkLibrary, BUILD_LIBJVM, \
     DISABLED_WARNINGS_clang_management.cpp := missing-field-initializers, \
     DISABLED_WARNINGS_clang_notificationThread.cpp := bitwise-instead-of-logical, \
     DISABLED_WARNINGS_clang_os_posix.cpp := mismatched-tags missing-field-initializers, \
+    DISABLED_WARNINGS_clang_aix_os_posix.cpp := format-nonliteral, \
     DISABLED_WARNINGS_clang_postaloc.cpp := tautological-undefined-compare, \
     DISABLED_WARNINGS_clang_serviceThread.cpp := bitwise-instead-of-logical, \
     DISABLED_WARNINGS_clang_vm_version_x86.cpp := missing-field-initializers, \
     DISABLED_WARNINGS_clang_zTracer.cpp := undefined-var-template, \
+    DISABLED_WARNINGS_clang_aix_debug.cpp := format-nonliteral, \
+    DISABLED_WARNINGS_clang_aix_jvm.cpp := format-nonliteral, \
+    DISABLED_WARNINGS_clang_aix_osThread_aix.cpp := tautological-undefined-compare, \
     DISABLED_WARNINGS_xlc := $(DISABLED_WARNINGS_xlc), \
     DISABLED_WARNINGS_microsoft := $(DISABLED_WARNINGS_microsoft), \
     ASFLAGS := $(JVM_ASFLAGS), \

make/hotspot/lib/JvmFeatures.gmk

@@ -150,6 +150,7 @@ endif
 ifneq ($(call check-jvm-feature, zgc), true)
   JVM_CFLAGS_FEATURES += -DINCLUDE_ZGC=0
   JVM_EXCLUDE_PATTERNS += gc/z
+  JVM_EXCLUDE_PATTERNS += gc/x
 endif
 
 ifneq ($(call check-jvm-feature, shenandoahgc), true)

make/hotspot/lib/JvmOverrideFiles.gmk

@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2013, 2021, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2013, 2023, Oracle and/or its affiliates. All rights reserved.
 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 #
 # This code is free software; you can redistribute it and/or modify it
@@ -109,7 +109,11 @@ else ifeq ($(call isTargetOs, macosx), true)
   endif
 
 else ifeq ($(call isTargetOs, aix), true)
-  BUILD_LIBJVM_synchronizer.cpp_CXXFLAGS := -qnoinline
+  ifeq ($(TOOLCHAIN_TYPE), clang)
+    BUILD_LIBJVM_synchronizer.cpp_CXXFLAGS := -fno-inline
+  else
+    BUILD_LIBJVM_synchronizer.cpp_CXXFLAGS := -qnoinline
+  endif
   BUILD_LIBJVM_sharedRuntimeTrans.cpp_CXXFLAGS := $(CXX_O_FLAG_NONE)
   # Disable aggressive optimizations for functions in sharedRuntimeTrig.cpp
   # and sharedRuntimeTrans.cpp on ppc64.

make/jdk/src/classes/build/tools/generatelsrequivmaps/EquivMapsGenerator.java

@@ -60,7 +60,9 @@ public class EquivMapsGenerator {
         }
         copyrightYear = Integer.parseInt(args[2]);
         readLSRfile(args[0]);
+        // Builds the maps from the IANA data
         generateEquivalentMap();
+        // Writes the maps out to LocaleEquivalentMaps.java
         generateSourceCode(args[1]);
     }
 
@@ -213,63 +215,6 @@ public class EquivMapsGenerator {
         return list.toArray(new String[list.size()]);
     }
 
-    private static String generateValuesString(String[] values) {
-        String outputStr = "";
-        for (int i = 0; i < values.length; i++) {
-            if (i != values.length - 1) {
-                outputStr = outputStr + "\"" + values[i] + "\", ";
-            } else {
-                outputStr = outputStr + "\"" + values[i] + "\"";
-            }
-
-        }
-        return outputStr;
-    }
-
-    private static final String COPYRIGHT = "/*\n"
-        + " * Copyright (c) 2012, %d, Oracle and/or its affiliates. All rights reserved.\n"
-        + " * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.\n"
-        + " *\n"
-        + " * This code is free software; you can redistribute it and/or modify it\n"
-        + " * under the terms of the GNU General Public License version 2 only, as\n"
-        + " * published by the Free Software Foundation.  Oracle designates this\n"
-        + " * particular file as subject to the \"Classpath\" exception as provided\n"
-        + " * by Oracle in the LICENSE file that accompanied this code.\n"
-        + " *\n"
-        + " * This code is distributed in the hope that it will be useful, but WITHOUT\n"
-        + " * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or\n"
-        + " * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License\n"
-        + " * version 2 for more details (a copy is included in the LICENSE file that\n"
-        + " * accompanied this code).\n"
-        + " *\n"
-        + " * You should have received a copy of the GNU General Public License version\n"
-        + " * 2 along with this work; if not, write to the Free Software Foundation,\n"
-        + " * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.\n"
-        + " *\n"
-        + " * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA\n"
-        + " * or visit www.oracle.com if you need additional information or have any\n"
-        + " * questions.\n"
-        + "*/\n\n";
-
-    private static final String headerText =
-        "package sun.util.locale;\n\n"
-        + "import java.util.HashMap;\n"
-        + "import java.util.Map;\n\n"
-        + "final class LocaleEquivalentMaps {\n\n"
-        + "    static final Map<String, String> singleEquivMap;\n"
-        + "    static final Map<String, String[]> multiEquivsMap;\n"
-        + "    static final Map<String, String> regionVariantEquivMap;\n\n"
-        + "    static {\n"
-        + "        singleEquivMap = HashMap.newHashMap(";
-
-    private static final String footerText =
-        "    }\n\n"
-        + "}";
-
-    private static String getOpenJDKCopyright() {
-        return String.format(Locale.US, COPYRIGHT, copyrightYear);
-    }
-
     /**
      * The input lsr data file is in UTF-8, so theoretically for the characters
      * beyond US-ASCII, the generated Java String literals need to be Unicode
@@ -277,53 +222,132 @@ public class EquivMapsGenerator {
      * the case since we don't use "description", "comment" or alike.
      */
     private static void generateSourceCode(String fileName) {
-
         try (BufferedWriter writer = Files.newBufferedWriter(
                 Paths.get(fileName))) {
             writer.write(getOpenJDKCopyright());
-            writer.write(headerText
-                + sortedLanguageMap1.size() + ");\n"
-                + "        multiEquivsMap = HashMap.newHashMap("
-                + sortedLanguageMap2.size() + ");\n"
-                + "        regionVariantEquivMap = HashMap.newHashMap("
-                + sortedRegionVariantMap.size() + ");\n\n"
-                + "        // This is an auto-generated file and should not be manually edited.\n"
-                + "        //   LSR Revision: " + LSRrevisionDate);
+            writer.write(HEADER_TEXT);
+            writer.write(getMapsText());
+            writer.write(getLSRText());
+            writeEquiv(writer, "singleEquivMap", sortedLanguageMap1);
             writer.newLine();
-
-            for (String key : sortedLanguageMap1.keySet()) {
-                String value = sortedLanguageMap1.get(key);
-                writer.write("        singleEquivMap.put(\""
-                    + key + "\", \"" + value + "\");");
-                writer.newLine();
-            }
-
+            writeMultiEquiv(writer);
             writer.newLine();
-            for (String key : sortedLanguageMap2.keySet()) {
-                String[] values = sortedLanguageMap2.get(key);
-
-                if (values.length >= 2) {
-                    writer.write("        multiEquivsMap.put(\""
-                        + key + "\", new String[] {"
-                        + generateValuesString(values) + "});");
-                    writer.newLine();
-                }
-            }
-
-            writer.newLine();
-            for (String key : sortedRegionVariantMap.keySet()) {
-                String value = sortedRegionVariantMap.get(key);
-                writer.write("        regionVariantEquivMap.put(\""
-                    + key + "\", \"" + value + "\");");
-                writer.newLine();
-            }
-
-            writer.write(footerText);
+            writeEquiv(writer, "regionVariantEquivMap", sortedRegionVariantMap);
+            writer.write(FOOTER_TEXT);
         } catch (IOException ex) {
             ex.printStackTrace(System.err);
             System.exit(1);
         }
-
     }
 
+    private static String getOpenJDKCopyright() {
+        return String.format(Locale.US, COPYRIGHT, copyrightYear);
+    }
+
+    private static final String COPYRIGHT =
+            """
+            /*
+             * Copyright (c) 2012, %d, Oracle and/or its affiliates. All rights reserved.
+             * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+             *
+             * This code is free software; you can redistribute it and/or modify it
+             * under the terms of the GNU General Public License version 2 only, as
+             * published by the Free Software Foundation.  Oracle designates this
+             * particular file as subject to the \"Classpath\" exception as provided
+             * by Oracle in the LICENSE file that accompanied this code.
+             *
+             * This code is distributed in the hope that it will be useful, but WITHOUT
+             * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+             * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+             * version 2 for more details (a copy is included in the LICENSE file that
+             * accompanied this code).
+             *
+             * You should have received a copy of the GNU General Public License version
+             * 2 along with this work; if not, write to the Free Software Foundation,
+             * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+             *
+             * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+             * or visit www.oracle.com if you need additional information or have any
+             * questions.
+            */
+
+            """;
+
+    private static final String HEADER_TEXT =
+            """
+            package sun.util.locale;
+
+            import java.util.HashMap;
+            import java.util.Map;
+
+            final class LocaleEquivalentMaps {
+
+                static final Map<String, String> singleEquivMap;
+                static final Map<String, String[]> multiEquivsMap;
+                static final Map<String, String> regionVariantEquivMap;
+
+            """;
+
+    private static final String FOOTER_TEXT =
+            """
+                }
+
+            }
+            """;
+
+    private static String getMapsText() {
+        return  """
+                    static {
+                        singleEquivMap = HashMap.newHashMap(%s);
+                        multiEquivsMap = HashMap.newHashMap(%s);
+                        regionVariantEquivMap = HashMap.newHashMap(%s);
+
+                """.formatted(
+                sortedLanguageMap1.size(),
+                sortedLanguageMap2.size(),
+                sortedRegionVariantMap.size());
+    }
+
+    private static final String getLSRText(){
+        return  """
+                        // This is an auto-generated file and should not be manually edited.
+                        //   LSR Revision: %s
+                """.formatted(LSRrevisionDate);
+    }
+
+    private static void writeMultiEquiv(BufferedWriter writer) throws IOException {
+        for (String key : sortedLanguageMap2.keySet()) {
+            String[] values = sortedLanguageMap2.get(key);
+            if (values.length >= 2) {
+                writer.write(String.format(
+                        "        multiEquivsMap.put(\"%s\", new String[] {%s});"
+                        , key, generateValuesString(values)));
+                writer.newLine();
+            }
+        }
+    }
+
+    // Use for writing an equivlancy map with one value
+    private static void writeEquiv(BufferedWriter writer, String name, Map<String, String> map) throws IOException {
+        for (String key : map.keySet()) {
+            String value = map.get(key);
+            writer.write(String.format(
+                    "        %s.put(\"%s\", \"%s\");"
+                    , name, key, value));
+            writer.newLine();
+        }
+    }
+
+    private static String generateValuesString(String[] values) {
+        String outputStr = "";
+        for (int i = 0; i < values.length; i++) {
+            if (i != values.length - 1) {
+                outputStr = String.format("%s\"%s\", ", outputStr, values[i]);
+            } else {
+                outputStr = String.format("%s\"%s\"", outputStr, values[i]);
+            }
+
+        }
+        return outputStr;
+    }
 }

make/jdk/src/classes/build/tools/taglet/SealedGraph.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2022, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -145,11 +145,15 @@ public final class SealedGraph implements Taglet {
 
         static void traverse(State state, TypeElement node, Set<String> exports) {
             state.addNode(node);
-            for (TypeElement subNode : permittedSubclasses(node, exports)) {
-                if (isInPublicApi(node, exports) && isInPublicApi(subNode, exports)) {
-                    state.addEdge(node, subNode);
+            if (!(node.getModifiers().contains(Modifier.SEALED) || node.getModifiers().contains(Modifier.FINAL))) {
+                state.addNonSealedEdge(node);
+            } else {
+                for (TypeElement subNode : permittedSubclasses(node, exports)) {
+                    if (isInPublicApi(node, exports) && isInPublicApi(subNode, exports)) {
+                        state.addEdge(node, subNode);
+                    }
+                    traverse(state, subNode, exports);
                 }
-                traverse(state, subNode, exports);
             }
         }
 
@@ -158,13 +162,31 @@ public final class SealedGraph implements Taglet {
             private static final String LABEL = "label";
             private static final String TOOLTIP = "tooltip";
             private static final String LINK = "href";
-            private static final String STYLE = "style";
 
             private final TypeElement rootNode;
 
             private final StringBuilder builder;
 
-            private final Map<String, Map<String, String>> nodeStyleMap;
+            private final Map<String, Map<String, StyleItem>> nodeStyleMap;
+
+            private int nonSealedHierarchyCount = 0;
+
+            private sealed interface StyleItem {
+                String valueString();
+
+                record PlainString(String text) implements StyleItem {
+                    @Override
+                    public String valueString() {
+                        return "\"" + text + "\"";
+                    }
+                }
+                record HtmlString(String text) implements StyleItem {
+                    @Override
+                    public String valueString() {
+                        return "<" + text + ">";
+                    }
+                }
+            }
 
             public State(TypeElement rootNode) {
                 this.rootNode = rootNode;
@@ -188,13 +210,9 @@ public final class SealedGraph implements Taglet {
 
             public void addNode(TypeElement node) {
                 var styles = nodeStyleMap.computeIfAbsent(id(node), n -> new LinkedHashMap<>());
-                styles.put(LABEL, node.getSimpleName().toString());
-                styles.put(TOOLTIP, node.getQualifiedName().toString());
-                styles.put(LINK, relativeLink(node));
-                if (!(node.getModifiers().contains(Modifier.SEALED) || node.getModifiers().contains(Modifier.FINAL))) {
-                    // This indicates that the hierarchy is not closed
-                    styles.put(STYLE, "dashed");
-                }
+                styles.put(LABEL, new StyleItem.PlainString(node.getSimpleName().toString()));
+                styles.put(TOOLTIP, new StyleItem.PlainString(node.getQualifiedName().toString()));
+                styles.put(LINK, new StyleItem.PlainString(relativeLink(node)));
             }
 
             // A permitted class must be in the same package or in the same module.
@@ -223,12 +241,32 @@ public final class SealedGraph implements Taglet {
                         .append(lineSeparator());
             }
 
+            public void addNonSealedEdge(TypeElement node) {
+                // prepare open node
+                var openNodeId = "open node #" + nonSealedHierarchyCount++;
+                var styles = nodeStyleMap.computeIfAbsent(openNodeId, n -> new LinkedHashMap<>());
+                styles.put(LABEL, new StyleItem.HtmlString("<I>&lt;any&gt;</I>"));
+                styles.put(TOOLTIP, new StyleItem.PlainString("Non-sealed Hierarchy"));
+
+                // add link to parent node
+                builder.append("  ")
+                        .append('"')
+                        .append(openNodeId)
+                        .append('"')
+                        .append(" -> ")
+                        .append(quotedId(node))
+                        .append(" ")
+                        .append("[style=\"dashed\"]")
+                        .append(";")
+                        .append(lineSeparator());
+            }
+
             public String render() {
                 nodeStyleMap.forEach((nodeName, styles) -> {
                     builder.append("  ")
                             .append('"').append(nodeName).append("\" ")
                             .append(styles.entrySet().stream()
-                                    .map(e -> e.getKey() + "=\"" + e.getValue() + "\"")
+                                    .map(e -> e.getKey() + "=" + e.getValue().valueString())
                                     .collect(joining(" ", "[", "]")))
                             .append(lineSeparator());
                 });
@@ -265,16 +303,17 @@ public final class SealedGraph implements Taglet {
         }
 
         private static boolean isInPublicApi(TypeElement typeElement, Set<String> exports) {
-           return (exports.contains(packageName(typeElement.getQualifiedName().toString())) ||
-                   exports.contains(packageName(typeElement.getSuperclass().toString()))) &&
-                   typeElement.getModifiers().contains(Modifier.PUBLIC);
+            var packageName = packageName(typeElement);
+            return packageName.isPresent() && exports.contains(packageName.get()) &&
+                    typeElement.getModifiers().contains(Modifier.PUBLIC);
         }
 
-        private static String packageName(String name) {
-            int lastDot = name.lastIndexOf('.');
-            return lastDot < 0
-                    ? ""
-                    : name.substring(0, lastDot);
+        private static Optional<String> packageName(TypeElement element) {
+            return switch (element.getNestingKind()) {
+                case TOP_LEVEL -> Optional.of(((PackageElement) element.getEnclosingElement()).getQualifiedName().toString());
+                case ANONYMOUS, LOCAL -> Optional.empty();
+                case MEMBER -> packageName((TypeElement) element.getEnclosingElement());
+            };
         }
     }
 }

make/modules/java.base/Java.gmk

@@ -27,9 +27,7 @@ DISABLED_WARNINGS_java += this-escape
 
 DOCLINT += -Xdoclint:all/protected \
     '-Xdoclint/package:java.*,javax.*'
-JAVAC_FLAGS += -XDstringConcat=inline \
-    --enable-preview
-DISABLED_WARNINGS_java += preview
+JAVAC_FLAGS += -XDstringConcat=inline
 COPY += .icu .dat .spp .nrm content-types.properties \
     hijrah-config-Hijrah-umalqura_islamic-umalqura.properties
 CLEAN += intrinsic.properties

make/modules/java.base/Lib.gmk

@@ -46,6 +46,8 @@ $(eval $(call SetupJdkLibrary, BUILD_LIBNET, \
     DISABLED_WARNINGS_gcc_net_util_md.c := format-nonliteral, \
     DISABLED_WARNINGS_gcc_NetworkInterface.c := unused-function, \
     DISABLED_WARNINGS_clang_net_util_md.c := format-nonliteral, \
+    DISABLED_WARNINGS_clang_aix_DefaultProxySelector.c := deprecated-non-prototype, \
+    DISABLED_WARNINGS_clang_aix_NetworkInterface.c := gnu-pointer-arith, \
     DISABLED_WARNINGS_microsoft_InetAddress.c := 4244, \
     DISABLED_WARNINGS_microsoft_ResolverConfigurationImpl.c := 4996, \
     LDFLAGS := $(LDFLAGS_JDKLIB) \

make/modules/java.base/lib/CoreLibraries.gmk

@@ -198,6 +198,7 @@ ifeq ($(call isTargetOs, aix), true)
       OPTIMIZATION := HIGH, \
       CFLAGS := $(STATIC_LIBRARY_FLAGS) $(CFLAGS_JDKLIB) $(LIBJLI_CFLAGS) \
           $(addprefix -I, $(LIBJLI_SRC_DIRS)), \
+      DISABLED_WARNINGS_clang_aix := format-nonliteral deprecated-non-prototype, \
       ARFLAGS := $(ARFLAGS), \
       OBJECT_DIR := $(SUPPORT_OUTPUTDIR)/native/$(MODULE)/libjli_static))
 

make/modules/java.desktop/Java.gmk

@@ -68,10 +68,12 @@ EXCLUDE_FILES += \
 ifeq ($(call isTargetOs, macosx), true)
   # exclude all X11 on Mac.
   EXCLUDES += \
+      sun/awt/screencast \
       sun/awt/X11 \
       sun/awt/wl \
       sun/java2d/wl \
       sun/java2d/x11 \
+      sun/java2d/vulkan \
       sun/java2d/jules \
       sun/java2d/xr \
       com/sun/java/swing/plaf/gtk \

make/modules/java.desktop/lib/Awt2dLibraries.gmk

@@ -219,6 +219,34 @@ ifeq ($(call isTargetOs, windows)+$(ENABLE_HEADLESS_ONLY)+$(A11Y_NVDA_ANNOUNCING
   TARGETS += $(COPY_NVDA_DEPENDENCIES)
 endif
 
+# Compile Vulkan shaders
+define compile-spirv
+  $(call MakeTargetDir)
+  $(VULKAN_SHADER_COMPILER) '$(call DecodeSpace, $@)' '$(call DecodeSpace, $<)'
+endef
+spirv-name = $(strip $1).h
+
+ifeq ($(VULKAN_ENABLED), true)
+  $(eval $(call SetupCopyFiles, COMPILE_VULKAN_SHADERS, \
+      SRC := $(TOPDIR)/src/$(MODULE)/share/glsl/vulkan, \
+      FILES := $(call FindFiles, $(TOPDIR)/src/$(MODULE)/share/glsl/vulkan), \
+      DEST := $(SUPPORT_OUTPUTDIR)/headers/java.desktop/vulkan/spirv, \
+      MACRO := compile-spirv, \
+      NAME_MACRO := spirv-name, \
+  ))
+  VULKAN_SHADER_LIST = $(SUPPORT_OUTPUTDIR)/headers/java.desktop/vulkan/shader_list.h
+  $(VULKAN_SHADER_LIST): $(COMPILE_VULKAN_SHADERS)
+	> $(VULKAN_SHADER_LIST) $(NEWLINE) \
+	$(foreach f, $(patsubst $(TOPDIR)/src/$(MODULE)/share/glsl/vulkan/%,%,$(call FindFiles, $(TOPDIR)/src/$(MODULE)/share/glsl/vulkan)), \
+	  $(ECHO) SHADER_ENTRY\($(subst .,$(COMMA),$(subst /,_,$f))\) >> $(VULKAN_SHADER_LIST) $(NEWLINE) \
+	  $(ECHO) '#ifdef INCLUDE_BYTECODE' >> $(VULKAN_SHADER_LIST) $(NEWLINE) \
+	  $(ECHO) '#include "spirv/$f.h"' >> $(VULKAN_SHADER_LIST) $(NEWLINE) \
+	  $(ECHO) BYTECODE_END >> $(VULKAN_SHADER_LIST) $(NEWLINE) \
+	  $(ECHO) '#endif' >> $(VULKAN_SHADER_LIST) $(NEWLINE) \
+	)
+  $(BUILD_LIBAWT): $(VULKAN_SHADER_LIST)
+endif
+
 TARGETS += $(BUILD_LIBAWT)
 
 ################################################################################
@@ -246,6 +274,9 @@ ifeq ($(call isTargetOs, windows macosx), false)
 
     LIBAWT_XAWT_EXCLUDES := medialib debug wl vulkan
 
+    LIBPIPEWIRE_HEADER_DIRS := \
+        $(TOPDIR)/src/$(MODULE)/unix/native/libpipewire/include
+
     LIBAWT_XAWT_EXTRA_HEADER_DIRS := \
         $(LIBAWT_DEFAULT_HEADER_DIRS) \
         libawt_xawt/awt \
@@ -255,7 +286,7 @@ ifeq ($(call isTargetOs, windows macosx), false)
         common/font \
         common/java2d/opengl \
         common/java2d/x11 \
-        #
+        $(LIBPIPEWIRE_HEADER_DIRS)
 
     LIBAWT_XAWT_CFLAGS += -DXAWT -DXAWT_HACK \
         $(FONTCONFIG_CFLAGS) \
@@ -299,6 +330,14 @@ ifeq ($(call isTargetOs, windows macosx), false)
         DISABLED_WARNINGS_gcc_XToolkit.c := unused-result, \
         DISABLED_WARNINGS_gcc_XWindow.c := unused-function, \
         DISABLED_WARNINGS_gcc_keycode_cache.c := unused-function, \
+        DISABLED_WARNINGS_clang_aix := deprecated-non-prototype, \
+        DISABLED_WARNINGS_clang_aix_awt_Taskbar.c := parentheses, \
+        DISABLED_WARNINGS_clang_aix_OGLPaints.c := format-nonliteral, \
+        DISABLED_WARNINGS_clang_aix_OGLBufImgOps.c := format-nonliteral, \
+        DISABLED_WARNINGS_clang_aix_gtk2_interface.c := parentheses logical-op-parentheses, \
+        DISABLED_WARNINGS_clang_aix_gtk3_interface.c := parentheses logical-op-parentheses, \
+        DISABLED_WARNINGS_clang_aix_sun_awt_X11_GtkFileDialogPeer.c := parentheses, \
+        DISABLED_WARNINGS_clang_aix_awt_InputMethod.c := sign-compare, \
         LDFLAGS := $(LDFLAGS_JDKLIB) \
             $(call SET_SHARED_LIBRARY_ORIGIN) \
             -L$(INSTALL_LIBRARIES_HERE), \
@@ -343,6 +382,7 @@ ifeq ($(call isTargetOs, windows macosx), false)
         common/font \
         common/java2d/wl \
         common/java2d/vulkan \
+        libvmahpp \
         #
 
     # Enable 'wakefield' extension for java.awt.Robot support
@@ -390,6 +430,8 @@ ifeq ($(call isTargetOs, windows macosx), false)
         DISABLED_WARNINGS_CXX_gcc := undef, \
         DISABLED_WARNINGS_clang := parentheses format undef \
             logical-op-parentheses format-nonliteral int-conversion, \
+        DISABLED_WARNINGS_gcc_VKMemory.cpp := missing-field-initializers implicit-fallthrough parentheses, \
+        DISABLED_WARNINGS_clang_VKMemory.cpp := missing-field-initializers implicit-fallthrough parentheses, \
         LDFLAGS := $(LDFLAGS_JDKLIB) \
             $(call SET_SHARED_LIBRARY_ORIGIN) \
             -L$(INSTALL_LIBRARIES_HERE), \
@@ -513,6 +555,7 @@ ifeq ($(call isTargetOs, windows macosx), false)
       common/awt/debug \
       common/font \
       common/java2d/opengl \
+      common/java2d/vulkan \
       #
 
   LIBAWT_HEADLESS_CFLAGS := $(CUPS_CFLAGS) $(FONTCONFIG_CFLAGS) $(X_CFLAGS) \
@@ -525,6 +568,7 @@ ifeq ($(call isTargetOs, windows macosx), false)
       OPTIMIZATION := LOW, \
       CFLAGS := $(CFLAGS_JDKLIB) \
           $(LIBAWT_HEADLESS_CFLAGS), \
+      CXXFLAGS := $(CXXFLAGS_JDKLIB), \
       EXTRA_HEADER_DIRS := $(LIBAWT_HEADLESS_EXTRA_HEADER_DIRS), \
       DISABLED_WARNINGS_gcc_X11Renderer.c := unused-function, \
       DISABLED_WARNINGS_gcc_X11SurfaceData.c := unused-function, \
@@ -597,16 +641,19 @@ else
    # Early re-canonizing has to be disabled to workaround an internal XlC compiler error
    # when building libharfbuzz
    ifeq ($(call isTargetOs, aix), true)
+    ifneq ($(TOOLCHAIN_TYPE), clang)
      HARFBUZZ_CFLAGS += -qdebug=necan
+    endif
    endif
 
    # hb-ft.cc is not presently needed, and requires freetype 2.4.2 or later.
    LIBFONTMANAGER_EXCLUDE_FILES += libharfbuzz/hb-ft.cc
 
    HARFBUZZ_DISABLED_WARNINGS_gcc := missing-field-initializers strict-aliasing \
-       unused-result
+       unused-result array-bounds
    # noexcept-type required for GCC 7 builds. Not required for GCC 8+.
-   HARFBUZZ_DISABLED_WARNINGS_CXX_gcc := class-memaccess noexcept-type
+   # expansion-to-defined required for GCC 9 builds. Not required for GCC 10+.
+   HARFBUZZ_DISABLED_WARNINGS_CXX_gcc := class-memaccess noexcept-type expansion-to-defined dangling-reference
    HARFBUZZ_DISABLED_WARNINGS_clang := missing-field-initializers range-loop-analysis
    HARFBUZZ_DISABLED_WARNINGS_microsoft := 4267 4244
 
@@ -644,12 +691,10 @@ ifeq ($(call isTargetOs, windows), true)
 else ifeq ($(call isTargetOs, macosx), true)
   LIBFONTMANAGER_EXCLUDE_FILES += X11FontScaler.c \
       X11TextRenderer.c \
-      fontpath.c \
       lcdglyph.c \
       lcdglyphDW.cpp
 else
-  LIBFONTMANAGER_EXCLUDE_FILES += fontpath.c \
-      lcdglyph.c \
+  LIBFONTMANAGER_EXCLUDE_FILES += lcdglyph.c \
       lcdglyphDW.cpp
 endif
 
@@ -890,6 +935,20 @@ ifeq ($(ENABLE_HEADLESS_ONLY), false)
     endif
   endif
 
+  # The external libpng submitted in the jdk is a reduced version
+  # which does not contain .png_init_filter_functions_vsx.
+  # Therefore we need to disable PNG_POWERPC_VSX_OPT explicitly by setting
+  # it to 0. If this define is not set, it would be automatically set to 2,
+  # because
+  #   "#if defined(__PPC64__) && defined(__ALTIVEC__) && defined(__VSX__)"
+  # expands to true. This would results in the fact that
+  # .png_init_filter_functions_vsx is needed in libpng.
+  ifeq ($(call isTargetOs, aix), true)
+    ifeq ($(TOOLCHAIN_TYPE), clang)
+      LIBSPLASHSCREEN_CFLAGS += -DPNG_POWERPC_VSX_OPT=0
+    endif
+  endif
+
   ifeq ($(call isTargetOs, macosx), true)
     LIBSPLASHSCREEN_CFLAGS += -DWITH_MACOSX
 

make/modules/java.security.jgss/Lib.gmk

@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2011, 2022, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2011, 2023, Oracle and/or its affiliates. All rights reserved.
 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 #
 # This code is free software; you can redistribute it and/or modify it
@@ -32,6 +32,7 @@ $(eval $(call SetupJdkLibrary, BUILD_LIBJ2GSS, \
     OPTIMIZATION := LOW, \
     CFLAGS := $(CFLAGS_JDKLIB), \
     DISABLED_WARNINGS_gcc := undef, \
+    DISABLED_WARNINGS_clang_aix := undef, \
     LDFLAGS := $(LDFLAGS_JDKLIB) \
         $(call SET_SHARED_LIBRARY_ORIGIN), \
     LIBS := $(LIBDL), \

make/modules/java.xml/Copy.gmk

@@ -1,10 +1,12 @@
 #
-# Copyright (c) 2017, 2018, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved.
 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 #
 # This code is free software; you can redistribute it and/or modify it
 # under the terms of the GNU General Public License version 2 only, as
-# published by the Free Software Foundation.
+# published by the Free Software Foundation.  Oracle designates this
+# particular file as subject to the "Classpath" exception as provided
+# by Oracle in the LICENSE file that accompanied this code.
 #
 # This code is distributed in the hope that it will be useful, but WITHOUT
 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
@@ -20,4 +22,16 @@
 # or visit www.oracle.com if you need additional information or have any
 # questions.
 #
-exclusiveAccess.dirs=.
+
+include CopyCommon.gmk
+
+################################################################################
+
+XML_LIB_SRC := $(TOPDIR)/src/java.xml/share/conf
+
+$(CONF_DST_DIR)/jaxp.properties: $(XML_LIB_SRC)/jaxp.properties
+	$(call install-file)
+
+TARGETS := $(CONF_DST_DIR)/jaxp.properties
+
+################################################################################

make/modules/jdk.internal.le/Lib.gmk

@@ -27,14 +27,16 @@ include LibCommon.gmk
 
 ################################################################################
 
-ifeq ($(call isTargetOs, windows), true)
+ifeq ($(call isTargetOs, linux macosx windows), true)
 
   $(eval $(call SetupJdkLibrary, BUILD_LIBLE, \
       NAME := le, \
+      TOOLCHAIN := TOOLCHAIN_LINK_CXX, \
       OPTIMIZATION := LOW, \
-      CFLAGS := $(CFLAGS_JDKLIB), \
+      CFLAGS := $(CXXFLAGS_JDKLIB), \
       LDFLAGS := $(LDFLAGS_JDKLIB), \
-      LIBS := $(JDKLIB_LIBS) user32.lib, \
+      LIBS_unix := $(JDKLIB_LIBS) $(LIBCXX), \
+      LIBS_windows := $(JDKLIB_LIBS) user32.lib, \
   ))
 
   TARGETS += $(BUILD_LIBLE)

make/modules/jdk.jartool/Java.gmk

@@ -25,5 +25,3 @@
 
 DISABLED_WARNINGS_java += missing-explicit-ctor
 JAVAC_FLAGS += -XDstringConcat=inline
-JAVAC_FLAGS += --enable-preview
-DISABLED_WARNINGS_java += preview

make/modules/jdk.jdwp.agent/Lib.gmk

@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2011, 2022, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2011, 2023, Oracle and/or its affiliates. All rights reserved.
 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 #
 # This code is free software; you can redistribute it and/or modify it
@@ -30,6 +30,7 @@ include LibCommon.gmk
 $(eval $(call SetupJdkLibrary, BUILD_LIBDT_SOCKET, \
     NAME := dt_socket, \
     OPTIMIZATION := LOW, \
+    DISABLED_WARNINGS_clang_aix := missing-braces, \
     CFLAGS := $(CFLAGS_JDKLIB) $(LIBDT_SOCKET_CPPFLAGS), \
     EXTRA_HEADER_DIRS := \
         include \

make/modules/jdk.jfr/Java.gmk

@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2020, 2023, Oracle and/or its affiliates. All rights reserved.
 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 #
 # This code is free software; you can redistribute it and/or modify it
@@ -24,5 +24,5 @@
 #
 
 DISABLED_WARNINGS_java += exports
-COPY := .xsd .xml .dtd
+COPY := .xsd .xml .dtd .ini
 JAVAC_FLAGS := -XDstringConcat=inline

make/test/BuildMicrobenchmark.gmk

@@ -100,7 +100,6 @@ $(eval $(call SetupJavaCompilation, BUILD_JDK_MICROBENCHMARK, \
         --add-exports java.base/jdk.internal.classfile.attribute=ALL-UNNAMED \
         --add-exports java.base/jdk.internal.classfile.constantpool=ALL-UNNAMED \
         --add-exports java.base/jdk.internal.classfile.instruction=ALL-UNNAMED \
-        --add-exports java.base/jdk.internal.classfile.java.lang.constant=ALL-UNNAMED \
         --add-exports java.base/jdk.internal.classfile.components=ALL-UNNAMED \
         --add-exports java.base/jdk.internal.classfile.impl=ALL-UNNAMED \
         --add-exports java.base/jdk.internal.org.objectweb.asm=ALL-UNNAMED \

make/test/BuildTestLib.gmk

@@ -36,7 +36,7 @@ TEST_LIB_SUPPORT := $(SUPPORT_OUTPUTDIR)/test/lib
 
 $(eval $(call SetupJavaCompilation, BUILD_WB_JAR, \
     TARGET_RELEASE := $(TARGET_RELEASE_NEWJDK_UPGRADED), \
-    SRC := $(TEST_LIB_SOURCE_DIR)/jdk/test/whitebox/parser, \
+    SRC := $(TEST_LIB_SOURCE_DIR)/jdk/test/whitebox/, \
     BIN := $(TEST_LIB_SUPPORT)/wb_classes, \
     JAR := $(TEST_LIB_SUPPORT)/wb.jar, \
     DISABLED_WARNINGS := deprecation removal preview, \
@@ -53,7 +53,12 @@ $(eval $(call SetupJavaCompilation, BUILD_TEST_LIB_JAR, \
     HEADERS := $(TEST_LIB_SUPPORT)/test-lib_headers, \
     JAR := $(TEST_LIB_SUPPORT)/test-lib.jar, \
     DISABLED_WARNINGS := try deprecation rawtypes unchecked serial cast removal preview, \
-    JAVAC_FLAGS := --enable-preview, \
+    JAVAC_FLAGS := --add-exports java.base/sun.security.util=ALL-UNNAMED \
+        --add-exports java.base/jdk.internal.classfile=ALL-UNNAMED \
+        --add-exports java.base/jdk.internal.classfile.attribute=ALL-UNNAMED \
+        --add-exports java.base/jdk.internal.classfile.constantpool=ALL-UNNAMED \
+        --add-exports java.base/jdk.internal.module=ALL-UNNAMED \
+        --enable-preview, \
 ))
 
 TARGETS += $(BUILD_TEST_LIB_JAR)

make/test/JtregNativeHotspot.gmk

@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2015, 2022, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 #
 # This code is free software; you can redistribute it and/or modify it
@@ -99,11 +99,6 @@ NSK_GC_LOCK_JVMTI_INCLUDES := \
     -I$(VM_TESTBASE_DIR)/nsk/share/native \
     -I$(VM_TESTBASE_DIR)/nsk/share/jni
 
-NSK_GC_LOCK_MALLOC_INCLUDES := \
-    -I$(VM_TESTBASE_DIR)/nsk/share/gc/lock/malloc \
-    -I$(VM_TESTBASE_DIR)/nsk/share/native \
-    -I$(VM_TESTBASE_DIR)/nsk/share/jni
-
 NSK_GC_LOCK_JNI_INCLUDES := \
     -I$(VM_TESTBASE_DIR)/nsk/share/gc/lock/jni \
     -I$(VM_TESTBASE_DIR)/nsk/share/native \
@@ -179,8 +174,6 @@ BUILD_HOTSPOT_JTREG_LIBRARIES_CFLAGS_libnativeAndMH := $(MLVM_STRESS_INCLUDES)
 
 BUILD_HOTSPOT_JTREG_LIBRARIES_CFLAGS_libJVMTIAllocLocker := $(NSK_GC_LOCK_JVMTI_INCLUDES)
 
-BUILD_HOTSPOT_JTREG_LIBRARIES_CFLAGS_libMallocLocker := $(NSK_GC_LOCK_MALLOC_INCLUDES)
-
 BUILD_HOTSPOT_JTREG_LIBRARIES_CFLAGS_libBooleanArrayCriticalLocker := $(NSK_GC_LOCK_JNI_INCLUDES)
 BUILD_HOTSPOT_JTREG_LIBRARIES_CFLAGS_libByteArrayCriticalLocker := $(NSK_GC_LOCK_JNI_INCLUDES)
 BUILD_HOTSPOT_JTREG_LIBRARIES_CFLAGS_libCharArrayCriticalLocker := $(NSK_GC_LOCK_JNI_INCLUDES)
@@ -874,7 +867,7 @@ BUILD_HOTSPOT_JTREG_EXECUTABLES_LIBS_exesigtest := -ljvm
 
 ifeq ($(call isTargetOs, windows), true)
   BUILD_HOTSPOT_JTREG_EXECUTABLES_CFLAGS_exeFPRegs := -MT
-  BUILD_HOTSPOT_JTREG_EXCLUDE += exesigtest.c libterminatedThread.c libTestJNI.c libCompleteExit.c libTestPsig.c libnativeStack.c
+  BUILD_HOTSPOT_JTREG_EXCLUDE += exesigtest.c libterminatedThread.c libTestJNI.c libCompleteExit.c libTestPsig.c libnativeStack.c exeGetCreatedJavaVMs.c
   BUILD_HOTSPOT_JTREG_LIBRARIES_LIBS_libatExit := jvm.lib
   BUILD_HOTSPOT_JTREG_EXECUTABLES_LIBS_exedaemonDestroy := jvm.lib
 else
@@ -1516,6 +1509,7 @@ else
   BUILD_HOTSPOT_JTREG_LIBRARIES_LIBS_libatExit += -ljvm
   BUILD_HOTSPOT_JTREG_LIBRARIES_LIBS_libCompleteExit += -lpthread
   BUILD_HOTSPOT_JTREG_LIBRARIES_LIBS_libnativeStack += -lpthread
+  BUILD_HOTSPOT_JTREG_EXECUTABLES_LIBS_exeGetCreatedJavaVMs := -ljvm -lpthread
 endif
 
 ifeq ($(ASAN_ENABLED), true)

src/demo/share/java2d/J2DBench/build.xml

@@ -49,7 +49,7 @@
   <target name="compile" depends="init"
         description="compile the source " >
     <!-- Compile the java code from ${src} into ${build} -->
-    <javac debug="off" source="1.7" target="1.7" srcdir="${src}" destdir="${build}"/>
+    <javac debug="off" source="21" target="21" srcdir="${src}" destdir="${build}"/>
   </target>
 
   <target name="run" depends="dist"

src/hotspot/cpu/aarch64/aarch64.ad

@@ -664,11 +664,6 @@ reg_class method_reg(
     R12, R12_H
 );
 
-// Class for heapbase register
-reg_class heapbase_reg(
-    R27, R27_H
-);
-
 // Class for thread register
 reg_class thread_reg(
     R28, R28_H
@@ -1244,15 +1239,15 @@ source %{
     // zero, compressed klass pointers doesn't use r27 after JDK-8234794
     if (UseCompressedOops && (CompressedOops::ptrs_base() != NULL)) {
       _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
-      _NO_SPECIAL_REG_mask.SUBTRACT(_HEAPBASE_REG_mask);
-      _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_HEAPBASE_REG_mask);
+      _NO_SPECIAL_REG_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
+      _NO_SPECIAL_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
     }
 
     // r29 is not allocatable when PreserveFramePointer is on
     if (PreserveFramePointer) {
       _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
-      _NO_SPECIAL_REG_mask.SUBTRACT(_FP_REG_mask);
-      _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_FP_REG_mask);
+      _NO_SPECIAL_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
+      _NO_SPECIAL_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
     }
   }
 
@@ -3822,7 +3817,7 @@ encode %{
     Register tmp = as_Register($tmp2$$reg);
     Label cont;
     Label object_has_monitor;
-    Label no_count;
+    Label count, no_count;
 
     assert_different_registers(oop, box, tmp, disp_hdr);
 
@@ -3839,7 +3834,10 @@ encode %{
     // Check for existing monitor
     __ tbnz(disp_hdr, exact_log2(markWord::monitor_value), object_has_monitor);
 
-    if (!UseHeavyMonitors) {
+    if (LockingMode == LM_MONITOR) {
+      __ tst(oop, oop); // Set NE to indicate 'failure' -> take slow-path. We know that oop != 0.
+      __ b(cont);
+    } else if (LockingMode == LM_LEGACY) {
       // Set tmp to be (markWord of object | UNLOCK_VALUE).
       __ orr(tmp, disp_hdr, markWord::unlocked_value);
 
@@ -3867,10 +3865,12 @@ encode %{
       // displaced header in the box, which indicates that it is a recursive lock.
       __ ands(tmp/*==0?*/, disp_hdr, tmp);   // Sets flags for result
       __ str(tmp/*==0, perhaps*/, Address(box, BasicLock::displaced_header_offset_in_bytes()));
+      __ b(cont);
     } else {
-      __ tst(oop, oop); // Set NE to indicate 'failure' -> take slow-path. We know that oop != 0.
+      assert(LockingMode == LM_LIGHTWEIGHT, "must be");
+      __ fast_lock(oop, disp_hdr, tmp, rscratch1, no_count);
+      __ b(count);
     }
-    __ b(cont);
 
     // Handle existing monitor.
     __ bind(object_has_monitor);
@@ -3879,24 +3879,25 @@ encode %{
     // otherwise m->owner may contain a thread or a stack address.
     //
     // Try to CAS m->owner from NULL to current thread.
-    __ add(tmp, disp_hdr, (ObjectMonitor::owner_offset_in_bytes()-markWord::monitor_value));
+    __ add(tmp, disp_hdr, (in_bytes(ObjectMonitor::owner_offset())-markWord::monitor_value));
     __ cmpxchg(tmp, zr, rthread, Assembler::xword, /*acquire*/ true,
                /*release*/ true, /*weak*/ false, rscratch1); // Sets flags for result
 
-    // Store a non-null value into the box to avoid looking like a re-entrant
-    // lock. The fast-path monitor unlock code checks for
-    // markWord::monitor_value so use markWord::unused_mark which has the
-    // relevant bit set, and also matches ObjectSynchronizer::enter.
-    __ mov(tmp, (address)markWord::unused_mark().value());
-    __ str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
-
+    if (LockingMode != LM_LIGHTWEIGHT) {
+      // Store a non-null value into the box to avoid looking like a re-entrant
+      // lock. The fast-path monitor unlock code checks for
+      // markWord::monitor_value so use markWord::unused_mark which has the
+      // relevant bit set, and also matches ObjectSynchronizer::enter.
+      __ mov(tmp, (address)markWord::unused_mark().value());
+      __ str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
+    }
     __ br(Assembler::EQ, cont); // CAS success means locking succeeded
 
     __ cmp(rscratch1, rthread);
     __ br(Assembler::NE, cont); // Check for recursive locking
 
     // Recursive lock case
-    __ increment(Address(disp_hdr, ObjectMonitor::recursions_offset_in_bytes() - markWord::monitor_value), 1);
+    __ increment(Address(disp_hdr, in_bytes(ObjectMonitor::recursions_offset()) - markWord::monitor_value), 1);
     // flag == EQ still from the cmp above, checking if this is a reentrant lock
 
     __ bind(cont);
@@ -3904,6 +3905,7 @@ encode %{
     // flag == NE indicates failure
     __ br(Assembler::NE, no_count);
 
+    __ bind(count);
     __ increment(Address(rthread, JavaThread::held_monitor_count_offset()));
 
     __ bind(no_count);
@@ -3917,11 +3919,11 @@ encode %{
     Register tmp = as_Register($tmp2$$reg);
     Label cont;
     Label object_has_monitor;
-    Label no_count;
+    Label count, no_count;
 
     assert_different_registers(oop, box, tmp, disp_hdr);
 
-    if (!UseHeavyMonitors) {
+    if (LockingMode == LM_LEGACY) {
       // Find the lock address and load the displaced header from the stack.
       __ ldr(disp_hdr, Address(box, BasicLock::displaced_header_offset_in_bytes()));
 
@@ -3934,17 +3936,22 @@ encode %{
     __ ldr(tmp, Address(oop, oopDesc::mark_offset_in_bytes()));
     __ tbnz(tmp, exact_log2(markWord::monitor_value), object_has_monitor);
 
-    if (!UseHeavyMonitors) {
+    if (LockingMode == LM_MONITOR) {
+      __ tst(oop, oop); // Set NE to indicate 'failure' -> take slow-path. We know that oop != 0.
+      __ b(cont);
+    } else if (LockingMode == LM_LEGACY) {
       // Check if it is still a light weight lock, this is is true if we
       // see the stack address of the basicLock in the markWord of the
       // object.
 
       __ cmpxchg(oop, box, disp_hdr, Assembler::xword, /*acquire*/ false,
                  /*release*/ true, /*weak*/ false, tmp);
+      __ b(cont);
     } else {
-      __ tst(oop, oop); // Set NE to indicate 'failure' -> take slow-path. We know that oop != 0.
+      assert(LockingMode == LM_LIGHTWEIGHT, "must be");
+      __ fast_unlock(oop, tmp, box, disp_hdr, no_count);
+      __ b(count);
     }
-    __ b(cont);
 
     assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
 
@@ -3952,25 +3959,39 @@ encode %{
     __ bind(object_has_monitor);
     STATIC_ASSERT(markWord::monitor_value <= INT_MAX);
     __ add(tmp, tmp, -(int)markWord::monitor_value); // monitor
-    __ ldr(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset_in_bytes()));
+
+    if (LockingMode == LM_LIGHTWEIGHT) {
+      // If the owner is anonymous, we need to fix it -- in an outline stub.
+      Register tmp2 = disp_hdr;
+      __ ldr(tmp2, Address(tmp, ObjectMonitor::owner_offset()));
+      // We cannot use tbnz here, the target might be too far away and cannot
+      // be encoded.
+      __ tst(tmp2, (uint64_t)ObjectMonitor::ANONYMOUS_OWNER);
+      C2HandleAnonOMOwnerStub* stub = new (Compile::current()->comp_arena()) C2HandleAnonOMOwnerStub(tmp, tmp2);
+      Compile::current()->output()->add_stub(stub);
+      __ br(Assembler::NE, stub->entry());
+      __ bind(stub->continuation());
+    }
+
+    __ ldr(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset()));
 
     Label notRecursive;
     __ cbz(disp_hdr, notRecursive);
 
     // Recursive lock
     __ sub(disp_hdr, disp_hdr, 1u);
-    __ str(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset_in_bytes()));
+    __ str(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset()));
     __ cmp(disp_hdr, disp_hdr); // Sets flags for result
     __ b(cont);
 
     __ bind(notRecursive);
-    __ ldr(rscratch1, Address(tmp, ObjectMonitor::EntryList_offset_in_bytes()));
-    __ ldr(disp_hdr, Address(tmp, ObjectMonitor::cxq_offset_in_bytes()));
+    __ ldr(rscratch1, Address(tmp, ObjectMonitor::EntryList_offset()));
+    __ ldr(disp_hdr, Address(tmp, ObjectMonitor::cxq_offset()));
     __ orr(rscratch1, rscratch1, disp_hdr); // Will be 0 if both are 0.
     __ cmp(rscratch1, zr); // Sets flags for result
     __ cbnz(rscratch1, cont);
     // need a release store here
-    __ lea(tmp, Address(tmp, ObjectMonitor::owner_offset_in_bytes()));
+    __ lea(tmp, Address(tmp, ObjectMonitor::owner_offset()));
     __ stlr(zr, tmp); // set unowned
 
     __ bind(cont);
@@ -3978,6 +3999,7 @@ encode %{
     // flag == NE indicates failure
     __ br(Assembler::NE, no_count);
 
+    __ bind(count);
     __ decrement(Address(rthread, JavaThread::held_monitor_count_offset()));
 
     __ bind(no_count);
@@ -4318,7 +4340,18 @@ operand immI_positive()
 // BoolTest condition for signed compare
 operand immI_cmp_cond()
 %{
-  predicate(n->get_int() < (int)(BoolTest::unsigned_compare));
+  predicate(!Matcher::is_unsigned_booltest_pred(n->get_int()));
+  match(ConI);
+
+  op_cost(0);
+  format %{ %}
+  interface(CONST_INTER);
+%}
+
+// BoolTest condition for unsigned compare
+operand immI_cmpU_cond()
+%{
+  predicate(Matcher::is_unsigned_booltest_pred(n->get_int()));
   match(ConI);
 
   op_cost(0);
@@ -4425,6 +4458,28 @@ operand immI19()
   interface(CONST_INTER);
 %}
 
+// 5 bit signed integer
+operand immI5()
+%{
+  predicate(Assembler::is_simm(n->get_int(), 5));
+  match(ConI);
+
+  op_cost(0);
+  format %{ %}
+  interface(CONST_INTER);
+%}
+
+// 7 bit unsigned integer
+operand immIU7()
+%{
+  predicate(Assembler::is_uimm(n->get_int(), 7));
+  match(ConI);
+
+  op_cost(0);
+  format %{ %}
+  interface(CONST_INTER);
+%}
+
 // 12 bit unsigned offset -- for base plus immediate loads
 operand immIU12()
 %{
@@ -4567,6 +4622,28 @@ operand immLoffset16()
   interface(CONST_INTER);
 %}
 
+// 5 bit signed long integer
+operand immL5()
+%{
+  predicate(Assembler::is_simm(n->get_long(), 5));
+  match(ConL);
+
+  op_cost(0);
+  format %{ %}
+  interface(CONST_INTER);
+%}
+
+// 7 bit unsigned long integer
+operand immLU7()
+%{
+  predicate(Assembler::is_uimm(n->get_long(), 7));
+  match(ConL);
+
+  op_cost(0);
+  format %{ %}
+  interface(CONST_INTER);
+%}
+
 // 8 bit signed value.
 operand immI8()
 %{
@@ -5238,17 +5315,6 @@ operand iRegNNoSp()
   interface(REG_INTER);
 %}
 
-// heap base register -- used for encoding immN0
-
-operand iRegIHeapbase()
-%{
-  constraint(ALLOC_IN_RC(heapbase_reg));
-  match(RegI);
-  op_cost(0);
-  format %{ %}
-  interface(REG_INTER);
-%}
-
 // Float Register
 // Float register operands
 operand vRegF()
@@ -14952,42 +15018,6 @@ instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
   ins_pipe(ialu_reg);
 %}
 
-instruct convI2B(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
-%{
-  match(Set dst (Conv2B src));
-  effect(KILL cr);
-
-  format %{
-    "cmpw $src, zr\n\t"
-    "cset $dst, ne"
-  %}
-
-  ins_encode %{
-    __ cmpw(as_Register($src$$reg), zr);
-    __ cset(as_Register($dst$$reg), Assembler::NE);
-  %}
-
-  ins_pipe(ialu_reg);
-%}
-
-instruct convP2B(iRegINoSp dst, iRegP src, rFlagsReg cr)
-%{
-  match(Set dst (Conv2B src));
-  effect(KILL cr);
-
-  format %{
-    "cmp  $src, zr\n\t"
-    "cset $dst, ne"
-  %}
-
-  ins_encode %{
-    __ cmp(as_Register($src$$reg), zr);
-    __ cset(as_Register($dst$$reg), Assembler::NE);
-  %}
-
-  ins_pipe(ialu_reg);
-%}
-
 instruct convD2F_reg(vRegF dst, vRegD src) %{
   match(Set dst (ConvD2F src));
 
@@ -17023,14 +17053,17 @@ instruct string_compareUU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI
 %}
 
 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
-       iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
-       iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
+                          iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
+                          iRegINoSp tmp3, iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
+                          vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
 %{
   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
-         TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
-  format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU)" %}
+         TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
+         TEMP vtmp0, TEMP vtmp1, KILL cr);
+  format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU) "
+            "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
 
   ins_encode %{
     __ string_indexof($str1$$Register, $str2$$Register,
@@ -17044,14 +17077,17 @@ instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2
 %}
 
 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
-       iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
-       iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
+                          iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
+                          iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
+                          vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
 %{
   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
-         TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
-  format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL)" %}
+         TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
+         TEMP vtmp0, TEMP vtmp1, KILL cr);
+  format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL) "
+            "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
 
   ins_encode %{
     __ string_indexof($str1$$Register, $str2$$Register,
@@ -17065,14 +17101,17 @@ instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2
 %}
 
 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
-       iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
-       iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
+                          iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,iRegINoSp tmp3,
+                          iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
+                          vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
 %{
   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
-         TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
-  format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL)" %}
+         TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5,
+         TEMP tmp6, TEMP vtmp0, TEMP vtmp1, KILL cr);
+  format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL) "
+            "# KILL $str1 cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
 
   ins_encode %{
     __ string_indexof($str1$$Register, $str2$$Register,
@@ -17086,14 +17125,15 @@ instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2
 %}
 
 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
-                 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
-                 iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
+                              immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
+                              iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
 %{
   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
-  format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU)" %}
+  format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU) "
+            "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
 
   ins_encode %{
     int icnt2 = (int)$int_cnt2$$constant;
@@ -17107,14 +17147,15 @@ instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
 %}
 
 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
-                 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
-                 iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
+                              immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
+                              iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
 %{
   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
-  format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL)" %}
+  format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL) "
+            "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
 
   ins_encode %{
     int icnt2 = (int)$int_cnt2$$constant;
@@ -17128,14 +17169,15 @@ instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
 %}
 
 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
-                 immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
-                 iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
+                              immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
+                              iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
 %{
   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
-  format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL)" %}
+  format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL) "
+            "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
 
   ins_encode %{
     int icnt2 = (int)$int_cnt2$$constant;
@@ -17252,13 +17294,17 @@ instruct string_equalsU(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
 
 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
                        iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
+                       vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
+                       vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
                        iRegP_R10 tmp, rFlagsReg cr)
 %{
   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
   match(Set result (AryEq ary1 ary2));
-  effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
+  effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
+         TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
+         TEMP vtmp6, TEMP vtmp7, KILL cr);
 
-  format %{ "Array Equals $ary1,ary2 -> $result    // KILL $tmp" %}
+  format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
   ins_encode %{
     address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
                                    $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
@@ -17273,13 +17319,17 @@ instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
 
 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
                        iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
+                       vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
+                       vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
                        iRegP_R10 tmp, rFlagsReg cr)
 %{
   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
   match(Set result (AryEq ary1 ary2));
-  effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
+  effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
+         TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
+         TEMP vtmp6, TEMP vtmp7, KILL cr);
 
-  format %{ "Array Equals $ary1,ary2 -> $result    // KILL $tmp" %}
+  format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
   ins_encode %{
     address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
                                    $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
@@ -17309,36 +17359,39 @@ instruct count_positives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg
 
 // fast char[] to byte[] compression
 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
-                         vRegD_V0 tmp1, vRegD_V1 tmp2,
-                         vRegD_V2 tmp3, vRegD_V3 tmp4,
+                         vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
+                         vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
                          iRegI_R0 result, rFlagsReg cr)
 %{
   match(Set result (StrCompressedCopy src (Binary dst len)));
-  effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4,
+  effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
          USE_KILL src, USE_KILL dst, USE len, KILL cr);
 
-  format %{ "String Compress $src,$dst,$len -> $result  // KILL $src,$dst" %}
+  format %{ "String Compress $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
   ins_encode %{
     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
-                           $result$$Register,
-                           $tmp1$$FloatRegister, $tmp2$$FloatRegister,
-                           $tmp3$$FloatRegister, $tmp4$$FloatRegister);
+                           $result$$Register, $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
+                           $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
+                           $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
   %}
   ins_pipe(pipe_slow);
 %}
 
 // fast byte[] to char[] inflation
-instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len,
-                        vRegD_V0 tmp1, vRegD_V1 tmp2, vRegD_V2 tmp3, iRegP_R3 tmp4, rFlagsReg cr)
+instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len, iRegP_R3 tmp,
+                        vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
+                        vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, rFlagsReg cr)
 %{
   match(Set dummy (StrInflatedCopy src (Binary dst len)));
-  effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
+  effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3,
+         TEMP vtmp4, TEMP vtmp5, TEMP vtmp6, TEMP tmp,
+         USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
 
-  format %{ "String Inflate $src,$dst    // KILL $tmp1, $tmp2" %}
+  format %{ "String Inflate $src,$dst # KILL $tmp $src $dst $len V0-V6 cr" %}
   ins_encode %{
     address tpc = __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
-                                        $tmp1$$FloatRegister, $tmp2$$FloatRegister,
-                                        $tmp3$$FloatRegister, $tmp4$$Register);
+                                        $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
+                                        $vtmp2$$FloatRegister, $tmp$$Register);
     if (tpc == NULL) {
       ciEnv::current()->record_failure("CodeCache is full");
       return;
@@ -17349,41 +17402,43 @@ instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len
 
 // encode char[] to byte[] in ISO_8859_1
 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
-                          vRegD_V0 vtmp0, vRegD_V1 vtmp1,
-                          vRegD_V2 vtmp2, vRegD_V3 vtmp3,
+                          vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
+                          vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
                           iRegI_R0 result, rFlagsReg cr)
 %{
   predicate(!((EncodeISOArrayNode*)n)->is_ascii());
   match(Set result (EncodeISOArray src (Binary dst len)));
-  effect(USE_KILL src, USE_KILL dst, USE len,
-         KILL vtmp0, KILL vtmp1, KILL vtmp2, KILL vtmp3, KILL cr);
+  effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
+         KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
 
-  format %{ "Encode ISO array $src,$dst,$len -> $result" %}
+  format %{ "Encode ISO array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
   ins_encode %{
     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
                         $result$$Register, false,
                         $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
-                        $vtmp2$$FloatRegister, $vtmp3$$FloatRegister);
+                        $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
+                        $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
   %}
   ins_pipe(pipe_class_memory);
 %}
 
 instruct encode_ascii_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
-                            vRegD_V0 vtmp0, vRegD_V1 vtmp1,
-                            vRegD_V2 vtmp2, vRegD_V3 vtmp3,
+                            vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
+                            vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
                             iRegI_R0 result, rFlagsReg cr)
 %{
   predicate(((EncodeISOArrayNode*)n)->is_ascii());
   match(Set result (EncodeISOArray src (Binary dst len)));
-  effect(USE_KILL src, USE_KILL dst, USE len,
-         KILL vtmp0, KILL vtmp1, KILL vtmp2, KILL vtmp3, KILL cr);
+  effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
+         KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
 
-  format %{ "Encode ASCII array $src,$dst,$len -> $result" %}
+  format %{ "Encode ASCII array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
   ins_encode %{
     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
                         $result$$Register, true,
                         $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
-                        $vtmp2$$FloatRegister, $vtmp3$$FloatRegister);
+                        $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
+                        $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
   %}
   ins_pipe(pipe_class_memory);
 %}

src/hotspot/cpu/aarch64/aarch64_vector.ad

@@ -315,10 +315,6 @@ source %{
     }
   }
 
-  const bool Matcher::vector_needs_load_shuffle(BasicType elem_bt, int vlen) {
-    return false;
-  }
-
   // Assert that the given node is not a variable shift.
   bool assert_not_var_shift(const Node* n) {
     assert(!n->as_ShiftV()->is_var_shift(), "illegal variable shift");
@@ -5220,6 +5216,118 @@ instruct vmaskcmp_sve(pReg dst, vReg src1, vReg src2, immI cond, rFlagsReg cr) %
   ins_pipe(pipe_slow);
 %}
 
+instruct vmaskcmp_immB_sve(pReg dst, vReg src, immI5 imm, immI_cmp_cond cond, rFlagsReg cr) %{
+  predicate(UseSVE > 0);
+  match(Set dst (VectorMaskCmp (Binary src (ReplicateB imm)) cond));
+  effect(KILL cr);
+  format %{ "vmaskcmp_immB_sve $dst, $src, $imm, $cond\t# KILL cr" %}
+  ins_encode %{
+    Assembler::Condition condition = to_assembler_cond((BoolTest::mask)$cond$$constant);
+    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
+    assert(length_in_bytes == MaxVectorSize, "invalid vector length");
+    __ sve_cmp(condition, $dst$$PRegister, __ B, ptrue, $src$$FloatRegister, (int)$imm$$constant);
+  %}
+  ins_pipe(pipe_slow);
+%}
+
+instruct vmaskcmpU_immB_sve(pReg dst, vReg src, immIU7 imm, immI_cmpU_cond cond, rFlagsReg cr) %{
+  predicate(UseSVE > 0);
+  match(Set dst (VectorMaskCmp (Binary src (ReplicateB imm)) cond));
+  effect(KILL cr);
+  format %{ "vmaskcmpU_immB_sve $dst, $src, $imm, $cond\t# KILL cr" %}
+  ins_encode %{
+    Assembler::Condition condition = to_assembler_cond((BoolTest::mask)$cond$$constant);
+    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
+    assert(length_in_bytes == MaxVectorSize, "invalid vector length");
+    __ sve_cmp(condition, $dst$$PRegister, __ B, ptrue, $src$$FloatRegister, (int)$imm$$constant);
+  %}
+  ins_pipe(pipe_slow);
+%}
+
+instruct vmaskcmp_immS_sve(pReg dst, vReg src, immI5 imm, immI_cmp_cond cond, rFlagsReg cr) %{
+  predicate(UseSVE > 0);
+  match(Set dst (VectorMaskCmp (Binary src (ReplicateS imm)) cond));
+  effect(KILL cr);
+  format %{ "vmaskcmp_immS_sve $dst, $src, $imm, $cond\t# KILL cr" %}
+  ins_encode %{
+    Assembler::Condition condition = to_assembler_cond((BoolTest::mask)$cond$$constant);
+    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
+    assert(length_in_bytes == MaxVectorSize, "invalid vector length");
+    __ sve_cmp(condition, $dst$$PRegister, __ H, ptrue, $src$$FloatRegister, (int)$imm$$constant);
+  %}
+  ins_pipe(pipe_slow);
+%}
+
+instruct vmaskcmpU_immS_sve(pReg dst, vReg src, immIU7 imm, immI_cmpU_cond cond, rFlagsReg cr) %{
+  predicate(UseSVE > 0);
+  match(Set dst (VectorMaskCmp (Binary src (ReplicateS imm)) cond));
+  effect(KILL cr);
+  format %{ "vmaskcmpU_immS_sve $dst, $src, $imm, $cond\t# KILL cr" %}
+  ins_encode %{
+    Assembler::Condition condition = to_assembler_cond((BoolTest::mask)$cond$$constant);
+    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
+    assert(length_in_bytes == MaxVectorSize, "invalid vector length");
+    __ sve_cmp(condition, $dst$$PRegister, __ H, ptrue, $src$$FloatRegister, (int)$imm$$constant);
+  %}
+  ins_pipe(pipe_slow);
+%}
+
+instruct vmaskcmp_immI_sve(pReg dst, vReg src, immI5 imm, immI_cmp_cond cond, rFlagsReg cr) %{
+  predicate(UseSVE > 0);
+  match(Set dst (VectorMaskCmp (Binary src (ReplicateI imm)) cond));
+  effect(KILL cr);
+  format %{ "vmaskcmp_immI_sve $dst, $src, $imm, $cond\t# KILL cr" %}
+  ins_encode %{
+    Assembler::Condition condition = to_assembler_cond((BoolTest::mask)$cond$$constant);
+    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
+    assert(length_in_bytes == MaxVectorSize, "invalid vector length");
+    __ sve_cmp(condition, $dst$$PRegister, __ S, ptrue, $src$$FloatRegister, (int)$imm$$constant);
+  %}
+  ins_pipe(pipe_slow);
+%}
+
+instruct vmaskcmpU_immI_sve(pReg dst, vReg src, immIU7 imm, immI_cmpU_cond cond, rFlagsReg cr) %{
+  predicate(UseSVE > 0);
+  match(Set dst (VectorMaskCmp (Binary src (ReplicateI imm)) cond));
+  effect(KILL cr);
+  format %{ "vmaskcmpU_immI_sve $dst, $src, $imm, $cond\t# KILL cr" %}
+  ins_encode %{
+    Assembler::Condition condition = to_assembler_cond((BoolTest::mask)$cond$$constant);
+    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
+    assert(length_in_bytes == MaxVectorSize, "invalid vector length");
+    __ sve_cmp(condition, $dst$$PRegister, __ S, ptrue, $src$$FloatRegister, (int)$imm$$constant);
+  %}
+  ins_pipe(pipe_slow);
+%}
+
+instruct vmaskcmp_immL_sve(pReg dst, vReg src, immL5 imm, immI_cmp_cond cond, rFlagsReg cr) %{
+  predicate(UseSVE > 0);
+  match(Set dst (VectorMaskCmp (Binary src (ReplicateL imm)) cond));
+  effect(KILL cr);
+  format %{ "vmaskcmp_immL_sve $dst, $src, $imm, $cond\t# KILL cr" %}
+  ins_encode %{
+    Assembler::Condition condition = to_assembler_cond((BoolTest::mask)$cond$$constant);
+    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
+    assert(length_in_bytes == MaxVectorSize, "invalid vector length");
+    __ sve_cmp(condition, $dst$$PRegister, __ D, ptrue, $src$$FloatRegister, (int)$imm$$constant);
+  %}
+  ins_pipe(pipe_slow);
+%}
+
+instruct vmaskcmpU_immL_sve(pReg dst, vReg src, immLU7 imm, immI_cmpU_cond cond, rFlagsReg cr) %{
+  predicate(UseSVE > 0);
+  match(Set dst (VectorMaskCmp (Binary src (ReplicateL imm)) cond));
+  effect(KILL cr);
+  format %{ "vmaskcmpU_immL_sve $dst, $src, $imm, $cond\t# KILL cr" %}
+  ins_encode %{
+    Assembler::Condition condition = to_assembler_cond((BoolTest::mask)$cond$$constant);
+    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
+    assert(length_in_bytes == MaxVectorSize, "invalid vector length");
+    __ sve_cmp(condition, $dst$$PRegister, __ D, ptrue, $src$$FloatRegister, (int)$imm$$constant);
+  %}
+  ins_pipe(pipe_slow);
+%}
+
 instruct vmaskcmp_masked(pReg dst, vReg src1, vReg src2, immI cond,
                          pRegGov pg, rFlagsReg cr) %{
   predicate(UseSVE > 0);
@@ -5400,6 +5508,30 @@ instruct vmask_truecount_sve(iRegINoSp dst, pReg src) %{
   ins_pipe(pipe_slow);
 %}
 
+// Combined rule for VectorMaskTrueCount (VectorStoreMask) when the vector element type is not T_BYTE.
+
+instruct vstoremask_truecount_neon(iRegINoSp dst, vReg src, immI_gt_1 size, vReg vtmp) %{
+  match(Set dst (VectorMaskTrueCount (VectorStoreMask src size)));
+  effect(TEMP vtmp);
+  format %{ "vstoremask_truecount_neon $dst, $src\t# KILL $vtmp" %}
+  ins_encode %{
+    // Input "src" is a vector mask represented as lanes with
+    // 0/-1 as element values.
+    uint esize = (uint)$size$$constant;
+    uint length_in_bytes = Matcher::vector_length_in_bytes(this, $src);
+    Assembler::SIMD_Arrangement arrangement = Assembler::esize2arrangement(esize,
+                                                                           /* isQ */ length_in_bytes == 16);
+    if (arrangement == __ T2D || arrangement == __ T2S) {
+      __ addpv($vtmp$$FloatRegister, arrangement, $src$$FloatRegister, $src$$FloatRegister);
+    } else {
+      __ addv($vtmp$$FloatRegister, arrangement, $src$$FloatRegister);
+    }
+    __ smov($dst$$Register, $vtmp$$FloatRegister, __ B, 0);
+    __ neg($dst$$Register, $dst$$Register);
+  %}
+  ins_pipe(pipe_slow);
+%}
+
 // first true
 
 instruct vmask_firsttrue_lt8e(iRegINoSp dst, vReg src, rFlagsReg cr) %{
@@ -5880,49 +6012,6 @@ instruct vblend_sve(vReg dst, vReg src1, vReg src2, pReg pg) %{
   ins_pipe(pipe_slow);
 %}
 
-// ------------------------- Vector conditional move --------------------------
-
-instruct vcmove_neon(vReg dst, vReg src1, vReg src2, immI cond, cmpOp copnd) %{
-  predicate(UseSVE == 0 ||
-            (VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n)) &&
-             n->in(1)->in(2)->get_int() != BoolTest::ne));
-  match(Set dst (CMoveVF (Binary copnd cond) (Binary src1 src2)));
-  match(Set dst (CMoveVD (Binary copnd cond) (Binary src1 src2)));
-  effect(TEMP_DEF dst);
-  format %{ "vcmove_neon.$copnd $dst, $src1, $src2\t# vector conditional move fp" %}
-  ins_encode %{
-    Assembler::Condition condition = to_assembler_cond((BoolTest::mask)$cond$$constant);
-    BasicType bt = Matcher::vector_element_basic_type(this);
-    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
-    assert(length_in_bytes == 8 || length_in_bytes == 16, "must be");
-    __ neon_compare($dst$$FloatRegister, bt, $src1$$FloatRegister,
-                    $src2$$FloatRegister, condition, /* isQ */ length_in_bytes == 16);
-    __ bsl($dst$$FloatRegister, length_in_bytes == 16 ? __ T16B : __ T8B,
-           $src2$$FloatRegister, $src1$$FloatRegister);
-  %}
-  ins_pipe(pipe_slow);
-%}
-
-instruct vcmove_sve(vReg dst, vReg src1, vReg src2, immI cond, cmpOp copnd, pRegGov pgtmp) %{
-  predicate(!VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n)) ||
-            (UseSVE > 0 && n->in(1)->in(2)->get_int() == BoolTest::ne));
-  match(Set dst (CMoveVF (Binary copnd cond) (Binary src1 src2)));
-  match(Set dst (CMoveVD (Binary copnd cond) (Binary src1 src2)));
-  effect(TEMP pgtmp);
-  format %{ "vcmove_sve.$copnd $dst, $src1, $src2\t# vector conditional move fp. KILL $pgtmp" %}
-  ins_encode %{
-    assert(UseSVE > 0, "must be sve");
-    Assembler::Condition condition = to_assembler_cond((BoolTest::mask)$cond$$constant);
-    BasicType bt = Matcher::vector_element_basic_type(this);
-    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
-    __ sve_compare($pgtmp$$PRegister, bt, ptrue, $src1$$FloatRegister,
-                   $src2$$FloatRegister, condition);
-    __ sve_sel($dst$$FloatRegister, __ elemType_to_regVariant(bt),
-               $pgtmp$$PRegister, $src2$$FloatRegister, $src1$$FloatRegister);
-  %}
-  ins_pipe(pipe_slow);
-%}
-
 // ------------------------------ Vector round ---------------------------------
 
 // vector Math.round
@@ -6069,6 +6158,41 @@ instruct vtest_alltrue_sve(rFlagsReg cr, pReg src1, pReg src2, pReg ptmp) %{
   ins_pipe(pipe_slow);
 %}
 
+// ------------------------------ Vector shuffle -------------------------------
+
+instruct loadshuffle(vReg dst, vReg src) %{
+  match(Set dst (VectorLoadShuffle src));
+  format %{ "loadshuffle $dst, $src" %}
+  ins_encode %{
+    BasicType bt = Matcher::vector_element_basic_type(this);
+    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
+    if (bt == T_BYTE) {
+      if ($dst$$FloatRegister != $src$$FloatRegister) {
+        if (VM_Version::use_neon_for_vector(length_in_bytes)) {
+          __ orr($dst$$FloatRegister, length_in_bytes == 16 ? __ T16B : __ T8B,
+                 $src$$FloatRegister, $src$$FloatRegister);
+        } else {
+          assert(UseSVE > 0, "must be sve");
+          __ sve_orr($dst$$FloatRegister, $src$$FloatRegister, $src$$FloatRegister);
+        }
+      }
+    } else {
+      if (VM_Version::use_neon_for_vector(length_in_bytes)) {
+        // 4S/8S, 4I, 4F
+        __ uxtl($dst$$FloatRegister, __ T8H, $src$$FloatRegister, __ T8B);
+        if (type2aelembytes(bt) == 4) {
+          __ uxtl($dst$$FloatRegister, __ T4S, $dst$$FloatRegister, __ T4H);
+        }
+      } else {
+        assert(UseSVE > 0, "must be sve");
+        __ sve_vector_extend($dst$$FloatRegister,  __ elemType_to_regVariant(bt),
+                             $src$$FloatRegister, __ B);
+      }
+    }
+  %}
+  ins_pipe(pipe_slow);
+%}
+
 // ------------------------------ Vector rearrange -----------------------------
 
 // Here is an example that rearranges a NEON vector with 4 ints:
@@ -6091,7 +6215,6 @@ instruct vtest_alltrue_sve(rFlagsReg cr, pReg src1, pReg src2, pReg ptmp) %{
 //   need to lookup 2/4 bytes as a group. For VectorRearrange long, we use bsl
 //   to implement rearrange.
 
-// Maybe move the shuffle preparation to VectorLoadShuffle
 instruct rearrange_HS_neon(vReg dst, vReg src, vReg shuffle, vReg tmp1, vReg tmp2) %{
   predicate(UseSVE == 0 &&
             (Matcher::vector_element_basic_type(n) == T_SHORT ||

src/hotspot/cpu/aarch64/aarch64_vector_ad.m4

@@ -305,10 +305,6 @@ source %{
     }
   }
 
-  const bool Matcher::vector_needs_load_shuffle(BasicType elem_bt, int vlen) {
-    return false;
-  }
-
   // Assert that the given node is not a variable shift.
   bool assert_not_var_shift(const Node* n) {
     assert(!n->as_ShiftV()->is_var_shift(), "illegal variable shift");
@@ -3616,6 +3612,31 @@ instruct vmaskcmp_sve(pReg dst, vReg src1, vReg src2, immI cond, rFlagsReg cr) %
   %}
   ins_pipe(pipe_slow);
 %}
+dnl
+dnl VMASKCMP_SVE_IMM($1          , $2          , $3      , $4            )
+dnl VMASKCMP_SVE_IMM(element_size, element_type, type_imm, type_condition)
+define(`VMASKCMP_SVE_IMM', `
+instruct vmask$4_imm$2_sve(pReg dst, vReg src, $3 imm, immI_$4_cond cond, rFlagsReg cr) %{
+  predicate(UseSVE > 0);
+  match(Set dst (VectorMaskCmp (Binary src (Replicate$2 imm)) cond));
+  effect(KILL cr);
+  format %{ "vmask$4_imm$2_sve $dst, $src, $imm, $cond\t# KILL cr" %}
+  ins_encode %{
+    Assembler::Condition condition = to_assembler_cond((BoolTest::mask)$cond$$constant);
+    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
+    assert(length_in_bytes == MaxVectorSize, "invalid vector length");
+    __ sve_cmp(condition, $dst$$PRegister, __ $1, ptrue, $src$$FloatRegister, (int)$imm$$constant);
+  %}
+  ins_pipe(pipe_slow);
+%}')dnl
+VMASKCMP_SVE_IMM(B, B, immI5, cmp)
+VMASKCMP_SVE_IMM(B, B, immIU7, cmpU)
+VMASKCMP_SVE_IMM(H, S, immI5, cmp)
+VMASKCMP_SVE_IMM(H, S, immIU7, cmpU)
+VMASKCMP_SVE_IMM(S, I, immI5, cmp)
+VMASKCMP_SVE_IMM(S, I, immIU7, cmpU)
+VMASKCMP_SVE_IMM(D, L, immL5, cmp)
+VMASKCMP_SVE_IMM(D, L, immLU7, cmpU)
 
 instruct vmaskcmp_masked(pReg dst, vReg src1, vReg src2, immI cond,
                          pRegGov pg, rFlagsReg cr) %{
@@ -3797,6 +3818,30 @@ instruct vmask_truecount_sve(iRegINoSp dst, pReg src) %{
   ins_pipe(pipe_slow);
 %}
 
+// Combined rule for VectorMaskTrueCount (VectorStoreMask) when the vector element type is not T_BYTE.
+
+instruct vstoremask_truecount_neon(iRegINoSp dst, vReg src, immI_gt_1 size, vReg vtmp) %{
+  match(Set dst (VectorMaskTrueCount (VectorStoreMask src size)));
+  effect(TEMP vtmp);
+  format %{ "vstoremask_truecount_neon $dst, $src\t# KILL $vtmp" %}
+  ins_encode %{
+    // Input "src" is a vector mask represented as lanes with
+    // 0/-1 as element values.
+    uint esize = (uint)$size$$constant;
+    uint length_in_bytes = Matcher::vector_length_in_bytes(this, $src);
+    Assembler::SIMD_Arrangement arrangement = Assembler::esize2arrangement(esize,
+                                                                           /* isQ */ length_in_bytes == 16);
+    if (arrangement == __ T2D || arrangement == __ T2S) {
+      __ addpv($vtmp$$FloatRegister, arrangement, $src$$FloatRegister, $src$$FloatRegister);
+    } else {
+      __ addv($vtmp$$FloatRegister, arrangement, $src$$FloatRegister);
+    }
+    __ smov($dst$$Register, $vtmp$$FloatRegister, __ B, 0);
+    __ neg($dst$$Register, $dst$$Register);
+  %}
+  ins_pipe(pipe_slow);
+%}
+
 // first true
 
 instruct vmask_firsttrue_lt8e(iRegINoSp dst, vReg src, rFlagsReg cr) %{
@@ -4233,49 +4278,6 @@ instruct vblend_sve(vReg dst, vReg src1, vReg src2, pReg pg) %{
   ins_pipe(pipe_slow);
 %}
 
-// ------------------------- Vector conditional move --------------------------
-
-instruct vcmove_neon(vReg dst, vReg src1, vReg src2, immI cond, cmpOp copnd) %{
-  predicate(UseSVE == 0 ||
-            (VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n)) &&
-             n->in(1)->in(2)->get_int() != BoolTest::ne));
-  match(Set dst (CMoveVF (Binary copnd cond) (Binary src1 src2)));
-  match(Set dst (CMoveVD (Binary copnd cond) (Binary src1 src2)));
-  effect(TEMP_DEF dst);
-  format %{ "vcmove_neon.$copnd $dst, $src1, $src2\t# vector conditional move fp" %}
-  ins_encode %{
-    Assembler::Condition condition = to_assembler_cond((BoolTest::mask)$cond$$constant);
-    BasicType bt = Matcher::vector_element_basic_type(this);
-    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
-    assert(length_in_bytes == 8 || length_in_bytes == 16, "must be");
-    __ neon_compare($dst$$FloatRegister, bt, $src1$$FloatRegister,
-                    $src2$$FloatRegister, condition, /* isQ */ length_in_bytes == 16);
-    __ bsl($dst$$FloatRegister, length_in_bytes == 16 ? __ T16B : __ T8B,
-           $src2$$FloatRegister, $src1$$FloatRegister);
-  %}
-  ins_pipe(pipe_slow);
-%}
-
-instruct vcmove_sve(vReg dst, vReg src1, vReg src2, immI cond, cmpOp copnd, pRegGov pgtmp) %{
-  predicate(!VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n)) ||
-            (UseSVE > 0 && n->in(1)->in(2)->get_int() == BoolTest::ne));
-  match(Set dst (CMoveVF (Binary copnd cond) (Binary src1 src2)));
-  match(Set dst (CMoveVD (Binary copnd cond) (Binary src1 src2)));
-  effect(TEMP pgtmp);
-  format %{ "vcmove_sve.$copnd $dst, $src1, $src2\t# vector conditional move fp. KILL $pgtmp" %}
-  ins_encode %{
-    assert(UseSVE > 0, "must be sve");
-    Assembler::Condition condition = to_assembler_cond((BoolTest::mask)$cond$$constant);
-    BasicType bt = Matcher::vector_element_basic_type(this);
-    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
-    __ sve_compare($pgtmp$$PRegister, bt, ptrue, $src1$$FloatRegister,
-                   $src2$$FloatRegister, condition);
-    __ sve_sel($dst$$FloatRegister, __ elemType_to_regVariant(bt),
-               $pgtmp$$PRegister, $src2$$FloatRegister, $src1$$FloatRegister);
-  %}
-  ins_pipe(pipe_slow);
-%}
-
 // ------------------------------ Vector round ---------------------------------
 
 // vector Math.round
@@ -4422,6 +4424,41 @@ instruct vtest_alltrue_sve(rFlagsReg cr, pReg src1, pReg src2, pReg ptmp) %{
   ins_pipe(pipe_slow);
 %}
 
+// ------------------------------ Vector shuffle -------------------------------
+
+instruct loadshuffle(vReg dst, vReg src) %{
+  match(Set dst (VectorLoadShuffle src));
+  format %{ "loadshuffle $dst, $src" %}
+  ins_encode %{
+    BasicType bt = Matcher::vector_element_basic_type(this);
+    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
+    if (bt == T_BYTE) {
+      if ($dst$$FloatRegister != $src$$FloatRegister) {
+        if (VM_Version::use_neon_for_vector(length_in_bytes)) {
+          __ orr($dst$$FloatRegister, length_in_bytes == 16 ? __ T16B : __ T8B,
+                 $src$$FloatRegister, $src$$FloatRegister);
+        } else {
+          assert(UseSVE > 0, "must be sve");
+          __ sve_orr($dst$$FloatRegister, $src$$FloatRegister, $src$$FloatRegister);
+        }
+      }
+    } else {
+      if (VM_Version::use_neon_for_vector(length_in_bytes)) {
+        // 4S/8S, 4I, 4F
+        __ uxtl($dst$$FloatRegister, __ T8H, $src$$FloatRegister, __ T8B);
+        if (type2aelembytes(bt) == 4) {
+          __ uxtl($dst$$FloatRegister, __ T4S, $dst$$FloatRegister, __ T4H);
+        }
+      } else {
+        assert(UseSVE > 0, "must be sve");
+        __ sve_vector_extend($dst$$FloatRegister,  __ elemType_to_regVariant(bt),
+                             $src$$FloatRegister, __ B);
+      }
+    }
+  %}
+  ins_pipe(pipe_slow);
+%}
+
 // ------------------------------ Vector rearrange -----------------------------
 
 // Here is an example that rearranges a NEON vector with 4 ints:
@@ -4444,7 +4481,6 @@ instruct vtest_alltrue_sve(rFlagsReg cr, pReg src1, pReg src2, pReg ptmp) %{
 //   need to lookup 2/4 bytes as a group. For VectorRearrange long, we use bsl
 //   to implement rearrange.
 
-// Maybe move the shuffle preparation to VectorLoadShuffle
 instruct rearrange_HS_neon(vReg dst, vReg src, vReg shuffle, vReg tmp1, vReg tmp2) %{
   predicate(UseSVE == 0 &&
             (Matcher::vector_element_basic_type(n) == T_SHORT ||

src/hotspot/cpu/aarch64/assembler_aarch64.cpp

@@ -102,7 +102,7 @@ void Assembler::emit_data64(jlong data,
                             RelocationHolder const& rspec,
                             int format) {
 
-  assert(inst_mark() != NULL, "must be inside InstructionMark");
+  assert(inst_mark() != nullptr, "must be inside InstructionMark");
   // Do not use AbstractAssembler::relocate, which is not intended for
   // embedded words.  Instead, relocate to the enclosing instruction.
   code_section()->relocate(inst_mark(), rspec, format);

src/hotspot/cpu/aarch64/assembler_aarch64.hpp

@@ -228,7 +228,7 @@ public:
   static void spatch(address a, int msb, int lsb, int64_t val) {
     int nbits = msb - lsb + 1;
     int64_t chk = val >> (nbits - 1);
-    guarantee (chk == -1 || chk == 0, "Field too big for insn");
+    guarantee (chk == -1 || chk == 0, "Field too big for insn at " INTPTR_FORMAT, p2i(a));
     unsigned uval = val;
     unsigned mask = checked_cast<unsigned>(right_n_bits(nbits));
     uval &= mask;
@@ -3786,50 +3786,78 @@ public:
   INSN(sve_fac, 0b01100101, 0b11, 1); // Floating-point absolute compare vectors
 #undef INSN
 
-// SVE Integer Compare - Signed Immediate
-void sve_cmp(Condition cond, PRegister Pd, SIMD_RegVariant T,
-             PRegister Pg, FloatRegister Zn, int imm5) {
-  starti;
-  assert(T != Q, "invalid size");
-  guarantee(-16 <= imm5 && imm5 <= 15, "invalid immediate");
-  int cond_op;
-  switch(cond) {
-    case EQ: cond_op = 0b1000; break;
-    case NE: cond_op = 0b1001; break;
-    case GE: cond_op = 0b0000; break;
-    case GT: cond_op = 0b0001; break;
-    case LE: cond_op = 0b0011; break;
-    case LT: cond_op = 0b0010; break;
-    default:
-      ShouldNotReachHere();
-  }
-  f(0b00100101, 31, 24), f(T, 23, 22), f(0b0, 21), sf(imm5, 20, 16),
-  f((cond_op >> 1) & 0x7, 15, 13), pgrf(Pg, 10), rf(Zn, 5);
-  f(cond_op & 0x1, 4), prf(Pd, 0);
-}
+private:
+  // Convert Assembler::Condition to op encoding - used by sve integer compare encoding
+  static int assembler_cond_to_sve_op(Condition cond, bool &is_unsigned) {
+    if (cond == HI || cond == HS || cond == LO || cond == LS) {
+      is_unsigned = true;
+    } else {
+      is_unsigned = false;
+    }
 
-// SVE Floating-point compare vector with zero
-void sve_fcm(Condition cond, PRegister Pd, SIMD_RegVariant T,
-             PRegister Pg, FloatRegister Zn, double d) {
-  starti;
-  assert(T != Q, "invalid size");
-  guarantee(d == 0.0, "invalid immediate");
-  int cond_op;
-  switch(cond) {
-    case EQ: cond_op = 0b100; break;
-    case GT: cond_op = 0b001; break;
-    case GE: cond_op = 0b000; break;
-    case LT: cond_op = 0b010; break;
-    case LE: cond_op = 0b011; break;
-    case NE: cond_op = 0b110; break;
-    default:
-      ShouldNotReachHere();
+    switch (cond) {
+      case HI:
+      case GT:
+        return 0b0001;
+      case HS:
+      case GE:
+        return 0b0000;
+      case LO:
+      case LT:
+        return 0b0010;
+      case LS:
+      case LE:
+        return 0b0011;
+      case EQ:
+        return 0b1000;
+      case NE:
+        return 0b1001;
+      default:
+        ShouldNotReachHere();
+        return -1;
+    }
+  }
+
+public:
+  // SVE Integer Compare - 5 bits signed imm and 7 bits unsigned imm
+  void sve_cmp(Condition cond, PRegister Pd, SIMD_RegVariant T,
+               PRegister Pg, FloatRegister Zn, int imm) {
+    starti;
+    assert(T != Q, "invalid size");
+    bool is_unsigned = false;
+    int cond_op = assembler_cond_to_sve_op(cond, is_unsigned);
+    f(is_unsigned ? 0b00100100 : 0b00100101, 31, 24), f(T, 23, 22);
+    f(is_unsigned ? 0b1 : 0b0, 21);
+    if (is_unsigned) {
+      f(imm, 20, 14), f((cond_op >> 1) & 0x1, 13);
+    } else {
+      sf(imm, 20, 16), f((cond_op >> 1) & 0x7, 15, 13);
+    }
+    pgrf(Pg, 10), rf(Zn, 5), f(cond_op & 0x1, 4), prf(Pd, 0);
+  }
+
+  // SVE Floating-point compare vector with zero
+  void sve_fcm(Condition cond, PRegister Pd, SIMD_RegVariant T,
+               PRegister Pg, FloatRegister Zn, double d) {
+    starti;
+    assert(T != Q, "invalid size");
+    guarantee(d == 0.0, "invalid immediate");
+    int cond_op;
+    switch(cond) {
+      case EQ: cond_op = 0b100; break;
+      case GT: cond_op = 0b001; break;
+      case GE: cond_op = 0b000; break;
+      case LT: cond_op = 0b010; break;
+      case LE: cond_op = 0b011; break;
+      case NE: cond_op = 0b110; break;
+      default:
+        ShouldNotReachHere();
+    }
+    f(0b01100101, 31, 24), f(T, 23, 22), f(0b0100, 21, 18),
+    f((cond_op >> 1) & 0x3, 17, 16), f(0b001, 15, 13),
+    pgrf(Pg, 10), rf(Zn, 5);
+    f(cond_op & 0x1, 4), prf(Pd, 0);
   }
-  f(0b01100101, 31, 24), f(T, 23, 22), f(0b0100, 21, 18),
-  f((cond_op >> 1) & 0x3, 17, 16), f(0b001, 15, 13),
-  pgrf(Pg, 10), rf(Zn, 5);
-  f(cond_op & 0x1, 4), prf(Pd, 0);
-}
 
 // SVE unpack vector elements
 #define INSN(NAME, op) \

src/hotspot/cpu/aarch64/c1_CodeStubs_aarch64.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1999, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2023, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2014, Red Hat Inc. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
@@ -44,7 +44,7 @@ void C1SafepointPollStub::emit_code(LIR_Assembler* ce) {
   __ adr(rscratch1, safepoint_pc);
   __ str(rscratch1, Address(rthread, JavaThread::saved_exception_pc_offset()));
 
-  assert(SharedRuntime::polling_page_return_handler_blob() != NULL,
+  assert(SharedRuntime::polling_page_return_handler_blob() != nullptr,
          "polling page return stub not created yet");
   address stub = SharedRuntime::polling_page_return_handler_blob()->entry_point();
 
@@ -334,7 +334,7 @@ void ArrayCopyStub::emit_code(LIR_Assembler* ce) {
   Address resolve(SharedRuntime::get_resolve_static_call_stub(),
                   relocInfo::static_call_type);
   address call = __ trampoline_call(resolve);
-  if (call == NULL) {
+  if (call == nullptr) {
     ce->bailout("trampoline stub overflow");
     return;
   }

src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp

@@ -111,7 +111,7 @@ LIR_Opr LIR_Assembler::osrBufferPointer() {
 
 address LIR_Assembler::float_constant(float f) {
   address const_addr = __ float_constant(f);
-  if (const_addr == NULL) {
+  if (const_addr == nullptr) {
     bailout("const section overflow");
     return __ code()->consts()->start();
   } else {
@@ -122,7 +122,7 @@ address LIR_Assembler::float_constant(float f) {
 
 address LIR_Assembler::double_constant(double d) {
   address const_addr = __ double_constant(d);
-  if (const_addr == NULL) {
+  if (const_addr == nullptr) {
     bailout("const section overflow");
     return __ code()->consts()->start();
   } else {
@@ -132,7 +132,7 @@ address LIR_Assembler::double_constant(double d) {
 
 address LIR_Assembler::int_constant(jlong n) {
   address const_addr = __ long_constant(n);
-  if (const_addr == NULL) {
+  if (const_addr == nullptr) {
     bailout("const section overflow");
     return __ code()->consts()->start();
   } else {
@@ -278,7 +278,7 @@ void LIR_Assembler::osr_entry() {
         Label L;
         __ ldr(rscratch1, Address(OSR_buf, slot_offset + 1*BytesPerWord));
         __ cbnz(rscratch1, L);
-        __ stop("locked object is NULL");
+        __ stop("locked object is null");
         __ bind(L);
       }
 #endif
@@ -328,7 +328,7 @@ void LIR_Assembler::clinit_barrier(ciMethod* method) {
 }
 
 void LIR_Assembler::jobject2reg(jobject o, Register reg) {
-  if (o == NULL) {
+  if (o == nullptr) {
     __ mov(reg, zr);
   } else {
     __ movoop(reg, o);
@@ -336,7 +336,7 @@ void LIR_Assembler::jobject2reg(jobject o, Register reg) {
 }
 
 void LIR_Assembler::deoptimize_trap(CodeEmitInfo *info) {
-  address target = NULL;
+  address target = nullptr;
   relocInfo::relocType reloc_type = relocInfo::none;
 
   switch (patching_id(info)) {
@@ -379,7 +379,7 @@ int LIR_Assembler::initial_frame_size_in_bytes() const {
 int LIR_Assembler::emit_exception_handler() {
   // generate code for exception handler
   address handler_base = __ start_a_stub(exception_handler_size());
-  if (handler_base == NULL) {
+  if (handler_base == nullptr) {
     // not enough space left for the handler
     bailout("exception handler overflow");
     return -1;
@@ -427,11 +427,11 @@ int LIR_Assembler::emit_unwind_handler() {
   }
 
   // Perform needed unlocking
-  MonitorExitStub* stub = NULL;
+  MonitorExitStub* stub = nullptr;
   if (method()->is_synchronized()) {
     monitor_address(0, FrameMap::r0_opr);
     stub = new MonitorExitStub(FrameMap::r0_opr, true, 0);
-    if (UseHeavyMonitors) {
+    if (LockingMode == LM_MONITOR) {
       __ b(*stub->entry());
     } else {
       __ unlock_object(r5, r4, r0, *stub->entry());
@@ -455,7 +455,7 @@ int LIR_Assembler::emit_unwind_handler() {
   __ far_jump(RuntimeAddress(Runtime1::entry_for(Runtime1::unwind_exception_id)));
 
   // Emit the slow path assembly
-  if (stub != NULL) {
+  if (stub != nullptr) {
     stub->emit_code(this);
   }
 
@@ -466,7 +466,7 @@ int LIR_Assembler::emit_unwind_handler() {
 int LIR_Assembler::emit_deopt_handler() {
   // generate code for exception handler
   address handler_base = __ start_a_stub(deopt_handler_size());
-  if (handler_base == NULL) {
+  if (handler_base == nullptr) {
     // not enough space left for the handler
     bailout("deopt handler overflow");
     return -1;
@@ -487,7 +487,7 @@ void LIR_Assembler::add_debug_info_for_branch(address adr, CodeEmitInfo* info) {
   int pc_offset = code_offset();
   flush_debug_info(pc_offset);
   info->record_debug_info(compilation()->debug_info_recorder(), pc_offset);
-  if (info->exception_handlers() != NULL) {
+  if (info->exception_handlers() != nullptr) {
     compilation()->add_exception_handlers_for_pco(pc_offset, info->exception_handlers());
   }
 }
@@ -509,7 +509,7 @@ void LIR_Assembler::return_op(LIR_Opr result, C1SafepointPollStub* code_stub) {
 }
 
 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
-  guarantee(info != NULL, "Shouldn't be NULL");
+  guarantee(info != nullptr, "Shouldn't be null");
   __ get_polling_page(rscratch1, relocInfo::poll_type);
   add_debug_info_for_branch(info);  // This isn't just debug info:
                                     // it's the oop map
@@ -604,14 +604,14 @@ void LIR_Assembler::const2stack(LIR_Opr src, LIR_Opr dest) {
       if (! c->as_jobject())
         __ str(zr, frame_map()->address_for_slot(dest->single_stack_ix()));
       else {
-        const2reg(src, FrameMap::rscratch1_opr, lir_patch_none, NULL);
+        const2reg(src, FrameMap::rscratch1_opr, lir_patch_none, nullptr);
         reg2stack(FrameMap::rscratch1_opr, dest, c->type(), false);
       }
     }
     break;
   case T_ADDRESS:
     {
-      const2reg(src, FrameMap::rscratch1_opr, lir_patch_none, NULL);
+      const2reg(src, FrameMap::rscratch1_opr, lir_patch_none, nullptr);
       reg2stack(FrameMap::rscratch1_opr, dest, c->type(), false);
     }
   case T_INT:
@@ -775,7 +775,7 @@ void LIR_Assembler::reg2stack(LIR_Opr src, LIR_Opr dest, BasicType type, bool po
 
 void LIR_Assembler::reg2mem(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool pop_fpu_stack, bool wide) {
   LIR_Address* to_addr = dest->as_address_ptr();
-  PatchingStub* patch = NULL;
+  PatchingStub* patch = nullptr;
   Register compressed_src = rscratch1;
 
   if (patch_code != lir_patch_none) {
@@ -847,7 +847,7 @@ void LIR_Assembler::reg2mem(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_Patch
     default:
       ShouldNotReachHere();
   }
-  if (info != NULL) {
+  if (info != nullptr) {
     add_debug_info_for_null_check(null_check_here, info);
   }
 }
@@ -890,7 +890,7 @@ void LIR_Assembler::stack2reg(LIR_Opr src, LIR_Opr dest, BasicType type) {
 
 
 void LIR_Assembler::klass2reg_with_patching(Register reg, CodeEmitInfo* info) {
-  address target = NULL;
+  address target = nullptr;
   relocInfo::relocType reloc_type = relocInfo::none;
 
   switch (patching_id(info)) {
@@ -943,7 +943,7 @@ void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_Patch
     return;
   }
 
-  if (info != NULL) {
+  if (info != nullptr) {
     add_debug_info_for_null_check_here(info);
   }
   int null_check_here = code_offset();
@@ -1010,7 +1010,7 @@ void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_Patch
       __ decode_heap_oop(dest->as_register());
     }
 
-    if (!UseZGC) {
+    if (!(UseZGC && !ZGenerational)) {
       // Load barrier has not yet been applied, so ZGC can't verify the oop here
       __ verify_oop(dest->as_register());
     }
@@ -1053,13 +1053,13 @@ void LIR_Assembler::emit_op3(LIR_Op3* op) {
 
 void LIR_Assembler::emit_opBranch(LIR_OpBranch* op) {
 #ifdef ASSERT
-  assert(op->block() == NULL || op->block()->label() == op->label(), "wrong label");
-  if (op->block() != NULL)  _branch_target_blocks.append(op->block());
-  if (op->ublock() != NULL) _branch_target_blocks.append(op->ublock());
+  assert(op->block() == nullptr || op->block()->label() == op->label(), "wrong label");
+  if (op->block() != nullptr)  _branch_target_blocks.append(op->block());
+  if (op->ublock() != nullptr) _branch_target_blocks.append(op->ublock());
 #endif
 
   if (op->cond() == lir_cond_always) {
-    if (op->info() != NULL) add_debug_info_for_branch(op->info());
+    if (op->info() != nullptr) add_debug_info_for_branch(op->info());
     __ b(*(op->label()));
   } else {
     Assembler::Condition acond;
@@ -1288,12 +1288,12 @@ void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, L
 
   if (should_profile) {
     ciMethod* method = op->profiled_method();
-    assert(method != NULL, "Should have method");
+    assert(method != nullptr, "Should have method");
     int bci = op->profiled_bci();
     md = method->method_data_or_null();
-    assert(md != NULL, "Sanity");
+    assert(md != nullptr, "Sanity");
     data = md->bci_to_data(bci);
-    assert(data != NULL,                "need data for type check");
+    assert(data != nullptr,                "need data for type check");
     assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
   }
   Label profile_cast_success, profile_cast_failure;
@@ -1375,7 +1375,7 @@ void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, L
       }
     } else {
       // perform the fast part of the checking logic
-      __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, NULL);
+      __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, nullptr);
       // call out-of-line instance of __ check_klass_subtype_slow_path(...):
       __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize)));
       __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
@@ -1428,12 +1428,12 @@ void LIR_Assembler::emit_opTypeCheck(LIR_OpTypeCheck* op) {
 
     if (should_profile) {
       ciMethod* method = op->profiled_method();
-      assert(method != NULL, "Should have method");
+      assert(method != nullptr, "Should have method");
       int bci = op->profiled_bci();
       md = method->method_data_or_null();
-      assert(md != NULL, "Sanity");
+      assert(md != nullptr, "Sanity");
       data = md->bci_to_data(bci);
-      assert(data != NULL,                "need data for type check");
+      assert(data != nullptr,                "need data for type check");
       assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
     }
     Label profile_cast_success, profile_cast_failure, done;
@@ -1466,7 +1466,7 @@ void LIR_Assembler::emit_opTypeCheck(LIR_OpTypeCheck* op) {
     // get instance klass (it's already uncompressed)
     __ ldr(k_RInfo, Address(k_RInfo, ObjArrayKlass::element_klass_offset()));
     // perform the fast part of the checking logic
-    __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, NULL);
+    __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, nullptr);
     // call out-of-line instance of __ check_klass_subtype_slow_path(...):
     __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize)));
     __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
@@ -1620,7 +1620,7 @@ void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, L
   } else if (opr1->is_constant()) {
     LIR_Opr tmp
       = opr1->type() == T_LONG ? FrameMap::rscratch1_long_opr : FrameMap::rscratch1_opr;
-    const2reg(opr1, tmp, lir_patch_none, NULL);
+    const2reg(opr1, tmp, lir_patch_none, nullptr);
     opr1 = tmp;
   }
 
@@ -1630,7 +1630,7 @@ void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, L
   } else if (opr2->is_constant()) {
     LIR_Opr tmp
       = opr2->type() == T_LONG ? FrameMap::rscratch2_long_opr : FrameMap::rscratch2_opr;
-    const2reg(opr2, tmp, lir_patch_none, NULL);
+    const2reg(opr2, tmp, lir_patch_none, nullptr);
     opr2 = tmp;
   }
 
@@ -1641,7 +1641,7 @@ void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, L
 }
 
 void LIR_Assembler::arith_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dest, CodeEmitInfo* info, bool pop_fpu_stack) {
-  assert(info == NULL, "should never be used, idiv/irem and ldiv/lrem not handled by this method");
+  assert(info == nullptr, "should never be used, idiv/irem and ldiv/lrem not handled by this method");
 
   if (left->is_single_cpu()) {
     Register lreg = left->as_register();
@@ -2033,7 +2033,7 @@ void LIR_Assembler::align_call(LIR_Code code) {  }
 
 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
   address call = __ trampoline_call(Address(op->addr(), rtype));
-  if (call == NULL) {
+  if (call == nullptr) {
     bailout("trampoline stub overflow");
     return;
   }
@@ -2044,7 +2044,7 @@ void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
 
 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
   address call = __ ic_call(op->addr());
-  if (call == NULL) {
+  if (call == nullptr) {
     bailout("trampoline stub overflow");
     return;
   }
@@ -2055,7 +2055,7 @@ void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
 void LIR_Assembler::emit_static_call_stub() {
   address call_pc = __ pc();
   address stub = __ start_a_stub(call_stub_size());
-  if (stub == NULL) {
+  if (stub == nullptr) {
     bailout("static call stub overflow");
     return;
   }
@@ -2226,11 +2226,11 @@ void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
 
   CodeStub* stub = op->stub();
   int flags = op->flags();
-  BasicType basic_type = default_type != NULL ? default_type->element_type()->basic_type() : T_ILLEGAL;
+  BasicType basic_type = default_type != nullptr ? default_type->element_type()->basic_type() : T_ILLEGAL;
   if (is_reference_type(basic_type)) basic_type = T_OBJECT;
 
   // if we don't know anything, just go through the generic arraycopy
-  if (default_type == NULL // || basic_type == T_OBJECT
+  if (default_type == nullptr // || basic_type == T_OBJECT
       ) {
     Label done;
     assert(src == r1 && src_pos == r2, "mismatch in calling convention");
@@ -2242,7 +2242,7 @@ void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
     __ str(src,              Address(sp, 4*BytesPerWord));
 
     address copyfunc_addr = StubRoutines::generic_arraycopy();
-    assert(copyfunc_addr != NULL, "generic arraycopy stub required");
+    assert(copyfunc_addr != nullptr, "generic arraycopy stub required");
 
     // The arguments are in java calling convention so we shift them
     // to C convention
@@ -2282,7 +2282,7 @@ void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
     return;
   }
 
-  assert(default_type != NULL && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
+  assert(default_type != nullptr && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
 
   int elem_size = type2aelembytes(basic_type);
   int scale = exact_log2(elem_size);
@@ -2292,7 +2292,7 @@ void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
   Address src_klass_addr = Address(src, oopDesc::klass_offset_in_bytes());
   Address dst_klass_addr = Address(dst, oopDesc::klass_offset_in_bytes());
 
-  // test for NULL
+  // test for null
   if (flags & LIR_OpArrayCopy::src_null_check) {
     __ cbz(src, *stub->entry());
   }
@@ -2377,7 +2377,7 @@ void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
       __ load_klass(src, src);
       __ load_klass(dst, dst);
 
-      __ check_klass_subtype_fast_path(src, dst, tmp, &cont, &slow, NULL);
+      __ check_klass_subtype_fast_path(src, dst, tmp, &cont, &slow, nullptr);
 
       __ PUSH(src, dst);
       __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
@@ -2389,7 +2389,7 @@ void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
       __ POP(src, dst);
 
       address copyfunc_addr = StubRoutines::checkcast_arraycopy();
-      if (copyfunc_addr != NULL) { // use stub if available
+      if (copyfunc_addr != nullptr) { // use stub if available
         // src is not a sub class of dst so we have to do a
         // per-element check.
 
@@ -2558,8 +2558,8 @@ void LIR_Assembler::emit_lock(LIR_OpLock* op) {
   Register obj = op->obj_opr()->as_register();  // may not be an oop
   Register hdr = op->hdr_opr()->as_register();
   Register lock = op->lock_opr()->as_register();
-  if (UseHeavyMonitors) {
-    if (op->info() != NULL) {
+  if (LockingMode == LM_MONITOR) {
+    if (op->info() != nullptr) {
       add_debug_info_for_null_check_here(op->info());
       __ null_check(obj, -1);
     }
@@ -2568,7 +2568,7 @@ void LIR_Assembler::emit_lock(LIR_OpLock* op) {
     assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
     // add debug info for NullPointerException only if one is possible
     int null_check_offset = __ lock_object(hdr, obj, lock, *op->stub()->entry());
-    if (op->info() != NULL) {
+    if (op->info() != nullptr) {
       add_debug_info_for_null_check(null_check_offset, op->info());
     }
     // done
@@ -2586,7 +2586,7 @@ void LIR_Assembler::emit_load_klass(LIR_OpLoadKlass* op) {
   Register result = op->result_opr()->as_pointer_register();
 
   CodeEmitInfo* info = op->info();
-  if (info != NULL) {
+  if (info != nullptr) {
     add_debug_info_for_null_check_here(info);
   }
 
@@ -2605,9 +2605,9 @@ void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) {
 
   // Update counter for all call types
   ciMethodData* md = method->method_data_or_null();
-  assert(md != NULL, "Sanity");
+  assert(md != nullptr, "Sanity");
   ciProfileData* data = md->bci_to_data(bci);
-  assert(data != NULL && data->is_CounterData(), "need CounterData for calls");
+  assert(data != nullptr && data->is_CounterData(), "need CounterData for calls");
   assert(op->mdo()->is_single_cpu(),  "mdo must be allocated");
   Register mdo  = op->mdo()->as_register();
   __ mov_metadata(mdo, md->constant_encoding());
@@ -2620,7 +2620,7 @@ void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) {
     assert_different_registers(mdo, recv);
     assert(data->is_VirtualCallData(), "need VirtualCallData for virtual calls");
     ciKlass* known_klass = op->known_holder();
-    if (C1OptimizeVirtualCallProfiling && known_klass != NULL) {
+    if (C1OptimizeVirtualCallProfiling && known_klass != nullptr) {
       // We know the type that will be seen at this call site; we can
       // statically update the MethodData* rather than needing to do
       // dynamic tests on the receiver type
@@ -2645,7 +2645,7 @@ void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) {
       // VirtualCallData rather than just the first time
       for (i = 0; i < VirtualCallData::row_limit(); i++) {
         ciKlass* receiver = vc_data->receiver(i);
-        if (receiver == NULL) {
+        if (receiver == nullptr) {
           Address recv_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_offset(i)));
           __ mov_metadata(rscratch1, known_klass->constant_encoding());
           __ lea(rscratch2, recv_addr);
@@ -2712,7 +2712,7 @@ void LIR_Assembler::emit_profile_type(LIR_OpProfileType* op) {
   Label update, next, none;
 
   bool do_null = !not_null;
-  bool exact_klass_set = exact_klass != NULL && ciTypeEntries::valid_ciklass(current_klass) == exact_klass;
+  bool exact_klass_set = exact_klass != nullptr && ciTypeEntries::valid_ciklass(current_klass) == exact_klass;
   bool do_update = !TypeEntries::is_type_unknown(current_klass) && !exact_klass_set;
 
   assert(do_null || do_update, "why are we here?");
@@ -2748,7 +2748,7 @@ void LIR_Assembler::emit_profile_type(LIR_OpProfileType* op) {
 
   if (do_update) {
 #ifdef ASSERT
-    if (exact_klass != NULL) {
+    if (exact_klass != nullptr) {
       Label ok;
       __ load_klass(tmp, tmp);
       __ mov_metadata(rscratch1, exact_klass->constant_encoding());
@@ -2759,8 +2759,8 @@ void LIR_Assembler::emit_profile_type(LIR_OpProfileType* op) {
     }
 #endif
     if (!no_conflict) {
-      if (exact_klass == NULL || TypeEntries::is_type_none(current_klass)) {
-        if (exact_klass != NULL) {
+      if (exact_klass == nullptr || TypeEntries::is_type_none(current_klass)) {
+        if (exact_klass != nullptr) {
           __ mov_metadata(tmp, exact_klass->constant_encoding());
         } else {
           __ load_klass(tmp, tmp);
@@ -2789,7 +2789,7 @@ void LIR_Assembler::emit_profile_type(LIR_OpProfileType* op) {
           __ cbz(rscratch1, next);
         }
       } else {
-        assert(ciTypeEntries::valid_ciklass(current_klass) != NULL &&
+        assert(ciTypeEntries::valid_ciklass(current_klass) != nullptr &&
                ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "conflict only");
 
         __ ldr(tmp, mdo_addr);
@@ -2810,7 +2810,7 @@ void LIR_Assembler::emit_profile_type(LIR_OpProfileType* op) {
       }
     } else {
       // There's a single possible klass at this profile point
-      assert(exact_klass != NULL, "should be");
+      assert(exact_klass != nullptr, "should be");
       if (TypeEntries::is_type_none(current_klass)) {
         __ mov_metadata(tmp, exact_klass->constant_encoding());
         __ ldr(rscratch2, mdo_addr);
@@ -2839,7 +2839,7 @@ void LIR_Assembler::emit_profile_type(LIR_OpProfileType* op) {
         // first time here. Set profile type.
         __ str(tmp, mdo_addr);
       } else {
-        assert(ciTypeEntries::valid_ciklass(current_klass) != NULL &&
+        assert(ciTypeEntries::valid_ciklass(current_klass) != nullptr &&
                ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
 
         __ ldr(tmp, mdo_addr);
@@ -2903,7 +2903,7 @@ void LIR_Assembler::rt_call(LIR_Opr result, address dest, const LIR_OprList* arg
     __ blr(rscratch1);
   }
 
-  if (info != NULL) {
+  if (info != nullptr) {
     add_call_info_here(info);
   }
   __ post_call_nop();

src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2000, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2000, 2023, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2014, Red Hat Inc. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
@@ -60,7 +60,7 @@ friend class ArrayCopyStub;
   void casw(Register addr, Register newval, Register cmpval);
   void casl(Register addr, Register newval, Register cmpval);
 
-  void poll_for_safepoint(relocInfo::relocType rtype, CodeEmitInfo* info = NULL);
+  void poll_for_safepoint(relocInfo::relocType rtype, CodeEmitInfo* info = nullptr);
 
   static const int max_tableswitches = 20;
   struct tableswitch switches[max_tableswitches];

src/hotspot/cpu/aarch64/c1_LIRGenerator_aarch64.cpp

@@ -107,11 +107,11 @@ LIR_Opr LIRGenerator::rlock_byte(BasicType type) {
 
 
 bool LIRGenerator::can_store_as_constant(Value v, BasicType type) const {
-  if (v->type()->as_IntConstant() != NULL) {
+  if (v->type()->as_IntConstant() != nullptr) {
     return v->type()->as_IntConstant()->value() == 0L;
-  } else if (v->type()->as_LongConstant() != NULL) {
+  } else if (v->type()->as_LongConstant() != nullptr) {
     return v->type()->as_LongConstant()->value() == 0L;
-  } else if (v->type()->as_ObjectConstant() != NULL) {
+  } else if (v->type()->as_ObjectConstant() != nullptr) {
     return v->type()->as_ObjectConstant()->value()->is_null_object();
   } else {
     return false;
@@ -120,11 +120,11 @@ bool LIRGenerator::can_store_as_constant(Value v, BasicType type) const {
 
 bool LIRGenerator::can_inline_as_constant(Value v) const {
   // FIXME: Just a guess
-  if (v->type()->as_IntConstant() != NULL) {
+  if (v->type()->as_IntConstant() != nullptr) {
     return Assembler::operand_valid_for_add_sub_immediate(v->type()->as_IntConstant()->value());
-  } else if (v->type()->as_LongConstant() != NULL) {
+  } else if (v->type()->as_LongConstant() != nullptr) {
     return v->type()->as_LongConstant()->value() == 0L;
-  } else if (v->type()->as_ObjectConstant() != NULL) {
+  } else if (v->type()->as_ObjectConstant() != nullptr) {
     return v->type()->as_ObjectConstant()->value()->is_null_object();
   } else {
     return false;
@@ -315,7 +315,7 @@ void LIRGenerator::do_MonitorEnter(MonitorEnter* x) {
   // "lock" stores the address of the monitor stack slot, so this is not an oop
   LIR_Opr lock = new_register(T_INT);
 
-  CodeEmitInfo* info_for_exception = NULL;
+  CodeEmitInfo* info_for_exception = nullptr;
   if (x->needs_null_check()) {
     info_for_exception = state_for(x);
   }
@@ -466,7 +466,7 @@ void LIRGenerator::do_ArithmeticOp_Long(ArithmeticOp* x) {
       }
     }
     rlock_result(x);
-    arithmetic_op_long(x->op(), x->operand(), left.result(), right.result(), NULL);
+    arithmetic_op_long(x->op(), x->operand(), left.result(), right.result(), nullptr);
   }
 }
 
@@ -512,9 +512,9 @@ void LIRGenerator::do_ArithmeticOp_Int(ArithmeticOp* x) {
 
     LIR_Opr ill = LIR_OprFact::illegalOpr;
     if (x->op() == Bytecodes::_irem) {
-      __ irem(left_arg->result(), right_arg->result(), x->operand(), ill, NULL);
+      __ irem(left_arg->result(), right_arg->result(), x->operand(), ill, nullptr);
     } else if (x->op() == Bytecodes::_idiv) {
-      __ idiv(left_arg->result(), right_arg->result(), x->operand(), ill, NULL);
+      __ idiv(left_arg->result(), right_arg->result(), x->operand(), ill, nullptr);
     }
 
   } else if (x->op() == Bytecodes::_iadd || x->op() == Bytecodes::_isub) {
@@ -547,7 +547,7 @@ void LIRGenerator::do_ArithmeticOp_Int(ArithmeticOp* x) {
 void LIRGenerator::do_ArithmeticOp(ArithmeticOp* x) {
   // when an operand with use count 1 is the left operand, then it is
   // likely that no move for 2-operand-LIR-form is necessary
-  if (x->is_commutative() && x->y()->as_Constant() == NULL && x->x()->use_count() > x->y()->use_count()) {
+  if (x->is_commutative() && x->y()->as_Constant() == nullptr && x->x()->use_count() > x->y()->use_count()) {
     x->swap_operands();
   }
 
@@ -800,7 +800,7 @@ void LIRGenerator::do_LibmIntrinsic(Intrinsic* x) {
   LIR_Opr calc_result = rlock_result(x);
   LIR_Opr result_reg = result_register_for(x->type());
 
-  CallingConvention* cc = NULL;
+  CallingConvention* cc = nullptr;
 
   if (x->id() == vmIntrinsics::_dpow) {
     LIRItem value1(x->argument_at(1), this);
@@ -822,49 +822,49 @@ void LIRGenerator::do_LibmIntrinsic(Intrinsic* x) {
 
   switch (x->id()) {
     case vmIntrinsics::_dexp:
-      if (StubRoutines::dexp() != NULL) {
+      if (StubRoutines::dexp() != nullptr) {
         __ call_runtime_leaf(StubRoutines::dexp(), getThreadTemp(), result_reg, cc->args());
       } else {
         __ call_runtime_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dexp), getThreadTemp(), result_reg, cc->args());
       }
       break;
     case vmIntrinsics::_dlog:
-      if (StubRoutines::dlog() != NULL) {
+      if (StubRoutines::dlog() != nullptr) {
         __ call_runtime_leaf(StubRoutines::dlog(), getThreadTemp(), result_reg, cc->args());
       } else {
         __ call_runtime_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dlog), getThreadTemp(), result_reg, cc->args());
       }
       break;
     case vmIntrinsics::_dlog10:
-      if (StubRoutines::dlog10() != NULL) {
+      if (StubRoutines::dlog10() != nullptr) {
         __ call_runtime_leaf(StubRoutines::dlog10(), getThreadTemp(), result_reg, cc->args());
       } else {
         __ call_runtime_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dlog10), getThreadTemp(), result_reg, cc->args());
       }
       break;
     case vmIntrinsics::_dpow:
-      if (StubRoutines::dpow() != NULL) {
+      if (StubRoutines::dpow() != nullptr) {
         __ call_runtime_leaf(StubRoutines::dpow(), getThreadTemp(), result_reg, cc->args());
       } else {
         __ call_runtime_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dpow), getThreadTemp(), result_reg, cc->args());
       }
       break;
     case vmIntrinsics::_dsin:
-      if (StubRoutines::dsin() != NULL) {
+      if (StubRoutines::dsin() != nullptr) {
         __ call_runtime_leaf(StubRoutines::dsin(), getThreadTemp(), result_reg, cc->args());
       } else {
         __ call_runtime_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dsin), getThreadTemp(), result_reg, cc->args());
       }
       break;
     case vmIntrinsics::_dcos:
-      if (StubRoutines::dcos() != NULL) {
+      if (StubRoutines::dcos() != nullptr) {
         __ call_runtime_leaf(StubRoutines::dcos(), getThreadTemp(), result_reg, cc->args());
       } else {
         __ call_runtime_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dcos), getThreadTemp(), result_reg, cc->args());
       }
       break;
     case vmIntrinsics::_dtan:
-      if (StubRoutines::dtan() != NULL) {
+      if (StubRoutines::dtan() != nullptr) {
         __ call_runtime_leaf(StubRoutines::dtan(), getThreadTemp(), result_reg, cc->args());
       } else {
         __ call_runtime_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dtan), getThreadTemp(), result_reg, cc->args());
@@ -1161,7 +1161,7 @@ void LIRGenerator::do_NewObjectArray(NewObjectArray* x) {
   LIRItem length(x->length(), this);
   // in case of patching (i.e., object class is not yet loaded), we need to reexecute the instruction
   // and therefore provide the state before the parameters have been consumed
-  CodeEmitInfo* patching_info = NULL;
+  CodeEmitInfo* patching_info = nullptr;
   if (!x->klass()->is_loaded() || PatchALot) {
     patching_info =  state_for(x, x->state_before());
   }
@@ -1194,14 +1194,14 @@ void LIRGenerator::do_NewObjectArray(NewObjectArray* x) {
 void LIRGenerator::do_NewMultiArray(NewMultiArray* x) {
   Values* dims = x->dims();
   int i = dims->length();
-  LIRItemList* items = new LIRItemList(i, i, NULL);
+  LIRItemList* items = new LIRItemList(i, i, nullptr);
   while (i-- > 0) {
     LIRItem* size = new LIRItem(dims->at(i), this);
     items->at_put(i, size);
   }
 
   // Evaluate state_for early since it may emit code.
-  CodeEmitInfo* patching_info = NULL;
+  CodeEmitInfo* patching_info = nullptr;
   if (!x->klass()->is_loaded() || PatchALot) {
     patching_info = state_for(x, x->state_before());
 
@@ -1248,7 +1248,7 @@ void LIRGenerator::do_BlockBegin(BlockBegin* x) {
 void LIRGenerator::do_CheckCast(CheckCast* x) {
   LIRItem obj(x->obj(), this);
 
-  CodeEmitInfo* patching_info = NULL;
+  CodeEmitInfo* patching_info = nullptr;
   if (!x->klass()->is_loaded() || (PatchALot && !x->is_incompatible_class_change_check() && !x->is_invokespecial_receiver_check())) {
     // must do this before locking the destination register as an oop register,
     // and before the obj is loaded (the latter is for deoptimization)
@@ -1263,10 +1263,10 @@ void LIRGenerator::do_CheckCast(CheckCast* x) {
 
   CodeStub* stub;
   if (x->is_incompatible_class_change_check()) {
-    assert(patching_info == NULL, "can't patch this");
+    assert(patching_info == nullptr, "can't patch this");
     stub = new SimpleExceptionStub(Runtime1::throw_incompatible_class_change_error_id, LIR_OprFact::illegalOpr, info_for_exception);
   } else if (x->is_invokespecial_receiver_check()) {
-    assert(patching_info == NULL, "can't patch this");
+    assert(patching_info == nullptr, "can't patch this");
     stub = new DeoptimizeStub(info_for_exception,
                               Deoptimization::Reason_class_check,
                               Deoptimization::Action_none);
@@ -1289,7 +1289,7 @@ void LIRGenerator::do_InstanceOf(InstanceOf* x) {
 
   // result and test object may not be in same register
   LIR_Opr reg = rlock_result(x);
-  CodeEmitInfo* patching_info = NULL;
+  CodeEmitInfo* patching_info = nullptr;
   if ((!x->klass()->is_loaded() || PatchALot)) {
     // must do this before locking the destination register as an oop register
     patching_info = state_for(x, x->state_before());

src/hotspot/cpu/aarch64/c1_MacroAssembler_aarch64.cpp

@@ -63,14 +63,13 @@ void C1_MacroAssembler::float_cmp(bool is_float, int unordered_result,
 int C1_MacroAssembler::lock_object(Register hdr, Register obj, Register disp_hdr, Label& slow_case) {
   const int aligned_mask = BytesPerWord -1;
   const int hdr_offset = oopDesc::mark_offset_in_bytes();
-  assert(hdr != obj && hdr != disp_hdr && obj != disp_hdr, "registers must be different");
-  Label done;
+  assert_different_registers(hdr, obj, disp_hdr);
   int null_check_offset = -1;
 
   verify_oop(obj);
 
   // save object being locked into the BasicObjectLock
-  str(obj, Address(disp_hdr, BasicObjectLock::obj_offset_in_bytes()));
+  str(obj, Address(disp_hdr, BasicObjectLock::obj_offset()));
 
   null_check_offset = offset();
 
@@ -83,39 +82,44 @@ int C1_MacroAssembler::lock_object(Register hdr, Register obj, Register disp_hdr
 
   // Load object header
   ldr(hdr, Address(obj, hdr_offset));
-  // and mark it as unlocked
-  orr(hdr, hdr, markWord::unlocked_value);
-  // save unlocked object header into the displaced header location on the stack
-  str(hdr, Address(disp_hdr, 0));
-  // test if object header is still the same (i.e. unlocked), and if so, store the
-  // displaced header address in the object header - if it is not the same, get the
-  // object header instead
-  lea(rscratch2, Address(obj, hdr_offset));
-  cmpxchgptr(hdr, disp_hdr, rscratch2, rscratch1, done, /*fallthough*/NULL);
-  // if the object header was the same, we're done
-  // if the object header was not the same, it is now in the hdr register
-  // => test if it is a stack pointer into the same stack (recursive locking), i.e.:
-  //
-  // 1) (hdr & aligned_mask) == 0
-  // 2) sp <= hdr
-  // 3) hdr <= sp + page_size
-  //
-  // these 3 tests can be done by evaluating the following expression:
-  //
-  // (hdr - sp) & (aligned_mask - page_size)
-  //
-  // assuming both the stack pointer and page_size have their least
-  // significant 2 bits cleared and page_size is a power of 2
-  mov(rscratch1, sp);
-  sub(hdr, hdr, rscratch1);
-  ands(hdr, hdr, aligned_mask - (int)os::vm_page_size());
-  // for recursive locking, the result is zero => save it in the displaced header
-  // location (NULL in the displaced hdr location indicates recursive locking)
-  str(hdr, Address(disp_hdr, 0));
-  // otherwise we don't care about the result and handle locking via runtime call
-  cbnz(hdr, slow_case);
-  // done
-  bind(done);
+  if (LockingMode == LM_LIGHTWEIGHT) {
+    fast_lock(obj, hdr, rscratch1, rscratch2, slow_case);
+  } else if (LockingMode == LM_LEGACY) {
+    Label done;
+    // and mark it as unlocked
+    orr(hdr, hdr, markWord::unlocked_value);
+    // save unlocked object header into the displaced header location on the stack
+    str(hdr, Address(disp_hdr, 0));
+    // test if object header is still the same (i.e. unlocked), and if so, store the
+    // displaced header address in the object header - if it is not the same, get the
+    // object header instead
+    lea(rscratch2, Address(obj, hdr_offset));
+    cmpxchgptr(hdr, disp_hdr, rscratch2, rscratch1, done, /*fallthough*/nullptr);
+    // if the object header was the same, we're done
+    // if the object header was not the same, it is now in the hdr register
+    // => test if it is a stack pointer into the same stack (recursive locking), i.e.:
+    //
+    // 1) (hdr & aligned_mask) == 0
+    // 2) sp <= hdr
+    // 3) hdr <= sp + page_size
+    //
+    // these 3 tests can be done by evaluating the following expression:
+    //
+    // (hdr - sp) & (aligned_mask - page_size)
+    //
+    // assuming both the stack pointer and page_size have their least
+    // significant 2 bits cleared and page_size is a power of 2
+    mov(rscratch1, sp);
+    sub(hdr, hdr, rscratch1);
+    ands(hdr, hdr, aligned_mask - (int)os::vm_page_size());
+    // for recursive locking, the result is zero => save it in the displaced header
+    // location (null in the displaced hdr location indicates recursive locking)
+    str(hdr, Address(disp_hdr, 0));
+    // otherwise we don't care about the result and handle locking via runtime call
+    cbnz(hdr, slow_case);
+    // done
+    bind(done);
+  }
   increment(Address(rthread, JavaThread::held_monitor_count_offset()));
   return null_check_offset;
 }
@@ -127,27 +131,40 @@ void C1_MacroAssembler::unlock_object(Register hdr, Register obj, Register disp_
   assert(hdr != obj && hdr != disp_hdr && obj != disp_hdr, "registers must be different");
   Label done;
 
-  // load displaced header
-  ldr(hdr, Address(disp_hdr, 0));
-  // if the loaded hdr is NULL we had recursive locking
-  // if we had recursive locking, we are done
-  cbz(hdr, done);
-  // load object
-  ldr(obj, Address(disp_hdr, BasicObjectLock::obj_offset_in_bytes()));
-  verify_oop(obj);
-  // test if object header is pointing to the displaced header, and if so, restore
-  // the displaced header in the object - if the object header is not pointing to
-  // the displaced header, get the object header instead
-  // if the object header was not pointing to the displaced header,
-  // we do unlocking via runtime call
-  if (hdr_offset) {
-    lea(rscratch1, Address(obj, hdr_offset));
-    cmpxchgptr(disp_hdr, hdr, rscratch1, rscratch2, done, &slow_case);
-  } else {
-    cmpxchgptr(disp_hdr, hdr, obj, rscratch2, done, &slow_case);
+  if (LockingMode != LM_LIGHTWEIGHT) {
+    // load displaced header
+    ldr(hdr, Address(disp_hdr, 0));
+    // if the loaded hdr is null we had recursive locking
+    // if we had recursive locking, we are done
+    cbz(hdr, done);
+  }
+
+  // load object
+  ldr(obj, Address(disp_hdr, BasicObjectLock::obj_offset()));
+  verify_oop(obj);
+
+  if (LockingMode == LM_LIGHTWEIGHT) {
+    ldr(hdr, Address(obj, oopDesc::mark_offset_in_bytes()));
+    // We cannot use tbnz here, the target might be too far away and cannot
+    // be encoded.
+    tst(hdr, markWord::monitor_value);
+    br(Assembler::NE, slow_case);
+    fast_unlock(obj, hdr, rscratch1, rscratch2, slow_case);
+  } else if (LockingMode == LM_LEGACY) {
+    // test if object header is pointing to the displaced header, and if so, restore
+    // the displaced header in the object - if the object header is not pointing to
+    // the displaced header, get the object header instead
+    // if the object header was not pointing to the displaced header,
+    // we do unlocking via runtime call
+    if (hdr_offset) {
+      lea(rscratch1, Address(obj, hdr_offset));
+      cmpxchgptr(disp_hdr, hdr, rscratch1, rscratch2, done, &slow_case);
+    } else {
+      cmpxchgptr(disp_hdr, hdr, obj, rscratch2, done, &slow_case);
+    }
+    // done
+    bind(done);
   }
-  // done
-  bind(done);
   decrement(Address(rthread, JavaThread::held_monitor_count_offset()));
 }
 
@@ -294,7 +311,7 @@ void C1_MacroAssembler::allocate_array(Register obj, Register len, Register t1,
 
 void C1_MacroAssembler::inline_cache_check(Register receiver, Register iCache) {
   verify_oop(receiver);
-  // explicit NULL check not needed since load from [klass_offset] causes a trap
+  // explicit null check not needed since load from [klass_offset] causes a trap
   // check against inline cache
   assert(!MacroAssembler::needs_explicit_null_check(oopDesc::klass_offset_in_bytes()), "must add explicit null check");
 
@@ -311,7 +328,7 @@ void C1_MacroAssembler::build_frame(int framesize, int bang_size_in_bytes) {
 
   // Insert nmethod entry barrier into frame.
   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
-  bs->nmethod_entry_barrier(this, NULL /* slow_path */, NULL /* continuation */, NULL /* guard */);
+  bs->nmethod_entry_barrier(this, nullptr /* slow_path */, nullptr /* continuation */, nullptr /* guard */);
 }
 
 void C1_MacroAssembler::remove_frame(int framesize) {

src/hotspot/cpu/aarch64/c1_MacroAssembler_aarch64.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1999, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2023, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2014, 2021, Red Hat Inc. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
@@ -106,7 +106,7 @@ using MacroAssembler::null_check;
   void invalidate_registers(bool inv_r0, bool inv_r19, bool inv_r2, bool inv_r3, bool inv_r4, bool inv_r5) PRODUCT_RETURN;
 
   // This platform only uses signal-based null checks. The Label is not needed.
-  void null_check(Register r, Label *Lnull = NULL) { MacroAssembler::null_check(r); }
+  void null_check(Register r, Label *Lnull = nullptr) { MacroAssembler::null_check(r); }
 
   void load_parameter(int offset_in_words, Register reg);
 

src/hotspot/cpu/aarch64/c1_Runtime1_aarch64.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1999, 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2023, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2014, 2021, Red Hat Inc. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
@@ -369,7 +369,7 @@ OopMapSet* Runtime1::generate_handle_exception(StubID id, StubAssembler *sasm) {
 
   // Save registers, if required.
   OopMapSet* oop_maps = new OopMapSet();
-  OopMap* oop_map = NULL;
+  OopMap* oop_map = nullptr;
   switch (id) {
   case forward_exception_id:
     // We're handling an exception in the context of a compiled frame.
@@ -542,7 +542,7 @@ OopMapSet* Runtime1::generate_patching(StubAssembler* sasm, address target) {
   // Note: This number affects also the RT-Call in generate_handle_exception because
   //       the oop-map is shared for all calls.
   DeoptimizationBlob* deopt_blob = SharedRuntime::deopt_blob();
-  assert(deopt_blob != NULL, "deoptimization blob must have been created");
+  assert(deopt_blob != nullptr, "deoptimization blob must have been created");
 
   OopMap* oop_map = save_live_registers(sasm);
 
@@ -616,8 +616,8 @@ OopMapSet* Runtime1::generate_code_for(StubID id, StubAssembler* sasm) {
   bool save_fpu_registers = true;
 
   // stub code & info for the different stubs
-  OopMapSet* oop_maps = NULL;
-  OopMap* oop_map = NULL;
+  OopMapSet* oop_maps = nullptr;
+  OopMap* oop_map = nullptr;
   switch (id) {
     {
     case forward_exception_id:
@@ -834,7 +834,7 @@ OopMapSet* Runtime1::generate_code_for(StubID id, StubAssembler* sasm) {
         __ ldp(r4, r0, Address(sp, (sup_k_off) * VMRegImpl::stack_slot_size));
 
         Label miss;
-        __ check_klass_subtype_slow_path(r4, r0, r2, r5, NULL, &miss);
+        __ check_klass_subtype_slow_path(r4, r0, r2, r5, nullptr, &miss);
 
         // fallthrough on success:
         __ mov(rscratch1, 1);
@@ -904,7 +904,7 @@ OopMapSet* Runtime1::generate_code_for(StubID id, StubAssembler* sasm) {
         oop_maps->add_gc_map(call_offset, oop_map);
         restore_live_registers(sasm);
         DeoptimizationBlob* deopt_blob = SharedRuntime::deopt_blob();
-        assert(deopt_blob != NULL, "deoptimization blob must have been created");
+        assert(deopt_blob != nullptr, "deoptimization blob must have been created");
         __ leave();
         __ far_jump(RuntimeAddress(deopt_blob->unpack_with_reexecution()));
       }
@@ -991,7 +991,7 @@ OopMapSet* Runtime1::generate_code_for(StubID id, StubAssembler* sasm) {
         restore_live_registers(sasm);
         __ leave();
         DeoptimizationBlob* deopt_blob = SharedRuntime::deopt_blob();
-        assert(deopt_blob != NULL, "deoptimization blob must have been created");
+        assert(deopt_blob != nullptr, "deoptimization blob must have been created");
 
         __ far_jump(RuntimeAddress(deopt_blob->unpack_with_reexecution()));
       }

src/hotspot/cpu/aarch64/c2_CodeStubs_aarch64.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2020, 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2020, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -25,6 +25,7 @@
 #include "precompiled.hpp"
 #include "opto/c2_MacroAssembler.hpp"
 #include "opto/c2_CodeStubs.hpp"
+#include "runtime/objectMonitor.hpp"
 #include "runtime/sharedRuntime.hpp"
 #include "runtime/stubRoutines.hpp"
 
@@ -35,7 +36,7 @@ int C2SafepointPollStub::max_size() const {
 }
 
 void C2SafepointPollStub::emit(C2_MacroAssembler& masm) {
-  assert(SharedRuntime::polling_page_return_handler_blob() != NULL,
+  assert(SharedRuntime::polling_page_return_handler_blob() != nullptr,
          "polling page return stub not created yet");
   address stub = SharedRuntime::polling_page_return_handler_blob()->entry_point();
 
@@ -63,4 +64,31 @@ void C2EntryBarrierStub::emit(C2_MacroAssembler& masm) {
   __ emit_int32(0);   // nmethod guard value
 }
 
+int C2HandleAnonOMOwnerStub::max_size() const {
+  // Max size of stub has been determined by testing with 0, in which case
+  // C2CodeStubList::emit() will throw an assertion and report the actual size that
+  // is needed.
+  return 24;
+}
+
+void C2HandleAnonOMOwnerStub::emit(C2_MacroAssembler& masm) {
+  __ bind(entry());
+  Register mon = monitor();
+  Register t = tmp();
+  assert(t != noreg, "need tmp register");
+
+  // Fix owner to be the current thread.
+  __ str(rthread, Address(mon, ObjectMonitor::owner_offset()));
+
+  // Pop owner object from lock-stack.
+  __ ldrw(t, Address(rthread, JavaThread::lock_stack_top_offset()));
+  __ subw(t, t, oopSize);
+#ifdef ASSERT
+  __ str(zr, Address(rthread, t));
+#endif
+  __ strw(t, Address(rthread, JavaThread::lock_stack_top_offset()));
+
+  __ b(continuation());
+}
+
 #undef __

src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.cpp

@@ -46,6 +46,7 @@
 typedef void (MacroAssembler::* chr_insn)(Register Rt, const Address &adr);
 
 // Search for str1 in str2 and return index or -1
+// Clobbers: rscratch1, rscratch2, rflags. May also clobber v0-v1, when icnt1==-1.
 void C2_MacroAssembler::string_indexof(Register str2, Register str1,
                                        Register cnt2, Register cnt1,
                                        Register tmp1, Register tmp2,
@@ -287,16 +288,16 @@ void C2_MacroAssembler::string_indexof(Register str2, Register str1,
     cmp(cnt1, (u1)16); // small patterns still should be handled by simple algorithm
     br(LT, LINEAR_MEDIUM);
     mov(result, zr);
-    RuntimeAddress stub = NULL;
+    RuntimeAddress stub = nullptr;
     if (isL) {
       stub = RuntimeAddress(StubRoutines::aarch64::string_indexof_linear_ll());
-      assert(stub.target() != NULL, "string_indexof_linear_ll stub has not been generated");
+      assert(stub.target() != nullptr, "string_indexof_linear_ll stub has not been generated");
     } else if (str1_isL) {
       stub = RuntimeAddress(StubRoutines::aarch64::string_indexof_linear_ul());
-       assert(stub.target() != NULL, "string_indexof_linear_ul stub has not been generated");
+       assert(stub.target() != nullptr, "string_indexof_linear_ul stub has not been generated");
     } else {
       stub = RuntimeAddress(StubRoutines::aarch64::string_indexof_linear_uu());
-      assert(stub.target() != NULL, "string_indexof_linear_uu stub has not been generated");
+      assert(stub.target() != nullptr, "string_indexof_linear_uu stub has not been generated");
     }
     address call = trampoline_call(stub);
     if (call == nullptr) {
@@ -844,7 +845,7 @@ void C2_MacroAssembler::string_compare(Register str1, Register str2,
   }
 
   bind(STUB);
-    RuntimeAddress stub = NULL;
+    RuntimeAddress stub = nullptr;
     switch(ae) {
       case StrIntrinsicNode::LL:
         stub = RuntimeAddress(StubRoutines::aarch64::compare_long_string_LL());
@@ -861,7 +862,7 @@ void C2_MacroAssembler::string_compare(Register str1, Register str2,
       default:
         ShouldNotReachHere();
      }
-    assert(stub.target() != NULL, "compare_long_string stub has not been generated");
+    assert(stub.target() != nullptr, "compare_long_string stub has not been generated");
     address call = trampoline_call(stub);
     if (call == nullptr) {
       DEBUG_ONLY(reset_labels(DONE, SHORT_LOOP, SHORT_STRING, SHORT_LAST, SHORT_LOOP_TAIL, SHORT_LAST2, SHORT_LAST_INIT, SHORT_LOOP_START));
@@ -2049,9 +2050,9 @@ void C2_MacroAssembler::vector_signum_sve(FloatRegister dst, FloatRegister src,
 }
 
 bool C2_MacroAssembler::in_scratch_emit_size() {
-  if (ciEnv::current()->task() != NULL) {
+  if (ciEnv::current()->task() != nullptr) {
     PhaseOutput* phase_output = Compile::current()->output();
-    if (phase_output != NULL && phase_output->in_scratch_emit_size()) {
+    if (phase_output != nullptr && phase_output->in_scratch_emit_size()) {
       return true;
     }
   }

src/hotspot/cpu/aarch64/codeBuffer_aarch64.cpp

@@ -69,7 +69,7 @@ static bool emit_shared_trampolines(CodeBuffer* cb, CodeBuffer::SharedTrampoline
 
   assert(requests->number_of_entries() >= 1, "at least one");
   const int total_requested_size = MacroAssembler::max_trampoline_stub_size() * requests->number_of_entries();
-  if (cb->stubs()->maybe_expand_to_ensure_remaining(total_requested_size) && cb->blob() == NULL) {
+  if (cb->stubs()->maybe_expand_to_ensure_remaining(total_requested_size) && cb->blob() == nullptr) {
     return false;
   }
 

src/hotspot/cpu/aarch64/compiledIC_aarch64.cpp

@@ -44,7 +44,7 @@ address CompiledStaticCall::emit_to_interp_stub(CodeBuffer &cbuf, address mark)
   // mov rmethod, 0
   // jmp -4 # to self
 
-  if (mark == NULL) {
+  if (mark == nullptr) {
     mark = cbuf.insts_mark();  // Get mark within main instrs section.
   }
 
@@ -54,8 +54,8 @@ address CompiledStaticCall::emit_to_interp_stub(CodeBuffer &cbuf, address mark)
 
   address base = __ start_a_stub(to_interp_stub_size());
   int offset = __ offset();
-  if (base == NULL) {
-    return NULL;  // CodeBuffer::expand failed
+  if (base == nullptr) {
+    return nullptr;  // CodeBuffer::expand failed
   }
   // static stub relocation stores the instruction address of the call
   __ relocate(static_stub_Relocation::spec(mark));
@@ -88,7 +88,7 @@ int CompiledStaticCall::reloc_to_interp_stub() {
 
 void CompiledDirectStaticCall::set_to_interpreted(const methodHandle& callee, address entry) {
   address stub = find_stub();
-  guarantee(stub != NULL, "stub not found");
+  guarantee(stub != nullptr, "stub not found");
 
   if (TraceICs) {
     ResourceMark rm;
@@ -117,7 +117,7 @@ void CompiledDirectStaticCall::set_to_interpreted(const methodHandle& callee, ad
 void CompiledDirectStaticCall::set_stub_to_clean(static_stub_Relocation* static_stub) {
   // Reset stub.
   address stub = static_stub->addr();
-  assert(stub != NULL, "stub not found");
+  assert(stub != nullptr, "stub not found");
   assert(CompiledICLocker::is_safe(stub), "mt unsafe call");
   // Creation also verifies the object.
   NativeMovConstReg* method_holder
@@ -138,7 +138,7 @@ void CompiledDirectStaticCall::verify() {
 
   // Verify stub.
   address stub = find_stub();
-  assert(stub != NULL, "no stub found for static call");
+  assert(stub != nullptr, "no stub found for static call");
   // Creation also verifies the object.
   NativeMovConstReg* method_holder
     = nativeMovConstReg_at(stub + NativeInstruction::instruction_size);

src/hotspot/cpu/aarch64/continuationFreezeThaw_aarch64.inline.hpp

@@ -149,7 +149,7 @@ inline void FreezeBase::relativize_interpreted_frame_metadata(const frame& f, co
   // because we freeze the padding word (see recurse_freeze_interpreted_frame) in order to keep the same relativized
   // locals value, we don't need to change the locals value here.
 
-  // at(frame::interpreter_frame_last_sp_offset) can be NULL at safepoint preempts
+  // at(frame::interpreter_frame_last_sp_offset) can be null at safepoint preempts
   *hf.addr_at(frame::interpreter_frame_last_sp_offset) = hf.unextended_sp() - hf.fp();
 
   relativize_one(vfp, hfp, frame::interpreter_frame_initial_sp_offset); // == block_top == block_bottom

src/hotspot/cpu/aarch64/disassembler_aarch64.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1997, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2014, Red Hat Inc. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
@@ -39,7 +39,7 @@
   // the perfect job. In those cases, decode_instruction0 may kick in
   // and do it right.
   // If nothing had to be done, just return "here", otherwise return "here + instr_len(here)"
-  static address decode_instruction0(address here, outputStream* st, address virtual_begin = NULL) {
+  static address decode_instruction0(address here, outputStream* st, address virtual_begin = nullptr) {
     return here;
   }
 

src/hotspot/cpu/aarch64/downcallLinker_aarch64.cpp

@@ -75,7 +75,7 @@ public:
      _needs_transition(needs_transition),
      _frame_complete(0),
      _frame_size_slots(0),
-     _oop_maps(NULL) {
+     _oop_maps(nullptr) {
   }
 
   void generate();
@@ -168,7 +168,7 @@ void DowncallStubGenerator::generate() {
   assert(_abi._shadow_space_bytes == 0, "not expecting shadow space on AArch64");
   allocated_frame_size += arg_shuffle.out_arg_bytes();
 
-  bool should_save_return_value = !_needs_return_buffer && _needs_transition;
+  bool should_save_return_value = !_needs_return_buffer;
   RegSpiller out_reg_spiller(_output_registers);
   int spill_offset = -1;
 

src/hotspot/cpu/aarch64/frame_aarch64.cpp

@@ -96,7 +96,7 @@ bool frame::safe_for_sender(JavaThread *thread) {
   // to construct the sender and do some validation of it. This goes a long way
   // toward eliminating issues when we get in frame construction code
 
-  if (_cb != NULL ) {
+  if (_cb != nullptr ) {
 
     // First check if frame is complete and tester is reliable
     // Unfortunately we can only check frame complete for runtime stubs and nmethod
@@ -122,10 +122,10 @@ bool frame::safe_for_sender(JavaThread *thread) {
       return fp_safe;
     }
 
-    intptr_t* sender_sp = NULL;
-    intptr_t* sender_unextended_sp = NULL;
-    address   sender_pc = NULL;
-    intptr_t* saved_fp =  NULL;
+    intptr_t* sender_sp = nullptr;
+    intptr_t* sender_unextended_sp = nullptr;
+    address   sender_pc = nullptr;
+    intptr_t* saved_fp =  nullptr;
 
     if (is_interpreted_frame()) {
       // fp must be safe
@@ -189,7 +189,7 @@ bool frame::safe_for_sender(JavaThread *thread) {
 
     // We must always be able to find a recognizable pc
     CodeBlob* sender_blob = CodeCache::find_blob(sender_pc);
-    if (sender_pc == NULL ||  sender_blob == NULL) {
+    if (sender_pc == nullptr ||  sender_blob == nullptr) {
       return false;
     }
 
@@ -222,7 +222,7 @@ bool frame::safe_for_sender(JavaThread *thread) {
     }
 
     CompiledMethod* nm = sender_blob->as_compiled_method_or_null();
-    if (nm != NULL) {
+    if (nm != nullptr) {
       if (nm->is_deopt_mh_entry(sender_pc) || nm->is_deopt_entry(sender_pc) ||
           nm->method()->is_method_handle_intrinsic()) {
         return false;
@@ -264,7 +264,7 @@ bool frame::safe_for_sender(JavaThread *thread) {
 
   // Will the pc we fetch be non-zero (which we'll find at the oldest frame)
 
-  if ( (address) this->fp()[return_addr_offset] == NULL) return false;
+  if ( (address) this->fp()[return_addr_offset] == nullptr) return false;
 
 
   // could try and do some more potential verification of native frame if we could think of some...
@@ -298,7 +298,7 @@ void frame::patch_pc(Thread* thread, address pc) {
   *pc_addr = signed_pc;
   _pc = pc; // must be set before call to get_deopt_original_pc
   address original_pc = CompiledMethod::get_deopt_original_pc(this);
-  if (original_pc != NULL) {
+  if (original_pc != nullptr) {
     assert(original_pc == old_pc, "expected original PC to be stored before patching");
     _deopt_state = is_deoptimized;
     _pc = original_pc;
@@ -364,7 +364,7 @@ void frame::interpreter_frame_set_extended_sp(intptr_t* sp) {
 }
 
 frame frame::sender_for_entry_frame(RegisterMap* map) const {
-  assert(map != NULL, "map must be set");
+  assert(map != nullptr, "map must be set");
   // Java frame called from C; skip all C frames and return top C
   // frame of that chunk as the sender
   JavaFrameAnchor* jfa = entry_frame_call_wrapper()->anchor();
@@ -392,11 +392,11 @@ bool frame::upcall_stub_frame_is_first() const {
   assert(is_upcall_stub_frame(), "must be optimzed entry frame");
   UpcallStub* blob = _cb->as_upcall_stub();
   JavaFrameAnchor* jfa = blob->jfa_for_frame(*this);
-  return jfa->last_Java_sp() == NULL;
+  return jfa->last_Java_sp() == nullptr;
 }
 
 frame frame::sender_for_upcall_stub_frame(RegisterMap* map) const {
-  assert(map != NULL, "map must be set");
+  assert(map != nullptr, "map must be set");
   UpcallStub* blob = _cb->as_upcall_stub();
   // Java frame called from C; skip all C frames and return top C
   // frame of that chunk as the sender
@@ -441,9 +441,9 @@ void frame::adjust_unextended_sp() {
   // as any other call site. Therefore, no special action is needed when we are
   // returning to any of these call sites.
 
-  if (_cb != NULL) {
+  if (_cb != nullptr) {
     CompiledMethod* sender_cm = _cb->as_compiled_method_or_null();
-    if (sender_cm != NULL) {
+    if (sender_cm != nullptr) {
       // If the sender PC is a deoptimization point, get the original PC.
       if (sender_cm->is_deopt_entry(_pc) ||
           sender_cm->is_deopt_mh_entry(_pc)) {
@@ -568,7 +568,7 @@ BasicType frame::interpreter_frame_result(oop* oop_result, jvalue* value_result)
         obj = cast_to_oop(at(interpreter_frame_oop_temp_offset));
       } else {
         oop* obj_p = (oop*)tos_addr;
-        obj = (obj_p == NULL) ? (oop)NULL : *obj_p;
+        obj = (obj_p == nullptr) ? (oop)nullptr : *obj_p;
       }
       assert(Universe::is_in_heap_or_null(obj), "sanity check");
       *oop_result = obj;
@@ -636,7 +636,7 @@ void frame::describe_pd(FrameValues& values, int frame_no) {
 
 intptr_t *frame::initial_deoptimization_info() {
   // Not used on aarch64, but we must return something.
-  return NULL;
+  return nullptr;
 }
 
 #undef DESCRIBE_FP_OFFSET
@@ -716,7 +716,7 @@ void internal_pf(uintptr_t sp, uintptr_t fp, uintptr_t pc, uintptr_t bcx) {
       printf("not a Method\n");
   } else {
     CodeBlob *cb = CodeCache::find_blob((address)pc);
-    if (cb != NULL) {
+    if (cb != nullptr) {
       if (cb->is_nmethod()) {
         ResourceMark rm;
         nmethod* nm = (nmethod*)cb;
@@ -782,11 +782,11 @@ frame::frame(void* sp, void* fp, void* pc) {
 
 void JavaFrameAnchor::make_walkable() {
   // last frame set?
-  if (last_Java_sp() == NULL) return;
+  if (last_Java_sp() == nullptr) return;
   // already walkable?
   if (walkable()) return;
-  vmassert(last_Java_sp() != NULL, "not called from Java code?");
-  vmassert(last_Java_pc() == NULL, "already walkable");
+  vmassert(last_Java_sp() != nullptr, "not called from Java code?");
+  vmassert(last_Java_pc() == nullptr, "already walkable");
   _last_Java_pc = (address)_last_Java_sp[-1];
   vmassert(walkable(), "something went wrong");
 }

src/hotspot/cpu/aarch64/frame_aarch64.inline.hpp

@@ -39,11 +39,11 @@
 // Constructors:
 
 inline frame::frame() {
-  _pc = NULL;
-  _sp = NULL;
-  _unextended_sp = NULL;
-  _fp = NULL;
-  _cb = NULL;
+  _pc = nullptr;
+  _sp = nullptr;
+  _unextended_sp = nullptr;
+  _fp = nullptr;
+  _cb = nullptr;
   _deopt_state = unknown;
   _sp_is_trusted = false;
   _on_heap = false;
@@ -60,11 +60,11 @@ inline void frame::init(intptr_t* sp, intptr_t* fp, address pc) {
   _unextended_sp = sp;
   _fp = fp;
   _pc = pc;
-  _oop_map = NULL;
+  _oop_map = nullptr;
   _on_heap = false;
   DEBUG_ONLY(_frame_index = -1;)
 
-  assert(pc != NULL, "no pc?");
+  assert(pc != nullptr, "no pc?");
   _cb = CodeCache::find_blob(pc);
   setup(pc);
 }
@@ -73,10 +73,10 @@ inline void frame::setup(address pc) {
   adjust_unextended_sp();
 
   address original_pc = CompiledMethod::get_deopt_original_pc(this);
-  if (original_pc != NULL) {
+  if (original_pc != nullptr) {
     _pc = original_pc;
     _deopt_state = is_deoptimized;
-    assert(_cb == NULL || _cb->as_compiled_method()->insts_contains_inclusive(_pc),
+    assert(_cb == nullptr || _cb->as_compiled_method()->insts_contains_inclusive(_pc),
            "original PC must be in the main code section of the compiled method (or must be immediately following it)");
   } else {
     if (_cb == SharedRuntime::deopt_blob()) {
@@ -100,10 +100,10 @@ inline frame::frame(intptr_t* sp, intptr_t* unextended_sp, intptr_t* fp, address
   _unextended_sp = unextended_sp;
   _fp = fp;
   _pc = pc;
-  assert(pc != NULL, "no pc?");
+  assert(pc != nullptr, "no pc?");
   _cb = cb;
-  _oop_map = NULL;
-  assert(_cb != NULL, "pc: " INTPTR_FORMAT, p2i(pc));
+  _oop_map = nullptr;
+  assert(_cb != nullptr, "pc: " INTPTR_FORMAT, p2i(pc));
   _on_heap = false;
   DEBUG_ONLY(_frame_index = -1;)
 
@@ -124,7 +124,7 @@ inline frame::frame(intptr_t* sp, intptr_t* unextended_sp, intptr_t* fp, address
 
   // In thaw, non-heap frames use this constructor to pass oop_map.  I don't know why.
   assert(_on_heap || _cb != nullptr, "these frames are always heap frames");
-  if (cb != NULL) {
+  if (cb != nullptr) {
     setup(pc);
   }
 #ifdef ASSERT
@@ -144,8 +144,8 @@ inline frame::frame(intptr_t* sp, intptr_t* unextended_sp, intptr_t* fp, address
   _fp = fp;
   _pc = pc;
   _cb = CodeCache::find_blob_fast(pc);
-  _oop_map = NULL;
-  assert(_cb != NULL, "pc: " INTPTR_FORMAT " sp: " INTPTR_FORMAT " unextended_sp: " INTPTR_FORMAT " fp: " INTPTR_FORMAT, p2i(pc), p2i(sp), p2i(unextended_sp), p2i(fp));
+  _oop_map = nullptr;
+  assert(_cb != nullptr, "pc: " INTPTR_FORMAT " sp: " INTPTR_FORMAT " unextended_sp: " INTPTR_FORMAT " fp: " INTPTR_FORMAT, p2i(pc), p2i(sp), p2i(unextended_sp), p2i(fp));
   _on_heap = false;
   DEBUG_ONLY(_frame_index = -1;)
 
@@ -171,13 +171,13 @@ inline frame::frame(intptr_t* sp, intptr_t* fp) {
   // call a specilaized frame constructor instead of this one.
   // Then we could use the assert below. However this assert is of somewhat dubious
   // value.
-  // assert(_pc != NULL, "no pc?");
+  // assert(_pc != nullptr, "no pc?");
 
   _cb = CodeCache::find_blob(_pc);
   adjust_unextended_sp();
 
   address original_pc = CompiledMethod::get_deopt_original_pc(this);
-  if (original_pc != NULL) {
+  if (original_pc != nullptr) {
     _pc = original_pc;
     _deopt_state = is_deoptimized;
   } else {
@@ -198,19 +198,19 @@ inline bool frame::equal(frame other) const {
 }
 
 // Return unique id for this frame. The id must have a value where we can distinguish
-// identity and younger/older relationship. NULL represents an invalid (incomparable)
+// identity and younger/older relationship. null represents an invalid (incomparable)
 // frame.
 inline intptr_t* frame::id(void) const { return unextended_sp(); }
 
 // Return true if the frame is older (less recent activation) than the frame represented by id
-inline bool frame::is_older(intptr_t* id) const   { assert(this->id() != NULL && id != NULL, "NULL frame id");
+inline bool frame::is_older(intptr_t* id) const   { assert(this->id() != nullptr && id != nullptr, "null frame id");
                                                     return this->id() > id ; }
 
 inline intptr_t* frame::link() const              { return (intptr_t*) *(intptr_t **)addr_at(link_offset); }
 
 inline intptr_t* frame::link_or_null() const {
   intptr_t** ptr = (intptr_t **)addr_at(link_offset);
-  return os::is_readable_pointer(ptr) ? *ptr : NULL;
+  return os::is_readable_pointer(ptr) ? *ptr : nullptr;
 }
 
 inline intptr_t* frame::unextended_sp() const          { assert_absolute(); return _unextended_sp; }
@@ -219,7 +219,7 @@ inline int  frame::offset_unextended_sp() const        { assert_offset();   retu
 inline void frame::set_offset_unextended_sp(int value) { assert_on_heap();  _offset_unextended_sp = value; }
 
 inline intptr_t* frame::real_fp() const {
-  if (_cb != NULL) {
+  if (_cb != nullptr) {
     // use the frame size if valid
     int size = _cb->frame_size();
     if (size > 0) {
@@ -243,7 +243,7 @@ inline int frame::compiled_frame_stack_argsize() const {
 }
 
 inline void frame::interpreted_frame_oop_map(InterpreterOopMap* mask) const {
-  assert(mask != NULL, "");
+  assert(mask != nullptr, "");
   Method* m = interpreter_frame_method();
   int   bci = interpreter_frame_bci();
   m->mask_for(bci, mask); // OopMapCache::compute_one_oop_map(m, bci, mask);
@@ -296,7 +296,7 @@ inline oop* frame::interpreter_frame_mirror_addr() const {
 // top of expression stack
 inline intptr_t* frame::interpreter_frame_tos_address() const {
   intptr_t* last_sp = interpreter_frame_last_sp();
-  if (last_sp == NULL) {
+  if (last_sp == nullptr) {
     return sp();
   } else {
     // sp() may have been extended or shrunk by an adapter.  At least
@@ -336,13 +336,13 @@ inline JavaCallWrapper** frame::entry_frame_call_wrapper_addr() const {
 
 inline oop frame::saved_oop_result(RegisterMap* map) const {
   oop* result_adr = (oop *)map->location(r0->as_VMReg(), sp());
-  guarantee(result_adr != NULL, "bad register save location");
+  guarantee(result_adr != nullptr, "bad register save location");
   return *result_adr;
 }
 
 inline void frame::set_saved_oop_result(RegisterMap* map, oop obj) {
   oop* result_adr = (oop *)map->location(r0->as_VMReg(), sp());
-  guarantee(result_adr != NULL, "bad register save location");
+  guarantee(result_adr != nullptr, "bad register save location");
 
   *result_adr = obj;
 }
@@ -356,17 +356,17 @@ inline int frame::sender_sp_ret_address_offset() {
 }
 
 inline const ImmutableOopMap* frame::get_oop_map() const {
-  if (_cb == NULL) return NULL;
-  if (_cb->oop_maps() != NULL) {
+  if (_cb == nullptr) return nullptr;
+  if (_cb->oop_maps() != nullptr) {
     NativePostCallNop* nop = nativePostCallNop_at(_pc);
-    if (nop != NULL && nop->displacement() != 0) {
+    if (nop != nullptr && nop->displacement() != 0) {
       int slot = ((nop->displacement() >> 24) & 0xff);
       return _cb->oop_map_for_slot(slot, _pc);
     }
     const ImmutableOopMap* oop_map = OopMapSet::find_map(this);
     return oop_map;
   }
-  return NULL;
+  return nullptr;
 }
 
 //------------------------------------------------------------------------------
@@ -395,7 +395,7 @@ inline frame frame::sender_raw(RegisterMap* map) const {
   if (is_interpreted_frame()) return sender_for_interpreter_frame(map);
 
   assert(_cb == CodeCache::find_blob(pc()), "Must be the same");
-  if (_cb != NULL) return sender_for_compiled_frame(map);
+  if (_cb != nullptr) return sender_for_compiled_frame(map);
 
   // Must be native-compiled frame, i.e. the marshaling code for native
   // methods that exists in the core system.
@@ -428,13 +428,13 @@ inline frame frame::sender_for_compiled_frame(RegisterMap* map) const {
     // outside of update_register_map.
     if (!_cb->is_compiled()) { // compiled frames do not use callee-saved registers
       map->set_include_argument_oops(_cb->caller_must_gc_arguments(map->thread()));
-      if (oop_map() != NULL) {
+      if (oop_map() != nullptr) {
         _oop_map->update_register_map(this, map);
       }
     } else {
       assert(!_cb->caller_must_gc_arguments(map->thread()), "");
       assert(!map->include_argument_oops(), "");
-      assert(oop_map() == NULL || !oop_map()->has_any(OopMapValue::callee_saved_value), "callee-saved value in compiled frame");
+      assert(oop_map() == nullptr || !oop_map()->has_any(OopMapValue::callee_saved_value), "callee-saved value in compiled frame");
     }
 
     // Since the prolog does the save and restore of FP there is no oopmap

src/hotspot/cpu/aarch64/gc/g1/g1BarrierSetAssembler_aarch64.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2018, 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -159,7 +159,7 @@ void G1BarrierSetAssembler::g1_write_barrier_pre(MacroAssembler* masm,
 
   // Calling the runtime using the regular call_VM_leaf mechanism generates
   // code (generated by InterpreterMacroAssember::call_VM_leaf_base)
-  // that checks that the *(rfp+frame::interpreter_frame_last_sp) == NULL.
+  // that checks that the *(rfp+frame::interpreter_frame_last_sp) == nullptr.
   //
   // If we care generating the pre-barrier without a frame (e.g. in the
   // intrinsified Reference.get() routine) then rfp might be pointing to
@@ -210,11 +210,11 @@ void G1BarrierSetAssembler::g1_write_barrier_post(MacroAssembler* masm,
   __ lsr(tmp1, tmp1, HeapRegion::LogOfHRGrainBytes);
   __ cbz(tmp1, done);
 
-  // crosses regions, storing NULL?
+  // crosses regions, storing null?
 
   __ cbz(new_val, done);
 
-  // storing region crossing non-NULL, is card already dirty?
+  // storing region crossing non-null, is card already dirty?
 
   const Register card_addr = tmp1;
 
@@ -234,7 +234,7 @@ void G1BarrierSetAssembler::g1_write_barrier_post(MacroAssembler* masm,
   __ ldrb(tmp2, Address(card_addr));
   __ cbzw(tmp2, done);
 
-  // storing a region crossing, non-NULL oop, card is clean.
+  // storing a region crossing, non-null oop, card is clean.
   // dirty card and log.
 
   __ strb(zr, Address(card_addr));
@@ -427,7 +427,7 @@ void G1BarrierSetAssembler::generate_c1_post_barrier_runtime_stub(StubAssembler*
   Label done;
   Label runtime;
 
-  // At this point we know new_value is non-NULL and the new_value crosses regions.
+  // At this point we know new_value is non-null and the new_value crosses regions.
   // Must check to see if card is already dirty
 
   const Register thread = rthread;
@@ -454,7 +454,7 @@ void G1BarrierSetAssembler::generate_c1_post_barrier_runtime_stub(StubAssembler*
   __ ldrb(rscratch1, Address(byte_map_base, card_offset));
   __ cbzw(rscratch1, done);
 
-  // storing region crossing non-NULL, card is clean.
+  // storing region crossing non-null, card is clean.
   // dirty card and log.
   __ strb(zr, Address(byte_map_base, card_offset));
 

src/hotspot/cpu/aarch64/gc/shared/barrierSetAssembler_aarch64.cpp

@@ -297,7 +297,7 @@ void BarrierSetAssembler::clear_patching_epoch() {
 void BarrierSetAssembler::nmethod_entry_barrier(MacroAssembler* masm, Label* slow_path, Label* continuation, Label* guard) {
   BarrierSetNMethod* bs_nm = BarrierSet::barrier_set()->barrier_set_nmethod();
 
-  if (bs_nm == NULL) {
+  if (bs_nm == nullptr) {
     return;
   }
 
@@ -305,13 +305,13 @@ void BarrierSetAssembler::nmethod_entry_barrier(MacroAssembler* masm, Label* slo
   Label skip_barrier;
   NMethodPatchingType patching_type = nmethod_patching_type();
 
-  if (slow_path == NULL) {
+  if (slow_path == nullptr) {
     guard = &local_guard;
   }
 
   // If the slow path is out of line in a stub, we flip the condition
-  Assembler::Condition condition = slow_path == NULL ? Assembler::EQ : Assembler::NE;
-  Label& barrier_target = slow_path == NULL ? skip_barrier : *slow_path;
+  Assembler::Condition condition = slow_path == nullptr ? Assembler::EQ : Assembler::NE;
+  Label& barrier_target = slow_path == nullptr ? skip_barrier : *slow_path;
 
   __ ldrw(rscratch1, *guard);
 
@@ -357,7 +357,7 @@ void BarrierSetAssembler::nmethod_entry_barrier(MacroAssembler* masm, Label* slo
   }
   __ br(condition, barrier_target);
 
-  if (slow_path == NULL) {
+  if (slow_path == nullptr) {
     __ movptr(rscratch1, (uintptr_t) StubRoutines::aarch64::method_entry_barrier());
     __ blr(rscratch1);
     __ b(skip_barrier);
@@ -374,7 +374,7 @@ void BarrierSetAssembler::nmethod_entry_barrier(MacroAssembler* masm, Label* slo
 
 void BarrierSetAssembler::c2i_entry_barrier(MacroAssembler* masm) {
   BarrierSetNMethod* bs = BarrierSet::barrier_set()->barrier_set_nmethod();
-  if (bs == NULL) {
+  if (bs == nullptr) {
     return;
   }
 
@@ -417,5 +417,5 @@ void BarrierSetAssembler::check_oop(MacroAssembler* masm, Register obj, Register
 
   // make sure klass is 'reasonable', which is not zero.
   __ load_klass(obj, obj); // get klass
-  __ cbz(obj, error);      // if klass is NULL it is broken
+  __ cbz(obj, error);      // if klass is null it is broken
 }

src/hotspot/cpu/aarch64/gc/shared/barrierSetNMethod_aarch64.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2018, 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -32,17 +32,21 @@
 #include "memory/resourceArea.hpp"
 #include "runtime/frame.inline.hpp"
 #include "runtime/javaThread.hpp"
-#include "runtime/sharedRuntime.hpp"
 #include "runtime/registerMap.hpp"
+#include "runtime/sharedRuntime.hpp"
 #include "utilities/align.hpp"
 #include "utilities/debug.hpp"
+#include "utilities/formatBuffer.hpp"
+#if INCLUDE_JVMCI
+#include "jvmci/jvmciRuntime.hpp"
+#endif
 
 static int slow_path_size(nmethod* nm) {
   // The slow path code is out of line with C2
   return nm->is_compiled_by_c2() ? 0 : 6;
 }
 
-// This is the offset of the entry barrier from where the frame is completed.
+// This is the offset of the entry barrier relative to where the frame is completed.
 // If any code changes between the end of the verified entry where the entry
 // barrier resides, and the completion of the frame, then
 // NativeNMethodCmpBarrier::verify() will immediately complain when it does
@@ -62,58 +66,80 @@ static int entry_barrier_offset(nmethod* nm) {
   return 0;
 }
 
-class NativeNMethodBarrier: public NativeInstruction {
-  address instruction_address() const { return addr_at(0); }
+class NativeNMethodBarrier {
+  address  _instruction_address;
+  int*     _guard_addr;
+  nmethod* _nm;
+
+  address instruction_address() const { return _instruction_address; }
+
+  int *guard_addr() {
+    return _guard_addr;
+  }
 
   int local_guard_offset(nmethod* nm) {
     // It's the last instruction
     return (-entry_barrier_offset(nm)) - 4;
   }
 
-  int *guard_addr(nmethod* nm) {
-    if (nm->is_compiled_by_c2()) {
-      // With c2 compiled code, the guard is out-of-line in a stub
-      // We find it using the RelocIterator.
-      RelocIterator iter(nm);
-      while (iter.next()) {
-        if (iter.type() == relocInfo::entry_guard_type) {
-          entry_guard_Relocation* const reloc = iter.entry_guard_reloc();
-          return reinterpret_cast<int*>(reloc->addr());
-        }
-      }
-      ShouldNotReachHere();
-    }
-    return reinterpret_cast<int*>(instruction_address() + local_guard_offset(nm));
-  }
-
 public:
-  int get_value(nmethod* nm) {
-    return Atomic::load_acquire(guard_addr(nm));
+  NativeNMethodBarrier(nmethod* nm): _nm(nm) {
+#if INCLUDE_JVMCI
+    if (nm->is_compiled_by_jvmci()) {
+      address pc = nm->code_begin() + nm->jvmci_nmethod_data()->nmethod_entry_patch_offset();
+      RelocIterator iter(nm, pc, pc + 4);
+      guarantee(iter.next(), "missing relocs");
+      guarantee(iter.type() == relocInfo::section_word_type, "unexpected reloc");
+
+      _guard_addr = (int*) iter.section_word_reloc()->target();
+      _instruction_address = pc;
+    } else
+#endif
+      {
+        _instruction_address = nm->code_begin() + nm->frame_complete_offset() + entry_barrier_offset(nm);
+        if (nm->is_compiled_by_c2()) {
+          // With c2 compiled code, the guard is out-of-line in a stub
+          // We find it using the RelocIterator.
+          RelocIterator iter(nm);
+          while (iter.next()) {
+            if (iter.type() == relocInfo::entry_guard_type) {
+              entry_guard_Relocation* const reloc = iter.entry_guard_reloc();
+              _guard_addr = reinterpret_cast<int*>(reloc->addr());
+              return;
+            }
+          }
+          ShouldNotReachHere();
+        }
+        _guard_addr =  reinterpret_cast<int*>(instruction_address() + local_guard_offset(nm));
+      }
   }
 
-  void set_value(nmethod* nm, int value) {
-    Atomic::release_store(guard_addr(nm), value);
+  int get_value() {
+    return Atomic::load_acquire(guard_addr());
   }
 
-  void verify() const;
-};
+  void set_value(int value) {
+    Atomic::release_store(guard_addr(), value);
+  }
 
-// Store the instruction bitmask, bits and name for checking the barrier.
-struct CheckInsn {
-  uint32_t mask;
-  uint32_t bits;
-  const char *name;
+  bool check_barrier(err_msg& msg) const;
+  void verify() const {
+    err_msg msg("%s", "");
+    assert(check_barrier(msg), "%s", msg.buffer());
+  }
 };
 
 // The first instruction of the nmethod entry barrier is an ldr (literal)
 // instruction. Verify that it's really there, so the offsets are not skewed.
-void NativeNMethodBarrier::verify() const {
+bool NativeNMethodBarrier::check_barrier(err_msg& msg) const {
   uint32_t* addr = (uint32_t*) instruction_address();
   uint32_t inst = *addr;
   if ((inst & 0xff000000) != 0x18000000) {
-    tty->print_cr("Addr: " INTPTR_FORMAT " Code: 0x%x", (intptr_t)addr, inst);
-    fatal("not an ldr (literal) instruction.");
+    msg.print("Nmethod entry barrier did not start with ldr (literal) as expected. "
+              "Addr: " PTR_FORMAT " Code: " UINT32_FORMAT, p2i(addr), inst);
+    return false;
   }
+  return true;
 }
 
 
@@ -156,13 +182,6 @@ void BarrierSetNMethod::deoptimize(nmethod* nm, address* return_address_ptr) {
   new_frame->pc = SharedRuntime::get_handle_wrong_method_stub();
 }
 
-static NativeNMethodBarrier* native_nmethod_barrier(nmethod* nm) {
-  address barrier_address = nm->code_begin() + nm->frame_complete_offset() + entry_barrier_offset(nm);
-  NativeNMethodBarrier* barrier = reinterpret_cast<NativeNMethodBarrier*>(barrier_address);
-  debug_only(barrier->verify());
-  return barrier;
-}
-
 void BarrierSetNMethod::set_guard_value(nmethod* nm, int value) {
   if (!supports_entry_barrier(nm)) {
     return;
@@ -179,8 +198,8 @@ void BarrierSetNMethod::set_guard_value(nmethod* nm, int value) {
     bs_asm->increment_patching_epoch();
   }
 
-  NativeNMethodBarrier* barrier = native_nmethod_barrier(nm);
-  barrier->set_value(nm, value);
+  NativeNMethodBarrier barrier(nm);
+  barrier.set_value(value);
 }
 
 int BarrierSetNMethod::guard_value(nmethod* nm) {
@@ -188,6 +207,13 @@ int BarrierSetNMethod::guard_value(nmethod* nm) {
     return disarmed_guard_value();
   }
 
-  NativeNMethodBarrier* barrier = native_nmethod_barrier(nm);
-  return barrier->get_value(nm);
+  NativeNMethodBarrier barrier(nm);
+  return barrier.get_value();
 }
+
+#if INCLUDE_JVMCI
+bool BarrierSetNMethod::verify_barrier(nmethod* nm, err_msg& msg) {
+  NativeNMethodBarrier barrier(nm);
+  return barrier.check_barrier(msg);
+}
+#endif

src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp

@@ -157,7 +157,7 @@ void ShenandoahBarrierSetAssembler::satb_write_barrier_pre(MacroAssembler* masm,
 
   // Calling the runtime using the regular call_VM_leaf mechanism generates
   // code (generated by InterpreterMacroAssember::call_VM_leaf_base)
-  // that checks that the *(rfp+frame::interpreter_frame_last_sp) == NULL.
+  // that checks that the *(rfp+frame::interpreter_frame_last_sp) == nullptr.
   //
   // If we care generating the pre-barrier without a frame (e.g. in the
   // intrinsified Reference.get() routine) then rfp might be pointing to
@@ -447,7 +447,7 @@ void ShenandoahBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler
 //  b) A parallel thread may heal the contents of addr, replacing a
 //     from-space pointer held in addr with the to-space pointer
 //     representing the new location of the object.
-// Upon entry to cmpxchg_oop, it is assured that new_val equals NULL
+// Upon entry to cmpxchg_oop, it is assured that new_val equals null
 // or it refers to an object that is not being evacuated out of
 // from-space, or it refers to the to-space version of an object that
 // is being evacuated out of from-space.

src/hotspot/cpu/aarch64/gc/x/xBarrierSetAssembler_aarch64.cpp

@@ -0,0 +1,462 @@
+/*
+ * Copyright (c) 2019, 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "asm/macroAssembler.inline.hpp"
+#include "code/codeBlob.hpp"
+#include "code/vmreg.inline.hpp"
+#include "gc/x/xBarrier.inline.hpp"
+#include "gc/x/xBarrierSet.hpp"
+#include "gc/x/xBarrierSetAssembler.hpp"
+#include "gc/x/xBarrierSetRuntime.hpp"
+#include "gc/x/xThreadLocalData.hpp"
+#include "memory/resourceArea.hpp"
+#include "runtime/sharedRuntime.hpp"
+#include "utilities/macros.hpp"
+#ifdef COMPILER1
+#include "c1/c1_LIRAssembler.hpp"
+#include "c1/c1_MacroAssembler.hpp"
+#include "gc/x/c1/xBarrierSetC1.hpp"
+#endif // COMPILER1
+#ifdef COMPILER2
+#include "gc/x/c2/xBarrierSetC2.hpp"
+#endif // COMPILER2
+
+#ifdef PRODUCT
+#define BLOCK_COMMENT(str) /* nothing */
+#else
+#define BLOCK_COMMENT(str) __ block_comment(str)
+#endif
+
+#undef __
+#define __ masm->
+
+void XBarrierSetAssembler::load_at(MacroAssembler* masm,
+                                   DecoratorSet decorators,
+                                   BasicType type,
+                                   Register dst,
+                                   Address src,
+                                   Register tmp1,
+                                   Register tmp2) {
+  if (!XBarrierSet::barrier_needed(decorators, type)) {
+    // Barrier not needed
+    BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp2);
+    return;
+  }
+
+  assert_different_registers(rscratch1, rscratch2, src.base());
+  assert_different_registers(rscratch1, rscratch2, dst);
+
+  Label done;
+
+  // Load bad mask into scratch register.
+  __ ldr(rscratch1, address_bad_mask_from_thread(rthread));
+  __ lea(rscratch2, src);
+  __ ldr(dst, src);
+
+  // Test reference against bad mask. If mask bad, then we need to fix it up.
+  __ tst(dst, rscratch1);
+  __ br(Assembler::EQ, done);
+
+  __ enter(/*strip_ret_addr*/true);
+
+  __ push_call_clobbered_registers_except(RegSet::of(dst));
+
+  if (c_rarg0 != dst) {
+    __ mov(c_rarg0, dst);
+  }
+  __ mov(c_rarg1, rscratch2);
+
+  __ call_VM_leaf(XBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(decorators), 2);
+
+  // Make sure dst has the return value.
+  if (dst != r0) {
+    __ mov(dst, r0);
+  }
+
+  __ pop_call_clobbered_registers_except(RegSet::of(dst));
+  __ leave();
+
+  __ bind(done);
+}
+
+#ifdef ASSERT
+
+void XBarrierSetAssembler::store_at(MacroAssembler* masm,
+                                        DecoratorSet decorators,
+                                        BasicType type,
+                                        Address dst,
+                                        Register val,
+                                        Register tmp1,
+                                        Register tmp2,
+                                        Register tmp3) {
+  // Verify value
+  if (is_reference_type(type)) {
+    // Note that src could be noreg, which means we
+    // are storing null and can skip verification.
+    if (val != noreg) {
+      Label done;
+
+      // tmp1, tmp2 and tmp3 are often set to noreg.
+      RegSet savedRegs = RegSet::of(rscratch1);
+      __ push(savedRegs, sp);
+
+      __ ldr(rscratch1, address_bad_mask_from_thread(rthread));
+      __ tst(val, rscratch1);
+      __ br(Assembler::EQ, done);
+      __ stop("Verify oop store failed");
+      __ should_not_reach_here();
+      __ bind(done);
+      __ pop(savedRegs, sp);
+    }
+  }
+
+  // Store value
+  BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2, noreg);
+}
+
+#endif // ASSERT
+
+void XBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm,
+                                              DecoratorSet decorators,
+                                              bool is_oop,
+                                              Register src,
+                                              Register dst,
+                                              Register count,
+                                              RegSet saved_regs) {
+  if (!is_oop) {
+    // Barrier not needed
+    return;
+  }
+
+  BLOCK_COMMENT("XBarrierSetAssembler::arraycopy_prologue {");
+
+  assert_different_registers(src, count, rscratch1);
+
+  __ push(saved_regs, sp);
+
+  if (count == c_rarg0) {
+    if (src == c_rarg1) {
+      // exactly backwards!!
+      __ mov(rscratch1, c_rarg0);
+      __ mov(c_rarg0, c_rarg1);
+      __ mov(c_rarg1, rscratch1);
+    } else {
+      __ mov(c_rarg1, count);
+      __ mov(c_rarg0, src);
+    }
+  } else {
+    __ mov(c_rarg0, src);
+    __ mov(c_rarg1, count);
+  }
+
+  __ call_VM_leaf(XBarrierSetRuntime::load_barrier_on_oop_array_addr(), 2);
+
+  __ pop(saved_regs, sp);
+
+  BLOCK_COMMENT("} XBarrierSetAssembler::arraycopy_prologue");
+}
+
+void XBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm,
+                                                         Register jni_env,
+                                                         Register robj,
+                                                         Register tmp,
+                                                         Label& slowpath) {
+  BLOCK_COMMENT("XBarrierSetAssembler::try_resolve_jobject_in_native {");
+
+  assert_different_registers(jni_env, robj, tmp);
+
+  // Resolve jobject
+  BarrierSetAssembler::try_resolve_jobject_in_native(masm, jni_env, robj, tmp, slowpath);
+
+  // The Address offset is too large to direct load - -784. Our range is +127, -128.
+  __ mov(tmp, (int64_t)(in_bytes(XThreadLocalData::address_bad_mask_offset()) -
+              in_bytes(JavaThread::jni_environment_offset())));
+
+  // Load address bad mask
+  __ add(tmp, jni_env, tmp);
+  __ ldr(tmp, Address(tmp));
+
+  // Check address bad mask
+  __ tst(robj, tmp);
+  __ br(Assembler::NE, slowpath);
+
+  BLOCK_COMMENT("} XBarrierSetAssembler::try_resolve_jobject_in_native");
+}
+
+#ifdef COMPILER1
+
+#undef __
+#define __ ce->masm()->
+
+void XBarrierSetAssembler::generate_c1_load_barrier_test(LIR_Assembler* ce,
+                                                         LIR_Opr ref) const {
+  assert_different_registers(rscratch1, rthread, ref->as_register());
+
+  __ ldr(rscratch1, address_bad_mask_from_thread(rthread));
+  __ tst(ref->as_register(), rscratch1);
+}
+
+void XBarrierSetAssembler::generate_c1_load_barrier_stub(LIR_Assembler* ce,
+                                                         XLoadBarrierStubC1* stub) const {
+  // Stub entry
+  __ bind(*stub->entry());
+
+  Register ref = stub->ref()->as_register();
+  Register ref_addr = noreg;
+  Register tmp = noreg;
+
+  if (stub->tmp()->is_valid()) {
+    // Load address into tmp register
+    ce->leal(stub->ref_addr(), stub->tmp());
+    ref_addr = tmp = stub->tmp()->as_pointer_register();
+  } else {
+    // Address already in register
+    ref_addr = stub->ref_addr()->as_address_ptr()->base()->as_pointer_register();
+  }
+
+  assert_different_registers(ref, ref_addr, noreg);
+
+  // Save r0 unless it is the result or tmp register
+  // Set up SP to accommodate parameters and maybe r0..
+  if (ref != r0 && tmp != r0) {
+    __ sub(sp, sp, 32);
+    __ str(r0, Address(sp, 16));
+  } else {
+    __ sub(sp, sp, 16);
+  }
+
+  // Setup arguments and call runtime stub
+  ce->store_parameter(ref_addr, 1);
+  ce->store_parameter(ref, 0);
+
+  __ far_call(stub->runtime_stub());
+
+  // Verify result
+  __ verify_oop(r0);
+
+  // Move result into place
+  if (ref != r0) {
+    __ mov(ref, r0);
+  }
+
+  // Restore r0 unless it is the result or tmp register
+  if (ref != r0 && tmp != r0) {
+    __ ldr(r0, Address(sp, 16));
+    __ add(sp, sp, 32);
+  } else {
+    __ add(sp, sp, 16);
+  }
+
+  // Stub exit
+  __ b(*stub->continuation());
+}
+
+#undef __
+#define __ sasm->
+
+void XBarrierSetAssembler::generate_c1_load_barrier_runtime_stub(StubAssembler* sasm,
+                                                                 DecoratorSet decorators) const {
+  __ prologue("zgc_load_barrier stub", false);
+
+  __ push_call_clobbered_registers_except(RegSet::of(r0));
+
+  // Setup arguments
+  __ load_parameter(0, c_rarg0);
+  __ load_parameter(1, c_rarg1);
+
+  __ call_VM_leaf(XBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(decorators), 2);
+
+  __ pop_call_clobbered_registers_except(RegSet::of(r0));
+
+  __ epilogue();
+}
+#endif // COMPILER1
+
+#ifdef COMPILER2
+
+OptoReg::Name XBarrierSetAssembler::refine_register(const Node* node, OptoReg::Name opto_reg) {
+  if (!OptoReg::is_reg(opto_reg)) {
+    return OptoReg::Bad;
+  }
+
+  const VMReg vm_reg = OptoReg::as_VMReg(opto_reg);
+  if (vm_reg->is_FloatRegister()) {
+    return opto_reg & ~1;
+  }
+
+  return opto_reg;
+}
+
+#undef __
+#define __ _masm->
+
+class XSaveLiveRegisters {
+private:
+  MacroAssembler* const _masm;
+  RegSet                _gp_regs;
+  FloatRegSet           _fp_regs;
+  PRegSet               _p_regs;
+
+public:
+  void initialize(XLoadBarrierStubC2* stub) {
+    // Record registers that needs to be saved/restored
+    RegMaskIterator rmi(stub->live());
+    while (rmi.has_next()) {
+      const OptoReg::Name opto_reg = rmi.next();
+      if (OptoReg::is_reg(opto_reg)) {
+        const VMReg vm_reg = OptoReg::as_VMReg(opto_reg);
+        if (vm_reg->is_Register()) {
+          _gp_regs += RegSet::of(vm_reg->as_Register());
+        } else if (vm_reg->is_FloatRegister()) {
+          _fp_regs += FloatRegSet::of(vm_reg->as_FloatRegister());
+        } else if (vm_reg->is_PRegister()) {
+          _p_regs += PRegSet::of(vm_reg->as_PRegister());
+        } else {
+          fatal("Unknown register type");
+        }
+      }
+    }
+
+    // Remove C-ABI SOE registers, scratch regs and _ref register that will be updated
+    _gp_regs -= RegSet::range(r19, r30) + RegSet::of(r8, r9, stub->ref());
+  }
+
+  XSaveLiveRegisters(MacroAssembler* masm, XLoadBarrierStubC2* stub) :
+      _masm(masm),
+      _gp_regs(),
+      _fp_regs(),
+      _p_regs() {
+
+    // Figure out what registers to save/restore
+    initialize(stub);
+
+    // Save registers
+    __ push(_gp_regs, sp);
+    __ push_fp(_fp_regs, sp);
+    __ push_p(_p_regs, sp);
+  }
+
+  ~XSaveLiveRegisters() {
+    // Restore registers
+    __ pop_p(_p_regs, sp);
+    __ pop_fp(_fp_regs, sp);
+
+    // External runtime call may clobber ptrue reg
+    __ reinitialize_ptrue();
+
+    __ pop(_gp_regs, sp);
+  }
+};
+
+#undef __
+#define __ _masm->
+
+class XSetupArguments {
+private:
+  MacroAssembler* const _masm;
+  const Register        _ref;
+  const Address         _ref_addr;
+
+public:
+  XSetupArguments(MacroAssembler* masm, XLoadBarrierStubC2* stub) :
+      _masm(masm),
+      _ref(stub->ref()),
+      _ref_addr(stub->ref_addr()) {
+
+    // Setup arguments
+    if (_ref_addr.base() == noreg) {
+      // No self healing
+      if (_ref != c_rarg0) {
+        __ mov(c_rarg0, _ref);
+      }
+      __ mov(c_rarg1, 0);
+    } else {
+      // Self healing
+      if (_ref == c_rarg0) {
+        // _ref is already at correct place
+        __ lea(c_rarg1, _ref_addr);
+      } else if (_ref != c_rarg1) {
+        // _ref is in wrong place, but not in c_rarg1, so fix it first
+        __ lea(c_rarg1, _ref_addr);
+        __ mov(c_rarg0, _ref);
+      } else if (_ref_addr.base() != c_rarg0 && _ref_addr.index() != c_rarg0) {
+        assert(_ref == c_rarg1, "Mov ref first, vacating c_rarg0");
+        __ mov(c_rarg0, _ref);
+        __ lea(c_rarg1, _ref_addr);
+      } else {
+        assert(_ref == c_rarg1, "Need to vacate c_rarg1 and _ref_addr is using c_rarg0");
+        if (_ref_addr.base() == c_rarg0 || _ref_addr.index() == c_rarg0) {
+          __ mov(rscratch2, c_rarg1);
+          __ lea(c_rarg1, _ref_addr);
+          __ mov(c_rarg0, rscratch2);
+        } else {
+          ShouldNotReachHere();
+        }
+      }
+    }
+  }
+
+  ~XSetupArguments() {
+    // Transfer result
+    if (_ref != r0) {
+      __ mov(_ref, r0);
+    }
+  }
+};
+
+#undef __
+#define __ masm->
+
+void XBarrierSetAssembler::generate_c2_load_barrier_stub(MacroAssembler* masm, XLoadBarrierStubC2* stub) const {
+  BLOCK_COMMENT("XLoadBarrierStubC2");
+
+  // Stub entry
+  __ bind(*stub->entry());
+
+  {
+    XSaveLiveRegisters save_live_registers(masm, stub);
+    XSetupArguments setup_arguments(masm, stub);
+    __ mov(rscratch1, stub->slow_path());
+    __ blr(rscratch1);
+  }
+  // Stub exit
+  __ b(*stub->continuation());
+}
+
+#endif // COMPILER2
+
+#undef __
+#define __ masm->
+
+void XBarrierSetAssembler::check_oop(MacroAssembler* masm, Register obj, Register tmp1, Register tmp2, Label& error) {
+  // Check if mask is good.
+  // verifies that XAddressBadMask & r0 == 0
+  __ ldr(tmp2, Address(rthread, XThreadLocalData::address_bad_mask_offset()));
+  __ andr(tmp1, obj, tmp2);
+  __ cbnz(tmp1, error);
+
+  BarrierSetAssembler::check_oop(masm, obj, tmp1, tmp2, error);
+}
+
+#undef __

src/hotspot/cpu/aarch64/gc/x/xBarrierSetAssembler_aarch64.hpp

@@ -0,0 +1,110 @@
+/*
+ * Copyright (c) 2019, 2022, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef CPU_AARCH64_GC_X_XBARRIERSETASSEMBLER_AARCH64_HPP
+#define CPU_AARCH64_GC_X_XBARRIERSETASSEMBLER_AARCH64_HPP
+
+#include "code/vmreg.hpp"
+#include "oops/accessDecorators.hpp"
+#ifdef COMPILER2
+#include "opto/optoreg.hpp"
+#endif // COMPILER2
+
+#ifdef COMPILER1
+class LIR_Assembler;
+class LIR_Opr;
+class StubAssembler;
+#endif // COMPILER1
+
+#ifdef COMPILER2
+class Node;
+#endif // COMPILER2
+
+#ifdef COMPILER1
+class XLoadBarrierStubC1;
+#endif // COMPILER1
+
+#ifdef COMPILER2
+class XLoadBarrierStubC2;
+#endif // COMPILER2
+
+class XBarrierSetAssembler : public XBarrierSetAssemblerBase {
+public:
+  virtual void load_at(MacroAssembler* masm,
+                       DecoratorSet decorators,
+                       BasicType type,
+                       Register dst,
+                       Address src,
+                       Register tmp1,
+                       Register tmp2);
+
+#ifdef ASSERT
+  virtual void store_at(MacroAssembler* masm,
+                        DecoratorSet decorators,
+                        BasicType type,
+                        Address dst,
+                        Register val,
+                        Register tmp1,
+                        Register tmp2,
+                        Register tmp3);
+#endif // ASSERT
+
+  virtual void arraycopy_prologue(MacroAssembler* masm,
+                                  DecoratorSet decorators,
+                                  bool is_oop,
+                                  Register src,
+                                  Register dst,
+                                  Register count,
+                                  RegSet saved_regs);
+
+  virtual void try_resolve_jobject_in_native(MacroAssembler* masm,
+                                             Register jni_env,
+                                             Register robj,
+                                             Register tmp,
+                                             Label& slowpath);
+
+  virtual NMethodPatchingType nmethod_patching_type() { return NMethodPatchingType::conc_data_patch; }
+
+#ifdef COMPILER1
+  void generate_c1_load_barrier_test(LIR_Assembler* ce,
+                                     LIR_Opr ref) const;
+
+  void generate_c1_load_barrier_stub(LIR_Assembler* ce,
+                                     XLoadBarrierStubC1* stub) const;
+
+  void generate_c1_load_barrier_runtime_stub(StubAssembler* sasm,
+                                             DecoratorSet decorators) const;
+#endif // COMPILER1
+
+#ifdef COMPILER2
+  OptoReg::Name refine_register(const Node* node,
+                                OptoReg::Name opto_reg);
+
+  void generate_c2_load_barrier_stub(MacroAssembler* masm,
+                                     XLoadBarrierStubC2* stub) const;
+#endif // COMPILER2
+
+  void check_oop(MacroAssembler* masm, Register obj, Register tmp1, Register tmp2, Label& error);
+};
+
+#endif // CPU_AARCH64_GC_X_XBARRIERSETASSEMBLER_AARCH64_HPP

src/hotspot/cpu/aarch64/gc/x/xGlobals_aarch64.cpp

@@ -24,7 +24,7 @@
 #include "precompiled.hpp"
 #include "gc/shared/gcLogPrecious.hpp"
 #include "gc/shared/gc_globals.hpp"
-#include "gc/z/zGlobals.hpp"
+#include "gc/x/xGlobals.hpp"
 #include "runtime/globals.hpp"
 #include "runtime/os.hpp"
 #include "utilities/globalDefinitions.hpp"
@@ -196,15 +196,15 @@ static size_t probe_valid_max_address_bit() {
 #endif // LINUX
 }
 
-size_t ZPlatformAddressOffsetBits() {
+size_t XPlatformAddressOffsetBits() {
   const static size_t valid_max_address_offset_bits = probe_valid_max_address_bit() + 1;
   const size_t max_address_offset_bits = valid_max_address_offset_bits - 3;
   const size_t min_address_offset_bits = max_address_offset_bits - 2;
-  const size_t address_offset = round_up_power_of_2(MaxHeapSize * ZVirtualToPhysicalRatio);
+  const size_t address_offset = round_up_power_of_2(MaxHeapSize * XVirtualToPhysicalRatio);
   const size_t address_offset_bits = log2i_exact(address_offset);
   return clamp(address_offset_bits, min_address_offset_bits, max_address_offset_bits);
 }
 
-size_t ZPlatformAddressMetadataShift() {
-  return ZPlatformAddressOffsetBits();
+size_t XPlatformAddressMetadataShift() {
+  return XPlatformAddressOffsetBits();
 }

src/hotspot/cpu/aarch64/gc/x/xGlobals_aarch64.hpp

@@ -0,0 +1,33 @@
+/*
+ * Copyright (c) 2015, 2022, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef CPU_AARCH64_GC_X_XGLOBALS_AARCH64_HPP
+#define CPU_AARCH64_GC_X_XGLOBALS_AARCH64_HPP
+
+const size_t XPlatformHeapViews        = 3;
+const size_t XPlatformCacheLineSize    = 64;
+
+size_t XPlatformAddressOffsetBits();
+size_t XPlatformAddressMetadataShift();
+
+#endif // CPU_AARCH64_GC_X_XGLOBALS_AARCH64_HPP

src/hotspot/cpu/aarch64/gc/x/x_aarch64.ad

@@ -0,0 +1,243 @@
+//
+// Copyright (c) 2019, 2021, Oracle and/or its affiliates. All rights reserved.
+// DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+//
+// This code is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License version 2 only, as
+// published by the Free Software Foundation.
+//
+// This code is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+// version 2 for more details (a copy is included in the LICENSE file that
+// accompanied this code).
+//
+// You should have received a copy of the GNU General Public License version
+// 2 along with this work; if not, write to the Free Software Foundation,
+// Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+//
+// Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+// or visit www.oracle.com if you need additional information or have any
+// questions.
+//
+
+source_hpp %{
+
+#include "gc/shared/gc_globals.hpp"
+#include "gc/x/c2/xBarrierSetC2.hpp"
+#include "gc/x/xThreadLocalData.hpp"
+
+%}
+
+source %{
+
+static void x_load_barrier(MacroAssembler& _masm, const MachNode* node, Address ref_addr, Register ref, Register tmp, uint8_t barrier_data) {
+  if (barrier_data == XLoadBarrierElided) {
+    return;
+  }
+  XLoadBarrierStubC2* const stub = XLoadBarrierStubC2::create(node, ref_addr, ref, tmp, barrier_data);
+  __ ldr(tmp, Address(rthread, XThreadLocalData::address_bad_mask_offset()));
+  __ andr(tmp, tmp, ref);
+  __ cbnz(tmp, *stub->entry());
+  __ bind(*stub->continuation());
+}
+
+static void x_load_barrier_slow_path(MacroAssembler& _masm, const MachNode* node, Address ref_addr, Register ref, Register tmp) {
+  XLoadBarrierStubC2* const stub = XLoadBarrierStubC2::create(node, ref_addr, ref, tmp, XLoadBarrierStrong);
+  __ b(*stub->entry());
+  __ bind(*stub->continuation());
+}
+
+%}
+
+// Load Pointer
+instruct xLoadP(iRegPNoSp dst, memory8 mem, rFlagsReg cr)
+%{
+  match(Set dst (LoadP mem));
+  predicate(UseZGC && !ZGenerational && !needs_acquiring_load(n) && (n->as_Load()->barrier_data() != 0));
+  effect(TEMP dst, KILL cr);
+
+  ins_cost(4 * INSN_COST);
+
+  format %{ "ldr  $dst, $mem" %}
+
+  ins_encode %{
+    const Address ref_addr = mem2address($mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
+    __ ldr($dst$$Register, ref_addr);
+    x_load_barrier(_masm, this, ref_addr, $dst$$Register, rscratch2 /* tmp */, barrier_data());
+  %}
+
+  ins_pipe(iload_reg_mem);
+%}
+
+// Load Pointer Volatile
+instruct xLoadPVolatile(iRegPNoSp dst, indirect mem /* sync_memory */, rFlagsReg cr)
+%{
+  match(Set dst (LoadP mem));
+  predicate(UseZGC && !ZGenerational && needs_acquiring_load(n) && n->as_Load()->barrier_data() != 0);
+  effect(TEMP dst, KILL cr);
+
+  ins_cost(VOLATILE_REF_COST);
+
+  format %{ "ldar  $dst, $mem\t" %}
+
+  ins_encode %{
+    __ ldar($dst$$Register, $mem$$Register);
+    x_load_barrier(_masm, this, Address($mem$$Register), $dst$$Register, rscratch2 /* tmp */, barrier_data());
+  %}
+
+  ins_pipe(pipe_serial);
+%}
+
+instruct xCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
+  match(Set res (CompareAndSwapP mem (Binary oldval newval)));
+  match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
+  predicate(UseZGC && !ZGenerational && !needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == XLoadBarrierStrong);
+  effect(KILL cr, TEMP_DEF res);
+
+  ins_cost(2 * VOLATILE_REF_COST);
+
+  format %{ "cmpxchg $mem, $oldval, $newval\n\t"
+            "cset    $res, EQ" %}
+
+  ins_encode %{
+    guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
+    __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword,
+               false /* acquire */, true /* release */, false /* weak */, rscratch2);
+    __ cset($res$$Register, Assembler::EQ);
+    if (barrier_data() != XLoadBarrierElided) {
+      Label good;
+      __ ldr(rscratch1, Address(rthread, XThreadLocalData::address_bad_mask_offset()));
+      __ andr(rscratch1, rscratch1, rscratch2);
+      __ cbz(rscratch1, good);
+      x_load_barrier_slow_path(_masm, this, Address($mem$$Register), rscratch2 /* ref */, rscratch1 /* tmp */);
+      __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword,
+                 false /* acquire */, true /* release */, false /* weak */, rscratch2);
+      __ cset($res$$Register, Assembler::EQ);
+      __ bind(good);
+    }
+  %}
+
+  ins_pipe(pipe_slow);
+%}
+
+instruct xCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
+  match(Set res (CompareAndSwapP mem (Binary oldval newval)));
+  match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
+  predicate(UseZGC && !ZGenerational && needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == XLoadBarrierStrong));
+  effect(KILL cr, TEMP_DEF res);
+
+  ins_cost(2 * VOLATILE_REF_COST);
+
+ format %{ "cmpxchg $mem, $oldval, $newval\n\t"
+           "cset    $res, EQ" %}
+
+  ins_encode %{
+    guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
+    __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword,
+               true /* acquire */, true /* release */, false /* weak */, rscratch2);
+    __ cset($res$$Register, Assembler::EQ);
+    if (barrier_data() != XLoadBarrierElided) {
+      Label good;
+      __ ldr(rscratch1, Address(rthread, XThreadLocalData::address_bad_mask_offset()));
+      __ andr(rscratch1, rscratch1, rscratch2);
+      __ cbz(rscratch1, good);
+      x_load_barrier_slow_path(_masm, this, Address($mem$$Register), rscratch2 /* ref */, rscratch1 /* tmp */ );
+      __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword,
+                 true /* acquire */, true /* release */, false /* weak */, rscratch2);
+      __ cset($res$$Register, Assembler::EQ);
+      __ bind(good);
+    }
+  %}
+
+  ins_pipe(pipe_slow);
+%}
+
+instruct xCompareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
+  match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
+  predicate(UseZGC && !ZGenerational && !needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == XLoadBarrierStrong);
+  effect(TEMP_DEF res, KILL cr);
+
+  ins_cost(2 * VOLATILE_REF_COST);
+
+  format %{ "cmpxchg $res = $mem, $oldval, $newval" %}
+
+  ins_encode %{
+    guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
+    __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword,
+               false /* acquire */, true /* release */, false /* weak */, $res$$Register);
+    if (barrier_data() != XLoadBarrierElided) {
+      Label good;
+      __ ldr(rscratch1, Address(rthread, XThreadLocalData::address_bad_mask_offset()));
+      __ andr(rscratch1, rscratch1, $res$$Register);
+      __ cbz(rscratch1, good);
+      x_load_barrier_slow_path(_masm, this, Address($mem$$Register), $res$$Register /* ref */, rscratch1 /* tmp */);
+      __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword,
+                 false /* acquire */, true /* release */, false /* weak */, $res$$Register);
+      __ bind(good);
+    }
+  %}
+
+  ins_pipe(pipe_slow);
+%}
+
+instruct xCompareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
+  match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
+  predicate(UseZGC && !ZGenerational && needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == XLoadBarrierStrong);
+  effect(TEMP_DEF res, KILL cr);
+
+  ins_cost(2 * VOLATILE_REF_COST);
+
+  format %{ "cmpxchg $res = $mem, $oldval, $newval" %}
+
+  ins_encode %{
+    guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
+    __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword,
+               true /* acquire */, true /* release */, false /* weak */, $res$$Register);
+    if (barrier_data() != XLoadBarrierElided) {
+      Label good;
+      __ ldr(rscratch1, Address(rthread, XThreadLocalData::address_bad_mask_offset()));
+      __ andr(rscratch1, rscratch1, $res$$Register);
+      __ cbz(rscratch1, good);
+      x_load_barrier_slow_path(_masm, this, Address($mem$$Register), $res$$Register /* ref */, rscratch1 /* tmp */);
+      __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword,
+                 true /* acquire */, true /* release */, false /* weak */, $res$$Register);
+      __ bind(good);
+    }
+  %}
+
+  ins_pipe(pipe_slow);
+%}
+
+instruct xGetAndSetP(indirect mem, iRegP newv, iRegPNoSp prev, rFlagsReg cr) %{
+  match(Set prev (GetAndSetP mem newv));
+  predicate(UseZGC && !ZGenerational && !needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
+  effect(TEMP_DEF prev, KILL cr);
+
+  ins_cost(2 * VOLATILE_REF_COST);
+
+  format %{ "atomic_xchg  $prev, $newv, [$mem]" %}
+
+  ins_encode %{
+    __ atomic_xchg($prev$$Register, $newv$$Register, $mem$$Register);
+    x_load_barrier(_masm, this, Address(noreg, 0), $prev$$Register, rscratch2 /* tmp */, barrier_data());
+  %}
+
+  ins_pipe(pipe_serial);
+%}
+
+instruct xGetAndSetPAcq(indirect mem, iRegP newv, iRegPNoSp prev, rFlagsReg cr) %{
+  match(Set prev (GetAndSetP mem newv));
+  predicate(UseZGC && !ZGenerational && needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() != 0));
+  effect(TEMP_DEF prev, KILL cr);
+
+  ins_cost(VOLATILE_REF_COST);
+
+  format %{ "atomic_xchg_acq  $prev, $newv, [$mem]" %}
+
+  ins_encode %{
+    __ atomic_xchgal($prev$$Register, $newv$$Register, $mem$$Register);
+    x_load_barrier(_masm, this, Address(noreg, 0), $prev$$Register, rscratch2 /* tmp */, barrier_data());
+  %}
+  ins_pipe(pipe_serial);
+%}

src/hotspot/cpu/aarch64/gc/z/zAddress_aarch64.cpp

@@ -0,0 +1,108 @@
+/*
+ * Copyright (c) 2017, 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/shared/gcLogPrecious.hpp"
+#include "gc/shared/gc_globals.hpp"
+#include "gc/z/zAddress.hpp"
+#include "gc/z/zBarrierSetAssembler.hpp"
+#include "gc/z/zGlobals.hpp"
+#include "runtime/globals.hpp"
+#include "runtime/os.hpp"
+#include "utilities/globalDefinitions.hpp"
+#include "utilities/powerOfTwo.hpp"
+
+#ifdef LINUX
+#include <sys/mman.h>
+#endif // LINUX
+
+// Default value if probing is not implemented for a certain platform: 128TB
+static const size_t DEFAULT_MAX_ADDRESS_BIT = 47;
+// Minimum value returned, if probing fails: 64GB
+static const size_t MINIMUM_MAX_ADDRESS_BIT = 36;
+
+static size_t probe_valid_max_address_bit() {
+#ifdef LINUX
+  size_t max_address_bit = 0;
+  const size_t page_size = os::vm_page_size();
+  for (size_t i = DEFAULT_MAX_ADDRESS_BIT; i > MINIMUM_MAX_ADDRESS_BIT; --i) {
+    const uintptr_t base_addr = ((uintptr_t) 1U) << i;
+    if (msync((void*)base_addr, page_size, MS_ASYNC) == 0) {
+      // msync suceeded, the address is valid, and maybe even already mapped.
+      max_address_bit = i;
+      break;
+    }
+    if (errno != ENOMEM) {
+      // Some error occured. This should never happen, but msync
+      // has some undefined behavior, hence ignore this bit.
+#ifdef ASSERT
+      fatal("Received '%s' while probing the address space for the highest valid bit", os::errno_name(errno));
+#else // ASSERT
+      log_warning_p(gc)("Received '%s' while probing the address space for the highest valid bit", os::errno_name(errno));
+#endif // ASSERT
+      continue;
+    }
+    // Since msync failed with ENOMEM, the page might not be mapped.
+    // Try to map it, to see if the address is valid.
+    void* const result_addr = mmap((void*) base_addr, page_size, PROT_NONE, MAP_PRIVATE|MAP_ANONYMOUS|MAP_NORESERVE, -1, 0);
+    if (result_addr != MAP_FAILED) {
+      munmap(result_addr, page_size);
+    }
+    if ((uintptr_t) result_addr == base_addr) {
+      // address is valid
+      max_address_bit = i;
+      break;
+    }
+  }
+  if (max_address_bit == 0) {
+    // probing failed, allocate a very high page and take that bit as the maximum
+    const uintptr_t high_addr = ((uintptr_t) 1U) << DEFAULT_MAX_ADDRESS_BIT;
+    void* const result_addr = mmap((void*) high_addr, page_size, PROT_NONE, MAP_PRIVATE|MAP_ANONYMOUS|MAP_NORESERVE, -1, 0);
+    if (result_addr != MAP_FAILED) {
+      max_address_bit = BitsPerSize_t - count_leading_zeros((size_t) result_addr) - 1;
+      munmap(result_addr, page_size);
+    }
+  }
+  log_info_p(gc, init)("Probing address space for the highest valid bit: " SIZE_FORMAT, max_address_bit);
+  return MAX2(max_address_bit, MINIMUM_MAX_ADDRESS_BIT);
+#else // LINUX
+  return DEFAULT_MAX_ADDRESS_BIT;
+#endif // LINUX
+}
+
+size_t ZPlatformAddressOffsetBits() {
+  const static size_t valid_max_address_offset_bits = probe_valid_max_address_bit() + 1;
+  const size_t max_address_offset_bits = valid_max_address_offset_bits - 3;
+  const size_t min_address_offset_bits = max_address_offset_bits - 2;
+  const size_t address_offset = round_up_power_of_2(MaxHeapSize * ZVirtualToPhysicalRatio);
+  const size_t address_offset_bits = log2i_exact(address_offset);
+  return clamp(address_offset_bits, min_address_offset_bits, max_address_offset_bits);
+}
+
+size_t ZPlatformAddressHeapBaseShift() {
+  return ZPlatformAddressOffsetBits();
+}
+
+void ZGlobalsPointers::pd_set_good_masks() {
+  BarrierSetAssembler::clear_patching_epoch();
+}

src/hotspot/cpu/aarch64/gc/z/zAddress_aarch64.hpp

@@ -0,0 +1,34 @@
+/*
+ * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef CPU_AARCH64_GC_Z_ZADDRESS_AARCH64_HPP
+#define CPU_AARCH64_GC_Z_ZADDRESS_AARCH64_HPP
+
+#include "utilities/globalDefinitions.hpp"
+
+const size_t ZPointerLoadShift = 16;
+
+size_t ZPlatformAddressOffsetBits();
+size_t ZPlatformAddressHeapBaseShift();
+
+#endif // CPU_AARCH64_GC_Z_ZADDRESS_AARCH64_HPP

src/hotspot/cpu/aarch64/gc/z/zAddress_aarch64.inline.hpp

@@ -0,0 +1,37 @@
+/*
+ * Copyright (c) 2019, 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef CPU_AARCH64_GC_Z_ZADDRESS_AARCH64_INLINE_HPP
+#define CPU_AARCH64_GC_Z_ZADDRESS_AARCH64_INLINE_HPP
+
+#include "utilities/globalDefinitions.hpp"
+
+inline uintptr_t ZPointer::remap_bits(uintptr_t colored) {
+  return (colored ^ ZPointerRemappedMask) & ZPointerRemappedMask;
+}
+
+inline constexpr int ZPointer::load_shift_lookup(uintptr_t value) {
+  return ZPointerLoadShift;
+}
+
+#endif // CPU_AARCH64_GC_Z_ZADDRESS_AARCH64_INLINE_HPP

src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019, 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2019, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -27,21 +27,32 @@
 #include "code/vmreg.hpp"
 #include "oops/accessDecorators.hpp"
 #ifdef COMPILER2
+#include "gc/z/c2/zBarrierSetC2.hpp"
 #include "opto/optoreg.hpp"
 #endif // COMPILER2
 
 #ifdef COMPILER1
+class LIR_Address;
 class LIR_Assembler;
 class LIR_Opr;
 class StubAssembler;
 class ZLoadBarrierStubC1;
+class ZStoreBarrierStubC1;
 #endif // COMPILER1
 
 #ifdef COMPILER2
+class MachNode;
 class Node;
-class ZLoadBarrierStubC2;
 #endif // COMPILER2
 
+// ZBarrierRelocationFormatLoadGoodBeforeTbX is used for both tbnz and tbz
+// They are patched in the same way, their immediate value has the same
+// structure
+const int ZBarrierRelocationFormatLoadGoodBeforeTbX  = 0;
+const int ZBarrierRelocationFormatMarkBadBeforeMov   = 1;
+const int ZBarrierRelocationFormatStoreGoodBeforeMov = 2;
+const int ZBarrierRelocationFormatStoreBadBeforeMov  = 3;
+
 class ZBarrierSetAssembler : public ZBarrierSetAssemblerBase {
 public:
   virtual void load_at(MacroAssembler* masm,
@@ -52,7 +63,27 @@ public:
                        Register tmp1,
                        Register tmp2);
 
-#ifdef ASSERT
+  void store_barrier_fast(MacroAssembler* masm,
+                          Address ref_addr,
+                          Register rnew_zaddress,
+                          Register rnew_zpointer,
+                          Register rtmp,
+                          bool in_nmethod,
+                          bool is_atomic,
+                          Label& medium_path,
+                          Label& medium_path_continuation) const;
+
+  void store_barrier_medium(MacroAssembler* masm,
+                            Address ref_addr,
+                            Register rtmp1,
+                            Register rtmp2,
+                            Register rtmp3,
+                            bool is_native,
+                            bool is_atomic,
+                            Label& medium_path_continuation,
+                            Label& slow_path,
+                            Label& slow_path_continuation) const;
+
   virtual void store_at(MacroAssembler* masm,
                         DecoratorSet decorators,
                         BasicType type,
@@ -61,7 +92,6 @@ public:
                         Register tmp1,
                         Register tmp2,
                         Register tmp3);
-#endif // ASSERT
 
   virtual void arraycopy_prologue(MacroAssembler* masm,
                                   DecoratorSet decorators,
@@ -71,23 +101,89 @@ public:
                                   Register count,
                                   RegSet saved_regs);
 
+  virtual void copy_load_at(MacroAssembler* masm,
+                            DecoratorSet decorators,
+                            BasicType type,
+                            size_t bytes,
+                            Register dst1,
+                            Register dst2,
+                            Address src,
+                            Register tmp);
+
+  virtual void copy_store_at(MacroAssembler* masm,
+                             DecoratorSet decorators,
+                             BasicType type,
+                             size_t bytes,
+                             Address dst,
+                             Register src1,
+                             Register src2,
+                             Register tmp1,
+                             Register tmp2,
+                             Register tmp3);
+
+  virtual void copy_load_at(MacroAssembler* masm,
+                            DecoratorSet decorators,
+                            BasicType type,
+                            size_t bytes,
+                            FloatRegister dst1,
+                            FloatRegister dst2,
+                            Address src,
+                            Register tmp1,
+                            Register tmp2,
+                            FloatRegister vec_tmp);
+
+  virtual void copy_store_at(MacroAssembler* masm,
+                             DecoratorSet decorators,
+                             BasicType type,
+                             size_t bytes,
+                             Address dst,
+                             FloatRegister src1,
+                             FloatRegister src2,
+                             Register tmp1,
+                             Register tmp2,
+                             Register tmp3,
+                             FloatRegister vec_tmp1,
+                             FloatRegister vec_tmp2,
+                             FloatRegister vec_tmp3);
+
   virtual void try_resolve_jobject_in_native(MacroAssembler* masm,
                                              Register jni_env,
                                              Register robj,
                                              Register tmp,
                                              Label& slowpath);
 
-  virtual NMethodPatchingType nmethod_patching_type() { return NMethodPatchingType::conc_data_patch; }
+  virtual NMethodPatchingType nmethod_patching_type() { return NMethodPatchingType::conc_instruction_and_data_patch; }
+
+  void patch_barrier_relocation(address addr, int format);
+
+  void patch_barriers() {}
 
 #ifdef COMPILER1
-  void generate_c1_load_barrier_test(LIR_Assembler* ce,
-                                     LIR_Opr ref) const;
+  void generate_c1_color(LIR_Assembler* ce, LIR_Opr ref) const;
+  void generate_c1_uncolor(LIR_Assembler* ce, LIR_Opr ref) const;
+
+  void generate_c1_load_barrier(LIR_Assembler* ce,
+                                LIR_Opr ref,
+                                ZLoadBarrierStubC1* stub,
+                                bool on_non_strong) const;
 
   void generate_c1_load_barrier_stub(LIR_Assembler* ce,
                                      ZLoadBarrierStubC1* stub) const;
 
   void generate_c1_load_barrier_runtime_stub(StubAssembler* sasm,
                                              DecoratorSet decorators) const;
+
+  void generate_c1_store_barrier(LIR_Assembler* ce,
+                                 LIR_Address* addr,
+                                 LIR_Opr new_zaddress,
+                                 LIR_Opr new_zpointer,
+                                 ZStoreBarrierStubC1* stub) const;
+
+  void generate_c1_store_barrier_stub(LIR_Assembler* ce,
+                                      ZStoreBarrierStubC1* stub) const;
+
+  void generate_c1_store_barrier_runtime_stub(StubAssembler* sasm,
+                                              bool self_healing) const;
 #endif // COMPILER1
 
 #ifdef COMPILER2
@@ -96,9 +192,103 @@ public:
 
   void generate_c2_load_barrier_stub(MacroAssembler* masm,
                                      ZLoadBarrierStubC2* stub) const;
+  void generate_c2_store_barrier_stub(MacroAssembler* masm,
+                                      ZStoreBarrierStubC2* stub) const;
 #endif // COMPILER2
 
   void check_oop(MacroAssembler* masm, Register obj, Register tmp1, Register tmp2, Label& error);
 };
 
+#ifdef COMPILER2
+
+// Load barriers on aarch64 are implemented with a test-and-branch immediate instruction.
+// This immediate has a max delta of 32K. Because of this the branch is implemented with
+// a small jump, as follows:
+//      __ tbz(ref, barrier_Relocation::unpatched, good);
+//      __ b(*stub->entry());
+//      __ bind(good);
+//
+// If we can guarantee that the *stub->entry() label is within 32K we can replace the above
+// code with:
+//      __ tbnz(ref, barrier_Relocation::unpatched, *stub->entry());
+//
+// From the branch shortening part of PhaseOutput we get a pessimistic code size that the code
+// will not grow beyond.
+//
+// The stubs objects are created and registered when the load barriers are emitted. The decision
+// between emitting the long branch or the test and branch is done at this point and uses the
+// pessimistic code size from branch shortening.
+//
+// After the code has been emitted the barrier set will emit all the stubs. When the stubs are
+// emitted we know the real code size. Because of this the trampoline jump can be skipped in
+// favour of emitting the stub directly if it does not interfere with the next trampoline stub.
+// (With respect to test and branch distance)
+//
+// The algorithm for emitting the load barrier branches and stubs now have three versions
+// depending on the distance between the barrier and the stub.
+// Version 1: Not Reachable with a test-and-branch immediate
+// Version 2: Reachable with a test-and-branch immediate via trampoline
+// Version 3: Reachable with a test-and-branch immediate without trampoline
+//
+//     +--------------------- Code ----------------------+
+//     |                      ***                        |
+//     | b(stub1)                                        | (Version 1)
+//     |                      ***                        |
+//     | tbnz(ref, barrier_Relocation::unpatched, tramp) | (Version 2)
+//     |                      ***                        |
+//     | tbnz(ref, barrier_Relocation::unpatched, stub3) | (Version 3)
+//     |                      ***                        |
+//     +--------------------- Stub ----------------------+
+//     | tramp: b(stub2)                                 | (Trampoline slot)
+//     | stub3:                                          |
+//     |                  * Stub Code*                   |
+//     | stub1:                                          |
+//     |                  * Stub Code*                   |
+//     | stub2:                                          |
+//     |                  * Stub Code*                   |
+//     +-------------------------------------------------+
+//
+//  Version 1: Is emitted if the pessimistic distance between the branch instruction and the current
+//             trampoline slot cannot fit in a test and branch immediate.
+//
+//  Version 2: Is emitted if the distance between the branch instruction and the current trampoline
+//             slot can fit in a test and branch immediate. But emitting the stub directly would
+//             interfere with the next trampoline.
+//
+//  Version 3: Same as version two but emitting the stub directly (skipping the trampoline) does not
+//             interfere with the next trampoline.
+//
+class ZLoadBarrierStubC2Aarch64 : public ZLoadBarrierStubC2 {
+private:
+  Label _test_and_branch_reachable_entry;
+  const int _offset;
+  bool _deferred_emit;
+  bool _test_and_branch_reachable;
+
+  ZLoadBarrierStubC2Aarch64(const MachNode* node, Address ref_addr, Register ref, int offset);
+
+  int get_stub_size();
+public:
+  static ZLoadBarrierStubC2Aarch64* create(const MachNode* node, Address ref_addr, Register ref, int offset);
+
+  virtual void emit_code(MacroAssembler& masm);
+  bool is_test_and_branch_reachable();
+  Label* entry();
+};
+
+
+class ZStoreBarrierStubC2Aarch64 : public ZStoreBarrierStubC2 {
+private:
+  bool _deferred_emit;
+
+  ZStoreBarrierStubC2Aarch64(const MachNode* node, Address ref_addr, Register new_zaddress, Register new_zpointer, bool is_native, bool is_atomic);
+
+public:
+  static ZStoreBarrierStubC2Aarch64* create(const MachNode* node, Address ref_addr, Register new_zaddress, Register new_zpointer, bool is_native, bool is_atomic);
+
+  virtual void emit_code(MacroAssembler& masm);
+};
+
+#endif // COMPILER2
+
 #endif // CPU_AARCH64_GC_Z_ZBARRIERSETASSEMBLER_AARCH64_HPP

src/hotspot/cpu/aarch64/gc/z/zGlobals_aarch64.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -24,10 +24,8 @@
 #ifndef CPU_AARCH64_GC_Z_ZGLOBALS_AARCH64_HPP
 #define CPU_AARCH64_GC_Z_ZGLOBALS_AARCH64_HPP
 
-const size_t ZPlatformHeapViews        = 3;
+#include "utilities/globalDefinitions.hpp"
+
 const size_t ZPlatformCacheLineSize    = 64;
 
-size_t ZPlatformAddressOffsetBits();
-size_t ZPlatformAddressMetadataShift();
-
 #endif // CPU_AARCH64_GC_Z_ZGLOBALS_AARCH64_HPP

src/hotspot/cpu/aarch64/gc/z/z_aarch64.ad

@@ -1,5 +1,5 @@
 //
-// Copyright (c) 2019, 2021, Oracle and/or its affiliates. All rights reserved.
+// Copyright (c) 2019, 2023, Oracle and/or its affiliates. All rights reserved.
 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 //
 // This code is free software; you can redistribute it and/or modify it
@@ -31,30 +31,79 @@ source_hpp %{
 
 source %{
 
-static void z_load_barrier(MacroAssembler& _masm, const MachNode* node, Address ref_addr, Register ref, Register tmp, uint8_t barrier_data) {
-  if (barrier_data == ZLoadBarrierElided) {
-    return;
-  }
-  ZLoadBarrierStubC2* const stub = ZLoadBarrierStubC2::create(node, ref_addr, ref, tmp, barrier_data);
-  __ ldr(tmp, Address(rthread, ZThreadLocalData::address_bad_mask_offset()));
-  __ andr(tmp, tmp, ref);
-  __ cbnz(tmp, *stub->entry());
+#include "gc/z/zBarrierSetAssembler.hpp"
+
+static void z_color(MacroAssembler& _masm, const MachNode* node, Register dst, Register src) {
+  assert_different_registers(src, dst);
+  __ relocate(barrier_Relocation::spec(), ZBarrierRelocationFormatStoreGoodBeforeMov);
+  __ movzw(dst, barrier_Relocation::unpatched);
+  __ orr(dst, dst, src, Assembler::LSL, ZPointerLoadShift);
+}
+
+static void z_uncolor(MacroAssembler& _masm, const MachNode* node, Register ref) {
+  __ lsr(ref, ref, ZPointerLoadShift);
+}
+
+static void z_keep_alive_load_barrier(MacroAssembler& _masm, const MachNode* node, Address ref_addr, Register ref, Register tmp) {
+  __ relocate(barrier_Relocation::spec(), ZBarrierRelocationFormatMarkBadBeforeMov);
+  __ movzw(tmp, barrier_Relocation::unpatched);
+  __ tst(ref, tmp);
+  ZLoadBarrierStubC2* const stub = ZLoadBarrierStubC2::create(node, ref_addr, ref);
+  __ br(Assembler::NE, *stub->entry());
+  z_uncolor(_masm, node, ref);
   __ bind(*stub->continuation());
 }
 
-static void z_load_barrier_slow_path(MacroAssembler& _masm, const MachNode* node, Address ref_addr, Register ref, Register tmp) {
-  ZLoadBarrierStubC2* const stub = ZLoadBarrierStubC2::create(node, ref_addr, ref, tmp, ZLoadBarrierStrong);
-  __ b(*stub->entry());
+static void z_load_barrier(MacroAssembler& _masm, const MachNode* node, Address ref_addr, Register ref, Register tmp) {
+  Assembler::InlineSkippedInstructionsCounter skipped_counter(&_masm);
+  const bool on_non_strong =
+      ((node->barrier_data() & ZBarrierWeak) != 0) ||
+      ((node->barrier_data() & ZBarrierPhantom) != 0);
+
+  if (on_non_strong) {
+    z_keep_alive_load_barrier(_masm, node, ref_addr, ref, tmp);
+    return;
+  }
+
+  if (node->barrier_data() == ZBarrierElided) {
+    z_uncolor(_masm, node, ref);
+    return;
+  }
+
+  ZLoadBarrierStubC2Aarch64* const stub = ZLoadBarrierStubC2Aarch64::create(node, ref_addr, ref, __ offset());
+  if (stub->is_test_and_branch_reachable()) {
+    __ relocate(barrier_Relocation::spec(), ZBarrierRelocationFormatLoadGoodBeforeTbX);
+    __ tbnz(ref, barrier_Relocation::unpatched, *stub->entry());
+  } else {
+    Label good;
+    __ relocate(barrier_Relocation::spec(), ZBarrierRelocationFormatLoadGoodBeforeTbX);
+    __ tbz(ref, barrier_Relocation::unpatched, good);
+    __ b(*stub->entry());
+    __ bind(good);
+  }
+  z_uncolor(_masm, node, ref);
   __ bind(*stub->continuation());
 }
 
+static void z_store_barrier(MacroAssembler& _masm, const MachNode* node, Address ref_addr, Register rnew_zaddress, Register rnew_zpointer, Register tmp, bool is_atomic) {
+  Assembler::InlineSkippedInstructionsCounter skipped_counter(&_masm);
+  if (node->barrier_data() == ZBarrierElided) {
+    z_color(_masm, node, rnew_zpointer, rnew_zaddress);
+  } else {
+    bool is_native = (node->barrier_data() & ZBarrierNative) != 0;
+    ZStoreBarrierStubC2Aarch64* const stub = ZStoreBarrierStubC2Aarch64::create(node, ref_addr, rnew_zaddress, rnew_zpointer, is_native, is_atomic);
+    ZBarrierSetAssembler* bs_asm = ZBarrierSet::assembler();
+    bs_asm->store_barrier_fast(&_masm, ref_addr, rnew_zaddress, rnew_zpointer, tmp, true /* in_nmethod */, is_atomic, *stub->entry(), *stub->continuation());
+  }
+}
+
 %}
 
 // Load Pointer
 instruct zLoadP(iRegPNoSp dst, memory8 mem, rFlagsReg cr)
 %{
   match(Set dst (LoadP mem));
-  predicate(UseZGC && !needs_acquiring_load(n) && (n->as_Load()->barrier_data() != 0));
+  predicate(UseZGC && ZGenerational && !needs_acquiring_load(n) && n->as_Load()->barrier_data() != 0);
   effect(TEMP dst, KILL cr);
 
   ins_cost(4 * INSN_COST);
@@ -64,7 +113,7 @@ instruct zLoadP(iRegPNoSp dst, memory8 mem, rFlagsReg cr)
   ins_encode %{
     const Address ref_addr = mem2address($mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
     __ ldr($dst$$Register, ref_addr);
-    z_load_barrier(_masm, this, ref_addr, $dst$$Register, rscratch2 /* tmp */, barrier_data());
+    z_load_barrier(_masm, this, ref_addr, $dst$$Register, rscratch1);
   %}
 
   ins_pipe(iload_reg_mem);
@@ -74,7 +123,7 @@ instruct zLoadP(iRegPNoSp dst, memory8 mem, rFlagsReg cr)
 instruct zLoadPVolatile(iRegPNoSp dst, indirect mem /* sync_memory */, rFlagsReg cr)
 %{
   match(Set dst (LoadP mem));
-  predicate(UseZGC && needs_acquiring_load(n) && n->as_Load()->barrier_data() != 0);
+  predicate(UseZGC && ZGenerational && needs_acquiring_load(n) && n->as_Load()->barrier_data() != 0);
   effect(TEMP dst, KILL cr);
 
   ins_cost(VOLATILE_REF_COST);
@@ -82,18 +131,53 @@ instruct zLoadPVolatile(iRegPNoSp dst, indirect mem /* sync_memory */, rFlagsReg
   format %{ "ldar  $dst, $mem\t" %}
 
   ins_encode %{
+    const Address ref_addr = Address($mem$$Register);
     __ ldar($dst$$Register, $mem$$Register);
-    z_load_barrier(_masm, this, Address($mem$$Register), $dst$$Register, rscratch2 /* tmp */, barrier_data());
+    z_load_barrier(_masm, this, ref_addr, $dst$$Register, rscratch1);
   %}
 
   ins_pipe(pipe_serial);
 %}
 
-instruct zCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
+// Store Pointer
+instruct zStoreP(memory mem, iRegP src, iRegPNoSp tmp, rFlagsReg cr)
+%{
+  predicate(UseZGC && ZGenerational && !needs_releasing_store(n) && n->as_Store()->barrier_data() != 0);
+  match(Set mem (StoreP mem src));
+  effect(TEMP tmp, KILL cr);
+
+  ins_cost(125); // XXX
+  format %{ "movq    $mem, $src\t# ptr" %}
+  ins_encode %{
+    const Address ref_addr = mem2address($mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
+    z_store_barrier(_masm, this, ref_addr, $src$$Register, $tmp$$Register, rscratch2, false /* is_atomic */);
+    __ str($tmp$$Register, ref_addr);
+  %}
+  ins_pipe(pipe_serial);
+%}
+
+// Store Pointer Volatile
+instruct zStorePVolatile(indirect mem, iRegP src, iRegPNoSp tmp, rFlagsReg cr)
+%{
+  predicate(UseZGC && ZGenerational && needs_releasing_store(n) && n->as_Store()->barrier_data() != 0);
+  match(Set mem (StoreP mem src));
+  effect(TEMP tmp, KILL cr);
+
+  ins_cost(125); // XXX
+  format %{ "movq    $mem, $src\t# ptr" %}
+  ins_encode %{
+    const Address ref_addr = Address($mem$$Register);
+    z_store_barrier(_masm, this, ref_addr, $src$$Register, $tmp$$Register, rscratch2, false /* is_atomic */);
+    __ stlr($tmp$$Register, $mem$$Register);
+  %}
+  ins_pipe(pipe_serial);
+%}
+
+instruct zCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp oldval_tmp, iRegPNoSp newval_tmp, rFlagsReg cr) %{
   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
   match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
-  predicate(UseZGC && !needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong);
-  effect(KILL cr, TEMP_DEF res);
+  predicate(UseZGC && ZGenerational && !needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
+  effect(TEMP oldval_tmp, TEMP newval_tmp, TEMP res, KILL cr);
 
   ins_cost(2 * VOLATILE_REF_COST);
 
@@ -102,108 +186,83 @@ instruct zCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newva
 
   ins_encode %{
     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
-    __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword,
-               false /* acquire */, true /* release */, false /* weak */, rscratch2);
+    Address ref_addr($mem$$Register);
+    z_store_barrier(_masm, this, ref_addr, $newval$$Register, $newval_tmp$$Register, rscratch2, true /* is_atomic */);
+    z_color(_masm, this, $oldval_tmp$$Register, $oldval$$Register);
+    __ cmpxchg($mem$$Register, $oldval_tmp$$Register, $newval_tmp$$Register, Assembler::xword,
+               false /* acquire */, true /* release */, false /* weak */, noreg);
     __ cset($res$$Register, Assembler::EQ);
-    if (barrier_data() != ZLoadBarrierElided) {
-      Label good;
-      __ ldr(rscratch1, Address(rthread, ZThreadLocalData::address_bad_mask_offset()));
-      __ andr(rscratch1, rscratch1, rscratch2);
-      __ cbz(rscratch1, good);
-      z_load_barrier_slow_path(_masm, this, Address($mem$$Register), rscratch2 /* ref */, rscratch1 /* tmp */);
-      __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword,
-                 false /* acquire */, true /* release */, false /* weak */, rscratch2);
-      __ cset($res$$Register, Assembler::EQ);
-      __ bind(good);
-    }
   %}
 
   ins_pipe(pipe_slow);
 %}
 
-instruct zCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
+instruct zCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp oldval_tmp, iRegPNoSp newval_tmp, rFlagsReg cr) %{
   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
   match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
-  predicate(UseZGC && needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong));
-  effect(KILL cr, TEMP_DEF res);
+  predicate(UseZGC && ZGenerational && needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
+  effect(TEMP oldval_tmp, TEMP newval_tmp, TEMP res, KILL cr);
 
   ins_cost(2 * VOLATILE_REF_COST);
 
- format %{ "cmpxchg $mem, $oldval, $newval\n\t"
-           "cset    $res, EQ" %}
+  format %{ "cmpxchg $mem, $oldval, $newval\n\t"
+            "cset    $res, EQ" %}
 
   ins_encode %{
     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
-    __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword,
-               true /* acquire */, true /* release */, false /* weak */, rscratch2);
+    Address ref_addr($mem$$Register);
+    z_store_barrier(_masm, this, ref_addr, $newval$$Register, $newval_tmp$$Register, rscratch2, true /* is_atomic */);
+    z_color(_masm, this, $oldval_tmp$$Register, $oldval$$Register);
+    __ cmpxchg($mem$$Register, $oldval_tmp$$Register, $newval_tmp$$Register, Assembler::xword,
+               true /* acquire */, true /* release */, false /* weak */, noreg);
     __ cset($res$$Register, Assembler::EQ);
-    if (barrier_data() != ZLoadBarrierElided) {
-      Label good;
-      __ ldr(rscratch1, Address(rthread, ZThreadLocalData::address_bad_mask_offset()));
-      __ andr(rscratch1, rscratch1, rscratch2);
-      __ cbz(rscratch1, good);
-      z_load_barrier_slow_path(_masm, this, Address($mem$$Register), rscratch2 /* ref */, rscratch1 /* tmp */ );
-      __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword,
-                 true /* acquire */, true /* release */, false /* weak */, rscratch2);
-      __ cset($res$$Register, Assembler::EQ);
-      __ bind(good);
-    }
   %}
 
   ins_pipe(pipe_slow);
 %}
 
-instruct zCompareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
+
+instruct zCompareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp oldval_tmp, iRegPNoSp newval_tmp, rFlagsReg cr) %{
   match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
-  predicate(UseZGC && !needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong);
-  effect(TEMP_DEF res, KILL cr);
+  predicate(UseZGC && ZGenerational && !needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
+  effect(TEMP oldval_tmp, TEMP newval_tmp, TEMP res, KILL cr);
 
   ins_cost(2 * VOLATILE_REF_COST);
 
-  format %{ "cmpxchg $res = $mem, $oldval, $newval" %}
+  format %{ "cmpxchg $mem, $oldval, $newval\n\t"
+            "cset    $res, EQ" %}
 
   ins_encode %{
     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
-    __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword,
+    Address ref_addr($mem$$Register);
+    z_store_barrier(_masm, this, ref_addr, $newval$$Register, $newval_tmp$$Register, rscratch2, true /* is_atomic */);
+    z_color(_masm, this, $oldval_tmp$$Register, $oldval$$Register);
+    __ cmpxchg($mem$$Register, $oldval_tmp$$Register, $newval_tmp$$Register, Assembler::xword,
                false /* acquire */, true /* release */, false /* weak */, $res$$Register);
-    if (barrier_data() != ZLoadBarrierElided) {
-      Label good;
-      __ ldr(rscratch1, Address(rthread, ZThreadLocalData::address_bad_mask_offset()));
-      __ andr(rscratch1, rscratch1, $res$$Register);
-      __ cbz(rscratch1, good);
-      z_load_barrier_slow_path(_masm, this, Address($mem$$Register), $res$$Register /* ref */, rscratch1 /* tmp */);
-      __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword,
-                 false /* acquire */, true /* release */, false /* weak */, $res$$Register);
-      __ bind(good);
-    }
+    z_uncolor(_masm, this, $res$$Register);
   %}
 
   ins_pipe(pipe_slow);
 %}
 
-instruct zCompareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
+instruct zCompareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp oldval_tmp, iRegPNoSp newval_tmp, rFlagsReg cr) %{
   match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
-  predicate(UseZGC && needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong);
-  effect(TEMP_DEF res, KILL cr);
+  predicate(UseZGC && ZGenerational && needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
+  effect(TEMP oldval_tmp, TEMP newval_tmp, TEMP res, KILL cr);
 
   ins_cost(2 * VOLATILE_REF_COST);
 
-  format %{ "cmpxchg $res = $mem, $oldval, $newval" %}
+  format %{ "cmpxchg $mem, $oldval, $newval\n\t"
+            "cset    $res, EQ" %}
 
   ins_encode %{
     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
-    __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword,
+    Address ref_addr($mem$$Register);
+    z_store_barrier(_masm, this, ref_addr, $newval$$Register, $newval_tmp$$Register, rscratch2, true /* is_atomic */);
+    z_color(_masm, this, $oldval_tmp$$Register, $oldval$$Register);
+    __ cmpxchg($mem$$Register, $oldval_tmp$$Register, $newval_tmp$$Register, Assembler::xword,
                true /* acquire */, true /* release */, false /* weak */, $res$$Register);
-    if (barrier_data() != ZLoadBarrierElided) {
-      Label good;
-      __ ldr(rscratch1, Address(rthread, ZThreadLocalData::address_bad_mask_offset()));
-      __ andr(rscratch1, rscratch1, $res$$Register);
-      __ cbz(rscratch1, good);
-      z_load_barrier_slow_path(_masm, this, Address($mem$$Register), $res$$Register /* ref */, rscratch1 /* tmp */);
-      __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword,
-                 true /* acquire */, true /* release */, false /* weak */, $res$$Register);
-      __ bind(good);
-    }
+    z_uncolor(_masm, this, $res$$Register);
   %}
 
   ins_pipe(pipe_slow);
@@ -211,16 +270,17 @@ instruct zCompareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iReg
 
 instruct zGetAndSetP(indirect mem, iRegP newv, iRegPNoSp prev, rFlagsReg cr) %{
   match(Set prev (GetAndSetP mem newv));
-  predicate(UseZGC && !needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
-  effect(TEMP_DEF prev, KILL cr);
+  predicate(UseZGC && ZGenerational && !needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
+  effect(TEMP prev, KILL cr);
 
   ins_cost(2 * VOLATILE_REF_COST);
 
   format %{ "atomic_xchg  $prev, $newv, [$mem]" %}
 
   ins_encode %{
-    __ atomic_xchg($prev$$Register, $newv$$Register, $mem$$Register);
-    z_load_barrier(_masm, this, Address(noreg, 0), $prev$$Register, rscratch2 /* tmp */, barrier_data());
+    z_store_barrier(_masm, this, Address($mem$$Register), $newv$$Register, $prev$$Register, rscratch2, true /* is_atomic */);
+    __ atomic_xchg($prev$$Register, $prev$$Register, $mem$$Register);
+    z_uncolor(_masm, this, $prev$$Register);
   %}
 
   ins_pipe(pipe_serial);
@@ -228,16 +288,18 @@ instruct zGetAndSetP(indirect mem, iRegP newv, iRegPNoSp prev, rFlagsReg cr) %{
 
 instruct zGetAndSetPAcq(indirect mem, iRegP newv, iRegPNoSp prev, rFlagsReg cr) %{
   match(Set prev (GetAndSetP mem newv));
-  predicate(UseZGC && needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() != 0));
-  effect(TEMP_DEF prev, KILL cr);
+  predicate(UseZGC && ZGenerational && needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
+  effect(TEMP prev, KILL cr);
 
-  ins_cost(VOLATILE_REF_COST);
+  ins_cost(2 * VOLATILE_REF_COST);
 
-  format %{ "atomic_xchg_acq  $prev, $newv, [$mem]" %}
+  format %{ "atomic_xchg  $prev, $newv, [$mem]" %}
 
   ins_encode %{
-    __ atomic_xchgal($prev$$Register, $newv$$Register, $mem$$Register);
-    z_load_barrier(_masm, this, Address(noreg, 0), $prev$$Register, rscratch2 /* tmp */, barrier_data());
+    z_store_barrier(_masm, this, Address($mem$$Register), $newv$$Register, $prev$$Register, rscratch2, true /* is_atomic */);
+    __ atomic_xchgal($prev$$Register, $prev$$Register, $mem$$Register);
+    z_uncolor(_masm, this, $prev$$Register);
   %}
+
   ins_pipe(pipe_serial);
 %}