diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 00f64d2aedf..aed03f55536 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -284,7 +284,7 @@ jobs: with: platform: macos-aarch64 runs-on: 'macos-14' - xcode-toolset-version: '14.3.1' + xcode-toolset-version: '15.4' configure-arguments: ${{ github.event.inputs.configure-arguments }} make-arguments: ${{ github.event.inputs.make-arguments }} if: needs.prepare.outputs.macos-aarch64 == 'true' @@ -354,6 +354,7 @@ jobs: platform: macos-x64 bootjdk-platform: macos-x64 runs-on: macos-13 + xcode-toolset-version: '14.3.1' test-macos-aarch64: name: macos-aarch64 @@ -364,6 +365,7 @@ jobs: platform: macos-aarch64 bootjdk-platform: macos-aarch64 runs-on: macos-14 + xcode-toolset-version: '15.4' test-windows-x64: name: windows-x64 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a8885866c12..3517fa53941 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -37,6 +37,9 @@ on: runs-on: required: true type: string + xcode-toolset-version: + required: false + type: string env: # These are needed to make the MSYS2 bash work properly @@ -147,7 +150,7 @@ jobs: run: | # On macOS we need to install some dependencies for testing brew install make - sudo xcode-select --switch /Applications/Xcode_14.3.1.app/Contents/Developer + sudo xcode-select --switch /Applications/Xcode_${{ inputs.xcode-toolset-version }}.app/Contents/Developer # This will make GNU make available as 'make' and not only as 'gmake' echo '/usr/local/opt/make/libexec/gnubin' >> $GITHUB_PATH if: runner.os == 'macOS' diff --git a/make/Images.gmk b/make/Images.gmk index 5703a74afa5..acdc594b009 100644 --- a/make/Images.gmk +++ b/make/Images.gmk @@ -96,6 +96,10 @@ JLINK_DISABLE_WARNINGS := | ( $(GREP) -v -e "WARNING: Using incubator module" || JDK_IMAGE_SUPPORT_DIR := $(SUPPORT_OUTPUTDIR)/images/jdk JRE_IMAGE_SUPPORT_DIR := $(SUPPORT_OUTPUTDIR)/images/jre +ifeq ($(JLINK_PRODUCE_LINKABLE_RUNTIME), true) + JLINK_JDK_EXTRA_OPTS += --generate-linkable-runtime +endif + $(eval $(call SetupExecute, jlink_jdk, \ WARN := Creating jdk image, \ DEPS := $(JDK_JMODS) $(BASE_RELEASE_FILE) \ @@ -132,10 +136,16 @@ CDS_DUMP_FLAGS = -Xmx128M -Xms128M # Helper function for creating the CDS archives for the JDK and JRE # # Param1 - VM variant (e.g., server, client, zero, ...) -# Param2 - _nocoops, or empty +# Param2 - _nocoops, _coh, _nocoops_coh, or empty define CreateCDSArchive - $1_$2_DUMP_EXTRA_ARG := $(if $(filter _nocoops, $2), -XX:-UseCompressedOops, ) - $1_$2_DUMP_TYPE := $(if $(filter _nocoops, $2), -NOCOOPS, ) + $1_$2_COOPS_OPTION := $(if $(findstring _nocoops, $2),-XX:-UseCompressedOops) + # enable and also explicitly disable coh as needed. + ifeq ($(call isTargetCpuBits, 64), true) + $1_$2_COH_OPTION := -XX:+UnlockExperimentalVMOptions \ + $(if $(findstring _coh, $2),-XX:+UseCompactObjectHeaders,-XX:-UseCompactObjectHeaders) + endif + $1_$2_DUMP_EXTRA_ARG := $$($1_$2_COOPS_OPTION) $$($1_$2_COH_OPTION) + $1_$2_DUMP_TYPE := $(if $(findstring _nocoops, $2),-NOCOOPS,)$(if $(findstring _coh, $2),-COH,) # Only G1 supports dumping the shared heap, so explicitly use G1 if the JVM supports it. $1_$2_CDS_DUMP_FLAGS := $(CDS_DUMP_FLAGS) $(if $(filter g1gc, $(JVM_FEATURES_$1)), -XX:+UseG1GC) @@ -190,6 +200,14 @@ ifeq ($(BUILD_CDS_ARCHIVE), true) $(foreach v, $(JVM_VARIANTS), \ $(eval $(call CreateCDSArchive,$v,_nocoops)) \ ) + ifeq ($(BUILD_CDS_ARCHIVE_COH), true) + $(foreach v, $(JVM_VARIANTS), \ + $(eval $(call CreateCDSArchive,$v,_coh)) \ + ) + $(foreach v, $(JVM_VARIANTS), \ + $(eval $(call CreateCDSArchive,$v,_nocoops_coh)) \ + ) + endif endif endif diff --git a/make/autoconf/basic.m4 b/make/autoconf/basic.m4 index 585575240ea..f574174a12e 100644 --- a/make/autoconf/basic.m4 +++ b/make/autoconf/basic.m4 @@ -76,6 +76,9 @@ AC_DEFUN_ONCE([BASIC_SETUP_PATHS], fi if test "x$OPENJDK_TARGET_OS" = "xwindows"; then + if test "x$OPENJDK_TARGET_CPU_BITS" = "x32"; then + AC_MSG_ERROR([32-bit Windows builds are not supported]) + fi BASIC_SETUP_PATHS_WINDOWS fi diff --git a/make/autoconf/configure.ac b/make/autoconf/configure.ac index 21bf61a3fa1..66809127a75 100644 --- a/make/autoconf/configure.ac +++ b/make/autoconf/configure.ac @@ -261,6 +261,7 @@ JDKOPT_ENABLE_DISABLE_GENERATE_CLASSLIST JDKOPT_EXCLUDE_TRANSLATIONS JDKOPT_ENABLE_DISABLE_MANPAGES JDKOPT_ENABLE_DISABLE_CDS_ARCHIVE +JDKOPT_ENABLE_DISABLE_CDS_ARCHIVE_COH JDKOPT_ENABLE_DISABLE_COMPATIBLE_CDS_ALIGNMENT JDKOPT_SETUP_MACOSX_SIGNING diff --git a/make/autoconf/flags-ldflags.m4 b/make/autoconf/flags-ldflags.m4 index bdbc898f987..851b3b7a0ef 100644 --- a/make/autoconf/flags-ldflags.m4 +++ b/make/autoconf/flags-ldflags.m4 @@ -74,7 +74,7 @@ AC_DEFUN([FLAGS_SETUP_LDFLAGS_HELPER], # Clang needs the lld linker to work correctly BASIC_LDFLAGS="-fuse-ld=lld -Wl,--exclude-libs,ALL" if test "x$CXX_IS_USER_SUPPLIED" = xfalse && test "x$CC_IS_USER_SUPPLIED" = xfalse; then - UTIL_REQUIRE_PROGS(LLD, lld) + UTIL_REQUIRE_PROGS(LLD, lld, $TOOLCHAIN_PATH:$PATH) fi fi if test "x$OPENJDK_TARGET_OS" = xaix; then diff --git a/make/autoconf/jdk-options.m4 b/make/autoconf/jdk-options.m4 index 11c55c78930..cf8a856de96 100644 --- a/make/autoconf/jdk-options.m4 +++ b/make/autoconf/jdk-options.m4 @@ -586,13 +586,42 @@ AC_DEFUN_ONCE([JDKOPT_SETUP_JMOD_OPTIONS], ################################################################################ # # jlink options. -# We always keep packaged modules in JDK image. # AC_DEFUN_ONCE([JDKOPT_SETUP_JLINK_OPTIONS], [ - UTIL_ARG_ENABLE(NAME: keep-packaged-modules, DEFAULT: true, + + ################################################################################ + # + # Configure option for building a JDK that is suitable for linking from the + # run-time image without JMODs. + # + # Determines whether or not a suitable run-time image is being produced from + # packaged modules. If set to 'true, changes the *default* of packaged + # modules to 'false'. + # + UTIL_ARG_ENABLE(NAME: linkable-runtime, DEFAULT: false, + RESULT: JLINK_PRODUCE_LINKABLE_RUNTIME, + DESC: [enable a JDK build suitable for linking from the run-time image], + CHECKING_MSG: [whether or not a JDK suitable for linking from the run-time image should be produced]) + AC_SUBST(JLINK_PRODUCE_LINKABLE_RUNTIME) + + if test "x$JLINK_PRODUCE_LINKABLE_RUNTIME" = xtrue; then + DEFAULT_PACKAGED_MODULES=false + else + DEFAULT_PACKAGED_MODULES=true + fi + + ################################################################################ + # + # Configure option for packaged modules + # + # We keep packaged modules in the JDK image unless --enable-linkable-runtime is + # requested. + # + UTIL_ARG_ENABLE(NAME: keep-packaged-modules, DEFAULT: $DEFAULT_PACKAGED_MODULES, RESULT: JLINK_KEEP_PACKAGED_MODULES, DESC: [enable keeping of packaged modules in jdk image], + DEFAULT_DESC: [enabled by default unless --enable-linkable-runtime is set], CHECKING_MSG: [if packaged modules are kept]) AC_SUBST(JLINK_KEEP_PACKAGED_MODULES) ]) @@ -638,14 +667,11 @@ AC_DEFUN([JDKOPT_EXCLUDE_TRANSLATIONS], ################################################################################ # -# Optionally disable man pages +# Optionally disable man pages (deprecated) # AC_DEFUN([JDKOPT_ENABLE_DISABLE_MANPAGES], [ - UTIL_ARG_ENABLE(NAME: manpages, DEFAULT: true, RESULT: BUILD_MANPAGES, - DESC: [enable copying of static man pages], - CHECKING_MSG: [if static man pages should be copied]) - AC_SUBST(BUILD_MANPAGES) + UTIL_DEPRECATED_ARG_ENABLE(manpages) ]) ################################################################################ @@ -673,6 +699,37 @@ AC_DEFUN([JDKOPT_ENABLE_DISABLE_CDS_ARCHIVE], AC_SUBST(BUILD_CDS_ARCHIVE) ]) +################################################################################ +# +# Enable or disable the default CDS archive generation for Compact Object Headers +# +AC_DEFUN([JDKOPT_ENABLE_DISABLE_CDS_ARCHIVE_COH], +[ + UTIL_ARG_ENABLE(NAME: cds-archive-coh, DEFAULT: auto, RESULT: BUILD_CDS_ARCHIVE_COH, + DESC: [enable generation of default CDS archives for compact object headers (requires --enable-cds-archive)], + DEFAULT_DESC: [auto], + CHECKING_MSG: [if default CDS archives for compact object headers should be generated], + CHECK_AVAILABLE: [ + AC_MSG_CHECKING([if CDS archive with compact object headers is available]) + if test "x$BUILD_CDS_ARCHIVE" = "xfalse"; then + AC_MSG_RESULT([no (CDS default archive generation is disabled)]) + AVAILABLE=false + elif test "x$OPENJDK_TARGET_CPU" != "xx86_64" && + test "x$OPENJDK_TARGET_CPU" != "xaarch64" && + test "x$OPENJDK_TARGET_CPU" != "xppc64" && + test "x$OPENJDK_TARGET_CPU" != "xppc64le" && + test "x$OPENJDK_TARGET_CPU" != "xriscv64" && + test "x$OPENJDK_TARGET_CPU" != "xs390x"; then + AC_MSG_RESULT([no (compact object headers not supported for this platform)]) + AVAILABLE=false + else + AC_MSG_RESULT([yes]) + AVAILABLE=true + fi + ]) + AC_SUBST(BUILD_CDS_ARCHIVE_COH) +]) + ################################################################################ # # Enable the alternative CDS core region alignment diff --git a/make/autoconf/platform.m4 b/make/autoconf/platform.m4 index 9ca808c4c17..3f977058a51 100644 --- a/make/autoconf/platform.m4 +++ b/make/autoconf/platform.m4 @@ -1,5 +1,5 @@ # -# Copyright (c) 2011, 2023, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2011, 2024, Oracle and/or its affiliates. All rights reserved. # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. # # This code is free software; you can redistribute it and/or modify it @@ -666,14 +666,14 @@ AC_DEFUN([PLATFORM_CHECK_DEPRECATION], [ AC_ARG_ENABLE(deprecated-ports, [AS_HELP_STRING([--enable-deprecated-ports@<:@=yes/no@:>@], [Suppress the error when configuring for a deprecated port @<:@no@:>@])]) - if test "x$OPENJDK_TARGET_OS" = xwindows && test "x$OPENJDK_TARGET_CPU" = xx86; then - if test "x$enable_deprecated_ports" = "xyes"; then - AC_MSG_WARN([The Windows 32-bit x86 port is deprecated and may be removed in a future release.]) - else - AC_MSG_ERROR(m4_normalize([The Windows 32-bit x86 port is deprecated and may be removed in a future release. - Use --enable-deprecated-ports=yes to suppress this error.])) - fi - fi + # if test "x$OPENJDK_TARGET_CPU" = xx86; then + # if test "x$enable_deprecated_ports" = "xyes"; then + # AC_MSG_WARN([The x86 port is deprecated and may be removed in a future release.]) + # else + # AC_MSG_ERROR(m4_normalize([The 32-bit x86 port is deprecated and may be removed in a future release. + # Use --enable-deprecated-ports=yes to suppress this error.])) + # fi + # fi ]) AC_DEFUN_ONCE([PLATFORM_SETUP_OPENJDK_BUILD_OS_VERSION], diff --git a/make/autoconf/spec.gmk.template b/make/autoconf/spec.gmk.template index eb2b1a688e1..bcd54058c28 100644 --- a/make/autoconf/spec.gmk.template +++ b/make/autoconf/spec.gmk.template @@ -367,9 +367,8 @@ ENABLE_GENERATE_CLASSLIST := @ENABLE_GENERATE_CLASSLIST@ EXCLUDE_TRANSLATIONS := @EXCLUDE_TRANSLATIONS@ -BUILD_MANPAGES := @BUILD_MANPAGES@ - BUILD_CDS_ARCHIVE := @BUILD_CDS_ARCHIVE@ +BUILD_CDS_ARCHIVE_COH := @BUILD_CDS_ARCHIVE_COH@ ENABLE_COMPATIBLE_CDS_ALIGNMENT := @ENABLE_COMPATIBLE_CDS_ALIGNMENT@ @@ -707,6 +706,7 @@ NEW_JAVADOC = $(INTERIM_LANGTOOLS_ARGS) $(JAVADOC_MAIN_CLASS) JMOD_COMPRESS := @JMOD_COMPRESS@ JLINK_KEEP_PACKAGED_MODULES := @JLINK_KEEP_PACKAGED_MODULES@ +JLINK_PRODUCE_LINKABLE_RUNTIME := @JLINK_PRODUCE_LINKABLE_RUNTIME@ RCFLAGS := @RCFLAGS@ diff --git a/make/autoconf/toolchain_microsoft.m4 b/make/autoconf/toolchain_microsoft.m4 index 387e3717718..4f970df7b5f 100644 --- a/make/autoconf/toolchain_microsoft.m4 +++ b/make/autoconf/toolchain_microsoft.m4 @@ -1,5 +1,5 @@ # -# Copyright (c) 2011, 2022, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2011, 2024, Oracle and/or its affiliates. All rights reserved. # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. # # This code is free software; you can redistribute it and/or modify it @@ -82,9 +82,7 @@ AC_DEFUN([TOOLCHAIN_CHECK_POSSIBLE_VISUAL_STUDIO_ROOT], fi AC_MSG_NOTICE([Found Visual Studio installation at $VS_BASE using $METHOD]) - if test "x$TARGET_CPU" = xx86; then - VCVARSFILES="vcvars32.bat vcvarsamd64_x86.bat" - elif test "x$TARGET_CPU" = xx86_64; then + if test "x$TARGET_CPU" = xx86_64; then VCVARSFILES="vcvars64.bat vcvarsx86_amd64.bat" elif test "x$TARGET_CPU" = xaarch64; then # for host x86-64, target aarch64 @@ -132,9 +130,7 @@ AC_DEFUN([TOOLCHAIN_CHECK_POSSIBLE_WIN_SDK_ROOT], elif test -f "$WIN_SDK_BASE/bin/setenv.cmd"; then AC_MSG_NOTICE([Found Windows SDK installation at $WIN_SDK_BASE using $METHOD]) VS_ENV_CMD="$WIN_SDK_BASE/bin/setenv.cmd" - if test "x$TARGET_CPU" = xx86; then - VS_ENV_ARGS="/x86" - elif test "x$TARGET_CPU" = xx86_64; then + if test "x$TARGET_CPU" = xx86_64; then VS_ENV_ARGS="/x64" elif test "x$TARGET_CPU" = xaarch64; then VS_ENV_ARGS="/arm64" @@ -438,9 +434,7 @@ AC_DEFUN([TOOLCHAIN_CHECK_POSSIBLE_MSVC_DLL], # Need to check if the found msvcr is correct architecture AC_MSG_CHECKING([found $DLL_NAME architecture]) MSVC_DLL_FILETYPE=`$FILE -b "$POSSIBLE_MSVC_DLL"` - if test "x$OPENJDK_TARGET_CPU" = xx86; then - CORRECT_MSVCR_ARCH=386 - elif test "x$OPENJDK_TARGET_CPU" = xx86_64; then + if test "x$OPENJDK_TARGET_CPU" = xx86_64; then CORRECT_MSVCR_ARCH=x86-64 elif test "x$OPENJDK_TARGET_CPU" = xaarch64; then # The cygwin 'file' command only returns "PE32+ executable (DLL) (console), for MS Windows", @@ -466,9 +460,7 @@ AC_DEFUN([TOOLCHAIN_SETUP_MSVC_DLL], DLL_HELP="$2" MSVC_DLL= - if test "x$OPENJDK_TARGET_CPU" = xx86; then - vs_target_cpu=x86 - elif test "x$OPENJDK_TARGET_CPU" = xx86_64; then + if test "x$OPENJDK_TARGET_CPU" = xx86_64; then vs_target_cpu=x64 elif test "x$OPENJDK_TARGET_CPU" = xaarch64; then vs_target_cpu=arm64 @@ -522,18 +514,8 @@ AC_DEFUN([TOOLCHAIN_SETUP_MSVC_DLL], # Probe: Search wildly in the VCINSTALLDIR. We've probably lost by now. # (This was the original behaviour; kept since it might turn something up) if test "x$VCINSTALLDIR" != x; then - if test "x$OPENJDK_TARGET_CPU" = xx86; then - POSSIBLE_MSVC_DLL=`$FIND "$VCINSTALLDIR" -name $DLL_NAME \ - | $GREP x86 | $GREP -v ia64 | $GREP -v x64 | $GREP -v arm64 | $HEAD --lines 1` - if test "x$POSSIBLE_MSVC_DLL" = x; then - # We're grasping at straws now... - POSSIBLE_MSVC_DLL=`$FIND "$VCINSTALLDIR" -name $DLL_NAME \ - | $HEAD --lines 1` - fi - else - POSSIBLE_MSVC_DLL=`$FIND "$VCINSTALLDIR" -name $DLL_NAME \ - | $GREP x64 | $HEAD --lines 1` - fi + POSSIBLE_MSVC_DLL=`$FIND "$VCINSTALLDIR" -name $DLL_NAME \ + | $GREP x64 | $HEAD --lines 1` TOOLCHAIN_CHECK_POSSIBLE_MSVC_DLL([$DLL_NAME], [$POSSIBLE_MSVC_DLL], [search of VCINSTALLDIR]) diff --git a/make/common/modules/LauncherCommon.gmk b/make/common/modules/LauncherCommon.gmk index 7c2cef58835..aa721a683d7 100644 --- a/make/common/modules/LauncherCommon.gmk +++ b/make/common/modules/LauncherCommon.gmk @@ -228,14 +228,11 @@ ifeq ($(call isTargetOsType, unix)+$(MAKEFILE_PREFIX), true+Launcher) endif else # No markdown man pages present - ifeq ($(BUILD_MANPAGES), true) - # BUILD_MANPAGES is a mis-nomer. It really means "copy the pre-generated man pages". - $(eval $(call SetupCopyFiles, COPY_MAN_PAGES, \ - DEST := $(SUPPORT_OUTPUTDIR)/modules_man/$(MODULE)/man1, \ - FILES := $(MAN_FILES_TROFF), \ - )) + $(eval $(call SetupCopyFiles, COPY_MAN_PAGES, \ + DEST := $(SUPPORT_OUTPUTDIR)/modules_man/$(MODULE)/man1, \ + FILES := $(MAN_FILES_TROFF), \ + )) - TARGETS += $(COPY_MAN_PAGES) - endif + TARGETS += $(COPY_MAN_PAGES) endif endif diff --git a/make/conf/jib-profiles.js b/make/conf/jib-profiles.js index c43c424632e..dc026a800ad 100644 --- a/make/conf/jib-profiles.js +++ b/make/conf/jib-profiles.js @@ -252,7 +252,6 @@ var getJibProfilesCommon = function (input, data) { default_make_targets: ["product-bundles", "test-bundles", "static-libs-bundles"], configure_args: concat( "--with-exclude-translations=es,fr,it,ko,pt_BR,sv,ca,tr,cs,sk,ja_JP_A,ja_JP_HA,ja_JP_HI,ja_JP_I,zh_TW,zh_HK", - "--disable-manpages", "--disable-jvm-feature-shenandoahgc", versionArgs(input, common)) }; diff --git a/make/hotspot/gensrc/GensrcAdlc.gmk b/make/hotspot/gensrc/GensrcAdlc.gmk index ce3f2684026..d43a8b8d3ab 100644 --- a/make/hotspot/gensrc/GensrcAdlc.gmk +++ b/make/hotspot/gensrc/GensrcAdlc.gmk @@ -109,9 +109,7 @@ ifeq ($(call check-jvm-feature, compiler2), true) ADLCFLAGS += -DR18_RESERVED endif else ifeq ($(call isTargetOs, windows), true) - ifeq ($(call isTargetCpuBits, 64), true) - ADLCFLAGS += -D_WIN64=1 - endif + ADLCFLAGS += -D_WIN64=1 ifeq ($(HOTSPOT_TARGET_CPU_ARCH), aarch64) ADLCFLAGS += -DR18_RESERVED endif diff --git a/make/modules/java.base/Copy.gmk b/make/modules/java.base/Copy.gmk index fea544a47a1..7e3a0e9c3ab 100644 --- a/make/modules/java.base/Copy.gmk +++ b/make/modules/java.base/Copy.gmk @@ -123,46 +123,6 @@ TARGETS += $(JVMCFG) ################################################################################ -POLICY_SRC := $(TOPDIR)/src/java.base/share/conf/security/java.policy -POLICY_DST := $(CONF_DST_DIR)/security/java.policy - -POLICY_SRC_LIST := $(POLICY_SRC) - -$(POLICY_DST): $(POLICY_SRC_LIST) - $(call MakeTargetDir) - $(RM) $@ $@.tmp - $(foreach f, $(POLICY_SRC_LIST), $(CAT) $(f) >> $@.tmp;) - $(MV) $@.tmp $@ - -TARGETS += $(POLICY_DST) - -################################################################################ - -DEF_POLICY_SRC := $(TOPDIR)/src/java.base/share/lib/security/default.policy -DEF_POLICY_DST := $(LIB_DST_DIR)/security/default.policy - -DEF_POLICY_SRC_LIST := $(DEF_POLICY_SRC) -DEF_POLICY_SRC_LIST += $(CUSTOM_POLICY_SRC_LIST) - -ifeq ($(call isTargetOs, windows), true) - DEF_POLICY_SRC_LIST += $(TOPDIR)/src/java.base/$(OPENJDK_TARGET_OS)/lib/security/default.policy -endif - -# Allow imported modules to modify the java.policy -ifneq ($(IMPORT_MODULES_CONF), ) - DEF_POLICY_SRC_LIST += $(wildcard $(IMPORT_MODULES_CONF)/java.base/security/java.policy.extra) -endif - -$(DEF_POLICY_DST): $(DEF_POLICY_SRC_LIST) - $(call MakeTargetDir) - $(RM) $@ $@.tmp - $(foreach f, $(DEF_POLICY_SRC_LIST), $(CAT) $(f) >> $@.tmp;) - $(MV) $@.tmp $@ - -TARGETS += $(DEF_POLICY_DST) - -################################################################################ - # CACERTS_FILE is optionally set in configure to override the default cacerts # which is otherwise generated in Gendata-java.base.gmk CACERTS_DST := $(LIB_DST_DIR)/security/cacerts diff --git a/make/modules/java.desktop/lib/AwtLibraries.gmk b/make/modules/java.desktop/lib/AwtLibraries.gmk index b2904f961fd..90eda1c4e64 100644 --- a/make/modules/java.desktop/lib/AwtLibraries.gmk +++ b/make/modules/java.desktop/lib/AwtLibraries.gmk @@ -85,11 +85,7 @@ LIBAWT_EXTRA_HEADER_DIRS := \ LIBAWT_CFLAGS := -D__MEDIALIB_OLD_NAMES -D__USE_J2D_NAMES -DMLIB_NO_LIBSUNMATH ifeq ($(call isTargetOs, windows), true) - LIBAWT_CFLAGS += -EHsc -DUNICODE -D_UNICODE - ifeq ($(call isTargetCpuBits, 64), true) - LIBAWT_CFLAGS += -DMLIB_OS64BIT - endif - + LIBAWT_CFLAGS += -EHsc -DUNICODE -D_UNICODE -DMLIB_OS64BIT LIBAWT_RCFLAGS ?= -I$(TOPDIR)/src/java.base/windows/native/launcher/icons LIBAWT_VERSIONINFO_RESOURCE := \ $(TOPDIR)/src/$(MODULE)/windows/native/libawt/windows/awt.rc diff --git a/make/modules/java.rmi/Launcher.gmk b/make/modules/java.rmi/Launcher.gmk index 14b381a540e..8c335711da0 100644 --- a/make/modules/java.rmi/Launcher.gmk +++ b/make/modules/java.rmi/Launcher.gmk @@ -31,5 +31,4 @@ include LauncherCommon.gmk $(eval $(call SetupBuildLauncher, rmiregistry, \ MAIN_CLASS := sun.rmi.registry.RegistryImpl, \ - JAVA_ARGS := -Djava.security.manager=allow, \ )) diff --git a/make/modules/jdk.accessibility/Launcher.gmk b/make/modules/jdk.accessibility/Launcher.gmk index 2499cf56616..86ad96b3a1e 100644 --- a/make/modules/jdk.accessibility/Launcher.gmk +++ b/make/modules/jdk.accessibility/Launcher.gmk @@ -48,63 +48,44 @@ ifeq ($(call isTargetOs, windows), true) TARGETS += $(BUILD_JABSWITCH) ############################################################################## - # Setup rules to create 32/64 bit version of jaccessinspector - # - # Parameter 1 File name suffix - # Parameter 2 ACCESSBRIDGE_ARCH_ -D suffix + # Build jaccessinspector ############################################################################## - define SetupInspector - $$(eval $$(call SetupJdkExecutable, BUILD_JACCESSINSPECTOR$1, \ - NAME := jaccessinspector$1, \ - SRC := jaccessinspector, \ - EXTRA_SRC := \ - bridge \ - common \ - toolscommon, \ - EXTRA_HEADER_DIRS := include/bridge, \ - CFLAGS := -DACCESSBRIDGE_ARCH_$2 -EHsc, \ - CXXFLAGS := -DACCESSBRIDGE_ARCH_$2 -EHsc, \ - LDFLAGS_windows := -stack:655360, \ - LIBS_windows := advapi32.lib user32.lib, \ - VERSIONINFO_RESOURCE := \ - $(ACCESSIBILITY_SRCDIR)/jaccessinspector/jaccessinspectorWindow.rc, \ - )) - TARGETS += $$(BUILD_JACCESSINSPECTOR$1) - endef + $(eval $(call SetupJdkExecutable, BUILD_JACCESSINSPECTOR, \ + NAME := jaccessinspector, \ + EXTRA_SRC := \ + bridge \ + common \ + toolscommon, \ + EXTRA_HEADER_DIRS := include/bridge, \ + CFLAGS := -DACCESSBRIDGE_ARCH_64 -EHsc, \ + CXXFLAGS := -DACCESSBRIDGE_ARCH_64 -EHsc, \ + LDFLAGS_windows := -stack:655360, \ + LIBS_windows := advapi32.lib user32.lib, \ + VERSIONINFO_RESOURCE := \ + $(ACCESSIBILITY_SRCDIR)/jaccessinspector/jaccessinspectorWindow.rc, \ + )) + + TARGETS += $(BUILD_JACCESSINSPECTOR) ############################################################################## - # Setup rules to create 32/64 bit version of jaccesswalker - # Parameter 1 File name suffix - # Parameter 2 ACCESSBRIDGE_ARCH_ -D suffix + # Build jaccesswalker ############################################################################## - define SetupWalker - $$(eval $$(call SetupJdkExecutable, BUILD_JACCESSWALKER$1, \ - NAME := jaccesswalker$1, \ - SRC := jaccesswalker, \ - EXTRA_SRC := \ - bridge \ - common \ - toolscommon, \ - EXTRA_HEADER_DIRS := include/bridge, \ - CFLAGS := -DACCESSBRIDGE_ARCH_$2 -EHsc, \ - CXXFLAGS := -DACCESSBRIDGE_ARCH_$2 -EHsc, \ - LDFLAGS_windows := -stack:655360, \ - LIBS_windows := advapi32.lib comctl32.lib gdi32.lib user32.lib, \ - VERSIONINFO_RESOURCE := \ - $(ACCESSIBILITY_SRCDIR)/jaccesswalker/jaccesswalkerWindow.rc, \ - )) - TARGETS += $$(BUILD_JACCESSWALKER$1) - endef + $(eval $(call SetupJdkExecutable, BUILD_JACCESSWALKER, \ + NAME := jaccesswalker, \ + EXTRA_SRC := \ + bridge \ + common \ + toolscommon, \ + EXTRA_HEADER_DIRS := include/bridge, \ + CFLAGS := -DACCESSBRIDGE_ARCH_64 -EHsc, \ + CXXFLAGS := -DACCESSBRIDGE_ARCH_64 -EHsc, \ + LDFLAGS_windows := -stack:655360, \ + LIBS_windows := advapi32.lib comctl32.lib gdi32.lib user32.lib, \ + VERSIONINFO_RESOURCE := \ + $(ACCESSIBILITY_SRCDIR)/jaccesswalker/jaccesswalkerWindow.rc, \ + )) - ifeq ($(call isTargetCpuBits, 32), true) - $(eval $(call SetupInspector,-32,32)) - $(eval $(call SetupWalker,-32,32)) - $(eval $(call SetupInspector,,LEGACY)) - $(eval $(call SetupWalker,,LEGACY)) - else - $(eval $(call SetupInspector,,64)) - $(eval $(call SetupWalker,,64)) - endif + TARGETS += $(BUILD_JACCESSWALKER) endif diff --git a/make/modules/jdk.accessibility/Lib.gmk b/make/modules/jdk.accessibility/Lib.gmk index 657eec2ad4e..6a429a56375 100644 --- a/make/modules/jdk.accessibility/Lib.gmk +++ b/make/modules/jdk.accessibility/Lib.gmk @@ -31,84 +31,50 @@ ifeq ($(call isTargetOs, windows), true) ACCESSIBILITY_SRCDIR := $(TOPDIR)/src/jdk.accessibility/windows/native ############################################################################## - # Setup rules to create 32/64 bit version of javaaccessbridge - # - # Parameter 1 Suffix - # Parameter 2 ACCESSBRIDGE_ARCH_ suffix + ## Build libjavaaccessbridge ############################################################################## - define SetupJavaDLL - $(call SetupJdkLibrary, BUILD_LIBJAVAACCESSBRIDGE$1, \ - NAME := javaaccessbridge$1, \ - SRC := libjavaaccessbridge, \ - EXTRA_SRC := common, \ - OPTIMIZATION := LOW, \ - DISABLED_WARNINGS_microsoft := 4311 4302 4312, \ - CXXFLAGS_FILTER_OUT := -MD, \ - CXXFLAGS := -MT -DACCESSBRIDGE_ARCH_$2, \ - EXTRA_HEADER_DIRS := \ - include/bridge \ - java.desktop:include, \ - JDK_LIBS := java.desktop:libjawt, \ - LIBS_windows := advapi32.lib comdlg32.lib gdi32.lib kernel32.lib \ - odbc32.lib odbccp32.lib ole32.lib oleaut32.lib shell32.lib \ - user32.lib uuid.lib winspool.lib, \ - VERSIONINFO_RESOURCE := \ - $(ACCESSIBILITY_SRCDIR)/common/AccessBridgeStatusWindow.rc, \ - ) - TARGETS += $$(BUILD_LIBJAVAACCESSBRIDGE$1) - endef + $(eval $(call SetupJdkLibrary, BUILD_LIBJAVAACCESSBRIDGE, \ + NAME := javaaccessbridge, \ + EXTRA_SRC := common, \ + OPTIMIZATION := LOW, \ + DISABLED_WARNINGS_microsoft := 4311 4302 4312, \ + CXXFLAGS_FILTER_OUT := -MD, \ + CXXFLAGS := -MT -DACCESSBRIDGE_ARCH_64, \ + EXTRA_HEADER_DIRS := \ + include/bridge \ + java.desktop:include, \ + JDK_LIBS := java.desktop:libjawt, \ + LIBS_windows := advapi32.lib comdlg32.lib gdi32.lib kernel32.lib \ + odbc32.lib odbccp32.lib ole32.lib oleaut32.lib shell32.lib \ + user32.lib uuid.lib winspool.lib, \ + VERSIONINFO_RESOURCE := \ + $(ACCESSIBILITY_SRCDIR)/common/AccessBridgeStatusWindow.rc, \ + )) + + TARGETS += $(BUILD_LIBJAVAACCESSBRIDGE) ############################################################################## - # Setup rules to create 32/64 bit version of windowsaccessbridge - # - # Parameter 1 Suffix - # Parameter 2 ACCESSBRIDGE_ARCH_ suffix + ## Build libwindowsaccessbridge ############################################################################## - define SetupWinDLL - $(call SetupJdkLibrary, BUILD_LIBWINDOWSACCESSBRIDGE$1, \ - NAME := windowsaccessbridge$1, \ - SRC := libwindowsaccessbridge, \ - EXTRA_SRC := common, \ - OPTIMIZATION := LOW, \ - DISABLED_WARNINGS_microsoft_WinAccessBridge.cpp := 4302 4311, \ - CXXFLAGS := -DACCESSBRIDGE_ARCH_$2, \ - EXTRA_HEADER_DIRS := \ - include/bridge, \ - LDFLAGS := \ - -def:$(ACCESSIBILITY_SRCDIR)/libwindowsaccessbridge/WinAccessBridge.DEF, \ - LIBS_windows := advapi32.lib comdlg32.lib gdi32.lib kernel32.lib \ - odbc32.lib odbccp32.lib ole32.lib oleaut32.lib shell32.lib \ - user32.lib uuid.lib winspool.lib, \ - VERSIONINFO_RESOURCE := \ - $(ACCESSIBILITY_SRCDIR)/common/AccessBridgeStatusWindow.rc, \ - ) - - TARGETS += $$(BUILD_LIBWINDOWSACCESSBRIDGE$1) - endef - - ifeq ($(call isTargetCpuBits, 32), true) - ############################################################################ - # Build libjabsysinfo - ############################################################################ - - $(eval $(call SetupJdkLibrary, BUILD_LIBJABSYSINFO, \ - NAME := jabsysinfo, \ - OPTIMIZATION := LOW, \ - VERSIONINFO_RESOURCE := \ - $(ACCESSIBILITY_SRCDIR)/common/AccessBridgeStatusWindow.rc, \ - ) - TARGETS += $(BUILD_LIBJABSYSINFO) - endif + $(eval $(call SetupJdkLibrary, BUILD_LIBWINDOWSACCESSBRIDGE, \ + NAME := windowsaccessbridge-64, \ + SRC := libwindowsaccessbridge, \ + EXTRA_SRC := common, \ + OPTIMIZATION := LOW, \ + DISABLED_WARNINGS_microsoft_WinAccessBridge.cpp := 4302 4311, \ + CXXFLAGS := -DACCESSBRIDGE_ARCH_64, \ + EXTRA_HEADER_DIRS := \ + include/bridge, \ + LDFLAGS := \ + -def:$(ACCESSIBILITY_SRCDIR)/libwindowsaccessbridge/WinAccessBridge.DEF, \ + LIBS_windows := advapi32.lib comdlg32.lib gdi32.lib kernel32.lib \ + odbc32.lib odbccp32.lib ole32.lib oleaut32.lib shell32.lib \ + user32.lib uuid.lib winspool.lib, \ + VERSIONINFO_RESOURCE := \ + $(ACCESSIBILITY_SRCDIR)/common/AccessBridgeStatusWindow.rc, \ + )) - ifeq ($(call isTargetCpuBits, 32), true) - $(eval $(call SetupJavaDLL,-32,32)) - $(eval $(call SetupJavaDLL,,LEGACY)) - $(eval $(call SetupWinDLL,-32,32)) - $(eval $(call SetupWinDLL,,LEGACY)) - else - $(eval $(call SetupJavaDLL,,64)) - $(eval $(call SetupWinDLL,-64,64)) - endif + TARGETS += $(BUILD_LIBWINDOWSACCESSBRIDGE) endif diff --git a/make/scripts/compare.sh b/make/scripts/compare.sh index e81c460225c..cc2f4adf2ed 100644 --- a/make/scripts/compare.sh +++ b/make/scripts/compare.sh @@ -76,26 +76,13 @@ fi # disassembly, such as hard-coded addresses, to be able to catch "actual" differences. if [ "$OPENJDK_TARGET_OS" = "windows" ]; then - if [ "$OPENJDK_TARGET_CPU" = "x86" ]; then - DIS_DIFF_FILTER="$SED -r \ - -e 's/^ [0-9A-F]{16}: //' \ - -e 's/^ [0-9A-F]{8}: / : /' \ - -e 's/(offset \?\?)_C@_.*/\1/' \ - -e 's/[@?][A-Za-z0-9_]{1,25}//' \ - -e 's/([-,+])[0-9A-F]{2,16}/\1/g' \ - -e 's/\[[0-9A-F]{4,16}h\]/[]/' \ - -e 's/: ([a-z]{2}[a-z ]{2}) [0-9A-F]{2,16}h?$/: \1 /' \ - -e 's/_20[0-9]{2}_[0-1][0-9]_[0-9]{2}/_/' \ - " - elif [ "$OPENJDK_TARGET_CPU" = "x86_64" ]; then - DIS_DIFF_FILTER="$SED -r \ - -e 's/^ [0-9A-F]{16}: //' \ - -e 's/\[[0-9A-F]{4,16}h\]/[]/' \ - -e 's/([,+])[0-9A-F]{2,16}h/\1/' \ - -e 's/([a-z]{2}[a-z ]{2}) [0-9A-F]{4,16}$/\1 /' \ - -e 's/\[\?\?_C@_.*/[]/' \ - " - fi + DIS_DIFF_FILTER="$SED -r \ + -e 's/^ [0-9A-F]{16}: //' \ + -e 's/\[[0-9A-F]{4,16}h\]/[]/' \ + -e 's/([,+])[0-9A-F]{2,16}h/\1/' \ + -e 's/([a-z]{2}[a-z ]{2}) [0-9A-F]{4,16}$/\1 /' \ + -e 's/\[\?\?_C@_.*/[]/' \ + " elif [ "$OPENJDK_TARGET_OS" = "macosx" ]; then DIS_DIFF_FILTER="$SED \ -e 's/0x[0-9a-f]\{3,16\}//g' -e 's/^[0-9a-f]\{12,20\}//' \ @@ -1467,11 +1454,7 @@ if [ "$SKIP_DEFAULT" != "true" ]; then OTHER_SEC_BIN="$OTHER_SEC_DIR/sec-bin.zip" THIS_SEC_BIN="$THIS_SEC_DIR/sec-bin.zip" if [ "$OPENJDK_TARGET_OS" = "windows" ]; then - if [ "$OPENJDK_TARGET_CPU" = "x86_64" ]; then - JGSS_WINDOWS_BIN="jgss-windows-x64-bin.zip" - else - JGSS_WINDOWS_BIN="jgss-windows-i586-bin.zip" - fi + JGSS_WINDOWS_BIN="jgss-windows-x64-bin.zip" OTHER_SEC_WINDOWS_BIN="$OTHER_SEC_DIR/sec-windows-bin.zip" OTHER_JGSS_WINDOWS_BIN="$OTHER_SEC_DIR/$JGSS_WINDOWS_BIN" THIS_SEC_WINDOWS_BIN="$THIS_SEC_DIR/sec-windows-bin.zip" diff --git a/src/hotspot/cpu/aarch64/aarch64.ad b/src/hotspot/cpu/aarch64/aarch64.ad index eedf29cc563..a97d01b7683 100644 --- a/src/hotspot/cpu/aarch64/aarch64.ad +++ b/src/hotspot/cpu/aarch64/aarch64.ad @@ -1648,8 +1648,8 @@ int MachCallRuntimeNode::ret_addr_offset() { // for real runtime callouts it will be six instructions // see aarch64_enc_java_to_runtime // adr(rscratch2, retaddr) + // str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset())); // lea(rscratch1, RuntimeAddress(addr) - // stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize))) // blr(rscratch1) CodeBlob *cb = CodeCache::find_blob(_entry_point); if (cb) { @@ -3696,14 +3696,13 @@ encode %{ __ post_call_nop(); } else { Label retaddr; + // Make the anchor frame walkable __ adr(rscratch2, retaddr); + __ str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset())); __ lea(rscratch1, RuntimeAddress(entry)); - // Leave a breadcrumb for JavaFrameAnchor::capture_last_Java_pc() - __ stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize))); __ blr(rscratch1); __ bind(retaddr); __ post_call_nop(); - __ add(sp, sp, 2 * wordSize); } if (Compile::current()->max_vector_size() > 0) { __ reinitialize_ptrue(); @@ -5756,6 +5755,10 @@ opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indInde indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P); +opclass memory_noindex(indirect, + indOffI1, indOffL1,indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8, + indirectN, indOffIN, indOffLN, indirectX2P, indOffX2P); + // iRegIorL2I is used for src inputs in rules for 32 bit int (I) // operations. it allows the src to be either an iRegI or a (ConvL2I // iRegL). in the latter case the l2i normally planted for a ConvL2I @@ -6682,7 +6685,7 @@ instruct loadKlass(iRegPNoSp dst, memory8 mem) instruct loadNKlass(iRegNNoSp dst, memory4 mem) %{ match(Set dst (LoadNKlass mem)); - predicate(!needs_acquiring_load(n)); + predicate(!needs_acquiring_load(n) && !UseCompactObjectHeaders); ins_cost(4 * INSN_COST); format %{ "ldrw $dst, $mem\t# compressed class ptr" %} @@ -6692,6 +6695,20 @@ instruct loadNKlass(iRegNNoSp dst, memory4 mem) ins_pipe(iload_reg_mem); %} +instruct loadNKlassCompactHeaders(iRegNNoSp dst, memory_noindex mem) +%{ + match(Set dst (LoadNKlass mem)); + predicate(!needs_acquiring_load(n) && UseCompactObjectHeaders); + + ins_cost(4 * INSN_COST); + format %{ "load_narrow_klass_compact $dst, $mem\t# compressed class ptr" %} + ins_encode %{ + assert($mem$$index$$Register == noreg, "must not have indexed address"); + __ load_narrow_klass_compact_c2($dst$$Register, $mem$$base$$Register, $mem$$disp); + %} + ins_pipe(iload_reg_mem); +%} + // Load Float instruct loadF(vRegF dst, memory4 mem) %{ diff --git a/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp index 3d1be91e9b2..b01360c3f7e 100644 --- a/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp @@ -2243,8 +2243,6 @@ void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) { Address src_length_addr = Address(src, arrayOopDesc::length_offset_in_bytes()); Address dst_length_addr = Address(dst, arrayOopDesc::length_offset_in_bytes()); - Address src_klass_addr = Address(src, oopDesc::klass_offset_in_bytes()); - Address dst_klass_addr = Address(dst, oopDesc::klass_offset_in_bytes()); // test for null if (flags & LIR_OpArrayCopy::src_null_check) { @@ -2305,15 +2303,7 @@ void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) { // We don't know the array types are compatible if (basic_type != T_OBJECT) { // Simple test for basic type arrays - if (UseCompressedClassPointers) { - __ ldrw(tmp, src_klass_addr); - __ ldrw(rscratch1, dst_klass_addr); - __ cmpw(tmp, rscratch1); - } else { - __ ldr(tmp, src_klass_addr); - __ ldr(rscratch1, dst_klass_addr); - __ cmp(tmp, rscratch1); - } + __ cmp_klasses_from_objects(src, dst, tmp, rscratch1); __ br(Assembler::NE, *stub->entry()); } else { // For object arrays, if src is a sub class of dst then we can @@ -2435,36 +2425,14 @@ void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) { // but not necessarily exactly of type default_type. Label known_ok, halt; __ mov_metadata(tmp, default_type->constant_encoding()); - if (UseCompressedClassPointers) { - __ encode_klass_not_null(tmp); - } if (basic_type != T_OBJECT) { - - if (UseCompressedClassPointers) { - __ ldrw(rscratch1, dst_klass_addr); - __ cmpw(tmp, rscratch1); - } else { - __ ldr(rscratch1, dst_klass_addr); - __ cmp(tmp, rscratch1); - } + __ cmp_klass(dst, tmp, rscratch1); __ br(Assembler::NE, halt); - if (UseCompressedClassPointers) { - __ ldrw(rscratch1, src_klass_addr); - __ cmpw(tmp, rscratch1); - } else { - __ ldr(rscratch1, src_klass_addr); - __ cmp(tmp, rscratch1); - } + __ cmp_klass(src, tmp, rscratch1); __ br(Assembler::EQ, known_ok); } else { - if (UseCompressedClassPointers) { - __ ldrw(rscratch1, dst_klass_addr); - __ cmpw(tmp, rscratch1); - } else { - __ ldr(rscratch1, dst_klass_addr); - __ cmp(tmp, rscratch1); - } + __ cmp_klass(dst, tmp, rscratch1); __ br(Assembler::EQ, known_ok); __ cmp(src, dst); __ br(Assembler::EQ, known_ok); @@ -2547,12 +2515,7 @@ void LIR_Assembler::emit_load_klass(LIR_OpLoadKlass* op) { add_debug_info_for_null_check_here(info); } - if (UseCompressedClassPointers) { - __ ldrw(result, Address (obj, oopDesc::klass_offset_in_bytes())); - __ decode_klass_not_null(result); - } else { - __ ldr(result, Address (obj, oopDesc::klass_offset_in_bytes())); - } + __ load_klass(result, obj); } void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) { diff --git a/src/hotspot/cpu/aarch64/c1_MacroAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/c1_MacroAssembler_aarch64.cpp index 8d1b3902ce4..fc6cadc8450 100644 --- a/src/hotspot/cpu/aarch64/c1_MacroAssembler_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/c1_MacroAssembler_aarch64.cpp @@ -119,8 +119,8 @@ int C1_MacroAssembler::lock_object(Register hdr, Register obj, Register disp_hdr cbnz(hdr, slow_case); // done bind(done); + inc_held_monitor_count(rscratch1); } - increment(Address(rthread, JavaThread::held_monitor_count_offset())); return null_check_offset; } @@ -159,8 +159,8 @@ void C1_MacroAssembler::unlock_object(Register hdr, Register obj, Register disp_ } // done bind(done); + dec_held_monitor_count(rscratch1); } - decrement(Address(rthread, JavaThread::held_monitor_count_offset())); } @@ -175,15 +175,19 @@ void C1_MacroAssembler::try_allocate(Register obj, Register var_size_in_bytes, i void C1_MacroAssembler::initialize_header(Register obj, Register klass, Register len, Register t1, Register t2) { assert_different_registers(obj, klass, len); - // This assumes that all prototype bits fit in an int32_t - mov(t1, (int32_t)(intptr_t)markWord::prototype().value()); - str(t1, Address(obj, oopDesc::mark_offset_in_bytes())); - if (UseCompressedClassPointers) { // Take care not to kill klass - encode_klass_not_null(t1, klass); - strw(t1, Address(obj, oopDesc::klass_offset_in_bytes())); + if (UseCompactObjectHeaders) { + ldr(t1, Address(klass, Klass::prototype_header_offset())); + str(t1, Address(obj, oopDesc::mark_offset_in_bytes())); } else { - str(klass, Address(obj, oopDesc::klass_offset_in_bytes())); + mov(t1, checked_cast(markWord::prototype().value())); + str(t1, Address(obj, oopDesc::mark_offset_in_bytes())); + if (UseCompressedClassPointers) { // Take care not to kill klass + encode_klass_not_null(t1, klass); + strw(t1, Address(obj, oopDesc::klass_offset_in_bytes())); + } else { + str(klass, Address(obj, oopDesc::klass_offset_in_bytes())); + } } if (len->is_valid()) { @@ -194,7 +198,7 @@ void C1_MacroAssembler::initialize_header(Register obj, Register klass, Register // Clear gap/first 4 bytes following the length field. strw(zr, Address(obj, base_offset)); } - } else if (UseCompressedClassPointers) { + } else if (UseCompressedClassPointers && !UseCompactObjectHeaders) { store_klass_gap(obj, zr); } } diff --git a/src/hotspot/cpu/aarch64/c1_Runtime1_aarch64.cpp b/src/hotspot/cpu/aarch64/c1_Runtime1_aarch64.cpp index 00b678d5405..19665d1b894 100644 --- a/src/hotspot/cpu/aarch64/c1_Runtime1_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/c1_Runtime1_aarch64.cpp @@ -160,16 +160,15 @@ int StubAssembler::call_RT(Register oop_result1, Register metadata_result, addre } enum return_state_t { - does_not_return, requires_return + does_not_return, requires_return, requires_pop_epilogue_return }; - // Implementation of StubFrame class StubFrame: public StackObj { private: StubAssembler* _sasm; - bool _return_state; + return_state_t _return_state; public: StubFrame(StubAssembler* sasm, const char* name, bool must_gc_arguments, return_state_t return_state=requires_return); @@ -183,8 +182,17 @@ void StubAssembler::prologue(const char* name, bool must_gc_arguments) { enter(); } -void StubAssembler::epilogue() { - leave(); +void StubAssembler::epilogue(bool use_pop) { + // Avoid using a leave instruction when this frame may + // have been frozen, since the current value of rfp + // restored from the stub would be invalid. We still + // must restore the rfp value saved on enter though. + if (use_pop) { + ldp(rfp, lr, Address(post(sp, 2 * wordSize))); + authenticate_return_address(); + } else { + leave(); + } ret(lr); } @@ -203,10 +211,10 @@ void StubFrame::load_argument(int offset_in_words, Register reg) { } StubFrame::~StubFrame() { - if (_return_state == requires_return) { - __ epilogue(); - } else { + if (_return_state == does_not_return) { __ should_not_reach_here(); + } else { + __ epilogue(_return_state == requires_pop_epilogue_return); } } @@ -252,7 +260,7 @@ static OopMap* generate_oop_map(StubAssembler* sasm, bool save_fpu_registers) { for (int i = 0; i < FrameMap::nof_cpu_regs; i++) { Register r = as_Register(i); - if (i <= 18 && i != rscratch1->encoding() && i != rscratch2->encoding()) { + if (r == rthread || (i <= 18 && i != rscratch1->encoding() && i != rscratch2->encoding())) { int sp_offset = cpu_reg_save_offsets[i]; oop_map->set_callee_saved(VMRegImpl::stack2reg(sp_offset), r->as_VMReg()); @@ -337,6 +345,15 @@ void Runtime1::initialize_pd() { } } +// return: offset in 64-bit words. +uint Runtime1::runtime_blob_current_thread_offset(frame f) { + CodeBlob* cb = f.cb(); + assert(cb == Runtime1::blob_for(C1StubId::monitorenter_id) || + cb == Runtime1::blob_for(C1StubId::monitorenter_nofpu_id), "must be"); + assert(cb != nullptr && cb->is_runtime_stub(), "invalid frame"); + int offset = cpu_reg_save_offsets[rthread->encoding()]; + return offset / 2; // SP offsets are in halfwords +} // target: the entry point of the method that creates and posts the exception oop // has_argument: true if the exception needs arguments (passed in rscratch1 and rscratch2) @@ -868,7 +885,7 @@ OopMapSet* Runtime1::generate_code_for(C1StubId id, StubAssembler* sasm) { // fall through case C1StubId::monitorenter_id: { - StubFrame f(sasm, "monitorenter", dont_gc_arguments); + StubFrame f(sasm, "monitorenter", dont_gc_arguments, requires_pop_epilogue_return); OopMap* map = save_live_registers(sasm, save_fpu_registers); // Called with store_parameter and not C abi diff --git a/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.cpp index 5ce76106c9e..4c22133c056 100644 --- a/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.cpp @@ -153,7 +153,7 @@ void C2_MacroAssembler::fast_lock(Register objectReg, Register boxReg, Register Label count, no_count; assert(LockingMode != LM_LIGHTWEIGHT, "lightweight locking should use fast_lock_lightweight"); - assert_different_registers(oop, box, tmp, disp_hdr); + assert_different_registers(oop, box, tmp, disp_hdr, rscratch2); // Load markWord from object into displaced_header. ldr(disp_hdr, Address(oop, oopDesc::mark_offset_in_bytes())); @@ -206,12 +206,10 @@ void C2_MacroAssembler::fast_lock(Register objectReg, Register boxReg, Register // Handle existing monitor. bind(object_has_monitor); - // The object's monitor m is unlocked iff m->owner == nullptr, - // otherwise m->owner may contain a thread or a stack address. - // - // Try to CAS m->owner from null to current thread. + // Try to CAS owner (no owner => current thread's _lock_id). + ldr(rscratch2, Address(rthread, JavaThread::lock_id_offset())); add(tmp, disp_hdr, (in_bytes(ObjectMonitor::owner_offset())-markWord::monitor_value)); - cmpxchg(tmp, zr, rthread, Assembler::xword, /*acquire*/ true, + cmpxchg(tmp, zr, rscratch2, Assembler::xword, /*acquire*/ true, /*release*/ true, /*weak*/ false, tmp3Reg); // Sets flags for result // Store a non-null value into the box to avoid looking like a re-entrant @@ -223,7 +221,7 @@ void C2_MacroAssembler::fast_lock(Register objectReg, Register boxReg, Register br(Assembler::EQ, cont); // CAS success means locking succeeded - cmp(tmp3Reg, rthread); + cmp(tmp3Reg, rscratch2); br(Assembler::NE, cont); // Check for recursive locking // Recursive lock case @@ -236,7 +234,9 @@ void C2_MacroAssembler::fast_lock(Register objectReg, Register boxReg, Register br(Assembler::NE, no_count); bind(count); - increment(Address(rthread, JavaThread::held_monitor_count_offset())); + if (LockingMode == LM_LEGACY) { + inc_held_monitor_count(rscratch1); + } bind(no_count); } @@ -343,7 +343,9 @@ void C2_MacroAssembler::fast_unlock(Register objectReg, Register boxReg, Registe br(Assembler::NE, no_count); bind(count); - decrement(Address(rthread, JavaThread::held_monitor_count_offset())); + if (LockingMode == LM_LEGACY) { + dec_held_monitor_count(rscratch1); + } bind(no_count); } @@ -351,7 +353,7 @@ void C2_MacroAssembler::fast_unlock(Register objectReg, Register boxReg, Registe void C2_MacroAssembler::fast_lock_lightweight(Register obj, Register box, Register t1, Register t2, Register t3) { assert(LockingMode == LM_LIGHTWEIGHT, "must be"); - assert_different_registers(obj, box, t1, t2, t3); + assert_different_registers(obj, box, t1, t2, t3, rscratch2); // Handle inflated monitor. Label inflated; @@ -467,13 +469,14 @@ void C2_MacroAssembler::fast_lock_lightweight(Register obj, Register box, Regist // Compute owner address. lea(t2_owner_addr, owner_address); - // CAS owner (null => current thread). - cmpxchg(t2_owner_addr, zr, rthread, Assembler::xword, /*acquire*/ true, + // Try to CAS owner (no owner => current thread's _lock_id). + ldr(rscratch2, Address(rthread, JavaThread::lock_id_offset())); + cmpxchg(t2_owner_addr, zr, rscratch2, Assembler::xword, /*acquire*/ true, /*release*/ false, /*weak*/ false, t3_owner); br(Assembler::EQ, monitor_locked); // Check if recursive. - cmp(t3_owner, rthread); + cmp(t3_owner, rscratch2); br(Assembler::NE, slow_path); // Recursive. @@ -486,7 +489,6 @@ void C2_MacroAssembler::fast_lock_lightweight(Register obj, Register box, Regist } bind(locked); - increment(Address(rthread, JavaThread::held_monitor_count_offset())); #ifdef ASSERT // Check that locked label is reached with Flags == EQ. @@ -655,7 +657,6 @@ void C2_MacroAssembler::fast_unlock_lightweight(Register obj, Register box, Regi } bind(unlocked); - decrement(Address(rthread, JavaThread::held_monitor_count_offset())); cmp(zr, zr); // Set Flags to EQ => fast path #ifdef ASSERT @@ -2689,3 +2690,12 @@ bool C2_MacroAssembler::in_scratch_emit_size() { } return MacroAssembler::in_scratch_emit_size(); } + +void C2_MacroAssembler::load_narrow_klass_compact_c2(Register dst, Register obj, int disp) { + // Note: Don't clobber obj anywhere in that method! + + // The incoming address is pointing into obj-start + klass_offset_in_bytes. We need to extract + // obj-start, so that we can load from the object's mark-word instead. + ldr(dst, Address(obj, disp - oopDesc::klass_offset_in_bytes())); + lsr(dst, dst, markWord::klass_shift); +} diff --git a/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.hpp b/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.hpp index d61b050407d..c6ddcf46cba 100644 --- a/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.hpp +++ b/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.hpp @@ -186,4 +186,6 @@ void vector_signum_sve(FloatRegister dst, FloatRegister src, FloatRegister zero, FloatRegister one, FloatRegister vtmp, PRegister pgtmp, SIMD_RegVariant T); + void load_narrow_klass_compact_c2(Register dst, Register obj, int disp); + #endif // CPU_AARCH64_C2_MACROASSEMBLER_AARCH64_HPP diff --git a/src/hotspot/cpu/aarch64/compressedKlass_aarch64.cpp b/src/hotspot/cpu/aarch64/compressedKlass_aarch64.cpp index fc78813c161..b96241aab19 100644 --- a/src/hotspot/cpu/aarch64/compressedKlass_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/compressedKlass_aarch64.cpp @@ -117,19 +117,3 @@ char* CompressedKlassPointers::reserve_address_space_for_compressed_classes(size return result; } - -void CompressedKlassPointers::initialize(address addr, size_t len) { - constexpr uintptr_t unscaled_max = nth_bit(32); - assert(len <= unscaled_max, "Klass range larger than 32 bits?"); - - // Shift is always 0 on aarch64. - _shift = 0; - - // On aarch64, we don't bother with zero-based encoding (base=0 shift>0). - address const end = addr + len; - _base = (end <= (address)unscaled_max) ? nullptr : addr; - - // Remember the Klass range: - _klass_range_start = addr; - _klass_range_end = addr + len; -} diff --git a/src/hotspot/cpu/aarch64/continuationFreezeThaw_aarch64.inline.hpp b/src/hotspot/cpu/aarch64/continuationFreezeThaw_aarch64.inline.hpp index d3b9e89773c..e82162df4ab 100644 --- a/src/hotspot/cpu/aarch64/continuationFreezeThaw_aarch64.inline.hpp +++ b/src/hotspot/cpu/aarch64/continuationFreezeThaw_aarch64.inline.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2019, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -129,6 +129,11 @@ void FreezeBase::adjust_interpreted_frame_unextended_sp(frame& f) { } } +inline void FreezeBase::prepare_freeze_interpreted_top_frame(frame& f) { + assert(f.interpreter_frame_last_sp() == nullptr, "should be null for top frame"); + f.interpreter_frame_set_last_sp(f.unextended_sp()); +} + inline void FreezeBase::relativize_interpreted_frame_metadata(const frame& f, const frame& hf) { assert(hf.fp() == hf.unextended_sp() + (f.fp() - f.unextended_sp()), ""); assert((f.at(frame::interpreter_frame_last_sp_offset) != 0) @@ -149,10 +154,16 @@ inline void FreezeBase::relativize_interpreted_frame_metadata(const frame& f, co // extended_sp is already relativized by TemplateInterpreterGenerator::generate_normal_entry or // AbstractInterpreter::layout_activation + // The interpreter native wrapper code adds space in the stack equal to size_of_parameters() + // after the fixed part of the frame. For wait0 this is equal to 3 words (this + long parameter). + // We adjust by this size since otherwise the saved last sp will be less than the extended_sp. + DEBUG_ONLY(Method* m = hf.interpreter_frame_method();) + DEBUG_ONLY(int extra_space = m->is_object_wait0() ? m->size_of_parameters() : 0;) + assert((hf.fp() - hf.unextended_sp()) == (f.fp() - f.unextended_sp()), ""); assert(hf.unextended_sp() == (intptr_t*)hf.at(frame::interpreter_frame_last_sp_offset), ""); assert(hf.unextended_sp() <= (intptr_t*)hf.at(frame::interpreter_frame_initial_sp_offset), ""); - assert(hf.unextended_sp() > (intptr_t*)hf.at(frame::interpreter_frame_extended_sp_offset), ""); + assert(hf.unextended_sp() + extra_space > (intptr_t*)hf.at(frame::interpreter_frame_extended_sp_offset), ""); assert(hf.fp() > (intptr_t*)hf.at(frame::interpreter_frame_initial_sp_offset), ""); assert(hf.fp() <= (intptr_t*)hf.at(frame::interpreter_frame_locals_offset), ""); } @@ -213,7 +224,6 @@ template frame ThawBase::new_stack_frame(const frame& hf, frame& // If caller is interpreted it already made room for the callee arguments int overlap = caller.is_interpreted_frame() ? ContinuationHelper::InterpretedFrame::stack_argsize(hf) : 0; const int fsize = (int)(ContinuationHelper::InterpretedFrame::frame_bottom(hf) - hf.unextended_sp() - overlap); - const int locals = hf.interpreter_frame_method()->max_locals(); intptr_t* frame_sp = caller.unextended_sp() - fsize; intptr_t* fp = frame_sp + (hf.fp() - heap_sp); if ((intptr_t)fp % frame::frame_alignment != 0) { @@ -235,7 +245,7 @@ template frame ThawBase::new_stack_frame(const frame& hf, frame& int fsize = FKind::size(hf); intptr_t* frame_sp = caller.unextended_sp() - fsize; if (bottom || caller.is_interpreted_frame()) { - int argsize = hf.compiled_frame_stack_argsize(); + int argsize = FKind::stack_argsize(hf); fsize += argsize; frame_sp -= argsize; @@ -252,8 +262,8 @@ template frame ThawBase::new_stack_frame(const frame& hf, frame& // we need to recreate a "real" frame pointer, pointing into the stack fp = frame_sp + FKind::size(hf) - frame::sender_sp_offset; } else { - fp = FKind::stub - ? frame_sp + fsize - frame::sender_sp_offset // on AArch64, this value is used for the safepoint stub + fp = FKind::stub || FKind::native + ? frame_sp + fsize - frame::sender_sp_offset // fp always points to the address below the pushed return pc. We need correct address. : *(intptr_t**)(hf.sp() - frame::sender_sp_offset); // we need to re-read fp because it may be an oop and we might have fixed the frame. } return frame(frame_sp, frame_sp, fp, hf.pc(), hf.cb(), hf.oop_map(), false); // TODO PERF : this computes deopt state; is it necessary? @@ -277,6 +287,22 @@ inline void ThawBase::patch_pd(frame& f, const frame& caller) { patch_callee_link(caller, caller.fp()); } +inline void ThawBase::patch_pd(frame& f, intptr_t* caller_sp) { + intptr_t* fp = caller_sp - frame::sender_sp_offset; + patch_callee_link(f, fp); +} + +inline intptr_t* ThawBase::push_cleanup_continuation() { + frame enterSpecial = new_entry_frame(); + intptr_t* sp = enterSpecial.sp(); + + sp[-1] = (intptr_t)ContinuationEntry::cleanup_pc(); + sp[-2] = (intptr_t)enterSpecial.fp(); + + log_develop_trace(continuations, preempt)("push_cleanup_continuation initial sp: " INTPTR_FORMAT " final sp: " INTPTR_FORMAT, p2i(sp + 2 * frame::metadata_words), p2i(sp)); + return sp; +} + inline void ThawBase::derelativize_interpreted_frame_metadata(const frame& hf, const frame& f) { // Make sure that last_sp is kept relativized. assert((intptr_t*)f.at_relative(frame::interpreter_frame_last_sp_offset) == f.unextended_sp(), ""); @@ -285,7 +311,9 @@ inline void ThawBase::derelativize_interpreted_frame_metadata(const frame& hf, c assert(f.at_absolute(frame::interpreter_frame_monitor_block_top_offset) <= frame::interpreter_frame_initial_sp_offset, ""); // Make sure that extended_sp is kept relativized. - assert((intptr_t*)f.at_relative(frame::interpreter_frame_extended_sp_offset) < f.unextended_sp(), ""); + DEBUG_ONLY(Method* m = hf.interpreter_frame_method();) + DEBUG_ONLY(int extra_space = m->is_object_wait0() ? m->size_of_parameters() : 0;) // see comment in relativize_interpreted_frame_metadata() + assert((intptr_t*)f.at_relative(frame::interpreter_frame_extended_sp_offset) < f.unextended_sp() + extra_space, ""); } #endif // CPU_AARCH64_CONTINUATIONFREEZETHAW_AARCH64_INLINE_HPP diff --git a/src/hotspot/cpu/aarch64/continuationHelper_aarch64.inline.hpp b/src/hotspot/cpu/aarch64/continuationHelper_aarch64.inline.hpp index 2715ebef8f0..e39580369db 100644 --- a/src/hotspot/cpu/aarch64/continuationHelper_aarch64.inline.hpp +++ b/src/hotspot/cpu/aarch64/continuationHelper_aarch64.inline.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2022, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -40,6 +40,22 @@ static inline intptr_t** link_address(const frame& f) { : (intptr_t**)(f.unextended_sp() + f.cb()->frame_size() - frame::sender_sp_offset); } +static inline void patch_return_pc_with_preempt_stub(frame& f) { + if (f.is_runtime_frame()) { + // Unlike x86 we don't know where in the callee frame the return pc is + // saved so we can't patch the return from the VM call back to Java. + // Instead, we will patch the return from the runtime stub back to the + // compiled method so that the target returns to the preempt cleanup stub. + intptr_t* caller_sp = f.sp() + f.cb()->frame_size(); + caller_sp[-1] = (intptr_t)StubRoutines::cont_preempt_stub(); + } else { + // The target will check for preemption once it returns to the interpreter + // or the native wrapper code and will manually jump to the preempt stub. + JavaThread *thread = JavaThread::current(); + thread->set_preempt_alternate_return(StubRoutines::cont_preempt_stub()); + } +} + inline int ContinuationHelper::frame_align_words(int size) { #ifdef _LP64 return size & 1; @@ -83,12 +99,12 @@ inline void ContinuationHelper::set_anchor_to_entry_pd(JavaFrameAnchor* anchor, anchor->set_last_Java_fp(entry->entry_fp()); } -#ifdef ASSERT inline void ContinuationHelper::set_anchor_pd(JavaFrameAnchor* anchor, intptr_t* sp) { intptr_t* fp = *(intptr_t**)(sp - frame::sender_sp_offset); anchor->set_last_Java_fp(fp); } +#ifdef ASSERT inline bool ContinuationHelper::Frame::assert_frame_laid_out(frame f) { intptr_t* sp = f.sp(); address pc = ContinuationHelper::return_address_at( diff --git a/src/hotspot/cpu/aarch64/frame_aarch64.cpp b/src/hotspot/cpu/aarch64/frame_aarch64.cpp index c2b127f31bb..361b913fd2e 100644 --- a/src/hotspot/cpu/aarch64/frame_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/frame_aarch64.cpp @@ -420,6 +420,36 @@ frame frame::sender_for_upcall_stub_frame(RegisterMap* map) const { return fr; } +#if defined(ASSERT) +static address get_register_address_in_stub(const frame& stub_fr, VMReg reg) { + RegisterMap map(nullptr, + RegisterMap::UpdateMap::include, + RegisterMap::ProcessFrames::skip, + RegisterMap::WalkContinuation::skip); + stub_fr.oop_map()->update_register_map(&stub_fr, &map); + return map.location(reg, stub_fr.sp()); +} +#endif + +JavaThread** frame::saved_thread_address(const frame& f) { + CodeBlob* cb = f.cb(); + assert(cb != nullptr && cb->is_runtime_stub(), "invalid frame"); + + JavaThread** thread_addr; +#ifdef COMPILER1 + if (cb == Runtime1::blob_for(C1StubId::monitorenter_id) || + cb == Runtime1::blob_for(C1StubId::monitorenter_nofpu_id)) { + thread_addr = (JavaThread**)(f.sp() + Runtime1::runtime_blob_current_thread_offset(f)); + } else +#endif + { + // c2 only saves rbp in the stub frame so nothing to do. + thread_addr = nullptr; + } + assert(get_register_address_in_stub(f, SharedRuntime::thread_register()) == (address)thread_addr, "wrong thread address"); + return thread_addr; +} + //------------------------------------------------------------------------------ // frame::verify_deopt_original_pc // diff --git a/src/hotspot/cpu/aarch64/frame_aarch64.hpp b/src/hotspot/cpu/aarch64/frame_aarch64.hpp index 401e2c6ae97..da020b4234d 100644 --- a/src/hotspot/cpu/aarch64/frame_aarch64.hpp +++ b/src/hotspot/cpu/aarch64/frame_aarch64.hpp @@ -73,7 +73,8 @@ sender_sp_offset = 2, // Interpreter frames - interpreter_frame_oop_temp_offset = 3, // for native calls only + interpreter_frame_result_handler_offset = 3, // for native calls only + interpreter_frame_oop_temp_offset = 2, // for native calls only interpreter_frame_sender_sp_offset = -1, // outgoing sp before a call to an invoked method diff --git a/src/hotspot/cpu/aarch64/interp_masm_aarch64.cpp b/src/hotspot/cpu/aarch64/interp_masm_aarch64.cpp index c6af74c3dcd..b892489af70 100644 --- a/src/hotspot/cpu/aarch64/interp_masm_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/interp_masm_aarch64.cpp @@ -666,7 +666,7 @@ void InterpreterMacroAssembler::lock_object(Register lock_reg) { assert(lock_reg == c_rarg1, "The argument is only for looks. It must be c_rarg1"); if (LockingMode == LM_MONITOR) { - call_VM(noreg, + call_VM_preemptable(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter), lock_reg); } else { @@ -697,7 +697,7 @@ void InterpreterMacroAssembler::lock_object(Register lock_reg) if (LockingMode == LM_LIGHTWEIGHT) { lightweight_lock(lock_reg, obj_reg, tmp, tmp2, tmp3, slow_case); - b(count); + b(done); } else if (LockingMode == LM_LEGACY) { // Load (object->mark() | 1) into swap_reg ldr(rscratch1, Address(obj_reg, oopDesc::mark_offset_in_bytes())); @@ -747,18 +747,18 @@ void InterpreterMacroAssembler::lock_object(Register lock_reg) // Save the test result, for recursive case, the result is zero str(swap_reg, Address(lock_reg, mark_offset)); - br(Assembler::EQ, count); + br(Assembler::NE, slow_case); + + bind(count); + inc_held_monitor_count(rscratch1); + b(done); } bind(slow_case); // Call the runtime routine for slow case - call_VM(noreg, + call_VM_preemptable(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter), lock_reg); - b(done); - - bind(count); - increment(Address(rthread, JavaThread::held_monitor_count_offset())); bind(done); } @@ -804,11 +804,10 @@ void InterpreterMacroAssembler::unlock_object(Register lock_reg) // Free entry str(zr, Address(lock_reg, BasicObjectLock::obj_offset())); + Label slow_case; if (LockingMode == LM_LIGHTWEIGHT) { - Label slow_case; lightweight_unlock(obj_reg, header_reg, swap_reg, tmp_reg, slow_case); - b(count); - bind(slow_case); + b(done); } else if (LockingMode == LM_LEGACY) { // Load the old header from BasicLock structure ldr(header_reg, Address(swap_reg, @@ -818,16 +817,17 @@ void InterpreterMacroAssembler::unlock_object(Register lock_reg) cbz(header_reg, count); // Atomic swap back the old header - cmpxchg_obj_header(swap_reg, header_reg, obj_reg, rscratch1, count, /*fallthrough*/nullptr); + cmpxchg_obj_header(swap_reg, header_reg, obj_reg, rscratch1, count, &slow_case); + + bind(count); + dec_held_monitor_count(rscratch1); + b(done); } + + bind(slow_case); // Call the runtime routine for slow case. str(obj_reg, Address(lock_reg, BasicObjectLock::obj_offset())); // restore obj call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorexit), lock_reg); - b(done); - - bind(count); - decrement(Address(rthread, JavaThread::held_monitor_count_offset())); - bind(done); restore_bcp(); } @@ -1531,6 +1531,55 @@ void InterpreterMacroAssembler::call_VM_base(Register oop_result, restore_locals(); } +void InterpreterMacroAssembler::call_VM_preemptable(Register oop_result, + address entry_point, + Register arg_1) { + assert(arg_1 == c_rarg1, ""); + Label resume_pc, not_preempted; + +#ifdef ASSERT + { + Label L; + ldr(rscratch1, Address(rthread, JavaThread::preempt_alternate_return_offset())); + cbz(rscratch1, L); + stop("Should not have alternate return address set"); + bind(L); + } +#endif /* ASSERT */ + + // Force freeze slow path. + push_cont_fastpath(); + + // Make VM call. In case of preemption set last_pc to the one we want to resume to. + adr(rscratch1, resume_pc); + str(rscratch1, Address(rthread, JavaThread::last_Java_pc_offset())); + call_VM_base(oop_result, noreg, noreg, entry_point, 1, false /*check_exceptions*/); + + pop_cont_fastpath(); + + // Check if preempted. + ldr(rscratch1, Address(rthread, JavaThread::preempt_alternate_return_offset())); + cbz(rscratch1, not_preempted); + str(zr, Address(rthread, JavaThread::preempt_alternate_return_offset())); + br(rscratch1); + + // In case of preemption, this is where we will resume once we finally acquire the monitor. + bind(resume_pc); + restore_after_resume(false /* is_native */); + + bind(not_preempted); +} + +void InterpreterMacroAssembler::restore_after_resume(bool is_native) { + lea(rscratch1, ExternalAddress(Interpreter::cont_resume_interpreter_adapter())); + blr(rscratch1); + if (is_native) { + // On resume we need to set up stack as expected + push(dtos); + push(ltos); + } +} + void InterpreterMacroAssembler::profile_obj_type(Register obj, const Address& mdo_addr) { assert_different_registers(obj, rscratch1, mdo_addr.base(), mdo_addr.index()); Label update, next, none; diff --git a/src/hotspot/cpu/aarch64/interp_masm_aarch64.hpp b/src/hotspot/cpu/aarch64/interp_masm_aarch64.hpp index 34dca56eaf7..bc2070d283d 100644 --- a/src/hotspot/cpu/aarch64/interp_masm_aarch64.hpp +++ b/src/hotspot/cpu/aarch64/interp_masm_aarch64.hpp @@ -58,6 +58,11 @@ class InterpreterMacroAssembler: public MacroAssembler { void load_earlyret_value(TosState state); + void call_VM_preemptable(Register oop_result, + address entry_point, + Register arg_1); + void restore_after_resume(bool is_native); + void jump_to_entry(address entry); virtual void check_and_handle_popframe(Register java_thread); diff --git a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp index 9f35a6e75a7..ce71d3a2262 100644 --- a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp @@ -39,6 +39,7 @@ #include "gc/shared/tlab_globals.hpp" #include "interpreter/bytecodeHistogram.hpp" #include "interpreter/interpreter.hpp" +#include "interpreter/interpreterRuntime.hpp" #include "jvm.h" #include "memory/resourceArea.hpp" #include "memory/universe.hpp" @@ -775,6 +776,10 @@ static void pass_arg3(MacroAssembler* masm, Register arg) { } } +static bool is_preemptable(address entry_point) { + return entry_point == CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter); +} + void MacroAssembler::call_VM_base(Register oop_result, Register java_thread, Register last_java_sp, @@ -810,7 +815,12 @@ void MacroAssembler::call_VM_base(Register oop_result, assert(last_java_sp != rfp, "can't use rfp"); Label l; - set_last_Java_frame(last_java_sp, rfp, l, rscratch1); + if (is_preemptable(entry_point)) { + // skip setting last_pc since we already set it to desired value. + set_last_Java_frame(last_java_sp, rfp, noreg, rscratch1); + } else { + set_last_Java_frame(last_java_sp, rfp, l, rscratch1); + } // do the call, remove parameters MacroAssembler::call_VM_leaf_base(entry_point, number_of_arguments, &l); @@ -1002,10 +1012,11 @@ address MacroAssembler::ic_call(address entry, jint method_index) { } int MacroAssembler::ic_check_size() { + int extra_instructions = UseCompactObjectHeaders ? 1 : 0; if (target_needs_far_branch(CAST_FROM_FN_PTR(address, SharedRuntime::get_ic_miss_stub()))) { - return NativeInstruction::instruction_size * 7; + return NativeInstruction::instruction_size * (7 + extra_instructions); } else { - return NativeInstruction::instruction_size * 5; + return NativeInstruction::instruction_size * (5 + extra_instructions); } } @@ -1023,7 +1034,11 @@ int MacroAssembler::ic_check(int end_alignment) { int uep_offset = offset(); - if (UseCompressedClassPointers) { + if (UseCompactObjectHeaders) { + load_narrow_klass_compact(tmp1, receiver); + ldrw(tmp2, Address(data, CompiledICData::speculated_klass_offset())); + cmpw(tmp1, tmp2); + } else if (UseCompressedClassPointers) { ldrw(tmp1, Address(receiver, oopDesc::klass_offset_in_bytes())); ldrw(tmp2, Address(data, CompiledICData::speculated_klass_offset())); cmpw(tmp1, tmp2); @@ -5009,8 +5024,22 @@ void MacroAssembler::load_method_holder(Register holder, Register method) { ldr(holder, Address(holder, ConstantPool::pool_holder_offset())); // InstanceKlass* } +// Loads the obj's Klass* into dst. +// Preserves all registers (incl src, rscratch1 and rscratch2). +// Input: +// src - the oop we want to load the klass from. +// dst - output narrow klass. +void MacroAssembler::load_narrow_klass_compact(Register dst, Register src) { + assert(UseCompactObjectHeaders, "expects UseCompactObjectHeaders"); + ldr(dst, Address(src, oopDesc::mark_offset_in_bytes())); + lsr(dst, dst, markWord::klass_shift); +} + void MacroAssembler::load_klass(Register dst, Register src) { - if (UseCompressedClassPointers) { + if (UseCompactObjectHeaders) { + load_narrow_klass_compact(dst, src); + decode_klass_not_null(dst); + } else if (UseCompressedClassPointers) { ldrw(dst, Address(src, oopDesc::klass_offset_in_bytes())); decode_klass_not_null(dst); } else { @@ -5065,28 +5094,50 @@ void MacroAssembler::load_mirror(Register dst, Register method, Register tmp1, R resolve_oop_handle(dst, tmp1, tmp2); } -void MacroAssembler::cmp_klass(Register oop, Register trial_klass, Register tmp) { +void MacroAssembler::cmp_klass(Register obj, Register klass, Register tmp) { + assert_different_registers(obj, klass, tmp); if (UseCompressedClassPointers) { - ldrw(tmp, Address(oop, oopDesc::klass_offset_in_bytes())); + if (UseCompactObjectHeaders) { + load_narrow_klass_compact(tmp, obj); + } else { + ldrw(tmp, Address(obj, oopDesc::klass_offset_in_bytes())); + } if (CompressedKlassPointers::base() == nullptr) { - cmp(trial_klass, tmp, LSL, CompressedKlassPointers::shift()); + cmp(klass, tmp, LSL, CompressedKlassPointers::shift()); return; } else if (((uint64_t)CompressedKlassPointers::base() & 0xffffffff) == 0 && CompressedKlassPointers::shift() == 0) { // Only the bottom 32 bits matter - cmpw(trial_klass, tmp); + cmpw(klass, tmp); return; } decode_klass_not_null(tmp); } else { - ldr(tmp, Address(oop, oopDesc::klass_offset_in_bytes())); + ldr(tmp, Address(obj, oopDesc::klass_offset_in_bytes())); + } + cmp(klass, tmp); +} + +void MacroAssembler::cmp_klasses_from_objects(Register obj1, Register obj2, Register tmp1, Register tmp2) { + if (UseCompactObjectHeaders) { + load_narrow_klass_compact(tmp1, obj1); + load_narrow_klass_compact(tmp2, obj2); + cmpw(tmp1, tmp2); + } else if (UseCompressedClassPointers) { + ldrw(tmp1, Address(obj1, oopDesc::klass_offset_in_bytes())); + ldrw(tmp2, Address(obj2, oopDesc::klass_offset_in_bytes())); + cmpw(tmp1, tmp2); + } else { + ldr(tmp1, Address(obj1, oopDesc::klass_offset_in_bytes())); + ldr(tmp2, Address(obj2, oopDesc::klass_offset_in_bytes())); + cmp(tmp1, tmp2); } - cmp(trial_klass, tmp); } void MacroAssembler::store_klass(Register dst, Register src) { // FIXME: Should this be a store release? concurrent gcs assumes // klass length is valid if klass field is not null. + assert(!UseCompactObjectHeaders, "not with compact headers"); if (UseCompressedClassPointers) { encode_klass_not_null(src); strw(src, Address(dst, oopDesc::klass_offset_in_bytes())); @@ -5096,6 +5147,7 @@ void MacroAssembler::store_klass(Register dst, Register src) { } void MacroAssembler::store_klass_gap(Register dst, Register src) { + assert(!UseCompactObjectHeaders, "not with compact headers"); if (UseCompressedClassPointers) { // Store to klass gap in destination strw(src, Address(dst, oopDesc::klass_gap_offset_in_bytes())); @@ -5246,9 +5298,6 @@ MacroAssembler::KlassDecodeMode MacroAssembler::klass_decode_mode() { return _klass_decode_mode; } - assert(LogKlassAlignmentInBytes == CompressedKlassPointers::shift() - || 0 == CompressedKlassPointers::shift(), "decode alg wrong"); - if (CompressedKlassPointers::base() == nullptr) { return (_klass_decode_mode = KlassDecodeZero); } @@ -5274,7 +5323,7 @@ void MacroAssembler::encode_klass_not_null(Register dst, Register src) { switch (klass_decode_mode()) { case KlassDecodeZero: if (CompressedKlassPointers::shift() != 0) { - lsr(dst, src, LogKlassAlignmentInBytes); + lsr(dst, src, CompressedKlassPointers::shift()); } else { if (dst != src) mov(dst, src); } @@ -5283,7 +5332,7 @@ void MacroAssembler::encode_klass_not_null(Register dst, Register src) { case KlassDecodeXor: if (CompressedKlassPointers::shift() != 0) { eor(dst, src, (uint64_t)CompressedKlassPointers::base()); - lsr(dst, dst, LogKlassAlignmentInBytes); + lsr(dst, dst, CompressedKlassPointers::shift()); } else { eor(dst, src, (uint64_t)CompressedKlassPointers::base()); } @@ -5291,7 +5340,7 @@ void MacroAssembler::encode_klass_not_null(Register dst, Register src) { case KlassDecodeMovk: if (CompressedKlassPointers::shift() != 0) { - ubfx(dst, src, LogKlassAlignmentInBytes, 32); + ubfx(dst, src, CompressedKlassPointers::shift(), 32); } else { movw(dst, src); } @@ -5313,7 +5362,7 @@ void MacroAssembler::decode_klass_not_null(Register dst, Register src) { switch (klass_decode_mode()) { case KlassDecodeZero: if (CompressedKlassPointers::shift() != 0) { - lsl(dst, src, LogKlassAlignmentInBytes); + lsl(dst, src, CompressedKlassPointers::shift()); } else { if (dst != src) mov(dst, src); } @@ -5321,7 +5370,7 @@ void MacroAssembler::decode_klass_not_null(Register dst, Register src) { case KlassDecodeXor: if (CompressedKlassPointers::shift() != 0) { - lsl(dst, src, LogKlassAlignmentInBytes); + lsl(dst, src, CompressedKlassPointers::shift()); eor(dst, dst, (uint64_t)CompressedKlassPointers::base()); } else { eor(dst, src, (uint64_t)CompressedKlassPointers::base()); @@ -5336,7 +5385,7 @@ void MacroAssembler::decode_klass_not_null(Register dst, Register src) { movk(dst, shifted_base >> 32, 32); if (CompressedKlassPointers::shift() != 0) { - lsl(dst, dst, LogKlassAlignmentInBytes); + lsl(dst, dst, CompressedKlassPointers::shift()); } break; @@ -5497,6 +5546,38 @@ void MacroAssembler::tlab_allocate(Register obj, bs->tlab_allocate(this, obj, var_size_in_bytes, con_size_in_bytes, t1, t2, slow_case); } +void MacroAssembler::inc_held_monitor_count(Register tmp) { + Address dst(rthread, JavaThread::held_monitor_count_offset()); +#ifdef ASSERT + ldr(tmp, dst); + increment(tmp); + str(tmp, dst); + Label ok; + tbz(tmp, 63, ok); + STOP("assert(held monitor count underflow)"); + should_not_reach_here(); + bind(ok); +#else + increment(dst); +#endif +} + +void MacroAssembler::dec_held_monitor_count(Register tmp) { + Address dst(rthread, JavaThread::held_monitor_count_offset()); +#ifdef ASSERT + ldr(tmp, dst); + decrement(tmp); + str(tmp, dst); + Label ok; + tbz(tmp, 63, ok); + STOP("assert(held monitor count underflow)"); + should_not_reach_here(); + bind(ok); +#else + decrement(dst); +#endif +} + void MacroAssembler::verify_tlab() { #ifdef ASSERT if (UseTLAB && VerifyOops) { diff --git a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp index b23acc15718..d9686aec3a9 100644 --- a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp +++ b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp @@ -875,9 +875,11 @@ class MacroAssembler: public Assembler { void load_method_holder(Register holder, Register method); // oop manipulations + void load_narrow_klass_compact(Register dst, Register src); void load_klass(Register dst, Register src); void store_klass(Register dst, Register src); - void cmp_klass(Register oop, Register trial_klass, Register tmp); + void cmp_klass(Register obj, Register klass, Register tmp); + void cmp_klasses_from_objects(Register obj1, Register obj2, Register tmp1, Register tmp2); void resolve_weak_handle(Register result, Register tmp1, Register tmp2); void resolve_oop_handle(Register result, Register tmp1, Register tmp2); @@ -938,8 +940,11 @@ class MacroAssembler: public Assembler { void pop_CPU_state(bool restore_vectors = false, bool use_sve = false, int sve_vector_size_in_bytes = 0, int total_predicate_in_bytes = 0); - void push_cont_fastpath(Register java_thread); - void pop_cont_fastpath(Register java_thread); + void push_cont_fastpath(Register java_thread = rthread); + void pop_cont_fastpath(Register java_thread = rthread); + + void inc_held_monitor_count(Register tmp); + void dec_held_monitor_count(Register tmp); // Round up to a power of two void round_to(Register reg, int modulus); diff --git a/src/hotspot/cpu/aarch64/sharedRuntime_aarch64.cpp b/src/hotspot/cpu/aarch64/sharedRuntime_aarch64.cpp index fb8ae80cb49..94d73b6bc27 100644 --- a/src/hotspot/cpu/aarch64/sharedRuntime_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/sharedRuntime_aarch64.cpp @@ -1179,12 +1179,14 @@ static void gen_continuation_enter(MacroAssembler* masm, __ bind(call_thaw); + ContinuationEntry::_thaw_call_pc_offset = __ pc() - start; __ rt_call(CAST_FROM_FN_PTR(address, StubRoutines::cont_thaw())); oop_maps->add_gc_map(__ pc() - start, map->deep_copy()); ContinuationEntry::_return_pc_offset = __ pc() - start; __ post_call_nop(); __ bind(exit); + ContinuationEntry::_cleanup_offset = __ pc() - start; continuation_enter_cleanup(masm); __ leave(); __ ret(lr); @@ -1281,6 +1283,10 @@ static void gen_continuation_yield(MacroAssembler* masm, oop_maps->add_gc_map(the_pc - start, map); } +void SharedRuntime::continuation_enter_cleanup(MacroAssembler* masm) { + ::continuation_enter_cleanup(masm); +} + static void gen_special_dispatch(MacroAssembler* masm, const methodHandle& method, const BasicType* sig_bt, @@ -1747,11 +1753,20 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm, } // Change state to native (we save the return address in the thread, since it might not - // be pushed on the stack when we do a stack traversal). - // We use the same pc/oopMap repeatedly when we call out + // be pushed on the stack when we do a stack traversal). It is enough that the pc() + // points into the right code segment. It does not have to be the correct return pc. + // We use the same pc/oopMap repeatedly when we call out. Label native_return; - __ set_last_Java_frame(sp, noreg, native_return, rscratch1); + if (LockingMode != LM_LEGACY && method->is_object_wait0()) { + // For convenience we use the pc we want to resume to in case of preemption on Object.wait. + __ set_last_Java_frame(sp, noreg, native_return, rscratch1); + } else { + intptr_t the_pc = (intptr_t) __ pc(); + oop_maps->add_gc_map(the_pc - start, map); + + __ set_last_Java_frame(sp, noreg, __ pc(), rscratch1); + } Label dtrace_method_entry, dtrace_method_entry_done; if (DTraceMethodProbes) { @@ -1829,12 +1844,13 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm, // Save the test result, for recursive case, the result is zero __ str(swap_reg, Address(lock_reg, mark_word_offset)); __ br(Assembler::NE, slow_path_lock); + + __ bind(count); + __ inc_held_monitor_count(rscratch1); } else { assert(LockingMode == LM_LIGHTWEIGHT, "must be"); __ lightweight_lock(lock_reg, obj_reg, swap_reg, tmp, lock_tmp, slow_path_lock); } - __ bind(count); - __ increment(Address(rthread, JavaThread::held_monitor_count_offset())); // Slow path will re-enter here __ bind(lock_done); @@ -1853,11 +1869,6 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm, __ rt_call(native_func); - __ bind(native_return); - - intptr_t return_pc = (intptr_t) __ pc(); - oop_maps->add_gc_map(return_pc - start, map); - // Verify or restore cpu control state after JNI call __ restore_cpu_control_state_after_jni(rscratch1, rscratch2); @@ -1916,6 +1927,18 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm, __ stlrw(rscratch1, rscratch2); __ bind(after_transition); + if (LockingMode != LM_LEGACY && method->is_object_wait0()) { + // Check preemption for Object.wait() + __ ldr(rscratch1, Address(rthread, JavaThread::preempt_alternate_return_offset())); + __ cbz(rscratch1, native_return); + __ str(zr, Address(rthread, JavaThread::preempt_alternate_return_offset())); + __ br(rscratch1); + __ bind(native_return); + + intptr_t the_pc = (intptr_t) __ pc(); + oop_maps->add_gc_map(the_pc - start, map); + } + Label reguard; Label reguard_done; __ ldrb(rscratch1, Address(rthread, JavaThread::stack_guard_state_offset())); @@ -1939,7 +1962,7 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm, // Simple recursive lock? __ ldr(rscratch1, Address(sp, lock_slot_offset * VMRegImpl::stack_slot_size)); __ cbnz(rscratch1, not_recursive); - __ decrement(Address(rthread, JavaThread::held_monitor_count_offset())); + __ dec_held_monitor_count(rscratch1); __ b(done); } @@ -1962,11 +1985,10 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm, Label count; __ cmpxchg_obj_header(r0, old_hdr, obj_reg, rscratch1, count, &slow_path_unlock); __ bind(count); - __ decrement(Address(rthread, JavaThread::held_monitor_count_offset())); + __ dec_held_monitor_count(rscratch1); } else { assert(LockingMode == LM_LIGHTWEIGHT, ""); __ lightweight_unlock(obj_reg, old_hdr, swap_reg, lock_tmp, slow_path_unlock); - __ decrement(Address(rthread, JavaThread::held_monitor_count_offset())); } // slow path re-enters here @@ -2033,8 +2055,14 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm, __ mov(c_rarg1, lock_reg); __ mov(c_rarg2, rthread); - // Not a leaf but we have last_Java_frame setup as we want + // Not a leaf but we have last_Java_frame setup as we want. + // We don't want to unmount in case of contention since that would complicate preserving + // the arguments that had already been marshalled into the native convention. So we force + // the freeze slow path to find this native wrapper frame (see recurse_freeze_native_frame()) + // and pin the vthread. Otherwise the fast path won't find it since we don't walk the stack. + __ push_cont_fastpath(); __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::complete_monitor_locking_C), 3); + __ pop_cont_fastpath(); restore_args(masm, total_c_args, c_arg, out_regs); #ifdef ASSERT @@ -2575,6 +2603,10 @@ uint SharedRuntime::out_preserve_stack_slots() { } +VMReg SharedRuntime::thread_register() { + return rthread->as_VMReg(); +} + //------------------------------generate_handler_blob------ // // Generate a special Compile2Runtime blob that saves all registers, diff --git a/src/hotspot/cpu/aarch64/stackChunkFrameStream_aarch64.inline.hpp b/src/hotspot/cpu/aarch64/stackChunkFrameStream_aarch64.inline.hpp index 7c5cf63e382..8a221f13772 100644 --- a/src/hotspot/cpu/aarch64/stackChunkFrameStream_aarch64.inline.hpp +++ b/src/hotspot/cpu/aarch64/stackChunkFrameStream_aarch64.inline.hpp @@ -116,6 +116,7 @@ inline int StackChunkFrameStream::interpreter_frame_num_oops() const f.interpreted_frame_oop_map(&mask); return mask.num_oops() + 1 // for the mirror oop + + (f.interpreter_frame_method()->is_native() ? 1 : 0) // temp oop slot + pointer_delta_as_int((intptr_t*)f.interpreter_frame_monitor_begin(), (intptr_t*)f.interpreter_frame_monitor_end())/BasicObjectLock::size(); } diff --git a/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp b/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp index 26462eed7de..de5df5c1af1 100644 --- a/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp @@ -7466,6 +7466,37 @@ class StubGenerator: public StubCodeGenerator { return start; } + address generate_cont_preempt_stub() { + if (!Continuations::enabled()) return nullptr; + StubCodeMark mark(this, "StubRoutines","Continuation preempt stub"); + address start = __ pc(); + + __ reset_last_Java_frame(true); + + // Set sp to enterSpecial frame, i.e. remove all frames copied into the heap. + __ ldr(rscratch2, Address(rthread, JavaThread::cont_entry_offset())); + __ mov(sp, rscratch2); + + Label preemption_cancelled; + __ ldrb(rscratch1, Address(rthread, JavaThread::preemption_cancelled_offset())); + __ cbnz(rscratch1, preemption_cancelled); + + // Remove enterSpecial frame from the stack and return to Continuation.run() to unmount. + SharedRuntime::continuation_enter_cleanup(_masm); + __ leave(); + __ ret(lr); + + // We acquired the monitor after freezing the frames so call thaw to continue execution. + __ bind(preemption_cancelled); + __ strb(zr, Address(rthread, JavaThread::preemption_cancelled_offset())); + __ lea(rfp, Address(sp, checked_cast(ContinuationEntry::size()))); + __ lea(rscratch1, ExternalAddress(ContinuationEntry::thaw_call_pc_address())); + __ ldr(rscratch1, Address(rscratch1)); + __ br(rscratch1); + + return start; + } + // In sun.security.util.math.intpoly.IntegerPolynomial1305, integers // are represented as long[5], with BITS_PER_LIMB = 26. // Pack five 26-bit limbs into three 64-bit registers. @@ -8620,6 +8651,7 @@ class StubGenerator: public StubCodeGenerator { StubRoutines::_cont_thaw = generate_cont_thaw(); StubRoutines::_cont_returnBarrier = generate_cont_returnBarrier(); StubRoutines::_cont_returnBarrierExc = generate_cont_returnBarrier_exception(); + StubRoutines::_cont_preempt_stub = generate_cont_preempt_stub(); } void generate_final_stubs() { diff --git a/src/hotspot/cpu/aarch64/templateInterpreterGenerator_aarch64.cpp b/src/hotspot/cpu/aarch64/templateInterpreterGenerator_aarch64.cpp index 9894841e933..8eefcba7ba2 100644 --- a/src/hotspot/cpu/aarch64/templateInterpreterGenerator_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/templateInterpreterGenerator_aarch64.cpp @@ -607,6 +607,40 @@ address TemplateInterpreterGenerator::generate_safept_entry_for( return entry; } +address TemplateInterpreterGenerator::generate_cont_resume_interpreter_adapter() { + if (!Continuations::enabled()) return nullptr; + address start = __ pc(); + + __ restore_bcp(); + __ restore_locals(); + + // Restore constant pool cache + __ ldr(rcpool, Address(rfp, frame::interpreter_frame_cache_offset * wordSize)); + + // Restore Java expression stack pointer + __ ldr(rscratch1, Address(rfp, frame::interpreter_frame_last_sp_offset * wordSize)); + __ lea(esp, Address(rfp, rscratch1, Address::lsl(Interpreter::logStackElementSize))); + // and NULL it as marker that esp is now tos until next java call + __ str(zr, Address(rfp, frame::interpreter_frame_last_sp_offset * wordSize)); + + // Restore machine SP + __ ldr(rscratch1, Address(rfp, frame::interpreter_frame_extended_sp_offset * wordSize)); + __ lea(sp, Address(rfp, rscratch1, Address::lsl(LogBytesPerWord))); + + // Restore method + __ ldr(rmethod, Address(rfp, frame::interpreter_frame_method_offset * wordSize)); + + // Restore dispatch + uint64_t offset; + __ adrp(rdispatch, ExternalAddress((address)Interpreter::dispatch_table()), offset); + __ add(rdispatch, rdispatch, offset); + + __ ret(lr); + + return start; +} + + // Helpers for commoning out cases in the various type of method entries. // @@ -1314,6 +1348,9 @@ address TemplateInterpreterGenerator::generate_native_entry(bool synchronized) { // result handler is in r0 // set result handler __ mov(result_handler, r0); + // Save it in the frame in case of preemption; we cannot rely on callee saved registers. + __ str(r0, Address(rfp, frame::interpreter_frame_result_handler_offset * wordSize)); + // pass mirror handle if static call { Label L; @@ -1349,9 +1386,10 @@ address TemplateInterpreterGenerator::generate_native_entry(bool synchronized) { // pass JNIEnv __ add(c_rarg0, rthread, in_bytes(JavaThread::jni_environment_offset())); - // Set the last Java PC in the frame anchor to be the return address from - // the call to the native method: this will allow the debugger to - // generate an accurate stack trace. + // It is enough that the pc() points into the right code + // segment. It does not have to be the correct return pc. + // For convenience we use the pc we want to resume to in + // case of preemption on Object.wait. Label native_return; __ set_last_Java_frame(esp, rfp, native_return, rscratch1); @@ -1372,9 +1410,13 @@ address TemplateInterpreterGenerator::generate_native_entry(bool synchronized) { __ lea(rscratch2, Address(rthread, JavaThread::thread_state_offset())); __ stlrw(rscratch1, rscratch2); + __ push_cont_fastpath(); + // Call the native method. __ blr(r10); - __ bind(native_return); + + __ pop_cont_fastpath(); + __ get_method(rmethod); // result potentially in r0 or v0 @@ -1432,6 +1474,23 @@ address TemplateInterpreterGenerator::generate_native_entry(bool synchronized) { __ lea(rscratch2, Address(rthread, JavaThread::thread_state_offset())); __ stlrw(rscratch1, rscratch2); + if (LockingMode != LM_LEGACY) { + // Check preemption for Object.wait() + Label not_preempted; + __ ldr(rscratch1, Address(rthread, JavaThread::preempt_alternate_return_offset())); + __ cbz(rscratch1, not_preempted); + __ str(zr, Address(rthread, JavaThread::preempt_alternate_return_offset())); + __ br(rscratch1); + __ bind(native_return); + __ restore_after_resume(true /* is_native */); + // reload result_handler + __ ldr(result_handler, Address(rfp, frame::interpreter_frame_result_handler_offset*wordSize)); + __ bind(not_preempted); + } else { + // any pc will do so just use this one for LM_LEGACY to keep code together. + __ bind(native_return); + } + // reset_last_Java_frame __ reset_last_Java_frame(true); diff --git a/src/hotspot/cpu/aarch64/templateTable_aarch64.cpp b/src/hotspot/cpu/aarch64/templateTable_aarch64.cpp index 48ff356f9a5..60d4c3c5110 100644 --- a/src/hotspot/cpu/aarch64/templateTable_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/templateTable_aarch64.cpp @@ -3629,12 +3629,14 @@ void TemplateTable::_new() { // The object is initialized before the header. If the object size is // zero, go directly to the header initialization. - __ sub(r3, r3, sizeof(oopDesc)); + int header_size = oopDesc::header_size() * HeapWordSize; + assert(is_aligned(header_size, BytesPerLong), "oop header size must be 8-byte-aligned"); + __ sub(r3, r3, header_size); __ cbz(r3, initialize_header); // Initialize object fields { - __ add(r2, r0, sizeof(oopDesc)); + __ add(r2, r0, header_size); Label loop; __ bind(loop); __ str(zr, Address(__ post(r2, BytesPerLong))); @@ -3644,10 +3646,15 @@ void TemplateTable::_new() { // initialize object header only. __ bind(initialize_header); - __ mov(rscratch1, (intptr_t)markWord::prototype().value()); - __ str(rscratch1, Address(r0, oopDesc::mark_offset_in_bytes())); - __ store_klass_gap(r0, zr); // zero klass gap for compressed oops - __ store_klass(r0, r4); // store klass last + if (UseCompactObjectHeaders) { + __ ldr(rscratch1, Address(r4, Klass::prototype_header_offset())); + __ str(rscratch1, Address(r0, oopDesc::mark_offset_in_bytes())); + } else { + __ mov(rscratch1, (intptr_t)markWord::prototype().value()); + __ str(rscratch1, Address(r0, oopDesc::mark_offset_in_bytes())); + __ store_klass_gap(r0, zr); // zero klass gap for compressed oops + __ store_klass(r0, r4); // store klass last + } if (DTraceAllocProbes) { // Trigger dtrace event for fastpath diff --git a/src/hotspot/cpu/arm/c1_Runtime1_arm.cpp b/src/hotspot/cpu/arm/c1_Runtime1_arm.cpp index b5117dedc42..bae882ea93d 100644 --- a/src/hotspot/cpu/arm/c1_Runtime1_arm.cpp +++ b/src/hotspot/cpu/arm/c1_Runtime1_arm.cpp @@ -251,6 +251,10 @@ void StubAssembler::restore_live_registers_without_return() { void Runtime1::initialize_pd() { } +uint Runtime1::runtime_blob_current_thread_offset(frame f) { + Unimplemented(); + return 0; +} OopMapSet* Runtime1::generate_exception_throw(StubAssembler* sasm, address target, bool has_argument) { OopMap* oop_map = save_live_registers(sasm); diff --git a/src/hotspot/cpu/arm/continuationFreezeThaw_arm.inline.hpp b/src/hotspot/cpu/arm/continuationFreezeThaw_arm.inline.hpp index 29af5681a67..fb588318170 100644 --- a/src/hotspot/cpu/arm/continuationFreezeThaw_arm.inline.hpp +++ b/src/hotspot/cpu/arm/continuationFreezeThaw_arm.inline.hpp @@ -48,6 +48,10 @@ void FreezeBase::adjust_interpreted_frame_unextended_sp(frame& f) { Unimplemented(); } +inline void FreezeBase::prepare_freeze_interpreted_top_frame(frame& f) { + Unimplemented(); +} + inline void FreezeBase::relativize_interpreted_frame_metadata(const frame& f, const frame& hf) { Unimplemented(); } @@ -83,6 +87,15 @@ inline void ThawBase::patch_pd(frame& f, const frame& caller) { Unimplemented(); } +inline void ThawBase::patch_pd(frame& f, intptr_t* caller_sp) { + Unimplemented(); +} + +inline intptr_t* ThawBase::push_cleanup_continuation() { + Unimplemented(); + return nullptr; +} + template inline void Thaw::patch_caller_links(intptr_t* sp, intptr_t* bottom) { Unimplemented(); diff --git a/src/hotspot/cpu/arm/continuationHelper_arm.inline.hpp b/src/hotspot/cpu/arm/continuationHelper_arm.inline.hpp index 2f96ccaf322..445a013b3e5 100644 --- a/src/hotspot/cpu/arm/continuationHelper_arm.inline.hpp +++ b/src/hotspot/cpu/arm/continuationHelper_arm.inline.hpp @@ -35,6 +35,10 @@ static inline intptr_t** link_address(const frame& f) { return nullptr; } +static inline void patch_return_pc_with_preempt_stub(frame& f) { + Unimplemented(); +} + inline int ContinuationHelper::frame_align_words(int size) { Unimplemented(); return 0; @@ -62,11 +66,11 @@ inline void ContinuationHelper::set_anchor_to_entry_pd(JavaFrameAnchor* anchor, Unimplemented(); } -#ifdef ASSERT inline void ContinuationHelper::set_anchor_pd(JavaFrameAnchor* anchor, intptr_t* sp) { Unimplemented(); } +#ifdef ASSERT inline bool ContinuationHelper::Frame::assert_frame_laid_out(frame f) { Unimplemented(); return false; diff --git a/src/hotspot/cpu/arm/frame_arm.cpp b/src/hotspot/cpu/arm/frame_arm.cpp index 6221191e926..13a5c471c6f 100644 --- a/src/hotspot/cpu/arm/frame_arm.cpp +++ b/src/hotspot/cpu/arm/frame_arm.cpp @@ -325,6 +325,11 @@ bool frame::upcall_stub_frame_is_first() const { return false; } +JavaThread** frame::saved_thread_address(const frame& f) { + Unimplemented(); + return nullptr; +} + //------------------------------------------------------------------------------ // frame::verify_deopt_original_pc // diff --git a/src/hotspot/cpu/arm/sharedRuntime_arm.cpp b/src/hotspot/cpu/arm/sharedRuntime_arm.cpp index 7c1f3aafe7d..a0cc2f44199 100644 --- a/src/hotspot/cpu/arm/sharedRuntime_arm.cpp +++ b/src/hotspot/cpu/arm/sharedRuntime_arm.cpp @@ -1358,6 +1358,11 @@ uint SharedRuntime::out_preserve_stack_slots() { return 0; } +VMReg SharedRuntime::thread_register() { + Unimplemented(); + return nullptr; +} + //------------------------------generate_deopt_blob---------------------------- void SharedRuntime::generate_deopt_blob() { ResourceMark rm; diff --git a/src/hotspot/cpu/arm/templateInterpreterGenerator_arm.cpp b/src/hotspot/cpu/arm/templateInterpreterGenerator_arm.cpp index ec9d237e50d..61f6026dbe3 100644 --- a/src/hotspot/cpu/arm/templateInterpreterGenerator_arm.cpp +++ b/src/hotspot/cpu/arm/templateInterpreterGenerator_arm.cpp @@ -459,6 +459,10 @@ address TemplateInterpreterGenerator::generate_safept_entry_for(TosState state, return entry; } +address TemplateInterpreterGenerator::generate_cont_resume_interpreter_adapter() { + return nullptr; +} + // Helpers for commoning out cases in the various type of method entries. // diff --git a/src/hotspot/cpu/ppc/assembler_ppc.hpp b/src/hotspot/cpu/ppc/assembler_ppc.hpp index b2711ac43b0..d2584cc1af3 100644 --- a/src/hotspot/cpu/ppc/assembler_ppc.hpp +++ b/src/hotspot/cpu/ppc/assembler_ppc.hpp @@ -1816,6 +1816,7 @@ class Assembler : public AbstractAssembler { relocInfo::relocType rt = relocInfo::none); // helper function for b, bcxx + inline bool is_branch(address a); inline bool is_within_range_of_b(address a, address pc); inline bool is_within_range_of_bcxx(address a, address pc); diff --git a/src/hotspot/cpu/ppc/assembler_ppc.inline.hpp b/src/hotspot/cpu/ppc/assembler_ppc.inline.hpp index b0eaaccf0d0..07de57babfa 100644 --- a/src/hotspot/cpu/ppc/assembler_ppc.inline.hpp +++ b/src/hotspot/cpu/ppc/assembler_ppc.inline.hpp @@ -484,6 +484,12 @@ inline void Assembler::bclrl( int boint, int biint, int bhint, relocInfo::relocT inline void Assembler::bcctr( int boint, int biint, int bhint, relocInfo::relocType rt) { emit_data(BCCTR_OPCODE| bo(boint) | bi(biint) | bh(bhint) | aa(0) | lk(0), rt); } inline void Assembler::bcctrl(int boint, int biint, int bhint, relocInfo::relocType rt) { emit_data(BCCTR_OPCODE| bo(boint) | bi(biint) | bh(bhint) | aa(0) | lk(1), rt); } +inline bool Assembler::is_branch(address a) { + int32_t instr = *(int32_t*) a; + int op = inv_op_ppc(instr); + return op == b_op || op == bc_op; +} + // helper function for b inline bool Assembler::is_within_range_of_b(address a, address pc) { // Guard against illegal branch targets, e.g. -1 (see CompiledDirectCall and ad-file). diff --git a/src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.cpp b/src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.cpp index 36e1ac82334..b8a271ffc3b 100644 --- a/src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.cpp +++ b/src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.cpp @@ -1996,16 +1996,7 @@ void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) { // We don't know the array types are compatible. if (basic_type != T_OBJECT) { // Simple test for basic type arrays. - if (UseCompressedClassPointers) { - // We don't need decode because we just need to compare. - __ lwz(tmp, oopDesc::klass_offset_in_bytes(), src); - __ lwz(tmp2, oopDesc::klass_offset_in_bytes(), dst); - __ cmpw(CCR0, tmp, tmp2); - } else { - __ ld(tmp, oopDesc::klass_offset_in_bytes(), src); - __ ld(tmp2, oopDesc::klass_offset_in_bytes(), dst); - __ cmpd(CCR0, tmp, tmp2); - } + __ cmp_klasses_from_objects(CCR0, src, dst, tmp, tmp2); __ beq(CCR0, cont); } else { // For object arrays, if src is a sub class of dst then we can @@ -2128,39 +2119,15 @@ void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) { // but not necessarily exactly of type default_type. Label known_ok, halt; metadata2reg(default_type->constant_encoding(), tmp); - if (UseCompressedClassPointers) { - // Tmp holds the default type. It currently comes uncompressed after the - // load of a constant, so encode it. - __ encode_klass_not_null(tmp); - // Load the raw value of the dst klass, since we will be comparing - // uncompressed values directly. - __ lwz(tmp2, oopDesc::klass_offset_in_bytes(), dst); - __ cmpw(CCR0, tmp, tmp2); - if (basic_type != T_OBJECT) { - __ bne(CCR0, halt); - // Load the raw value of the src klass. - __ lwz(tmp2, oopDesc::klass_offset_in_bytes(), src); - __ cmpw(CCR0, tmp, tmp2); - __ beq(CCR0, known_ok); - } else { - __ beq(CCR0, known_ok); - __ cmpw(CCR0, src, dst); - __ beq(CCR0, known_ok); - } + __ cmp_klass(CCR0, dst, tmp, R11_scratch1, R12_scratch2); + if (basic_type != T_OBJECT) { + __ bne(CCR0, halt); + __ cmp_klass(CCR0, src, tmp, R11_scratch1, R12_scratch2); + __ beq(CCR0, known_ok); } else { - __ ld(tmp2, oopDesc::klass_offset_in_bytes(), dst); - __ cmpd(CCR0, tmp, tmp2); - if (basic_type != T_OBJECT) { - __ bne(CCR0, halt); - // Load the raw value of the src klass. - __ ld(tmp2, oopDesc::klass_offset_in_bytes(), src); - __ cmpd(CCR0, tmp, tmp2); - __ beq(CCR0, known_ok); - } else { - __ beq(CCR0, known_ok); - __ cmpd(CCR0, src, dst); - __ beq(CCR0, known_ok); - } + __ beq(CCR0, known_ok); + __ cmpw(CCR0, src, dst); + __ beq(CCR0, known_ok); } __ bind(halt); __ stop("incorrect type information in arraycopy"); @@ -2738,12 +2705,7 @@ void LIR_Assembler::emit_load_klass(LIR_OpLoadKlass* op) { } } - if (UseCompressedClassPointers) { - __ lwz(result, oopDesc::klass_offset_in_bytes(), obj); - __ decode_klass_not_null(result); - } else { - __ ld(result, oopDesc::klass_offset_in_bytes(), obj); - } + __ load_klass(result, obj); } void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) { diff --git a/src/hotspot/cpu/ppc/c1_MacroAssembler_ppc.cpp b/src/hotspot/cpu/ppc/c1_MacroAssembler_ppc.cpp index ea4d76e200f..8cd21478d41 100644 --- a/src/hotspot/cpu/ppc/c1_MacroAssembler_ppc.cpp +++ b/src/hotspot/cpu/ppc/c1_MacroAssembler_ppc.cpp @@ -133,7 +133,9 @@ void C1_MacroAssembler::lock_object(Register Rmark, Register Roop, Register Rbox } bind(done); - inc_held_monitor_count(Rmark /*tmp*/); + if (LockingMode == LM_LEGACY) { + inc_held_monitor_count(Rmark /*tmp*/); + } } @@ -179,7 +181,9 @@ void C1_MacroAssembler::unlock_object(Register Rmark, Register Roop, Register Rb // Done bind(done); - dec_held_monitor_count(Rmark /*tmp*/); + if (LockingMode == LM_LEGACY) { + dec_held_monitor_count(Rmark /*tmp*/); + } } @@ -201,12 +205,19 @@ void C1_MacroAssembler::try_allocate( void C1_MacroAssembler::initialize_header(Register obj, Register klass, Register len, Register t1, Register t2) { assert_different_registers(obj, klass, len, t1, t2); - load_const_optimized(t1, (intx)markWord::prototype().value()); - std(t1, oopDesc::mark_offset_in_bytes(), obj); - store_klass(obj, klass); + + if (UseCompactObjectHeaders) { + ld(t1, in_bytes(Klass::prototype_header_offset()), klass); + std(t1, oopDesc::mark_offset_in_bytes(), obj); + } else { + load_const_optimized(t1, (intx)markWord::prototype().value()); + std(t1, oopDesc::mark_offset_in_bytes(), obj); + store_klass(obj, klass); + } + if (len->is_valid()) { stw(len, arrayOopDesc::length_offset_in_bytes(), obj); - } else if (UseCompressedClassPointers) { + } else if (UseCompressedClassPointers && !UseCompactObjectHeaders) { // Otherwise length is in the class gap. store_klass_gap(obj); } diff --git a/src/hotspot/cpu/ppc/c1_Runtime1_ppc.cpp b/src/hotspot/cpu/ppc/c1_Runtime1_ppc.cpp index 654626d66d8..47cafc45f33 100644 --- a/src/hotspot/cpu/ppc/c1_Runtime1_ppc.cpp +++ b/src/hotspot/cpu/ppc/c1_Runtime1_ppc.cpp @@ -64,7 +64,8 @@ int StubAssembler::call_RT(Register oop_result1, Register metadata_result, address return_pc = call_c(entry_point); - reset_last_Java_frame(); + // Last java sp can be null when the RT call was preempted + reset_last_Java_frame(false /* check_last_java_sp */); // Check for pending exceptions. { @@ -257,6 +258,11 @@ void Runtime1::initialize_pd() { frame_size_in_bytes = align_up(sp_offset, frame::alignment_in_bytes); } +uint Runtime1::runtime_blob_current_thread_offset(frame f) { + // On PPC virtual threads don't save the JavaThread* in their context (e.g. C1 stub frames). + ShouldNotCallThis(); + return 0; +} OopMapSet* Runtime1::generate_exception_throw(StubAssembler* sasm, address target, bool has_argument) { // Make a frame and preserve the caller's caller-save registers. diff --git a/src/hotspot/cpu/ppc/c2_MacroAssembler_ppc.cpp b/src/hotspot/cpu/ppc/c2_MacroAssembler_ppc.cpp index 1147c3b42b2..82d9a046bc6 100644 --- a/src/hotspot/cpu/ppc/c2_MacroAssembler_ppc.cpp +++ b/src/hotspot/cpu/ppc/c2_MacroAssembler_ppc.cpp @@ -47,6 +47,15 @@ void C2_MacroAssembler::fast_unlock_lightweight(ConditionRegister flag, Register compiler_fast_unlock_lightweight_object(flag, obj, box, tmp1, tmp2, tmp3); } +void C2_MacroAssembler::load_narrow_klass_compact_c2(Register dst, Register obj, int disp) { + // Note: Don't clobber obj anywhere in that method! + + // The incoming address is pointing into obj-start + klass_offset_in_bytes. We need to extract + // obj-start, so that we can load from the object's mark-word instead. + ld(dst, disp - oopDesc::klass_offset_in_bytes(), obj); + srdi(dst, dst, markWord::klass_shift); +} + // Intrinsics for CompactStrings // Compress char[] to byte[] by compressing 16 bytes at once. diff --git a/src/hotspot/cpu/ppc/c2_MacroAssembler_ppc.hpp b/src/hotspot/cpu/ppc/c2_MacroAssembler_ppc.hpp index 5096810ef91..48a362aa63c 100644 --- a/src/hotspot/cpu/ppc/c2_MacroAssembler_ppc.hpp +++ b/src/hotspot/cpu/ppc/c2_MacroAssembler_ppc.hpp @@ -34,6 +34,8 @@ void fast_unlock_lightweight(ConditionRegister flag, Register obj, Register box, Register tmp1, Register tmp2, Register tmp3); + void load_narrow_klass_compact_c2(Register dst, Register obj, int disp); + // Intrinsics for CompactStrings // Compress char[] to byte[] by compressing 16 bytes at once. void string_compress_16(Register src, Register dst, Register cnt, diff --git a/src/hotspot/cpu/ppc/c2_init_ppc.cpp b/src/hotspot/cpu/ppc/c2_init_ppc.cpp index d15f880e413..e517cdf7de1 100644 --- a/src/hotspot/cpu/ppc/c2_init_ppc.cpp +++ b/src/hotspot/cpu/ppc/c2_init_ppc.cpp @@ -34,21 +34,5 @@ // Processor dependent initialization of C2 compiler for ppc. void Compile::pd_compiler2_init() { - - // Power7 and later. - if (PowerArchitecturePPC64 > 6) { - if (FLAG_IS_DEFAULT(UsePopCountInstruction)) { - FLAG_SET_ERGO(UsePopCountInstruction, true); - } - } - - if (!VM_Version::has_isel() && FLAG_IS_DEFAULT(ConditionalMoveLimit)) { - FLAG_SET_ERGO(ConditionalMoveLimit, 0); - } - - if (OptimizeFill) { - warning("OptimizeFill is not supported on this CPU."); - FLAG_SET_DEFAULT(OptimizeFill, false); - } - + guarantee(CodeEntryAlignment >= InteriorEntryAlignment, ""); } diff --git a/src/hotspot/cpu/ppc/continuationFreezeThaw_ppc.inline.hpp b/src/hotspot/cpu/ppc/continuationFreezeThaw_ppc.inline.hpp index c1a98b734da..e49bc548ad4 100644 --- a/src/hotspot/cpu/ppc/continuationFreezeThaw_ppc.inline.hpp +++ b/src/hotspot/cpu/ppc/continuationFreezeThaw_ppc.inline.hpp @@ -71,6 +71,12 @@ void FreezeBase::adjust_interpreted_frame_unextended_sp(frame& f) { // nothing to do } +inline void FreezeBase::prepare_freeze_interpreted_top_frame(frame& f) { + // nothing to do + DEBUG_ONLY( intptr_t* lspp = (intptr_t*) &(f.get_ijava_state()->top_frame_sp); ) + assert(*lspp == f.unextended_sp() - f.fp(), "should be " INTPTR_FORMAT " usp:" INTPTR_FORMAT " fp:" INTPTR_FORMAT, *lspp, p2i(f.unextended_sp()), p2i(f.fp())); +} + inline void FreezeBase::relativize_interpreted_frame_metadata(const frame& f, const frame& hf) { intptr_t* vfp = f.fp(); intptr_t* hfp = hf.fp(); @@ -350,6 +356,7 @@ inline void Thaw::patch_caller_links(intptr_t* sp, intptr_t* bottom) { if (is_entry_frame) { callers_sp = _cont.entryFP(); } else { + assert(!Interpreter::contains(pc), "sp:" PTR_FORMAT " pc:" PTR_FORMAT, p2i(sp), p2i(pc)); CodeBlob* cb = CodeCache::find_blob_fast(pc); callers_sp = sp + cb->frame_size(); } @@ -480,8 +487,8 @@ inline frame ThawBase::new_entry_frame() { template frame ThawBase::new_stack_frame(const frame& hf, frame& caller, bool bottom) { assert(FKind::is_instance(hf), ""); - assert(is_aligned(caller.fp(), frame::frame_alignment), ""); - assert(is_aligned(caller.sp(), frame::frame_alignment), ""); + assert(is_aligned(caller.fp(), frame::frame_alignment), PTR_FORMAT, p2i(caller.fp())); + // caller.sp() can be unaligned. This is fixed below. if (FKind::interpreted) { // Note: we have to overlap with the caller, at least if it is interpreted, to match the // max_thawing_size calculation during freeze. See also comment above. @@ -510,7 +517,7 @@ template frame ThawBase::new_stack_frame(const frame& hf, frame& return f; } else { int fsize = FKind::size(hf); - int argsize = hf.compiled_frame_stack_argsize(); + int argsize = FKind::stack_argsize(hf); intptr_t* frame_sp = caller.sp() - fsize; if ((bottom && argsize > 0) || caller.is_interpreted_frame()) { @@ -543,12 +550,29 @@ inline void ThawBase::derelativize_interpreted_frame_metadata(const frame& hf, c // Keep top_frame_sp relativized. } +inline intptr_t* ThawBase::push_cleanup_continuation() { + frame enterSpecial = new_entry_frame(); + frame::common_abi* enterSpecial_abi = (frame::common_abi*)enterSpecial.sp(); + + enterSpecial_abi->lr = (intptr_t)ContinuationEntry::cleanup_pc(); + + log_develop_trace(continuations, preempt)("push_cleanup_continuation enterSpecial sp: " INTPTR_FORMAT " cleanup pc: " INTPTR_FORMAT, + p2i(enterSpecial_abi), + p2i(ContinuationEntry::cleanup_pc())); + + return enterSpecial.sp(); +} + inline void ThawBase::patch_pd(frame& f, const frame& caller) { patch_callee_link(caller, caller.fp()); // Prevent assertion if f gets deoptimized right away before it's fully initialized f.mark_not_fully_initialized(); } +inline void ThawBase::patch_pd(frame& f, intptr_t* caller_sp) { + assert(f.own_abi()->callers_sp == (uint64_t)caller_sp, "should have been fixed by patch_caller_links"); +} + // // Interpreter Calling Procedure on PPC // diff --git a/src/hotspot/cpu/ppc/continuationHelper_ppc.inline.hpp b/src/hotspot/cpu/ppc/continuationHelper_ppc.inline.hpp index 528f1f8ad96..ae5a02451e2 100644 --- a/src/hotspot/cpu/ppc/continuationHelper_ppc.inline.hpp +++ b/src/hotspot/cpu/ppc/continuationHelper_ppc.inline.hpp @@ -27,10 +27,18 @@ #include "runtime/continuationHelper.hpp" -template -static inline intptr_t** link_address(const frame& f) { - Unimplemented(); - return nullptr; +static inline void patch_return_pc_with_preempt_stub(frame& f) { + if (f.is_runtime_frame()) { + // Patch the pc of the now old last Java frame (we already set the anchor to enterSpecial) + // so that when target goes back to Java it will actually return to the preempt cleanup stub. + frame::common_abi* abi = (frame::common_abi*)f.sp(); + abi->lr = (uint64_t)StubRoutines::cont_preempt_stub(); + } else { + // The target will check for preemption once it returns to the interpreter + // or the native wrapper code and will manually jump to the preempt stub. + JavaThread *thread = JavaThread::current(); + thread->set_preempt_alternate_return(StubRoutines::cont_preempt_stub()); + } } inline int ContinuationHelper::frame_align_words(int size) { @@ -59,11 +67,11 @@ inline void ContinuationHelper::set_anchor_to_entry_pd(JavaFrameAnchor* anchor, // nothing to do } -#ifdef ASSERT inline void ContinuationHelper::set_anchor_pd(JavaFrameAnchor* anchor, intptr_t* sp) { // nothing to do } +#ifdef ASSERT inline bool ContinuationHelper::Frame::assert_frame_laid_out(frame f) { intptr_t* sp = f.sp(); address pc = *(address*)(sp - frame::sender_sp_ret_address_offset()); diff --git a/src/hotspot/cpu/ppc/frame_ppc.cpp b/src/hotspot/cpu/ppc/frame_ppc.cpp index eb16af5e9db..935def11e1a 100644 --- a/src/hotspot/cpu/ppc/frame_ppc.cpp +++ b/src/hotspot/cpu/ppc/frame_ppc.cpp @@ -243,6 +243,11 @@ frame frame::sender_for_upcall_stub_frame(RegisterMap* map) const { return fr; } +JavaThread** frame::saved_thread_address(const frame& f) { + // The current thread (JavaThread*) is never stored on the stack + return nullptr; +} + frame frame::sender_for_interpreter_frame(RegisterMap *map) const { // This is the sp before any possible extension (adapter/locals). intptr_t* unextended_sp = interpreter_frame_sender_sp(); diff --git a/src/hotspot/cpu/ppc/frame_ppc.inline.hpp b/src/hotspot/cpu/ppc/frame_ppc.inline.hpp index 2beb7a84d34..c390449dc33 100644 --- a/src/hotspot/cpu/ppc/frame_ppc.inline.hpp +++ b/src/hotspot/cpu/ppc/frame_ppc.inline.hpp @@ -53,7 +53,10 @@ inline void frame::setup(kind knd) { // The back link for compiled frames on the heap is not valid if (is_heap_frame()) { // fp for interpreted frames should have been derelativized and passed to the constructor - assert(is_compiled_frame(), ""); + assert(is_compiled_frame() + || is_native_frame() // native wrapper (nmethod) for j.l.Object::wait0 + || is_runtime_frame(), // e.g. Runtime1::monitorenter, SharedRuntime::complete_monitor_locking_C + "sp:" PTR_FORMAT " fp:" PTR_FORMAT " name:%s", p2i(_sp), p2i(_unextended_sp + _cb->frame_size()), _cb->name()); // The back link for compiled frames on the heap is invalid. _fp = _unextended_sp + _cb->frame_size(); } else { diff --git a/src/hotspot/cpu/ppc/interp_masm_ppc.hpp b/src/hotspot/cpu/ppc/interp_masm_ppc.hpp index dd0a597ec8d..4c39f88ce51 100644 --- a/src/hotspot/cpu/ppc/interp_masm_ppc.hpp +++ b/src/hotspot/cpu/ppc/interp_masm_ppc.hpp @@ -49,6 +49,14 @@ class InterpreterMacroAssembler: public MacroAssembler { virtual void check_and_handle_popframe(Register scratch_reg); virtual void check_and_handle_earlyret(Register scratch_reg); + void call_VM_preemptable(Register oop_result, address entry_point, Register arg_1, bool check_exceptions = true); + void restore_after_resume(Register fp); + // R22 and R31 are preserved when a vthread gets preempted in the interpreter. + // The interpreter already assumes that these registers are nonvolatile across native calls. + bool nonvolatile_accross_vthread_preemtion(Register r) const { + return r->is_nonvolatile() && ((r == R22) || (r == R31)); + } + // Base routine for all dispatches. void dispatch_base(TosState state, address* table); @@ -182,7 +190,7 @@ class InterpreterMacroAssembler: public MacroAssembler { // Special call VM versions that check for exceptions and forward exception // via short cut (not via expensive forward exception stub). void check_and_forward_exception(Register Rscratch1, Register Rscratch2); - void call_VM(Register oop_result, address entry_point, bool check_exceptions = true); + void call_VM(Register oop_result, address entry_point, bool check_exceptions = true, Label* last_java_pc = nullptr); void call_VM(Register oop_result, address entry_point, Register arg_1, bool check_exceptions = true); void call_VM(Register oop_result, address entry_point, Register arg_1, Register arg_2, bool check_exceptions = true); void call_VM(Register oop_result, address entry_point, Register arg_1, Register arg_2, Register arg_3, bool check_exceptions = true); diff --git a/src/hotspot/cpu/ppc/interp_masm_ppc_64.cpp b/src/hotspot/cpu/ppc/interp_masm_ppc_64.cpp index aa77f0169ea..632eb97e852 100644 --- a/src/hotspot/cpu/ppc/interp_masm_ppc_64.cpp +++ b/src/hotspot/cpu/ppc/interp_masm_ppc_64.cpp @@ -932,7 +932,7 @@ void InterpreterMacroAssembler::remove_activation(TosState state, // void InterpreterMacroAssembler::lock_object(Register monitor, Register object) { if (LockingMode == LM_MONITOR) { - call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter), monitor); + call_VM_preemptable(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter), monitor); } else { // template code (for LM_LEGACY): // @@ -953,8 +953,7 @@ void InterpreterMacroAssembler::lock_object(Register monitor, Register object) { const Register current_header = R9_ARG7; const Register tmp = R10_ARG8; - Label count_locking, done; - Label cas_failed, slow_case; + Label count_locking, done, slow_case, cas_failed; assert_different_registers(header, object_mark_addr, current_header, tmp); @@ -969,7 +968,7 @@ void InterpreterMacroAssembler::lock_object(Register monitor, Register object) { if (LockingMode == LM_LIGHTWEIGHT) { lightweight_lock(monitor, object, header, tmp, slow_case); - b(count_locking); + b(done); } else if (LockingMode == LM_LEGACY) { // Load markWord from object into header. ld(header, oopDesc::mark_offset_in_bytes(), object); @@ -1035,12 +1034,15 @@ void InterpreterMacroAssembler::lock_object(Register monitor, Register object) { // None of the above fast optimizations worked so we have to get into the // slow case of monitor enter. bind(slow_case); - call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter), monitor); - b(done); + call_VM_preemptable(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter), monitor); // } - align(32, 12); - bind(count_locking); - inc_held_monitor_count(current_header /*tmp*/); + + if (LockingMode == LM_LEGACY) { + b(done); + align(32, 12); + bind(count_locking); + inc_held_monitor_count(current_header /*tmp*/); + } bind(done); } } @@ -1137,7 +1139,9 @@ void InterpreterMacroAssembler::unlock_object(Register monitor) { bind(free_slot); li(R0, 0); std(R0, in_bytes(BasicObjectLock::obj_offset()), monitor); - dec_held_monitor_count(current_header /*tmp*/); + if (LockingMode == LM_LEGACY) { + dec_held_monitor_count(current_header /*tmp*/); + } bind(done); } } @@ -2133,10 +2137,10 @@ void InterpreterMacroAssembler::check_and_forward_exception(Register Rscratch1, bind(Ldone); } -void InterpreterMacroAssembler::call_VM(Register oop_result, address entry_point, bool check_exceptions) { +void InterpreterMacroAssembler::call_VM(Register oop_result, address entry_point, bool check_exceptions, Label* last_java_pc) { save_interpreter_state(R11_scratch1); - MacroAssembler::call_VM(oop_result, entry_point, false); + MacroAssembler::call_VM(oop_result, entry_point, false /*check_exceptions*/, last_java_pc); restore_interpreter_state(R11_scratch1, /*bcp_and_mdx_only*/ true); @@ -2155,6 +2159,74 @@ void InterpreterMacroAssembler::call_VM(Register oop_result, address entry_point call_VM(oop_result, entry_point, check_exceptions); } +void InterpreterMacroAssembler::call_VM_preemptable(Register oop_result, address entry_point, + Register arg_1, bool check_exceptions) { + if (!Continuations::enabled()) { + call_VM(oop_result, entry_point, arg_1, check_exceptions); + return; + } + + Label resume_pc, not_preempted; + + DEBUG_ONLY(ld(R0, in_bytes(JavaThread::preempt_alternate_return_offset()), R16_thread)); + DEBUG_ONLY(cmpdi(CCR0, R0, 0)); + asm_assert_eq("Should not have alternate return address set"); + + // Preserve 2 registers + assert(nonvolatile_accross_vthread_preemtion(R31) && nonvolatile_accross_vthread_preemtion(R22), ""); + ld(R3_ARG1, _abi0(callers_sp), R1_SP); // load FP + std(R31, _ijava_state_neg(lresult), R3_ARG1); + std(R22, _ijava_state_neg(fresult), R3_ARG1); + + // We set resume_pc as last java pc. It will be saved if the vthread gets preempted. + // Later execution will continue right there. + mr_if_needed(R4_ARG2, arg_1); + push_cont_fastpath(); + call_VM(oop_result, entry_point, false /*check_exceptions*/, &resume_pc /* last_java_pc */); + pop_cont_fastpath(); + + // Jump to handler if the call was preempted + ld(R0, in_bytes(JavaThread::preempt_alternate_return_offset()), R16_thread); + cmpdi(CCR0, R0, 0); + beq(CCR0, not_preempted); + mtlr(R0); + li(R0, 0); + std(R0, in_bytes(JavaThread::preempt_alternate_return_offset()), R16_thread); + blr(); + + bind(resume_pc); // Location to resume execution + restore_after_resume(noreg /* fp */); + bind(not_preempted); +} + +void InterpreterMacroAssembler::restore_after_resume(Register fp) { + if (!Continuations::enabled()) return; + + const address resume_adapter = TemplateInterpreter::cont_resume_interpreter_adapter(); + add_const_optimized(R31, R29_TOC, MacroAssembler::offset_to_global_toc(resume_adapter)); + mtctr(R31); + bctrl(); + // Restore registers that are preserved across vthread preemption + assert(nonvolatile_accross_vthread_preemtion(R31) && nonvolatile_accross_vthread_preemtion(R22), ""); + ld(R3_ARG1, _abi0(callers_sp), R1_SP); // load FP + ld(R31, _ijava_state_neg(lresult), R3_ARG1); + ld(R22, _ijava_state_neg(fresult), R3_ARG1); +#ifdef ASSERT + // Assert FP is in R11_scratch1 (see generate_cont_resume_interpreter_adapter()) + { + Label ok; + ld(R12_scratch2, 0, R1_SP); // load fp + cmpd(CCR0, R12_scratch2, R11_scratch1); + beq(CCR0, ok); + stop(FILE_AND_LINE ": FP is expected in R11_scratch1"); + bind(ok); + } +#endif + if (fp != noreg && fp != R11_scratch1) { + mr(fp, R11_scratch1); + } +} + void InterpreterMacroAssembler::call_VM(Register oop_result, address entry_point, Register arg_1, Register arg_2, bool check_exceptions) { diff --git a/src/hotspot/cpu/ppc/macroAssembler_ppc.cpp b/src/hotspot/cpu/ppc/macroAssembler_ppc.cpp index 190e0c39fd7..994521ba48f 100644 --- a/src/hotspot/cpu/ppc/macroAssembler_ppc.cpp +++ b/src/hotspot/cpu/ppc/macroAssembler_ppc.cpp @@ -31,6 +31,7 @@ #include "gc/shared/barrierSet.hpp" #include "gc/shared/barrierSetAssembler.hpp" #include "interpreter/interpreter.hpp" +#include "interpreter/interpreterRuntime.hpp" #include "memory/resourceArea.hpp" #include "nativeInst_ppc.hpp" #include "oops/compressedKlass.inline.hpp" @@ -115,7 +116,8 @@ void MacroAssembler::align_prefix() { // Issue instructions that calculate given TOC from global TOC. void MacroAssembler::calculate_address_from_global_toc(Register dst, address addr, bool hi16, bool lo16, - bool add_relocation, bool emit_dummy_addr) { + bool add_relocation, bool emit_dummy_addr, + bool add_addr_to_reloc) { int offset = -1; if (emit_dummy_addr) { offset = -128; // dummy address @@ -129,7 +131,10 @@ void MacroAssembler::calculate_address_from_global_toc(Register dst, address add if (lo16) { if (add_relocation) { // Relocate at the addi to avoid confusion with a load from the method's TOC. - relocate(internal_word_Relocation::spec(addr)); + RelocationHolder rh = add_addr_to_reloc ? + internal_word_Relocation::spec(addr) : + internal_word_Relocation::spec_for_immediate(); + relocate(rh); } addi(dst, dst, MacroAssembler::largeoffset_si16_si16_lo(offset)); } @@ -714,6 +719,7 @@ address MacroAssembler::get_dest_of_bxx64_patchable_at(address instruction_addr, } } +#ifdef ASSERT void MacroAssembler::clobber_volatile_gprs(Register excluded_register) { const int magic_number = 0x42; @@ -729,6 +735,37 @@ void MacroAssembler::clobber_volatile_gprs(Register excluded_register) { } } +void MacroAssembler::clobber_nonvolatile_registers() { + BLOCK_COMMENT("clobber nonvolatile registers {"); + static const Register regs[] = { + R14, + R15, + // don't zap R16_thread + R17, + R18, + R19, + R20, + R21, + R22, + R23, + R24, + R25, + R26, + R27, + R28, + // don't zap R29_TOC + R30, + R31 + }; + Register bad = regs[0]; + load_const_optimized(bad, 0xbad0101babe11111); + for (uint32_t i = 1; i < (sizeof(regs) / sizeof(Register)); i++) { + mr(regs[i], bad); + } + BLOCK_COMMENT("} clobber nonvolatile registers"); +} +#endif // ASSERT + void MacroAssembler::clobber_carg_stack_slots(Register tmp) { const int magic_number = 0x43; @@ -1218,6 +1255,9 @@ int MacroAssembler::ic_check_size() { num_ins = 7; if (!implicit_null_checks_available) num_ins += 2; } + + if (UseCompactObjectHeaders) num_ins++; + return num_ins * BytesPerInstWord; } @@ -1245,7 +1285,9 @@ int MacroAssembler::ic_check(int end_alignment) { if (use_trap_based_null_check) { trap_null_check(receiver); } - if (UseCompressedClassPointers) { + if (UseCompactObjectHeaders) { + load_narrow_klass_compact(tmp1, receiver); + } else if (UseCompressedClassPointers) { lwz(tmp1, oopDesc::klass_offset_in_bytes(), receiver); } else { ld(tmp1, oopDesc::klass_offset_in_bytes(), receiver); @@ -1283,13 +1325,14 @@ int MacroAssembler::ic_check(int end_alignment) { void MacroAssembler::call_VM_base(Register oop_result, Register last_java_sp, address entry_point, - bool check_exceptions) { + bool check_exceptions, + Label* last_java_pc) { BLOCK_COMMENT("call_VM {"); // Determine last_java_sp register. if (!last_java_sp->is_valid()) { last_java_sp = R1_SP; } - set_top_ijava_frame_at_SP_as_last_Java_frame(last_java_sp, R11_scratch1); + set_top_ijava_frame_at_SP_as_last_Java_frame(last_java_sp, R11_scratch1, last_java_pc); // ARG1 must hold thread address. mr(R3_ARG1, R16_thread); @@ -1318,8 +1361,8 @@ void MacroAssembler::call_VM_leaf_base(address entry_point) { BLOCK_COMMENT("} call_VM_leaf"); } -void MacroAssembler::call_VM(Register oop_result, address entry_point, bool check_exceptions) { - call_VM_base(oop_result, noreg, entry_point, check_exceptions); +void MacroAssembler::call_VM(Register oop_result, address entry_point, bool check_exceptions, Label* last_java_pc) { + call_VM_base(oop_result, noreg, entry_point, check_exceptions, last_java_pc); } void MacroAssembler::call_VM(Register oop_result, address entry_point, Register arg_1, @@ -2620,15 +2663,15 @@ void MacroAssembler::compiler_fast_lock_object(ConditionRegister flag, Register // Handle existing monitor. bind(object_has_monitor); - // The object's monitor m is unlocked iff m->owner is null, - // otherwise m->owner may contain a thread or a stack address. - // Try to CAS m->owner from null to current thread. + // Try to CAS owner (no owner => current thread's _lock_id). addi(temp, displaced_header, in_bytes(ObjectMonitor::owner_offset()) - markWord::monitor_value); + Register thread_id = displaced_header; + ld(thread_id, in_bytes(JavaThread::lock_id_offset()), R16_thread); cmpxchgd(/*flag=*/flag, /*current_value=*/current_header, /*compare_value=*/(intptr_t)0, - /*exchange_value=*/R16_thread, + /*exchange_value=*/thread_id, /*where=*/temp, MacroAssembler::MemBarRel | MacroAssembler::MemBarAcq, MacroAssembler::cmpxchgx_hint_acquire_lock()); @@ -2638,7 +2681,7 @@ void MacroAssembler::compiler_fast_lock_object(ConditionRegister flag, Register beq(flag, success); // Check for recursive locking. - cmpd(flag, current_header, R16_thread); + cmpd(flag, current_header, thread_id); bne(flag, failure); // Current thread already owns the lock. Just increment recursions. @@ -2647,10 +2690,12 @@ void MacroAssembler::compiler_fast_lock_object(ConditionRegister flag, Register addi(recursions, recursions, 1); std(recursions, in_bytes(ObjectMonitor::recursions_offset() - ObjectMonitor::owner_offset()), temp); - // flag == EQ indicates success, increment held monitor count + // flag == EQ indicates success, increment held monitor count if LM_LEGACY is enabled // flag == NE indicates failure bind(success); - inc_held_monitor_count(temp); + if (LockingMode == LM_LEGACY) { + inc_held_monitor_count(temp); + } #ifdef ASSERT // Check that unlocked label is reached with flag == EQ. Label flag_correct; @@ -2670,7 +2715,7 @@ void MacroAssembler::compiler_fast_unlock_object(ConditionRegister flag, Registe Register temp, Register displaced_header, Register current_header) { assert(LockingMode != LM_LIGHTWEIGHT, "uses fast_unlock_lightweight"); assert_different_registers(oop, box, temp, displaced_header, current_header); - Label success, failure, object_has_monitor, notRecursive; + Label success, failure, object_has_monitor, not_recursive; if (LockingMode == LM_LEGACY) { // Find the lock address and load the displaced header from the stack. @@ -2716,7 +2761,7 @@ void MacroAssembler::compiler_fast_unlock_object(ConditionRegister flag, Registe ld(displaced_header, in_bytes(ObjectMonitor::recursions_offset()), current_header); addic_(displaced_header, displaced_header, -1); - blt(CCR0, notRecursive); // Not recursive if negative after decrement. + blt(CCR0, not_recursive); // Not recursive if negative after decrement. // Recursive unlock std(displaced_header, in_bytes(ObjectMonitor::recursions_offset()), current_header); @@ -2725,7 +2770,7 @@ void MacroAssembler::compiler_fast_unlock_object(ConditionRegister flag, Registe } b(success); - bind(notRecursive); + bind(not_recursive); // Set owner to null. // Release to satisfy the JMM @@ -2755,10 +2800,12 @@ void MacroAssembler::compiler_fast_unlock_object(ConditionRegister flag, Registe std(current_header, in_bytes(JavaThread::unlocked_inflated_monitor_offset()), R16_thread); b(failure); // flag == NE - // flag == EQ indicates success, decrement held monitor count + // flag == EQ indicates success, decrement held monitor count if LM_LEGACY is enabled // flag == NE indicates failure bind(success); - dec_held_monitor_count(temp); + if (LockingMode == LM_LEGACY) { + dec_held_monitor_count(temp); + } #ifdef ASSERT // Check that unlocked label is reached with flag == EQ. Label flag_correct; @@ -2777,6 +2824,7 @@ void MacroAssembler::compiler_fast_unlock_object(ConditionRegister flag, Registe void MacroAssembler::compiler_fast_lock_lightweight_object(ConditionRegister flag, Register obj, Register box, Register tmp1, Register tmp2, Register tmp3) { assert_different_registers(obj, box, tmp1, tmp2, tmp3); + assert(UseObjectMonitorTable || tmp3 == noreg, "tmp3 not needed"); assert(flag == CCR0, "bad condition register"); // Handle inflated monitor. @@ -2799,8 +2847,7 @@ void MacroAssembler::compiler_fast_lock_lightweight_object(ConditionRegister fla bne(CCR0, slow_path); } - const Register mark = tmp1; - const Register t = tmp3; // Usage of R0 allowed! + Register mark = tmp1; { // Lightweight locking @@ -2818,15 +2865,15 @@ void MacroAssembler::compiler_fast_lock_lightweight_object(ConditionRegister fla // when the lock stack is empty because of the _bad_oop_sentinel field. // Check if recursive. - subi(t, top, oopSize); - ldx(t, R16_thread, t); - cmpd(CCR0, obj, t); + subi(R0, top, oopSize); + ldx(R0, R16_thread, R0); + cmpd(CCR0, obj, R0); beq(CCR0, push); // Check for monitor (0b10) or locked (0b00). ld(mark, oopDesc::mark_offset_in_bytes(), obj); - andi_(t, mark, markWord::lock_mask_in_place); - cmpldi(CCR0, t, markWord::unlocked_value); + andi_(R0, mark, markWord::lock_mask_in_place); + cmpldi(CCR0, R0, markWord::unlocked_value); bgt(CCR0, inflated); bne(CCR0, slow_path); @@ -2849,13 +2896,15 @@ void MacroAssembler::compiler_fast_lock_lightweight_object(ConditionRegister fla // mark contains the tagged ObjectMonitor*. const uintptr_t monitor_tag = markWord::monitor_value; - const Register monitor = mark; + const Register monitor = UseObjectMonitorTable ? tmp1 : noreg; const Register owner_addr = tmp2; + const Register thread_id = UseObjectMonitorTable ? tmp3 : tmp1; Label monitor_locked; if (!UseObjectMonitorTable) { // Compute owner address. addi(owner_addr, mark, in_bytes(ObjectMonitor::owner_offset()) - monitor_tag); + mark = noreg; } else { Label monitor_found; Register cache_addr = tmp2; @@ -2865,8 +2914,8 @@ void MacroAssembler::compiler_fast_lock_lightweight_object(ConditionRegister fla const int num_unrolled = 2; for (int i = 0; i < num_unrolled; i++) { - ld(tmp3, 0, cache_addr); - cmpd(CCR0, tmp3, obj); + ld(R0, 0, cache_addr); + cmpd(CCR0, R0, obj); beq(CCR0, monitor_found); addi(cache_addr, cache_addr, in_bytes(OMCache::oop_to_oop_difference())); } @@ -2877,13 +2926,13 @@ void MacroAssembler::compiler_fast_lock_lightweight_object(ConditionRegister fla bind(loop); // Check for match. - ld(tmp3, 0, cache_addr); - cmpd(CCR0, tmp3, obj); + ld(R0, 0, cache_addr); + cmpd(CCR0, R0, obj); beq(CCR0, monitor_found); // Search until null encountered, guaranteed _null_sentinel at end. addi(cache_addr, cache_addr, in_bytes(OMCache::oop_to_oop_difference())); - cmpdi(CCR1, tmp3, 0); + cmpdi(CCR1, R0, 0); bne(CCR1, loop); // Cache Miss, CCR0.NE set from cmp above b(slow_path); @@ -2895,18 +2944,20 @@ void MacroAssembler::compiler_fast_lock_lightweight_object(ConditionRegister fla addi(owner_addr, monitor, in_bytes(ObjectMonitor::owner_offset())); } - // CAS owner (null => current thread). + // Try to CAS owner (no owner => current thread's _lock_id). + assert_different_registers(thread_id, monitor, owner_addr, box, R0); + ld(thread_id, in_bytes(JavaThread::lock_id_offset()), R16_thread); cmpxchgd(/*flag=*/CCR0, - /*current_value=*/t, + /*current_value=*/R0, /*compare_value=*/(intptr_t)0, - /*exchange_value=*/R16_thread, + /*exchange_value=*/thread_id, /*where=*/owner_addr, MacroAssembler::MemBarRel | MacroAssembler::MemBarAcq, MacroAssembler::cmpxchgx_hint_acquire_lock()); beq(CCR0, monitor_locked); // Check if recursive. - cmpd(CCR0, t, R16_thread); + cmpd(CCR0, R0, thread_id); bne(CCR0, slow_path); // Recursive. @@ -2929,7 +2980,6 @@ void MacroAssembler::compiler_fast_lock_lightweight_object(ConditionRegister fla } bind(locked); - inc_held_monitor_count(tmp1); #ifdef ASSERT // Check that locked label is reached with flag == EQ. @@ -3104,7 +3154,6 @@ void MacroAssembler::compiler_fast_unlock_lightweight_object(ConditionRegister f } bind(unlocked); - dec_held_monitor_count(t); #ifdef ASSERT // Check that unlocked label is reached with flag == EQ. @@ -3186,9 +3235,11 @@ void MacroAssembler::set_last_Java_frame(Register last_Java_sp, Register last_Ja std(last_Java_sp, in_bytes(JavaThread::last_Java_sp_offset()), R16_thread); } -void MacroAssembler::reset_last_Java_frame(void) { - asm_assert_mem8_isnot_zero(in_bytes(JavaThread::last_Java_sp_offset()), - R16_thread, "SP was not set, still zero"); +void MacroAssembler::reset_last_Java_frame(bool check_last_java_sp) { + if (check_last_java_sp) { + asm_assert_mem8_isnot_zero(in_bytes(JavaThread::last_Java_sp_offset()), + R16_thread, "SP was not set, still zero"); + } BLOCK_COMMENT("reset_last_Java_frame {"); li(R0, 0); @@ -3201,14 +3252,14 @@ void MacroAssembler::reset_last_Java_frame(void) { BLOCK_COMMENT("} reset_last_Java_frame"); } -void MacroAssembler::set_top_ijava_frame_at_SP_as_last_Java_frame(Register sp, Register tmp1) { +void MacroAssembler::set_top_ijava_frame_at_SP_as_last_Java_frame(Register sp, Register tmp1, Label* jpc) { assert_different_registers(sp, tmp1); - // sp points to a TOP_IJAVA_FRAME, retrieve frame's PC via - // TOP_IJAVA_FRAME_ABI. - // FIXME: assert that we really have a TOP_IJAVA_FRAME here! - address entry = pc(); - load_const_optimized(tmp1, entry); + if (jpc == nullptr || jpc->is_bound()) { + load_const_optimized(tmp1, jpc == nullptr ? pc() : target(*jpc)); + } else { + load_const(tmp1, *jpc, R12_scratch2); + } set_last_Java_frame(/*sp=*/sp, /*pc=*/tmp1); } @@ -3258,6 +3309,7 @@ Register MacroAssembler::encode_klass_not_null(Register dst, Register src) { } void MacroAssembler::store_klass(Register dst_oop, Register klass, Register ck) { + assert(!UseCompactObjectHeaders, "not with compact headers"); if (UseCompressedClassPointers) { Register compressedKlass = encode_klass_not_null(ck, klass); stw(compressedKlass, oopDesc::klass_offset_in_bytes(), dst_oop); @@ -3267,12 +3319,13 @@ void MacroAssembler::store_klass(Register dst_oop, Register klass, Register ck) } void MacroAssembler::store_klass_gap(Register dst_oop, Register val) { + assert(!UseCompactObjectHeaders, "not with compact headers"); if (UseCompressedClassPointers) { if (val == noreg) { val = R0; li(val, 0); } - stw(val, oopDesc::klass_gap_offset_in_bytes(), dst_oop); // klass gap if compressed + stw(val, oopDesc::klass_gap_offset_in_bytes(), dst_oop); } } @@ -3313,15 +3366,60 @@ void MacroAssembler::decode_klass_not_null(Register dst, Register src) { } void MacroAssembler::load_klass(Register dst, Register src) { - if (UseCompressedClassPointers) { + if (UseCompactObjectHeaders) { + load_narrow_klass_compact(dst, src); + decode_klass_not_null(dst); + } else if (UseCompressedClassPointers) { lwz(dst, oopDesc::klass_offset_in_bytes(), src); - // Attention: no null check here! - decode_klass_not_null(dst, dst); + decode_klass_not_null(dst); } else { ld(dst, oopDesc::klass_offset_in_bytes(), src); } } +// Loads the obj's Klass* into dst. +// Preserves all registers (incl src, rscratch1 and rscratch2). +// Input: +// src - the oop we want to load the klass from. +// dst - output nklass. +void MacroAssembler::load_narrow_klass_compact(Register dst, Register src) { + assert(UseCompactObjectHeaders, "expects UseCompactObjectHeaders"); + ld(dst, oopDesc::mark_offset_in_bytes(), src); + srdi(dst, dst, markWord::klass_shift); +} + +void MacroAssembler::cmp_klass(ConditionRegister dst, Register obj, Register klass, Register tmp, Register tmp2) { + assert_different_registers(obj, klass, tmp); + if (UseCompressedClassPointers) { + if (UseCompactObjectHeaders) { + load_narrow_klass_compact(tmp, obj); + } else { + lwz(tmp, oopDesc::klass_offset_in_bytes(), obj); + } + Register encoded_klass = encode_klass_not_null(tmp2, klass); + cmpw(dst, tmp, encoded_klass); + } else { + ld(tmp, oopDesc::klass_offset_in_bytes(), obj); + cmpd(dst, tmp, klass); + } +} + +void MacroAssembler::cmp_klasses_from_objects(ConditionRegister dst, Register obj1, Register obj2, Register tmp1, Register tmp2) { + if (UseCompactObjectHeaders) { + load_narrow_klass_compact(tmp1, obj1); + load_narrow_klass_compact(tmp2, obj2); + cmpw(dst, tmp1, tmp2); + } else if (UseCompressedClassPointers) { + lwz(tmp1, oopDesc::klass_offset_in_bytes(), obj1); + lwz(tmp2, oopDesc::klass_offset_in_bytes(), obj2); + cmpw(dst, tmp1, tmp2); + } else { + ld(tmp1, oopDesc::klass_offset_in_bytes(), obj1); + ld(tmp2, oopDesc::klass_offset_in_bytes(), obj2); + cmpd(dst, tmp1, tmp2); + } +} + void MacroAssembler::load_klass_check_null(Register dst, Register src, Label* is_null) { null_check(src, oopDesc::klass_offset_in_bytes(), is_null); load_klass(dst, src); @@ -4478,9 +4576,9 @@ void MacroAssembler::asm_assert(bool check_equal, const char *msg) { #endif } +#ifdef ASSERT void MacroAssembler::asm_assert_mems_zero(bool check_equal, int size, int mem_offset, Register mem_base, const char* msg) { -#ifdef ASSERT switch (size) { case 4: lwz(R0, mem_offset, mem_base); @@ -4494,8 +4592,8 @@ void MacroAssembler::asm_assert_mems_zero(bool check_equal, int size, int mem_of ShouldNotReachHere(); } asm_assert(check_equal, msg); -#endif // ASSERT } +#endif // ASSERT void MacroAssembler::verify_coop(Register coop, const char* msg) { if (!VerifyOops) { return; } @@ -4639,6 +4737,8 @@ void MacroAssembler::cache_wbsync(bool is_presync) { } void MacroAssembler::push_cont_fastpath() { + if (!Continuations::enabled()) return; + Label done; ld_ptr(R0, JavaThread::cont_fastpath_offset(), R16_thread); cmpld(CCR0, R1_SP, R0); @@ -4648,6 +4748,8 @@ void MacroAssembler::push_cont_fastpath() { } void MacroAssembler::pop_cont_fastpath() { + if (!Continuations::enabled()) return; + Label done; ld_ptr(R0, JavaThread::cont_fastpath_offset(), R16_thread); cmpld(CCR0, R1_SP, R0); @@ -4659,6 +4761,7 @@ void MacroAssembler::pop_cont_fastpath() { // Note: Must preserve CCR0 EQ (invariant). void MacroAssembler::inc_held_monitor_count(Register tmp) { + assert(LockingMode == LM_LEGACY, ""); ld(tmp, in_bytes(JavaThread::held_monitor_count_offset()), R16_thread); #ifdef ASSERT Label ok; @@ -4674,6 +4777,7 @@ void MacroAssembler::inc_held_monitor_count(Register tmp) { // Note: Must preserve CCR0 EQ (invariant). void MacroAssembler::dec_held_monitor_count(Register tmp) { + assert(LockingMode == LM_LEGACY, ""); ld(tmp, in_bytes(JavaThread::held_monitor_count_offset()), R16_thread); #ifdef ASSERT Label ok; diff --git a/src/hotspot/cpu/ppc/macroAssembler_ppc.hpp b/src/hotspot/cpu/ppc/macroAssembler_ppc.hpp index f0e7c644535..237024fa05e 100644 --- a/src/hotspot/cpu/ppc/macroAssembler_ppc.hpp +++ b/src/hotspot/cpu/ppc/macroAssembler_ppc.hpp @@ -115,7 +115,13 @@ class MacroAssembler: public Assembler { // Global TOC. void calculate_address_from_global_toc(Register dst, address addr, bool hi16 = true, bool lo16 = true, - bool add_relocation = true, bool emit_dummy_addr = false); + bool add_relocation = true, bool emit_dummy_addr = false, + bool add_addr_to_reloc = true); + void calculate_address_from_global_toc(Register dst, Label& addr, + bool hi16 = true, bool lo16 = true, + bool add_relocation = true, bool emit_dummy_addr = false) { + calculate_address_from_global_toc(dst, target(addr), hi16, lo16, add_relocation, emit_dummy_addr, false); + } inline void calculate_address_from_global_toc_hi16only(Register dst, address addr) { calculate_address_from_global_toc(dst, addr, true, false); }; @@ -284,7 +290,10 @@ class MacroAssembler: public Assembler { // Clobbers all volatile, (non-floating-point) general-purpose registers for debugging purposes. // This is especially useful for making calls to the JRT in places in which this hasn't been done before; // e.g. with the introduction of LRBs (load reference barriers) for concurrent garbage collection. - void clobber_volatile_gprs(Register excluded_register = noreg); + void clobber_volatile_gprs(Register excluded_register = noreg) NOT_DEBUG_RETURN; + // Load bad values into registers that are nonvolatile according to the ABI except R16_thread and R29_TOC. + // This is done after vthread preemption and before vthread resume. + void clobber_nonvolatile_registers() NOT_DEBUG_RETURN; void clobber_carg_stack_slots(Register tmp); void save_nonvolatile_gprs( Register dst_base, int offset); @@ -398,7 +407,8 @@ class MacroAssembler: public Assembler { // the entry point address entry_point, // flag which indicates if exception should be checked - bool check_exception = true + bool check_exception = true, + Label* last_java_pc = nullptr ); // Support for VM calls. This is the base routine called by the @@ -411,7 +421,7 @@ class MacroAssembler: public Assembler { // Call into the VM. // Passes the thread pointer (in R3_ARG1) as a prepended argument. // Makes sure oop return values are visible to the GC. - void call_VM(Register oop_result, address entry_point, bool check_exceptions = true); + void call_VM(Register oop_result, address entry_point, bool check_exceptions = true, Label* last_java_pc = nullptr); void call_VM(Register oop_result, address entry_point, Register arg_1, bool check_exceptions = true); void call_VM(Register oop_result, address entry_point, Register arg_1, Register arg_2, bool check_exceptions = true); void call_VM(Register oop_result, address entry_point, Register arg_1, Register arg_2, Register arg3, bool check_exceptions = true); @@ -695,8 +705,8 @@ class MacroAssembler: public Assembler { // Support for last Java frame (but use call_VM instead where possible): // access R16_thread->last_Java_sp. void set_last_Java_frame(Register last_java_sp, Register last_Java_pc); - void reset_last_Java_frame(void); - void set_top_ijava_frame_at_SP_as_last_Java_frame(Register sp, Register tmp1); + void reset_last_Java_frame(bool check_last_java_sp = true); + void set_top_ijava_frame_at_SP_as_last_Java_frame(Register sp, Register tmp1, Label* jpc = nullptr); // Read vm result from thread: oop_result = R16_thread->result; void get_vm_result (Register oop_result); @@ -757,6 +767,9 @@ class MacroAssembler: public Assembler { // Load/Store klass oop from klass field. Compress. void load_klass(Register dst, Register src); + void load_narrow_klass_compact(Register dst, Register src); + void cmp_klass(ConditionRegister dst, Register obj, Register klass, Register tmp, Register tmp2); + void cmp_klasses_from_objects(ConditionRegister dst, Register obj1, Register obj2, Register tmp1, Register tmp2); void load_klass_check_null(Register dst, Register src, Label* is_null = nullptr); void store_klass(Register dst_oop, Register klass, Register tmp = R0); void store_klass_gap(Register dst_oop, Register val = noreg); // Will store 0 if val not specified. @@ -909,7 +922,7 @@ class MacroAssembler: public Assembler { private: void asm_assert_mems_zero(bool check_equal, int size, int mem_offset, Register mem_base, - const char* msg); + const char* msg) NOT_DEBUG_RETURN; public: diff --git a/src/hotspot/cpu/ppc/macroAssembler_ppc.inline.hpp b/src/hotspot/cpu/ppc/macroAssembler_ppc.inline.hpp index e9c6fd38f45..f98ad592388 100644 --- a/src/hotspot/cpu/ppc/macroAssembler_ppc.inline.hpp +++ b/src/hotspot/cpu/ppc/macroAssembler_ppc.inline.hpp @@ -191,8 +191,18 @@ inline void MacroAssembler::set_oop(AddressLiteral obj_addr, Register d) { } inline void MacroAssembler::pd_patch_instruction(address branch, address target, const char* file, int line) { - jint& stub_inst = *(jint*) branch; - stub_inst = patched_branch(target - branch, stub_inst, 0); + if (is_branch(branch)) { + jint& stub_inst = *(jint*) branch; + stub_inst = patched_branch(target - branch, stub_inst, 0); + } else if (is_calculate_address_from_global_toc_at(branch + BytesPerInstWord, branch)) { + const address inst1_addr = branch; + const address inst2_addr = branch + BytesPerInstWord; + patch_calculate_address_from_global_toc_at(inst2_addr, inst1_addr, target); + } else if (is_load_const_at(branch)) { + patch_const(branch, (long)target); + } else { + assert(false, "instruction at " PTR_FORMAT " not recognized", p2i(branch)); + } } // Relocation of conditional far branches. diff --git a/src/hotspot/cpu/ppc/nativeInst_ppc.cpp b/src/hotspot/cpu/ppc/nativeInst_ppc.cpp index f61328fc736..dbc43c45e09 100644 --- a/src/hotspot/cpu/ppc/nativeInst_ppc.cpp +++ b/src/hotspot/cpu/ppc/nativeInst_ppc.cpp @@ -205,12 +205,14 @@ intptr_t NativeMovConstReg::data() const { // Therefore we use raw decoding. if (CompressedOops::is_null(no)) return 0; return cast_from_oop(CompressedOops::decode_raw(no)); - } else { - assert(MacroAssembler::is_load_const_from_method_toc_at(addr), "must be load_const_from_pool"); - + } else if (MacroAssembler::is_load_const_from_method_toc_at(addr)) { address ctable = cb->content_begin(); int offset = MacroAssembler::get_offset_of_load_const_from_method_toc_at(addr); return *(intptr_t *)(ctable + offset); + } else { + assert(MacroAssembler::is_calculate_address_from_global_toc_at(addr, addr - BytesPerInstWord), + "must be calculate_address_from_global_toc"); + return (intptr_t) MacroAssembler::get_address_of_calculate_address_from_global_toc_at(addr, addr - BytesPerInstWord); } } diff --git a/src/hotspot/cpu/ppc/ppc.ad b/src/hotspot/cpu/ppc/ppc.ad index 6d3daa025e8..cfed8674e89 100644 --- a/src/hotspot/cpu/ppc/ppc.ad +++ b/src/hotspot/cpu/ppc/ppc.ad @@ -5496,6 +5496,7 @@ instruct loadP2X(iRegLdst dst, memoryAlg4 mem) %{ // Load compressed klass pointer. instruct loadNKlass(iRegNdst dst, memory mem) %{ match(Set dst (LoadNKlass mem)); + predicate(!UseCompactObjectHeaders); ins_cost(MEMORY_REF_COST); format %{ "LWZ $dst, $mem \t// compressed klass ptr" %} @@ -5504,6 +5505,20 @@ instruct loadNKlass(iRegNdst dst, memory mem) %{ ins_pipe(pipe_class_memory); %} +instruct loadNKlassCompactHeaders(iRegNdst dst, memory mem) %{ + match(Set dst (LoadNKlass mem)); + predicate(UseCompactObjectHeaders); + ins_cost(MEMORY_REF_COST); + + format %{ "load_narrow_klass_compact $dst, $mem \t// compressed class ptr" %} + size(8); + ins_encode %{ + assert($mem$$index$$Register == R0, "must not have indexed address: %s[%s]", $mem$$base$$Register.name(), $mem$$index$$Register.name()); + __ load_narrow_klass_compact_c2($dst$$Register, $mem$$base$$Register, $mem$$disp); + %} + ins_pipe(pipe_class_memory); +%} + // Load Klass Pointer instruct loadKlass(iRegPdst dst, memoryAlg4 mem) %{ match(Set dst (LoadKlass mem)); @@ -12093,15 +12108,31 @@ instruct cmpFastUnlock(flagsRegCR0 crx, iRegPdst oop, iRegPdst box, iRegPdst tmp ins_pipe(pipe_class_compare); %} -instruct cmpFastLockLightweight(flagsRegCR0 crx, iRegPdst oop, iRegPdst box, iRegPdst tmp1, iRegPdst tmp2, flagsRegCR1 cr1) %{ - predicate(LockingMode == LM_LIGHTWEIGHT); +instruct cmpFastLockLightweight(flagsRegCR0 crx, iRegPdst oop, iRegPdst box, iRegPdst tmp1, iRegPdst tmp2) %{ + predicate(LockingMode == LM_LIGHTWEIGHT && !UseObjectMonitorTable); match(Set crx (FastLock oop box)); - effect(TEMP tmp1, TEMP tmp2, KILL cr1); + effect(TEMP tmp1, TEMP tmp2); format %{ "FASTLOCK $oop, $box, $tmp1, $tmp2" %} ins_encode %{ __ fast_lock_lightweight($crx$$CondRegister, $oop$$Register, $box$$Register, - $tmp1$$Register, $tmp2$$Register, /*tmp3*/ R0); + $tmp1$$Register, $tmp2$$Register, noreg /*tmp3*/); + // If locking was successful, crx should indicate 'EQ'. + // The compiler generates a branch to the runtime call to + // _complete_monitor_locking_Java for the case where crx is 'NE'. + %} + ins_pipe(pipe_class_compare); +%} + +instruct cmpFastLockMonitorTable(flagsRegCR0 crx, iRegPdst oop, iRegPdst box, iRegPdst tmp1, iRegPdst tmp2, iRegPdst tmp3, flagsRegCR1 cr1) %{ + predicate(LockingMode == LM_LIGHTWEIGHT && UseObjectMonitorTable); + match(Set crx (FastLock oop box)); + effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr1); + + format %{ "FASTLOCK $oop, $box, $tmp1, $tmp2, $tmp3" %} + ins_encode %{ + __ fast_lock_lightweight($crx$$CondRegister, $oop$$Register, $box$$Register, + $tmp1$$Register, $tmp2$$Register, $tmp3$$Register); // If locking was successful, crx should indicate 'EQ'. // The compiler generates a branch to the runtime call to // _complete_monitor_locking_Java for the case where crx is 'NE'. diff --git a/src/hotspot/cpu/ppc/sharedRuntime_ppc.cpp b/src/hotspot/cpu/ppc/sharedRuntime_ppc.cpp index aa8ae6070b6..84102ef41e8 100644 --- a/src/hotspot/cpu/ppc/sharedRuntime_ppc.cpp +++ b/src/hotspot/cpu/ppc/sharedRuntime_ppc.cpp @@ -1602,6 +1602,7 @@ static void fill_continuation_entry(MacroAssembler* masm, Register reg_cont_obj, #ifdef ASSERT __ load_const_optimized(tmp2, ContinuationEntry::cookie_value()); __ stw(tmp2, in_bytes(ContinuationEntry::cookie_offset()), R1_SP); + __ std(tmp2, _abi0(cr), R1_SP); #endif //ASSERT __ li(zero, 0); @@ -1645,6 +1646,10 @@ static void continuation_enter_cleanup(MacroAssembler* masm) { __ ld_ptr(tmp1, JavaThread::cont_entry_offset(), R16_thread); __ cmpd(CCR0, R1_SP, tmp1); __ asm_assert_eq(FILE_AND_LINE ": incorrect R1_SP"); + __ load_const_optimized(tmp1, ContinuationEntry::cookie_value()); + __ ld(tmp2, _abi0(cr), R1_SP); + __ cmpd(CCR0, tmp1, tmp2); + __ asm_assert_eq(FILE_AND_LINE ": cookie not found"); #endif __ ld_ptr(tmp1, ContinuationEntry::parent_cont_fastpath_offset(), R1_SP); @@ -1853,6 +1858,7 @@ static void gen_continuation_enter(MacroAssembler* masm, // --- Thawing path __ bind(L_thaw); + ContinuationEntry::_thaw_call_pc_offset = __ pc() - start; __ add_const_optimized(R0, R29_TOC, MacroAssembler::offset_to_global_toc(StubRoutines::cont_thaw())); __ mtctr(R0); __ bctrl(); @@ -1863,6 +1869,7 @@ static void gen_continuation_enter(MacroAssembler* masm, // --- Normal exit (resolve/thawing) __ bind(L_exit); + ContinuationEntry::_cleanup_offset = __ pc() - start; continuation_enter_cleanup(masm); // Pop frame and return @@ -1970,6 +1977,10 @@ static void gen_continuation_yield(MacroAssembler* masm, __ bctr(); } +void SharedRuntime::continuation_enter_cleanup(MacroAssembler* masm) { + ::continuation_enter_cleanup(masm); +} + // --------------------------------------------------------------------------- // Generate a native wrapper for a given method. The method takes arguments // in the Java compiled code convention, marshals them to the native @@ -2190,9 +2201,9 @@ nmethod *SharedRuntime::generate_native_wrapper(MacroAssembler *masm, intptr_t start_pc = (intptr_t)__ pc(); intptr_t vep_start_pc; intptr_t frame_done_pc; - intptr_t oopmap_pc; Label handle_pending_exception; + Label last_java_pc; Register r_callers_sp = R21; Register r_temp_1 = R22; @@ -2201,7 +2212,7 @@ nmethod *SharedRuntime::generate_native_wrapper(MacroAssembler *masm, Register r_temp_4 = R25; Register r_temp_5 = R26; Register r_temp_6 = R27; - Register r_return_pc = R28; + Register r_last_java_pc = R28; Register r_carg1_jnienv = noreg; Register r_carg2_classorobject = noreg; @@ -2363,15 +2374,9 @@ nmethod *SharedRuntime::generate_native_wrapper(MacroAssembler *masm, // We MUST NOT touch any outgoing regs from this point on. // So if we must call out we must push a new frame. - // Get current pc for oopmap, and load it patchable relative to global toc. - oopmap_pc = (intptr_t) __ pc(); - __ calculate_address_from_global_toc(r_return_pc, (address)oopmap_pc, true, true, true, true); - - // We use the same pc/oopMap repeatedly when we call out. - oop_maps->add_gc_map(oopmap_pc - start_pc, oop_map); - - // r_return_pc now has the pc loaded that we will use when we finally call - // to native. + // The last java pc will also be used as resume pc if this is the wrapper for wait0. + // For this purpose the precise location matters but not for oopmap lookup. + __ calculate_address_from_global_toc(r_last_java_pc, last_java_pc, true, true, true, true); // Make sure that thread is non-volatile; it crosses a bunch of VM calls below. assert(R16_thread->is_nonvolatile(), "thread must be in non-volatile register"); @@ -2399,7 +2404,8 @@ nmethod *SharedRuntime::generate_native_wrapper(MacroAssembler *masm, // Try fastpath for locking. if (LockingMode == LM_LIGHTWEIGHT) { // fast_lock kills r_temp_1, r_temp_2, r_temp_3. - __ compiler_fast_lock_lightweight_object(CCR0, r_oop, r_box, r_temp_1, r_temp_2, r_temp_3); + Register r_temp_3_or_noreg = UseObjectMonitorTable ? r_temp_3 : noreg; + __ compiler_fast_lock_lightweight_object(CCR0, r_oop, r_box, r_temp_1, r_temp_2, r_temp_3_or_noreg); } else { // fast_lock kills r_temp_1, r_temp_2, r_temp_3. __ compiler_fast_lock_object(CCR0, r_oop, r_box, r_temp_1, r_temp_2, r_temp_3); @@ -2416,9 +2422,14 @@ nmethod *SharedRuntime::generate_native_wrapper(MacroAssembler *masm, RegisterSaver::push_frame_and_save_argument_registers(masm, R12_scratch2, frame_size, total_c_args, out_regs); // Do the call. - __ set_last_Java_frame(R11_scratch1, r_return_pc); - assert(r_return_pc->is_nonvolatile(), "expecting return pc to be in non-volatile register"); + __ set_last_Java_frame(R11_scratch1, r_last_java_pc); + assert(r_last_java_pc->is_nonvolatile(), "r_last_java_pc needs to be preserved accross complete_monitor_locking_C call"); + // The following call will not be preempted. + // push_cont_fastpath forces freeze slow path in case we try to preempt where we will pin the + // vthread to the carrier (see FreezeBase::recurse_freeze_native_frame()). + __ push_cont_fastpath(); __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::complete_monitor_locking_C), r_oop, r_box, R16_thread); + __ pop_cont_fastpath(); __ reset_last_Java_frame(); RegisterSaver::restore_argument_registers_and_pop_frame(masm, frame_size, total_c_args, out_regs); @@ -2429,8 +2440,7 @@ nmethod *SharedRuntime::generate_native_wrapper(MacroAssembler *masm, __ bind(locked); } - // Use that pc we placed in r_return_pc a while back as the current frame anchor. - __ set_last_Java_frame(R1_SP, r_return_pc); + __ set_last_Java_frame(R1_SP, r_last_java_pc); // Publish thread state // -------------------------------------------------------------------------- @@ -2490,8 +2500,6 @@ nmethod *SharedRuntime::generate_native_wrapper(MacroAssembler *masm, break; } - Label after_transition; - // Publish thread state // -------------------------------------------------------------------------- @@ -2566,7 +2574,23 @@ nmethod *SharedRuntime::generate_native_wrapper(MacroAssembler *masm, __ lwsync(); // Acquire safepoint and suspend state, release thread state. // TODO: PPC port assert(4 == JavaThread::sz_thread_state(), "unexpected field size"); __ stw(R0, thread_(thread_state)); - __ bind(after_transition); + + // Check preemption for Object.wait() + if (LockingMode != LM_LEGACY && method->is_object_wait0()) { + Label not_preempted; + __ ld(R0, in_bytes(JavaThread::preempt_alternate_return_offset()), R16_thread); + __ cmpdi(CCR0, R0, 0); + __ beq(CCR0, not_preempted); + __ mtlr(R0); + __ li(R0, 0); + __ std(R0, in_bytes(JavaThread::preempt_alternate_return_offset()), R16_thread); + __ blr(); + __ bind(not_preempted); + } + __ bind(last_java_pc); + // We use the same pc/oopMap repeatedly when we call out above. + intptr_t oopmap_pc = (intptr_t) __ pc(); + oop_maps->add_gc_map(oopmap_pc - start_pc, oop_map); } // Reguard any pages if necessary. @@ -2648,7 +2672,9 @@ nmethod *SharedRuntime::generate_native_wrapper(MacroAssembler *masm, // Clear "last Java frame" SP and PC. // -------------------------------------------------------------------------- - __ reset_last_Java_frame(); + // Last java frame won't be set if we're resuming after preemption + bool maybe_preempted = LockingMode != LM_LEGACY && method->is_object_wait0(); + __ reset_last_Java_frame(!maybe_preempted /* check_last_java_sp */); // Unbox oop result, e.g. JNIHandles::resolve value. // -------------------------------------------------------------------------- @@ -2733,6 +2759,12 @@ uint SharedRuntime::out_preserve_stack_slots() { #endif } +VMReg SharedRuntime::thread_register() { + // On PPC virtual threads don't save the JavaThread* in their context (e.g. C1 stub frames). + ShouldNotCallThis(); + return nullptr; +} + #if defined(COMPILER1) || defined(COMPILER2) // Frame generation for deopt and uncommon trap blobs. static void push_skeleton_frame(MacroAssembler* masm, bool deopt, diff --git a/src/hotspot/cpu/ppc/stackChunkFrameStream_ppc.inline.hpp b/src/hotspot/cpu/ppc/stackChunkFrameStream_ppc.inline.hpp index cb4af1a3ff7..07f1c9c1c6f 100644 --- a/src/hotspot/cpu/ppc/stackChunkFrameStream_ppc.inline.hpp +++ b/src/hotspot/cpu/ppc/stackChunkFrameStream_ppc.inline.hpp @@ -184,8 +184,9 @@ inline int StackChunkFrameStream::interpreter_frame_num_oops() const f.interpreted_frame_oop_map(&mask); return mask.num_oops() + 1 // for the mirror oop - + ((intptr_t*)f.interpreter_frame_monitor_begin() - - (intptr_t*)f.interpreter_frame_monitor_end())/BasicObjectLock::size(); + + (f.interpreter_frame_method()->is_native() ? 1 : 0) // temp oop slot + + pointer_delta_as_int((intptr_t*)f.interpreter_frame_monitor_begin(), + (intptr_t*)f.interpreter_frame_monitor_end())/BasicObjectLock::size(); } template<> diff --git a/src/hotspot/cpu/ppc/stubGenerator_ppc.cpp b/src/hotspot/cpu/ppc/stubGenerator_ppc.cpp index b3ace8898ad..9f2b668b9c8 100644 --- a/src/hotspot/cpu/ppc/stubGenerator_ppc.cpp +++ b/src/hotspot/cpu/ppc/stubGenerator_ppc.cpp @@ -4483,6 +4483,10 @@ address generate_lookup_secondary_supers_table_stub(u1 super_klass_index) { address start = __ pc(); + if (kind == Continuation::thaw_top) { + __ clobber_nonvolatile_registers(); // Except R16_thread and R29_TOC + } + if (return_barrier) { __ mr(nvtmp, R3_RET); __ fmr(nvftmp, F1_RET); // preserve possible return value from a method returning to the return barrier DEBUG_ONLY(__ ld_ptr(tmp1, _abi0(callers_sp), R1_SP);) @@ -4571,6 +4575,41 @@ address generate_lookup_secondary_supers_table_stub(u1 super_klass_index) { return generate_cont_thaw("Cont thaw return barrier exception", Continuation::thaw_return_barrier_exception); } + address generate_cont_preempt_stub() { + if (!Continuations::enabled()) return nullptr; + StubCodeMark mark(this, "StubRoutines","Continuation preempt stub"); + address start = __ pc(); + + __ clobber_nonvolatile_registers(); // Except R16_thread and R29_TOC + + __ reset_last_Java_frame(false /*check_last_java_sp*/); + + // Set sp to enterSpecial frame, i.e. remove all frames copied into the heap. + __ ld_ptr(R1_SP, JavaThread::cont_entry_offset(), R16_thread); + + Label preemption_cancelled; + __ lbz(R11_scratch1, in_bytes(JavaThread::preemption_cancelled_offset()), R16_thread); + __ cmpwi(CCR0, R11_scratch1, 0); + __ bne(CCR0, preemption_cancelled); + + // Remove enterSpecial frame from the stack and return to Continuation.run() to unmount. + SharedRuntime::continuation_enter_cleanup(_masm); + __ pop_frame(); + __ restore_LR(R11_scratch1); + __ blr(); + + // We acquired the monitor after freezing the frames so call thaw to continue execution. + __ bind(preemption_cancelled); + __ li(R11_scratch1, 0); // false + __ stb(R11_scratch1, in_bytes(JavaThread::preemption_cancelled_offset()), R16_thread); + int simm16_offs = __ load_const_optimized(R11_scratch1, ContinuationEntry::thaw_call_pc_address(), R0, true); + __ ld(R11_scratch1, simm16_offs, R11_scratch1); + __ mtctr(R11_scratch1); + __ bctr(); + + return start; + } + // exception handler for upcall stubs address generate_upcall_stub_exception_handler() { StubCodeMark mark(this, "StubRoutines", "upcall stub exception handler"); @@ -4646,6 +4685,7 @@ address generate_lookup_secondary_supers_table_stub(u1 super_klass_index) { StubRoutines::_cont_thaw = generate_cont_thaw(); StubRoutines::_cont_returnBarrier = generate_cont_returnBarrier(); StubRoutines::_cont_returnBarrierExc = generate_cont_returnBarrier_exception(); + StubRoutines::_cont_preempt_stub = generate_cont_preempt_stub(); } void generate_final_stubs() { diff --git a/src/hotspot/cpu/ppc/templateInterpreterGenerator_ppc.cpp b/src/hotspot/cpu/ppc/templateInterpreterGenerator_ppc.cpp index cf3dd4cbd34..e320349583d 100644 --- a/src/hotspot/cpu/ppc/templateInterpreterGenerator_ppc.cpp +++ b/src/hotspot/cpu/ppc/templateInterpreterGenerator_ppc.cpp @@ -696,6 +696,17 @@ address TemplateInterpreterGenerator::generate_safept_entry_for(TosState state, return entry; } +address TemplateInterpreterGenerator::generate_cont_resume_interpreter_adapter() { + if (!Continuations::enabled()) return nullptr; + address start = __ pc(); + + __ load_const_optimized(R25_templateTableBase, (address)Interpreter::dispatch_table((TosState)0), R12_scratch2); + __ restore_interpreter_state(R11_scratch1, false, true /*restore_top_frame_sp*/); + __ blr(); + + return start; +} + // Helpers for commoning out cases in the various type of method entries. // Increment invocation count & check for overflow. @@ -1197,7 +1208,7 @@ address TemplateInterpreterGenerator::generate_native_entry(bool synchronized) { const Register signature_handler_fd = R11_scratch1; const Register pending_exception = R0; const Register result_handler_addr = R31; - const Register native_method_fd = R11_scratch1; + const Register native_method_fd = R12_scratch2; // preferred in MacroAssembler::branch_to const Register access_flags = R22_tmp2; const Register active_handles = R11_scratch1; // R26_monitor saved to state. const Register sync_state = R12_scratch2; @@ -1211,10 +1222,6 @@ address TemplateInterpreterGenerator::generate_native_entry(bool synchronized) { Label exception_return_sync_check; Label stack_overflow_return; - // Generate new interpreter state and jump to stack_overflow_return in case of - // a stack overflow. - //generate_compute_interpreter_state(stack_overflow_return); - Register size_of_parameters = R22_tmp2; generate_fixed_frame(true, size_of_parameters, noreg /* unused */); @@ -1253,8 +1260,8 @@ address TemplateInterpreterGenerator::generate_native_entry(bool synchronized) { // access_flags = method->access_flags(); // Load access flags. - assert(access_flags->is_nonvolatile(), - "access_flags must be in a non-volatile register"); + assert(__ nonvolatile_accross_vthread_preemtion(access_flags), + "access_flags not preserved"); // Type check. assert(4 == sizeof(AccessFlags), "unexpected field size"); __ lwz(access_flags, method_(access_flags)); @@ -1315,8 +1322,12 @@ address TemplateInterpreterGenerator::generate_native_entry(bool synchronized) { // convenient and the slow signature handler can use this same frame // anchor. + bool support_vthread_preemption = Continuations::enabled() && LockingMode != LM_LEGACY; + // We have a TOP_IJAVA_FRAME here, which belongs to us. - __ set_top_ijava_frame_at_SP_as_last_Java_frame(R1_SP, R12_scratch2/*tmp*/); + Label last_java_pc; + Label *resume_pc = support_vthread_preemption ? &last_java_pc : nullptr; + __ set_top_ijava_frame_at_SP_as_last_Java_frame(R1_SP, R3_ARG1/*tmp*/, resume_pc); // Now the interpreter frame (and its call chain) have been // invalidated and flushed. We are now protected against eager @@ -1335,16 +1346,11 @@ address TemplateInterpreterGenerator::generate_native_entry(bool synchronized) { __ call_stub(signature_handler_fd); - // Remove the register parameter varargs slots we allocated in - // compute_interpreter_state. SP+16 ends up pointing to the ABI - // outgoing argument area. - // - // Not needed on PPC64. - //__ add(SP, SP, Argument::n_int_register_parameters_c*BytesPerWord); - - assert(result_handler_addr->is_nonvolatile(), "result_handler_addr must be in a non-volatile register"); + assert(__ nonvolatile_accross_vthread_preemtion(result_handler_addr), + "result_handler_addr not preserved"); // Save across call to native method. __ mr(result_handler_addr, R3_RET); + __ ld(R11_scratch1, _abi0(callers_sp), R1_SP); // load FP __ isync(); // Acquire signature handler before trying to fetch the native entry point and klass mirror. @@ -1358,12 +1364,11 @@ address TemplateInterpreterGenerator::generate_native_entry(bool synchronized) { __ testbitdi(CCR0, R0, access_flags, JVM_ACC_STATIC_BIT); __ bfalse(CCR0, method_is_not_static); - __ ld(R11_scratch1, _abi0(callers_sp), R1_SP); - // Load mirror from interpreter frame. - __ ld(R12_scratch2, _ijava_state_neg(mirror), R11_scratch1); + // Load mirror from interpreter frame (FP in R11_scratch1) + __ ld(R21_tmp1, _ijava_state_neg(mirror), R11_scratch1); // R4_ARG2 = &state->_oop_temp; __ addi(R4_ARG2, R11_scratch1, _ijava_state_neg(oop_tmp)); - __ std(R12_scratch2/*mirror*/, _ijava_state_neg(oop_tmp), R11_scratch1); + __ std(R21_tmp1/*mirror*/, _ijava_state_neg(oop_tmp), R11_scratch1); BIND(method_is_not_static); } @@ -1397,7 +1402,18 @@ address TemplateInterpreterGenerator::generate_native_entry(bool synchronized) { // Call the native method. Argument registers must not have been // overwritten since "__ call_stub(signature_handler);" (except for // ARG1 and ARG2 for static methods). + + if (support_vthread_preemption) { + // result_handler_addr is a nonvolatile register. Its value will be preserved across + // the native call but only if the call isn't preempted. To preserve its value even + // in the case of preemption we save it in the lresult slot. It is restored at + // resume_pc if, and only if the call was preempted. This works because only + // j.l.Object::wait calls are preempted which don't return a result. + __ std(result_handler_addr, _ijava_state_neg(lresult), R11_scratch1); + } + __ push_cont_fastpath(); __ call_c(native_method_fd); + __ pop_cont_fastpath(); __ li(R0, 0); __ ld(R11_scratch1, 0, R1_SP); @@ -1495,6 +1511,35 @@ address TemplateInterpreterGenerator::generate_native_entry(bool synchronized) { __ lwsync(); // Acquire safepoint and suspend state, release thread state. __ stw(R0/*thread_state*/, thread_(thread_state)); + if (support_vthread_preemption) { + // Check preemption for Object.wait() + Label not_preempted; + __ ld(R0, in_bytes(JavaThread::preempt_alternate_return_offset()), R16_thread); + __ cmpdi(CCR0, R0, 0); + __ beq(CCR0, not_preempted); + __ mtlr(R0); + __ li(R0, 0); + __ std(R0, in_bytes(JavaThread::preempt_alternate_return_offset()), R16_thread); + __ blr(); + + // Execution will be resumed here when the vthread becomes runnable again. + __ bind(*resume_pc); + __ restore_after_resume(R11_scratch1 /* fp */); + // We saved the result handler before the call + __ ld(result_handler_addr, _ijava_state_neg(lresult), R11_scratch1); +#ifdef ASSERT + // Clobber result slots. Only native methods returning void can be preemted currently. + __ load_const(R3_RET, UCONST64(0xbad01001)); + __ std(R3_RET, _ijava_state_neg(lresult), R11_scratch1); + __ std(R3_RET, _ijava_state_neg(fresult), R11_scratch1); + // reset_last_Java_frame() below asserts that a last java sp is set + __ asm_assert_mem8_is_zero(in_bytes(JavaThread::last_Java_sp_offset()), + R16_thread, FILE_AND_LINE ": Last java sp should not be set when resuming"); + __ std(R3_RET, in_bytes(JavaThread::last_Java_sp_offset()), R16_thread); +#endif + __ bind(not_preempted); + } + if (CheckJNICalls) { // clear_pending_jni_exception_check __ load_const_optimized(R0, 0L); diff --git a/src/hotspot/cpu/ppc/templateTable_ppc_64.cpp b/src/hotspot/cpu/ppc/templateTable_ppc_64.cpp index a55f30eb67d..0e88b2d3eb4 100644 --- a/src/hotspot/cpu/ppc/templateTable_ppc_64.cpp +++ b/src/hotspot/cpu/ppc/templateTable_ppc_64.cpp @@ -3840,8 +3840,9 @@ void TemplateTable::_new() { // Init1: Zero out newly allocated memory. // Initialize remaining object fields. Register Rbase = Rtags; - __ addi(Rinstance_size, Rinstance_size, 7 - (int)sizeof(oopDesc)); - __ addi(Rbase, RallocatedObject, sizeof(oopDesc)); + int header_size = oopDesc::header_size() * HeapWordSize; + __ addi(Rinstance_size, Rinstance_size, 7 - header_size); + __ addi(Rbase, RallocatedObject, header_size); __ srdi(Rinstance_size, Rinstance_size, 3); // Clear out object skipping header. Takes also care of the zero length case. @@ -3851,12 +3852,15 @@ void TemplateTable::_new() { // -------------------------------------------------------------------------- // Init2: Initialize the header: mark, klass // Init mark. - __ load_const_optimized(Rscratch, markWord::prototype().value(), R0); - __ std(Rscratch, oopDesc::mark_offset_in_bytes(), RallocatedObject); - - // Init klass. - __ store_klass_gap(RallocatedObject); - __ store_klass(RallocatedObject, RinstanceKlass, Rscratch); // klass (last for cms) + if (UseCompactObjectHeaders) { + __ ld(Rscratch, in_bytes(Klass::prototype_header_offset()), RinstanceKlass); + __ std(Rscratch, oopDesc::mark_offset_in_bytes(), RallocatedObject); + } else { + __ load_const_optimized(Rscratch, markWord::prototype().value(), R0); + __ std(Rscratch, oopDesc::mark_offset_in_bytes(), RallocatedObject); + __ store_klass_gap(RallocatedObject); + __ store_klass(RallocatedObject, RinstanceKlass, Rscratch); + } // Check and trigger dtrace event. if (DTraceAllocProbes) { diff --git a/src/hotspot/cpu/ppc/vm_version_ppc.cpp b/src/hotspot/cpu/ppc/vm_version_ppc.cpp index 20578ed3b8b..37e15548b4f 100644 --- a/src/hotspot/cpu/ppc/vm_version_ppc.cpp +++ b/src/hotspot/cpu/ppc/vm_version_ppc.cpp @@ -110,6 +110,17 @@ void VM_Version::initialize() { FLAG_SET_ERGO(TrapBasedRangeChecks, false); } + // Power7 and later. + if (PowerArchitecturePPC64 > 6) { + if (FLAG_IS_DEFAULT(UsePopCountInstruction)) { + FLAG_SET_ERGO(UsePopCountInstruction, true); + } + } + + if (!VM_Version::has_isel() && FLAG_IS_DEFAULT(ConditionalMoveLimit)) { + FLAG_SET_ERGO(ConditionalMoveLimit, 0); + } + if (PowerArchitecturePPC64 >= 8) { if (FLAG_IS_DEFAULT(SuperwordUseVSX)) { FLAG_SET_ERGO(SuperwordUseVSX, true); @@ -169,6 +180,17 @@ void VM_Version::initialize() { FLAG_SET_DEFAULT(UseByteReverseInstructions, false); } } + + if (OptimizeFill) { + warning("OptimizeFill is not supported on this CPU."); + FLAG_SET_DEFAULT(OptimizeFill, false); + } + + if (OptoScheduling) { + // The OptoScheduling information is not maintained in ppd.ad. + warning("OptoScheduling is not supported on this CPU."); + FLAG_SET_DEFAULT(OptoScheduling, false); + } #endif // Create and print feature-string. diff --git a/src/hotspot/cpu/riscv/c1_CodeStubs_riscv.cpp b/src/hotspot/cpu/riscv/c1_CodeStubs_riscv.cpp index 4c1d4ad9f69..46af27a011f 100644 --- a/src/hotspot/cpu/riscv/c1_CodeStubs_riscv.cpp +++ b/src/hotspot/cpu/riscv/c1_CodeStubs_riscv.cpp @@ -316,7 +316,7 @@ void ArrayCopyStub::emit_code(LIR_Assembler* ce) { relocInfo::static_call_type); address call = __ reloc_call(resolve); if (call == nullptr) { - ce->bailout("trampoline stub overflow"); + ce->bailout("reloc call address stub overflow"); return; } ce->add_call_info_here(info()); diff --git a/src/hotspot/cpu/riscv/c1_LIRAssembler_arraycopy_riscv.cpp b/src/hotspot/cpu/riscv/c1_LIRAssembler_arraycopy_riscv.cpp index a45add1032a..7d673383cad 100644 --- a/src/hotspot/cpu/riscv/c1_LIRAssembler_arraycopy_riscv.cpp +++ b/src/hotspot/cpu/riscv/c1_LIRAssembler_arraycopy_riscv.cpp @@ -194,7 +194,10 @@ void LIR_Assembler::arraycopy_type_check(Register src, Register src_pos, Registe // We don't know the array types are compatible if (basic_type != T_OBJECT) { // Simple test for basic type arrays - if (UseCompressedClassPointers) { + if (UseCompactObjectHeaders) { + __ load_narrow_klass_compact(tmp, src); + __ load_narrow_klass_compact(t0, dst); + } else if (UseCompressedClassPointers) { __ lwu(tmp, Address(src, oopDesc::klass_offset_in_bytes())); __ lwu(t0, Address(dst, oopDesc::klass_offset_in_bytes())); } else { @@ -244,7 +247,6 @@ void LIR_Assembler::arraycopy_type_check(Register src, Register src_pos, Registe void LIR_Assembler::arraycopy_assert(Register src, Register dst, Register tmp, ciArrayKlass *default_type, int flags) { assert(default_type != nullptr, "null default_type!"); BasicType basic_type = default_type->element_type()->basic_type(); - if (basic_type == T_ARRAY) { basic_type = T_OBJECT; } if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) { // Sanity check the known type with the incoming class. For the @@ -261,25 +263,10 @@ void LIR_Assembler::arraycopy_assert(Register src, Register dst, Register tmp, c } if (basic_type != T_OBJECT) { - if (UseCompressedClassPointers) { - __ lwu(t0, Address(dst, oopDesc::klass_offset_in_bytes())); - } else { - __ ld(t0, Address(dst, oopDesc::klass_offset_in_bytes())); - } - __ bne(tmp, t0, halt); - if (UseCompressedClassPointers) { - __ lwu(t0, Address(src, oopDesc::klass_offset_in_bytes())); - } else { - __ ld(t0, Address(src, oopDesc::klass_offset_in_bytes())); - } - __ beq(tmp, t0, known_ok); + __ cmp_klass_compressed(dst, tmp, t0, halt, false); + __ cmp_klass_compressed(src, tmp, t0, known_ok, true); } else { - if (UseCompressedClassPointers) { - __ lwu(t0, Address(dst, oopDesc::klass_offset_in_bytes())); - } else { - __ ld(t0, Address(dst, oopDesc::klass_offset_in_bytes())); - } - __ beq(tmp, t0, known_ok); + __ cmp_klass_compressed(dst, tmp, t0, known_ok, true); __ beq(src, dst, known_ok); } __ bind(halt); diff --git a/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.cpp b/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.cpp index 542f8d8ba58..eb715227f7e 100644 --- a/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.cpp +++ b/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.cpp @@ -1346,7 +1346,7 @@ void LIR_Assembler::align_call(LIR_Code code) { void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) { address call = __ reloc_call(Address(op->addr(), rtype)); if (call == nullptr) { - bailout("trampoline stub overflow"); + bailout("reloc call address stub overflow"); return; } add_call_info(code_offset(), op->info()); @@ -1356,7 +1356,7 @@ void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) { void LIR_Assembler::ic_call(LIR_OpJavaCall* op) { address call = __ ic_call(op->addr()); if (call == nullptr) { - bailout("trampoline stub overflow"); + bailout("reloc call address stub overflow"); return; } add_call_info(code_offset(), op->info()); @@ -1518,12 +1518,7 @@ void LIR_Assembler::emit_load_klass(LIR_OpLoadKlass* op) { add_debug_info_for_null_check_here(info); } - if (UseCompressedClassPointers) { - __ lwu(result, Address(obj, oopDesc::klass_offset_in_bytes())); - __ decode_klass_not_null(result); - } else { - __ ld(result, Address(obj, oopDesc::klass_offset_in_bytes())); - } + __ load_klass(result, obj); } void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) { diff --git a/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.hpp b/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.hpp index 4ce7eb2bf73..22f1d694b92 100644 --- a/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.hpp +++ b/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.hpp @@ -65,17 +65,16 @@ friend class ArrayCopyStub; void deoptimize_trap(CodeEmitInfo *info); enum { - // See emit_static_call_stub for detail - // CompiledDirectCall::to_interp_stub_size() (14) + CompiledDirectCall::to_trampoline_stub_size() (1 + 3 + address) - _call_stub_size = 14 * MacroAssembler::instruction_size + - (MacroAssembler::instruction_size + MacroAssembler::NativeShortCall::trampoline_size), + // call stub: CompiledDirectCall::to_interp_stub_size() + + // CompiledDirectCall::to_trampoline_stub_size() + _call_stub_size = 11 * MacroAssembler::instruction_size + + 1 * MacroAssembler::instruction_size + wordSize, // See emit_exception_handler for detail - // verify_not_null_oop + far_call + should_not_reach_here + invalidate_registers(DEBUG_ONLY) - _exception_handler_size = DEBUG_ONLY(584) NOT_DEBUG(548), // or smaller + _exception_handler_size = DEBUG_ONLY(256) NOT_DEBUG(32), // or smaller // See emit_deopt_handler for detail - // auipc (1) + far_jump (6 or 2) + // auipc (1) + far_jump (2) _deopt_handler_size = 1 * MacroAssembler::instruction_size + - 6 * MacroAssembler::instruction_size // or smaller + 2 * MacroAssembler::instruction_size }; void check_conflict(ciKlass* exact_klass, intptr_t current_klass, Register tmp, diff --git a/src/hotspot/cpu/riscv/c1_MacroAssembler_riscv.cpp b/src/hotspot/cpu/riscv/c1_MacroAssembler_riscv.cpp index 1e4b66069ee..ed932dddcd8 100644 --- a/src/hotspot/cpu/riscv/c1_MacroAssembler_riscv.cpp +++ b/src/hotspot/cpu/riscv/c1_MacroAssembler_riscv.cpp @@ -83,8 +83,8 @@ int C1_MacroAssembler::lock_object(Register hdr, Register obj, Register disp_hdr // displaced header address in the object header - if it is not the same, get the // object header instead la(temp, Address(obj, hdr_offset)); - cmpxchgptr(hdr, disp_hdr, temp, t1, done, /*fallthough*/nullptr); // if the object header was the same, we're done + cmpxchgptr(hdr, disp_hdr, temp, t1, done, /*fallthough*/nullptr); // if the object header was not the same, it is now in the hdr register // => test if it is a stack pointer into the same stack (recursive locking), i.e.: // @@ -106,11 +106,12 @@ int C1_MacroAssembler::lock_object(Register hdr, Register obj, Register disp_hdr sd(hdr, Address(disp_hdr, 0)); // otherwise we don't care about the result and handle locking via runtime call bnez(hdr, slow_case, /* is_far */ true); + // done bind(done); + inc_held_monitor_count(t0); } - increment(Address(xthread, JavaThread::held_monitor_count_offset())); return null_check_offset; } @@ -146,11 +147,11 @@ void C1_MacroAssembler::unlock_object(Register hdr, Register obj, Register disp_ } else { cmpxchgptr(disp_hdr, hdr, obj, t1, done, &slow_case); } + // done bind(done); + dec_held_monitor_count(t0); } - - decrement(Address(xthread, JavaThread::held_monitor_count_offset())); } // Defines obj, preserves var_size_in_bytes @@ -164,15 +165,19 @@ void C1_MacroAssembler::try_allocate(Register obj, Register var_size_in_bytes, i void C1_MacroAssembler::initialize_header(Register obj, Register klass, Register len, Register tmp1, Register tmp2) { assert_different_registers(obj, klass, len, tmp1, tmp2); - // This assumes that all prototype bits fitr in an int32_t - mv(tmp1, (int32_t)(intptr_t)markWord::prototype().value()); - sd(tmp1, Address(obj, oopDesc::mark_offset_in_bytes())); - - if (UseCompressedClassPointers) { // Take care not to kill klass - encode_klass_not_null(tmp1, klass, tmp2); - sw(tmp1, Address(obj, oopDesc::klass_offset_in_bytes())); + if (UseCompactObjectHeaders) { + ld(tmp1, Address(klass, Klass::prototype_header_offset())); + sd(tmp1, Address(obj, oopDesc::mark_offset_in_bytes())); } else { - sd(klass, Address(obj, oopDesc::klass_offset_in_bytes())); + // This assumes that all prototype bits fitr in an int32_t + mv(tmp1, checked_cast(markWord::prototype().value())); + sd(tmp1, Address(obj, oopDesc::mark_offset_in_bytes())); + if (UseCompressedClassPointers) { // Take care not to kill klass + encode_klass_not_null(tmp1, klass, tmp2); + sw(tmp1, Address(obj, oopDesc::klass_offset_in_bytes())); + } else { + sd(klass, Address(obj, oopDesc::klass_offset_in_bytes())); + } } if (len->is_valid()) { @@ -183,7 +188,7 @@ void C1_MacroAssembler::initialize_header(Register obj, Register klass, Register // Clear gap/first 4 bytes following the length field. sw(zr, Address(obj, base_offset)); } - } else if (UseCompressedClassPointers) { + } else if (UseCompressedClassPointers && !UseCompactObjectHeaders) { store_klass_gap(obj, zr); } } diff --git a/src/hotspot/cpu/riscv/c1_Runtime1_riscv.cpp b/src/hotspot/cpu/riscv/c1_Runtime1_riscv.cpp index 5e4031727c8..6f59f5c2b95 100644 --- a/src/hotspot/cpu/riscv/c1_Runtime1_riscv.cpp +++ b/src/hotspot/cpu/riscv/c1_Runtime1_riscv.cpp @@ -165,7 +165,7 @@ int StubAssembler::call_RT(Register oop_result, Register metadata_result, addres } enum return_state_t { - does_not_return, requires_return + does_not_return, requires_return, requires_pop_epilogue_return }; // Implementation of StubFrame @@ -173,7 +173,7 @@ enum return_state_t { class StubFrame: public StackObj { private: StubAssembler* _sasm; - bool _return_state; + return_state_t _return_state; public: StubFrame(StubAssembler* sasm, const char* name, bool must_gc_arguments, return_state_t return_state=requires_return); @@ -187,8 +187,18 @@ void StubAssembler::prologue(const char* name, bool must_gc_arguments) { enter(); } -void StubAssembler::epilogue() { - leave(); +void StubAssembler::epilogue(bool use_pop) { + // Avoid using a leave instruction when this frame may + // have been frozen, since the current value of fp + // restored from the stub would be invalid. We still + // must restore the fp value saved on enter though. + if (use_pop) { + ld(fp, Address(sp)); + ld(ra, Address(sp, wordSize)); + addi(sp, sp, 2 * wordSize); + } else { + leave(); + } ret(); } @@ -208,10 +218,10 @@ void StubFrame::load_argument(int offset_in_words, Register reg) { StubFrame::~StubFrame() { - if (_return_state == requires_return) { - __ epilogue(); - } else { + if (_return_state == does_not_return) { __ should_not_reach_here(); + } else { + __ epilogue(_return_state == requires_pop_epilogue_return); } _sasm = nullptr; } @@ -266,6 +276,10 @@ static OopMap* generate_oop_map(StubAssembler* sasm, bool save_fpu_registers) { r->as_VMReg()); } + int sp_offset = cpu_reg_save_offsets[xthread->encoding()]; + oop_map->set_callee_saved(VMRegImpl::stack2reg(sp_offset), + xthread->as_VMReg()); + // fpu_regs if (save_fpu_registers) { for (int i = 0; i < FrameMap::nof_fpu_regs; i++) { @@ -354,6 +368,16 @@ void Runtime1::initialize_pd() { } } +// return: offset in 64-bit words. +uint Runtime1::runtime_blob_current_thread_offset(frame f) { + CodeBlob* cb = f.cb(); + assert(cb == Runtime1::blob_for(C1StubId::monitorenter_id) || + cb == Runtime1::blob_for(C1StubId::monitorenter_nofpu_id), "must be"); + assert(cb != nullptr && cb->is_runtime_stub(), "invalid frame"); + int offset = cpu_reg_save_offsets[xthread->encoding()]; + return offset / 2; // SP offsets are in halfwords +} + // target: the entry point of the method that creates and posts the exception oop // has_argument: true if the exception needs arguments (passed in t0 and t1) @@ -879,7 +903,7 @@ OopMapSet* Runtime1::generate_code_for(C1StubId id, StubAssembler* sasm) { // fall through case C1StubId::monitorenter_id: { - StubFrame f(sasm, "monitorenter", dont_gc_arguments); + StubFrame f(sasm, "monitorenter", dont_gc_arguments, requires_pop_epilogue_return); OopMap* map = save_live_registers(sasm, save_fpu_registers); assert_cond(map != nullptr); diff --git a/src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.cpp b/src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.cpp index 0ffdcbca723..7b0316e208f 100644 --- a/src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.cpp +++ b/src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.cpp @@ -45,7 +45,7 @@ #define BIND(label) bind(label); BLOCK_COMMENT(#label ":") void C2_MacroAssembler::fast_lock(Register objectReg, Register boxReg, - Register tmp1Reg, Register tmp2Reg, Register tmp3Reg) { + Register tmp1Reg, Register tmp2Reg, Register tmp3Reg, Register tmp4Reg) { // Use cr register to indicate the fast_lock result: zero for success; non-zero for failure. Register flag = t1; Register oop = objectReg; @@ -104,9 +104,9 @@ void C2_MacroAssembler::fast_lock(Register objectReg, Register boxReg, // markWord of object (disp_hdr) with the stack pointer. sub(disp_hdr, disp_hdr, sp); mv(tmp, (intptr_t) (~(os::vm_page_size()-1) | (uintptr_t)markWord::lock_mask_in_place)); - // If (mark & lock_mask) == 0 and mark - sp < page_size, we are stack-locking and goto label locked, - // hence we can store 0 as the displaced header in the box, which indicates that it is a - // recursive lock. + // If (mark & lock_mask) == 0 and mark - sp < page_size, we are stack-locking and goto label + // locked, hence we can store 0 as the displaced header in the box, which indicates that it + // is a recursive lock. andr(tmp/*==0?*/, disp_hdr, tmp); sd(tmp/*==0, perhaps*/, Address(box, BasicLock::displaced_header_offset_in_bytes())); beqz(tmp, locked); @@ -115,12 +115,12 @@ void C2_MacroAssembler::fast_lock(Register objectReg, Register boxReg, // Handle existing monitor. bind(object_has_monitor); - // The object's monitor m is unlocked iff m->owner == nullptr, - // otherwise m->owner may contain a thread or a stack address. - // - // Try to CAS m->owner from null to current thread. + + // Try to CAS owner (no owner => current thread's _lock_id). add(tmp, disp_hdr, (in_bytes(ObjectMonitor::owner_offset()) - markWord::monitor_value)); - cmpxchg(/*memory address*/tmp, /*expected value*/zr, /*new value*/xthread, Assembler::int64, + Register tid = tmp4Reg; + ld(tid, Address(xthread, JavaThread::lock_id_offset())); + cmpxchg(/*memory address*/tmp, /*expected value*/zr, /*new value*/tid, Assembler::int64, Assembler::aq, Assembler::rl, /*result*/tmp3Reg); // cas succeeds if tmp3Reg == zr(expected) // Store a non-null value into the box to avoid looking like a re-entrant @@ -132,14 +132,16 @@ void C2_MacroAssembler::fast_lock(Register objectReg, Register boxReg, beqz(tmp3Reg, locked); // CAS success means locking succeeded - bne(tmp3Reg, xthread, slow_path); // Check for recursive locking + bne(tmp3Reg, tid, slow_path); // Check for recursive locking // Recursive lock case increment(Address(disp_hdr, in_bytes(ObjectMonitor::recursions_offset()) - markWord::monitor_value), 1, tmp2Reg, tmp3Reg); bind(locked); mv(flag, zr); - increment(Address(xthread, JavaThread::held_monitor_count_offset()), 1, tmp2Reg, tmp3Reg); + if (LockingMode == LM_LEGACY) { + inc_held_monitor_count(t0); + } #ifdef ASSERT // Check that locked label is reached with flag == 0. @@ -253,7 +255,9 @@ void C2_MacroAssembler::fast_unlock(Register objectReg, Register boxReg, bind(unlocked); mv(flag, zr); - decrement(Address(xthread, JavaThread::held_monitor_count_offset()), 1, tmp1Reg, tmp2Reg); + if (LockingMode == LM_LEGACY) { + dec_held_monitor_count(t0); + } #ifdef ASSERT // Check that unlocked label is reached with flag == 0. @@ -273,12 +277,12 @@ void C2_MacroAssembler::fast_unlock(Register objectReg, Register boxReg, } void C2_MacroAssembler::fast_lock_lightweight(Register obj, Register box, - Register tmp1, Register tmp2, Register tmp3) { + Register tmp1, Register tmp2, Register tmp3, Register tmp4) { // Flag register, zero for success; non-zero for failure. Register flag = t1; assert(LockingMode == LM_LIGHTWEIGHT, "must be"); - assert_different_registers(obj, box, tmp1, tmp2, tmp3, flag, t0); + assert_different_registers(obj, box, tmp1, tmp2, tmp3, tmp4, flag, t0); mv(flag, 1); @@ -349,6 +353,7 @@ void C2_MacroAssembler::fast_lock_lightweight(Register obj, Register box, bind(inflated); const Register tmp1_monitor = tmp1; + if (!UseObjectMonitorTable) { assert(tmp1_monitor == tmp1_mark, "should be the same here"); } else { @@ -395,13 +400,15 @@ void C2_MacroAssembler::fast_lock_lightweight(Register obj, Register box, // Compute owner address. la(tmp2_owner_addr, owner_address); - // CAS owner (null => current thread). - cmpxchg(/*addr*/ tmp2_owner_addr, /*expected*/ zr, /*new*/ xthread, Assembler::int64, + // Try to CAS owner (no owner => current thread's _lock_id). + Register tid = tmp4; + ld(tid, Address(xthread, JavaThread::lock_id_offset())); + cmpxchg(/*addr*/ tmp2_owner_addr, /*expected*/ zr, /*new*/ tid, Assembler::int64, /*acquire*/ Assembler::aq, /*release*/ Assembler::relaxed, /*result*/ tmp3_owner); beqz(tmp3_owner, monitor_locked); // Check if recursive. - bne(tmp3_owner, xthread, slow_path); + bne(tmp3_owner, tid, slow_path); // Recursive. increment(recursions_address, 1, tmp2, tmp3); @@ -414,7 +421,6 @@ void C2_MacroAssembler::fast_lock_lightweight(Register obj, Register box, bind(locked); mv(flag, zr); - increment(Address(xthread, JavaThread::held_monitor_count_offset()), 1, tmp2, tmp3); #ifdef ASSERT // Check that locked label is reached with flag == 0. @@ -586,7 +592,6 @@ void C2_MacroAssembler::fast_unlock_lightweight(Register obj, Register box, bind(unlocked); mv(flag, zr); - decrement(Address(xthread, JavaThread::held_monitor_count_offset()), 1, tmp2, tmp3); #ifdef ASSERT // Check that unlocked label is reached with flag == 0. @@ -3119,3 +3124,13 @@ void C2_MacroAssembler::extract_fp_v(FloatRegister dst, VectorRegister src, Basi vfmv_f_s(dst, tmp); } } + +void C2_MacroAssembler::load_narrow_klass_compact_c2(Register dst, Address src) { + // The incoming address is pointing into obj-start + klass_offset_in_bytes. We need to extract + // obj-start, so that we can load from the object's mark-word instead. Usually the address + // comes as obj-start in obj and klass_offset_in_bytes in disp. + assert(UseCompactObjectHeaders, "must"); + int offset = oopDesc::mark_offset_in_bytes() - oopDesc::klass_offset_in_bytes(); + ld(dst, Address(src.base(), src.offset() + offset)); + srli(dst, dst, markWord::klass_shift); +} diff --git a/src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.hpp b/src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.hpp index 38351565cc6..8f9d9cd2ccd 100644 --- a/src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.hpp +++ b/src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.hpp @@ -44,11 +44,15 @@ public: // Code used by cmpFastLock and cmpFastUnlock mach instructions in .ad file. - void fast_lock(Register object, Register box, Register tmp1, Register tmp2, Register tmp3); + void fast_lock(Register object, Register box, + Register tmp1, Register tmp2, Register tmp3, Register tmp4); void fast_unlock(Register object, Register box, Register tmp1, Register tmp2); + // Code used by cmpFastLockLightweight and cmpFastUnlockLightweight mach instructions in .ad file. - void fast_lock_lightweight(Register object, Register box, Register tmp1, Register tmp2, Register tmp3); - void fast_unlock_lightweight(Register object, Register box, Register tmp1, Register tmp2, Register tmp3); + void fast_lock_lightweight(Register object, Register box, + Register tmp1, Register tmp2, Register tmp3, Register tmp4); + void fast_unlock_lightweight(Register object, Register box, + Register tmp1, Register tmp2, Register tmp3); void string_compare(Register str1, Register str2, Register cnt1, Register cnt2, Register result, @@ -277,4 +281,6 @@ void extract_v(Register dst, VectorRegister src, BasicType bt, int idx, VectorRegister tmp); void extract_fp_v(FloatRegister dst, VectorRegister src, BasicType bt, int idx, VectorRegister tmp); + void load_narrow_klass_compact_c2(Register dst, Address src); + #endif // CPU_RISCV_C2_MACROASSEMBLER_RISCV_HPP diff --git a/src/hotspot/cpu/riscv/codeBuffer_riscv.cpp b/src/hotspot/cpu/riscv/codeBuffer_riscv.cpp index e772959ed08..e99183a5b5d 100644 --- a/src/hotspot/cpu/riscv/codeBuffer_riscv.cpp +++ b/src/hotspot/cpu/riscv/codeBuffer_riscv.cpp @@ -28,61 +28,6 @@ #include "asm/codeBuffer.inline.hpp" #include "asm/macroAssembler.hpp" -void CodeBuffer::share_trampoline_for(address dest, int caller_offset) { - if (_shared_trampoline_requests == nullptr) { - constexpr unsigned init_size = 8; - constexpr unsigned max_size = 256; - _shared_trampoline_requests = new (mtCompiler)SharedTrampolineRequests(init_size, max_size); - } - - bool created; - Offsets* offsets = _shared_trampoline_requests->put_if_absent(dest, &created); - if (created) { - _shared_trampoline_requests->maybe_grow(); - } - offsets->add(caller_offset); - _finalize_stubs = true; -} - -#define __ masm. - -static bool emit_shared_trampolines(CodeBuffer* cb, CodeBuffer::SharedTrampolineRequests* requests) { - if (requests == nullptr) { - return true; - } - assert(UseTrampolines, "We are not using trampolines"); - - MacroAssembler masm(cb); - - auto emit = [&](address dest, const CodeBuffer::Offsets &offsets) { - assert(cb->stubs()->remaining() >= MacroAssembler::max_reloc_call_stub_size(), "pre-allocated trampolines"); - LinkedListIterator it(offsets.head()); - int offset = *it.next(); - address stub = __ emit_trampoline_stub(offset, dest); - assert(stub, "pre-allocated trampolines"); - - address reloc_pc = cb->stubs()->end() - MacroAssembler::NativeShortCall::trampoline_size; - while (!it.is_empty()) { - offset = *it.next(); - address caller_pc = cb->insts()->start() + offset; - cb->stubs()->relocate(reloc_pc, trampoline_stub_Relocation::spec(caller_pc)); - } - return true; - }; - - assert(requests->number_of_entries() >= 1, "at least one"); - const int total_requested_size = MacroAssembler::max_reloc_call_stub_size() * requests->number_of_entries(); - if (cb->stubs()->maybe_expand_to_ensure_remaining(total_requested_size) && cb->blob() == nullptr) { - return false; - } - - requests->iterate(emit); - return true; -} - -#undef __ - bool CodeBuffer::pd_finalize_stubs() { - return emit_shared_stubs_to_interp(this, _shared_stub_to_interp_requests) - && emit_shared_trampolines(this, _shared_trampoline_requests); + return emit_shared_stubs_to_interp(this, _shared_stub_to_interp_requests); } diff --git a/src/hotspot/cpu/riscv/codeBuffer_riscv.hpp b/src/hotspot/cpu/riscv/codeBuffer_riscv.hpp index de70bc2ecc9..0d90c7d866e 100644 --- a/src/hotspot/cpu/riscv/codeBuffer_riscv.hpp +++ b/src/hotspot/cpu/riscv/codeBuffer_riscv.hpp @@ -33,8 +33,6 @@ public: void flush_bundle(bool start_new_bundle) {} - static bool supports_shared_stubs() { return UseTrampolines; } - - void share_trampoline_for(address dest, int caller_offset); + static bool supports_shared_stubs() { return false; } #endif // CPU_RISCV_CODEBUFFER_RISCV_HPP diff --git a/src/hotspot/cpu/riscv/compiledIC_riscv.cpp b/src/hotspot/cpu/riscv/compiledIC_riscv.cpp index 4bbea8f356f..c95521a95bd 100644 --- a/src/hotspot/cpu/riscv/compiledIC_riscv.cpp +++ b/src/hotspot/cpu/riscv/compiledIC_riscv.cpp @@ -69,9 +69,9 @@ int CompiledDirectCall::to_interp_stub_size() { } int CompiledDirectCall::to_trampoline_stub_size() { - // We count instructions and an additional alignment nop. - // Trampoline stubs are always word aligned. - return MacroAssembler::max_reloc_call_stub_size(); + // We count size of target address and an additional alignment nop. + // Reloc call address stubs are always word aligned. + return MacroAssembler::max_reloc_call_address_stub_size(); } // Relocation entries for call stub, compiled java to interpreter. diff --git a/src/hotspot/cpu/riscv/compressedKlass_riscv.cpp b/src/hotspot/cpu/riscv/compressedKlass_riscv.cpp index cffadb4189b..7c8d6b8f5bb 100644 --- a/src/hotspot/cpu/riscv/compressedKlass_riscv.cpp +++ b/src/hotspot/cpu/riscv/compressedKlass_riscv.cpp @@ -56,9 +56,9 @@ char* CompressedKlassPointers::reserve_address_space_for_compressed_classes(size result = reserve_address_space_for_zerobased_encoding(size, aslr); } - // Failing that, optimize for case (3) - a base with only bits set between [33-44) + // Failing that, optimize for case (3) - a base with only bits set between [32-44) if (result == nullptr) { - const uintptr_t from = nth_bit(32 + (optimize_for_zero_base ? LogKlassAlignmentInBytes : 0)); + const uintptr_t from = nth_bit(32); constexpr uintptr_t to = nth_bit(44); constexpr size_t alignment = nth_bit(32); result = reserve_address_space_X(from, to, size, alignment, aslr); diff --git a/src/hotspot/cpu/riscv/continuationFreezeThaw_riscv.inline.hpp b/src/hotspot/cpu/riscv/continuationFreezeThaw_riscv.inline.hpp index cea43511514..283334bbf41 100644 --- a/src/hotspot/cpu/riscv/continuationFreezeThaw_riscv.inline.hpp +++ b/src/hotspot/cpu/riscv/continuationFreezeThaw_riscv.inline.hpp @@ -127,6 +127,11 @@ void FreezeBase::adjust_interpreted_frame_unextended_sp(frame& f) { } } +inline void FreezeBase::prepare_freeze_interpreted_top_frame(frame& f) { + assert(f.interpreter_frame_last_sp() == nullptr, "should be null for top frame"); + f.interpreter_frame_set_last_sp(f.unextended_sp()); +} + inline void FreezeBase::relativize_interpreted_frame_metadata(const frame& f, const frame& hf) { assert(hf.fp() == hf.unextended_sp() + (f.fp() - f.unextended_sp()), ""); assert((f.at(frame::interpreter_frame_last_sp_offset) != 0) @@ -147,10 +152,16 @@ inline void FreezeBase::relativize_interpreted_frame_metadata(const frame& f, co // extended_sp is already relativized by TemplateInterpreterGenerator::generate_normal_entry or // AbstractInterpreter::layout_activation + // The interpreter native wrapper code adds space in the stack equal to size_of_parameters() + // after the fixed part of the frame. For wait0 this is equal to 3 words (this + long parameter). + // We adjust by this size since otherwise the saved last sp will be less than the extended_sp. + DEBUG_ONLY(Method* m = hf.interpreter_frame_method();) + DEBUG_ONLY(int extra_space = m->is_object_wait0() ? m->size_of_parameters() : 0;) + assert((hf.fp() - hf.unextended_sp()) == (f.fp() - f.unextended_sp()), ""); assert(hf.unextended_sp() == (intptr_t*)hf.at(frame::interpreter_frame_last_sp_offset), ""); assert(hf.unextended_sp() <= (intptr_t*)hf.at(frame::interpreter_frame_initial_sp_offset), ""); - assert(hf.unextended_sp() > (intptr_t*)hf.at(frame::interpreter_frame_extended_sp_offset), ""); + assert(hf.unextended_sp() + extra_space > (intptr_t*)hf.at(frame::interpreter_frame_extended_sp_offset), ""); assert(hf.fp() > (intptr_t*)hf.at(frame::interpreter_frame_initial_sp_offset), ""); #ifdef ASSERT if (f.interpreter_frame_method()->max_locals() > 0) { @@ -203,7 +214,8 @@ inline void Thaw::patch_caller_links(intptr_t* sp, intptr_t* bottom) { inline frame ThawBase::new_entry_frame() { intptr_t* sp = _cont.entrySP(); - return frame(sp, sp, _cont.entryFP(), _cont.entryPC()); // TODO PERF: This finds code blob and computes deopt state + // TODO PERF: This finds code blob and computes deopt state + return frame(sp, sp, _cont.entryFP(), _cont.entryPC()); } template frame ThawBase::new_stack_frame(const frame& hf, frame& caller, bool bottom) { @@ -215,7 +227,6 @@ template frame ThawBase::new_stack_frame(const frame& hf, frame& // If caller is interpreted it already made room for the callee arguments int overlap = caller.is_interpreted_frame() ? ContinuationHelper::InterpretedFrame::stack_argsize(hf) : 0; const int fsize = (int)(ContinuationHelper::InterpretedFrame::frame_bottom(hf) - hf.unextended_sp() - overlap); - const int locals = hf.interpreter_frame_method()->max_locals(); intptr_t* frame_sp = caller.unextended_sp() - fsize; intptr_t* fp = frame_sp + (hf.fp() - heap_sp); if ((intptr_t)fp % frame::frame_alignment != 0) { @@ -237,7 +248,7 @@ template frame ThawBase::new_stack_frame(const frame& hf, frame& int fsize = FKind::size(hf); intptr_t* frame_sp = caller.unextended_sp() - fsize; if (bottom || caller.is_interpreted_frame()) { - int argsize = hf.compiled_frame_stack_argsize(); + int argsize = FKind::stack_argsize(hf); fsize += argsize; frame_sp -= argsize; @@ -252,13 +263,16 @@ template frame ThawBase::new_stack_frame(const frame& hf, frame& intptr_t* fp; if (PreserveFramePointer) { // we need to recreate a "real" frame pointer, pointing into the stack - fp = frame_sp + FKind::size(hf) - 2; + fp = frame_sp + FKind::size(hf) - frame::sender_sp_offset; } else { - fp = FKind::stub - ? frame_sp + fsize - 2 // On RISCV, this value is used for the safepoint stub - : *(intptr_t**)(hf.sp() - 2); // we need to re-read fp because it may be an oop and we might have fixed the frame. + fp = FKind::stub || FKind::native + // fp always points to the address above the pushed return pc. We need correct address. + ? frame_sp + fsize - frame::sender_sp_offset + // we need to re-read fp because it may be an oop and we might have fixed the frame. + : *(intptr_t**)(hf.sp() - 2); } - return frame(frame_sp, frame_sp, fp, hf.pc(), hf.cb(), hf.oop_map(), false); // TODO PERF : this computes deopt state; is it necessary? + // TODO PERF : this computes deopt state; is it necessary? + return frame(frame_sp, frame_sp, fp, hf.pc(), hf.cb(), hf.oop_map(), false); } } @@ -279,6 +293,22 @@ inline void ThawBase::patch_pd(frame& f, const frame& caller) { patch_callee_link(caller, caller.fp()); } +inline void ThawBase::patch_pd(frame& f, intptr_t* caller_sp) { + intptr_t* fp = caller_sp - frame::sender_sp_offset; + patch_callee_link(f, fp); +} + +inline intptr_t* ThawBase::push_cleanup_continuation() { + frame enterSpecial = new_entry_frame(); + intptr_t* sp = enterSpecial.sp(); + + sp[-1] = (intptr_t)ContinuationEntry::cleanup_pc(); + sp[-2] = (intptr_t)enterSpecial.fp(); + + log_develop_trace(continuations, preempt)("push_cleanup_continuation initial sp: " INTPTR_FORMAT " final sp: " INTPTR_FORMAT, p2i(sp + 2 * frame::metadata_words), p2i(sp)); + return sp; +} + inline void ThawBase::derelativize_interpreted_frame_metadata(const frame& hf, const frame& f) { // Make sure that last_sp is kept relativized. assert((intptr_t*)f.at_relative(frame::interpreter_frame_last_sp_offset) == f.unextended_sp(), ""); @@ -286,8 +316,11 @@ inline void ThawBase::derelativize_interpreted_frame_metadata(const frame& hf, c // Make sure that monitor_block_top is still relativized. assert(f.at_absolute(frame::interpreter_frame_monitor_block_top_offset) <= frame::interpreter_frame_initial_sp_offset, ""); + DEBUG_ONLY(Method* m = hf.interpreter_frame_method();) + DEBUG_ONLY(int extra_space = m->is_object_wait0() ? m->size_of_parameters() : 0;) // see comment in relativize_interpreted_frame_metadata() + // Make sure that extended_sp is kept relativized. - assert((intptr_t*)f.at_relative(frame::interpreter_frame_extended_sp_offset) < f.unextended_sp(), ""); + assert((intptr_t*)f.at_relative(frame::interpreter_frame_extended_sp_offset) < f.unextended_sp() + extra_space, ""); } #endif // CPU_RISCV_CONTINUATIONFREEZETHAW_RISCV_INLINE_HPP diff --git a/src/hotspot/cpu/riscv/continuationHelper_riscv.inline.hpp b/src/hotspot/cpu/riscv/continuationHelper_riscv.inline.hpp index 1c668531a07..8c0fb728dc7 100644 --- a/src/hotspot/cpu/riscv/continuationHelper_riscv.inline.hpp +++ b/src/hotspot/cpu/riscv/continuationHelper_riscv.inline.hpp @@ -40,6 +40,22 @@ static inline intptr_t** link_address(const frame& f) { : (intptr_t**)(f.unextended_sp() + f.cb()->frame_size() - 2); } +static inline void patch_return_pc_with_preempt_stub(frame& f) { + if (f.is_runtime_frame()) { + // Unlike x86 we don't know where in the callee frame the return pc is + // saved so we can't patch the return from the VM call back to Java. + // Instead, we will patch the return from the runtime stub back to the + // compiled method so that the target returns to the preempt cleanup stub. + intptr_t* caller_sp = f.sp() + f.cb()->frame_size(); + caller_sp[-1] = (intptr_t)StubRoutines::cont_preempt_stub(); + } else { + // The target will check for preemption once it returns to the interpreter + // or the native wrapper code and will manually jump to the preempt stub. + JavaThread *thread = JavaThread::current(); + thread->set_preempt_alternate_return(StubRoutines::cont_preempt_stub()); + } +} + inline int ContinuationHelper::frame_align_words(int size) { #ifdef _LP64 return size & 1; @@ -72,12 +88,12 @@ inline void ContinuationHelper::set_anchor_to_entry_pd(JavaFrameAnchor* anchor, anchor->set_last_Java_fp(entry->entry_fp()); } -#ifdef ASSERT inline void ContinuationHelper::set_anchor_pd(JavaFrameAnchor* anchor, intptr_t* sp) { intptr_t* fp = *(intptr_t**)(sp - 2); anchor->set_last_Java_fp(fp); } +#ifdef ASSERT inline bool ContinuationHelper::Frame::assert_frame_laid_out(frame f) { intptr_t* sp = f.sp(); address pc = *(address*)(sp - frame::sender_sp_ret_address_offset()); diff --git a/src/hotspot/cpu/riscv/frame_riscv.cpp b/src/hotspot/cpu/riscv/frame_riscv.cpp index 96dca4704ad..ecc450bd6b2 100644 --- a/src/hotspot/cpu/riscv/frame_riscv.cpp +++ b/src/hotspot/cpu/riscv/frame_riscv.cpp @@ -393,6 +393,36 @@ frame frame::sender_for_upcall_stub_frame(RegisterMap* map) const { return fr; } +#if defined(ASSERT) +static address get_register_address_in_stub(const frame& stub_fr, VMReg reg) { + RegisterMap map(nullptr, + RegisterMap::UpdateMap::include, + RegisterMap::ProcessFrames::skip, + RegisterMap::WalkContinuation::skip); + stub_fr.oop_map()->update_register_map(&stub_fr, &map); + return map.location(reg, stub_fr.sp()); +} +#endif + +JavaThread** frame::saved_thread_address(const frame& f) { + CodeBlob* cb = f.cb(); + assert(cb != nullptr && cb->is_runtime_stub(), "invalid frame"); + + JavaThread** thread_addr; +#ifdef COMPILER1 + if (cb == Runtime1::blob_for(C1StubId::monitorenter_id) || + cb == Runtime1::blob_for(C1StubId::monitorenter_nofpu_id)) { + thread_addr = (JavaThread**)(f.sp() + Runtime1::runtime_blob_current_thread_offset(f)); + } else +#endif + { + // c2 only saves rbp in the stub frame so nothing to do. + thread_addr = nullptr; + } + assert(get_register_address_in_stub(f, SharedRuntime::thread_register()) == (address)thread_addr, "wrong thread address"); + return thread_addr; +} + //------------------------------------------------------------------------------ // frame::verify_deopt_original_pc // diff --git a/src/hotspot/cpu/riscv/frame_riscv.hpp b/src/hotspot/cpu/riscv/frame_riscv.hpp index 3692c99193e..b4540c45ab8 100644 --- a/src/hotspot/cpu/riscv/frame_riscv.hpp +++ b/src/hotspot/cpu/riscv/frame_riscv.hpp @@ -111,7 +111,8 @@ sender_sp_offset = 0, // Interpreter frames - interpreter_frame_oop_temp_offset = 1, // for native calls only + interpreter_frame_result_handler_offset = 1, // for native calls only + interpreter_frame_oop_temp_offset = 0, // for native calls only interpreter_frame_sender_sp_offset = -3, // outgoing sp before a call to an invoked method diff --git a/src/hotspot/cpu/riscv/globals_riscv.hpp b/src/hotspot/cpu/riscv/globals_riscv.hpp index e0b7a400cb8..cc3c3b4a63c 100644 --- a/src/hotspot/cpu/riscv/globals_riscv.hpp +++ b/src/hotspot/cpu/riscv/globals_riscv.hpp @@ -98,12 +98,12 @@ define_pd_global(intx, InlineSmallCode, 1000); product(bool, UseRVA20U64, true, "Use RVA20U64 profile") \ product(bool, UseRVA22U64, false, EXPERIMENTAL, "Use RVA22U64 profile") \ product(bool, UseRVA23U64, false, EXPERIMENTAL, "Use RVA23U64 profile") \ - product(bool, UseRVC, false, "Use RVC instructions") \ - product(bool, UseRVV, false, "Use RVV instructions") \ - product(bool, UseZba, false, "Use Zba instructions") \ - product(bool, UseZbb, false, "Use Zbb instructions") \ - product(bool, UseZbs, false, "Use Zbs instructions") \ - product(bool, UseZfh, false, "Use Zfh instructions") \ + product(bool, UseRVC, false, DIAGNOSTIC, "Use RVC instructions") \ + product(bool, UseRVV, false, DIAGNOSTIC, "Use RVV instructions") \ + product(bool, UseZba, false, DIAGNOSTIC, "Use Zba instructions") \ + product(bool, UseZbb, false, DIAGNOSTIC, "Use Zbb instructions") \ + product(bool, UseZbs, false, DIAGNOSTIC, "Use Zbs instructions") \ + product(bool, UseZfh, false, DIAGNOSTIC, "Use Zfh instructions") \ product(bool, UseZacas, false, EXPERIMENTAL, "Use Zacas instructions") \ product(bool, UseZcb, false, EXPERIMENTAL, "Use Zcb instructions") \ product(bool, UseZic64b, false, EXPERIMENTAL, "Use Zic64b instructions") \ @@ -114,13 +114,11 @@ define_pd_global(intx, InlineSmallCode, 1000); "Use Zihintpause instructions") \ product(bool, UseZtso, false, EXPERIMENTAL, "Assume Ztso memory model") \ product(bool, UseZvbb, false, EXPERIMENTAL, "Use Zvbb instructions") \ - product(bool, UseZvfh, false, "Use Zvfh instructions") \ + product(bool, UseZvfh, false, DIAGNOSTIC, "Use Zvfh instructions") \ product(bool, UseZvkn, false, EXPERIMENTAL, \ "Use Zvkn group extension, Zvkned, Zvknhb, Zvkb, Zvkt") \ product(bool, UseRVVForBigIntegerShiftIntrinsics, true, \ "Use RVV instructions for left/right shift of BigInteger") \ - product(bool, UseTrampolines, false, EXPERIMENTAL, \ - "Far calls uses jal to trampoline.") \ product(bool, UseCtxFencei, false, EXPERIMENTAL, \ "Use PR_RISCV_CTX_SW_FENCEI_ON to avoid explicit icache flush") diff --git a/src/hotspot/cpu/riscv/interp_masm_riscv.cpp b/src/hotspot/cpu/riscv/interp_masm_riscv.cpp index a1d806a17b3..b00b4474d69 100644 --- a/src/hotspot/cpu/riscv/interp_masm_riscv.cpp +++ b/src/hotspot/cpu/riscv/interp_masm_riscv.cpp @@ -721,7 +721,7 @@ void InterpreterMacroAssembler::lock_object(Register lock_reg) { assert(lock_reg == c_rarg1, "The argument is only for looks. It must be c_rarg1"); if (LockingMode == LM_MONITOR) { - call_VM(noreg, + call_VM_preemptable(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter), lock_reg); } else { @@ -752,7 +752,7 @@ void InterpreterMacroAssembler::lock_object(Register lock_reg) if (LockingMode == LM_LIGHTWEIGHT) { lightweight_lock(lock_reg, obj_reg, tmp, tmp2, tmp3, slow_case); - j(count); + j(done); } else if (LockingMode == LM_LEGACY) { // Load (object->mark() | 1) into swap_reg ld(t0, Address(obj_reg, oopDesc::mark_offset_in_bytes())); @@ -781,19 +781,19 @@ void InterpreterMacroAssembler::lock_object(Register lock_reg) // Save the test result, for recursive case, the result is zero sd(swap_reg, Address(lock_reg, mark_offset)); - beqz(swap_reg, count); + bnez(swap_reg, slow_case); + + bind(count); + inc_held_monitor_count(t0); + j(done); } bind(slow_case); // Call the runtime routine for slow case - call_VM(noreg, + call_VM_preemptable(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter), lock_reg); - j(done); - - bind(count); - increment(Address(xthread, JavaThread::held_monitor_count_offset())); bind(done); } @@ -839,12 +839,10 @@ void InterpreterMacroAssembler::unlock_object(Register lock_reg) // Free entry sd(zr, Address(lock_reg, BasicObjectLock::obj_offset())); + Label slow_case; if (LockingMode == LM_LIGHTWEIGHT) { - Label slow_case; lightweight_unlock(obj_reg, header_reg, swap_reg, tmp_reg, slow_case); - j(count); - - bind(slow_case); + j(done); } else if (LockingMode == LM_LEGACY) { // Load the old header from BasicLock structure ld(header_reg, Address(swap_reg, @@ -854,20 +852,19 @@ void InterpreterMacroAssembler::unlock_object(Register lock_reg) beqz(header_reg, count); // Atomic swap back the old header - cmpxchg_obj_header(swap_reg, header_reg, obj_reg, tmp_reg, count, /*fallthrough*/nullptr); + cmpxchg_obj_header(swap_reg, header_reg, obj_reg, tmp_reg, count, &slow_case); + + bind(count); + dec_held_monitor_count(t0); + j(done); } + bind(slow_case); // Call the runtime routine for slow case. sd(obj_reg, Address(lock_reg, BasicObjectLock::obj_offset())); // restore obj call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorexit), lock_reg); - j(done); - - bind(count); - decrement(Address(xthread, JavaThread::held_monitor_count_offset())); - bind(done); - restore_bcp(); } } @@ -1575,6 +1572,55 @@ void InterpreterMacroAssembler::call_VM_base(Register oop_result, restore_locals(); } +void InterpreterMacroAssembler::call_VM_preemptable(Register oop_result, + address entry_point, + Register arg_1) { + assert(arg_1 == c_rarg1, ""); + Label resume_pc, not_preempted; + +#ifdef ASSERT + { + Label L; + ld(t0, Address(xthread, JavaThread::preempt_alternate_return_offset())); + beqz(t0, L); + stop("Should not have alternate return address set"); + bind(L); + } +#endif /* ASSERT */ + + // Force freeze slow path. + push_cont_fastpath(); + + // Make VM call. In case of preemption set last_pc to the one we want to resume to. + la(t0, resume_pc); + sd(t0, Address(xthread, JavaThread::last_Java_pc_offset())); + call_VM_base(oop_result, noreg, noreg, entry_point, 1, false /*check_exceptions*/); + + pop_cont_fastpath(); + + // Check if preempted. + ld(t1, Address(xthread, JavaThread::preempt_alternate_return_offset())); + beqz(t1, not_preempted); + sd(zr, Address(xthread, JavaThread::preempt_alternate_return_offset())); + jr(t1); + + // In case of preemption, this is where we will resume once we finally acquire the monitor. + bind(resume_pc); + restore_after_resume(false /* is_native */); + + bind(not_preempted); +} + +void InterpreterMacroAssembler::restore_after_resume(bool is_native) { + la(t1, ExternalAddress(Interpreter::cont_resume_interpreter_adapter())); + jalr(t1); + if (is_native) { + // On resume we need to set up stack as expected + push(dtos); + push(ltos); + } +} + void InterpreterMacroAssembler::profile_obj_type(Register obj, const Address& mdo_addr, Register tmp) { assert_different_registers(obj, tmp, t0, mdo_addr.base()); Label update, next, none; diff --git a/src/hotspot/cpu/riscv/interp_masm_riscv.hpp b/src/hotspot/cpu/riscv/interp_masm_riscv.hpp index d3959d68d9c..ddc0a94a6e4 100644 --- a/src/hotspot/cpu/riscv/interp_masm_riscv.hpp +++ b/src/hotspot/cpu/riscv/interp_masm_riscv.hpp @@ -59,6 +59,11 @@ class InterpreterMacroAssembler: public MacroAssembler { void load_earlyret_value(TosState state); + void call_VM_preemptable(Register oop_result, + address entry_point, + Register arg_1); + void restore_after_resume(bool is_native); + void jump_to_entry(address entry); virtual void check_and_handle_popframe(Register java_thread); diff --git a/src/hotspot/cpu/riscv/macroAssembler_riscv.cpp b/src/hotspot/cpu/riscv/macroAssembler_riscv.cpp index 1e7a3f65e8e..ac0445e5e4e 100644 --- a/src/hotspot/cpu/riscv/macroAssembler_riscv.cpp +++ b/src/hotspot/cpu/riscv/macroAssembler_riscv.cpp @@ -36,6 +36,7 @@ #include "gc/shared/collectedHeap.hpp" #include "interpreter/bytecodeHistogram.hpp" #include "interpreter/interpreter.hpp" +#include "interpreter/interpreterRuntime.hpp" #include "memory/resourceArea.hpp" #include "memory/universe.hpp" #include "oops/accessDecorators.hpp" @@ -226,6 +227,36 @@ void MacroAssembler::pop_cont_fastpath(Register java_thread) { bind(done); } +void MacroAssembler::inc_held_monitor_count(Register tmp) { + Address dst(xthread, JavaThread::held_monitor_count_offset()); + ld(tmp, dst); + addi(tmp, tmp, 1); + sd(tmp, dst); +#ifdef ASSERT + Label ok; + test_bit(tmp, tmp, 63); + beqz(tmp, ok); + STOP("assert(held monitor count overflow)"); + should_not_reach_here(); + bind(ok); +#endif +} + +void MacroAssembler::dec_held_monitor_count(Register tmp) { + Address dst(xthread, JavaThread::held_monitor_count_offset()); + ld(tmp, dst); + addi(tmp, tmp, -1); + sd(tmp, dst); +#ifdef ASSERT + Label ok; + test_bit(tmp, tmp, 63); + beqz(tmp, ok); + STOP("assert(held monitor count underflow)"); + should_not_reach_here(); + bind(ok); +#endif +} + int MacroAssembler::align(int modulus, int extra_offset) { CompressibleRegion cr(this); intptr_t before = offset(); @@ -407,6 +438,10 @@ void MacroAssembler::reset_last_Java_frame(bool clear_fp) { sd(zr, Address(xthread, JavaThread::last_Java_pc_offset())); } +static bool is_preemptable(address entry_point) { + return entry_point == CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter); +} + void MacroAssembler::call_VM_base(Register oop_result, Register java_thread, Register last_java_sp, @@ -436,7 +471,12 @@ void MacroAssembler::call_VM_base(Register oop_result, assert(last_java_sp != fp, "can't use fp"); Label l; - set_last_Java_frame(last_java_sp, fp, l, t0); + if (is_preemptable(entry_point)) { + // skip setting last_pc since we already set it to desired value. + set_last_Java_frame(last_java_sp, fp, noreg); + } else { + set_last_Java_frame(last_java_sp, fp, l, t0); + } // do the call, remove parameters MacroAssembler::call_VM_leaf_base(entry_point, number_of_arguments, &l); @@ -927,26 +967,6 @@ void MacroAssembler::li(Register Rd, int64_t imm) { } } -void MacroAssembler::load_link_jump(const address source, Register temp) { - assert(temp != noreg && temp != x0, "expecting a register"); - assert(temp != x5, "temp register must not be x5."); - assert_cond(source != nullptr); - int64_t distance = source - pc(); - assert(is_simm32(distance), "Must be"); - auipc(temp, (int32_t)distance + 0x800); - ld(temp, Address(temp, ((int32_t)distance << 20) >> 20)); - jalr(temp); -} - -void MacroAssembler::jump_link(const address dest, Register temp) { - assert(UseTrampolines, "Must be"); - assert_cond(dest != nullptr); - int64_t distance = dest - pc(); - assert(is_simm21(distance), "Must be"); - assert((distance % 2) == 0, "Must be"); - jal(x1, distance); -} - void MacroAssembler::j(const address dest, Register temp) { assert(CodeCache::contains(dest), "Must be"); assert_cond(dest != nullptr); @@ -2503,20 +2523,19 @@ void MacroAssembler::orptr(Address adr, RegisterOrConstant src, Register tmp1, R sd(tmp1, adr); } -void MacroAssembler::cmp_klass(Register oop, Register trial_klass, Register tmp1, Register tmp2, Label &L) { - assert_different_registers(oop, trial_klass, tmp1, tmp2); - if (UseCompressedClassPointers) { - lwu(tmp1, Address(oop, oopDesc::klass_offset_in_bytes())); - if (CompressedKlassPointers::base() == nullptr) { - slli(tmp1, tmp1, CompressedKlassPointers::shift()); - beq(trial_klass, tmp1, L); - return; - } - decode_klass_not_null(tmp1, tmp2); +void MacroAssembler::cmp_klass_compressed(Register oop, Register trial_klass, Register tmp, Label &L, bool equal) { + if (UseCompactObjectHeaders) { + load_narrow_klass_compact(tmp, oop); + } else if (UseCompressedClassPointers) { + lwu(tmp, Address(oop, oopDesc::klass_offset_in_bytes())); } else { - ld(tmp1, Address(oop, oopDesc::klass_offset_in_bytes())); + ld(tmp, Address(oop, oopDesc::klass_offset_in_bytes())); + } + if (equal) { + beq(trial_klass, tmp, L); + } else { + bne(trial_klass, tmp, L); } - beq(trial_klass, tmp1, L); } // Move an oop into a register. @@ -2722,10 +2741,19 @@ void MacroAssembler::encode_heap_oop_not_null(Register dst, Register src) { } } +void MacroAssembler::load_narrow_klass_compact(Register dst, Register src) { + assert(UseCompactObjectHeaders, "expects UseCompactObjectHeaders"); + ld(dst, Address(src, oopDesc::mark_offset_in_bytes())); + srli(dst, dst, markWord::klass_shift); +} + void MacroAssembler::load_klass(Register dst, Register src, Register tmp) { assert_different_registers(dst, tmp); assert_different_registers(src, tmp); - if (UseCompressedClassPointers) { + if (UseCompactObjectHeaders) { + load_narrow_klass_compact(dst, src); + decode_klass_not_null(dst, tmp); + } else if (UseCompressedClassPointers) { lwu(dst, Address(src, oopDesc::klass_offset_in_bytes())); decode_klass_not_null(dst, tmp); } else { @@ -2736,6 +2764,7 @@ void MacroAssembler::load_klass(Register dst, Register src, Register tmp) { void MacroAssembler::store_klass(Register dst, Register src, Register tmp) { // FIXME: Should this be a store release? concurrent gcs assumes // klass length is valid if klass field is not null. + assert(!UseCompactObjectHeaders, "not with compact headers"); if (UseCompressedClassPointers) { encode_klass_not_null(src, tmp); sw(src, Address(dst, oopDesc::klass_offset_in_bytes())); @@ -2745,6 +2774,7 @@ void MacroAssembler::store_klass(Register dst, Register src, Register tmp) { } void MacroAssembler::store_klass_gap(Register dst, Register src) { + assert(!UseCompactObjectHeaders, "not with compact headers"); if (UseCompressedClassPointers) { // Store to klass gap in destination sw(src, Address(dst, oopDesc::klass_gap_offset_in_bytes())); @@ -2761,8 +2791,7 @@ void MacroAssembler::decode_klass_not_null(Register dst, Register src, Register if (CompressedKlassPointers::base() == nullptr) { if (CompressedKlassPointers::shift() != 0) { - assert(LogKlassAlignmentInBytes == CompressedKlassPointers::shift(), "decode alg wrong"); - slli(dst, src, LogKlassAlignmentInBytes); + slli(dst, src, CompressedKlassPointers::shift()); } else { mv(dst, src); } @@ -2778,9 +2807,9 @@ void MacroAssembler::decode_klass_not_null(Register dst, Register src, Register mv(xbase, (uintptr_t)CompressedKlassPointers::base()); if (CompressedKlassPointers::shift() != 0) { - assert(LogKlassAlignmentInBytes == CompressedKlassPointers::shift(), "decode alg wrong"); - assert_different_registers(t0, xbase); - shadd(dst, src, xbase, t0, LogKlassAlignmentInBytes); + Register t = src == dst ? dst : t0; + assert_different_registers(t, xbase); + shadd(dst, src, xbase, t, CompressedKlassPointers::shift()); } else { add(dst, xbase, src); } @@ -2796,8 +2825,7 @@ void MacroAssembler::encode_klass_not_null(Register dst, Register src, Register if (CompressedKlassPointers::base() == nullptr) { if (CompressedKlassPointers::shift() != 0) { - assert(LogKlassAlignmentInBytes == CompressedKlassPointers::shift(), "decode alg wrong"); - srli(dst, src, LogKlassAlignmentInBytes); + srli(dst, src, CompressedKlassPointers::shift()); } else { mv(dst, src); } @@ -2819,8 +2847,7 @@ void MacroAssembler::encode_klass_not_null(Register dst, Register src, Register mv(xbase, (uintptr_t)CompressedKlassPointers::base()); sub(dst, src, xbase); if (CompressedKlassPointers::shift() != 0) { - assert(LogKlassAlignmentInBytes == CompressedKlassPointers::shift(), "decode alg wrong"); - srli(dst, dst, LogKlassAlignmentInBytes); + srli(dst, dst, CompressedKlassPointers::shift()); } } @@ -4235,9 +4262,7 @@ void MacroAssembler::set_narrow_klass(Register dst, Klass* k) { zero_extend(dst, dst, 32); } -// Maybe emit a call via a trampoline. If the code cache is small -// trampolines won't be emitted. -address MacroAssembler::trampoline_call(Address entry) { +address MacroAssembler::reloc_call(Address entry, Register tmp) { assert(entry.rspec().type() == relocInfo::runtime_call_type || entry.rspec().type() == relocInfo::opt_virtual_call_type || entry.rspec().type() == relocInfo::static_call_type || @@ -4245,45 +4270,8 @@ address MacroAssembler::trampoline_call(Address entry) { address target = entry.target(); - // We need a trampoline if branches are far. if (!in_scratch_emit_size()) { - if (entry.rspec().type() == relocInfo::runtime_call_type) { - assert(CodeBuffer::supports_shared_stubs(), "must support shared stubs"); - code()->share_trampoline_for(entry.target(), offset()); - } else { - address stub = emit_trampoline_stub(offset(), target); - if (stub == nullptr) { - postcond(pc() == badAddress); - return nullptr; // CodeCache is full - } - } - } - target = pc(); - - address call_pc = pc(); -#ifdef ASSERT - if (entry.rspec().type() != relocInfo::runtime_call_type) { - assert_alignment(call_pc); - } -#endif - relocate(entry.rspec(), [&] { - jump_link(target, t0); - }); - - postcond(pc() != badAddress); - return call_pc; -} - -address MacroAssembler::load_and_call(Address entry) { - assert(entry.rspec().type() == relocInfo::runtime_call_type || - entry.rspec().type() == relocInfo::opt_virtual_call_type || - entry.rspec().type() == relocInfo::static_call_type || - entry.rspec().type() == relocInfo::virtual_call_type, "wrong reloc type"); - - address target = entry.target(); - - if (!in_scratch_emit_size()) { - address stub = emit_address_stub(offset(), target); + address stub = emit_reloc_call_address_stub(offset(), target); if (stub == nullptr) { postcond(pc() == badAddress); return nullptr; // CodeCache is full @@ -4296,8 +4284,13 @@ address MacroAssembler::load_and_call(Address entry) { assert_alignment(call_pc); } #endif + + // The relocation created while emitting the stub will ensure this + // call instruction is subsequently patched to call the stub. relocate(entry.rspec(), [&] { - load_link_jump(target, t1); + auipc(tmp, 0); + ld(tmp, Address(tmp, 0)); + jalr(tmp); }); postcond(pc() != badAddress); @@ -4315,7 +4308,7 @@ address MacroAssembler::ic_call(address entry, jint method_index) { int MacroAssembler::ic_check_size() { // No compressed return (MacroAssembler::instruction_size * (2 /* 2 loads */ + 1 /* branch */)) + - far_branch_size(); + far_branch_size() + (UseCompactObjectHeaders ? MacroAssembler::instruction_size * 1 : 0); } int MacroAssembler::ic_check(int end_alignment) { @@ -4335,7 +4328,10 @@ int MacroAssembler::ic_check(int end_alignment) { align(end_alignment, ic_check_size()); int uep_offset = offset(); - if (UseCompressedClassPointers) { + if (UseCompactObjectHeaders) { + load_narrow_klass_compact(tmp1, receiver); + lwu(tmp2, Address(data, CompiledICData::speculated_klass_offset())); + } else if (UseCompressedClassPointers) { lwu(tmp1, Address(receiver, oopDesc::klass_offset_in_bytes())); lwu(tmp2, Address(data, CompiledICData::speculated_klass_offset())); } else { @@ -4354,95 +4350,50 @@ int MacroAssembler::ic_check(int end_alignment) { return uep_offset; } -address MacroAssembler::emit_address_stub(int insts_call_instruction_offset, address dest) { - address stub = start_a_stub(max_reloc_call_stub_size()); - if (stub == nullptr) { - return nullptr; // CodeBuffer::expand failed - } - - // We are always 4-byte aligned here. - assert_alignment(pc()); - - // Make sure the address of destination 8-byte aligned. - align(wordSize, 0); - - RelocationHolder rh = trampoline_stub_Relocation::spec(code()->insts()->start() + - insts_call_instruction_offset); - const int stub_start_offset = offset(); - relocate(rh, [&] { - assert(offset() - stub_start_offset == 0, - "%ld - %ld == %ld : should be", (long)offset(), (long)stub_start_offset, (long)0); - assert(offset() % wordSize == 0, "bad alignment"); - emit_int64((int64_t)dest); - }); - - const address stub_start_addr = addr_at(stub_start_offset); - end_a_stub(); - - return stub_start_addr; -} - -// Emit a trampoline stub for a call to a target which is too far away. +// Emit an address stub for a call to a target which is too far away. +// Note that we only put the target address of the call in the stub. // // code sequences: // // call-site: -// branch-and-link to or +// load target address from stub +// jump-and-link target address // -// Related trampoline stub for this call site in the stub section: -// load the call target from the constant pool -// branch (RA still points to the call site above) - -address MacroAssembler::emit_trampoline_stub(int insts_call_instruction_offset, - address dest) { - // Max stub size: alignment nop, TrampolineStub. - address stub = start_a_stub(max_reloc_call_stub_size()); +// Related address stub for this call site in the stub section: +// alignment nop +// target address + +address MacroAssembler::emit_reloc_call_address_stub(int insts_call_instruction_offset, address dest) { + address stub = start_a_stub(max_reloc_call_address_stub_size()); if (stub == nullptr) { return nullptr; // CodeBuffer::expand failed } - assert(UseTrampolines, "Must be using trampos."); - // We are always 4-byte aligned here. assert_alignment(pc()); - // Create a trampoline stub relocation which relates this trampoline stub - // with the call instruction at insts_call_instruction_offset in the - // instructions code-section. - - // Make sure the address of destination 8-byte aligned after 3 instructions. - align(wordSize, MacroAssembler::NativeShortCall::trampoline_data_offset); + // Make sure the address of destination 8-byte aligned. + align(wordSize, 0); RelocationHolder rh = trampoline_stub_Relocation::spec(code()->insts()->start() + insts_call_instruction_offset); const int stub_start_offset = offset(); relocate(rh, [&] { - // Now, create the trampoline stub's code: - // - load the call - // - call - Label target; - ld(t1, target); // auipc + ld - jr(t1); // jalr - bind(target); - assert(offset() - stub_start_offset == MacroAssembler::NativeShortCall::trampoline_data_offset, - "should be"); + assert(offset() - stub_start_offset == 0, + "%ld - %ld == %ld : should be", (long)offset(), (long)stub_start_offset, (long)0); assert(offset() % wordSize == 0, "bad alignment"); emit_int64((int64_t)dest); }); const address stub_start_addr = addr_at(stub_start_offset); - end_a_stub(); return stub_start_addr; } -int MacroAssembler::max_reloc_call_stub_size() { - // Max stub size: alignment nop, TrampolineStub. - if (UseTrampolines) { - return instruction_size + MacroAssembler::NativeShortCall::trampoline_size; - } - return instruction_size + wordSize; +int MacroAssembler::max_reloc_call_address_stub_size() { + // Max stub size: alignment nop, target address. + return 1 * instruction_size + wordSize; } int MacroAssembler::static_call_stub_size() { diff --git a/src/hotspot/cpu/riscv/macroAssembler_riscv.hpp b/src/hotspot/cpu/riscv/macroAssembler_riscv.hpp index 8fca6357627..dc06708b0ff 100644 --- a/src/hotspot/cpu/riscv/macroAssembler_riscv.hpp +++ b/src/hotspot/cpu/riscv/macroAssembler_riscv.hpp @@ -195,8 +195,9 @@ class MacroAssembler: public Assembler { void access_store_at(BasicType type, DecoratorSet decorators, Address dst, Register val, Register tmp1, Register tmp2, Register tmp3); void load_klass(Register dst, Register src, Register tmp = t0); + void load_narrow_klass_compact(Register dst, Register src); void store_klass(Register dst, Register src, Register tmp = t0); - void cmp_klass(Register oop, Register trial_klass, Register tmp1, Register tmp2, Label &L); + void cmp_klass_compressed(Register oop, Register trial_klass, Register tmp, Label &L, bool equal); void encode_klass_not_null(Register r, Register tmp = t0); void decode_klass_not_null(Register r, Register tmp = t0); @@ -468,9 +469,8 @@ class MacroAssembler: public Assembler { return false; } - address emit_address_stub(int insts_call_instruction_offset, address target); - address emit_trampoline_stub(int insts_call_instruction_offset, address target); - static int max_reloc_call_stub_size(); + address emit_reloc_call_address_stub(int insts_call_instruction_offset, address target); + static int max_reloc_call_address_stub_size(); void emit_static_call_stub(); static int static_call_stub_size(); @@ -626,9 +626,6 @@ class MacroAssembler: public Assembler { void bltz(Register Rs, const address dest); void bgtz(Register Rs, const address dest); - private: - void load_link_jump(const address source, Register temp); - void jump_link(const address dest, Register temp); public: // We try to follow risc-v asm menomics. // But as we don't layout a reachable GOT, @@ -811,8 +808,11 @@ class MacroAssembler: public Assembler { void push_CPU_state(bool save_vectors = false, int vector_size_in_bytes = 0); void pop_CPU_state(bool restore_vectors = false, int vector_size_in_bytes = 0); - void push_cont_fastpath(Register java_thread); - void pop_cont_fastpath(Register java_thread); + void push_cont_fastpath(Register java_thread = xthread); + void pop_cont_fastpath(Register java_thread = xthread); + + void inc_held_monitor_count(Register tmp); + void dec_held_monitor_count(Register tmp); // if heap base register is used - reinit it with the correct value void reinit_heapbase(); @@ -1227,92 +1227,50 @@ class MacroAssembler: public Assembler { void get_polling_page(Register dest, relocInfo::relocType rtype); void read_polling_page(Register r, int32_t offset, relocInfo::relocType rtype); - // RISCV64 OpenJDK uses four different types of calls: - // - direct call: jal pc_relative_offset - // This is the shortest and the fastest, but the offset has the range: +/-1MB. + // RISCV64 OpenJDK uses three different types of calls: // // - far call: auipc reg, pc_relative_offset; jalr ra, reg, offset - // This is longer than a direct call. The offset has - // the range [-(2G + 2K), 2G - 2K). Addresses out of the range in the code cache - // requires indirect call. - // If a jump is needed rather than a call, a far jump 'jalr x0, reg, offset' can - // be used instead. + // The offset has the range [-(2G + 2K), 2G - 2K). Addresses out of the + // range in the code cache requires indirect call. + // If a jump is needed rather than a call, a far jump 'jalr x0, reg, offset' + // can be used instead. // All instructions are embedded at a call site. // // - indirect call: movptr + jalr - // This too can reach anywhere in the address space, but it cannot be - // patched while code is running, so it must only be modified at a safepoint. - // This form of call is most suitable for targets at fixed addresses, which - // will never be patched. + // This can reach anywhere in the address space, but it cannot be patched + // while code is running, so it must only be modified at a safepoint. + // This form of call is most suitable for targets at fixed addresses, + // which will never be patched. // // - reloc call: - // This is only available in C1/C2-generated code (nmethod). + // This too can reach anywhere in the address space but is only available + // in C1/C2-generated code (nmethod). // // [Main code section] // auipc // ld // jalr + // // [Stub section] - // trampoline: + // address stub: // <64-bit destination address> // // To change the destination we simply atomically store the new // address in the stub section. - // - // - trampoline call (old reloc call / -XX:+UseTrampolines): - // This is only available in C1/C2-generated code (nmethod). It is a combination - // of a direct call, which is used if the destination of a call is in range, - // and a register-indirect call. It has the advantages of reaching anywhere in - // the RISCV address space and being patchable at runtime when the generated - // code is being executed by other threads. - // - // [Main code section] - // jal trampoline - // [Stub code section] - // trampoline: - // ld reg, pc + 8 (auipc + ld) - // jr reg - // <64-bit destination address> - // - // If the destination is in range when the generated code is moved to the code - // cache, 'jal trampoline' is replaced with 'jal destination' and the trampoline - // is not used. - // The optimization does not remove the trampoline from the stub section. - // - // This is necessary because the trampoline may well be redirected later when - // code is patched, and the new destination may not be reachable by a simple JAL - // instruction. - // - // To patch a trampoline call when the JAL can't reach, we first modify - // the 64-bit destination address in the trampoline, then modify the - // JAL to point to the trampoline, then flush the instruction cache to - // broadcast the change to all executing threads. See - // NativeCall::set_destination_mt_safe for the details. - // - // There is a benign race in that the other thread might observe the - // modified JAL before it observes the modified 64-bit destination - // address. That does not matter because the destination method has been - // invalidated, so there will be a trap at its start. - // For this to work, the destination address in the trampoline is - // always updated, even if we're not using the trampoline. - // -- - - // Emit a direct call if the entry address will always be in range, - // otherwise a reloc call. + // There is a benign race in that the other thread might observe the old + // 64-bit destination address before it observes the new address. That does + // not matter because the destination method has been invalidated, so there + // will be a trap at its start. + + // Emit a reloc call and create a stub to hold the entry point address. // Supported entry.rspec(): // - relocInfo::runtime_call_type // - relocInfo::opt_virtual_call_type // - relocInfo::static_call_type // - relocInfo::virtual_call_type // - // Return: the call PC or null if CodeCache is full. - address reloc_call(Address entry) { - return UseTrampolines ? trampoline_call(entry) : load_and_call(entry); - } - private: - address trampoline_call(Address entry); - address load_and_call(Address entry); - public: + // Return: the call PC or nullptr if CodeCache is full. + address reloc_call(Address entry, Register tmp = t1); address ic_call(address entry, jint method_index = 0); static int ic_check_size(); @@ -1636,11 +1594,6 @@ class MacroAssembler: public Assembler { load_pc_relative_instruction_size = 2 * instruction_size // auipc, ld }; - enum NativeShortCall { - trampoline_size = 3 * instruction_size + wordSize, - trampoline_data_offset = 3 * instruction_size - }; - static bool is_load_pc_relative_at(address branch); static bool is_li16u_at(address instr); diff --git a/src/hotspot/cpu/riscv/nativeInst_riscv.cpp b/src/hotspot/cpu/riscv/nativeInst_riscv.cpp index 988a5b79332..3a273fdb3ed 100644 --- a/src/hotspot/cpu/riscv/nativeInst_riscv.cpp +++ b/src/hotspot/cpu/riscv/nativeInst_riscv.cpp @@ -46,264 +46,10 @@ bool NativeInstruction::is_call_at(address addr) { return NativeCall::is_at(addr); } -//----------------------------------------------------------------------------- -// NativeShortCallTrampoline -// -// Implements the trampoline part of reloc call - trampoline call. - -class NativeShortCall; - -class NativeShortCallTrampolineStub : public NativeInstruction { - private: - friend NativeShortCall; - enum RISCV_specific_constants { - trampoline_data_offset = 3 * NativeInstruction::instruction_size // auipc + ld + jr - }; - - address destination() const; - void set_destination(address new_destination); - - static bool is_at(address addr); - static NativeShortCallTrampolineStub* at(address addr); -}; - -address NativeShortCallTrampolineStub::destination() const { - return ptr_at(trampoline_data_offset); -} - -void NativeShortCallTrampolineStub::set_destination(address new_destination) { - set_ptr_at(trampoline_data_offset, new_destination); - OrderAccess::release(); -} - -bool NativeShortCallTrampolineStub::is_at(address addr) { - // Ensure that the stub is exactly - // ld t0, L--->auipc + ld - // jr t0 - // L: - - // judge inst + register + imm - // 1). check the instructions: auipc + ld + jalr - // 2). check if auipc[11:7] == t0 and ld[11:7] == t0 and ld[19:15] == t0 && jr[19:15] == t0 - // 3). check if the offset in ld[31:20] equals the data_offset - assert_cond(addr != nullptr); - const int instr_size = NativeInstruction::instruction_size; - if (MacroAssembler::is_auipc_at(addr) && - MacroAssembler::is_ld_at(addr + instr_size) && - MacroAssembler::is_jalr_at(addr + 2 * instr_size) && - (MacroAssembler::extract_rd(addr) == x6) && - (MacroAssembler::extract_rd(addr + instr_size) == x6) && - (MacroAssembler::extract_rs1(addr + instr_size) == x6) && - (MacroAssembler::extract_rs1(addr + 2 * instr_size) == x6) && - (Assembler::extract(Assembler::ld_instr(addr + 4), 31, 20) == trampoline_data_offset)) { - return true; - } - return false; -} - -NativeShortCallTrampolineStub* NativeShortCallTrampolineStub::at(address addr) { - assert_cond(addr != nullptr); - assert(NativeShortCallTrampolineStub::is_at(addr), "no call trampoline found"); - return (NativeShortCallTrampolineStub*)addr; -} - -//----------------------------------------------------------------------------- -// NativeShortCall -// -// Implements the trampoline call, a short call with a trampoline, version of reloc call. -// Enabled by setting the experimental UseTrampolines to true. - -class NativeShortCall: private NativeInstruction { - public: - enum RISCV_specific_constants { - return_address_offset = 1 * NativeInstruction::instruction_size // jal - }; - - address instruction_address() const { return addr_at(0); } - address next_instruction_address() const { return addr_at(return_address_offset); } - address return_address() const { return addr_at(return_address_offset); } - address destination() const; - address reloc_destination(address orig_address); - - void set_destination(address dest); - void verify(); - void print(); - - bool set_destination_mt_safe(address dest, bool assert_lock = true); - bool reloc_set_destination(address dest); - - private: - address get_trampoline(); - bool has_trampoline(); - address trampoline_destination(); - public: - - static NativeShortCall* at(address addr); - static bool is_at(address addr); - static bool is_call_before(address return_address); -}; - -address NativeShortCall::destination() const { - address addr = instruction_address(); - assert(MacroAssembler::is_jal_at(instruction_address()), "inst must be jal."); - - address destination = MacroAssembler::target_addr_for_insn(instruction_address()); - - // Do we use a trampoline stub for this call? - CodeBlob* cb = CodeCache::find_blob(addr); - assert(cb && cb->is_nmethod(), "sanity"); - nmethod *nm = (nmethod *)cb; - if (nm != nullptr && nm->stub_contains(destination) && NativeShortCallTrampolineStub::is_at(destination)) { - // Yes we do, so get the destination from the trampoline stub. - const address trampoline_stub_addr = destination; - destination = NativeShortCallTrampolineStub::at(trampoline_stub_addr)->destination(); - } - - return destination; -} - -address NativeShortCall::reloc_destination(address orig_address) { - address addr = instruction_address(); - if (NativeShortCall::is_at(addr)) { - NativeShortCall* call = NativeShortCall::at(addr); - if (call->has_trampoline()) { - return call->trampoline_destination(); - } - } - if (orig_address != nullptr) { - // the extracted address from the instructions in address orig_addr - address new_addr = MacroAssembler::pd_call_destination(orig_address); - // If call is branch to self, don't try to relocate it, just leave it - // as branch to self. This happens during code generation if the code - // buffer expands. It will be relocated to the trampoline above once - // code generation is complete. - new_addr = (new_addr == orig_address) ? addr : new_addr; - return new_addr; - } - return MacroAssembler::pd_call_destination(addr); -} - -void NativeShortCall::set_destination(address dest) { - assert(NativeShortCall::is_at(instruction_address()), "unexpected code at call site"); - assert(is_jal(), "Should be jal instruction!"); - intptr_t offset = (intptr_t)(dest - instruction_address()); - assert((offset & 0x1) == 0, "bad alignment"); - assert(Assembler::is_simm21(offset), "encoding constraint"); - unsigned int insn = 0b1101111; // jal - address pInsn = (address)(&insn); - Assembler::patch(pInsn, 31, 31, (offset >> 20) & 0x1); - Assembler::patch(pInsn, 30, 21, (offset >> 1) & 0x3ff); - Assembler::patch(pInsn, 20, 20, (offset >> 11) & 0x1); - Assembler::patch(pInsn, 19, 12, (offset >> 12) & 0xff); - Assembler::patch(pInsn, 11, 7, ra->encoding()); // Rd must be x1, need ra - set_int_at(0, insn); -} - -void NativeShortCall::verify() { - assert(NativeShortCall::is_at(instruction_address()), - "unexpected code at call site: %p", instruction_address()); -} - -void NativeShortCall::print() { - assert(NativeShortCall::is_at(instruction_address()), "unexpected code at call site"); - tty->print_cr(PTR_FORMAT ": jal/auipc,ld,jalr x1, offset/reg", p2i(instruction_address())); -} - -// The important thing is that threads are able to execute this -// call instruction at all times. (cmodx) -// -// Used in the runtime linkage of calls; see class CompiledIC. -// -// Add parameter assert_lock to switch off assertion -// during code generation, where no lock is needed. -bool NativeShortCall::set_destination_mt_safe(address dest, bool assert_lock) { - assert(!assert_lock || - (CodeCache_lock->is_locked() || SafepointSynchronize::is_at_safepoint()) || - CompiledICLocker::is_safe(instruction_address()), - "concurrent code patching"); - - address call_addr = instruction_address(); - assert(NativeCall::is_at(call_addr), "unexpected code at call site"); - - reloc_set_destination(dest); - - ICache::invalidate_range(call_addr, instruction_size); - return true; -} - -bool NativeShortCall::reloc_set_destination(address dest) { - address call_addr = instruction_address(); - assert(NativeCall::is_at(call_addr), "unexpected code at call site"); - - // Patch the constant in the call's trampoline stub. - address trampoline_stub_addr = get_trampoline(); - if (trampoline_stub_addr != nullptr) { - assert(!NativeShortCallTrampolineStub::is_at(dest), "chained trampolines"); - NativeShortCallTrampolineStub::at(trampoline_stub_addr)->set_destination(dest); - } - - // Patch the call. - if (Assembler::reachable_from_branch_at(call_addr, dest)) { - set_destination(dest); - } else { - assert (trampoline_stub_addr != nullptr, "we need a trampoline"); - set_destination(trampoline_stub_addr); - } - - return true; -} - -address NativeShortCall::get_trampoline() { - address call_addr = instruction_address(); - - CodeBlob *code = CodeCache::find_blob(call_addr); - assert(code != nullptr, "Could not find the containing code blob"); - - address jal_destination = MacroAssembler::pd_call_destination(call_addr); - if (code != nullptr && code->contains(jal_destination) && NativeShortCallTrampolineStub::is_at(jal_destination)) { - return jal_destination; - } - - if (code != nullptr && code->is_nmethod()) { - return trampoline_stub_Relocation::get_trampoline_for(call_addr, (nmethod*)code); - } - - return nullptr; -} - -bool NativeShortCall::has_trampoline() { - return NativeShortCall::get_trampoline() != nullptr; -} - -address NativeShortCall::trampoline_destination() { - return NativeShortCallTrampolineStub::at(get_trampoline())->destination(); -} - -NativeShortCall* NativeShortCall::at(address addr) { - assert_cond(addr != nullptr); - assert(NativeShortCall::is_at(addr), "unexpected code at call site: %p", addr); - NativeShortCall* call = (NativeShortCall*)(addr); - return call; -} - -bool NativeShortCall::is_at(address addr) { - if (MacroAssembler::is_jal_at(addr)) { - if (MacroAssembler::extract_rd(addr) == x1) { - return true; - } - } - return false; -} - -bool NativeShortCall::is_call_before(address return_address) { - return NativeShortCall::is_at(return_address - instruction_size); -} - //----------------------------------------------------------------------------- // NativeFarCall // // Implements direct far calling loading an address from the stub section version of reloc call. -// This is the default (experimental flag UseTrampolines, default false). class NativeFarCall: public NativeInstruction { public: @@ -478,99 +224,51 @@ bool NativeFarCall::is_call_before(address return_address) { // NativeCall address NativeCall::instruction_address() const { - if (UseTrampolines) { - return NativeShortCall::at(addr_at(0))->instruction_address(); - } else { - return NativeFarCall::at(addr_at(0))->instruction_address(); - } + return NativeFarCall::at(addr_at(0))->instruction_address(); } address NativeCall::next_instruction_address() const { - if (UseTrampolines) { - return NativeShortCall::at(addr_at(0))->next_instruction_address(); - } else { - return NativeFarCall::at(addr_at(0))->next_instruction_address(); - } + return NativeFarCall::at(addr_at(0))->next_instruction_address(); } address NativeCall::return_address() const { - if (UseTrampolines) { - return NativeShortCall::at(addr_at(0))->return_address(); - } else { - return NativeFarCall::at(addr_at(0))->return_address(); - } + return NativeFarCall::at(addr_at(0))->return_address(); } address NativeCall::destination() const { - if (UseTrampolines) { - return NativeShortCall::at(addr_at(0))->destination(); - } else { - return NativeFarCall::at(addr_at(0))->destination(); - } + return NativeFarCall::at(addr_at(0))->destination(); } address NativeCall::reloc_destination(address orig_address) { - if (UseTrampolines) { - return NativeShortCall::at(addr_at(0))->reloc_destination(orig_address); - } else { - return NativeFarCall::at(addr_at(0))->reloc_destination(orig_address); - } + return NativeFarCall::at(addr_at(0))->reloc_destination(orig_address); } void NativeCall::set_destination(address dest) { - if (UseTrampolines) { - NativeShortCall::at(addr_at(0))->set_destination(dest); - } else { - NativeFarCall::at(addr_at(0))->set_destination(dest); - } + NativeFarCall::at(addr_at(0))->set_destination(dest); } void NativeCall::verify() { - if (UseTrampolines) { - NativeShortCall::at(addr_at(0))->verify(); - } else { - NativeFarCall::at(addr_at(0))->verify();; - } + NativeFarCall::at(addr_at(0))->verify();; } void NativeCall::print() { - if (UseTrampolines) { - NativeShortCall::at(addr_at(0))->print(); - } else { - NativeFarCall::at(addr_at(0))->print();; - } + NativeFarCall::at(addr_at(0))->print();; } bool NativeCall::set_destination_mt_safe(address dest, bool assert_lock) { - if (UseTrampolines) { - return NativeShortCall::at(addr_at(0))->set_destination_mt_safe(dest, assert_lock); - } else { - return NativeFarCall::at(addr_at(0))->set_destination_mt_safe(dest, assert_lock); - } + return NativeFarCall::at(addr_at(0))->set_destination_mt_safe(dest, assert_lock); } bool NativeCall::reloc_set_destination(address dest) { - if (UseTrampolines) { - return NativeShortCall::at(addr_at(0))->reloc_set_destination(dest); - } else { - return NativeFarCall::at(addr_at(0))->reloc_set_destination(dest); - } + return NativeFarCall::at(addr_at(0))->reloc_set_destination(dest); } bool NativeCall::is_at(address addr) { - if (UseTrampolines) { - return NativeShortCall::is_at(addr); - } else { - return NativeFarCall::is_at(addr); - } + return NativeFarCall::is_at(addr); } bool NativeCall::is_call_before(address return_address) { - if (UseTrampolines) { - return NativeShortCall::is_call_before(return_address); - } else { - return NativeFarCall::is_call_before(return_address); - } + return NativeFarCall::is_call_before(return_address); } NativeCall* nativeCall_at(address addr) { @@ -583,11 +281,7 @@ NativeCall* nativeCall_at(address addr) { NativeCall* nativeCall_before(address return_address) { assert_cond(return_address != nullptr); NativeCall* call = nullptr; - if (UseTrampolines) { - call = (NativeCall*)(return_address - NativeShortCall::return_address_offset); - } else { - call = (NativeCall*)(return_address - NativeFarCall::return_address_offset); - } + call = (NativeCall*)(return_address - NativeFarCall::return_address_offset); DEBUG_ONLY(call->verify()); return call; } diff --git a/src/hotspot/cpu/riscv/nativeInst_riscv.hpp b/src/hotspot/cpu/riscv/nativeInst_riscv.hpp index 0ac3a89969a..5b276dde368 100644 --- a/src/hotspot/cpu/riscv/nativeInst_riscv.hpp +++ b/src/hotspot/cpu/riscv/nativeInst_riscv.hpp @@ -124,12 +124,9 @@ class NativeCall: private NativeInstruction { public: static int byte_size() { - if (UseTrampolines) { - return NativeInstruction::instruction_size; // jal - } else { - return 3 * NativeInstruction::instruction_size; // auipc + ld + jalr - } + return 3 * NativeInstruction::instruction_size; // auipc + ld + jalr } + // Creation friend NativeCall* nativeCall_at(address addr); friend NativeCall* nativeCall_before(address return_address); diff --git a/src/hotspot/cpu/riscv/riscv.ad b/src/hotspot/cpu/riscv/riscv.ad index 6df41722d86..b944cc5b4b9 100644 --- a/src/hotspot/cpu/riscv/riscv.ad +++ b/src/hotspot/cpu/riscv/riscv.ad @@ -1238,41 +1238,27 @@ bool needs_acquiring_load_reserved(const Node *n) int MachCallStaticJavaNode::ret_addr_offset() { - if (UseTrampolines) { - return 1 * NativeInstruction::instruction_size; // jal - } return 3 * NativeInstruction::instruction_size; // auipc + ld + jalr } int MachCallDynamicJavaNode::ret_addr_offset() { - if (UseTrampolines) { - return NativeMovConstReg::movptr2_instruction_size + NativeInstruction::instruction_size; // movptr2, jal - } return NativeMovConstReg::movptr2_instruction_size + (3 * NativeInstruction::instruction_size); // movptr2, auipc + ld + jal } int MachCallRuntimeNode::ret_addr_offset() { - // For generated stubs the call will be: - // auipc + ld + jalr - // Using trampos: - // jal(addr) - // or with far branches - // jal(trampoline_stub) - // for real runtime callouts it will be 9 instructions + // For address inside the code cache the call will be: + // auipc + jalr + // For real runtime callouts it will be 8 instructions // see riscv_enc_java_to_runtime - // la(t0, retaddr) -> auipc + addi - // addi(sp, sp, -2 * wordSize) -> addi - // sd(t0, Address(sp, wordSize)) -> sd - // movptr(t1, addr, offset, t0) -> lui + lui + slli + add - // jalr(t1, offset) -> jalr + // la(t0, retaddr) -> auipc + addi + // sd(t0, Address(xthread, JavaThread::last_Java_pc_offset())) -> sd + // movptr(t1, addr, offset, t0) -> lui + lui + slli + add + // jalr(t1, offset) -> jalr if (CodeCache::contains(_entry_point)) { - if (UseTrampolines) { - return 1 * NativeInstruction::instruction_size; - } - return 3 * NativeInstruction::instruction_size; + return 2 * NativeInstruction::instruction_size; } else { - return 9 * NativeInstruction::instruction_size; + return 8 * NativeInstruction::instruction_size; } } @@ -2434,10 +2420,8 @@ encode %{ // The NOP here is purely to ensure that eliding a call to // JVM_EnsureMaterializedForStackWalk doesn't change the code size. __ nop(); - if (!UseTrampolines) { - __ nop(); - __ nop(); - } + __ nop(); + __ nop(); __ block_comment("call JVM_EnsureMaterializedForStackWalk (elided)"); } else { int method_index = resolved_method_index(masm); @@ -2488,31 +2472,25 @@ encode %{ enc_class riscv_enc_java_to_runtime(method meth) %{ Assembler::IncompressibleRegion ir(masm); // Fixed length: see ret_addr_offset - // some calls to generated routines (arraycopy code) are scheduled - // by C2 as runtime calls. if so we can call them using a jr (they - // will be in a reachable segment) otherwise we have to use a jalr - // which loads the absolute address into a register. + // Some calls to generated routines (arraycopy code) are scheduled by C2 + // as runtime calls. if so we can call them using a far call (they will be + // in the code cache, thus in a reachable segment) otherwise we have to use + // a movptr+jalr pair which loads the absolute address into a register. address entry = (address)$meth$$method; if (CodeCache::contains(entry)) { - address call = __ reloc_call(Address(entry, relocInfo::runtime_call_type)); - if (call == nullptr) { - ciEnv::current()->record_failure("CodeCache is full"); - return; - } + __ far_call(Address(entry, relocInfo::runtime_call_type)); __ post_call_nop(); } else { Label retaddr; + // Make the anchor frame walkable __ la(t0, retaddr); - // Leave a breadcrumb for JavaFrameAnchor::capture_last_Java_pc() - __ addi(sp, sp, -2 * wordSize); - __ sd(t0, Address(sp, wordSize)); + __ sd(t0, Address(xthread, JavaThread::last_Java_pc_offset())); int32_t offset = 0; // No relocation needed __ movptr(t1, entry, offset, t0); // lui + lui + slli + add __ jalr(t1, offset); __ bind(retaddr); __ post_call_nop(); - __ addi(sp, sp, 2 * wordSize); } %} @@ -4790,7 +4768,7 @@ instruct loadN(iRegNNoSp dst, memory mem) match(Set dst (LoadN mem)); ins_cost(LOAD_COST); - format %{ "lwu $dst, $mem\t# loadN, compressed ptr, #@loadN" %} + format %{ "lwu $dst, $mem\t# compressed ptr, #@loadN" %} ins_encode %{ __ lwu(as_Register($dst$$reg), Address(as_Register($mem$$base), $mem$$disp)); @@ -4817,10 +4795,11 @@ instruct loadKlass(iRegPNoSp dst, memory mem) // Load Narrow Klass Pointer instruct loadNKlass(iRegNNoSp dst, memory mem) %{ + predicate(!UseCompactObjectHeaders); match(Set dst (LoadNKlass mem)); ins_cost(LOAD_COST); - format %{ "lwu $dst, $mem\t# loadNKlass, compressed class ptr, #@loadNKlass" %} + format %{ "lwu $dst, $mem\t# compressed class ptr, #@loadNKlass" %} ins_encode %{ __ lwu(as_Register($dst$$reg), Address(as_Register($mem$$base), $mem$$disp)); @@ -4829,6 +4808,21 @@ instruct loadNKlass(iRegNNoSp dst, memory mem) ins_pipe(iload_reg_mem); %} +instruct loadNKlassCompactHeaders(iRegNNoSp dst, memory mem) +%{ + predicate(UseCompactObjectHeaders); + match(Set dst (LoadNKlass mem)); + + ins_cost(LOAD_COST); + format %{ "load_narrow_klass_compact $dst, $mem\t# compressed class ptr, #@loadNKlassCompactHeaders" %} + + ins_encode %{ + __ load_narrow_klass_compact_c2(as_Register($dst$$reg), Address(as_Register($mem$$base), $mem$$disp)); + %} + + ins_pipe(iload_reg_mem); +%} + // Load Float instruct loadF(fRegF dst, memory mem) %{ @@ -10523,17 +10517,19 @@ instruct tlsLoadP(javaThread_RegP dst) // inlined locking and unlocking // using t1 as the 'flag' register to bridge the BoolNode producers and consumers -instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3) +instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, + iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, iRegPNoSp tmp4) %{ predicate(LockingMode != LM_LIGHTWEIGHT); match(Set cr (FastLock object box)); - effect(TEMP tmp1, TEMP tmp2, TEMP tmp3); + effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4); ins_cost(10 * DEFAULT_COST); - format %{ "fastlock $object,$box\t! kills $tmp1,$tmp2,$tmp3, #@cmpFastLock" %} + format %{ "fastlock $object,$box\t! kills $tmp1,$tmp2,$tmp3,$tmp4 #@cmpFastLock" %} ins_encode %{ - __ fast_lock($object$$Register, $box$$Register, $tmp1$$Register, $tmp2$$Register, $tmp3$$Register); + __ fast_lock($object$$Register, $box$$Register, + $tmp1$$Register, $tmp2$$Register, $tmp3$$Register, $tmp4$$Register); %} ins_pipe(pipe_serial); @@ -10556,23 +10552,26 @@ instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp1, iR ins_pipe(pipe_serial); %} -instruct cmpFastLockLightweight(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3) +instruct cmpFastLockLightweight(rFlagsReg cr, iRegP object, iRegP box, + iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, iRegPNoSp tmp4) %{ predicate(LockingMode == LM_LIGHTWEIGHT); match(Set cr (FastLock object box)); - effect(TEMP tmp1, TEMP tmp2, TEMP tmp3); + effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4); ins_cost(10 * DEFAULT_COST); - format %{ "fastlock $object,$box\t! kills $tmp1,$tmp2,$tmp3 #@cmpFastLockLightweight" %} + format %{ "fastlock $object,$box\t! kills $tmp1,$tmp2,$tmp3,$tmp4 #@cmpFastLockLightweight" %} ins_encode %{ - __ fast_lock_lightweight($object$$Register, $box$$Register, $tmp1$$Register, $tmp2$$Register, $tmp3$$Register); + __ fast_lock_lightweight($object$$Register, $box$$Register, + $tmp1$$Register, $tmp2$$Register, $tmp3$$Register, $tmp4$$Register); %} ins_pipe(pipe_serial); %} -instruct cmpFastUnlockLightweight(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3) +instruct cmpFastUnlockLightweight(rFlagsReg cr, iRegP object, iRegP box, + iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3) %{ predicate(LockingMode == LM_LIGHTWEIGHT); match(Set cr (FastUnlock object box)); @@ -10582,7 +10581,8 @@ instruct cmpFastUnlockLightweight(rFlagsReg cr, iRegP object, iRegP box, iRegPNo format %{ "fastunlock $object,$box\t! kills $tmp1,$tmp2,$tmp3 #@cmpFastUnlockLightweight" %} ins_encode %{ - __ fast_unlock_lightweight($object$$Register, $box$$Register, $tmp1$$Register, $tmp2$$Register, $tmp3$$Register); + __ fast_unlock_lightweight($object$$Register, $box$$Register, + $tmp1$$Register, $tmp2$$Register, $tmp3$$Register); %} ins_pipe(pipe_serial); diff --git a/src/hotspot/cpu/riscv/sharedRuntime_riscv.cpp b/src/hotspot/cpu/riscv/sharedRuntime_riscv.cpp index 29c96112ead..5ab2a9f00b9 100644 --- a/src/hotspot/cpu/riscv/sharedRuntime_riscv.cpp +++ b/src/hotspot/cpu/riscv/sharedRuntime_riscv.cpp @@ -1050,12 +1050,14 @@ static void gen_continuation_enter(MacroAssembler* masm, __ bind(call_thaw); + ContinuationEntry::_thaw_call_pc_offset = __ pc() - start; __ rt_call(CAST_FROM_FN_PTR(address, StubRoutines::cont_thaw())); oop_maps->add_gc_map(__ pc() - start, map->deep_copy()); ContinuationEntry::_return_pc_offset = __ pc() - start; __ post_call_nop(); __ bind(exit); + ContinuationEntry::_cleanup_offset = __ pc() - start; continuation_enter_cleanup(masm); __ leave(); __ ret(); @@ -1149,6 +1151,10 @@ static void gen_continuation_yield(MacroAssembler* masm, oop_maps->add_gc_map(the_pc - start, map); } +void SharedRuntime::continuation_enter_cleanup(MacroAssembler* masm) { + ::continuation_enter_cleanup(masm); +} + static void gen_special_dispatch(MacroAssembler* masm, const methodHandle& method, const BasicType* sig_bt, @@ -1629,11 +1635,20 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm, } // Change state to native (we save the return address in the thread, since it might not - // be pushed on the stack when we do a stack traversal). - // We use the same pc/oopMap repeatedly when we call out + // be pushed on the stack when we do a stack traversal). It is enough that the pc() + // points into the right code segment. It does not have to be the correct return pc. + // We use the same pc/oopMap repeatedly when we call out. Label native_return; - __ set_last_Java_frame(sp, noreg, native_return, t0); + if (LockingMode != LM_LEGACY && method->is_object_wait0()) { + // For convenience we use the pc we want to resume to in case of preemption on Object.wait. + __ set_last_Java_frame(sp, noreg, native_return, t0); + } else { + intptr_t the_pc = (intptr_t) __ pc(); + oop_maps->add_gc_map(the_pc - start, map); + + __ set_last_Java_frame(sp, noreg, __ pc(), t0); + } Label dtrace_method_entry, dtrace_method_entry_done; if (DTraceMethodProbes) { @@ -1709,14 +1724,14 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm, // Save the test result, for recursive case, the result is zero __ sd(swap_reg, Address(lock_reg, mark_word_offset)); __ bnez(swap_reg, slow_path_lock); + + __ bind(count); + __ inc_held_monitor_count(t0); } else { assert(LockingMode == LM_LIGHTWEIGHT, "must be"); __ lightweight_lock(lock_reg, obj_reg, swap_reg, tmp, lock_tmp, slow_path_lock); } - __ bind(count); - __ increment(Address(xthread, JavaThread::held_monitor_count_offset())); - // Slow path will re-enter here __ bind(lock_done); } @@ -1736,11 +1751,6 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm, // Clobbers t1 __ rt_call(native_func); - __ bind(native_return); - - intptr_t return_pc = (intptr_t) __ pc(); - oop_maps->add_gc_map(return_pc - start, map); - // Verify or restore cpu control state after JNI call __ restore_cpu_control_state_after_jni(t0); @@ -1791,6 +1801,18 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm, __ sw(t0, Address(t1)); __ bind(after_transition); + if (LockingMode != LM_LEGACY && method->is_object_wait0()) { + // Check preemption for Object.wait() + __ ld(t1, Address(xthread, JavaThread::preempt_alternate_return_offset())); + __ beqz(t1, native_return); + __ sd(zr, Address(xthread, JavaThread::preempt_alternate_return_offset())); + __ jr(t1); + __ bind(native_return); + + intptr_t the_pc = (intptr_t) __ pc(); + oop_maps->add_gc_map(the_pc - start, map); + } + Label reguard; Label reguard_done; __ lbu(t0, Address(xthread, JavaThread::stack_guard_state_offset())); @@ -1814,7 +1836,7 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm, // Simple recursive lock? __ ld(t0, Address(sp, lock_slot_offset * VMRegImpl::stack_slot_size)); __ bnez(t0, not_recursive); - __ decrement(Address(xthread, JavaThread::held_monitor_count_offset())); + __ dec_held_monitor_count(t0); __ j(done); } @@ -1837,11 +1859,10 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm, Label count; __ cmpxchg_obj_header(x10, old_hdr, obj_reg, lock_tmp, count, &slow_path_unlock); __ bind(count); - __ decrement(Address(xthread, JavaThread::held_monitor_count_offset())); + __ dec_held_monitor_count(t0); } else { assert(LockingMode == LM_LIGHTWEIGHT, ""); __ lightweight_unlock(obj_reg, old_hdr, swap_reg, lock_tmp, slow_path_unlock); - __ decrement(Address(xthread, JavaThread::held_monitor_count_offset())); } // slow path re-enters here @@ -1909,8 +1930,14 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm, __ mv(c_rarg1, lock_reg); __ mv(c_rarg2, xthread); - // Not a leaf but we have last_Java_frame setup as we want + // Not a leaf but we have last_Java_frame setup as we want. + // We don't want to unmount in case of contention since that would complicate preserving + // the arguments that had already been marshalled into the native convention. So we force + // the freeze slow path to find this native wrapper frame (see recurse_freeze_native_frame()) + // and pin the vthread. Otherwise the fast path won't find it since we don't walk the stack. + __ push_cont_fastpath(); __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::complete_monitor_locking_C), 3); + __ pop_cont_fastpath(); restore_args(masm, total_c_args, c_arg, out_regs); #ifdef ASSERT @@ -2441,6 +2468,10 @@ uint SharedRuntime::out_preserve_stack_slots() { return 0; } +VMReg SharedRuntime::thread_register() { + return xthread->as_VMReg(); +} + //------------------------------generate_handler_blob------ // // Generate a special Compile2Runtime blob that saves all registers, diff --git a/src/hotspot/cpu/riscv/stackChunkFrameStream_riscv.inline.hpp b/src/hotspot/cpu/riscv/stackChunkFrameStream_riscv.inline.hpp index e226c7b7a53..fa8a8fb47f0 100644 --- a/src/hotspot/cpu/riscv/stackChunkFrameStream_riscv.inline.hpp +++ b/src/hotspot/cpu/riscv/stackChunkFrameStream_riscv.inline.hpp @@ -114,6 +114,7 @@ inline int StackChunkFrameStream::interpreter_frame_num_oops() const f.interpreted_frame_oop_map(&mask); return mask.num_oops() + 1 // for the mirror oop + + (f.interpreter_frame_method()->is_native() ? 1 : 0) // temp oop slot + pointer_delta_as_int((intptr_t*)f.interpreter_frame_monitor_begin(), (intptr_t*)f.interpreter_frame_monitor_end()) / BasicObjectLock::size(); } diff --git a/src/hotspot/cpu/riscv/stubGenerator_riscv.cpp b/src/hotspot/cpu/riscv/stubGenerator_riscv.cpp index bce0c8f1f3d..e4ae4f3cfd1 100644 --- a/src/hotspot/cpu/riscv/stubGenerator_riscv.cpp +++ b/src/hotspot/cpu/riscv/stubGenerator_riscv.cpp @@ -4040,6 +4040,36 @@ class StubGenerator: public StubCodeGenerator { return start; } + address generate_cont_preempt_stub() { + if (!Continuations::enabled()) return nullptr; + StubCodeMark mark(this, "StubRoutines","Continuation preempt stub"); + address start = __ pc(); + + __ reset_last_Java_frame(true); + + // Set sp to enterSpecial frame, i.e. remove all frames copied into the heap. + __ ld(sp, Address(xthread, JavaThread::cont_entry_offset())); + + Label preemption_cancelled; + __ lbu(t0, Address(xthread, JavaThread::preemption_cancelled_offset())); + __ bnez(t0, preemption_cancelled); + + // Remove enterSpecial frame from the stack and return to Continuation.run() to unmount. + SharedRuntime::continuation_enter_cleanup(_masm); + __ leave(); + __ ret(); + + // We acquired the monitor after freezing the frames so call thaw to continue execution. + __ bind(preemption_cancelled); + __ sb(zr, Address(xthread, JavaThread::preemption_cancelled_offset())); + __ la(fp, Address(sp, checked_cast(ContinuationEntry::size() + 2 * wordSize))); + __ la(t1, ExternalAddress(ContinuationEntry::thaw_call_pc_address())); + __ ld(t1, Address(t1)); + __ jr(t1); + + return start; + } + #if COMPILER2_OR_JVMCI #undef __ @@ -6402,6 +6432,7 @@ static const int64_t right_3_bits = right_n_bits(3); StubRoutines::_cont_thaw = generate_cont_thaw(); StubRoutines::_cont_returnBarrier = generate_cont_returnBarrier(); StubRoutines::_cont_returnBarrierExc = generate_cont_returnBarrier_exception(); + StubRoutines::_cont_preempt_stub = generate_cont_preempt_stub(); } void generate_final_stubs() { diff --git a/src/hotspot/cpu/riscv/stubRoutines_riscv.hpp b/src/hotspot/cpu/riscv/stubRoutines_riscv.hpp index 46b5461d777..3d1f4a61f00 100644 --- a/src/hotspot/cpu/riscv/stubRoutines_riscv.hpp +++ b/src/hotspot/cpu/riscv/stubRoutines_riscv.hpp @@ -39,7 +39,7 @@ enum platform_dependent_constants { // simply increase sizes if too small (assembler will crash if too small) _initial_stubs_code_size = 10000, _continuation_stubs_code_size = 2000, - _compiler_stubs_code_size = 25000 ZGC_ONLY(+5000), + _compiler_stubs_code_size = 45000, _final_stubs_code_size = 20000 ZGC_ONLY(+10000) }; diff --git a/src/hotspot/cpu/riscv/templateInterpreterGenerator_riscv.cpp b/src/hotspot/cpu/riscv/templateInterpreterGenerator_riscv.cpp index 0281c66b97d..ac28f4b3514 100644 --- a/src/hotspot/cpu/riscv/templateInterpreterGenerator_riscv.cpp +++ b/src/hotspot/cpu/riscv/templateInterpreterGenerator_riscv.cpp @@ -539,6 +539,38 @@ address TemplateInterpreterGenerator::generate_safept_entry_for(TosState state, return entry; } +address TemplateInterpreterGenerator::generate_cont_resume_interpreter_adapter() { + if (!Continuations::enabled()) return nullptr; + address start = __ pc(); + + __ restore_bcp(); + __ restore_locals(); + + // Restore constant pool cache + __ ld(xcpool, Address(fp, frame::interpreter_frame_cache_offset * wordSize)); + + // Restore Java expression stack pointer + __ ld(t0, Address(fp, frame::interpreter_frame_last_sp_offset * wordSize)); + __ shadd(esp, t0, fp, t0, Interpreter::logStackElementSize); + // and NULL it as marker that esp is now tos until next java call + __ sd(zr, Address(fp, frame::interpreter_frame_last_sp_offset * wordSize)); + + // Restore machine SP + __ ld(t0, Address(fp, frame::interpreter_frame_extended_sp_offset * wordSize)); + __ shadd(sp, t0, fp, t0, LogBytesPerWord); + + // Restore method + __ ld(xmethod, Address(fp, frame::interpreter_frame_method_offset * wordSize)); + + // Restore dispatch + __ la(xdispatch, ExternalAddress((address)Interpreter::dispatch_table())); + + __ ret(); + + return start; +} + + // Helpers for commoning out cases in the various type of method entries. // @@ -1092,6 +1124,9 @@ address TemplateInterpreterGenerator::generate_native_entry(bool synchronized) { // result handler is in x10 // set result handler __ mv(result_handler, x10); + // Save it in the frame in case of preemption; we cannot rely on callee saved registers. + __ sd(x10, Address(fp, frame::interpreter_frame_result_handler_offset * wordSize)); + // pass mirror handle if static call { Label L; @@ -1130,6 +1165,8 @@ address TemplateInterpreterGenerator::generate_native_entry(bool synchronized) { // It is enough that the pc() points into the right code // segment. It does not have to be the correct return pc. + // For convenience we use the pc we want to resume to in + // case of preemption on Object.wait. Label native_return; __ set_last_Java_frame(esp, fp, native_return, x30); @@ -1151,9 +1188,13 @@ address TemplateInterpreterGenerator::generate_native_entry(bool synchronized) { __ membar(MacroAssembler::LoadStore | MacroAssembler::StoreStore); __ sw(t0, Address(t1)); + __ push_cont_fastpath(); + // Call the native method. __ jalr(x28); - __ bind(native_return); + + __ pop_cont_fastpath(); + __ get_method(xmethod); // result potentially in x10 or f10 @@ -1219,6 +1260,23 @@ address TemplateInterpreterGenerator::generate_native_entry(bool synchronized) { __ mv(t0, _thread_in_Java); __ sw(t0, Address(xthread, JavaThread::thread_state_offset())); + if (LockingMode != LM_LEGACY) { + // Check preemption for Object.wait() + Label not_preempted; + __ ld(t1, Address(xthread, JavaThread::preempt_alternate_return_offset())); + __ beqz(t1, not_preempted); + __ sd(zr, Address(xthread, JavaThread::preempt_alternate_return_offset())); + __ jr(t1); + __ bind(native_return); + __ restore_after_resume(true /* is_native */); + // reload result_handler + __ ld(result_handler, Address(fp, frame::interpreter_frame_result_handler_offset * wordSize)); + __ bind(not_preempted); + } else { + // any pc will do so just use this one for LM_LEGACY to keep code together. + __ bind(native_return); + } + // reset_last_Java_frame __ reset_last_Java_frame(true); diff --git a/src/hotspot/cpu/riscv/templateTable_riscv.cpp b/src/hotspot/cpu/riscv/templateTable_riscv.cpp index a204c9fdd45..52b33c62616 100644 --- a/src/hotspot/cpu/riscv/templateTable_riscv.cpp +++ b/src/hotspot/cpu/riscv/templateTable_riscv.cpp @@ -3546,12 +3546,22 @@ void TemplateTable::_new() { // The object is initialized before the header. If the object size is // zero, go directly to the header initialization. - __ sub(x13, x13, sizeof(oopDesc)); + if (UseCompactObjectHeaders) { + assert(is_aligned(oopDesc::base_offset_in_bytes(), BytesPerLong), "oop base offset must be 8-byte-aligned"); + __ sub(x13, x13, oopDesc::base_offset_in_bytes()); + } else { + __ sub(x13, x13, sizeof(oopDesc)); + } __ beqz(x13, initialize_header); // Initialize object fields { - __ add(x12, x10, sizeof(oopDesc)); + if (UseCompactObjectHeaders) { + assert(is_aligned(oopDesc::base_offset_in_bytes(), BytesPerLong), "oop base offset must be 8-byte-aligned"); + __ add(x12, x10, oopDesc::base_offset_in_bytes()); + } else { + __ add(x12, x10, sizeof(oopDesc)); + } Label loop; __ bind(loop); __ sd(zr, Address(x12)); @@ -3562,10 +3572,15 @@ void TemplateTable::_new() { // initialize object hader only. __ bind(initialize_header); - __ mv(t0, (intptr_t)markWord::prototype().value()); - __ sd(t0, Address(x10, oopDesc::mark_offset_in_bytes())); - __ store_klass_gap(x10, zr); // zero klass gap for compressed oops - __ store_klass(x10, x14); // store klass last + if (UseCompactObjectHeaders) { + __ ld(t0, Address(x14, Klass::prototype_header_offset())); + __ sd(t0, Address(x10, oopDesc::mark_offset_in_bytes())); + } else { + __ mv(t0, (intptr_t)markWord::prototype().value()); + __ sd(t0, Address(x10, oopDesc::mark_offset_in_bytes())); + __ store_klass_gap(x10, zr); // zero klass gap for compressed oops + __ store_klass(x10, x14); // store klass last + } if (DTraceAllocProbes) { // Trigger dtrace event for fastpath diff --git a/src/hotspot/cpu/riscv/vm_version_riscv.cpp b/src/hotspot/cpu/riscv/vm_version_riscv.cpp index 5c4e3ec1df2..6e8ee1ab7e0 100644 --- a/src/hotspot/cpu/riscv/vm_version_riscv.cpp +++ b/src/hotspot/cpu/riscv/vm_version_riscv.cpp @@ -122,22 +122,6 @@ void VM_Version::common_initialize() { FLAG_SET_DEFAULT(AllocatePrefetchDistance, 0); } - if (UseZba) { - if (FLAG_IS_DEFAULT(UseCRC32Intrinsics)) { - FLAG_SET_DEFAULT(UseCRC32Intrinsics, true); - } - } else { - if (!FLAG_IS_DEFAULT(UseCRC32Intrinsics)) { - warning("CRC32 intrinsic requires Zba instructions (not available on this CPU)"); - } - FLAG_SET_DEFAULT(UseCRC32Intrinsics, false); - } - - if (UseCRC32CIntrinsics) { - warning("CRC32C intrinsics are not available on this CPU."); - FLAG_SET_DEFAULT(UseCRC32CIntrinsics, false); - } - if (UseVectorizedMismatchIntrinsic) { warning("VectorizedMismatch intrinsic is not available on this CPU."); FLAG_SET_DEFAULT(UseVectorizedMismatchIntrinsic, false); @@ -217,6 +201,24 @@ void VM_Version::common_initialize() { _initial_vector_length = cpu_vector_length(); } } + + // Misc Intrinsics could depend on RVV + + if (UseZba || UseRVV) { + if (FLAG_IS_DEFAULT(UseCRC32Intrinsics)) { + FLAG_SET_DEFAULT(UseCRC32Intrinsics, true); + } + } else { + if (!FLAG_IS_DEFAULT(UseCRC32Intrinsics)) { + warning("CRC32 intrinsic requires Zba or RVV instructions (not available on this CPU)"); + } + FLAG_SET_DEFAULT(UseCRC32Intrinsics, false); + } + + if (UseCRC32CIntrinsics) { + warning("CRC32C intrinsics are not available on this CPU."); + FLAG_SET_DEFAULT(UseCRC32CIntrinsics, false); + } } #ifdef COMPILER2 diff --git a/src/hotspot/cpu/s390/assembler_s390.hpp b/src/hotspot/cpu/s390/assembler_s390.hpp index ab09460bd86..c98c100a068 100644 --- a/src/hotspot/cpu/s390/assembler_s390.hpp +++ b/src/hotspot/cpu/s390/assembler_s390.hpp @@ -56,24 +56,23 @@ class Immediate { } // Test if x is within signed immediate range for nbits. - static bool is_uimm(int64_t x, unsigned int nbits) { + static bool is_uimm(uint64_t x, unsigned int nbits) { // nbits == 0 --> false // nbits >= 64 --> true assert(1 <= nbits && nbits < 64, "don't call, use statically known result"); - const uint64_t xu = (unsigned long)x; const uint64_t maxplus1 = 1UL << nbits; - return xu < maxplus1; // Unsigned comparison. Negative inputs appear to be very large. + return x < maxplus1; // Unsigned comparison. Negative inputs appear to be very large. } - static bool is_uimm32(int64_t x) { + static bool is_uimm32(uint64_t x) { return is_uimm(x, 32); } - static bool is_uimm16(int64_t x) { + static bool is_uimm16(uint64_t x) { return is_uimm(x, 16); } - static bool is_uimm12(int64_t x) { + static bool is_uimm12(uint64_t x) { return is_uimm(x, 12); } - static bool is_uimm8(int64_t x) { + static bool is_uimm8(uint64_t x) { return is_uimm(x, 8); } }; diff --git a/src/hotspot/cpu/s390/c1_LIRAssembler_s390.cpp b/src/hotspot/cpu/s390/c1_LIRAssembler_s390.cpp index d2e860aa320..213aa5efe1e 100644 --- a/src/hotspot/cpu/s390/c1_LIRAssembler_s390.cpp +++ b/src/hotspot/cpu/s390/c1_LIRAssembler_s390.cpp @@ -2047,8 +2047,6 @@ void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) { Address src_length_addr = Address(src, arrayOopDesc::length_offset_in_bytes()); Address dst_length_addr = Address(dst, arrayOopDesc::length_offset_in_bytes()); - Address src_klass_addr = Address(src, oopDesc::klass_offset_in_bytes()); - Address dst_klass_addr = Address(dst, oopDesc::klass_offset_in_bytes()); // Length and pos's are all sign extended at this point on 64bit. @@ -2112,13 +2110,7 @@ void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) { // We don't know the array types are compatible. if (basic_type != T_OBJECT) { // Simple test for basic type arrays. - if (UseCompressedClassPointers) { - __ z_l(tmp, src_klass_addr); - __ z_c(tmp, dst_klass_addr); - } else { - __ z_lg(tmp, src_klass_addr); - __ z_cg(tmp, dst_klass_addr); - } + __ cmp_klasses_from_objects(src, dst, tmp, Z_R1_scratch); __ branch_optimized(Assembler::bcondNotEqual, *stub->entry()); } else { // For object arrays, if src is a sub class of dst then we can @@ -2252,15 +2244,13 @@ void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) { } if (basic_type != T_OBJECT) { - if (UseCompressedClassPointers) { __ z_c (tmp, dst_klass_addr); } - else { __ z_cg(tmp, dst_klass_addr); } + __ cmp_klass(tmp, dst, Z_R1_scratch); __ branch_optimized(Assembler::bcondNotEqual, halt); - if (UseCompressedClassPointers) { __ z_c (tmp, src_klass_addr); } - else { __ z_cg(tmp, src_klass_addr); } + + __ cmp_klass(tmp, src, Z_R1_scratch); __ branch_optimized(Assembler::bcondEqual, known_ok); } else { - if (UseCompressedClassPointers) { __ z_c (tmp, dst_klass_addr); } - else { __ z_cg(tmp, dst_klass_addr); } + __ cmp_klass(tmp, dst, Z_R1_scratch); __ branch_optimized(Assembler::bcondEqual, known_ok); __ compareU64_and_branch(src, dst, Assembler::bcondEqual, known_ok); } @@ -2755,12 +2745,7 @@ void LIR_Assembler::emit_load_klass(LIR_OpLoadKlass* op) { add_debug_info_for_null_check_here(info); } - if (UseCompressedClassPointers) { - __ z_llgf(result, Address(obj, oopDesc::klass_offset_in_bytes())); - __ decode_klass_not_null(result); - } else { - __ z_lg(result, Address(obj, oopDesc::klass_offset_in_bytes())); - } + __ load_klass(result, obj); } void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) { ciMethod* method = op->profiled_method(); diff --git a/src/hotspot/cpu/s390/c1_MacroAssembler_s390.cpp b/src/hotspot/cpu/s390/c1_MacroAssembler_s390.cpp index 41c2ae260a6..bc269f9353c 100644 --- a/src/hotspot/cpu/s390/c1_MacroAssembler_s390.cpp +++ b/src/hotspot/cpu/s390/c1_MacroAssembler_s390.cpp @@ -177,17 +177,21 @@ void C1_MacroAssembler::try_allocate( void C1_MacroAssembler::initialize_header(Register obj, Register klass, Register len, Register Rzero, Register t1) { assert_different_registers(obj, klass, len, t1, Rzero); - // This assumes that all prototype bits fit in an int32_t. - load_const_optimized(t1, (intx)markWord::prototype().value()); - z_stg(t1, Address(obj, oopDesc::mark_offset_in_bytes())); + if (UseCompactObjectHeaders) { + z_lg(t1, Address(klass, in_bytes(Klass::prototype_header_offset()))); + z_stg(t1, Address(obj, oopDesc::mark_offset_in_bytes())); + } else { + load_const_optimized(t1, (intx)markWord::prototype().value()); + z_stg(t1, Address(obj, oopDesc::mark_offset_in_bytes())); + store_klass(klass, obj, t1); + } if (len->is_valid()) { // Length will be in the klass gap, if one exists. z_st(len, Address(obj, arrayOopDesc::length_offset_in_bytes())); - } else if (UseCompressedClassPointers) { + } else if (UseCompressedClassPointers && !UseCompactObjectHeaders) { store_klass_gap(Rzero, obj); // Zero klass gap for compressed oops. } - store_klass(klass, obj, t1); } void C1_MacroAssembler::initialize_body(Register objectFields, Register len_in_bytes, Register Rzero) { diff --git a/src/hotspot/cpu/s390/c1_Runtime1_s390.cpp b/src/hotspot/cpu/s390/c1_Runtime1_s390.cpp index 2f629c108c9..8b30adb4785 100644 --- a/src/hotspot/cpu/s390/c1_Runtime1_s390.cpp +++ b/src/hotspot/cpu/s390/c1_Runtime1_s390.cpp @@ -208,6 +208,11 @@ void Runtime1::initialize_pd() { // Nothing to do. } +uint Runtime1::runtime_blob_current_thread_offset(frame f) { + Unimplemented(); + return 0; +} + OopMapSet* Runtime1::generate_exception_throw(StubAssembler* sasm, address target, bool has_argument) { // Make a frame and preserve the caller's caller-save registers. OopMap* oop_map = save_live_registers(sasm); diff --git a/src/hotspot/cpu/s390/c2_MacroAssembler_s390.cpp b/src/hotspot/cpu/s390/c2_MacroAssembler_s390.cpp index 025ef4c8915..378d5e4cfe1 100644 --- a/src/hotspot/cpu/s390/c2_MacroAssembler_s390.cpp +++ b/src/hotspot/cpu/s390/c2_MacroAssembler_s390.cpp @@ -42,6 +42,13 @@ void C2_MacroAssembler::fast_unlock_lightweight(Register obj, Register box, Regi compiler_fast_unlock_lightweight_object(obj, box, temp1, temp2); } +void C2_MacroAssembler::load_narrow_klass_compact_c2(Register dst, Address src) { + // The incoming address is pointing into obj-start + klass_offset_in_bytes. We need to extract + // obj-start, so that we can load from the object's mark-word instead. + z_lg(dst, src.plus_disp(-oopDesc::klass_offset_in_bytes())); + z_srlg(dst, dst, markWord::klass_shift); // TODO: could be z_sra +} + //------------------------------------------------------ // Special String Intrinsics. Implementation //------------------------------------------------------ diff --git a/src/hotspot/cpu/s390/c2_MacroAssembler_s390.hpp b/src/hotspot/cpu/s390/c2_MacroAssembler_s390.hpp index aecb483f0a6..26c3c9a2bb5 100644 --- a/src/hotspot/cpu/s390/c2_MacroAssembler_s390.hpp +++ b/src/hotspot/cpu/s390/c2_MacroAssembler_s390.hpp @@ -33,6 +33,8 @@ void fast_lock_lightweight(Register obj, Register box, Register temp1, Register temp2); void fast_unlock_lightweight(Register obj, Register box, Register temp1, Register temp2); + void load_narrow_klass_compact_c2(Register dst, Address src); + //------------------------------------------- // Special String Intrinsics Implementation. //------------------------------------------- diff --git a/src/hotspot/cpu/s390/continuationFreezeThaw_s390.inline.hpp b/src/hotspot/cpu/s390/continuationFreezeThaw_s390.inline.hpp index 084ee189211..7223facaaeb 100644 --- a/src/hotspot/cpu/s390/continuationFreezeThaw_s390.inline.hpp +++ b/src/hotspot/cpu/s390/continuationFreezeThaw_s390.inline.hpp @@ -48,6 +48,10 @@ void FreezeBase::adjust_interpreted_frame_unextended_sp(frame& f) { Unimplemented(); } +inline void FreezeBase::prepare_freeze_interpreted_top_frame(frame& f) { + Unimplemented(); +} + inline void FreezeBase::relativize_interpreted_frame_metadata(const frame& f, const frame& hf) { Unimplemented(); } @@ -83,6 +87,15 @@ inline void ThawBase::patch_pd(frame& f, const frame& caller) { Unimplemented(); } +inline void ThawBase::patch_pd(frame& f, intptr_t* caller_sp) { + Unimplemented(); +} + +inline intptr_t* ThawBase::push_cleanup_continuation() { + Unimplemented(); + return nullptr; +} + template inline void Thaw::patch_caller_links(intptr_t* sp, intptr_t* bottom) { Unimplemented(); diff --git a/src/hotspot/cpu/s390/continuationHelper_s390.inline.hpp b/src/hotspot/cpu/s390/continuationHelper_s390.inline.hpp index 4cea0459551..8c57f1810e5 100644 --- a/src/hotspot/cpu/s390/continuationHelper_s390.inline.hpp +++ b/src/hotspot/cpu/s390/continuationHelper_s390.inline.hpp @@ -35,6 +35,10 @@ static inline intptr_t** link_address(const frame& f) { return nullptr; } +static inline void patch_return_pc_with_preempt_stub(frame& f) { + Unimplemented(); +} + inline int ContinuationHelper::frame_align_words(int size) { Unimplemented(); return 0; @@ -62,11 +66,11 @@ inline void ContinuationHelper::set_anchor_to_entry_pd(JavaFrameAnchor* anchor, Unimplemented(); } -#ifdef ASSERT inline void ContinuationHelper::set_anchor_pd(JavaFrameAnchor* anchor, intptr_t* sp) { Unimplemented(); } +#ifdef ASSERT inline bool ContinuationHelper::Frame::assert_frame_laid_out(frame f) { Unimplemented(); return false; diff --git a/src/hotspot/cpu/s390/frame_s390.cpp b/src/hotspot/cpu/s390/frame_s390.cpp index 11c7386dfd7..75f29be13ab 100644 --- a/src/hotspot/cpu/s390/frame_s390.cpp +++ b/src/hotspot/cpu/s390/frame_s390.cpp @@ -246,6 +246,11 @@ frame frame::sender_for_upcall_stub_frame(RegisterMap* map) const { return fr; } +JavaThread** frame::saved_thread_address(const frame& f) { + Unimplemented(); + return nullptr; +} + frame frame::sender_for_interpreter_frame(RegisterMap *map) const { // Pass callers sender_sp as unextended_sp. return frame(sender_sp(), sender_pc(), (intptr_t*)(ijava_state()->sender_sp)); diff --git a/src/hotspot/cpu/s390/macroAssembler_s390.cpp b/src/hotspot/cpu/s390/macroAssembler_s390.cpp index 84d09b9c1dc..9e1c5cbced3 100644 --- a/src/hotspot/cpu/s390/macroAssembler_s390.cpp +++ b/src/hotspot/cpu/s390/macroAssembler_s390.cpp @@ -2160,7 +2160,16 @@ void MacroAssembler::call_VM_leaf_base(address entry_point) { } int MacroAssembler::ic_check_size() { - return 30 + (ImplicitNullChecks ? 0 : 6); + int ic_size = 24; + if (!ImplicitNullChecks) { + ic_size += 6; + } + if (UseCompactObjectHeaders) { + ic_size += 12; + } else { + ic_size += 6; // either z_llgf or z_lg + } + return ic_size; } int MacroAssembler::ic_check(int end_alignment) { @@ -2181,7 +2190,9 @@ int MacroAssembler::ic_check(int end_alignment) { z_cgij(R2_receiver, 0, Assembler::bcondEqual, failure); } - if (UseCompressedClassPointers) { + if (UseCompactObjectHeaders) { + load_narrow_klass_compact(R1_scratch, R2_receiver); + } else if (UseCompressedClassPointers) { z_llgf(R1_scratch, Address(R2_receiver, oopDesc::klass_offset_in_bytes())); } else { z_lg(R1_scratch, Address(R2_receiver, oopDesc::klass_offset_in_bytes())); @@ -3492,7 +3503,7 @@ void MacroAssembler::increment_counter_eq(address counter_address, Register tmp1 void MacroAssembler::compiler_fast_lock_object(Register oop, Register box, Register temp1, Register temp2) { assert(LockingMode != LM_LIGHTWEIGHT, "uses fast_lock_lightweight"); - assert_different_registers(oop, box, temp1, temp2); + assert_different_registers(oop, box, temp1, temp2, Z_R0_scratch); Register displacedHeader = temp1; Register currentHeader = temp1; @@ -3561,14 +3572,13 @@ void MacroAssembler::compiler_fast_lock_object(Register oop, Register box, Regis Register zero = temp; Register monitor_tagged = displacedHeader; // Tagged with markWord::monitor_value. - // The object's monitor m is unlocked iff m->owner is null, - // otherwise m->owner may contain a thread or a stack address. - // Try to CAS m->owner from null to current thread. - // If m->owner is null, then csg succeeds and sets m->owner=THREAD and CR=EQ. - // Otherwise, register zero is filled with the current owner. + // Try to CAS owner (no owner => current thread's _lock_id). + // If csg succeeds then CR=EQ, otherwise, register zero is filled + // with the current owner. z_lghi(zero, 0); - z_csg(zero, Z_thread, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner), monitor_tagged); + z_lg(Z_R0_scratch, Address(Z_thread, JavaThread::lock_id_offset())); + z_csg(zero, Z_R0_scratch, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner), monitor_tagged); // Store a non-null value into the box. z_stg(box, BasicLock::displaced_header_offset_in_bytes(), box); @@ -3577,7 +3587,7 @@ void MacroAssembler::compiler_fast_lock_object(Register oop, Register box, Regis BLOCK_COMMENT("fast_path_recursive_lock {"); // Check if we are already the owner (recursive lock) - z_cgr(Z_thread, zero); // owner is stored in zero by "z_csg" above + z_cgr(Z_R0_scratch, zero); // owner is stored in zero by "z_csg" above z_brne(done); // not a recursive lock // Current thread already owns the lock. Just increment recursion count. @@ -3595,7 +3605,7 @@ void MacroAssembler::compiler_fast_lock_object(Register oop, Register box, Regis void MacroAssembler::compiler_fast_unlock_object(Register oop, Register box, Register temp1, Register temp2) { assert(LockingMode != LM_LIGHTWEIGHT, "uses fast_unlock_lightweight"); - assert_different_registers(oop, box, temp1, temp2); + assert_different_registers(oop, box, temp1, temp2, Z_R0_scratch); Register displacedHeader = temp1; Register currentHeader = temp2; @@ -3642,7 +3652,8 @@ void MacroAssembler::compiler_fast_unlock_object(Register oop, Register box, Reg // Handle existing monitor. bind(object_has_monitor); - z_cg(Z_thread, Address(currentHeader, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner))); + z_lg(Z_R0_scratch, Address(Z_thread, JavaThread::lock_id_offset())); + z_cg(Z_R0_scratch, Address(currentHeader, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner))); z_brne(done); BLOCK_COMMENT("fast_path_recursive_unlock {"); @@ -3852,7 +3863,7 @@ void MacroAssembler::encode_klass_not_null(Register dst, Register src) { #ifdef ASSERT Label ok; - z_tmll(current, KlassAlignmentInBytes-1); // Check alignment. + z_tmll(current, CompressedKlassPointers::klass_alignment_in_bytes() - 1); // Check alignment. z_brc(Assembler::bcondAllZero, ok); // The plain disassembler does not recognize illtrap. It instead displays // a 32-bit value. Issuing two illtraps assures the disassembler finds @@ -3866,7 +3877,6 @@ void MacroAssembler::encode_klass_not_null(Register dst, Register src) { // We then can be sure we calculate an offset that fits into 32 bit. // More generally speaking: all subsequent calculations are purely 32-bit. if (shift != 0) { - assert (LogKlassAlignmentInBytes == shift, "decode alg wrong"); z_srlg(dst, current, shift); current = dst; } @@ -3996,7 +4006,7 @@ void MacroAssembler::decode_klass_not_null(Register dst) { #ifdef ASSERT Label ok; - z_tmll(dst, KlassAlignmentInBytes-1); // Check alignment. + z_tmll(dst, CompressedKlassPointers::klass_alignment_in_bytes() - 1); // Check alignment. z_brc(Assembler::bcondAllZero, ok); // The plain disassembler does not recognize illtrap. It instead displays // a 32-bit value. Issuing two illtraps assures the disassembler finds @@ -4043,7 +4053,7 @@ void MacroAssembler::decode_klass_not_null(Register dst, Register src) { #ifdef ASSERT Label ok; - z_tmll(dst, KlassAlignmentInBytes-1); // Check alignment. + z_tmll(dst, CompressedKlassPointers::klass_alignment_in_bytes() - 1); // Check alignment. z_brc(Assembler::bcondAllZero, ok); // The plain disassembler does not recognize illtrap. It instead displays // a 32-bit value. Issuing two illtraps assures the disassembler finds @@ -4065,10 +4075,58 @@ void MacroAssembler::load_klass(Register klass, Address mem) { } } +// Loads the obj's Klass* into dst. +// Input: +// src - the oop we want to load the klass from. +// dst - output nklass. +void MacroAssembler::load_narrow_klass_compact(Register dst, Register src) { + BLOCK_COMMENT("load_narrow_klass_compact {"); + assert(UseCompactObjectHeaders, "expects UseCompactObjectHeaders"); + z_lg(dst, Address(src, oopDesc::mark_offset_in_bytes())); + z_srlg(dst, dst, markWord::klass_shift); + BLOCK_COMMENT("} load_narrow_klass_compact"); +} + +void MacroAssembler::cmp_klass(Register klass, Register obj, Register tmp) { + BLOCK_COMMENT("cmp_klass {"); + assert_different_registers(obj, klass, tmp); + if (UseCompactObjectHeaders) { + assert(tmp != noreg, "required"); + assert_different_registers(klass, obj, tmp); + load_narrow_klass_compact(tmp, obj); + z_cr(klass, tmp); + } else if (UseCompressedClassPointers) { + z_c(klass, Address(obj, oopDesc::klass_offset_in_bytes())); + } else { + z_cg(klass, Address(obj, oopDesc::klass_offset_in_bytes())); + } + BLOCK_COMMENT("} cmp_klass"); +} + +void MacroAssembler::cmp_klasses_from_objects(Register obj1, Register obj2, Register tmp1, Register tmp2) { + BLOCK_COMMENT("cmp_klasses_from_objects {"); + if (UseCompactObjectHeaders) { + assert(tmp1 != noreg && tmp2 != noreg, "required"); + assert_different_registers(obj1, obj2, tmp1, tmp2); + load_narrow_klass_compact(tmp1, obj1); + load_narrow_klass_compact(tmp2, obj2); + z_cr(tmp1, tmp2); + } else if (UseCompressedClassPointers) { + z_l(tmp1, Address(obj1, oopDesc::klass_offset_in_bytes())); + z_c(tmp1, Address(obj2, oopDesc::klass_offset_in_bytes())); + } else { + z_lg(tmp1, Address(obj1, oopDesc::klass_offset_in_bytes())); + z_cg(tmp1, Address(obj2, oopDesc::klass_offset_in_bytes())); + } + BLOCK_COMMENT("} cmp_klasses_from_objects"); +} + void MacroAssembler::load_klass(Register klass, Register src_oop) { - if (UseCompressedClassPointers) { + if (UseCompactObjectHeaders) { + load_narrow_klass_compact(klass, src_oop); + decode_klass_not_null(klass); + } else if (UseCompressedClassPointers) { z_llgf(klass, oopDesc::klass_offset_in_bytes(), src_oop); - // Attention: no null check here! decode_klass_not_null(klass); } else { z_lg(klass, oopDesc::klass_offset_in_bytes(), src_oop); @@ -4076,6 +4134,7 @@ void MacroAssembler::load_klass(Register klass, Register src_oop) { } void MacroAssembler::store_klass(Register klass, Register dst_oop, Register ck) { + assert(!UseCompactObjectHeaders, "Don't use with compact headers"); if (UseCompressedClassPointers) { assert_different_registers(dst_oop, klass, Z_R0); if (ck == noreg) ck = klass; @@ -4087,6 +4146,7 @@ void MacroAssembler::store_klass(Register klass, Register dst_oop, Register ck) } void MacroAssembler::store_klass_gap(Register s, Register d) { + assert(!UseCompactObjectHeaders, "Don't use with compact headers"); if (UseCompressedClassPointers) { assert(s != d, "not enough registers"); // Support s = noreg. @@ -4112,7 +4172,11 @@ void MacroAssembler::compare_klass_ptr(Register Rop1, int64_t disp, Register Rba const int shift = CompressedKlassPointers::shift(); address base = CompressedKlassPointers::base(); - assert((shift == 0) || (shift == LogKlassAlignmentInBytes), "cKlass encoder detected bad shift"); + if (UseCompactObjectHeaders) { + assert(shift >= 3, "cKlass encoder detected bad shift"); + } else { + assert((shift == 0) || (shift == 3), "cKlass encoder detected bad shift"); + } assert_different_registers(Rop1, Z_R0); assert_different_registers(Rop1, Rbase, Z_R1); @@ -6164,7 +6228,7 @@ void MacroAssembler::lightweight_unlock(Register obj, Register temp1, Register t } void MacroAssembler::compiler_fast_lock_lightweight_object(Register obj, Register box, Register tmp1, Register tmp2) { - assert_different_registers(obj, box, tmp1, tmp2); + assert_different_registers(obj, box, tmp1, tmp2, Z_R0_scratch); // Handle inflated monitor. NearLabel inflated; @@ -6292,15 +6356,16 @@ void MacroAssembler::compiler_fast_lock_lightweight_object(Register obj, Registe const Address recursions_address(tmp1_monitor, ObjectMonitor::recursions_offset() - monitor_tag); - // Try to CAS m->owner from null to current thread. - // If m->owner is null, then csg succeeds and sets m->owner=THREAD and CR=EQ. - // Otherwise, register zero is filled with the current owner. + // Try to CAS owner (no owner => current thread's _lock_id). + // If csg succeeds then CR=EQ, otherwise, register zero is filled + // with the current owner. z_lghi(zero, 0); - z_csg(zero, Z_thread, owner_address); + z_lg(Z_R0_scratch, Address(Z_thread, JavaThread::lock_id_offset())); + z_csg(zero, Z_R0_scratch, owner_address); z_bre(monitor_locked); // Check if recursive. - z_cgr(Z_thread, zero); // zero contains the owner from z_csg instruction + z_cgr(Z_R0_scratch, zero); // zero contains the owner from z_csg instruction z_brne(slow_path); // Recursive diff --git a/src/hotspot/cpu/s390/macroAssembler_s390.hpp b/src/hotspot/cpu/s390/macroAssembler_s390.hpp index 061817a1289..7806fef3ce8 100644 --- a/src/hotspot/cpu/s390/macroAssembler_s390.hpp +++ b/src/hotspot/cpu/s390/macroAssembler_s390.hpp @@ -803,6 +803,13 @@ class MacroAssembler: public Assembler { void load_klass(Register klass, Register src_oop); void store_klass(Register klass, Register dst_oop, Register ck = noreg); // Klass will get compressed if ck not provided. void store_klass_gap(Register s, Register dst_oop); + void load_narrow_klass_compact(Register dst, Register src); + // Compares the Klass pointer of an object to a given Klass (which might be narrow, + // depending on UseCompressedClassPointers). + void cmp_klass(Register klass, Register obj, Register tmp); + // Compares the Klass pointer of two objects obj1 and obj2. Result is in the condition flags. + // Uses tmp1 and tmp2 as temporary registers. + void cmp_klasses_from_objects(Register obj1, Register obj2, Register tmp1, Register tmp2); // This function calculates the size of the code generated by // decode_klass_not_null(register dst) diff --git a/src/hotspot/cpu/s390/s390.ad b/src/hotspot/cpu/s390/s390.ad index 288de48c105..63e150c9e9c 100644 --- a/src/hotspot/cpu/s390/s390.ad +++ b/src/hotspot/cpu/s390/s390.ad @@ -2547,7 +2547,7 @@ operand uimmI8() %{ // operand type int // Unsigned Integer Immediate: 9-bit operand SSlenDW() %{ - predicate(Immediate::is_uimm8(n->get_long()-1)); + predicate(Immediate::is_uimm8((julong)n->get_long()-1)); match(ConL); op_cost(1); format %{ %} @@ -4410,6 +4410,7 @@ instruct loadN(iRegN dst, memory mem) %{ // Load narrow Klass Pointer instruct loadNKlass(iRegN dst, memory mem) %{ + predicate(!UseCompactObjectHeaders); match(Set dst (LoadNKlass mem)); ins_cost(MEMORY_REF_COST); size(Z_DISP3_SIZE); @@ -4419,6 +4420,21 @@ instruct loadNKlass(iRegN dst, memory mem) %{ ins_pipe(pipe_class_dummy); %} +instruct loadNKlassCompactHeaders(iRegN dst, memory mem, flagsReg cr) %{ + match(Set dst (LoadNKlass mem)); + predicate(UseCompactObjectHeaders); + effect(KILL cr); + ins_cost(MEMORY_REF_COST); + format %{ "load_narrow_klass_compact $dst,$mem \t# compressed class ptr" %} + // TODO: size() + ins_encode %{ + __ block_comment("load_narrow_klass_compact_c2 {"); + __ load_narrow_klass_compact_c2($dst$$Register, $mem$$Address); + __ block_comment("} load_narrow_klass_compact"); + %} + ins_pipe(pipe_class_dummy); +%} + // Load constant Compressed Pointer instruct loadConN(iRegN dst, immN src) %{ diff --git a/src/hotspot/cpu/s390/sharedRuntime_s390.cpp b/src/hotspot/cpu/s390/sharedRuntime_s390.cpp index 468610b588e..c60f6ef3295 100644 --- a/src/hotspot/cpu/s390/sharedRuntime_s390.cpp +++ b/src/hotspot/cpu/s390/sharedRuntime_s390.cpp @@ -2387,6 +2387,11 @@ uint SharedRuntime::out_preserve_stack_slots() { return frame::z_jit_out_preserve_size/VMRegImpl::stack_slot_size; } +VMReg SharedRuntime::thread_register() { + Unimplemented(); + return nullptr; +} + // // Frame generation for deopt and uncommon trap blobs. // diff --git a/src/hotspot/cpu/s390/templateInterpreterGenerator_s390.cpp b/src/hotspot/cpu/s390/templateInterpreterGenerator_s390.cpp index 2c2e8ed9e3b..820417fc743 100644 --- a/src/hotspot/cpu/s390/templateInterpreterGenerator_s390.cpp +++ b/src/hotspot/cpu/s390/templateInterpreterGenerator_s390.cpp @@ -721,6 +721,11 @@ address TemplateInterpreterGenerator::generate_safept_entry_for (TosState state, return entry; } +address TemplateInterpreterGenerator::generate_cont_resume_interpreter_adapter() { + return nullptr; +} + + // // Helpers for commoning out cases in the various type of method entries. // diff --git a/src/hotspot/cpu/s390/templateTable_s390.cpp b/src/hotspot/cpu/s390/templateTable_s390.cpp index 0c9f9e031b0..3cb1aba810d 100644 --- a/src/hotspot/cpu/s390/templateTable_s390.cpp +++ b/src/hotspot/cpu/s390/templateTable_s390.cpp @@ -3952,7 +3952,12 @@ void TemplateTable::_new() { if (!ZeroTLAB) { // The object is initialized before the header. If the object size is // zero, go directly to the header initialization. - __ z_aghi(Rsize, (int)-sizeof(oopDesc)); // Subtract header size, set CC. + if (UseCompactObjectHeaders) { + assert(is_aligned(oopDesc::base_offset_in_bytes(), BytesPerLong), "oop base offset must be 8-byte-aligned"); + __ z_aghi(Rsize, (int)-oopDesc::base_offset_in_bytes()); + } else { + __ z_aghi(Rsize, (int)-sizeof(oopDesc)); // Subtract header size, set CC. + } __ z_bre(initialize_header); // Jump if size of fields is zero. // Initialize object fields. @@ -3964,17 +3969,25 @@ void TemplateTable::_new() { // Set Rzero to 0 and use it as src length, then mvcle will copy nothing // and fill the object with the padding value 0. - __ add2reg(RobjectFields, sizeof(oopDesc), RallocatedObject); + if (UseCompactObjectHeaders) { + __ add2reg(RobjectFields, oopDesc::base_offset_in_bytes(), RallocatedObject); + } else { + __ add2reg(RobjectFields, sizeof(oopDesc), RallocatedObject); + } __ move_long_ext(RobjectFields, as_Register(Rzero->encoding() - 1), 0); } // Initialize object header only. __ bind(initialize_header); - __ store_const(Address(RallocatedObject, oopDesc::mark_offset_in_bytes()), - (long)markWord::prototype().value()); - - __ store_klass_gap(Rzero, RallocatedObject); // Zero klass gap for compressed oops. - __ store_klass(iklass, RallocatedObject); // Store klass last. + if (UseCompactObjectHeaders) { + __ z_lg(tmp, Address(iklass, in_bytes(Klass::prototype_header_offset()))); + __ z_stg(tmp, Address(RallocatedObject, oopDesc::mark_offset_in_bytes())); + } else { + __ store_const(Address(RallocatedObject, oopDesc::mark_offset_in_bytes()), + (long) markWord::prototype().value()); + __ store_klass_gap(Rzero, RallocatedObject); // Zero klass gap for compressed oops. + __ store_klass(iklass, RallocatedObject); // Store klass last. + } if (DTraceAllocProbes) { // Trigger dtrace event for fastpath. diff --git a/src/hotspot/cpu/x86/assembler_x86.cpp b/src/hotspot/cpu/x86/assembler_x86.cpp index 03ce06726c6..36a122d2946 100644 --- a/src/hotspot/cpu/x86/assembler_x86.cpp +++ b/src/hotspot/cpu/x86/assembler_x86.cpp @@ -2854,6 +2854,26 @@ void Assembler::leal(Register dst, Address src) { emit_operand(dst, src, 0); } +#ifdef _LP64 +void Assembler::lea(Register dst, Label& L) { + emit_prefix_and_int8(get_prefixq(Address(), dst), (unsigned char)0x8D); + if (!L.is_bound()) { + // Patch @0x8D opcode + L.add_patch_at(code(), CodeBuffer::locator(offset() - 1, sect())); + // Register and [rip+disp] operand + emit_modrm(0b00, raw_encode(dst), 0b101); + emit_int32(0); + } else { + // Register and [rip+disp] operand + emit_modrm(0b00, raw_encode(dst), 0b101); + // Adjust displacement by sizeof lea instruction + int32_t disp = checked_cast(target(L) - (pc() + sizeof(int32_t))); + assert(is_simm32(disp), "must be 32bit offset [rip+offset]"); + emit_int32(disp); + } +} +#endif + void Assembler::lfence() { emit_int24(0x0F, (unsigned char)0xAE, (unsigned char)0xE8); } diff --git a/src/hotspot/cpu/x86/assembler_x86.hpp b/src/hotspot/cpu/x86/assembler_x86.hpp index 420c28254d5..25be0d6a48d 100644 --- a/src/hotspot/cpu/x86/assembler_x86.hpp +++ b/src/hotspot/cpu/x86/assembler_x86.hpp @@ -954,20 +954,13 @@ class Assembler : public AbstractAssembler { // the product flag UseIncDec value. void decl(Register dst); - void edecl(Register dst, Register src, bool no_flags); void decl(Address dst); - void edecl(Register dst, Address src, bool no_flags); void decq(Address dst); - void edecq(Register dst, Address src, bool no_flags); void incl(Register dst); - void eincl(Register dst, Register src, bool no_flags); void incl(Address dst); - void eincl(Register dst, Address src, bool no_flags); void incq(Register dst); - void eincq(Register dst, Register src, bool no_flags); void incq(Address dst); - void eincq(Register dst, Address src, bool no_flags); // New cpus require use of movsd and movss to avoid partial register stall // when loading from memory. But for old Opteron use movlpd instead of movsd. @@ -1117,6 +1110,14 @@ class Assembler : public AbstractAssembler { void addq(Register dst, Register src); void eaddq(Register dst, Register src1, Register src2, bool no_flags); + void edecl(Register dst, Register src, bool no_flags); + void edecl(Register dst, Address src, bool no_flags); + void edecq(Register dst, Address src, bool no_flags); + void eincl(Register dst, Register src, bool no_flags); + void eincl(Register dst, Address src, bool no_flags); + void eincq(Register dst, Register src, bool no_flags); + void eincq(Register dst, Address src, bool no_flags); + #ifdef _LP64 //Add Unsigned Integers with Carry Flag void adcxq(Register dst, Register src); @@ -1625,6 +1626,10 @@ class Assembler : public AbstractAssembler { void leaq(Register dst, Address src); +#ifdef _LP64 + void lea(Register dst, Label& L); +#endif + void lfence(); void lock(); diff --git a/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp b/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp index 64265a96909..ff6d18e48e1 100644 --- a/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp +++ b/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp @@ -3046,6 +3046,7 @@ void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) { Register length = op->length()->as_register(); Register tmp = op->tmp()->as_register(); Register tmp_load_klass = LP64_ONLY(rscratch1) NOT_LP64(noreg); + Register tmp2 = UseCompactObjectHeaders ? rscratch2 : noreg; CodeStub* stub = op->stub(); int flags = op->flags(); @@ -3170,8 +3171,6 @@ void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) { Address src_length_addr = Address(src, arrayOopDesc::length_offset_in_bytes()); Address dst_length_addr = Address(dst, arrayOopDesc::length_offset_in_bytes()); - Address src_klass_addr = Address(src, oopDesc::klass_offset_in_bytes()); - Address dst_klass_addr = Address(dst, oopDesc::klass_offset_in_bytes()); // length and pos's are all sign extended at this point on 64bit @@ -3237,13 +3236,7 @@ void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) { // We don't know the array types are compatible if (basic_type != T_OBJECT) { // Simple test for basic type arrays - if (UseCompressedClassPointers) { - __ movl(tmp, src_klass_addr); - __ cmpl(tmp, dst_klass_addr); - } else { - __ movptr(tmp, src_klass_addr); - __ cmpptr(tmp, dst_klass_addr); - } + __ cmp_klasses_from_objects(src, dst, tmp, tmp2); __ jcc(Assembler::notEqual, *stub->entry()); } else { // For object arrays, if src is a sub class of dst then we can @@ -3302,6 +3295,7 @@ void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) { store_parameter(src, 4); #ifndef _LP64 + Address dst_klass_addr = Address(dst, oopDesc::klass_offset_in_bytes()); __ movptr(tmp, dst_klass_addr); __ movptr(tmp, Address(tmp, ObjArrayKlass::element_klass_offset())); __ push(tmp); @@ -3405,16 +3399,12 @@ void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) { #endif if (basic_type != T_OBJECT) { - - if (UseCompressedClassPointers) __ cmpl(tmp, dst_klass_addr); - else __ cmpptr(tmp, dst_klass_addr); + __ cmp_klass(tmp, dst, tmp2); __ jcc(Assembler::notEqual, halt); - if (UseCompressedClassPointers) __ cmpl(tmp, src_klass_addr); - else __ cmpptr(tmp, src_klass_addr); + __ cmp_klass(tmp, src, tmp2); __ jcc(Assembler::equal, known_ok); } else { - if (UseCompressedClassPointers) __ cmpl(tmp, dst_klass_addr); - else __ cmpptr(tmp, dst_klass_addr); + __ cmp_klass(tmp, dst, tmp2); __ jcc(Assembler::equal, known_ok); __ cmpptr(src, dst); __ jcc(Assembler::equal, known_ok); @@ -3511,13 +3501,7 @@ void LIR_Assembler::emit_load_klass(LIR_OpLoadKlass* op) { add_debug_info_for_null_check_here(info); } -#ifdef _LP64 - if (UseCompressedClassPointers) { - __ movl(result, Address(obj, oopDesc::klass_offset_in_bytes())); - __ decode_klass_not_null(result, rscratch1); - } else -#endif - __ movptr(result, Address(obj, oopDesc::klass_offset_in_bytes())); + __ load_klass(result, obj, rscratch1); } void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) { diff --git a/src/hotspot/cpu/x86/c1_MacroAssembler_x86.cpp b/src/hotspot/cpu/x86/c1_MacroAssembler_x86.cpp index bf5b90db5fc..f53a25ed3e6 100644 --- a/src/hotspot/cpu/x86/c1_MacroAssembler_x86.cpp +++ b/src/hotspot/cpu/x86/c1_MacroAssembler_x86.cpp @@ -109,10 +109,9 @@ int C1_MacroAssembler::lock_object(Register hdr, Register obj, Register disp_hdr jcc(Assembler::notZero, slow_case); // done bind(done); + inc_held_monitor_count(); } - inc_held_monitor_count(); - return null_check_offset; } @@ -153,9 +152,9 @@ void C1_MacroAssembler::unlock_object(Register hdr, Register obj, Register disp_ // we do unlocking via runtime call jcc(Assembler::notEqual, slow_case); // done + bind(done); + dec_held_monitor_count(); } - bind(done); - dec_held_monitor_count(); } @@ -170,16 +169,20 @@ void C1_MacroAssembler::try_allocate(Register obj, Register var_size_in_bytes, i void C1_MacroAssembler::initialize_header(Register obj, Register klass, Register len, Register t1, Register t2) { - assert_different_registers(obj, klass, len); - movptr(Address(obj, oopDesc::mark_offset_in_bytes()), checked_cast(markWord::prototype().value())); + assert_different_registers(obj, klass, len, t1, t2); #ifdef _LP64 - if (UseCompressedClassPointers) { // Take care not to kill klass + if (UseCompactObjectHeaders) { + movptr(t1, Address(klass, Klass::prototype_header_offset())); + movptr(Address(obj, oopDesc::mark_offset_in_bytes()), t1); + } else if (UseCompressedClassPointers) { // Take care not to kill klass + movptr(Address(obj, oopDesc::mark_offset_in_bytes()), checked_cast(markWord::prototype().value())); movptr(t1, klass); encode_klass_not_null(t1, rscratch1); movl(Address(obj, oopDesc::klass_offset_in_bytes()), t1); } else #endif { + movptr(Address(obj, oopDesc::mark_offset_in_bytes()), checked_cast(markWord::prototype().value())); movptr(Address(obj, oopDesc::klass_offset_in_bytes()), klass); } @@ -196,7 +199,7 @@ void C1_MacroAssembler::initialize_header(Register obj, Register klass, Register #endif } #ifdef _LP64 - else if (UseCompressedClassPointers) { + else if (UseCompressedClassPointers && !UseCompactObjectHeaders) { xorptr(t1, t1); store_klass_gap(obj, t1); } @@ -230,7 +233,9 @@ void C1_MacroAssembler::initialize_object(Register obj, Register klass, Register assert((con_size_in_bytes & MinObjAlignmentInBytesMask) == 0, "con_size_in_bytes is not multiple of alignment"); const int hdr_size_in_bytes = instanceOopDesc::header_size() * HeapWordSize; - + if (UseCompactObjectHeaders) { + assert(hdr_size_in_bytes == 8, "check object headers size"); + } initialize_header(obj, klass, noreg, t1, t2); if (!(UseTLAB && ZeroTLAB && is_tlab_allocated)) { diff --git a/src/hotspot/cpu/x86/c1_Runtime1_x86.cpp b/src/hotspot/cpu/x86/c1_Runtime1_x86.cpp index 1ccb06df489..5cc8ffd9bef 100644 --- a/src/hotspot/cpu/x86/c1_Runtime1_x86.cpp +++ b/src/hotspot/cpu/x86/c1_Runtime1_x86.cpp @@ -206,9 +206,10 @@ int StubAssembler::call_RT(Register oop_result1, Register metadata_result, addre class StubFrame: public StackObj { private: StubAssembler* _sasm; + bool _use_pop_on_epilog; public: - StubFrame(StubAssembler* sasm, const char* name, bool must_gc_arguments); + StubFrame(StubAssembler* sasm, const char* name, bool must_gc_arguments, bool use_pop_on_epilog = false); void load_argument(int offset_in_words, Register reg); ~StubFrame(); @@ -219,15 +220,20 @@ void StubAssembler::prologue(const char* name, bool must_gc_arguments) { enter(); } -void StubAssembler::epilogue() { - leave(); +void StubAssembler::epilogue(bool use_pop) { + // Avoid using a leave instruction when this frame may + // have been frozen, since the current value of rbp + // restored from the stub would be invalid. We still + // must restore the rbp value saved on enter though. + use_pop ? pop(rbp) : leave(); ret(0); } #define __ _sasm-> -StubFrame::StubFrame(StubAssembler* sasm, const char* name, bool must_gc_arguments) { +StubFrame::StubFrame(StubAssembler* sasm, const char* name, bool must_gc_arguments, bool use_pop_on_epilog) { _sasm = sasm; + _use_pop_on_epilog = use_pop_on_epilog; __ prologue(name, must_gc_arguments); } @@ -239,7 +245,7 @@ void StubFrame::load_argument(int offset_in_words, Register reg) { StubFrame::~StubFrame() { - __ epilogue(); + __ epilogue(_use_pop_on_epilog); } #undef __ @@ -632,6 +638,15 @@ void Runtime1::initialize_pd() { // nothing to do } +// return: offset in 64-bit words. +uint Runtime1::runtime_blob_current_thread_offset(frame f) { +#ifdef _LP64 + return r15_off / 2; // rsp offsets are in halfwords +#else + Unimplemented(); + return 0; +#endif +} // Target: the entry point of the method that creates and posts the exception oop. // has_argument: true if the exception needs arguments (passed on the stack because @@ -1308,7 +1323,7 @@ OopMapSet* Runtime1::generate_code_for(C1StubId id, StubAssembler* sasm) { // fall through case C1StubId::monitorenter_id: { - StubFrame f(sasm, "monitorenter", dont_gc_arguments); + StubFrame f(sasm, "monitorenter", dont_gc_arguments, true /* use_pop_on_epilog */); OopMap* map = save_live_registers(sasm, 3, save_fpu_registers); // Called with store_parameter and not C abi diff --git a/src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp b/src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp index 3a8f1ab839d..06d93ddea26 100644 --- a/src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp +++ b/src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp @@ -311,61 +311,24 @@ void C2_MacroAssembler::fast_lock(Register objReg, Register boxReg, Register tmp // The object is inflated. tmpReg contains pointer to ObjectMonitor* + markWord::monitor_value #ifndef _LP64 - // The object is inflated. - - // boxReg refers to the on-stack BasicLock in the current frame. - // We'd like to write: - // set box->_displaced_header = markWord::unused_mark(). Any non-0 value suffices. - // This is convenient but results a ST-before-CAS penalty. The following CAS suffers - // additional latency as we have another ST in the store buffer that must drain. - - // avoid ST-before-CAS - // register juggle because we need tmpReg for cmpxchgptr below - movptr(scrReg, boxReg); - movptr(boxReg, tmpReg); // consider: LEA box, [tmp-2] - - // Optimistic form: consider XORL tmpReg,tmpReg - movptr(tmpReg, NULL_WORD); - - // Appears unlocked - try to swing _owner from null to non-null. - // Ideally, I'd manifest "Self" with get_thread and then attempt - // to CAS the register containing Self into m->Owner. - // But we don't have enough registers, so instead we can either try to CAS - // rsp or the address of the box (in scr) into &m->owner. If the CAS succeeds - // we later store "Self" into m->Owner. Transiently storing a stack address - // (rsp or the address of the box) into m->owner is harmless. - // Invariant: tmpReg == 0. tmpReg is EAX which is the implicit cmpxchg comparand. - lock(); - cmpxchgptr(scrReg, Address(boxReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner))); - movptr(Address(scrReg, 0), 3); // box->_displaced_header = 3 - // If we weren't able to swing _owner from null to the BasicLock - // then take the slow path. - jccb (Assembler::notZero, NO_COUNT); - // update _owner from BasicLock to thread - get_thread (scrReg); // beware: clobbers ICCs - movptr(Address(boxReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)), scrReg); - xorptr(boxReg, boxReg); // set icc.ZFlag = 1 to indicate success - - // If the CAS fails we can either retry or pass control to the slow path. - // We use the latter tactic. - // Pass the CAS result in the icc.ZFlag into DONE_LABEL - // If the CAS was successful ... - // Self has acquired the lock - // Invariant: m->_recursions should already be 0, so we don't need to explicitly set it. - // Intentional fall-through into DONE_LABEL ... -#else // _LP64 + // Just take slow path to avoid dealing with 64 bit atomic instructions here. + orl(boxReg, 1); // set ICC.ZF=0 to indicate failure +#else + // Unconditionally set box->_displaced_header = markWord::unused_mark(). + // Without cast to int32_t this style of movptr will destroy r10 which is typically obj. + movptr(Address(boxReg, 0), checked_cast(markWord::unused_mark().value())); + // It's inflated and we use scrReg for ObjectMonitor* in this section. + movptr(boxReg, Address(r15_thread, JavaThread::lock_id_offset())); movq(scrReg, tmpReg); xorq(tmpReg, tmpReg); lock(); - cmpxchgptr(thread, Address(scrReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner))); - // Unconditionally set box->_displaced_header = markWord::unused_mark(). - // Without cast to int32_t this style of movptr will destroy r10 which is typically obj. - movptr(Address(boxReg, 0), checked_cast(markWord::unused_mark().value())); + cmpxchgptr(boxReg, Address(scrReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner))); + // Propagate ICC.ZF from CAS above into DONE_LABEL. - jccb(Assembler::equal, COUNT); // CAS above succeeded; propagate ZF = 1 (success) + jccb(Assembler::equal, COUNT); // CAS above succeeded; propagate ZF = 1 (success) - cmpptr(thread, rax); // Check if we are already the owner (recursive lock) + cmpptr(boxReg, rax); // Check if we are already the owner (recursive lock) jccb(Assembler::notEqual, NO_COUNT); // If not recursive, ZF = 0 at this point (fail) incq(Address(scrReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(recursions))); xorq(rax, rax); // Set ZF = 1 (success) for recursive lock, denoting locking success @@ -377,9 +340,12 @@ void C2_MacroAssembler::fast_lock(Register objReg, Register boxReg, Register tmp jccb(Assembler::notZero, NO_COUNT); // jump if ZFlag == 0 bind(COUNT); - // Count monitors in fast path - increment(Address(thread, JavaThread::held_monitor_count_offset())); - + if (LockingMode == LM_LEGACY) { +#ifdef _LP64 + // Count monitors in fast path + increment(Address(thread, JavaThread::held_monitor_count_offset())); +#endif + } xorl(tmpReg, tmpReg); // Set ZF == 1 bind(NO_COUNT); @@ -441,6 +407,11 @@ void C2_MacroAssembler::fast_unlock(Register objReg, Register boxReg, Register t // It's inflated. +#ifndef _LP64 + // Just take slow path to avoid dealing with 64 bit atomic instructions here. + orl(boxReg, 1); // set ICC.ZF=0 to indicate failure + jmpb(DONE_LABEL); +#else // Despite our balanced locking property we still check that m->_owner == Self // as java routines or native JNI code called by this thread might // have released the lock. @@ -489,12 +460,7 @@ void C2_MacroAssembler::fast_unlock(Register objReg, Register boxReg, Register t // Save the monitor pointer in the current thread, so we can try to // reacquire the lock in SharedRuntime::monitor_exit_helper(). andptr(tmpReg, ~(int32_t)markWord::monitor_value); -#ifndef _LP64 - get_thread(boxReg); - movptr(Address(boxReg, JavaThread::unlocked_inflated_monitor_offset()), tmpReg); -#else // _LP64 movptr(Address(r15_thread, JavaThread::unlocked_inflated_monitor_offset()), tmpReg); -#endif orl (boxReg, 1); // set ICC.ZF=0 to indicate failure jmpb (DONE_LABEL); @@ -502,6 +468,7 @@ void C2_MacroAssembler::fast_unlock(Register objReg, Register boxReg, Register t bind (LSuccess); testl (boxReg, 0); // set ICC.ZF=1 to indicate success jmpb (DONE_LABEL); +#endif // _LP64 if (LockingMode == LM_LEGACY) { bind (Stacked); @@ -518,13 +485,13 @@ void C2_MacroAssembler::fast_unlock(Register objReg, Register boxReg, Register t jccb(Assembler::notZero, NO_COUNT); bind(COUNT); - // Count monitors in fast path -#ifndef _LP64 - get_thread(tmpReg); - decrementl(Address(tmpReg, JavaThread::held_monitor_count_offset())); -#else // _LP64 - decrementq(Address(r15_thread, JavaThread::held_monitor_count_offset())); + + if (LockingMode == LM_LEGACY) { + // Count monitors in fast path +#ifdef _LP64 + decrementq(Address(r15_thread, JavaThread::held_monitor_count_offset())); #endif + } xorl(tmpReg, tmpReg); // Set ZF == 1 @@ -602,6 +569,11 @@ void C2_MacroAssembler::fast_lock_lightweight(Register obj, Register box, Regist { // Handle inflated monitor. bind(inflated); +#ifndef _LP64 + // Just take slow path to avoid dealing with 64 bit atomic instructions here. + orl(box, 1); // set ICC.ZF=0 to indicate failure + jmpb(slow_path); +#else const Register monitor = t; if (!UseObjectMonitorTable) { @@ -647,27 +619,30 @@ void C2_MacroAssembler::fast_lock_lightweight(Register obj, Register box, Regist Label monitor_locked; // Lock the monitor. - // CAS owner (null => current thread). + if (UseObjectMonitorTable) { + // Cache the monitor for unlock before trashing box. On failure to acquire + // the lock, the slow path will reset the entry accordingly (see CacheSetter). + movptr(Address(box, BasicLock::object_monitor_cache_offset_in_bytes()), monitor); + } + + // Try to CAS owner (no owner => current thread's _lock_id). xorptr(rax_reg, rax_reg); - lock(); cmpxchgptr(thread, owner_address); + movptr(box, Address(thread, JavaThread::lock_id_offset())); + lock(); cmpxchgptr(box, owner_address); jccb(Assembler::equal, monitor_locked); // Check if recursive. - cmpptr(thread, rax_reg); + cmpptr(box, rax_reg); jccb(Assembler::notEqual, slow_path); // Recursive. increment(recursions_address); bind(monitor_locked); - if (UseObjectMonitorTable) { - // Cache the monitor for unlock - movptr(Address(box, BasicLock::object_monitor_cache_offset_in_bytes()), monitor); - } +#endif // _LP64 } bind(locked); - increment(Address(thread, JavaThread::held_monitor_count_offset())); // Set ZF = 1 xorl(rax_reg, rax_reg); @@ -777,6 +752,11 @@ void C2_MacroAssembler::fast_unlock_lightweight(Register obj, Register reg_rax, bind(inflated); +#ifndef _LP64 + // Just take slow path to avoid dealing with 64 bit atomic instructions here. + orl(t, 1); // set ICC.ZF=0 to indicate failure + jmpb(slow_path); +#else if (!UseObjectMonitorTable) { assert(mark == monitor, "should be the same here"); } else { @@ -828,10 +808,10 @@ void C2_MacroAssembler::fast_unlock_lightweight(Register obj, Register reg_rax, // Recursive unlock. bind(recursive); decrement(recursions_address); +#endif // _LP64 } bind(unlocked); - decrement(Address(thread, JavaThread::held_monitor_count_offset())); xorl(t, t); // Fast Unlock ZF = 1 #ifdef ASSERT @@ -7093,3 +7073,13 @@ void C2_MacroAssembler::vector_saturating_op(int ideal_opc, BasicType elem_bt, X vector_saturating_op(ideal_opc, elem_bt, dst, src1, src2, vlen_enc); } } + +#ifdef _LP64 +void C2_MacroAssembler::load_narrow_klass_compact_c2(Register dst, Address src) { + // The incoming address is pointing into obj-start + klass_offset_in_bytes. We need to extract + // obj-start, so that we can load from the object's mark-word instead. Usually the address + // comes as obj-start in obj and klass_offset_in_bytes in disp. + movq(dst, src.plus_disp(-oopDesc::klass_offset_in_bytes())); + shrq(dst, markWord::klass_shift); +} +#endif diff --git a/src/hotspot/cpu/x86/c2_MacroAssembler_x86.hpp b/src/hotspot/cpu/x86/c2_MacroAssembler_x86.hpp index 3a36fd75e3f..523200486cc 100644 --- a/src/hotspot/cpu/x86/c2_MacroAssembler_x86.hpp +++ b/src/hotspot/cpu/x86/c2_MacroAssembler_x86.hpp @@ -583,4 +583,8 @@ void select_from_two_vectors_evex(BasicType elem_bt, XMMRegister dst, XMMRegister src1, XMMRegister src2, int vlen_enc); +#ifdef _LP64 + void load_narrow_klass_compact_c2(Register dst, Address src); +#endif + #endif // CPU_X86_C2_MACROASSEMBLER_X86_HPP diff --git a/src/hotspot/cpu/x86/c2_stubGenerator_x86_64_string.cpp b/src/hotspot/cpu/x86/c2_stubGenerator_x86_64_string.cpp index 34f8bec8d11..2837a85800f 100644 --- a/src/hotspot/cpu/x86/c2_stubGenerator_x86_64_string.cpp +++ b/src/hotspot/cpu/x86/c2_stubGenerator_x86_64_string.cpp @@ -26,6 +26,7 @@ #include "precompiled.hpp" #include "macroAssembler_x86.hpp" #include "stubGenerator_x86_64.hpp" +#include "oops/arrayOop.hpp" #include "opto/c2_MacroAssembler.hpp" #include "opto/intrinsicnode.hpp" @@ -160,6 +161,9 @@ static void highly_optimized_short_cases(StrIntrinsicNode::ArgEncoding ae, Regis Register needle_len, XMMRegister XMM0, XMMRegister XMM1, Register mask, Register tmp, MacroAssembler *_masm); +static void copy_to_stack(Register haystack, Register haystack_len, bool isU, Register tmp, + XMMRegister xtmp, MacroAssembler *_masm); + static void setup_jump_tables(StrIntrinsicNode::ArgEncoding ae, Label &L_error, Label &L_checkRange, Label &L_fixup, address *big_jump_table, address *small_jump_table, MacroAssembler *_masm); @@ -395,41 +399,21 @@ static void generate_string_indexof_stubs(StubGenerator *stubgen, address *fnptr // Do "big switch" if haystack size > 32 __ cmpq(haystack_len, 0x20); - __ ja_b(L_bigSwitchTop); + __ ja(L_bigSwitchTop); // Copy the small (< 32 byte) haystack to the stack. Allows for vector reads without page fault // Only done for small haystacks // // NOTE: This code assumes that the haystack points to a java array type AND there are - // at least 16 bytes of header preceeding the haystack pointer. + // at least 8 bytes of header preceeding the haystack pointer. // - // This means that we're copying up to 15 bytes of the header onto the stack along + // This means that we're copying up to 7 bytes of the header onto the stack along // with the haystack bytes. After the copy completes, we adjust the haystack pointer // to the valid haystack bytes on the stack. { - Label L_moreThan16, L_adjustHaystack; - - const Register index = rax; + const Register tmp = rax; const Register haystack = rbx; - - // Only a single vector load/store of either 16 or 32 bytes - __ cmpq(haystack_len, 0x10); - __ ja_b(L_moreThan16); - - __ movq(index, COPIED_HAYSTACK_STACK_OFFSET + 0x10); - __ movdqu(XMM_TMP1, Address(haystack, haystack_len, Address::times_1, -0x10)); - __ movdqu(Address(rsp, COPIED_HAYSTACK_STACK_OFFSET), XMM_TMP1); - __ jmpb(L_adjustHaystack); - - __ bind(L_moreThan16); - __ movq(index, COPIED_HAYSTACK_STACK_OFFSET + 0x20); - __ vmovdqu(XMM_TMP1, Address(haystack, haystack_len, Address::times_1, -0x20)); - __ vmovdqu(Address(rsp, COPIED_HAYSTACK_STACK_OFFSET), XMM_TMP1); - - // Point the haystack at the correct location of the first byte of the "real" haystack on the stack - __ bind(L_adjustHaystack); - __ subq(index, haystack_len); - __ leaq(haystack, Address(rsp, index, Address::times_1)); + copy_to_stack(haystack, haystack_len, false, tmp, XMM_TMP1, _masm); } // Dispatch to handlers for small needle and small haystack @@ -760,39 +744,39 @@ static void generate_string_indexof_stubs(StubGenerator *stubgen, address *fnptr __ ja(L_wideNoExpand); // - // Reads of existing needle are 16-byte chunks - // Writes to copied needle are 32-byte chunks + // Reads of existing needle are 8-byte chunks + // Writes to copied needle are 16-byte chunks // Don't read past the end of the existing needle // - // Start first read at [((ndlLen % 16) - 16) & 0xf] - // outndx += 32 - // inndx += 16 + // Start first read at [((ndlLen % 8) - 8) & 0x7] + // outndx += 16 + // inndx += 8 // cmp nndx, ndlLen // jae done // - // Final index of start of needle at ((16 - (ndlLen %16)) & 0xf) << 1 + // Final index of start of needle at ((8 - (ndlLen % 8)) & 0x7) << 1 // - // Starting read for needle at -(16 - (nLen % 16)) - // Offset of needle in stack should be (16 - (nLen % 16)) * 2 + // Starting read for needle at -(8 - (nLen % 8)) + // Offset of needle in stack should be (8 - (nLen % 8)) * 2 __ movq(index, needle_len); - __ andq(index, 0xf); // nLen % 16 - __ movq(offset, 0x10); - __ subq(offset, index); // 16 - (nLen % 16) + __ andq(index, 0x7); // nLen % 8 + __ movq(offset, 0x8); + __ subq(offset, index); // 8 - (nLen % 8) __ movq(index, offset); __ shlq(offset, 1); // * 2 - __ negq(index); // -(16 - (nLen % 16)) + __ negq(index); // -(8 - (nLen % 8)) __ xorq(wr_index, wr_index); __ bind(L_top); // load needle and expand - __ vpmovzxbw(xmm0, Address(needle, index, Address::times_1), Assembler::AVX_256bit); + __ vpmovzxbw(xmm0, Address(needle, index, Address::times_1), Assembler::AVX_128bit); // store expanded needle to stack - __ vmovdqu(Address(rsp, wr_index, Address::times_1, EXPANDED_NEEDLE_STACK_OFFSET), xmm0); - __ addq(index, 0x10); + __ movdqu(Address(rsp, wr_index, Address::times_1, EXPANDED_NEEDLE_STACK_OFFSET), xmm0); + __ addq(index, 0x8); __ cmpq(index, needle_len); __ jae(L_finished); - __ addq(wr_index, 32); + __ addq(wr_index, 16); __ jmpb(L_top); // adjust pointer and length of needle @@ -1582,35 +1566,9 @@ static void highly_optimized_short_cases(StrIntrinsicNode::ArgEncoding ae, Regis assert((COPIED_HAYSTACK_STACK_OFFSET == 0), "Must be zero!"); assert((COPIED_HAYSTACK_STACK_SIZE == 64), "Must be 64!"); - // Copy incoming haystack onto stack - { - Label L_adjustHaystack, L_moreThan16; - - // Copy haystack to stack (haystack <= 32 bytes) - __ subptr(rsp, COPIED_HAYSTACK_STACK_SIZE); - __ cmpq(haystack_len, isU ? 0x8 : 0x10); - __ ja_b(L_moreThan16); - - __ movq(tmp, COPIED_HAYSTACK_STACK_OFFSET + 0x10); - __ movdqu(XMM0, Address(haystack, haystack_len, isU ? Address::times_2 : Address::times_1, -0x10)); - __ movdqu(Address(rsp, COPIED_HAYSTACK_STACK_OFFSET), XMM0); - __ jmpb(L_adjustHaystack); - - __ bind(L_moreThan16); - __ movq(tmp, COPIED_HAYSTACK_STACK_OFFSET + 0x20); - __ vmovdqu(XMM0, Address(haystack, haystack_len, isU ? Address::times_2 : Address::times_1, -0x20)); - __ vmovdqu(Address(rsp, COPIED_HAYSTACK_STACK_OFFSET), XMM0); - - __ bind(L_adjustHaystack); - __ subptr(tmp, haystack_len); - - if (isU) { - // For UTF-16, lengths are half - __ subptr(tmp, haystack_len); - } - // Point the haystack to the stack - __ leaq(haystack, Address(rsp, tmp, Address::times_1)); - } + // Copy incoming haystack onto stack (haystack <= 32 bytes) + __ subptr(rsp, COPIED_HAYSTACK_STACK_SIZE); + copy_to_stack(haystack, haystack_len, isU, tmp, XMM0, _masm); // Creates a mask of (n - k + 1) ones. This prevents recognizing any false-positives // past the end of the valid haystack. @@ -1672,6 +1630,86 @@ static void highly_optimized_short_cases(StrIntrinsicNode::ArgEncoding ae, Regis __ jmpb(L_out); } + + +// Copy the small (<= 32 byte) haystack to the stack. Allows for vector reads without page fault +// Only done for small haystacks +// NOTE: This code assumes that the haystack points to a java array type AND there are +// at least 8 bytes of header preceeding the haystack pointer. +// We're copying up to 7 bytes of the header onto the stack along with the haystack bytes. +// After the copy completes, we adjust the haystack pointer +// to the valid haystack bytes on the stack. +// +// Copy haystack array elements to stack at region +// (COPIED_HAYSTACK_STACK_OFFSET - COPIED_HAYSTACK_STACK_OFFSET+63) with the following conditions: +// It may copy up to 7 bytes that precede the array +// It doesn't read beyond the end of the array +// There are atleast 31 bytes of stack region beyond the end of array +// Inputs: +// haystack - Address of haystack +// haystack_len - Number of elements in haystack +// isU - Boolean indicating if each element is Latin1 or UTF16 +// tmp, xtmp - Scratch registers +// Output: +// haystack - Address of copied string on stack + +static void copy_to_stack(Register haystack, Register haystack_len, bool isU, + Register tmp, XMMRegister xtmp, MacroAssembler *_masm) { + Label L_moreThan8, L_moreThan16, L_moreThan24, L_adjustHaystack; + + assert(arrayOopDesc::base_offset_in_bytes(isU ? T_CHAR : T_BYTE) >= 8, + "Needs at least 8 bytes preceding the array body"); + + // Copy haystack to stack (haystack <= 32 bytes) + int scale = isU ? 2 : 1; // bytes per char + Address::ScaleFactor addrScale = isU ? Address::times_2 : Address::times_1; + + __ cmpq(haystack_len, 16/scale); + __ ja_b(L_moreThan16); + + __ cmpq(haystack_len, 8/scale); + __ ja_b(L_moreThan8); + // haystack length <= 8 bytes, copy 8 bytes upto haystack end reading at most 7 bytes into the header + __ movq(tmp, COPIED_HAYSTACK_STACK_OFFSET + 8); + __ movq(xtmp, Address(haystack, haystack_len, addrScale, -8)); + __ movq(Address(rsp, COPIED_HAYSTACK_STACK_OFFSET), xtmp); + __ jmpb(L_adjustHaystack); + + __ bind(L_moreThan8); + // haystack length > 8 and <=16 bytes, copy 16 bytes upto haystack end reading at most 7 bytes into the header + __ movq(tmp, COPIED_HAYSTACK_STACK_OFFSET + 16); + __ movdqu(xtmp, Address(haystack, haystack_len, addrScale, -16)); + __ movdqu(Address(rsp, COPIED_HAYSTACK_STACK_OFFSET), xtmp); + __ jmpb(L_adjustHaystack); + + __ bind(L_moreThan16); + __ cmpq(haystack_len, 24/scale); + __ ja_b(L_moreThan24); + // haystack length > 16 and <=24 bytes, copy 24 bytes upto haystack end reading at most 7 bytes into the header + __ movq(tmp, COPIED_HAYSTACK_STACK_OFFSET + 24); + __ movdqu(xtmp, Address(haystack, haystack_len, addrScale, -24)); + __ movdqu(Address(rsp, COPIED_HAYSTACK_STACK_OFFSET), xtmp); + __ movq(xtmp, Address(haystack, haystack_len, addrScale, -8)); + __ movq(Address(rsp, COPIED_HAYSTACK_STACK_OFFSET + 16), xtmp); + __ jmpb(L_adjustHaystack); + + __ bind(L_moreThan24); + // haystack length > 24 and < 32 bytes, copy 32 bytes upto haystack end reading at most 7 bytes into the header + __ movq(tmp, COPIED_HAYSTACK_STACK_OFFSET + 32); + __ vmovdqu(xtmp, Address(haystack, haystack_len, addrScale, -32)); + __ vmovdqu(Address(rsp, COPIED_HAYSTACK_STACK_OFFSET), xtmp); + + __ bind(L_adjustHaystack); + __ subptr(tmp, haystack_len); + + if (isU) { + __ subptr(tmp, haystack_len); + } + + // Point the haystack to the stack + __ leaq(haystack, Address(rsp, tmp, Address::times_1)); +} + //////////////////////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/hotspot/cpu/x86/continuationFreezeThaw_x86.inline.hpp b/src/hotspot/cpu/x86/continuationFreezeThaw_x86.inline.hpp index ecc2d81e672..ba8fcb3aa9c 100644 --- a/src/hotspot/cpu/x86/continuationFreezeThaw_x86.inline.hpp +++ b/src/hotspot/cpu/x86/continuationFreezeThaw_x86.inline.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2019, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -126,6 +126,11 @@ void FreezeBase::adjust_interpreted_frame_unextended_sp(frame& f) { } } +inline void FreezeBase::prepare_freeze_interpreted_top_frame(frame& f) { + assert(f.interpreter_frame_last_sp() == nullptr, "should be null for top frame"); + f.interpreter_frame_set_last_sp(f.unextended_sp()); +} + inline void FreezeBase::relativize_interpreted_frame_metadata(const frame& f, const frame& hf) { assert(hf.fp() == hf.unextended_sp() + (f.fp() - f.unextended_sp()), ""); assert((f.at(frame::interpreter_frame_last_sp_offset) != 0) @@ -136,7 +141,10 @@ inline void FreezeBase::relativize_interpreted_frame_metadata(const frame& f, co assert((intptr_t*)hf.at_relative(frame::interpreter_frame_last_sp_offset) == hf.unextended_sp(), ""); // Make sure that locals is already relativized. - assert((*hf.addr_at(frame::interpreter_frame_locals_offset) == frame::sender_sp_offset + f.interpreter_frame_method()->max_locals() - 1), ""); + DEBUG_ONLY(Method* m = f.interpreter_frame_method();) + // Frames for native methods have 2 extra words (temp oop/result handler) before fixed part of frame. + DEBUG_ONLY(int max_locals = !m->is_native() ? m->max_locals() : m->size_of_parameters() + 2;) + assert((*hf.addr_at(frame::interpreter_frame_locals_offset) == frame::sender_sp_offset + max_locals - 1), ""); // Make sure that monitor_block_top is already relativized. assert(hf.at_absolute(frame::interpreter_frame_monitor_block_top_offset) <= frame::interpreter_frame_initial_sp_offset, ""); @@ -207,7 +215,6 @@ template frame ThawBase::new_stack_frame(const frame& hf, frame& // If caller is interpreted it already made room for the callee arguments int overlap = caller.is_interpreted_frame() ? ContinuationHelper::InterpretedFrame::stack_argsize(hf) : 0; const int fsize = (int)(ContinuationHelper::InterpretedFrame::frame_bottom(hf) - hf.unextended_sp() - overlap); - const int locals = hf.interpreter_frame_method()->max_locals(); intptr_t* frame_sp = caller.unextended_sp() - fsize; intptr_t* fp = frame_sp + (hf.fp() - heap_sp); DEBUG_ONLY(intptr_t* unextended_sp = fp + *hf.addr_at(frame::interpreter_frame_last_sp_offset);) @@ -217,7 +224,10 @@ template frame ThawBase::new_stack_frame(const frame& hf, frame& // we need to set the locals so that the caller of new_stack_frame() can call // ContinuationHelper::InterpretedFrame::frame_bottom intptr_t locals_offset = *hf.addr_at(frame::interpreter_frame_locals_offset); - assert((int)locals_offset == frame::sender_sp_offset + locals - 1, ""); + DEBUG_ONLY(Method* m = hf.interpreter_frame_method();) + // Frames for native methods have 2 extra words (temp oop/result handler) before fixed part of frame. + DEBUG_ONLY(const int max_locals = !m->is_native() ? m->max_locals() : m->size_of_parameters() + 2;) + assert((int)locals_offset == frame::sender_sp_offset + max_locals - 1, ""); // copy relativized locals from the heap frame *f.addr_at(frame::interpreter_frame_locals_offset) = locals_offset; return f; @@ -225,7 +235,7 @@ template frame ThawBase::new_stack_frame(const frame& hf, frame& int fsize = FKind::size(hf); intptr_t* frame_sp = caller.unextended_sp() - fsize; if (bottom || caller.is_interpreted_frame()) { - int argsize = hf.compiled_frame_stack_argsize(); + int argsize = FKind::stack_argsize(hf); fsize += argsize; frame_sp -= argsize; @@ -242,8 +252,9 @@ template frame ThawBase::new_stack_frame(const frame& hf, frame& // we need to recreate a "real" frame pointer, pointing into the stack fp = frame_sp + FKind::size(hf) - frame::sender_sp_offset; } else { - // we need to re-read fp because it may be an oop and we might have fixed the frame. - fp = *(intptr_t**)(hf.sp() - frame::sender_sp_offset); + fp = FKind::stub || FKind::native + ? frame_sp + fsize - frame::sender_sp_offset // fp always points to the address below the pushed return pc. We need correct address. + : *(intptr_t**)(hf.sp() - frame::sender_sp_offset); // we need to re-read fp because it may be an oop and we might have fixed the frame. } return frame(frame_sp, frame_sp, fp, hf.pc(), hf.cb(), hf.oop_map(), false); // TODO PERF : this computes deopt state; is it necessary? } @@ -266,6 +277,22 @@ inline void ThawBase::patch_pd(frame& f, const frame& caller) { patch_callee_link(caller, caller.fp()); } +inline void ThawBase::patch_pd(frame& f, intptr_t* caller_sp) { + intptr_t* fp = caller_sp - frame::sender_sp_offset; + patch_callee_link(f, fp); +} + +inline intptr_t* ThawBase::push_cleanup_continuation() { + frame enterSpecial = new_entry_frame(); + intptr_t* sp = enterSpecial.sp(); + + sp[-1] = (intptr_t)ContinuationEntry::cleanup_pc(); + sp[-2] = (intptr_t)enterSpecial.fp(); + + log_develop_trace(continuations, preempt)("push_cleanup_continuation initial sp: " INTPTR_FORMAT " final sp: " INTPTR_FORMAT, p2i(sp + 2 * frame::metadata_words), p2i(sp)); + return sp; +} + inline void ThawBase::derelativize_interpreted_frame_metadata(const frame& hf, const frame& f) { // Make sure that last_sp is kept relativized. assert((intptr_t*)f.at_relative(frame::interpreter_frame_last_sp_offset) == f.unextended_sp(), ""); diff --git a/src/hotspot/cpu/x86/continuationHelper_x86.inline.hpp b/src/hotspot/cpu/x86/continuationHelper_x86.inline.hpp index dab6cec2a5f..46fe0946951 100644 --- a/src/hotspot/cpu/x86/continuationHelper_x86.inline.hpp +++ b/src/hotspot/cpu/x86/continuationHelper_x86.inline.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2022, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -40,6 +40,20 @@ static inline intptr_t** link_address(const frame& f) { : (intptr_t**)(f.unextended_sp() + f.cb()->frame_size() - frame::sender_sp_offset); } +static inline void patch_return_pc_with_preempt_stub(frame& f) { + if (f.is_runtime_frame()) { + // Patch the pc of the now old last Java frame (we already set the anchor to enterSpecial) + // so that when target goes back to Java it will actually return to the preempt cleanup stub. + intptr_t* sp = f.sp(); + sp[-1] = (intptr_t)StubRoutines::cont_preempt_stub(); + } else { + // The target will check for preemption once it returns to the interpreter + // or the native wrapper code and will manually jump to the preempt stub. + JavaThread *thread = JavaThread::current(); + thread->set_preempt_alternate_return(StubRoutines::cont_preempt_stub()); + } +} + inline int ContinuationHelper::frame_align_words(int size) { #ifdef _LP64 return size & 1; @@ -72,12 +86,12 @@ inline void ContinuationHelper::set_anchor_to_entry_pd(JavaFrameAnchor* anchor, anchor->set_last_Java_fp(entry->entry_fp()); } -#ifdef ASSERT inline void ContinuationHelper::set_anchor_pd(JavaFrameAnchor* anchor, intptr_t* sp) { intptr_t* fp = *(intptr_t**)(sp - frame::sender_sp_offset); anchor->set_last_Java_fp(fp); } +#ifdef ASSERT inline bool ContinuationHelper::Frame::assert_frame_laid_out(frame f) { intptr_t* sp = f.sp(); address pc = *(address*)(sp - frame::sender_sp_ret_address_offset()); diff --git a/src/hotspot/cpu/x86/frame_x86.cpp b/src/hotspot/cpu/x86/frame_x86.cpp index aeb77763e7f..4e28dc12534 100644 --- a/src/hotspot/cpu/x86/frame_x86.cpp +++ b/src/hotspot/cpu/x86/frame_x86.cpp @@ -409,6 +409,36 @@ frame frame::sender_for_upcall_stub_frame(RegisterMap* map) const { return fr; } +#if defined(ASSERT) +static address get_register_address_in_stub(const frame& stub_fr, VMReg reg) { + RegisterMap map(nullptr, + RegisterMap::UpdateMap::include, + RegisterMap::ProcessFrames::skip, + RegisterMap::WalkContinuation::skip); + stub_fr.oop_map()->update_register_map(&stub_fr, &map); + return map.location(reg, stub_fr.sp()); +} +#endif + +JavaThread** frame::saved_thread_address(const frame& f) { + CodeBlob* cb = f.cb(); + assert(cb != nullptr && cb->is_runtime_stub(), "invalid frame"); + + JavaThread** thread_addr; +#ifdef COMPILER1 + if (cb == Runtime1::blob_for(C1StubId::monitorenter_id) || + cb == Runtime1::blob_for(C1StubId::monitorenter_nofpu_id)) { + thread_addr = (JavaThread**)(f.sp() + Runtime1::runtime_blob_current_thread_offset(f)); + } else +#endif + { + // c2 only saves rbp in the stub frame so nothing to do. + thread_addr = nullptr; + } + assert(get_register_address_in_stub(f, SharedRuntime::thread_register()) == (address)thread_addr, "wrong thread address"); + return thread_addr; +} + //------------------------------------------------------------------------------ // frame::verify_deopt_original_pc // diff --git a/src/hotspot/cpu/x86/globalDefinitions_x86.hpp b/src/hotspot/cpu/x86/globalDefinitions_x86.hpp index 12ac26aff21..873cfbdcea0 100644 --- a/src/hotspot/cpu/x86/globalDefinitions_x86.hpp +++ b/src/hotspot/cpu/x86/globalDefinitions_x86.hpp @@ -34,7 +34,9 @@ const bool CCallingConventionRequiresIntsAsLongs = false; #define SUPPORTS_NATIVE_CX8 +#ifdef _LP64 #define SUPPORT_MONITOR_COUNT +#endif #define CPU_MULTI_COPY_ATOMIC diff --git a/src/hotspot/cpu/x86/interp_masm_x86.cpp b/src/hotspot/cpu/x86/interp_masm_x86.cpp index f4b95d846c6..3a3f01a6409 100644 --- a/src/hotspot/cpu/x86/interp_masm_x86.cpp +++ b/src/hotspot/cpu/x86/interp_masm_x86.cpp @@ -336,6 +336,66 @@ void InterpreterMacroAssembler::call_VM_base(Register oop_result, restore_locals(); } +#ifdef _LP64 +void InterpreterMacroAssembler::call_VM_preemptable(Register oop_result, + address entry_point, + Register arg_1) { + assert(arg_1 == c_rarg1, ""); + Label resume_pc, not_preempted; + +#ifdef ASSERT + { + Label L; + cmpptr(Address(r15_thread, JavaThread::preempt_alternate_return_offset()), NULL_WORD); + jcc(Assembler::equal, L); + stop("Should not have alternate return address set"); + bind(L); + } +#endif /* ASSERT */ + + // Force freeze slow path. + push_cont_fastpath(); + + // Make VM call. In case of preemption set last_pc to the one we want to resume to. + // Note: call_VM_helper requires last_Java_pc for anchor to be at the top of the stack. + lea(rscratch1, resume_pc); + push(rscratch1); + MacroAssembler::call_VM_helper(oop_result, entry_point, 1, false /*check_exceptions*/); + pop(rscratch1); + + pop_cont_fastpath(); + + // Check if preempted. + movptr(rscratch1, Address(r15_thread, JavaThread::preempt_alternate_return_offset())); + cmpptr(rscratch1, NULL_WORD); + jccb(Assembler::zero, not_preempted); + movptr(Address(r15_thread, JavaThread::preempt_alternate_return_offset()), NULL_WORD); + jmp(rscratch1); + + // In case of preemption, this is where we will resume once we finally acquire the monitor. + bind(resume_pc); + restore_after_resume(false /* is_native */); + + bind(not_preempted); +} + +void InterpreterMacroAssembler::restore_after_resume(bool is_native) { + lea(rscratch1, ExternalAddress(Interpreter::cont_resume_interpreter_adapter())); + call(rscratch1); + if (is_native) { + // On resume we need to set up stack as expected. + push(dtos); + push(ltos); + } +} +#else +void InterpreterMacroAssembler::call_VM_preemptable(Register oop_result, + address entry_point, + Register arg_1) { + MacroAssembler::call_VM(oop_result, entry_point, arg_1); +} +#endif // _LP64 + void InterpreterMacroAssembler::check_and_handle_popframe(Register java_thread) { if (JvmtiExport::can_pop_frame()) { Label L; @@ -1154,7 +1214,7 @@ void InterpreterMacroAssembler::lock_object(Register lock_reg) { "The argument is only for looks. It must be c_rarg1"); if (LockingMode == LM_MONITOR) { - call_VM(noreg, + call_VM_preemptable(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter), lock_reg); } else { @@ -1241,14 +1301,14 @@ void InterpreterMacroAssembler::lock_object(Register lock_reg) { jcc(Assembler::notZero, slow_case); bind(count_locking); + inc_held_monitor_count(); } - inc_held_monitor_count(); jmp(done); bind(slow_case); // Call the runtime routine for slow case - call_VM(noreg, + call_VM_preemptable(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter), lock_reg); bind(done); @@ -1321,8 +1381,8 @@ void InterpreterMacroAssembler::unlock_object(Register lock_reg) { jcc(Assembler::notZero, slow_case); bind(count_locking); + dec_held_monitor_count(); } - dec_held_monitor_count(); jmp(done); bind(slow_case); diff --git a/src/hotspot/cpu/x86/interp_masm_x86.hpp b/src/hotspot/cpu/x86/interp_masm_x86.hpp index a23058486c3..a122161b1c7 100644 --- a/src/hotspot/cpu/x86/interp_masm_x86.hpp +++ b/src/hotspot/cpu/x86/interp_masm_x86.hpp @@ -63,6 +63,11 @@ class InterpreterMacroAssembler: public MacroAssembler { void load_earlyret_value(TosState state); + void call_VM_preemptable(Register oop_result, + address entry_point, + Register arg_1); + void restore_after_resume(bool is_native); + // Interpreter-specific registers void save_bcp() { movptr(Address(rbp, frame::interpreter_frame_bcp_offset * wordSize), _bcp_register); diff --git a/src/hotspot/cpu/x86/jniFastGetField_x86_32.cpp b/src/hotspot/cpu/x86/jniFastGetField_x86_32.cpp index 52e4388c5d2..12336289412 100644 --- a/src/hotspot/cpu/x86/jniFastGetField_x86_32.cpp +++ b/src/hotspot/cpu/x86/jniFastGetField_x86_32.cpp @@ -36,17 +36,6 @@ #define BUFFER_SIZE 30 -#ifdef _WINDOWS -GetBooleanField_t JNI_FastGetField::jni_fast_GetBooleanField_fp; -GetByteField_t JNI_FastGetField::jni_fast_GetByteField_fp; -GetCharField_t JNI_FastGetField::jni_fast_GetCharField_fp; -GetShortField_t JNI_FastGetField::jni_fast_GetShortField_fp; -GetIntField_t JNI_FastGetField::jni_fast_GetIntField_fp; -GetLongField_t JNI_FastGetField::jni_fast_GetLongField_fp; -GetFloatField_t JNI_FastGetField::jni_fast_GetFloatField_fp; -GetDoubleField_t JNI_FastGetField::jni_fast_GetDoubleField_fp; -#endif - // Instead of issuing lfence for LoadLoad barrier, we create data dependency // between loads, which is much more efficient than lfence. @@ -119,12 +108,7 @@ address JNI_FastGetField::generate_fast_get_int_field0(BasicType type) { // ca1 is data dependent on rax,. __ jcc (Assembler::notEqual, slow); -#ifndef _WINDOWS __ ret (0); -#else - // __stdcall calling convention - __ ret (3*wordSize); -#endif slowcase_entry_pclist[count++] = __ pc(); __ bind (slow); @@ -142,18 +126,7 @@ address JNI_FastGetField::generate_fast_get_int_field0(BasicType type) { __ flush (); -#ifndef _WINDOWS return fast_entry; -#else - switch (type) { - case T_BOOLEAN: jni_fast_GetBooleanField_fp = (GetBooleanField_t) fast_entry; break; - case T_BYTE: jni_fast_GetByteField_fp = (GetByteField_t) fast_entry; break; - case T_CHAR: jni_fast_GetCharField_fp = (GetCharField_t) fast_entry; break; - case T_SHORT: jni_fast_GetShortField_fp = (GetShortField_t) fast_entry; break; - case T_INT: jni_fast_GetIntField_fp = (GetIntField_t) fast_entry; break; - } - return os::win32::fast_jni_accessor_wrapper(type); -#endif } address JNI_FastGetField::generate_fast_get_boolean_field() { @@ -238,12 +211,7 @@ address JNI_FastGetField::generate_fast_get_long_field() { __ pop (rsi); -#ifndef _WINDOWS __ ret (0); -#else - // __stdcall calling convention - __ ret (3*wordSize); -#endif slowcase_entry_pclist[count-1] = __ pc(); slowcase_entry_pclist[count++] = __ pc(); @@ -255,12 +223,7 @@ address JNI_FastGetField::generate_fast_get_long_field() { __ flush (); -#ifndef _WINDOWS return fast_entry; -#else - jni_fast_GetLongField_fp = (GetLongField_t) fast_entry; - return os::win32::fast_jni_accessor_wrapper(T_LONG); -#endif } address JNI_FastGetField::generate_fast_get_float_field0(BasicType type) { @@ -330,12 +293,7 @@ address JNI_FastGetField::generate_fast_get_float_field0(BasicType type) { // access. __ jcc (Assembler::notEqual, slow_with_pop); -#ifndef _WINDOWS __ ret (0); -#else - // __stdcall calling convention - __ ret (3*wordSize); -#endif __ bind (slow_with_pop); // invalid load. pop FPU stack. @@ -354,15 +312,7 @@ address JNI_FastGetField::generate_fast_get_float_field0(BasicType type) { __ flush (); -#ifndef _WINDOWS return fast_entry; -#else - switch (type) { - case T_FLOAT: jni_fast_GetFloatField_fp = (GetFloatField_t) fast_entry; break; - case T_DOUBLE: jni_fast_GetDoubleField_fp = (GetDoubleField_t) fast_entry; break; - } - return os::win32::fast_jni_accessor_wrapper(type); -#endif } address JNI_FastGetField::generate_fast_get_float_field() { diff --git a/src/hotspot/cpu/x86/macroAssembler_x86.cpp b/src/hotspot/cpu/x86/macroAssembler_x86.cpp index 2da8fe68502..08e089455f2 100644 --- a/src/hotspot/cpu/x86/macroAssembler_x86.cpp +++ b/src/hotspot/cpu/x86/macroAssembler_x86.cpp @@ -35,6 +35,7 @@ #include "gc/shared/tlab_globals.hpp" #include "interpreter/bytecodeHistogram.hpp" #include "interpreter/interpreter.hpp" +#include "interpreter/interpreterRuntime.hpp" #include "jvm.h" #include "memory/resourceArea.hpp" #include "memory/universe.hpp" @@ -528,7 +529,6 @@ void MacroAssembler::call_VM_leaf_base(address entry_point, int num_args) { // restore stack pointer addq(rsp, frame::arg_reg_save_area_bytes); #endif - } void MacroAssembler::cmp64(Register src1, AddressLiteral src2, Register rscratch) { @@ -1350,7 +1350,8 @@ void MacroAssembler::ic_call(address entry, jint method_index) { } int MacroAssembler::ic_check_size() { - return LP64_ONLY(14) NOT_LP64(12); + return + LP64_ONLY(UseCompactObjectHeaders ? 17 : 14) NOT_LP64(12); } int MacroAssembler::ic_check(int end_alignment) { @@ -1366,6 +1367,12 @@ int MacroAssembler::ic_check(int end_alignment) { int uep_offset = offset(); +#ifdef _LP64 + if (UseCompactObjectHeaders) { + load_narrow_klass_compact(temp, receiver); + cmpl(temp, Address(data, CompiledICData::speculated_klass_offset())); + } else +#endif if (UseCompressedClassPointers) { movl(temp, Address(receiver, oopDesc::klass_offset_in_bytes())); cmpl(temp, Address(data, CompiledICData::speculated_klass_offset())); @@ -1376,7 +1383,7 @@ int MacroAssembler::ic_check(int end_alignment) { // if inline cache check fails, then jump to runtime routine jump_cc(Assembler::notEqual, RuntimeAddress(SharedRuntime::get_ic_miss_stub())); - assert((offset() % end_alignment) == 0, "Misaligned verified entry point"); + assert((offset() % end_alignment) == 0, "Misaligned verified entry point (%d, %d, %d)", uep_offset, offset(), end_alignment); return uep_offset; } @@ -3033,25 +3040,13 @@ void MacroAssembler::pop_cont_fastpath() { } void MacroAssembler::inc_held_monitor_count() { -#ifndef _LP64 - Register thread = rax; - push(thread); - get_thread(thread); - incrementl(Address(thread, JavaThread::held_monitor_count_offset())); - pop(thread); -#else // LP64 +#ifdef _LP64 incrementq(Address(r15_thread, JavaThread::held_monitor_count_offset())); #endif } void MacroAssembler::dec_held_monitor_count() { -#ifndef _LP64 - Register thread = rax; - push(thread); - get_thread(thread); - decrementl(Address(thread, JavaThread::held_monitor_count_offset())); - pop(thread); -#else // LP64 +#ifdef _LP64 decrementq(Address(r15_thread, JavaThread::held_monitor_count_offset())); #endif } @@ -3148,6 +3143,17 @@ void MacroAssembler::set_last_Java_frame(Register java_thread, movptr(Address(java_thread, JavaThread::last_Java_sp_offset()), last_java_sp); } +#ifdef _LP64 +void MacroAssembler::set_last_Java_frame(Register last_java_sp, + Register last_java_fp, + Label &L, + Register scratch) { + lea(scratch, L); + movptr(Address(r15_thread, JavaThread::last_Java_pc_offset()), scratch); + set_last_Java_frame(r15_thread, last_java_sp, last_java_fp, nullptr, scratch); +} +#endif + void MacroAssembler::shlptr(Register dst, int imm8) { LP64_ONLY(shlq(dst, imm8)) NOT_LP64(shll(dst, imm8)); } @@ -4092,7 +4098,7 @@ RegSet MacroAssembler::call_clobbered_gp_registers() { RegSet regs; #ifdef _LP64 regs += RegSet::of(rax, rcx, rdx); -#ifndef WINDOWS +#ifndef _WINDOWS regs += RegSet::of(rsi, rdi); #endif regs += RegSet::range(r8, r11); @@ -4109,7 +4115,7 @@ RegSet MacroAssembler::call_clobbered_gp_registers() { XMMRegSet MacroAssembler::call_clobbered_xmm_registers() { int num_xmm_registers = XMMRegister::available_xmm_registers(); -#if defined(WINDOWS) && defined(_LP64) +#if defined(_WINDOWS) XMMRegSet result = XMMRegSet::range(xmm0, xmm5); if (num_xmm_registers > 16) { result += XMMRegSet::range(xmm16, as_XMMRegister(num_xmm_registers - 1)); @@ -5948,19 +5954,33 @@ void MacroAssembler::load_method_holder(Register holder, Register method) { movptr(holder, Address(holder, ConstantPool::pool_holder_offset())); // InstanceKlass* } +#ifdef _LP64 +void MacroAssembler::load_narrow_klass_compact(Register dst, Register src) { + assert(UseCompactObjectHeaders, "expect compact object headers"); + movq(dst, Address(src, oopDesc::mark_offset_in_bytes())); + shrq(dst, markWord::klass_shift); +} +#endif + void MacroAssembler::load_klass(Register dst, Register src, Register tmp) { assert_different_registers(src, tmp); assert_different_registers(dst, tmp); #ifdef _LP64 - if (UseCompressedClassPointers) { + if (UseCompactObjectHeaders) { + load_narrow_klass_compact(dst, src); + decode_klass_not_null(dst, tmp); + } else if (UseCompressedClassPointers) { movl(dst, Address(src, oopDesc::klass_offset_in_bytes())); decode_klass_not_null(dst, tmp); } else #endif + { movptr(dst, Address(src, oopDesc::klass_offset_in_bytes())); + } } void MacroAssembler::store_klass(Register dst, Register src, Register tmp) { + assert(!UseCompactObjectHeaders, "not with compact headers"); assert_different_registers(src, tmp); assert_different_registers(dst, tmp); #ifdef _LP64 @@ -5972,6 +5992,41 @@ void MacroAssembler::store_klass(Register dst, Register src, Register tmp) { movptr(Address(dst, oopDesc::klass_offset_in_bytes()), src); } +void MacroAssembler::cmp_klass(Register klass, Register obj, Register tmp) { +#ifdef _LP64 + if (UseCompactObjectHeaders) { + assert(tmp != noreg, "need tmp"); + assert_different_registers(klass, obj, tmp); + load_narrow_klass_compact(tmp, obj); + cmpl(klass, tmp); + } else if (UseCompressedClassPointers) { + cmpl(klass, Address(obj, oopDesc::klass_offset_in_bytes())); + } else +#endif + { + cmpptr(klass, Address(obj, oopDesc::klass_offset_in_bytes())); + } +} + +void MacroAssembler::cmp_klasses_from_objects(Register obj1, Register obj2, Register tmp1, Register tmp2) { +#ifdef _LP64 + if (UseCompactObjectHeaders) { + assert(tmp2 != noreg, "need tmp2"); + assert_different_registers(obj1, obj2, tmp1, tmp2); + load_narrow_klass_compact(tmp1, obj1); + load_narrow_klass_compact(tmp2, obj2); + cmpl(tmp1, tmp2); + } else if (UseCompressedClassPointers) { + movl(tmp1, Address(obj1, oopDesc::klass_offset_in_bytes())); + cmpl(tmp1, Address(obj2, oopDesc::klass_offset_in_bytes())); + } else +#endif + { + movptr(tmp1, Address(obj1, oopDesc::klass_offset_in_bytes())); + cmpptr(tmp1, Address(obj2, oopDesc::klass_offset_in_bytes())); + } +} + void MacroAssembler::access_load_at(BasicType type, DecoratorSet decorators, Register dst, Address src, Register tmp1, Register thread_tmp) { BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler(); @@ -6019,6 +6074,7 @@ void MacroAssembler::store_heap_oop_null(Address dst) { #ifdef _LP64 void MacroAssembler::store_klass_gap(Register dst, Register src) { + assert(!UseCompactObjectHeaders, "Don't use with compact headers"); if (UseCompressedClassPointers) { // Store to klass gap in destination movl(Address(dst, oopDesc::klass_gap_offset_in_bytes()), src); @@ -6183,8 +6239,7 @@ void MacroAssembler::encode_klass_not_null(Register r, Register tmp) { subq(r, tmp); } if (CompressedKlassPointers::shift() != 0) { - assert (LogKlassAlignmentInBytes == CompressedKlassPointers::shift(), "decode alg wrong"); - shrq(r, LogKlassAlignmentInBytes); + shrq(r, CompressedKlassPointers::shift()); } } @@ -6197,8 +6252,7 @@ void MacroAssembler::encode_and_move_klass_not_null(Register dst, Register src) movptr(dst, src); } if (CompressedKlassPointers::shift() != 0) { - assert (LogKlassAlignmentInBytes == CompressedKlassPointers::shift(), "decode alg wrong"); - shrq(dst, LogKlassAlignmentInBytes); + shrq(dst, CompressedKlassPointers::shift()); } } @@ -6210,8 +6264,7 @@ void MacroAssembler::decode_klass_not_null(Register r, Register tmp) { // vtableStubs also counts instructions in pd_code_size_limit. // Also do not verify_oop as this is called by verify_oop. if (CompressedKlassPointers::shift() != 0) { - assert(LogKlassAlignmentInBytes == CompressedKlassPointers::shift(), "decode alg wrong"); - shlq(r, LogKlassAlignmentInBytes); + shlq(r, CompressedKlassPointers::shift()); } if (CompressedKlassPointers::base() != nullptr) { mov64(tmp, (int64_t)CompressedKlassPointers::base()); @@ -6233,17 +6286,28 @@ void MacroAssembler::decode_and_move_klass_not_null(Register dst, Register src) // a pointer that needs nothing but a register rename. movl(dst, src); } else { - if (CompressedKlassPointers::base() != nullptr) { - mov64(dst, (int64_t)CompressedKlassPointers::base()); - } else { - xorq(dst, dst); - } - if (CompressedKlassPointers::shift() != 0) { - assert(LogKlassAlignmentInBytes == CompressedKlassPointers::shift(), "decode alg wrong"); - assert(LogKlassAlignmentInBytes == Address::times_8, "klass not aligned on 64bits?"); - leaq(dst, Address(dst, src, Address::times_8, 0)); + if (CompressedKlassPointers::shift() <= Address::times_8) { + if (CompressedKlassPointers::base() != nullptr) { + mov64(dst, (int64_t)CompressedKlassPointers::base()); + } else { + xorq(dst, dst); + } + if (CompressedKlassPointers::shift() != 0) { + assert(CompressedKlassPointers::shift() == Address::times_8, "klass not aligned on 64bits?"); + leaq(dst, Address(dst, src, Address::times_8, 0)); + } else { + addq(dst, src); + } } else { + if (CompressedKlassPointers::base() != nullptr) { + const uint64_t base_right_shifted = + (uint64_t)CompressedKlassPointers::base() >> CompressedKlassPointers::shift(); + mov64(dst, base_right_shifted); + } else { + xorq(dst, dst); + } addq(dst, src); + shlq(dst, CompressedKlassPointers::shift()); } } } @@ -10560,10 +10624,6 @@ Assembler::Condition MacroAssembler::negate_condition(Assembler::Condition cond) ShouldNotReachHere(); return Assembler::overflow; } -// 32-bit Windows has its own fast-path implementation -// of get_thread -#if !defined(WIN32) || defined(_LP64) - // This is simply a call to Thread::current() void MacroAssembler::get_thread(Register thread) { if (thread != rax) { @@ -10598,9 +10658,6 @@ void MacroAssembler::get_thread(Register thread) { } } - -#endif // !WIN32 || _LP64 - void MacroAssembler::check_stack_alignment(Register sp, const char* msg, unsigned bias, Register tmp) { Label L_stack_ok; if (bias == 0) { diff --git a/src/hotspot/cpu/x86/macroAssembler_x86.hpp b/src/hotspot/cpu/x86/macroAssembler_x86.hpp index 618ec87da86..c6e5b2a115f 100644 --- a/src/hotspot/cpu/x86/macroAssembler_x86.hpp +++ b/src/hotspot/cpu/x86/macroAssembler_x86.hpp @@ -111,7 +111,8 @@ class MacroAssembler: public Assembler { op == 0xEB /* short jmp */ || (op & 0xF0) == 0x70 /* short jcc */ || (op == 0x0F && (branch[1] & 0xF0) == 0x80) /* jcc */ || - (op == 0xC7 && branch[1] == 0xF8) /* xbegin */, + (op == 0xC7 && branch[1] == 0xF8) /* xbegin */ || + (op == 0x8D) /* lea */, "Invalid opcode at patch point"); if (op == 0xEB || (op & 0xF0) == 0x70) { @@ -122,7 +123,7 @@ class MacroAssembler: public Assembler { file == nullptr ? "" : file, line); *disp = (char)imm8; } else { - int* disp = (int*) &branch[(op == 0x0F || op == 0xC7)? 2: 1]; + int* disp = (int*) &branch[(op == 0x0F || op == 0xC7 || op == 0x8D) ? 2 : 1]; int imm32 = checked_cast(target - (address) &disp[1]); *disp = imm32; } @@ -335,6 +336,13 @@ class MacroAssembler: public Assembler { address last_java_pc, Register rscratch); +#ifdef _LP64 + void set_last_Java_frame(Register last_java_sp, + Register last_java_fp, + Label &last_java_pc, + Register scratch); +#endif + void reset_last_Java_frame(Register thread, bool clear_fp); // thread in the default location (r15_thread on 64bit) @@ -363,9 +371,20 @@ class MacroAssembler: public Assembler { void load_method_holder(Register holder, Register method); // oop manipulations +#ifdef _LP64 + void load_narrow_klass_compact(Register dst, Register src); +#endif void load_klass(Register dst, Register src, Register tmp); void store_klass(Register dst, Register src, Register tmp); + // Compares the Klass pointer of an object to a given Klass (which might be narrow, + // depending on UseCompressedClassPointers). + void cmp_klass(Register klass, Register obj, Register tmp); + + // Compares the Klass pointer of two objects obj1 and obj2. Result is in the condition flags. + // Uses tmp1 and tmp2 as temporary registers. + void cmp_klasses_from_objects(Register obj1, Register obj2, Register tmp1, Register tmp2); + void access_load_at(BasicType type, DecoratorSet decorators, Register dst, Address src, Register tmp1, Register thread_tmp); void access_store_at(BasicType type, DecoratorSet decorators, Address dst, Register val, @@ -943,7 +962,7 @@ class MacroAssembler: public Assembler { void atomic_incptr(AddressLiteral counter_addr, Register rscratch = noreg) { LP64_ONLY(atomic_incq(counter_addr, rscratch)) NOT_LP64(atomic_incl(counter_addr, rscratch)) ; } void atomic_incptr(Address counter_addr) { LP64_ONLY(atomic_incq(counter_addr)) NOT_LP64(atomic_incl(counter_addr)) ; } - void lea(Register dst, Address adr) { Assembler::lea(dst, adr); } + using Assembler::lea; void lea(Register dst, AddressLiteral adr); void lea(Address dst, AddressLiteral adr, Register rscratch); diff --git a/src/hotspot/cpu/x86/matcher_x86.hpp b/src/hotspot/cpu/x86/matcher_x86.hpp index 192e959451f..3d8b0ed092f 100644 --- a/src/hotspot/cpu/x86/matcher_x86.hpp +++ b/src/hotspot/cpu/x86/matcher_x86.hpp @@ -93,7 +93,7 @@ static bool narrow_klass_use_complex_address() { NOT_LP64(ShouldNotCallThis();) assert(UseCompressedClassPointers, "only for compressed klass code"); - return (LogKlassAlignmentInBytes <= 3); + return (CompressedKlassPointers::shift() <= 3); } // Prefer ConN+DecodeN over ConP. diff --git a/src/hotspot/cpu/x86/sharedRuntime_x86_32.cpp b/src/hotspot/cpu/x86/sharedRuntime_x86_32.cpp index c6ab00cc9ee..ee99f49321a 100644 --- a/src/hotspot/cpu/x86/sharedRuntime_x86_32.cpp +++ b/src/hotspot/cpu/x86/sharedRuntime_x86_32.cpp @@ -2056,6 +2056,11 @@ uint SharedRuntime::out_preserve_stack_slots() { return 0; } +VMReg SharedRuntime::thread_register() { + Unimplemented(); + return nullptr; +} + //------------------------------generate_deopt_blob---------------------------- void SharedRuntime::generate_deopt_blob() { // allocate space for the code diff --git a/src/hotspot/cpu/x86/sharedRuntime_x86_64.cpp b/src/hotspot/cpu/x86/sharedRuntime_x86_64.cpp index 174e2e02779..b98a17848df 100644 --- a/src/hotspot/cpu/x86/sharedRuntime_x86_64.cpp +++ b/src/hotspot/cpu/x86/sharedRuntime_x86_64.cpp @@ -171,6 +171,7 @@ class RegisterSaver { static int rax_offset_in_bytes(void) { return BytesPerInt * rax_off; } static int rdx_offset_in_bytes(void) { return BytesPerInt * rdx_off; } static int rbx_offset_in_bytes(void) { return BytesPerInt * rbx_off; } + static int r15_offset_in_bytes(void) { return BytesPerInt * r15_off; } static int xmm0_offset_in_bytes(void) { return BytesPerInt * xmm0_off; } static int return_offset_in_bytes(void) { return BytesPerInt * return_off; } @@ -1420,7 +1421,7 @@ static void fill_continuation_entry(MacroAssembler* masm, Register reg_cont_obj, // Kills: // rbx // -void static continuation_enter_cleanup(MacroAssembler* masm) { +static void continuation_enter_cleanup(MacroAssembler* masm) { #ifdef ASSERT Label L_good_sp; __ cmpptr(rsp, Address(r15_thread, JavaThread::cont_entry_offset())); @@ -1610,6 +1611,7 @@ static void gen_continuation_enter(MacroAssembler* masm, __ bind(L_thaw); + ContinuationEntry::_thaw_call_pc_offset = __ pc() - start; __ call(RuntimeAddress(StubRoutines::cont_thaw())); ContinuationEntry::_return_pc_offset = __ pc() - start; @@ -1619,7 +1621,7 @@ static void gen_continuation_enter(MacroAssembler* masm, // --- Normal exit (resolve/thawing) __ bind(L_exit); - + ContinuationEntry::_cleanup_offset = __ pc() - start; continuation_enter_cleanup(masm); __ pop(rbp); __ ret(0); @@ -1712,6 +1714,10 @@ static void gen_continuation_yield(MacroAssembler* masm, __ ret(0); } +void SharedRuntime::continuation_enter_cleanup(MacroAssembler* masm) { + ::continuation_enter_cleanup(masm); +} + static void gen_special_dispatch(MacroAssembler* masm, const methodHandle& method, const BasicType* sig_bt, @@ -2180,11 +2186,16 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm, // points into the right code segment. It does not have to be the correct return pc. // We use the same pc/oopMap repeatedly when we call out - intptr_t the_pc = (intptr_t) __ pc(); - oop_maps->add_gc_map(the_pc - start, map); - - __ set_last_Java_frame(rsp, noreg, (address)the_pc, rscratch1); + Label native_return; + if (LockingMode != LM_LEGACY && method->is_object_wait0()) { + // For convenience we use the pc we want to resume to in case of preemption on Object.wait. + __ set_last_Java_frame(rsp, noreg, native_return, rscratch1); + } else { + intptr_t the_pc = (intptr_t) __ pc(); + oop_maps->add_gc_map(the_pc - start, map); + __ set_last_Java_frame(rsp, noreg, __ pc(), rscratch1); + } // We have all of the arguments setup at this point. We must not touch any register // argument registers at this point (what if we save/restore them there are no oop? @@ -2271,12 +2282,13 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm, // Save the test result, for recursive case, the result is zero __ movptr(Address(lock_reg, mark_word_offset), swap_reg); __ jcc(Assembler::notEqual, slow_path_lock); + + __ bind(count_mon); + __ inc_held_monitor_count(); } else { assert(LockingMode == LM_LIGHTWEIGHT, "must be"); __ lightweight_lock(lock_reg, obj_reg, swap_reg, r15_thread, rscratch1, slow_path_lock); } - __ bind(count_mon); - __ inc_held_monitor_count(); // Slow path will re-enter here __ bind(lock_done); @@ -2367,6 +2379,20 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm, __ movl(Address(r15_thread, JavaThread::thread_state_offset()), _thread_in_Java); __ bind(after_transition); + if (LockingMode != LM_LEGACY && method->is_object_wait0()) { + // Check preemption for Object.wait() + __ movptr(rscratch1, Address(r15_thread, JavaThread::preempt_alternate_return_offset())); + __ cmpptr(rscratch1, NULL_WORD); + __ jccb(Assembler::equal, native_return); + __ movptr(Address(r15_thread, JavaThread::preempt_alternate_return_offset()), NULL_WORD); + __ jmp(rscratch1); + __ bind(native_return); + + intptr_t the_pc = (intptr_t) __ pc(); + oop_maps->add_gc_map(the_pc - start, map); + } + + Label reguard; Label reguard_done; __ cmpl(Address(r15_thread, JavaThread::stack_guard_state_offset()), StackOverflow::stack_guard_yellow_reserved_disabled); @@ -2416,7 +2442,6 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm, } else { assert(LockingMode == LM_LIGHTWEIGHT, "must be"); __ lightweight_unlock(obj_reg, swap_reg, r15_thread, lock_reg, slow_path_unlock); - __ dec_held_monitor_count(); } // slow path re-enters here @@ -2490,8 +2515,14 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm, __ mov(c_rarg1, lock_reg); __ mov(c_rarg2, r15_thread); - // Not a leaf but we have last_Java_frame setup as we want + // Not a leaf but we have last_Java_frame setup as we want. + // We don't want to unmount in case of contention since that would complicate preserving + // the arguments that had already been marshalled into the native convention. So we force + // the freeze slow path to find this native wrapper frame (see recurse_freeze_native_frame()) + // and pin the vthread. Otherwise the fast path won't find it since we don't walk the stack. + __ push_cont_fastpath(); __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::complete_monitor_locking_C), 3); + __ pop_cont_fastpath(); restore_args(masm, total_c_args, c_arg, out_regs); #ifdef ASSERT @@ -2606,6 +2637,10 @@ uint SharedRuntime::in_preserve_stack_slots() { return 4 + 2 * VerifyStackAtCalls; } +VMReg SharedRuntime::thread_register() { + return r15_thread->as_VMReg(); +} + //------------------------------generate_deopt_blob---------------------------- void SharedRuntime::generate_deopt_blob() { // Allocate space for the code diff --git a/src/hotspot/cpu/x86/stackChunkFrameStream_x86.inline.hpp b/src/hotspot/cpu/x86/stackChunkFrameStream_x86.inline.hpp index d69facadbea..6289b903ab1 100644 --- a/src/hotspot/cpu/x86/stackChunkFrameStream_x86.inline.hpp +++ b/src/hotspot/cpu/x86/stackChunkFrameStream_x86.inline.hpp @@ -114,6 +114,7 @@ inline int StackChunkFrameStream::interpreter_frame_num_oops() const f.interpreted_frame_oop_map(&mask); return mask.num_oops() + 1 // for the mirror oop + + (f.interpreter_frame_method()->is_native() ? 1 : 0) // temp oop slot + pointer_delta_as_int((intptr_t*)f.interpreter_frame_monitor_begin(), (intptr_t*)f.interpreter_frame_monitor_end())/BasicObjectLock::size(); } diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp index 82d2fd1e73b..3979237619c 100644 --- a/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp +++ b/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp @@ -35,6 +35,7 @@ #include "prims/jvmtiExport.hpp" #include "prims/upcallLinker.hpp" #include "runtime/arguments.hpp" +#include "runtime/continuationEntry.hpp" #include "runtime/javaThread.hpp" #include "runtime/sharedRuntime.hpp" #include "runtime/stubRoutines.hpp" @@ -3781,6 +3782,36 @@ address StubGenerator::generate_cont_returnBarrier_exception() { return generate_cont_thaw("Cont thaw return barrier exception", Continuation::thaw_return_barrier_exception); } +address StubGenerator::generate_cont_preempt_stub() { + if (!Continuations::enabled()) return nullptr; + StubCodeMark mark(this, "StubRoutines","Continuation preempt stub"); + address start = __ pc(); + + __ reset_last_Java_frame(true); + + // Set rsp to enterSpecial frame, i.e. remove all frames copied into the heap. + __ movptr(rsp, Address(r15_thread, JavaThread::cont_entry_offset())); + + Label preemption_cancelled; + __ movbool(rscratch1, Address(r15_thread, JavaThread::preemption_cancelled_offset())); + __ testbool(rscratch1); + __ jcc(Assembler::notZero, preemption_cancelled); + + // Remove enterSpecial frame from the stack and return to Continuation.run() to unmount. + SharedRuntime::continuation_enter_cleanup(_masm); + __ pop(rbp); + __ ret(0); + + // We acquired the monitor after freezing the frames so call thaw to continue execution. + __ bind(preemption_cancelled); + __ movbool(Address(r15_thread, JavaThread::preemption_cancelled_offset()), false); + __ lea(rbp, Address(rsp, checked_cast(ContinuationEntry::size()))); + __ movptr(rscratch1, ExternalAddress(ContinuationEntry::thaw_call_pc_address())); + __ jmp(rscratch1); + + return start; +} + // exception handler for upcall stubs address StubGenerator::generate_upcall_stub_exception_handler() { StubCodeMark mark(this, "StubRoutines", "upcall stub exception handler"); @@ -3953,6 +3984,7 @@ void StubGenerator::generate_continuation_stubs() { StubRoutines::_cont_thaw = generate_cont_thaw(); StubRoutines::_cont_returnBarrier = generate_cont_returnBarrier(); StubRoutines::_cont_returnBarrierExc = generate_cont_returnBarrier_exception(); + StubRoutines::_cont_preempt_stub = generate_cont_preempt_stub(); } void StubGenerator::generate_final_stubs() { diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64.hpp b/src/hotspot/cpu/x86/stubGenerator_x86_64.hpp index c6fa31c5213..f883b6453a6 100644 --- a/src/hotspot/cpu/x86/stubGenerator_x86_64.hpp +++ b/src/hotspot/cpu/x86/stubGenerator_x86_64.hpp @@ -602,6 +602,8 @@ class StubGenerator: public StubCodeGenerator { address generate_cont_returnBarrier(); address generate_cont_returnBarrier_exception(); + address generate_cont_preempt_stub(); + // Continuation point for throwing of implicit exceptions that are // not handled in the current activation. Fabricates an exception // oop and initiates normal exception dispatching in this diff --git a/src/hotspot/cpu/x86/stubRoutines_x86.hpp b/src/hotspot/cpu/x86/stubRoutines_x86.hpp index b5ed3719897..af57fda23fd 100644 --- a/src/hotspot/cpu/x86/stubRoutines_x86.hpp +++ b/src/hotspot/cpu/x86/stubRoutines_x86.hpp @@ -34,10 +34,10 @@ static bool returns_to_call_stub(address return_pc) { return return_pc == _call_ enum platform_dependent_constants { // simply increase sizes if too small (assembler will crash if too small) _initial_stubs_code_size = 20000 WINDOWS_ONLY(+1000), - _continuation_stubs_code_size = 1000 LP64_ONLY(+1000), + _continuation_stubs_code_size = 1000 LP64_ONLY(+2000), // AVX512 intrinsics add more code in 64-bit VM, // Windows have more code to save/restore registers - _compiler_stubs_code_size = 20000 LP64_ONLY(+46000) WINDOWS_ONLY(+2000), + _compiler_stubs_code_size = 20000 LP64_ONLY(+47000) WINDOWS_ONLY(+2000), _final_stubs_code_size = 10000 LP64_ONLY(+20000) WINDOWS_ONLY(+2000) ZGC_ONLY(+20000) }; diff --git a/src/hotspot/cpu/x86/templateInterpreterGenerator_x86.cpp b/src/hotspot/cpu/x86/templateInterpreterGenerator_x86.cpp index 76ad498be0e..beb80d83e2e 100644 --- a/src/hotspot/cpu/x86/templateInterpreterGenerator_x86.cpp +++ b/src/hotspot/cpu/x86/templateInterpreterGenerator_x86.cpp @@ -387,6 +387,26 @@ address TemplateInterpreterGenerator::generate_safept_entry_for( return entry; } +address TemplateInterpreterGenerator::generate_cont_resume_interpreter_adapter() { + if (!Continuations::enabled()) return nullptr; + address start = __ pc(); + + __ restore_bcp(); + __ restore_locals(); + + // Get return address before adjusting rsp + __ movptr(rax, Address(rsp, 0)); + + // Restore stack bottom + __ movptr(rcx, Address(rbp, frame::interpreter_frame_last_sp_offset * wordSize)); + __ lea(rsp, Address(rbp, rcx, Address::times_ptr)); + // and NULL it as marker that esp is now tos until next java call + __ movptr(Address(rbp, frame::interpreter_frame_last_sp_offset * wordSize), NULL_WORD); + + __ jmp(rax); + + return start; +} // Helpers for commoning out cases in the various type of method entries. @@ -1029,7 +1049,10 @@ address TemplateInterpreterGenerator::generate_native_entry(bool synchronized) { // It is enough that the pc() points into the right code // segment. It does not have to be the correct return pc. - __ set_last_Java_frame(rsp, rbp, (address) __ pc(), rscratch1); + // For convenience we use the pc we want to resume to in + // case of preemption on Object.wait. + Label native_return; + __ set_last_Java_frame(rsp, rbp, native_return, rscratch1); #endif // _LP64 // change thread state @@ -1049,11 +1072,15 @@ address TemplateInterpreterGenerator::generate_native_entry(bool synchronized) { __ movl(Address(thread, JavaThread::thread_state_offset()), _thread_in_native); + __ push_cont_fastpath(); + // Call the native method. __ call(rax); // 32: result potentially in rdx:rax or ST0 // 64: result potentially in rax or xmm0 + __ pop_cont_fastpath(); + // Verify or restore cpu control state after JNI call __ restore_cpu_control_state_after_jni(rscratch1); @@ -1077,10 +1104,10 @@ address TemplateInterpreterGenerator::generate_native_entry(bool synchronized) { Label push_double; ExternalAddress float_handler(AbstractInterpreter::result_handler(T_FLOAT)); ExternalAddress double_handler(AbstractInterpreter::result_handler(T_DOUBLE)); - __ cmpptr(Address(rbp, (frame::interpreter_frame_oop_temp_offset + 1)*wordSize), + __ cmpptr(Address(rbp, (frame::interpreter_frame_result_handler_offset)*wordSize), float_handler.addr(), noreg); __ jcc(Assembler::equal, push_double); - __ cmpptr(Address(rbp, (frame::interpreter_frame_oop_temp_offset + 1)*wordSize), + __ cmpptr(Address(rbp, (frame::interpreter_frame_result_handler_offset)*wordSize), double_handler.addr(), noreg); __ jcc(Assembler::notEqual, L); __ bind(push_double); @@ -1150,6 +1177,24 @@ address TemplateInterpreterGenerator::generate_native_entry(bool synchronized) { // change thread state __ movl(Address(thread, JavaThread::thread_state_offset()), _thread_in_Java); +#ifdef _LP64 + if (LockingMode != LM_LEGACY) { + // Check preemption for Object.wait() + Label not_preempted; + __ movptr(rscratch1, Address(r15_thread, JavaThread::preempt_alternate_return_offset())); + __ cmpptr(rscratch1, NULL_WORD); + __ jccb(Assembler::equal, not_preempted); + __ movptr(Address(r15_thread, JavaThread::preempt_alternate_return_offset()), NULL_WORD); + __ jmp(rscratch1); + __ bind(native_return); + __ restore_after_resume(true /* is_native */); + __ bind(not_preempted); + } else { + // any pc will do so just use this one for LM_LEGACY to keep code together. + __ bind(native_return); + } +#endif // _LP64 + // reset_last_Java_frame __ reset_last_Java_frame(thread, true); diff --git a/src/hotspot/cpu/x86/templateTable_x86.cpp b/src/hotspot/cpu/x86/templateTable_x86.cpp index 527d961259e..441e4c8a0b8 100644 --- a/src/hotspot/cpu/x86/templateTable_x86.cpp +++ b/src/hotspot/cpu/x86/templateTable_x86.cpp @@ -4084,7 +4084,12 @@ void TemplateTable::_new() { // The object is initialized before the header. If the object size is // zero, go directly to the header initialization. - __ decrement(rdx, sizeof(oopDesc)); + if (UseCompactObjectHeaders) { + assert(is_aligned(oopDesc::base_offset_in_bytes(), BytesPerLong), "oop base offset must be 8-byte-aligned"); + __ decrement(rdx, oopDesc::base_offset_in_bytes()); + } else { + __ decrement(rdx, sizeof(oopDesc)); + } __ jcc(Assembler::zero, initialize_header); // Initialize topmost object field, divide rdx by 8, check if odd and @@ -4106,22 +4111,30 @@ void TemplateTable::_new() { // initialize remaining object fields: rdx was a multiple of 8 { Label loop; __ bind(loop); - __ movptr(Address(rax, rdx, Address::times_8, sizeof(oopDesc) - 1*oopSize), rcx); - NOT_LP64(__ movptr(Address(rax, rdx, Address::times_8, sizeof(oopDesc) - 2*oopSize), rcx)); + int header_size_bytes = oopDesc::header_size() * HeapWordSize; + assert(is_aligned(header_size_bytes, BytesPerLong), "oop header size must be 8-byte-aligned"); + __ movptr(Address(rax, rdx, Address::times_8, header_size_bytes - 1*oopSize), rcx); + NOT_LP64(__ movptr(Address(rax, rdx, Address::times_8, header_size_bytes - 2*oopSize), rcx)); __ decrement(rdx); __ jcc(Assembler::notZero, loop); } // initialize object header only. __ bind(initialize_header); - __ movptr(Address(rax, oopDesc::mark_offset_in_bytes()), - (intptr_t)markWord::prototype().value()); // header - __ pop(rcx); // get saved klass back in the register. + if (UseCompactObjectHeaders) { + __ pop(rcx); // get saved klass back in the register. + __ movptr(rbx, Address(rcx, Klass::prototype_header_offset())); + __ movptr(Address(rax, oopDesc::mark_offset_in_bytes()), rbx); + } else { + __ movptr(Address(rax, oopDesc::mark_offset_in_bytes()), + (intptr_t)markWord::prototype().value()); // header + __ pop(rcx); // get saved klass back in the register. #ifdef _LP64 - __ xorl(rsi, rsi); // use zero reg to clear memory (shorter code) - __ store_klass_gap(rax, rsi); // zero klass gap for compressed oops + __ xorl(rsi, rsi); // use zero reg to clear memory (shorter code) + __ store_klass_gap(rax, rsi); // zero klass gap for compressed oops #endif - __ store_klass(rax, rcx, rscratch1); // klass + __ store_klass(rax, rcx, rscratch1); // klass + } if (DTraceAllocProbes) { // Trigger dtrace event for fastpath diff --git a/src/hotspot/cpu/x86/vm_version_x86.cpp b/src/hotspot/cpu/x86/vm_version_x86.cpp index f8c5de551cd..688cd4fa5a6 100644 --- a/src/hotspot/cpu/x86/vm_version_x86.cpp +++ b/src/hotspot/cpu/x86/vm_version_x86.cpp @@ -517,12 +517,10 @@ class VM_Version_StubGenerator: public StubCodeGenerator { // https://msdn.microsoft.com/en-us/library/9z1stfyw.aspx __ subptr(rsp, 64); __ evmovdqul(Address(rsp, 0), xmm7, Assembler::AVX_512bit); -#ifdef _LP64 __ subptr(rsp, 64); __ evmovdqul(Address(rsp, 0), xmm8, Assembler::AVX_512bit); __ subptr(rsp, 64); __ evmovdqul(Address(rsp, 0), xmm31, Assembler::AVX_512bit); -#endif // _LP64 #endif // _WINDOWS // load value into all 64 bytes of zmm7 register @@ -546,12 +544,10 @@ class VM_Version_StubGenerator: public StubCodeGenerator { #ifdef _WINDOWS __ subptr(rsp, 32); __ vmovdqu(Address(rsp, 0), xmm7); -#ifdef _LP64 __ subptr(rsp, 32); __ vmovdqu(Address(rsp, 0), xmm8); __ subptr(rsp, 32); __ vmovdqu(Address(rsp, 0), xmm15); -#endif // _LP64 #endif // _WINDOWS // load value into all 32 bytes of ymm7 register @@ -611,12 +607,10 @@ class VM_Version_StubGenerator: public StubCodeGenerator { #endif #ifdef _WINDOWS -#ifdef _LP64 __ evmovdqul(xmm31, Address(rsp, 0), Assembler::AVX_512bit); __ addptr(rsp, 64); __ evmovdqul(xmm8, Address(rsp, 0), Assembler::AVX_512bit); __ addptr(rsp, 64); -#endif // _LP64 __ evmovdqul(xmm7, Address(rsp, 0), Assembler::AVX_512bit); __ addptr(rsp, 64); #endif // _WINDOWS @@ -641,12 +635,10 @@ class VM_Version_StubGenerator: public StubCodeGenerator { #endif #ifdef _WINDOWS -#ifdef _LP64 __ vmovdqu(xmm15, Address(rsp, 0)); __ addptr(rsp, 32); __ vmovdqu(xmm8, Address(rsp, 0)); __ addptr(rsp, 32); -#endif // _LP64 __ vmovdqu(xmm7, Address(rsp, 0)); __ addptr(rsp, 32); #endif // _WINDOWS diff --git a/src/hotspot/cpu/x86/vm_version_x86.hpp b/src/hotspot/cpu/x86/vm_version_x86.hpp index 791f4a1fec7..004b64ebe6e 100644 --- a/src/hotspot/cpu/x86/vm_version_x86.hpp +++ b/src/hotspot/cpu/x86/vm_version_x86.hpp @@ -643,7 +643,7 @@ class VM_Version : public Abstract_VM_Version { static void set_avx_cpuFeatures() { _features |= (CPU_SSE | CPU_SSE2 | CPU_AVX | CPU_VZEROUPPER ); } static void set_evex_cpuFeatures() { _features |= (CPU_AVX512F | CPU_SSE | CPU_SSE2 | CPU_VZEROUPPER ); } static void set_apx_cpuFeatures() { _features |= CPU_APX_F; } - static void set_bmi_cpuFeatures() { _features |= (CPU_BMI1 | CPU_BMI2 | CPU_LZCNT); } + static void set_bmi_cpuFeatures() { _features |= (CPU_BMI1 | CPU_BMI2 | CPU_LZCNT | CPU_POPCNT); } // Initialization static void initialize(); diff --git a/src/hotspot/cpu/x86/x86_64.ad b/src/hotspot/cpu/x86/x86_64.ad index b4eb4c313d3..fc083ecfa24 100644 --- a/src/hotspot/cpu/x86/x86_64.ad +++ b/src/hotspot/cpu/x86/x86_64.ad @@ -4351,6 +4351,7 @@ instruct loadKlass(rRegP dst, memory mem) // Load narrow Klass Pointer instruct loadNKlass(rRegN dst, memory mem) %{ + predicate(!UseCompactObjectHeaders); match(Set dst (LoadNKlass mem)); ins_cost(125); // XXX @@ -4361,6 +4362,19 @@ instruct loadNKlass(rRegN dst, memory mem) ins_pipe(ialu_reg_mem); // XXX %} +instruct loadNKlassCompactHeaders(rRegN dst, memory mem, rFlagsReg cr) +%{ + predicate(UseCompactObjectHeaders); + match(Set dst (LoadNKlass mem)); + effect(KILL cr); + ins_cost(125); // XXX + format %{ "load_narrow_klass_compact $dst, $mem\t# compressed klass ptr" %} + ins_encode %{ + __ load_narrow_klass_compact_c2($dst$$Register, $mem$$Address); + %} + ins_pipe(pipe_slow); // XXX +%} + // Load Float instruct loadF(regF dst, memory mem) %{ @@ -11665,6 +11679,7 @@ instruct compN_rReg_imm_klass(rFlagsRegU cr, rRegN op1, immNKlass op2) %{ instruct compN_mem_imm_klass(rFlagsRegU cr, memory mem, immNKlass src) %{ + predicate(!UseCompactObjectHeaders); match(Set cr (CmpN src (LoadNKlass mem))); format %{ "cmpl $mem, $src\t# compressed klass ptr" %} diff --git a/src/hotspot/cpu/zero/continuationFreezeThaw_zero.inline.hpp b/src/hotspot/cpu/zero/continuationFreezeThaw_zero.inline.hpp index cb05ab1389e..b1bd138e357 100644 --- a/src/hotspot/cpu/zero/continuationFreezeThaw_zero.inline.hpp +++ b/src/hotspot/cpu/zero/continuationFreezeThaw_zero.inline.hpp @@ -48,6 +48,10 @@ void FreezeBase::adjust_interpreted_frame_unextended_sp(frame& f) { Unimplemented(); } +inline void FreezeBase::prepare_freeze_interpreted_top_frame(frame& f) { + Unimplemented(); +} + inline void FreezeBase::relativize_interpreted_frame_metadata(const frame& f, const frame& hf) { Unimplemented(); } @@ -83,6 +87,15 @@ inline void ThawBase::patch_pd(frame& f, const frame& caller) { Unimplemented(); } +inline void ThawBase::patch_pd(frame& f, intptr_t* caller_sp) { + Unimplemented(); +} + +inline intptr_t* ThawBase::push_cleanup_continuation() { + Unimplemented(); + return nullptr; +} + template inline void Thaw::patch_caller_links(intptr_t* sp, intptr_t* bottom) { Unimplemented(); diff --git a/src/hotspot/cpu/zero/continuationHelper_zero.inline.hpp b/src/hotspot/cpu/zero/continuationHelper_zero.inline.hpp index 56da168d9b1..9829d32dc7f 100644 --- a/src/hotspot/cpu/zero/continuationHelper_zero.inline.hpp +++ b/src/hotspot/cpu/zero/continuationHelper_zero.inline.hpp @@ -33,6 +33,10 @@ static inline intptr_t** link_address(const frame& f) { return nullptr; } +static inline void patch_return_pc_with_preempt_stub(frame& f) { + Unimplemented(); +} + inline int ContinuationHelper::frame_align_words(int size) { Unimplemented(); return 0; @@ -60,11 +64,11 @@ inline void ContinuationHelper::set_anchor_to_entry_pd(JavaFrameAnchor* anchor, Unimplemented(); } -#ifdef ASSERT inline void ContinuationHelper::set_anchor_pd(JavaFrameAnchor* anchor, intptr_t* sp) { Unimplemented(); } +#ifdef ASSERT inline bool ContinuationHelper::Frame::assert_frame_laid_out(frame f) { Unimplemented(); return false; diff --git a/src/hotspot/cpu/zero/frame_zero.cpp b/src/hotspot/cpu/zero/frame_zero.cpp index 5ddd23a9d59..400766298f5 100644 --- a/src/hotspot/cpu/zero/frame_zero.cpp +++ b/src/hotspot/cpu/zero/frame_zero.cpp @@ -72,6 +72,11 @@ bool frame::upcall_stub_frame_is_first() const { return false; } +JavaThread** frame::saved_thread_address(const frame& f) { + Unimplemented(); + return nullptr; +} + frame frame::sender_for_nonentry_frame(RegisterMap *map) const { assert(zeroframe()->is_interpreter_frame() || zeroframe()->is_fake_stub_frame(), "wrong type of frame"); diff --git a/src/hotspot/cpu/zero/globalDefinitions_zero.hpp b/src/hotspot/cpu/zero/globalDefinitions_zero.hpp index 0f78b3eeefb..d05e5deb0f8 100644 --- a/src/hotspot/cpu/zero/globalDefinitions_zero.hpp +++ b/src/hotspot/cpu/zero/globalDefinitions_zero.hpp @@ -36,8 +36,6 @@ // The default padding size for data structures to avoid false sharing. #define DEFAULT_PADDING_SIZE DEFAULT_CACHE_LINE_SIZE -#define SUPPORT_MONITOR_COUNT - #include // Indicates whether the C calling conventions require that diff --git a/src/hotspot/cpu/zero/sharedRuntime_zero.cpp b/src/hotspot/cpu/zero/sharedRuntime_zero.cpp index 18ceb9514d3..094ddc05a2e 100644 --- a/src/hotspot/cpu/zero/sharedRuntime_zero.cpp +++ b/src/hotspot/cpu/zero/sharedRuntime_zero.cpp @@ -85,6 +85,11 @@ uint SharedRuntime::out_preserve_stack_slots() { return 0; } +VMReg SharedRuntime::thread_register() { + Unimplemented(); + return nullptr; +} + JRT_LEAF(void, zero_stub()) ShouldNotCallThis(); JRT_END diff --git a/src/hotspot/cpu/zero/zeroInterpreter_zero.cpp b/src/hotspot/cpu/zero/zeroInterpreter_zero.cpp index aab43e73396..c7e6a139831 100644 --- a/src/hotspot/cpu/zero/zeroInterpreter_zero.cpp +++ b/src/hotspot/cpu/zero/zeroInterpreter_zero.cpp @@ -346,9 +346,6 @@ int ZeroInterpreter::native_entry(Method* method, intptr_t UNUSED, TRAPS) { success = false; } } - if (success) { - THREAD->inc_held_monitor_count(); - } } if (!success) { CALL_VM_NOCHECK(InterpreterRuntime::monitorenter(thread, monitor)); @@ -499,9 +496,6 @@ int ZeroInterpreter::native_entry(Method* method, intptr_t UNUSED, TRAPS) { success = false; } } - if (success) { - THREAD->dec_held_monitor_count(); - } } if (!success) { InterpreterRuntime::monitorexit(monitor); diff --git a/src/hotspot/os/aix/os_aix.cpp b/src/hotspot/os/aix/os_aix.cpp index 842ab0c6eeb..44b9571e102 100644 --- a/src/hotspot/os/aix/os_aix.cpp +++ b/src/hotspot/os/aix/os_aix.cpp @@ -468,7 +468,7 @@ static void query_multipage_support() { // Can we use mmap with 64K pages? (Should be available with AIX7.3 TL1) { - void* p = mmap(NULL, 64*K, PROT_READ | PROT_WRITE, MAP_ANON_64K | MAP_ANONYMOUS | MAP_SHARED, -1, 0); + void* p = mmap(nullptr, 64*K, PROT_READ | PROT_WRITE, MAP_ANON_64K | MAP_ANONYMOUS | MAP_SHARED, -1, 0); assert(p != (void*) -1, "mmap failed"); if (p != (void*) -1) { g_multipage_support.can_use_64K_mmap_pages = (64*K == os::Aix::query_pagesize(p)); diff --git a/src/hotspot/os/bsd/os_bsd.cpp b/src/hotspot/os/bsd/os_bsd.cpp index b50ebf22203..a1887aa5975 100644 --- a/src/hotspot/os/bsd/os_bsd.cpp +++ b/src/hotspot/os/bsd/os_bsd.cpp @@ -2148,13 +2148,12 @@ jint os::init_2(void) { // On macOS according to setrlimit(2), OPEN_MAX must be used instead // of RLIM_INFINITY, but testing on macOS >= 10.6, reveals that - // we can, in fact, use even RLIM_INFINITY, so try the max value - // that the system claims can be used first, same as other BSD OSes. - // However, some terminals (ksh) will internally use "int" type - // to store this value and since RLIM_INFINITY overflows an "int" - // we might end up with a negative value, so cap the system limit max - // at INT_MAX instead, just in case, for everyone. - nbr_files.rlim_cur = MIN(INT_MAX, nbr_files.rlim_max); + // we can, in fact, use even RLIM_INFINITY. + // However, we need to limit the value to 0x100000 (which is the max value + // allowed on Linux) so that any existing code that iterates over all allowed + // file descriptors, finishes in a reasonable time, without appearing + // to hang. + nbr_files.rlim_cur = MIN(0x100000, nbr_files.rlim_max); status = setrlimit(RLIMIT_NOFILE, &nbr_files); if (status != 0) { diff --git a/src/hotspot/os/linux/cgroupSubsystem_linux.cpp b/src/hotspot/os/linux/cgroupSubsystem_linux.cpp index d51499611a8..4a8fda432d8 100644 --- a/src/hotspot/os/linux/cgroupSubsystem_linux.cpp +++ b/src/hotspot/os/linux/cgroupSubsystem_linux.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2019, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -167,7 +167,7 @@ static bool find_ro_opt(char* mount_opts) { char* token; char* mo_ptr = mount_opts; // mount options are comma-separated (man proc). - while ((token = strsep(&mo_ptr, ",")) != NULL) { + while ((token = strsep(&mo_ptr, ",")) != nullptr) { if (strcmp(token, "ro") == 0) { return true; } diff --git a/src/hotspot/os/posix/include/jvm_md.h b/src/hotspot/os/posix/include/jvm_md.h index 2c8ebe06d70..e550ecbb000 100644 --- a/src/hotspot/os/posix/include/jvm_md.h +++ b/src/hotspot/os/posix/include/jvm_md.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -37,13 +37,6 @@ #include /* For ptrdiff_t */ #include /* For uintptr_t */ -#define JNI_ONLOAD_SYMBOLS {"JNI_OnLoad"} -#define JNI_ONUNLOAD_SYMBOLS {"JNI_OnUnload"} -#define JVM_ONLOAD_SYMBOLS {"JVM_OnLoad"} -#define AGENT_ONLOAD_SYMBOLS {"Agent_OnLoad"} -#define AGENT_ONUNLOAD_SYMBOLS {"Agent_OnUnload"} -#define AGENT_ONATTACH_SYMBOLS {"Agent_OnAttach"} - #define JNI_LIB_PREFIX "lib" #ifdef __APPLE__ #define JNI_LIB_SUFFIX ".dylib" diff --git a/src/hotspot/os/posix/os_posix.cpp b/src/hotspot/os/posix/os_posix.cpp index 75253843593..72330190b53 100644 --- a/src/hotspot/os/posix/os_posix.cpp +++ b/src/hotspot/os/posix/os_posix.cpp @@ -696,14 +696,6 @@ void os::print_active_locale(outputStream* st) { } } -void os::print_jni_name_prefix_on(outputStream* st, int args_size) { - // no prefix required -} - -void os::print_jni_name_suffix_on(outputStream* st, int args_size) { - // no suffix required -} - bool os::get_host_name(char* buf, size_t buflen) { struct utsname name; int retcode = uname(&name); @@ -816,7 +808,7 @@ void* os::dll_lookup(void* handle, const char* name) { void* ret = ::dlsym(handle, name); if (ret == nullptr) { const char* tmp = ::dlerror(); - // It is possible that we found a NULL symbol, hence no error. + // It is possible that we found a null symbol, hence no error. if (tmp != nullptr) { log_debug(os)("Symbol %s not found in dll: %s", name, tmp); } diff --git a/src/hotspot/os/posix/perfMemory_posix.cpp b/src/hotspot/os/posix/perfMemory_posix.cpp index 17bf63092c2..4eb46169878 100644 --- a/src/hotspot/os/posix/perfMemory_posix.cpp +++ b/src/hotspot/os/posix/perfMemory_posix.cpp @@ -1086,7 +1086,7 @@ static char* mmap_create_shared(size_t size) { static void unmap_shared(char* addr, size_t bytes) { int res; if (MemTracker::enabled()) { - NmtVirtualMemoryLocker ml; + ThreadCritical tc; res = ::munmap(addr, bytes); if (res == 0) { MemTracker::record_virtual_memory_release((address)addr, bytes); diff --git a/src/hotspot/os/windows/attachListener_windows.cpp b/src/hotspot/os/windows/attachListener_windows.cpp index bfa377d52cf..5a34639a1cf 100644 --- a/src/hotspot/os/windows/attachListener_windows.cpp +++ b/src/hotspot/os/windows/attachListener_windows.cpp @@ -84,7 +84,8 @@ class PipeChannel : public AttachOperation::RequestReader, public AttachOperatio 0, // default attributes nullptr); // no template file if (_hPipe == INVALID_HANDLE_VALUE) { - log_error(attach)("could not open (%d) pipe %s", GetLastError(), pipe); + log_error(attach)("could not open %s (%d) pipe %s", + (write_only ? "write-only" : "read-write"), GetLastError(), pipe); return false; } return true; @@ -106,7 +107,11 @@ class PipeChannel : public AttachOperation::RequestReader, public AttachOperatio (DWORD)size, &nread, nullptr); // not overlapped - return fSuccess ? (int)nread : -1; + if (!fSuccess) { + log_error(attach)("pipe read error (%d)", GetLastError()); + return -1; + } + return (int)nread; } // ReplyWriter @@ -118,7 +123,11 @@ class PipeChannel : public AttachOperation::RequestReader, public AttachOperatio (DWORD)size, &written, nullptr); // not overlapped - return fSuccess ? (int)written : -1; + if (!fSuccess) { + log_error(attach)("pipe write error (%d)", GetLastError()); + return -1; + } + return (int)written; } void flush() override { @@ -138,12 +147,12 @@ class Win32AttachOperation: public AttachOperation { public: // for v1 pipe must be write-only - void open_pipe(const char* pipe_name, bool write_only) { - _pipe.open(pipe_name, write_only); + bool open_pipe(const char* pipe_name, bool write_only) { + return _pipe.open(pipe_name, write_only); } bool read_request() { - return AttachOperation::read_request(&_pipe); + return AttachOperation::read_request(&_pipe); } public: @@ -390,13 +399,17 @@ Win32AttachOperation* Win32AttachListener::dequeue() { for (int i = 0; i < AttachOperation::arg_count_max; i++) { op->append_arg(request->arg(i)); } - op->open_pipe(request->pipe(), true/*write-only*/); + if (!op->open_pipe(request->pipe(), true/*write-only*/)) { + // error is already logged + delete op; + op = nullptr; + } break; case ATTACH_API_V2: op = new Win32AttachOperation(); - op->open_pipe(request->pipe(), false/*write-only*/); - if (!op->read_request()) { - log_error(attach)("AttachListener::dequeue, reading request ERROR"); + if (!op->open_pipe(request->pipe(), false/*write-only*/) + || !op->read_request()) { + // error is already logged delete op; op = nullptr; } diff --git a/src/hotspot/os/windows/include/jvm_md.h b/src/hotspot/os/windows/include/jvm_md.h index 4baba320855..a0e44a45613 100644 --- a/src/hotspot/os/windows/include/jvm_md.h +++ b/src/hotspot/os/windows/include/jvm_md.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2008, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -39,13 +39,6 @@ typedef int socklen_t; -#define JNI_ONLOAD_SYMBOLS {"_JNI_OnLoad@8", "JNI_OnLoad"} -#define JNI_ONUNLOAD_SYMBOLS {"_JNI_OnUnload@8", "JNI_OnUnload"} -#define JVM_ONLOAD_SYMBOLS {"_JVM_OnLoad@12", "JVM_OnLoad"} -#define AGENT_ONLOAD_SYMBOLS {"_Agent_OnLoad@12", "Agent_OnLoad"} -#define AGENT_ONUNLOAD_SYMBOLS {"_Agent_OnUnload@4", "Agent_OnUnload"} -#define AGENT_ONATTACH_SYMBOLS {"_Agent_OnAttach@12", "Agent_OnAttach"} - #define JNI_LIB_PREFIX "" #define JNI_LIB_SUFFIX ".dll" diff --git a/src/hotspot/os/windows/os_windows.cpp b/src/hotspot/os/windows/os_windows.cpp index 71efb57e0f2..9d514e584a4 100644 --- a/src/hotspot/os/windows/os_windows.cpp +++ b/src/hotspot/os/windows/os_windows.cpp @@ -133,7 +133,7 @@ static FILETIME process_kernel_time; #elif defined(_M_AMD64) #define __CPU__ amd64 #else - #define __CPU__ i486 + #error "Unknown CPU" #endif #if defined(USE_VECTORED_EXCEPTION_HANDLING) @@ -282,11 +282,6 @@ void os::run_periodic_checks(outputStream* st) { return; } -#ifndef _WIN64 -// previous UnhandledExceptionFilter, if there is one -static LPTOP_LEVEL_EXCEPTION_FILTER prev_uef_handler = nullptr; -#endif - static LONG WINAPI Uncaught_Exception_Handler(struct _EXCEPTION_POINTERS* exceptionInfo); void os::init_system_properties_values() { @@ -399,11 +394,6 @@ void os::init_system_properties_values() { #undef BIN_DIR #undef PACKAGE_DIR -#ifndef _WIN64 - // set our UnhandledExceptionFilter and save any previous one - prev_uef_handler = SetUnhandledExceptionFilter(Uncaught_Exception_Handler); -#endif - // Done return; } @@ -517,7 +507,7 @@ LONG WINAPI topLevelExceptionFilter(struct _EXCEPTION_POINTERS* exceptionInfo); // Thread start routine for all newly created threads. // Called with the associated Thread* as the argument. -static unsigned __stdcall thread_native_entry(void* t) { +static unsigned thread_native_entry(void* t) { Thread* thread = static_cast(t); thread->record_stack_base_and_size(); @@ -876,14 +866,8 @@ bool os::has_allocatable_memory_limit(size_t* limit) { MEMORYSTATUSEX ms; ms.dwLength = sizeof(ms); GlobalMemoryStatusEx(&ms); -#ifdef _LP64 *limit = (size_t)ms.ullAvailVirtual; return true; -#else - // Limit to 1400m because of the 2gb address space wall - *limit = MIN2((size_t)1400*M, (size_t)ms.ullAvailVirtual); - return true; -#endif } int os::active_processor_count() { @@ -1549,7 +1533,7 @@ void os::prepare_native_symbols() { //----------------------------------------------------------- // Helper functions for fatal error handler -#ifdef _WIN64 + // Helper routine which returns true if address in // within the NTDLL address space. // @@ -1571,7 +1555,6 @@ static bool _addr_in_ntdll(address addr) { return false; } } -#endif struct _modinfo { address addr; @@ -1749,7 +1732,6 @@ void * os::dll_load(const char *name, char *ebuf, int ebuflen) { } arch_t; static const arch_t arch_array[] = { - {IMAGE_FILE_MACHINE_I386, (char*)"IA 32"}, {IMAGE_FILE_MACHINE_AMD64, (char*)"AMD 64"}, {IMAGE_FILE_MACHINE_ARM64, (char*)"ARM 64"} }; @@ -1757,11 +1739,9 @@ void * os::dll_load(const char *name, char *ebuf, int ebuflen) { static const uint16_t running_arch = IMAGE_FILE_MACHINE_ARM64; #elif (defined _M_AMD64) static const uint16_t running_arch = IMAGE_FILE_MACHINE_AMD64; -#elif (defined _M_IX86) - static const uint16_t running_arch = IMAGE_FILE_MACHINE_I386; #else #error Method os::dll_load requires that one of following \ - is defined :_M_AMD64 or _M_IX86 or _M_ARM64 + is defined :_M_AMD64 or _M_ARM64 #endif @@ -2101,13 +2081,6 @@ void os::print_memory_info(outputStream* st) { (int64_t) ms.ullTotalPageFile >> 20); st->print("(AvailPageFile size " INT64_FORMAT "M)", (int64_t) ms.ullAvailPageFile >> 20); - - // on 32bit Total/AvailVirtual are interesting (show us how close we get to 2-4 GB per process borders) -#if defined(_M_IX86) - st->print(", user-mode portion of virtual address-space " INT64_FORMAT "M ", - (int64_t) ms.ullTotalVirtual >> 20); - st->print("(" INT64_FORMAT "M free)", (int64_t) ms.ullAvailVirtual >> 20); -#endif } else { st->print(", GlobalMemoryStatusEx did not succeed so we miss some memory values."); } @@ -2228,19 +2201,6 @@ void os::jvm_path(char *buf, jint buflen) { } -void os::print_jni_name_prefix_on(outputStream* st, int args_size) { -#ifndef _WIN64 - st->print("_"); -#endif -} - - -void os::print_jni_name_suffix_on(outputStream* st, int args_size) { -#ifndef _WIN64 - st->print("@%d", args_size * sizeof(int)); -#endif -} - // This method is a copy of JDK's sysGetLastErrorString // from src/windows/hpi/src/system_md.c @@ -2465,8 +2425,6 @@ LONG Handle_Exception(struct _EXCEPTION_POINTERS* exceptionInfo, #define PC_NAME Pc #elif defined(_M_AMD64) #define PC_NAME Rip -#elif defined(_M_IX86) - #define PC_NAME Eip #else #error unknown architecture #endif @@ -2590,21 +2548,12 @@ LONG Handle_IDiv_Exception(struct _EXCEPTION_POINTERS* exceptionInfo) { ctx->Rdx = (DWORD)0; // remainder // Continue the execution #else - PCONTEXT ctx = exceptionInfo->ContextRecord; - address pc = (address)ctx->Eip; - guarantee(pc[0] == 0xF7, "not an idiv opcode(0xF7), the actual value = 0x%x", pc[1]); - guarantee((pc[1] & ~0x7) == 0xF8, "cannot handle non-register operands, the actual value = 0x%x", pc[1]); - guarantee(ctx->Eax == min_jint, "unexpected idiv exception, the actual value = %d while the expected is %d", ctx->Eax, min_jint); - // set correct result values and continue after idiv instruction - ctx->Eip = (DWORD)pc + 2; // idiv reg, reg is 2 bytes - ctx->Eax = (DWORD)min_jint; // result - ctx->Edx = (DWORD)0; // remainder - // Continue the execution + #error unknown architecture #endif return EXCEPTION_CONTINUE_EXECUTION; } -#if defined(_M_AMD64) || defined(_M_IX86) +#if defined(_M_AMD64) //----------------------------------------------------------------------------- static bool handle_FLT_exception(struct _EXCEPTION_POINTERS* exceptionInfo) { // handle exception caused by native method modifying control word @@ -2619,16 +2568,6 @@ static bool handle_FLT_exception(struct _EXCEPTION_POINTERS* exceptionInfo) { case EXCEPTION_FLT_STACK_CHECK: case EXCEPTION_FLT_UNDERFLOW: { PCONTEXT ctx = exceptionInfo->ContextRecord; -#ifndef _WIN64 - jint fp_control_word = (* (jint*) StubRoutines::x86::addr_fpu_cntrl_wrd_std()); - if (fp_control_word != ctx->FloatSave.ControlWord) { - // Restore FPCW and mask out FLT exceptions - ctx->FloatSave.ControlWord = fp_control_word | 0xffffffc0; - // Mask out pending FLT exceptions - ctx->FloatSave.StatusWord &= 0xffffff00; - return true; - } -#else // !_WIN64 // On Windows, the mxcsr control bits are non-volatile across calls // See also CR 6192333 // @@ -2639,7 +2578,6 @@ static bool handle_FLT_exception(struct _EXCEPTION_POINTERS* exceptionInfo) { ctx->MxCsr = MxCsr; return true; } -#endif // !_WIN64 } } @@ -2647,23 +2585,6 @@ static bool handle_FLT_exception(struct _EXCEPTION_POINTERS* exceptionInfo) { } #endif -#ifndef _WIN64 -static LONG WINAPI Uncaught_Exception_Handler(struct _EXCEPTION_POINTERS* exceptionInfo) { - if (handle_FLT_exception(exceptionInfo)) { - return EXCEPTION_CONTINUE_EXECUTION; - } - - // we only override this on 32 bits, so only check it there - if (prev_uef_handler != nullptr) { - // We didn't handle this exception so pass it to the previous - // UnhandledExceptionFilter. - return (prev_uef_handler)(exceptionInfo); - } - - return EXCEPTION_CONTINUE_SEARCH; -} -#endif - static inline void report_error(Thread* t, DWORD exception_code, address addr, void* siginfo, void* context) { VMError::report_and_die(t, exception_code, addr, siginfo, context); @@ -2683,98 +2604,18 @@ LONG WINAPI topLevelExceptionFilter(struct _EXCEPTION_POINTERS* exceptionInfo) { #elif defined(_M_AMD64) address pc = (address) exceptionInfo->ContextRecord->Rip; #else - address pc = (address) exceptionInfo->ContextRecord->Eip; + #error unknown architecture #endif Thread* t = Thread::current_or_null_safe(); -#ifndef _WIN64 - // Execution protection violation - win32 running on AMD64 only - // Handled first to avoid misdiagnosis as a "normal" access violation; - // This is safe to do because we have a new/unique ExceptionInformation - // code for this condition. - if (exception_code == EXCEPTION_ACCESS_VIOLATION) { - int exception_subcode = (int) exception_record->ExceptionInformation[0]; - address addr = (address) exception_record->ExceptionInformation[1]; - - if (exception_subcode == EXCEPTION_INFO_EXEC_VIOLATION) { - size_t page_size = os::vm_page_size(); - - // Make sure the pc and the faulting address are sane. - // - // If an instruction spans a page boundary, and the page containing - // the beginning of the instruction is executable but the following - // page is not, the pc and the faulting address might be slightly - // different - we still want to unguard the 2nd page in this case. - // - // 15 bytes seems to be a (very) safe value for max instruction size. - bool pc_is_near_addr = - (pointer_delta((void*) addr, (void*) pc, sizeof(char)) < 15); - bool instr_spans_page_boundary = - (align_down((intptr_t) pc ^ (intptr_t) addr, - (intptr_t) page_size) > 0); - - if (pc == addr || (pc_is_near_addr && instr_spans_page_boundary)) { - static volatile address last_addr = - (address) os::non_memory_address_word(); - - // In conservative mode, don't unguard unless the address is in the VM - if (UnguardOnExecutionViolation > 0 && addr != last_addr && - (UnguardOnExecutionViolation > 1 || os::address_is_in_vm(addr))) { - - // Set memory to RWX and retry - address page_start = align_down(addr, page_size); - bool res = os::protect_memory((char*) page_start, page_size, - os::MEM_PROT_RWX); - - log_debug(os)("Execution protection violation " - "at " INTPTR_FORMAT - ", unguarding " INTPTR_FORMAT ": %s", p2i(addr), - p2i(page_start), (res ? "success" : os::strerror(errno))); - - // Set last_addr so if we fault again at the same address, we don't - // end up in an endless loop. - // - // There are two potential complications here. Two threads trapping - // at the same address at the same time could cause one of the - // threads to think it already unguarded, and abort the VM. Likely - // very rare. - // - // The other race involves two threads alternately trapping at - // different addresses and failing to unguard the page, resulting in - // an endless loop. This condition is probably even more unlikely - // than the first. - // - // Although both cases could be avoided by using locks or thread - // local last_addr, these solutions are unnecessary complication: - // this handler is a best-effort safety net, not a complete solution. - // It is disabled by default and should only be used as a workaround - // in case we missed any no-execute-unsafe VM code. - - last_addr = addr; - - return EXCEPTION_CONTINUE_EXECUTION; - } - } - - // Last unguard failed or not unguarding - tty->print_raw_cr("Execution protection violation"); -#if !defined(USE_VECTORED_EXCEPTION_HANDLING) - report_error(t, exception_code, addr, exception_record, - exceptionInfo->ContextRecord); -#endif - return EXCEPTION_CONTINUE_SEARCH; - } - } -#endif // _WIN64 - -#if defined(_M_AMD64) || defined(_M_IX86) +#if defined(_M_AMD64) if ((exception_code == EXCEPTION_ACCESS_VIOLATION) && VM_Version::is_cpuinfo_segv_addr(pc)) { // Verify that OS save/restore AVX registers. return Handle_Exception(exceptionInfo, VM_Version::cpuinfo_cont_addr()); } -#if !defined(PRODUCT) && defined(_LP64) +#if !defined(PRODUCT) if ((exception_code == EXCEPTION_ACCESS_VIOLATION) && VM_Version::is_cpuinfo_segv_addr_apx(pc)) { // Verify that OS save/restore APX registers. @@ -2842,7 +2683,6 @@ LONG WINAPI topLevelExceptionFilter(struct _EXCEPTION_POINTERS* exceptionInfo) { return Handle_Exception(exceptionInfo, stub); } } -#ifdef _WIN64 // If it's a legal stack address map the entire region in if (thread->is_in_usable_stack(addr)) { addr = (address)((uintptr_t)addr & @@ -2851,7 +2691,6 @@ LONG WINAPI topLevelExceptionFilter(struct _EXCEPTION_POINTERS* exceptionInfo) { !ExecMem); return EXCEPTION_CONTINUE_EXECUTION; } -#endif // Null pointer exception. if (MacroAssembler::uses_implicit_null_check((void*)addr)) { address stub = SharedRuntime::continuation_for_implicit_exception(thread, pc, SharedRuntime::IMPLICIT_NULL); @@ -2862,7 +2701,6 @@ LONG WINAPI topLevelExceptionFilter(struct _EXCEPTION_POINTERS* exceptionInfo) { return EXCEPTION_CONTINUE_SEARCH; } -#ifdef _WIN64 // Special care for fast JNI field accessors. // jni_fast_GetField can trap at certain pc's if a GC kicks // in and the heap gets shrunk before the field access. @@ -2870,7 +2708,6 @@ LONG WINAPI topLevelExceptionFilter(struct _EXCEPTION_POINTERS* exceptionInfo) { if (slowcase_pc != (address)-1) { return Handle_Exception(exceptionInfo, slowcase_pc); } -#endif // Stack overflow or null pointer exception in native code. #if !defined(USE_VECTORED_EXCEPTION_HANDLING) @@ -2923,7 +2760,7 @@ LONG WINAPI topLevelExceptionFilter(struct _EXCEPTION_POINTERS* exceptionInfo) { } // switch } -#if defined(_M_AMD64) || defined(_M_IX86) +#if defined(_M_AMD64) if ((in_java || in_native) && handle_FLT_exception(exceptionInfo)) { return EXCEPTION_CONTINUE_EXECUTION; } @@ -2967,7 +2804,7 @@ LONG WINAPI topLevelVectoredExceptionFilter(struct _EXCEPTION_POINTERS* exceptio #elif defined(_M_AMD64) address pc = (address) exceptionInfo->ContextRecord->Rip; #else - address pc = (address) exceptionInfo->ContextRecord->Eip; + #error unknown architecture #endif // Fast path for code part of the code cache @@ -3009,63 +2846,6 @@ LONG WINAPI topLevelUnhandledExceptionFilter(struct _EXCEPTION_POINTERS* excepti } #endif -#ifndef _WIN64 -// Special care for fast JNI accessors. -// jni_fast_GetField can trap at certain pc's if a GC kicks in and -// the heap gets shrunk before the field access. -// Need to install our own structured exception handler since native code may -// install its own. -LONG WINAPI fastJNIAccessorExceptionFilter(struct _EXCEPTION_POINTERS* exceptionInfo) { - DWORD exception_code = exceptionInfo->ExceptionRecord->ExceptionCode; - if (exception_code == EXCEPTION_ACCESS_VIOLATION) { - address pc = (address) exceptionInfo->ContextRecord->Eip; - address addr = JNI_FastGetField::find_slowcase_pc(pc); - if (addr != (address)-1) { - return Handle_Exception(exceptionInfo, addr); - } - } - return EXCEPTION_CONTINUE_SEARCH; -} - -#define DEFINE_FAST_GETFIELD(Return, Fieldname, Result) \ - Return JNICALL jni_fast_Get##Result##Field_wrapper(JNIEnv *env, \ - jobject obj, \ - jfieldID fieldID) { \ - __try { \ - return (*JNI_FastGetField::jni_fast_Get##Result##Field_fp)(env, \ - obj, \ - fieldID); \ - } __except(fastJNIAccessorExceptionFilter((_EXCEPTION_POINTERS*) \ - _exception_info())) { \ - } \ - return 0; \ - } - -DEFINE_FAST_GETFIELD(jboolean, bool, Boolean) -DEFINE_FAST_GETFIELD(jbyte, byte, Byte) -DEFINE_FAST_GETFIELD(jchar, char, Char) -DEFINE_FAST_GETFIELD(jshort, short, Short) -DEFINE_FAST_GETFIELD(jint, int, Int) -DEFINE_FAST_GETFIELD(jlong, long, Long) -DEFINE_FAST_GETFIELD(jfloat, float, Float) -DEFINE_FAST_GETFIELD(jdouble, double, Double) - -address os::win32::fast_jni_accessor_wrapper(BasicType type) { - switch (type) { - case T_BOOLEAN: return (address)jni_fast_GetBooleanField_wrapper; - case T_BYTE: return (address)jni_fast_GetByteField_wrapper; - case T_CHAR: return (address)jni_fast_GetCharField_wrapper; - case T_SHORT: return (address)jni_fast_GetShortField_wrapper; - case T_INT: return (address)jni_fast_GetIntField_wrapper; - case T_LONG: return (address)jni_fast_GetLongField_wrapper; - case T_FLOAT: return (address)jni_fast_GetFloatField_wrapper; - case T_DOUBLE: return (address)jni_fast_GetDoubleField_wrapper; - default: ShouldNotReachHere(); - } - return (address)-1; -} -#endif - // Virtual Memory // Windows large page support is available on Windows 2003. In order to use @@ -3341,12 +3121,7 @@ size_t os::win32::large_page_init_decide_size() { return 0; } -#if defined(IA32) - if (size > 4 * M || LargePageSizeInBytes > 4 * M) { - WARN("JVM cannot use large pages bigger than 4mb."); - return 0; - } -#elif defined(AMD64) +#if defined(AMD64) if (!EnableAllLargePageSizesForWindows) { if (size > 4 * M || LargePageSizeInBytes > 4 * M) { WARN("JVM cannot use large pages bigger than 4mb."); @@ -3430,13 +3205,8 @@ char* os::map_memory_to_file(char* base, size_t size, int fd) { assert(fd != -1, "File descriptor is not valid"); HANDLE fh = (HANDLE)_get_osfhandle(fd); -#ifdef _LP64 HANDLE fileMapping = CreateFileMapping(fh, nullptr, PAGE_READWRITE, (DWORD)(size >> 32), (DWORD)(size & 0xFFFFFFFF), nullptr); -#else - HANDLE fileMapping = CreateFileMapping(fh, nullptr, PAGE_READWRITE, - 0, (DWORD)size, nullptr); -#endif if (fileMapping == nullptr) { if (GetLastError() == ERROR_DISK_FULL) { vm_exit_during_initialization(err_msg("Could not allocate sufficient disk space for Java heap")); @@ -3825,8 +3595,7 @@ bool os::pd_release_memory(char* addr, size_t bytes) { if (err != nullptr) { log_warning(os)("bad release: [" PTR_FORMAT "-" PTR_FORMAT "): %s", p2i(start), p2i(end), err); #ifdef ASSERT - fileStream fs(stdout); - os::print_memory_mappings((char*)start, bytes, &fs); + os::print_memory_mappings((char*)start, bytes, tty); assert(false, "bad release: [" PTR_FORMAT "-" PTR_FORMAT "): %s", p2i(start), p2i(end), err); #endif return false; @@ -4151,7 +3920,7 @@ void getWindowsInstallationType(char* buffer, int bufferSize) { } // Query the value - if (RegQueryValueExA(hKey, valueName, NULL, NULL, (LPBYTE)buffer, &valueLength) != ERROR_SUCCESS) { + if (RegQueryValueExA(hKey, valueName, nullptr, nullptr, (LPBYTE)buffer, &valueLength) != ERROR_SUCCESS) { RegCloseKey(hKey); buffer[0] = '\0'; return; @@ -4619,32 +4388,6 @@ bool os::message_box(const char* title, const char* message) { return result == IDYES; } -#ifndef PRODUCT -#ifndef _WIN64 -// Helpers to check whether NX protection is enabled -int nx_exception_filter(_EXCEPTION_POINTERS *pex) { - if (pex->ExceptionRecord->ExceptionCode == EXCEPTION_ACCESS_VIOLATION && - pex->ExceptionRecord->NumberParameters > 0 && - pex->ExceptionRecord->ExceptionInformation[0] == - EXCEPTION_INFO_EXEC_VIOLATION) { - return EXCEPTION_EXECUTE_HANDLER; - } - return EXCEPTION_CONTINUE_SEARCH; -} - -void nx_check_protection() { - // If NX is enabled we'll get an exception calling into code on the stack - char code[] = { (char)0xC3 }; // ret - void *code_ptr = (void *)code; - __try { - __asm call code_ptr - } __except(nx_exception_filter((_EXCEPTION_POINTERS*)_exception_info())) { - tty->print_raw_cr("NX protection detected."); - } -} -#endif // _WIN64 -#endif // PRODUCT - // This is called _before_ the global arguments have been parsed void os::init(void) { _initial_pid = _getpid(); @@ -4664,9 +4407,6 @@ void os::init(void) { fatal("DuplicateHandle failed\n"); } main_thread_id = (int) GetCurrentThreadId(); - - // initialize fast thread access - only used for 32-bit - win32::initialize_thread_ptr_offset(); } // To install functions for atexit processing @@ -4682,11 +4422,7 @@ static jint initSock(); // HotSpot guard pages is added later. size_t os::_compiler_thread_min_stack_allowed = 48 * K; size_t os::_java_thread_min_stack_allowed = 40 * K; -#ifdef _LP64 size_t os::_vm_internal_thread_min_stack_allowed = 64 * K; -#else -size_t os::_vm_internal_thread_min_stack_allowed = (48 DEBUG_ONLY(+ 4)) * K; -#endif // _LP64 // If stack_commit_size is 0, windows will reserve the default size, // but only commit a small portion of it. This stack size is the size of this @@ -4714,16 +4450,6 @@ jint os::init_2(void) { previousUnhandledExceptionFilter = SetUnhandledExceptionFilter(topLevelUnhandledExceptionFilter); #endif - // for debugging float code generation bugs -#if defined(ASSERT) && !defined(_WIN64) - static long fp_control_word = 0; - __asm { fstcw fp_control_word } - // see Intel PPro Manual, Vol. 2, p 7-16 - const long invalid = 0x01; - fp_control_word |= invalid; - __asm { fldcw fp_control_word } -#endif - // Check and sets minimum stack sizes against command line options if (set_minimum_stack_sizes() == JNI_ERR) { return JNI_ERR; @@ -4747,11 +4473,6 @@ jint os::init_2(void) { } } -#ifndef _WIN64 - // Print something if NX is enabled (win32 on AMD64) - NOT_PRODUCT(if (PrintMiscellaneous && Verbose) nx_check_protection()); -#endif - // initialize thread priority policy prio_init(); @@ -6007,13 +5728,6 @@ ssize_t os::raw_send(int fd, char* buf, size_t nBytes, uint flags) { return ::send(fd, buf, (int)nBytes, flags); } -// WINDOWS CONTEXT Flags for THREAD_SAMPLING -#if defined(IA32) - #define sampling_context_flags (CONTEXT_FULL | CONTEXT_FLOATING_POINT | CONTEXT_EXTENDED_REGISTERS) -#elif defined(AMD64) || defined(_M_ARM64) - #define sampling_context_flags (CONTEXT_FULL | CONTEXT_FLOATING_POINT) -#endif - // returns true if thread could be suspended, // false otherwise static bool do_suspend(HANDLE* h) { @@ -6058,7 +5772,7 @@ void SuspendedThreadTask::internal_do_task() { // suspend the thread if (do_suspend(&h)) { - ctxt.ContextFlags = sampling_context_flags; + ctxt.ContextFlags = (CONTEXT_FULL | CONTEXT_FLOATING_POINT); // get thread context GetThreadContext(h, &ctxt); SuspendedThreadTaskContext context(_thread, &ctxt); @@ -6104,7 +5818,6 @@ void* os::get_default_process_handle() { // Builds a platform dependent Agent_OnLoad_ function name // which is used to find statically linked in agents. -// Additionally for windows, takes into account __stdcall names. // Parameters: // sym_name: Symbol in library we are looking for // lib_name: Name of library to look in, null for shared libs. @@ -6145,24 +5858,11 @@ char* os::build_agent_function_name(const char *sym_name, const char *lib_name, if (agent_entry_name == nullptr) { return nullptr; } + + strcpy(agent_entry_name, sym_name); if (lib_name != nullptr) { - const char *p = strrchr(sym_name, '@'); - if (p != nullptr && p != sym_name) { - // sym_name == _Agent_OnLoad@XX - strncpy(agent_entry_name, sym_name, (p - sym_name)); - agent_entry_name[(p-sym_name)] = '\0'; - // agent_entry_name == _Agent_OnLoad - strcat(agent_entry_name, "_"); - strncat(agent_entry_name, lib_name, name_len); - strcat(agent_entry_name, p); - // agent_entry_name == _Agent_OnLoad_lib_name@XX - } else { - strcpy(agent_entry_name, sym_name); - strcat(agent_entry_name, "_"); - strncat(agent_entry_name, lib_name, name_len); - } - } else { - strcpy(agent_entry_name, sym_name); + strcat(agent_entry_name, "_"); + strncat(agent_entry_name, lib_name, name_len); } return agent_entry_name; } @@ -6201,19 +5901,6 @@ int os::get_signal_number(const char* name) { return -1; } -// Fast current thread access - -int os::win32::_thread_ptr_offset = 0; - -static void call_wrapper_dummy() {} - -// We need to call the os_exception_wrapper once so that it sets -// up the offset from FS of the thread pointer. -void os::win32::initialize_thread_ptr_offset() { - os::os_exception_wrapper((java_call_t)call_wrapper_dummy, - nullptr, methodHandle(), nullptr, nullptr); -} - bool os::supports_map_sync() { return false; } @@ -6288,7 +5975,7 @@ bool os::win32::find_mapping(address addr, mapping_info_t* mi) { // Helper for print_one_mapping: print n words, both as hex and ascii. // Use Safefetch for all values. static void print_snippet(const void* p, outputStream* st) { - static const int num_words = LP64_ONLY(3) NOT_LP64(6); + static const int num_words = 3; static const int num_bytes = num_words * sizeof(int); intptr_t v[num_words]; const int errval = 0xDE210244; @@ -6331,8 +6018,7 @@ static address print_one_mapping(MEMORY_BASIC_INFORMATION* minfo, address start, if (first_line) { st->print("Base " PTR_FORMAT ": ", p2i(allocation_base)); } else { - st->print_raw(NOT_LP64 (" ") - LP64_ONLY(" ")); + st->print_raw(" "); } address region_start = (address)minfo->BaseAddress; address region_end = region_start + minfo->RegionSize; @@ -6432,7 +6118,7 @@ void os::print_memory_mappings(char* addr, size_t bytes, outputStream* st) { // Here, we advance the probe pointer by alloc granularity. But if the range to print // is large, this may take a long time. Therefore lets stop right away if the address // is outside of what we know are valid addresses on Windows. Also, add a loop fuse. - static const address end_virt = (address)(LP64_ONLY(0x7ffffffffffULL) NOT_LP64(3*G)); + static const address end_virt = (address)(0x7ffffffffffULL); if (p >= end_virt) { break; } else { diff --git a/src/hotspot/os/windows/os_windows.hpp b/src/hotspot/os/windows/os_windows.hpp index 1d523724300..1aba43fb3d2 100644 --- a/src/hotspot/os/windows/os_windows.hpp +++ b/src/hotspot/os/windows/os_windows.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -146,22 +146,7 @@ class os::win32 { // return information about that area. static bool find_mapping(address p, mapping_info_t* mapping_info); -#ifndef _WIN64 - // A wrapper to install a structured exception handler for fast JNI accessors. - static address fast_jni_accessor_wrapper(BasicType); -#endif - - // Fast access to current thread -protected: - static int _thread_ptr_offset; -private: - static void initialize_thread_ptr_offset(); public: - static inline void set_thread_ptr_offset(int offset) { - _thread_ptr_offset = offset; - } - static inline int get_thread_ptr_offset() { return _thread_ptr_offset; } - // signal support static void* install_signal_handler(int sig, signal_handler_t handler); static void* user_handler(); diff --git a/src/hotspot/os/windows/perfMemory_windows.cpp b/src/hotspot/os/windows/perfMemory_windows.cpp index 959be982fab..06b057315cb 100644 --- a/src/hotspot/os/windows/perfMemory_windows.cpp +++ b/src/hotspot/os/windows/perfMemory_windows.cpp @@ -1803,7 +1803,7 @@ void PerfMemory::detach(char* addr, size_t bytes) { if (MemTracker::enabled()) { // it does not go through os api, the operation has to record from here - NmtVirtualMemoryLocker ml; + ThreadCritical tc; remove_file_mapping(addr); MemTracker::record_virtual_memory_release((address)addr, bytes); } else { diff --git a/src/hotspot/os/windows/sharedRuntimeRem.cpp b/src/hotspot/os/windows/sharedRuntimeRem.cpp index cb47cb7dfff..62a85e55a01 100644 --- a/src/hotspot/os/windows/sharedRuntimeRem.cpp +++ b/src/hotspot/os/windows/sharedRuntimeRem.cpp @@ -1,5 +1,5 @@ /* -* Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved. +* Copyright (c) 2015, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -25,7 +25,6 @@ #include "precompiled.hpp" #include "runtime/sharedRuntime.hpp" -#ifdef _WIN64 // These are copied defines originally from fdlibm.h. #define __HI(x) *(1+(int*)&x) @@ -158,5 +157,3 @@ double SharedRuntime::fmod_winx64(double x, double y) } return x; /* exact output */ } - -#endif diff --git a/src/hotspot/os_cpu/aix_ppc/os_aix_ppc.cpp b/src/hotspot/os_cpu/aix_ppc/os_aix_ppc.cpp index f83aa603062..45d91c60ed4 100644 --- a/src/hotspot/os_cpu/aix_ppc/os_aix_ppc.cpp +++ b/src/hotspot/os_cpu/aix_ppc/os_aix_ppc.cpp @@ -123,13 +123,13 @@ frame os::fetch_frame_from_context(const void* ucVoid) { intptr_t* sp; intptr_t* fp; address epc = fetch_frame_from_context(ucVoid, &sp, &fp); - // Avoid crash during crash if pc broken. - if (epc) { - frame fr(sp, epc, frame::kind::unknown); - return fr; + if (epc == nullptr || !is_readable_pointer(epc)) { + // Try to recover from calling into bad memory + // Assume new frame has not been set up, the same as + // compiled frame stack bang + return fetch_compiled_frame_from_context(ucVoid); } - frame fr(sp); - return fr; + return frame(sp, epc, frame::kind::unknown); } frame os::fetch_compiled_frame_from_context(const void* ucVoid) { @@ -447,23 +447,6 @@ void os::print_context(outputStream *st, const void *context) { st->cr(); } -void os::print_tos_pc(outputStream *st, const void *context) { - if (context == nullptr) return; - - const ucontext_t* uc = (const ucontext_t*)context; - - address sp = (address)os::Aix::ucontext_get_sp(uc); - print_tos(st, sp); - st->cr(); - - // Note: it may be unsafe to inspect memory near pc. For example, pc may - // point to garbage if entry point in an nmethod is corrupted. Leave - // this at the end, and hope for the best. - address pc = os::Posix::ucontext_get_pc(uc); - print_instructions(st, pc); - st->cr(); -} - void os::print_register_info(outputStream *st, const void *context, int& continuation) { const int register_count = 32 /* r0-r32 */ + 3 /* pc, lr, sp */; int n = continuation; diff --git a/src/hotspot/os_cpu/bsd_aarch64/os_bsd_aarch64.cpp b/src/hotspot/os_cpu/bsd_aarch64/os_bsd_aarch64.cpp index ba14cb2ac12..7702dbd17ad 100644 --- a/src/hotspot/os_cpu/bsd_aarch64/os_bsd_aarch64.cpp +++ b/src/hotspot/os_cpu/bsd_aarch64/os_bsd_aarch64.cpp @@ -159,6 +159,12 @@ frame os::fetch_frame_from_context(const void* ucVoid) { intptr_t* sp; intptr_t* fp; address epc = fetch_frame_from_context(ucVoid, &sp, &fp); + if (!is_readable_pointer(epc)) { + // Try to recover from calling into bad memory + // Assume new frame has not been set up, the same as + // compiled frame stack bang + return fetch_compiled_frame_from_context(ucVoid); + } return frame(sp, fp, epc); } @@ -442,23 +448,6 @@ void os::print_context(outputStream *st, const void *context) { st->cr(); } -void os::print_tos_pc(outputStream *st, const void *context) { - if (context == nullptr) return; - - const ucontext_t* uc = (const ucontext_t*)context; - - address sp = (address)os::Bsd::ucontext_get_sp(uc); - print_tos(st, sp); - st->cr(); - - // Note: it may be unsafe to inspect memory near pc. For example, pc may - // point to garbage if entry point in an nmethod is corrupted. Leave - // this at the end, and hope for the best. - address pc = os::Posix::ucontext_get_pc(uc); - print_instructions(st, pc); - st->cr(); -} - void os::print_register_info(outputStream *st, const void *context, int& continuation) { const int register_count = 29 /* x0-x28 */ + 3 /* fp, lr, sp */; int n = continuation; diff --git a/src/hotspot/os_cpu/bsd_x86/os_bsd_x86.cpp b/src/hotspot/os_cpu/bsd_x86/os_bsd_x86.cpp index 437274a2cb1..153c5ad7e2b 100644 --- a/src/hotspot/os_cpu/bsd_x86/os_bsd_x86.cpp +++ b/src/hotspot/os_cpu/bsd_x86/os_bsd_x86.cpp @@ -333,6 +333,12 @@ frame os::fetch_frame_from_context(const void* ucVoid) { intptr_t* sp; intptr_t* fp; address epc = fetch_frame_from_context(ucVoid, &sp, &fp); + if (!is_readable_pointer(epc)) { + // Try to recover from calling into bad memory + // Assume new frame has not been set up, the same as + // compiled frame stack bang + return fetch_compiled_frame_from_context(ucVoid); + } return frame(sp, fp, epc); } @@ -836,23 +842,6 @@ void os::print_context(outputStream *st, const void *context) { st->cr(); } -void os::print_tos_pc(outputStream *st, const void *context) { - if (context == nullptr) return; - - const ucontext_t* uc = (const ucontext_t*)context; - - address sp = (address)os::Bsd::ucontext_get_sp(uc); - print_tos(st, sp); - st->cr(); - - // Note: it may be unsafe to inspect memory near pc. For example, pc may - // point to garbage if entry point in an nmethod is corrupted. Leave - // this at the end, and hope for the best. - address pc = os::Posix::ucontext_get_pc(uc); - print_instructions(st, pc); - st->cr(); -} - void os::print_register_info(outputStream *st, const void *context, int& continuation) { const int register_count = AMD64_ONLY(16) NOT_AMD64(8); int n = continuation; diff --git a/src/hotspot/os_cpu/bsd_zero/os_bsd_zero.cpp b/src/hotspot/os_cpu/bsd_zero/os_bsd_zero.cpp index 0fc9484ce23..4ecdbe93ebf 100644 --- a/src/hotspot/os_cpu/bsd_zero/os_bsd_zero.cpp +++ b/src/hotspot/os_cpu/bsd_zero/os_bsd_zero.cpp @@ -228,10 +228,6 @@ void os::print_context(outputStream* st, const void* context) { ShouldNotCallThis(); } -void os::print_tos_pc(outputStream *st, const void *context) { - ShouldNotCallThis(); -} - void os::print_register_info(outputStream *st, const void *context, int& continuation) { ShouldNotCallThis(); } diff --git a/src/hotspot/os_cpu/linux_aarch64/os_linux_aarch64.cpp b/src/hotspot/os_cpu/linux_aarch64/os_linux_aarch64.cpp index 9f44fe3aa25..a7ec163f785 100644 --- a/src/hotspot/os_cpu/linux_aarch64/os_linux_aarch64.cpp +++ b/src/hotspot/os_cpu/linux_aarch64/os_linux_aarch64.cpp @@ -354,23 +354,6 @@ void os::print_context(outputStream *st, const void *context) { st->cr(); } -void os::print_tos_pc(outputStream *st, const void *context) { - if (context == nullptr) return; - - const ucontext_t* uc = (const ucontext_t*)context; - - address sp = (address)os::Linux::ucontext_get_sp(uc); - print_tos(st, sp); - st->cr(); - - // Note: it may be unsafe to inspect memory near pc. For example, pc may - // point to garbage if entry point in an nmethod is corrupted. Leave - // this at the end, and hope for the best. - address pc = os::fetch_frame_from_context(uc).pc(); - print_instructions(st, pc); - st->cr(); -} - void os::print_register_info(outputStream *st, const void *context, int& continuation) { const int register_count = 32 /* r0-r31 */; int n = continuation; diff --git a/src/hotspot/os_cpu/linux_arm/os_linux_arm.cpp b/src/hotspot/os_cpu/linux_arm/os_linux_arm.cpp index 807fa765897..861d0d20153 100644 --- a/src/hotspot/os_cpu/linux_arm/os_linux_arm.cpp +++ b/src/hotspot/os_cpu/linux_arm/os_linux_arm.cpp @@ -188,9 +188,27 @@ frame os::fetch_frame_from_context(const void* ucVoid) { intptr_t* sp; intptr_t* fp; address epc = fetch_frame_from_context(ucVoid, &sp, &fp); + if (!is_readable_pointer(epc)) { + // Try to recover from calling into bad memory + // Assume new frame has not been set up, the same as + // compiled frame stack bang + return fetch_compiled_frame_from_context(ucVoid); + } return frame(sp, fp, epc); } +frame os::fetch_compiled_frame_from_context(const void* ucVoid) { + const ucontext_t* uc = (const ucontext_t*)ucVoid; + // In compiled code, the stack banging is performed before LR + // has been saved in the frame. LR is live, and SP and FP + // belong to the caller. + intptr_t* fp = os::Linux::ucontext_get_fp(uc); + intptr_t* sp = os::Linux::ucontext_get_sp(uc); + address pc = (address)(uc->uc_mcontext.arm_lr + - NativeInstruction::instruction_size); + return frame(sp, fp, pc); +} + frame os::get_sender_for_C_frame(frame* fr) { #ifdef __thumb__ // We can't reliably get anything from a thumb C frame. @@ -474,23 +492,6 @@ void os::print_context(outputStream *st, const void *context) { st->cr(); } -void os::print_tos_pc(outputStream *st, const void *context) { - if (context == nullptr) return; - - const ucontext_t* uc = (const ucontext_t*)context; - - address sp = (address)os::Linux::ucontext_get_sp(uc); - print_tos(st, sp); - st->cr(); - - // Note: it may be unsafe to inspect memory near pc. For example, pc may - // point to garbage if entry point in an nmethod is corrupted. Leave - // this at the end, and hope for the best. - address pc = os::Posix::ucontext_get_pc(uc); - print_instructions(st, pc); - st->cr(); -} - void os::print_register_info(outputStream *st, const void *context, int& continuation) { const int register_count = ARM_REGS_IN_CONTEXT; int n = continuation; diff --git a/src/hotspot/os_cpu/linux_ppc/os_linux_ppc.cpp b/src/hotspot/os_cpu/linux_ppc/os_linux_ppc.cpp index 1857097bd52..f3f9a3a88df 100644 --- a/src/hotspot/os_cpu/linux_ppc/os_linux_ppc.cpp +++ b/src/hotspot/os_cpu/linux_ppc/os_linux_ppc.cpp @@ -155,6 +155,12 @@ frame os::fetch_frame_from_context(const void* ucVoid) { intptr_t* sp; intptr_t* fp; address epc = fetch_frame_from_context(ucVoid, &sp, &fp); + if (!is_readable_pointer(epc)) { + // Try to recover from calling into bad memory + // Assume new frame has not been set up, the same as + // compiled frame stack bang + return fetch_compiled_frame_from_context(ucVoid); + } return frame(sp, epc, frame::kind::unknown); } @@ -461,23 +467,6 @@ void os::print_context(outputStream *st, const void *context) { st->cr(); } -void os::print_tos_pc(outputStream *st, const void *context) { - if (context == nullptr) return; - - const ucontext_t* uc = (const ucontext_t*)context; - - address sp = (address)os::Linux::ucontext_get_sp(uc); - print_tos(st, sp); - st->cr(); - - // Note: it may be unsafe to inspect memory near pc. For example, pc may - // point to garbage if entry point in an nmethod is corrupted. Leave - // this at the end, and hope for the best. - address pc = os::Posix::ucontext_get_pc(uc); - print_instructions(st, pc); - st->cr(); -} - void os::print_register_info(outputStream *st, const void *context, int& continuation) { const int register_count = 32 /* r0-r32 */ + 3 /* pc, lr, ctr */; int n = continuation; diff --git a/src/hotspot/os_cpu/linux_riscv/os_linux_riscv.cpp b/src/hotspot/os_cpu/linux_riscv/os_linux_riscv.cpp index fcb0e170af1..a00659f37cb 100644 --- a/src/hotspot/os_cpu/linux_riscv/os_linux_riscv.cpp +++ b/src/hotspot/os_cpu/linux_riscv/os_linux_riscv.cpp @@ -352,23 +352,6 @@ void os::print_context(outputStream *st, const void *context) { st->cr(); } -void os::print_tos_pc(outputStream *st, const void *context) { - if (context == nullptr) return; - - const ucontext_t* uc = (const ucontext_t*)context; - - address sp = (address)os::Linux::ucontext_get_sp(uc); - print_tos(st, sp); - st->cr(); - - // Note: it may be unsafe to inspect memory near pc. For example, pc may - // point to garbage if entry point in an nmethod is corrupted. Leave - // this at the end, and hope for the best. - address pc = os::fetch_frame_from_context(uc).pc(); - print_instructions(st, pc); - st->cr(); -} - void os::print_register_info(outputStream *st, const void *context, int& continuation) { const int register_count = 32; int n = continuation; diff --git a/src/hotspot/os_cpu/linux_s390/os_linux_s390.cpp b/src/hotspot/os_cpu/linux_s390/os_linux_s390.cpp index 9ac1152a013..bc8e3d10431 100644 --- a/src/hotspot/os_cpu/linux_s390/os_linux_s390.cpp +++ b/src/hotspot/os_cpu/linux_s390/os_linux_s390.cpp @@ -140,6 +140,12 @@ frame os::fetch_frame_from_context(const void* ucVoid) { intptr_t* sp; intptr_t* fp; address epc = fetch_frame_from_context(ucVoid, &sp, &fp); + if (!is_readable_pointer(epc)) { + // Try to recover from calling into bad memory + // Assume new frame has not been set up, the same as + // compiled frame stack bang + return fetch_compiled_frame_from_context(ucVoid); + } return frame(sp, epc); } @@ -442,23 +448,6 @@ void os::print_context(outputStream *st, const void *context) { st->cr(); } -void os::print_tos_pc(outputStream *st, const void *context) { - if (context == nullptr) return; - - const ucontext_t* uc = (const ucontext_t*)context; - - address sp = (address)os::Linux::ucontext_get_sp(uc); - print_tos(st, sp); - st->cr(); - - // Note: it may be unsafe to inspect memory near pc. For example, pc may - // point to garbage if entry point in an nmethod is corrupted. Leave - // this at the end, and hope for the best. - address pc = os::Posix::ucontext_get_pc(uc); - print_instructions(st, pc); - st->cr(); -} - void os::print_register_info(outputStream *st, const void *context, int& continuation) { const int register_count = 16 /* r0-r15 */ + 1 /* pc */; int n = continuation; diff --git a/src/hotspot/os_cpu/linux_x86/os_linux_x86.cpp b/src/hotspot/os_cpu/linux_x86/os_linux_x86.cpp index 8fdcbe63c7e..e357747bfea 100644 --- a/src/hotspot/os_cpu/linux_x86/os_linux_x86.cpp +++ b/src/hotspot/os_cpu/linux_x86/os_linux_x86.cpp @@ -578,23 +578,6 @@ void os::print_context(outputStream *st, const void *context) { st->cr(); } -void os::print_tos_pc(outputStream *st, const void *context) { - if (context == nullptr) return; - - const ucontext_t* uc = (const ucontext_t*)context; - - address sp = (address)os::Linux::ucontext_get_sp(uc); - print_tos(st, sp); - st->cr(); - - // Note: it may be unsafe to inspect memory near pc. For example, pc may - // point to garbage if entry point in an nmethod is corrupted. Leave - // this at the end, and hope for the best. - address pc = os::fetch_frame_from_context(uc).pc(); - print_instructions(st, pc); - st->cr(); -} - void os::print_register_info(outputStream *st, const void *context, int& continuation) { const int register_count = AMD64_ONLY(16) NOT_AMD64(8); int n = continuation; diff --git a/src/hotspot/os_cpu/linux_zero/os_linux_zero.cpp b/src/hotspot/os_cpu/linux_zero/os_linux_zero.cpp index d593c46d15d..d8498f104c2 100644 --- a/src/hotspot/os_cpu/linux_zero/os_linux_zero.cpp +++ b/src/hotspot/os_cpu/linux_zero/os_linux_zero.cpp @@ -376,21 +376,6 @@ void os::print_context(outputStream* st, const void* ucVoid) { st->print_cr("No context information."); } -void os::print_tos_pc(outputStream *st, const void* ucVoid) { - const ucontext_t* uc = (const ucontext_t*)ucVoid; - - address sp = (address)os::Linux::ucontext_get_sp(uc); - print_tos(st, sp); - st->cr(); - - // Note: it may be unsafe to inspect memory near pc. For example, pc may - // point to garbage if entry point in an nmethod is corrupted. Leave - // this at the end, and hope for the best. - address pc = os::Posix::ucontext_get_pc(uc); - print_instructions(st, pc); - st->cr(); -} - void os::print_register_info(outputStream *st, const void *context, int& continuation) { st->print_cr("No register info."); } diff --git a/src/hotspot/os_cpu/windows_aarch64/os_windows_aarch64.cpp b/src/hotspot/os_cpu/windows_aarch64/os_windows_aarch64.cpp index 722c3c8dd60..24410ed9203 100644 --- a/src/hotspot/os_cpu/windows_aarch64/os_windows_aarch64.cpp +++ b/src/hotspot/os_cpu/windows_aarch64/os_windows_aarch64.cpp @@ -205,24 +205,6 @@ void os::print_context(outputStream *st, const void *context) { st->cr(); } -void os::print_tos_pc(outputStream *st, const void *context) { - if (context == nullptr) return; - - const CONTEXT* uc = (const CONTEXT*)context; - - address sp = (address)uc->Sp; - print_tos(st, sp); - st->cr(); - - // Note: it may be unsafe to inspect memory near pc. For example, pc may - // point to garbage if entry point in an nmethod is corrupted. Leave - // this at the end, and hope for the best. - address pc = (address)uc->Pc; - st->print_cr("Instructions: (pc=" PTR_FORMAT ")", pc); - print_hex_dump(st, pc - 32, pc + 32, sizeof(char), /* print_ascii=*/false); - st->cr(); -} - void os::print_register_info(outputStream *st, const void *context, int& continuation) { const int register_count = 29 /* X0-X28 */; int n = continuation; diff --git a/src/hotspot/os_cpu/windows_x86/assembler_windows_x86.cpp b/src/hotspot/os_cpu/windows_x86/assembler_windows_x86.cpp index aa1d5820637..c541213615c 100644 --- a/src/hotspot/os_cpu/windows_x86/assembler_windows_x86.cpp +++ b/src/hotspot/os_cpu/windows_x86/assembler_windows_x86.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1999, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1999, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -24,41 +24,7 @@ #include "precompiled.hpp" #include "asm/macroAssembler.hpp" -#include "asm/macroAssembler.inline.hpp" -#include "os_windows.hpp" -#include "runtime/os.hpp" void MacroAssembler::int3() { emit_int8((unsigned char)0xCC); } - -#ifndef _LP64 -// The current scheme to accelerate access to the thread -// pointer is to store the current thread in the os_exception_wrapper -// and reference the current thread from stubs and compiled code -// via the FS register. FS[0] contains a pointer to the structured -// exception block which is actually a stack address. The first time -// we call the os exception wrapper, we calculate and store the -// offset from this exception block and use that offset here. -// -// The last mechanism we used was problematic in that the -// the offset we had hard coded in the VM kept changing as Microsoft -// evolved the OS. -// -// Warning: This mechanism assumes that we only attempt to get the -// thread when we are nested below a call wrapper. -// -// movl reg, fs:[0] Get exception pointer -// movl reg, [reg + thread_ptr_offset] Load thread -// -void MacroAssembler::get_thread(Register thread) { - prefix(FS_segment); - movptr(thread, ExternalAddress(nullptr)); - assert(os::win32::get_thread_ptr_offset() != 0, - "Thread Pointer Offset has not been initialized"); - movl(thread, Address(thread, os::win32::get_thread_ptr_offset())); -} - -// #else - use shared x86 implementation in cpu/x86/vm/macroAssembler_x86.cpp - -#endif diff --git a/src/hotspot/os_cpu/windows_x86/atomic_windows_x86.hpp b/src/hotspot/os_cpu/windows_x86/atomic_windows_x86.hpp index aa0fef3b83a..9f7da19b328 100644 --- a/src/hotspot/os_cpu/windows_x86/atomic_windows_x86.hpp +++ b/src/hotspot/os_cpu/windows_x86/atomic_windows_x86.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1999, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1999, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -111,79 +111,4 @@ DEFINE_INTRINSIC_CMPXCHG(InterlockedCompareExchange64, __int64) #undef DEFINE_INTRINSIC_CMPXCHG -#ifndef AMD64 - -#pragma warning(disable: 4035) // Disables warnings reporting missing return statement - -template<> -template -inline T Atomic::PlatformLoad<8>::operator()(T const volatile* src) const { - STATIC_ASSERT(8 == sizeof(T)); - volatile T dest; - volatile T* pdest = &dest; - __asm { - mov eax, src - fild qword ptr [eax] - mov eax, pdest - fistp qword ptr [eax] - } - return dest; -} - -template<> -template -inline void Atomic::PlatformStore<8>::operator()(T volatile* dest, - T store_value) const { - STATIC_ASSERT(8 == sizeof(T)); - volatile T* src = &store_value; - __asm { - mov eax, src - fild qword ptr [eax] - mov eax, dest - fistp qword ptr [eax] - } -} - -#pragma warning(default: 4035) // Enables warnings reporting missing return statement - -template<> -struct Atomic::PlatformOrderedStore<1, RELEASE_X_FENCE> -{ - template - void operator()(volatile T* p, T v) const { - __asm { - mov edx, p; - mov al, v; - xchg al, byte ptr [edx]; - } - } -}; - -template<> -struct Atomic::PlatformOrderedStore<2, RELEASE_X_FENCE> -{ - template - void operator()(volatile T* p, T v) const { - __asm { - mov edx, p; - mov ax, v; - xchg ax, word ptr [edx]; - } - } -}; - -template<> -struct Atomic::PlatformOrderedStore<4, RELEASE_X_FENCE> -{ - template - void operator()(volatile T* p, T v) const { - __asm { - mov edx, p; - mov eax, v; - xchg eax, dword ptr [edx]; - } - } -}; -#endif // AMD64 - #endif // OS_CPU_WINDOWS_X86_ATOMIC_WINDOWS_X86_HPP diff --git a/src/hotspot/os_cpu/windows_x86/copy_windows_x86.hpp b/src/hotspot/os_cpu/windows_x86/copy_windows_x86.hpp index 6f4678e4a5c..4298542629b 100644 --- a/src/hotspot/os_cpu/windows_x86/copy_windows_x86.hpp +++ b/src/hotspot/os_cpu/windows_x86/copy_windows_x86.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2003, 2022, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2003, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -53,35 +53,7 @@ static void pd_conjoint_jints_atomic(const jint* from, jint* to, size_t count) { } static void pd_conjoint_jlongs_atomic(const jlong* from, jlong* to, size_t count) { -#ifdef AMD64 pd_conjoint_atomic_helper(from, to, count); -#else - // Guarantee use of fild/fistp or xmm regs via some asm code, because compilers won't. - __asm { - mov eax, from; - mov edx, to; - mov ecx, count; - cmp eax, edx; - jbe downtest; - jmp uptest; - up: - fild qword ptr [eax]; - fistp qword ptr [edx]; - add eax, 8; - add edx, 8; - uptest: - sub ecx, 1; - jge up; - jmp done; - down: - fild qword ptr [eax][ecx*8]; - fistp qword ptr [edx][ecx*8]; - downtest: - sub ecx, 1; - jge down; - done:; - } -#endif // AMD64 } static void pd_conjoint_oops_atomic(const oop* from, oop* to, size_t count) { @@ -89,11 +61,7 @@ static void pd_conjoint_oops_atomic(const oop* from, oop* to, size_t count) { } static void pd_arrayof_conjoint_bytes(const HeapWord* from, HeapWord* to, size_t count) { -#ifdef AMD64 pd_conjoint_bytes_atomic(from, to, count); -#else - pd_conjoint_bytes(from, to, count); -#endif // AMD64 } static void pd_arrayof_conjoint_jshorts(const HeapWord* from, HeapWord* to, size_t count) { diff --git a/src/hotspot/os_cpu/windows_x86/orderAccess_windows_x86.hpp b/src/hotspot/os_cpu/windows_x86/orderAccess_windows_x86.hpp index 43764b14a01..a1f1aa76912 100644 --- a/src/hotspot/os_cpu/windows_x86/orderAccess_windows_x86.hpp +++ b/src/hotspot/os_cpu/windows_x86/orderAccess_windows_x86.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2003, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2003, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -48,13 +48,7 @@ inline void OrderAccess::acquire() { compiler_barrier(); } inline void OrderAccess::release() { compiler_barrier(); } inline void OrderAccess::fence() { -#ifdef AMD64 StubRoutines_fence(); -#else - __asm { - lock add dword ptr [esp], 0; - } -#endif // AMD64 compiler_barrier(); } diff --git a/src/hotspot/os_cpu/windows_x86/os_windows_x86.cpp b/src/hotspot/os_cpu/windows_x86/os_windows_x86.cpp index de59a74cc24..9a23ac67335 100644 --- a/src/hotspot/os_cpu/windows_x86/os_windows_x86.cpp +++ b/src/hotspot/os_cpu/windows_x86/os_windows_x86.cpp @@ -56,15 +56,9 @@ #undef REG_SP #undef REG_FP #undef REG_PC -#ifdef AMD64 #define REG_SP Rsp #define REG_FP Rbp #define REG_PC Rip -#else -#define REG_SP Esp -#define REG_FP Ebp -#define REG_PC Eip -#endif // AMD64 JNIEXPORT extern LONG WINAPI topLevelExceptionFilter(_EXCEPTION_POINTERS* ); @@ -72,52 +66,12 @@ extern LONG WINAPI topLevelExceptionFilter(_EXCEPTION_POINTERS* ); // Install a win32 structured exception handler around thread. void os::os_exception_wrapper(java_call_t f, JavaValue* value, const methodHandle& method, JavaCallArguments* args, JavaThread* thread) { __try { - -#ifndef AMD64 - // We store the current thread in this wrapperthread location - // and determine how far away this address is from the structured - // exception pointer that FS:[0] points to. This get_thread - // code can then get the thread pointer via FS. - // - // Warning: This routine must NEVER be inlined since we'd end up with - // multiple offsets. - // - volatile Thread* wrapperthread = thread; - - if (os::win32::get_thread_ptr_offset() == 0) { - int thread_ptr_offset; - __asm { - lea eax, dword ptr wrapperthread; - sub eax, dword ptr FS:[0H]; - mov thread_ptr_offset, eax - }; - os::win32::set_thread_ptr_offset(thread_ptr_offset); - } -#ifdef ASSERT - // Verify that the offset hasn't changed since we initially captured - // it. This might happen if we accidentally ended up with an - // inlined version of this routine. - else { - int test_thread_ptr_offset; - __asm { - lea eax, dword ptr wrapperthread; - sub eax, dword ptr FS:[0H]; - mov test_thread_ptr_offset, eax - }; - assert(test_thread_ptr_offset == os::win32::get_thread_ptr_offset(), - "thread pointer offset from SEH changed"); - } -#endif // ASSERT -#endif // !AMD64 - f(value, method, args, thread); } __except(topLevelExceptionFilter((_EXCEPTION_POINTERS*)_exception_info())) { // Nothing to do. } } -#ifdef AMD64 - // This is the language specific handler for exceptions // originating from dynamically generated code. // We call the standard structured exception handler @@ -157,7 +111,6 @@ typedef struct { UNWIND_INFO_EH_ONLY unw; } DynamicCodeData, *pDynamicCodeData; -#endif // AMD64 // // Register our CodeCache area with the OS so it will dispatch exceptions // to our topLevelExceptionFilter when we take an exception in our @@ -167,8 +120,6 @@ typedef struct { // codeCache area // bool os::win32::register_code_area(char *low, char *high) { -#ifdef AMD64 - ResourceMark rm; pDynamicCodeData pDCD; @@ -206,7 +157,6 @@ bool os::win32::register_code_area(char *low, char *high) { guarantee(RtlAddFunctionTable(prt, 1, (ULONGLONG)low), "Failed to register Dynamic Code Exception Handler with RtlAddFunctionTable"); -#endif // AMD64 return true; } @@ -334,20 +284,6 @@ frame os::fetch_frame_from_context(const void* ucVoid) { return frame(sp, fp, epc); } -#ifndef AMD64 -// Ignore "C4172: returning address of local variable or temporary" on 32bit -PRAGMA_DIAG_PUSH -PRAGMA_DISABLE_MSVC_WARNING(4172) -// Returns an estimate of the current stack pointer. Result must be guaranteed -// to point into the calling threads stack, and be no lower than the current -// stack pointer. -address os::current_stack_pointer() { - int dummy; - address sp = (address)&dummy; - return sp; -} -PRAGMA_DIAG_POP -#else // Returns the current stack pointer. Accurate value needed for // os::verify_stack_alignment(). address os::current_stack_pointer() { @@ -356,7 +292,6 @@ address os::current_stack_pointer() { StubRoutines::x86::get_previous_sp_entry()); return (*func)(); } -#endif bool os::win32::get_frame_at_stack_banging_point(JavaThread* thread, struct _EXCEPTION_POINTERS* exceptionInfo, address pc, frame* fr) { @@ -414,7 +349,6 @@ void os::print_context(outputStream *st, const void *context) { const CONTEXT* uc = (const CONTEXT*)context; st->print_cr("Registers:"); -#ifdef AMD64 st->print( "RAX=" INTPTR_FORMAT, uc->Rax); st->print(", RBX=" INTPTR_FORMAT, uc->Rbx); st->print(", RCX=" INTPTR_FORMAT, uc->Rcx); @@ -446,43 +380,12 @@ void os::print_context(outputStream *st, const void *context) { i, xmm[1], xmm[0]); } st->print(" MXCSR=" UINT32_FORMAT_X_0, uc->MxCsr); -#else - st->print( "EAX=" INTPTR_FORMAT, uc->Eax); - st->print(", EBX=" INTPTR_FORMAT, uc->Ebx); - st->print(", ECX=" INTPTR_FORMAT, uc->Ecx); - st->print(", EDX=" INTPTR_FORMAT, uc->Edx); - st->cr(); - st->print( "ESP=" INTPTR_FORMAT, uc->Esp); - st->print(", EBP=" INTPTR_FORMAT, uc->Ebp); - st->print(", ESI=" INTPTR_FORMAT, uc->Esi); - st->print(", EDI=" INTPTR_FORMAT, uc->Edi); - st->cr(); - st->print( "EIP=" INTPTR_FORMAT, uc->Eip); - st->print(", EFLAGS=" INTPTR_FORMAT, uc->EFlags); -#endif // AMD64 st->cr(); st->cr(); } -void os::print_tos_pc(outputStream *st, const void *context) { - if (context == nullptr) return; - - const CONTEXT* uc = (const CONTEXT*)context; - - address sp = (address)uc->REG_SP; - print_tos(st, sp); - st->cr(); - - // Note: it may be unsafe to inspect memory near pc. For example, pc may - // point to garbage if entry point in an nmethod is corrupted. Leave - // this at the end, and hope for the best. - address pc = os::fetch_frame_from_context(uc).pc(); - print_instructions(st, pc); - st->cr(); -} - void os::print_register_info(outputStream *st, const void *context, int& continuation) { - const int register_count = AMD64_ONLY(16) NOT_AMD64(8); + const int register_count = 16; int n = continuation; assert(n >= 0 && n <= register_count, "Invalid continuation value"); if (context == nullptr || n == register_count) { @@ -495,7 +398,6 @@ void os::print_register_info(outputStream *st, const void *context, int& continu continuation = n + 1; # define CASE_PRINT_REG(n, str, id) case n: st->print(str); print_location(st, uc->id); switch (n) { -#ifdef AMD64 CASE_PRINT_REG( 0, "RAX=", Rax); break; CASE_PRINT_REG( 1, "RBX=", Rbx); break; CASE_PRINT_REG( 2, "RCX=", Rcx); break; @@ -512,16 +414,6 @@ void os::print_register_info(outputStream *st, const void *context, int& continu CASE_PRINT_REG(13, "R13=", R13); break; CASE_PRINT_REG(14, "R14=", R14); break; CASE_PRINT_REG(15, "R15=", R15); break; -#else - CASE_PRINT_REG(0, "EAX=", Eax); break; - CASE_PRINT_REG(1, "EBX=", Ebx); break; - CASE_PRINT_REG(2, "ECX=", Ecx); break; - CASE_PRINT_REG(3, "EDX=", Edx); break; - CASE_PRINT_REG(4, "ESP=", Esp); break; - CASE_PRINT_REG(5, "EBP=", Ebp); break; - CASE_PRINT_REG(6, "ESI=", Esi); break; - CASE_PRINT_REG(7, "EDI=", Edi); break; -#endif // AMD64 } # undef CASE_PRINT_REG ++n; @@ -529,17 +421,7 @@ void os::print_register_info(outputStream *st, const void *context, int& continu } extern "C" int SpinPause () { -#ifdef AMD64 return 0 ; -#else - // pause == rep:nop - // On systems that don't support pause a rep:nop - // is executed as a nop. The rep: prefix is ignored. - _asm { - pause ; - }; - return 1 ; -#endif // AMD64 } juint os::cpu_microcode_revision() { @@ -561,21 +443,15 @@ juint os::cpu_microcode_revision() { } void os::setup_fpu() { -#ifndef AMD64 - int fpu_cntrl_word = StubRoutines::x86::fpu_cntrl_wrd_std(); - __asm fldcw fpu_cntrl_word; -#endif // !AMD64 } #ifndef PRODUCT void os::verify_stack_alignment() { -#ifdef AMD64 // The current_stack_pointer() calls generated get_previous_sp stub routine. // Only enable the assert after the routine becomes available. if (StubRoutines::initial_stubs_code() != nullptr) { assert(((intptr_t)os::current_stack_pointer() & (StackAlignmentInBytes-1)) == 0, "incorrect stack alignment"); } -#endif } #endif diff --git a/src/hotspot/os_cpu/windows_x86/os_windows_x86.inline.hpp b/src/hotspot/os_cpu/windows_x86/os_windows_x86.inline.hpp index f7622611da7..0cc457609cc 100644 --- a/src/hotspot/os_cpu/windows_x86/os_windows_x86.inline.hpp +++ b/src/hotspot/os_cpu/windows_x86/os_windows_x86.inline.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011, 2022, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2011, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -28,13 +28,11 @@ #include "runtime/os.hpp" #include "os_windows.hpp" -#ifdef AMD64 #define HAVE_PLATFORM_PRINT_NATIVE_STACK 1 inline bool os::platform_print_native_stack(outputStream* st, const void* context, char *buf, int buf_size, address& lastpc) { return os::win32::platform_print_native_stack(st, context, buf, buf_size, lastpc); } -#endif inline jlong os::rdtsc() { // 32 bit: 64 bit result in edx:eax diff --git a/src/hotspot/os_cpu/windows_x86/unwind_windows_x86.hpp b/src/hotspot/os_cpu/windows_x86/unwind_windows_x86.hpp index 1f66e839fab..ca402a13675 100644 --- a/src/hotspot/os_cpu/windows_x86/unwind_windows_x86.hpp +++ b/src/hotspot/os_cpu/windows_x86/unwind_windows_x86.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2004, 2022, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2004, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -25,8 +25,6 @@ #ifndef OS_CPU_WINDOWS_X86_UNWIND_WINDOWS_X86_HPP #define OS_CPU_WINDOWS_X86_UNWIND_WINDOWS_X86_HPP - -#ifdef AMD64 typedef unsigned char UBYTE; // This structure is used to define an UNWIND_INFO that @@ -54,6 +52,4 @@ typedef struct _RUNTIME_FUNCTION { } RUNTIME_FUNCTION, *PRUNTIME_FUNCTION; */ -#endif // AMD64 - #endif // OS_CPU_WINDOWS_X86_UNWIND_WINDOWS_X86_HPP diff --git a/src/hotspot/share/adlc/adlc.hpp b/src/hotspot/share/adlc/adlc.hpp index 8812066bafa..3e82fe0e210 100644 --- a/src/hotspot/share/adlc/adlc.hpp +++ b/src/hotspot/share/adlc/adlc.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1998, 2022, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1998, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -50,7 +50,7 @@ using namespace std; #ifdef _WIN64 typedef __int64 intptr_t; #else -typedef int intptr_t; +#error "Unsupported platform" #endif #define _INTPTR_T_DEFINED #endif @@ -59,7 +59,7 @@ typedef int intptr_t; #ifdef _WIN64 typedef unsigned __int64 uintptr_t; #else -typedef unsigned int uintptr_t; +#error "Unsupported platform" #endif #define _UINTPTR_T_DEFINED #endif diff --git a/src/hotspot/share/adlc/main.cpp b/src/hotspot/share/adlc/main.cpp index fd270b09443..f15b6e8813f 100644 --- a/src/hotspot/share/adlc/main.cpp +++ b/src/hotspot/share/adlc/main.cpp @@ -491,8 +491,6 @@ int get_legal_text(FileBuff &fbuf, char **legal_text) return (int) (legal_end - legal_start); } -#if !defined(_WIN32) || defined(_WIN64) void *operator new( size_t size, int, const char *, int ) throw() { return ::operator new( size ); } -#endif diff --git a/src/hotspot/share/c1/c1_Instruction.hpp b/src/hotspot/share/c1/c1_Instruction.hpp index e577e7fd90b..e950afc981d 100644 --- a/src/hotspot/share/c1/c1_Instruction.hpp +++ b/src/hotspot/share/c1/c1_Instruction.hpp @@ -346,11 +346,8 @@ class Instruction: public CompilationResourceObj { NeedsNullCheckFlag = 0, CanTrapFlag, DirectCompareFlag, - IsEliminatedFlag, IsSafepointFlag, IsStaticFlag, - NeedsStoreCheckFlag, - NeedsWriteBarrierFlag, PreservesStateFlag, TargetIsFinalFlag, TargetIsLoadedFlag, @@ -361,7 +358,6 @@ class Instruction: public CompilationResourceObj { ProfileMDOFlag, IsLinkedInBlockFlag, NeedsRangeCheckFlag, - InWorkListFlag, DeoptimizeOnException, KillsMemoryFlag, OmitChecksFlag, @@ -840,14 +836,12 @@ LEAF(StoreField, AccessField) : AccessField(obj, offset, field, is_static, state_before, needs_patching) , _value(value) { - set_flag(NeedsWriteBarrierFlag, as_ValueType(field_type())->is_object()); ASSERT_VALUES pin(); } // accessors Value value() const { return _value; } - bool needs_write_barrier() const { return check_flag(NeedsWriteBarrierFlag); } // generic virtual void input_values_do(ValueVisitor* f) { AccessField::input_values_do(f); f->visit(&_value); } @@ -974,16 +968,12 @@ LEAF(StoreIndexed, AccessIndexed) : AccessIndexed(array, index, length, elt_type, state_before, mismatched) , _value(value), _profiled_method(nullptr), _profiled_bci(0), _check_boolean(check_boolean) { - set_flag(NeedsWriteBarrierFlag, (as_ValueType(elt_type)->is_object())); - set_flag(NeedsStoreCheckFlag, (as_ValueType(elt_type)->is_object())); ASSERT_VALUES pin(); } // accessors Value value() const { return _value; } - bool needs_write_barrier() const { return check_flag(NeedsWriteBarrierFlag); } - bool needs_store_check() const { return check_flag(NeedsStoreCheckFlag); } bool check_boolean() const { return _check_boolean; } // Helpers for MethodData* profiling void set_should_profile(bool value) { set_flag(ProfileMDOFlag, value); } diff --git a/src/hotspot/share/c1/c1_MacroAssembler.hpp b/src/hotspot/share/c1/c1_MacroAssembler.hpp index 1e193ce0869..0fe0d0ff285 100644 --- a/src/hotspot/share/c1/c1_MacroAssembler.hpp +++ b/src/hotspot/share/c1/c1_MacroAssembler.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2000, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -89,7 +89,7 @@ class StubAssembler: public C1_MacroAssembler { int call_RT(Register oop_result1, Register metadata_result, address entry, Register arg1, Register arg2, Register arg3); void prologue(const char* name, bool must_gc_arguments); - void epilogue(); + void epilogue(bool use_pop = false); }; #endif // SHARE_C1_C1_MACROASSEMBLER_HPP diff --git a/src/hotspot/share/c1/c1_Runtime1.hpp b/src/hotspot/share/c1/c1_Runtime1.hpp index 330c4067504..6aed71fdfe5 100644 --- a/src/hotspot/share/c1/c1_Runtime1.hpp +++ b/src/hotspot/share/c1/c1_Runtime1.hpp @@ -135,6 +135,9 @@ class Runtime1: public AllStatic { static void initialize(BufferBlob* blob); static void initialize_pd(); + // return offset in words + static uint runtime_blob_current_thread_offset(frame f); + // stubs static CodeBlob* blob_for (C1StubId id); static address entry_for(C1StubId id) { return blob_for(id)->code_begin(); } diff --git a/src/hotspot/share/cds/archiveBuilder.cpp b/src/hotspot/share/cds/archiveBuilder.cpp index 3c67216d4c5..a181257a519 100644 --- a/src/hotspot/share/cds/archiveBuilder.cpp +++ b/src/hotspot/share/cds/archiveBuilder.cpp @@ -227,8 +227,10 @@ bool ArchiveBuilder::gather_klass_and_symbol(MetaspaceClosure::Ref* ref, bool re if (!is_excluded(klass)) { _klasses->append(klass); } - // See RunTimeClassInfo::get_for() - _estimated_metaspaceobj_bytes += align_up(BytesPerWord, SharedSpaceObjectAlignment); + // See RunTimeClassInfo::get_for(): make sure we have enough space for both maximum + // Klass alignment as well as the RuntimeInfo* pointer we will embed in front of a Klass. + _estimated_metaspaceobj_bytes += align_up(BytesPerWord, CompressedKlassPointers::klass_alignment_in_bytes()) + + align_up(sizeof(void*), SharedSpaceObjectAlignment); } else if (ref->msotype() == MetaspaceObj::SymbolType) { // Make sure the symbol won't be GC'ed while we are dumping the archive. Symbol* sym = (Symbol*)ref->obj(); @@ -661,7 +663,7 @@ void ArchiveBuilder::make_shallow_copy(DumpRegion *dump_region, SourceObjInfo* s oldtop = dump_region->top(); if (src_info->msotype() == MetaspaceObj::ClassType) { - // Save a pointer immediate in front of an InstanceKlass, so + // Allocate space for a pointer directly in front of the future InstanceKlass, so // we can do a quick lookup from InstanceKlass* -> RunTimeClassInfo* // without building another hashtable. See RunTimeClassInfo::get_for() // in systemDictionaryShared.cpp. @@ -670,8 +672,19 @@ void ArchiveBuilder::make_shallow_copy(DumpRegion *dump_region, SourceObjInfo* s SystemDictionaryShared::validate_before_archiving(InstanceKlass::cast(klass)); dump_region->allocate(sizeof(address)); } + // Allocate space for the future InstanceKlass with proper alignment + const size_t alignment = +#ifdef _LP64 + UseCompressedClassPointers ? + nth_bit(ArchiveBuilder::precomputed_narrow_klass_shift()) : + SharedSpaceObjectAlignment; +#else + SharedSpaceObjectAlignment; +#endif + dest = dump_region->allocate(bytes, alignment); + } else { + dest = dump_region->allocate(bytes); } - dest = dump_region->allocate(bytes); newtop = dump_region->top(); memcpy(dest, src, bytes); @@ -702,6 +715,8 @@ void ArchiveBuilder::make_shallow_copy(DumpRegion *dump_region, SourceObjInfo* s src_info->set_buffered_addr((address)dest); _alloc_stats.record(src_info->msotype(), int(newtop - oldtop), src_info->read_only()); + + DEBUG_ONLY(_alloc_stats.verify((int)dump_region->used(), src_info->read_only())); } // This is used by code that hand-assembles data structures, such as the LambdaProxyClassKey, that are @@ -780,6 +795,15 @@ void ArchiveBuilder::make_klasses_shareable() { const char* generated = ""; Klass* k = get_buffered_addr(klasses()->at(i)); k->remove_java_mirror(); +#ifdef _LP64 + if (UseCompactObjectHeaders) { + Klass* requested_k = to_requested(k); + address narrow_klass_base = _requested_static_archive_bottom; // runtime encoding base == runtime mapping start + const int narrow_klass_shift = precomputed_narrow_klass_shift(); + narrowKlass nk = CompressedKlassPointers::encode_not_null_without_asserts(requested_k, narrow_klass_base, narrow_klass_shift); + k->set_prototype_header(markWord::prototype().set_narrow_klass(nk)); + } +#endif //_LP64 if (k->is_objArray_klass()) { // InstanceKlass and TypeArrayKlass will in turn call remove_unshareable_info // on their array classes. @@ -872,14 +896,27 @@ uintx ArchiveBuilder::any_to_offset(address p) const { return buffer_to_offset(p); } +address ArchiveBuilder::offset_to_buffered_address(u4 offset) const { + address requested_addr = _requested_static_archive_bottom + offset; + address buffered_addr = requested_addr - _buffer_to_requested_delta; + assert(is_in_buffer_space(buffered_addr), "bad offset"); + return buffered_addr; +} + #if INCLUDE_CDS_JAVA_HEAP narrowKlass ArchiveBuilder::get_requested_narrow_klass(Klass* k) { assert(CDSConfig::is_dumping_heap(), "sanity"); k = get_buffered_klass(k); Klass* requested_k = to_requested(k); + const int narrow_klass_shift = ArchiveBuilder::precomputed_narrow_klass_shift(); +#ifdef ASSERT + const size_t klass_alignment = MAX2(SharedSpaceObjectAlignment, (size_t)nth_bit(narrow_klass_shift)); + assert(is_aligned(k, klass_alignment), "Klass " PTR_FORMAT " misaligned.", p2i(k)); +#endif address narrow_klass_base = _requested_static_archive_bottom; // runtime encoding base == runtime mapping start - const int narrow_klass_shift = ArchiveHeapWriter::precomputed_narrow_klass_shift; - return CompressedKlassPointers::encode_not_null(requested_k, narrow_klass_base, narrow_klass_shift); + // Note: use the "raw" version of encode that takes explicit narrow klass base and shift. Don't use any + // of the variants that do sanity checks, nor any of those that use the current - dump - JVM's encoding setting. + return CompressedKlassPointers::encode_not_null_without_asserts(requested_k, narrow_klass_base, narrow_klass_shift); } #endif // INCLUDE_CDS_JAVA_HEAP @@ -959,6 +996,20 @@ class RelocateBufferToRequested : public BitMapClosure { } }; +#ifdef _LP64 +int ArchiveBuilder::precomputed_narrow_klass_shift() { + // Legacy Mode: + // We use 32 bits for narrowKlass, which should cover the full 4G Klass range. Shift can be 0. + // CompactObjectHeader Mode: + // narrowKlass is much smaller, and we use the highest possible shift value to later get the maximum + // Klass encoding range. + // + // Note that all of this may change in the future, if we decide to correct the pre-calculated + // narrow Klass IDs at archive load time. + assert(UseCompressedClassPointers, "Only needed for compressed class pointers"); + return UseCompactObjectHeaders ? CompressedKlassPointers::max_shift() : 0; +} +#endif // _LP64 void ArchiveBuilder::relocate_to_requested() { ro_region()->pack(); diff --git a/src/hotspot/share/cds/archiveBuilder.hpp b/src/hotspot/share/cds/archiveBuilder.hpp index c17090ee53d..f306e4676b3 100644 --- a/src/hotspot/share/cds/archiveBuilder.hpp +++ b/src/hotspot/share/cds/archiveBuilder.hpp @@ -27,6 +27,7 @@ #include "cds/archiveUtils.hpp" #include "cds/dumpAllocStats.hpp" +#include "memory/metaspace.hpp" #include "memory/metaspaceClosure.hpp" #include "oops/array.hpp" #include "oops/klass.hpp" @@ -43,9 +44,9 @@ class Klass; class MemRegion; class Symbol; -// Metaspace::allocate() requires that all blocks must be aligned with KlassAlignmentInBytes. -// We enforce the same alignment rule in blocks allocated from the shared space. -const int SharedSpaceObjectAlignment = KlassAlignmentInBytes; +// The minimum alignment for non-Klass objects inside the CDS archive. Klass objects need +// to follow CompressedKlassPointers::klass_alignment_in_bytes(). +constexpr size_t SharedSpaceObjectAlignment = Metaspace::min_allocation_alignment_bytes; // Overview of CDS archive creation (for both static and dynamic dump): // @@ -321,7 +322,7 @@ class ArchiveBuilder : public StackObj { } public: - static const uintx MAX_SHARED_DELTA = 0x7FFFFFFF; + static const uintx MAX_SHARED_DELTA = ArchiveUtils::MAX_SHARED_DELTA;; // The address p points to an object inside the output buffer. When the archive is mapped // at the requested address, what's the offset of this object from _requested_static_archive_bottom? @@ -331,6 +332,9 @@ class ArchiveBuilder : public StackObj { // inside the output buffer, or (b), an object in the currently mapped static archive. uintx any_to_offset(address p) const; + // The reverse of buffer_to_offset() + address offset_to_buffered_address(u4 offset) const; + template u4 buffer_to_offset_u4(T p) const { uintx offset = buffer_to_offset((address)p); @@ -343,6 +347,11 @@ class ArchiveBuilder : public StackObj { return to_offset_u4(offset); } + template + T offset_to_buffered(u4 offset) const { + return (T)offset_to_buffered_address(offset); + } + public: ArchiveBuilder(); ~ArchiveBuilder(); @@ -460,6 +469,29 @@ class ArchiveBuilder : public StackObj { void print_stats(); void report_out_of_space(const char* name, size_t needed_bytes); + +#ifdef _LP64 + // The CDS archive contains pre-computed narrow Klass IDs. It carries them in the headers of + // archived heap objects. With +UseCompactObjectHeaders, it also carries them in prototypes + // in Klass. + // When generating the archive, these narrow Klass IDs are computed using the following scheme: + // 1) The future encoding base is assumed to point to the first address of the generated mapping. + // That means that at runtime, the narrow Klass encoding must be set up with base pointing to + // the start address of the mapped CDS metadata archive (wherever that may be). This precludes + // zero-based encoding. + // 2) The shift must be large enough to result in an encoding range that covers the future assumed + // runtime Klass range. That future Klass range will contain both the CDS metadata archive and + // the future runtime class space. Since we do not know the size of the future class space, we + // need to chose an encoding base/shift combination that will result in a "large enough" size. + // The details depend on whether we use compact object headers or legacy object headers. + // In Legacy Mode, a narrow Klass ID is 32 bit. This gives us an encoding range size of 4G even + // with shift = 0, which is all we need. Therefore, we use a shift=0 for pre-calculating the + // narrow Klass IDs. + // TinyClassPointer Mode: + // We use the highest possible shift value to maximize the encoding range size. + static int precomputed_narrow_klass_shift(); +#endif // _LP64 + }; #endif // SHARE_CDS_ARCHIVEBUILDER_HPP diff --git a/src/hotspot/share/cds/archiveHeapWriter.cpp b/src/hotspot/share/cds/archiveHeapWriter.cpp index a55325978de..636b368117d 100644 --- a/src/hotspot/share/cds/archiveHeapWriter.cpp +++ b/src/hotspot/share/cds/archiveHeapWriter.cpp @@ -186,8 +186,12 @@ objArrayOop ArchiveHeapWriter::allocate_root_segment(size_t offset, int element_ memset(mem, 0, objArrayOopDesc::object_size(element_count)); // The initialization code is copied from MemAllocator::finish and ObjArrayAllocator::initialize. - oopDesc::set_mark(mem, markWord::prototype()); - oopDesc::release_set_klass(mem, Universe::objectArrayKlass()); + if (UseCompactObjectHeaders) { + oopDesc::release_set_mark(mem, Universe::objectArrayKlass()->prototype_header()); + } else { + oopDesc::set_mark(mem, markWord::prototype()); + oopDesc::release_set_klass(mem, Universe::objectArrayKlass()); + } arrayOopDesc::set_length(mem, element_count); return objArrayOop(cast_to_oop(mem)); } @@ -350,9 +354,13 @@ HeapWord* ArchiveHeapWriter::init_filler_array_at_buffer_top(int array_length, s Klass* oak = Universe::objectArrayKlass(); // already relocated to point to archived klass HeapWord* mem = offset_to_buffered_address(_buffer_used); memset(mem, 0, fill_bytes); - oopDesc::set_mark(mem, markWord::prototype()); narrowKlass nk = ArchiveBuilder::current()->get_requested_narrow_klass(oak); - cast_to_oop(mem)->set_narrow_klass(nk); + if (UseCompactObjectHeaders) { + oopDesc::release_set_mark(mem, markWord::prototype().set_narrow_klass(nk)); + } else { + oopDesc::set_mark(mem, markWord::prototype()); + cast_to_oop(mem)->set_narrow_klass(nk); + } arrayOopDesc::set_length(mem, array_length); return mem; } @@ -556,7 +564,11 @@ void ArchiveHeapWriter::update_header_for_requested_obj(oop requested_obj, oop s address buffered_addr = requested_addr_to_buffered_addr(cast_from_oop
(requested_obj)); oop fake_oop = cast_to_oop(buffered_addr); - fake_oop->set_narrow_klass(nk); + if (UseCompactObjectHeaders) { + fake_oop->set_mark(markWord::prototype().set_narrow_klass(nk)); + } else { + fake_oop->set_narrow_klass(nk); + } if (src_obj == nullptr) { return; @@ -565,7 +577,11 @@ void ArchiveHeapWriter::update_header_for_requested_obj(oop requested_obj, oop s // in the shared heap. if (!src_obj->fast_no_hash_check()) { intptr_t src_hash = src_obj->identity_hash(); - fake_oop->set_mark(markWord::prototype().copy_set_hash(src_hash)); + if (UseCompactObjectHeaders) { + fake_oop->set_mark(markWord::prototype().set_narrow_klass(nk).copy_set_hash(src_hash)); + } else { + fake_oop->set_mark(markWord::prototype().copy_set_hash(src_hash)); + } assert(fake_oop->mark().is_unlocked(), "sanity"); DEBUG_ONLY(intptr_t archived_hash = fake_oop->identity_hash()); diff --git a/src/hotspot/share/cds/archiveHeapWriter.hpp b/src/hotspot/share/cds/archiveHeapWriter.hpp index 29ea50ba5fe..70c1207bb91 100644 --- a/src/hotspot/share/cds/archiveHeapWriter.hpp +++ b/src/hotspot/share/cds/archiveHeapWriter.hpp @@ -240,17 +240,6 @@ class ArchiveHeapWriter : AllStatic { static oop buffered_addr_to_source_obj(address buffered_addr); static address buffered_addr_to_requested_addr(address buffered_addr); - // Archived heap object headers carry pre-computed narrow Klass ids calculated with the - // following scheme: - // 1) the encoding base must be the mapping start address. - // 2) shift must be large enough to result in an encoding range that covers the runtime Klass range. - // That Klass range is defined by CDS archive size and runtime class space size. Luckily, the maximum - // size can be predicted: archive size is assumed to be <1G, class space size capped at 3G, and at - // runtime we put both regions adjacent to each other. Therefore, runtime Klass range size < 4G. - // Since nKlass itself is 32 bit, our encoding range len is 4G, and since we set the base directly - // at mapping start, these 4G are enough. Therefore, we don't need to shift at all (shift=0). - static constexpr int precomputed_narrow_klass_shift = 0; - }; #endif // INCLUDE_CDS_JAVA_HEAP #endif // SHARE_CDS_ARCHIVEHEAPWRITER_HPP diff --git a/src/hotspot/share/cds/archiveUtils.cpp b/src/hotspot/share/cds/archiveUtils.cpp index 4622a27cbec..c7282f7d97c 100644 --- a/src/hotspot/share/cds/archiveUtils.cpp +++ b/src/hotspot/share/cds/archiveUtils.cpp @@ -241,9 +241,10 @@ void DumpRegion::commit_to(char* newtop) { which, commit, _vs->actual_committed_size(), _vs->high()); } - -char* DumpRegion::allocate(size_t num_bytes) { - char* p = (char*)align_up(_top, (size_t)SharedSpaceObjectAlignment); +char* DumpRegion::allocate(size_t num_bytes, size_t alignment) { + // Always align to at least minimum alignment + alignment = MAX2(SharedSpaceObjectAlignment, alignment); + char* p = (char*)align_up(_top, alignment); char* newtop = p + align_up(num_bytes, (size_t)SharedSpaceObjectAlignment); expand_top_to(newtop); memset(p, 0, newtop - p); @@ -320,9 +321,8 @@ void WriteClosure::do_ptr(void** p) { void ReadClosure::do_ptr(void** p) { assert(*p == nullptr, "initializing previous initialized pointer."); intptr_t obj = nextPtr(); - assert((intptr_t)obj >= 0 || (intptr_t)obj < -100, - "hit tag while initializing ptrs."); - *p = (void*)obj != nullptr ? (void*)(SharedBaseAddress + obj) : (void*)obj; + assert(obj >= 0, "sanity."); + *p = (obj != 0) ? (void*)(_base_address + obj) : (void*)obj; } void ReadClosure::do_u4(u4* p) { @@ -344,7 +344,7 @@ void ReadClosure::do_tag(int tag) { int old_tag; old_tag = (int)(intptr_t)nextPtr(); // do_int(&old_tag); - assert(tag == old_tag, "old tag doesn't match"); + assert(tag == old_tag, "tag doesn't match (%d, expected %d)", old_tag, tag); FileMapInfo::assert_mark(tag == old_tag); } diff --git a/src/hotspot/share/cds/archiveUtils.hpp b/src/hotspot/share/cds/archiveUtils.hpp index 5a78bc26ee6..7fb8e538084 100644 --- a/src/hotspot/share/cds/archiveUtils.hpp +++ b/src/hotspot/share/cds/archiveUtils.hpp @@ -25,8 +25,10 @@ #ifndef SHARE_CDS_ARCHIVEUTILS_HPP #define SHARE_CDS_ARCHIVEUTILS_HPP +#include "cds/cds_globals.hpp" #include "cds/serializeClosure.hpp" #include "logging/log.hpp" +#include "memory/metaspace.hpp" #include "memory/virtualspace.hpp" #include "utilities/bitMap.hpp" #include "utilities/exceptions.hpp" @@ -156,10 +158,11 @@ class DumpRegion { public: DumpRegion(const char* name, uintx max_delta = 0) : _name(name), _base(nullptr), _top(nullptr), _end(nullptr), - _max_delta(max_delta), _is_packed(false) {} + _max_delta(max_delta), _is_packed(false), + _rs(NULL), _vs(NULL) {} char* expand_top_to(char* newtop); - char* allocate(size_t num_bytes); + char* allocate(size_t num_bytes, size_t alignment = 0); void append_intptr_t(intptr_t n, bool need_to_mark = false) NOT_CDS_RETURN; @@ -228,13 +231,14 @@ class WriteClosure : public SerializeClosure { class ReadClosure : public SerializeClosure { private: intptr_t** _ptr_array; - + intptr_t _base_address; inline intptr_t nextPtr() { return *(*_ptr_array)++; } public: - ReadClosure(intptr_t** ptr_array) { _ptr_array = ptr_array; } + ReadClosure(intptr_t** ptr_array, intptr_t base_address) : + _ptr_array(ptr_array), _base_address(base_address) {} void do_ptr(void** p); void do_u4(u4* p); @@ -247,7 +251,27 @@ class ReadClosure : public SerializeClosure { class ArchiveUtils { public: + static const uintx MAX_SHARED_DELTA = 0x7FFFFFFF; static void log_to_classlist(BootstrapInfo* bootstrap_specifier, TRAPS) NOT_CDS_RETURN; + + // offset must represent an object of type T in the mapped shared space. Return + // a direct pointer to this object. + template T static from_offset(u4 offset) { + T p = (T)(SharedBaseAddress + offset); + assert(Metaspace::is_in_shared_metaspace(p), "must be"); + return p; + } + + // p must be an archived object. Get its offset from SharedBaseAddress + template static u4 to_offset(T p) { + uintx pn = (uintx)p; + uintx base = (uintx)SharedBaseAddress; + assert(Metaspace::is_in_shared_metaspace(p), "must be"); + assert(pn > base, "sanity"); // No valid object is stored at 0 offset from SharedBaseAddress + uintx offset = pn - base; + assert(offset <= MAX_SHARED_DELTA, "range check"); + return static_cast(offset); + } }; class HeapRootSegments { diff --git a/src/hotspot/share/cds/cdsConfig.cpp b/src/hotspot/share/cds/cdsConfig.cpp index 9e8a46e105e..691a1983732 100644 --- a/src/hotspot/share/cds/cdsConfig.cpp +++ b/src/hotspot/share/cds/cdsConfig.cpp @@ -82,13 +82,20 @@ char* CDSConfig::default_archive_path() { os::jvm_path(jvm_path, sizeof(jvm_path)); char *end = strrchr(jvm_path, *os::file_separator()); if (end != nullptr) *end = '\0'; - size_t jvm_path_len = strlen(jvm_path); - size_t file_sep_len = strlen(os::file_separator()); - const size_t len = jvm_path_len + file_sep_len + 20; - _default_archive_path = NEW_C_HEAP_ARRAY(char, len, mtArguments); - jio_snprintf(_default_archive_path, len, - LP64_ONLY(!UseCompressedOops ? "%s%sclasses_nocoops.jsa":) "%s%sclasses.jsa", - jvm_path, os::file_separator()); + stringStream tmp; + tmp.print("%s%sclasses", jvm_path, os::file_separator()); +#ifdef _LP64 + if (!UseCompressedOops) { + tmp.print_raw("_nocoops"); + } + if (UseCompactObjectHeaders) { + // Note that generation of xxx_coh.jsa variants require + // --enable-cds-archive-coh at build time + tmp.print_raw("_coh"); + } +#endif + tmp.print_raw(".jsa"); + _default_archive_path = os::strdup(tmp.base()); } return _default_archive_path; } diff --git a/src/hotspot/share/cds/dumpAllocStats.cpp b/src/hotspot/share/cds/dumpAllocStats.cpp index a1dac41e322..25351f78d84 100644 --- a/src/hotspot/share/cds/dumpAllocStats.cpp +++ b/src/hotspot/share/cds/dumpAllocStats.cpp @@ -115,3 +115,15 @@ void DumpAllocStats::print_stats(int ro_all, int rw_all) { percent_of(_num_method_cp_entries_archived, _num_method_cp_entries), _num_method_cp_entries_reverted); } + +#ifdef ASSERT +void DumpAllocStats::verify(int expected_byte_size, bool read_only) const { + int bytes = 0; + const int what = (int)(read_only ? RO : RW); + for (int type = 0; type < int(_number_of_types); type ++) { + bytes += _bytes[what][type]; + } + assert(bytes == expected_byte_size, "counter mismatch (%s: %d vs %d)", + (read_only ? "RO" : "RW"), bytes, expected_byte_size); +} +#endif // ASSERT diff --git a/src/hotspot/share/cds/dumpAllocStats.hpp b/src/hotspot/share/cds/dumpAllocStats.hpp index 0332be73120..2ff81c52392 100644 --- a/src/hotspot/share/cds/dumpAllocStats.hpp +++ b/src/hotspot/share/cds/dumpAllocStats.hpp @@ -135,6 +135,9 @@ class DumpAllocStats : public StackObj { } void print_stats(int ro_all, int rw_all); + + DEBUG_ONLY(void verify(int expected_byte_size, bool read_only) const); + }; #endif // SHARE_CDS_DUMPALLOCSTATS_HPP diff --git a/src/hotspot/share/cds/filemap.cpp b/src/hotspot/share/cds/filemap.cpp index 223791cdd36..5e88b3c47e4 100644 --- a/src/hotspot/share/cds/filemap.cpp +++ b/src/hotspot/share/cds/filemap.cpp @@ -55,6 +55,7 @@ #include "nmt/memTracker.hpp" #include "oops/compressedOops.hpp" #include "oops/compressedOops.inline.hpp" +#include "oops/compressedKlass.hpp" #include "oops/objArrayOop.hpp" #include "oops/oop.inline.hpp" #include "prims/jvmtiExport.hpp" @@ -204,6 +205,7 @@ void FileMapHeader::populate(FileMapInfo *info, size_t core_region_alignment, _core_region_alignment = core_region_alignment; _obj_alignment = ObjectAlignmentInBytes; _compact_strings = CompactStrings; + _compact_headers = UseCompactObjectHeaders; if (CDSConfig::is_dumping_heap()) { _narrow_oop_mode = CompressedOops::mode(); _narrow_oop_base = CompressedOops::base(); @@ -211,6 +213,14 @@ void FileMapHeader::populate(FileMapInfo *info, size_t core_region_alignment, } _compressed_oops = UseCompressedOops; _compressed_class_ptrs = UseCompressedClassPointers; + if (UseCompressedClassPointers) { +#ifdef _LP64 + _narrow_klass_pointer_bits = CompressedKlassPointers::narrow_klass_pointer_bits(); + _narrow_klass_shift = ArchiveBuilder::precomputed_narrow_klass_shift(); +#endif + } else { + _narrow_klass_pointer_bits = _narrow_klass_shift = -1; + } _max_heap_size = MaxHeapSize; _use_optimized_module_handling = CDSConfig::is_using_optimized_module_handling(); _has_full_module_graph = CDSConfig::is_dumping_full_module_graph(); @@ -269,11 +279,15 @@ void FileMapHeader::print(outputStream* st) { st->print_cr("- narrow_oop_base: " INTPTR_FORMAT, p2i(_narrow_oop_base)); st->print_cr("- narrow_oop_shift %d", _narrow_oop_shift); st->print_cr("- compact_strings: %d", _compact_strings); + st->print_cr("- compact_headers: %d", _compact_headers); st->print_cr("- max_heap_size: " UINTX_FORMAT, _max_heap_size); st->print_cr("- narrow_oop_mode: %d", _narrow_oop_mode); st->print_cr("- compressed_oops: %d", _compressed_oops); st->print_cr("- compressed_class_ptrs: %d", _compressed_class_ptrs); + st->print_cr("- narrow_klass_pointer_bits: %d", _narrow_klass_pointer_bits); + st->print_cr("- narrow_klass_shift: %d", _narrow_klass_shift); st->print_cr("- cloned_vtables_offset: " SIZE_FORMAT_X, _cloned_vtables_offset); + st->print_cr("- early_serialized_data_offset: " SIZE_FORMAT_X, _early_serialized_data_offset); st->print_cr("- serialized_data_offset: " SIZE_FORMAT_X, _serialized_data_offset); st->print_cr("- jvm_ident: %s", _jvm_ident); st->print_cr("- shared_path_table_offset: " SIZE_FORMAT_X, _shared_path_table_offset); @@ -2082,22 +2096,23 @@ bool FileMapInfo::can_use_heap_region() { } // We pre-compute narrow Klass IDs with the runtime mapping start intended to be the base, and a shift of - // ArchiveHeapWriter::precomputed_narrow_klass_shift. We enforce this encoding at runtime (see + // ArchiveBuilder::precomputed_narrow_klass_shift. We enforce this encoding at runtime (see // CompressedKlassPointers::initialize_for_given_encoding()). Therefore, the following assertions must // hold: address archive_narrow_klass_base = (address)header()->mapped_base_address(); - const int archive_narrow_klass_shift = ArchiveHeapWriter::precomputed_narrow_klass_shift; + const int archive_narrow_klass_pointer_bits = header()->narrow_klass_pointer_bits(); + const int archive_narrow_klass_shift = header()->narrow_klass_shift(); log_info(cds)("CDS archive was created with max heap size = " SIZE_FORMAT "M, and the following configuration:", max_heap_size()/M); - log_info(cds)(" narrow_klass_base at mapping start address, narrow_klass_shift = %d", - archive_narrow_klass_shift); + log_info(cds)(" narrow_klass_base at mapping start address, narrow_klass_pointer_bits = %d, narrow_klass_shift = %d", + archive_narrow_klass_pointer_bits, archive_narrow_klass_shift); log_info(cds)(" narrow_oop_mode = %d, narrow_oop_base = " PTR_FORMAT ", narrow_oop_shift = %d", narrow_oop_mode(), p2i(narrow_oop_base()), narrow_oop_shift()); log_info(cds)("The current max heap size = " SIZE_FORMAT "M, G1HeapRegion::GrainBytes = " SIZE_FORMAT, MaxHeapSize/M, G1HeapRegion::GrainBytes); - log_info(cds)(" narrow_klass_base = " PTR_FORMAT ", narrow_klass_shift = %d", - p2i(CompressedKlassPointers::base()), CompressedKlassPointers::shift()); + log_info(cds)(" narrow_klass_base = " PTR_FORMAT ", arrow_klass_pointer_bits = %d, narrow_klass_shift = %d", + p2i(CompressedKlassPointers::base()), CompressedKlassPointers::narrow_klass_pointer_bits(), CompressedKlassPointers::shift()); log_info(cds)(" narrow_oop_mode = %d, narrow_oop_base = " PTR_FORMAT ", narrow_oop_shift = %d", CompressedOops::mode(), p2i(CompressedOops::base()), CompressedOops::shift()); log_info(cds)(" heap range = [" PTR_FORMAT " - " PTR_FORMAT "]", @@ -2106,10 +2121,35 @@ bool FileMapInfo::can_use_heap_region() { UseCompressedOops ? p2i(CompressedOops::end()) : UseG1GC ? p2i((address)G1CollectedHeap::heap()->reserved().end()) : 0L); - assert(archive_narrow_klass_base == CompressedKlassPointers::base(), "Unexpected encoding base encountered " - "(" PTR_FORMAT ", expected " PTR_FORMAT ")", p2i(CompressedKlassPointers::base()), p2i(archive_narrow_klass_base)); - assert(archive_narrow_klass_shift == CompressedKlassPointers::shift(), "Unexpected encoding shift encountered " - "(%d, expected %d)", CompressedKlassPointers::shift(), archive_narrow_klass_shift); + int err = 0; + if ( archive_narrow_klass_base != CompressedKlassPointers::base() || + (err = 1, archive_narrow_klass_pointer_bits != CompressedKlassPointers::narrow_klass_pointer_bits()) || + (err = 2, archive_narrow_klass_shift != CompressedKlassPointers::shift()) ) { + stringStream ss; + switch (err) { + case 0: + ss.print("Unexpected encoding base encountered (" PTR_FORMAT ", expected " PTR_FORMAT ")", + p2i(CompressedKlassPointers::base()), p2i(archive_narrow_klass_base)); + break; + case 1: + ss.print("Unexpected narrow Klass bit length encountered (%d, expected %d)", + CompressedKlassPointers::narrow_klass_pointer_bits(), archive_narrow_klass_pointer_bits); + break; + case 2: + ss.print("Unexpected narrow Klass shift encountered (%d, expected %d)", + CompressedKlassPointers::shift(), archive_narrow_klass_shift); + break; + default: + ShouldNotReachHere(); + }; + LogTarget(Info, cds) lt; + if (lt.is_enabled()) { + LogStream ls(lt); + ls.print_raw(ss.base()); + header()->print(&ls); + } + assert(false, "%s", ss.base()); + } return true; } @@ -2481,14 +2521,22 @@ bool FileMapHeader::validate() { "for testing purposes only and should not be used in a production environment"); } - log_info(cds)("Archive was created with UseCompressedOops = %d, UseCompressedClassPointers = %d", - compressed_oops(), compressed_class_pointers()); + log_info(cds)("Archive was created with UseCompressedOops = %d, UseCompressedClassPointers = %d, UseCompactObjectHeaders = %d", + compressed_oops(), compressed_class_pointers(), compact_headers()); if (compressed_oops() != UseCompressedOops || compressed_class_pointers() != UseCompressedClassPointers) { - log_info(cds)("Unable to use shared archive.\nThe saved state of UseCompressedOops and UseCompressedClassPointers is " + log_warning(cds)("Unable to use shared archive.\nThe saved state of UseCompressedOops and UseCompressedClassPointers is " "different from runtime, CDS will be disabled."); return false; } + if (compact_headers() != UseCompactObjectHeaders) { + log_warning(cds)("Unable to use shared archive.\nThe shared archive file's UseCompactObjectHeaders setting (%s)" + " does not equal the current UseCompactObjectHeaders setting (%s).", + _compact_headers ? "enabled" : "disabled", + UseCompactObjectHeaders ? "enabled" : "disabled"); + return false; + } + if (!_use_optimized_module_handling) { CDSConfig::stop_using_optimized_module_handling(); log_info(cds)("optimized module handling: disabled because archive was created without optimized module handling"); diff --git a/src/hotspot/share/cds/filemap.hpp b/src/hotspot/share/cds/filemap.hpp index f7c2e9c19de..6a15ff03c3a 100644 --- a/src/hotspot/share/cds/filemap.hpp +++ b/src/hotspot/share/cds/filemap.hpp @@ -188,11 +188,15 @@ class FileMapHeader: private CDSFileMapHeaderBase { address _narrow_oop_base; // compressed oop encoding base int _narrow_oop_shift; // compressed oop encoding shift bool _compact_strings; // value of CompactStrings + bool _compact_headers; // value of UseCompactObjectHeaders uintx _max_heap_size; // java max heap size during dumping CompressedOops::Mode _narrow_oop_mode; // compressed oop encoding mode bool _compressed_oops; // save the flag UseCompressedOops bool _compressed_class_ptrs; // save the flag UseCompressedClassPointers + int _narrow_klass_pointer_bits; // save number of bits in narrowKlass + int _narrow_klass_shift; // save shift width used to pre-compute narrowKlass IDs in archived heap objects size_t _cloned_vtables_offset; // The address of the first cloned vtable + size_t _early_serialized_data_offset; // Data accessed using {ReadClosure,WriteClosure}::serialize() size_t _serialized_data_offset; // Data accessed using {ReadClosure,WriteClosure}::serialize() bool _has_non_jar_in_classpath; // non-jar file entry exists in classpath unsigned int _common_app_classpath_prefix_size; // size of the common prefix of app class paths @@ -258,9 +262,11 @@ class FileMapHeader: private CDSFileMapHeaderBase { address narrow_oop_base() const { return _narrow_oop_base; } int narrow_oop_shift() const { return _narrow_oop_shift; } bool compact_strings() const { return _compact_strings; } + bool compact_headers() const { return _compact_headers; } uintx max_heap_size() const { return _max_heap_size; } CompressedOops::Mode narrow_oop_mode() const { return _narrow_oop_mode; } char* cloned_vtables() const { return from_mapped_offset(_cloned_vtables_offset); } + char* early_serialized_data() const { return from_mapped_offset(_early_serialized_data_offset); } char* serialized_data() const { return from_mapped_offset(_serialized_data_offset); } const char* jvm_ident() const { return _jvm_ident; } char* requested_base_address() const { return _requested_base_address; } @@ -269,6 +275,8 @@ class FileMapHeader: private CDSFileMapHeaderBase { bool has_non_jar_in_classpath() const { return _has_non_jar_in_classpath; } bool compressed_oops() const { return _compressed_oops; } bool compressed_class_pointers() const { return _compressed_class_ptrs; } + int narrow_klass_pointer_bits() const { return _narrow_klass_pointer_bits; } + int narrow_klass_shift() const { return _narrow_klass_shift; } HeapRootSegments heap_root_segments() const { return _heap_root_segments; } bool has_full_module_graph() const { return _has_full_module_graph; } size_t heap_oopmap_start_pos() const { return _heap_oopmap_start_pos; } @@ -283,6 +291,7 @@ class FileMapHeader: private CDSFileMapHeaderBase { void set_has_platform_or_app_classes(bool v) { _has_platform_or_app_classes = v; } void set_cloned_vtables(char* p) { set_as_offset(p, &_cloned_vtables_offset); } + void set_early_serialized_data(char* p) { set_as_offset(p, &_early_serialized_data_offset); } void set_serialized_data(char* p) { set_as_offset(p, &_serialized_data_offset); } void set_mapped_base_address(char* p) { _mapped_base_address = p; } void set_heap_root_segments(HeapRootSegments segments) { _heap_root_segments = segments; } @@ -396,6 +405,8 @@ class FileMapInfo : public CHeapObj { char* cloned_vtables() const { return header()->cloned_vtables(); } void set_cloned_vtables(char* p) const { header()->set_cloned_vtables(p); } + char* early_serialized_data() const { return header()->early_serialized_data(); } + void set_early_serialized_data(char* p) const { header()->set_early_serialized_data(p); } char* serialized_data() const { return header()->serialized_data(); } void set_serialized_data(char* p) const { header()->set_serialized_data(p); } diff --git a/src/hotspot/share/cds/lambdaFormInvokers.cpp b/src/hotspot/share/cds/lambdaFormInvokers.cpp index 8a556a48624..88b1f9277ff 100644 --- a/src/hotspot/share/cds/lambdaFormInvokers.cpp +++ b/src/hotspot/share/cds/lambdaFormInvokers.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -51,7 +51,7 @@ #include "runtime/mutexLocker.hpp" GrowableArrayCHeap* LambdaFormInvokers::_lambdaform_lines = nullptr; -Array*>* LambdaFormInvokers::_static_archive_invokers = nullptr; +Array* LambdaFormInvokers::_static_archive_invokers = nullptr; #define NUM_FILTER 4 static const char* filter[NUM_FILTER] = {"java.lang.invoke.Invokers$Holder", @@ -216,7 +216,7 @@ void LambdaFormInvokers::dump_static_archive_invokers() { } } if (count > 0) { - _static_archive_invokers = ArchiveBuilder::new_ro_array*>(count); + _static_archive_invokers = ArchiveBuilder::new_ro_array(count); int index = 0; for (int i = 0; i < len; i++) { char* str = _lambdaform_lines->at(i); @@ -225,8 +225,7 @@ void LambdaFormInvokers::dump_static_archive_invokers() { Array* line = ArchiveBuilder::new_ro_array((int)str_len); strncpy(line->adr_at(0), str, str_len); - _static_archive_invokers->at_put(index, line); - ArchivePtrMarker::mark_pointer(_static_archive_invokers->adr_at(index)); + _static_archive_invokers->at_put(index, ArchiveBuilder::current()->any_to_offset_u4(line)); index++; } } @@ -239,7 +238,8 @@ void LambdaFormInvokers::dump_static_archive_invokers() { void LambdaFormInvokers::read_static_archive_invokers() { if (_static_archive_invokers != nullptr) { for (int i = 0; i < _static_archive_invokers->length(); i++) { - Array* line = _static_archive_invokers->at(i); + u4 offset = _static_archive_invokers->at(i); + Array* line = ArchiveUtils::from_offset*>(offset); char* str = line->adr_at(0); append(str); } diff --git a/src/hotspot/share/cds/lambdaFormInvokers.hpp b/src/hotspot/share/cds/lambdaFormInvokers.hpp index e78ddb1a1bc..583a863a1c2 100644 --- a/src/hotspot/share/cds/lambdaFormInvokers.hpp +++ b/src/hotspot/share/cds/lambdaFormInvokers.hpp @@ -38,7 +38,7 @@ class LambdaFormInvokers : public AllStatic { private: static GrowableArrayCHeap* _lambdaform_lines; // For storing LF form lines (LF_RESOLVE only) in read only table. - static Array*>* _static_archive_invokers; + static Array* _static_archive_invokers; static void regenerate_class(char* name, ClassFileStream& st, TRAPS); public: static void append(char* line); diff --git a/src/hotspot/share/cds/lambdaProxyClassDictionary.cpp b/src/hotspot/share/cds/lambdaProxyClassDictionary.cpp index 2eef543f9db..b8220a34047 100644 --- a/src/hotspot/share/cds/lambdaProxyClassDictionary.cpp +++ b/src/hotspot/share/cds/lambdaProxyClassDictionary.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2021, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -34,17 +34,6 @@ DumpTimeLambdaProxyClassInfo::~DumpTimeLambdaProxyClassInfo() { } } -void LambdaProxyClassKey::init_for_archive(LambdaProxyClassKey& dumptime_key) { - ArchiveBuilder* b = ArchiveBuilder::current(); - - b->write_pointer_in_buffer(&_caller_ik, dumptime_key._caller_ik); - b->write_pointer_in_buffer(&_instantiated_method_type, dumptime_key._instantiated_method_type); - b->write_pointer_in_buffer(&_invoked_name, dumptime_key._invoked_name); - b->write_pointer_in_buffer(&_invoked_type, dumptime_key._invoked_type); - b->write_pointer_in_buffer(&_member_method, dumptime_key._member_method); - b->write_pointer_in_buffer(&_method_type, dumptime_key._method_type); -} - unsigned int LambdaProxyClassKey::hash() const { return SystemDictionaryShared::hash_for_shared_dictionary((address)_caller_ik) + SystemDictionaryShared::hash_for_shared_dictionary((address)_invoked_name) + @@ -53,6 +42,14 @@ unsigned int LambdaProxyClassKey::hash() const { SystemDictionaryShared::hash_for_shared_dictionary((address)_instantiated_method_type); } +unsigned int RunTimeLambdaProxyClassKey::hash() const { + return primitive_hash(_caller_ik) + + primitive_hash(_invoked_name) + + primitive_hash(_invoked_type) + + primitive_hash(_method_type) + + primitive_hash(_instantiated_method_type); +} + #ifndef PRODUCT void LambdaProxyClassKey::print_on(outputStream* st) const { ResourceMark rm; @@ -65,13 +62,24 @@ void LambdaProxyClassKey::print_on(outputStream* st) const { st->print_cr("_method_type : %s", _method_type->as_C_string()); } +void RunTimeLambdaProxyClassKey::print_on(outputStream* st) const { + ResourceMark rm; + st->print_cr("LambdaProxyClassKey : " INTPTR_FORMAT " hash: %0x08x", p2i(this), hash()); + st->print_cr("_caller_ik : %d", _caller_ik); + st->print_cr("_instantiated_method_type : %d", _instantiated_method_type); + st->print_cr("_invoked_name : %d", _invoked_name); + st->print_cr("_invoked_type : %d", _invoked_type); + st->print_cr("_member_method : %d", _member_method); + st->print_cr("_method_type : %d", _method_type); +} + void RunTimeLambdaProxyClassInfo::print_on(outputStream* st) const { _key.print_on(st); } #endif void RunTimeLambdaProxyClassInfo::init(LambdaProxyClassKey& key, DumpTimeLambdaProxyClassInfo& info) { - _key.init_for_archive(key); + _key = RunTimeLambdaProxyClassKey::init_for_dumptime(key); ArchiveBuilder::current()->write_pointer_in_buffer(&_proxy_klass_head, info._proxy_klasses->at(0)); } diff --git a/src/hotspot/share/cds/lambdaProxyClassDictionary.hpp b/src/hotspot/share/cds/lambdaProxyClassDictionary.hpp index b9766066484..2a9f87681db 100644 --- a/src/hotspot/share/cds/lambdaProxyClassDictionary.hpp +++ b/src/hotspot/share/cds/lambdaProxyClassDictionary.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2021, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -101,9 +101,80 @@ class LambdaProxyClassKey { return (k1.equals(k2)); } - InstanceKlass* caller_ik() const { return _caller_ik; } + InstanceKlass* caller_ik() const { return _caller_ik; } + Symbol* invoked_name() const { return _invoked_name; } + Symbol* invoked_type() const { return _invoked_type; } + Symbol* method_type() const { return _method_type; } + Method* member_method() const { return _member_method; } + Symbol* instantiated_method_type() const { return _instantiated_method_type; } - void init_for_archive(LambdaProxyClassKey& dumptime_key); +#ifndef PRODUCT + void print_on(outputStream* st) const; +#endif +}; + +class RunTimeLambdaProxyClassKey { + u4 _caller_ik; + u4 _invoked_name; + u4 _invoked_type; + u4 _method_type; + u4 _member_method; + u4 _instantiated_method_type; + + RunTimeLambdaProxyClassKey(u4 caller_ik, + u4 invoked_name, + u4 invoked_type, + u4 method_type, + u4 member_method, + u4 instantiated_method_type) : + _caller_ik(caller_ik), + _invoked_name(invoked_name), + _invoked_type(invoked_type), + _method_type(method_type), + _member_method(member_method), + _instantiated_method_type(instantiated_method_type) {} + +public: + static RunTimeLambdaProxyClassKey init_for_dumptime(LambdaProxyClassKey& key) { + assert(ArchiveBuilder::is_active(), "sanity"); + ArchiveBuilder* b = ArchiveBuilder::current(); + + u4 caller_ik = b->any_to_offset_u4(key.caller_ik()); + u4 invoked_name = b->any_to_offset_u4(key.invoked_name()); + u4 invoked_type = b->any_to_offset_u4(key.invoked_type()); + u4 method_type = b->any_to_offset_u4(key.method_type()); + u4 member_method = b->any_to_offset_u4(key.member_method()); + u4 instantiated_method_type = b->any_to_offset_u4(key.instantiated_method_type()); + + return RunTimeLambdaProxyClassKey(caller_ik, invoked_name, invoked_type, method_type, + member_method, instantiated_method_type); + } + + static RunTimeLambdaProxyClassKey init_for_runtime(InstanceKlass* caller_ik, + Symbol* invoked_name, + Symbol* invoked_type, + Symbol* method_type, + Method* member_method, + Symbol* instantiated_method_type) { + // All parameters must be in shared space, or else you'd get an assert in + // ArchiveUtils::to_offset(). + return RunTimeLambdaProxyClassKey(ArchiveUtils::to_offset(caller_ik), + ArchiveUtils::to_offset(invoked_name), + ArchiveUtils::to_offset(invoked_type), + ArchiveUtils::to_offset(method_type), + ArchiveUtils::to_offset(member_method), + ArchiveUtils::to_offset(instantiated_method_type)); + } + + unsigned int hash() const; + bool equals(RunTimeLambdaProxyClassKey const& other) const { + return _caller_ik == other._caller_ik && + _invoked_name == other._invoked_name && + _invoked_type == other._invoked_type && + _method_type == other._method_type && + _member_method == other._member_method && + _instantiated_method_type == other._instantiated_method_type; + } #ifndef PRODUCT void print_on(outputStream* st) const; @@ -133,17 +204,17 @@ class DumpTimeLambdaProxyClassInfo { }; class RunTimeLambdaProxyClassInfo { - LambdaProxyClassKey _key; + RunTimeLambdaProxyClassKey _key; InstanceKlass* _proxy_klass_head; public: - RunTimeLambdaProxyClassInfo(LambdaProxyClassKey key, InstanceKlass* proxy_klass_head) : + RunTimeLambdaProxyClassInfo(RunTimeLambdaProxyClassKey key, InstanceKlass* proxy_klass_head) : _key(key), _proxy_klass_head(proxy_klass_head) {} InstanceKlass* proxy_klass_head() const { return _proxy_klass_head; } // Used by LambdaProxyClassDictionary to implement OffsetCompactHashtable::EQUALS static inline bool EQUALS( - const RunTimeLambdaProxyClassInfo* value, LambdaProxyClassKey* key, int len_unused) { + const RunTimeLambdaProxyClassInfo* value, RunTimeLambdaProxyClassKey* key, int len_unused) { return (value->_key.equals(*key)); } void init(LambdaProxyClassKey& key, DumpTimeLambdaProxyClassInfo& info); @@ -151,7 +222,7 @@ class RunTimeLambdaProxyClassInfo { unsigned int hash() const { return _key.hash(); } - LambdaProxyClassKey key() const { + RunTimeLambdaProxyClassKey key() const { return _key; } #ifndef PRODUCT @@ -173,7 +244,7 @@ class DumpTimeLambdaProxyClassDictionary }; class LambdaProxyClassDictionary : public OffsetCompactHashtable< - LambdaProxyClassKey*, + RunTimeLambdaProxyClassKey*, const RunTimeLambdaProxyClassInfo*, RunTimeLambdaProxyClassInfo::EQUALS> {}; diff --git a/src/hotspot/share/cds/metaspaceShared.cpp b/src/hotspot/share/cds/metaspaceShared.cpp index 2a43fd2dd4f..3cb1374efb1 100644 --- a/src/hotspot/share/cds/metaspaceShared.cpp +++ b/src/hotspot/share/cds/metaspaceShared.cpp @@ -87,6 +87,7 @@ #include "utilities/align.hpp" #include "utilities/bitMap.inline.hpp" #include "utilities/defaultStream.hpp" +#include "utilities/macros.hpp" #include "utilities/ostream.hpp" #include "utilities/resourceHash.hpp" @@ -359,8 +360,43 @@ void MetaspaceShared::read_extra_data(JavaThread* current, const char* filename) } } -// Read/write a data stream for restoring/preserving metadata pointers and -// miscellaneous data from/to the shared archive file. +// About "serialize" -- +// +// This is (probably a badly named) way to read/write a data stream of pointers and +// miscellaneous data from/to the shared archive file. The usual code looks like this: +// +// // These two global C++ variables are initialized during dump time. +// static int _archived_int; +// static MetaspaceObj* archived_ptr; +// +// void MyClass::serialize(SerializeClosure* soc) { +// soc->do_int(&_archived_int); +// soc->do_int(&_archived_ptr); +// } +// +// At dumptime, these two variables are stored into the CDS archive. +// At runtime, these two variables are loaded from the CDS archive. +// In addition, the pointer is relocated as necessary. +// +// Some of the xxx::serialize() functions may have side effects and assume that +// the archive is already mapped. For example, SymbolTable::serialize_shared_table_header() +// unconditionally makes the set of archived symbols available. Therefore, we put most +// of these xxx::serialize() functions inside MetaspaceShared::serialize(), which +// is called AFTER we made the decision to map the archive. +// +// However, some of the "serialized" data are used to decide whether an archive should +// be mapped or not (e.g., for checking if the -Djdk.module.main property is compatible +// with the archive). The xxx::serialize() functions for these data must be put inside +// MetaspaceShared::early_serialize(). Such functions must not produce side effects that +// assume we will always decides to map the archive. + +void MetaspaceShared::early_serialize(SerializeClosure* soc) { + int tag = 0; + soc->do_tag(--tag); + CDS_JAVA_HEAP_ONLY(Modules::serialize(soc);) + CDS_JAVA_HEAP_ONLY(Modules::serialize_addmods_names(soc);) + soc->do_tag(666); +} void MetaspaceShared::serialize(SerializeClosure* soc) { int tag = 0; @@ -402,8 +438,6 @@ void MetaspaceShared::serialize(SerializeClosure* soc) { SystemDictionaryShared::serialize_vm_classes(soc); soc->do_tag(--tag); - CDS_JAVA_HEAP_ONLY(Modules::serialize(soc);) - CDS_JAVA_HEAP_ONLY(Modules::serialize_addmods_names(soc);) CDS_JAVA_HEAP_ONLY(ClassLoaderDataShared::serialize(soc);) LambdaFormInvokers::serialize(soc); @@ -455,6 +489,7 @@ class VM_PopulateDumpSharedSpace : public VM_Operation { log_info(cds)("Dumping symbol table ..."); SymbolTable::write_to_archive(symbols); } + char* dump_early_read_only_tables(); char* dump_read_only_tables(); public: @@ -494,6 +529,21 @@ class StaticArchiveBuilder : public ArchiveBuilder { } }; +char* VM_PopulateDumpSharedSpace::dump_early_read_only_tables() { + ArchiveBuilder::OtherROAllocMark mark; + + // Write module name into archive + CDS_JAVA_HEAP_ONLY(Modules::dump_main_module_name();) + // Write module names from --add-modules into archive + CDS_JAVA_HEAP_ONLY(Modules::dump_addmods_names();) + + DumpRegion* ro_region = ArchiveBuilder::current()->ro_region(); + char* start = ro_region->top(); + WriteClosure wc(ro_region); + MetaspaceShared::early_serialize(&wc); + return start; +} + char* VM_PopulateDumpSharedSpace::dump_read_only_tables() { ArchiveBuilder::OtherROAllocMark mark; @@ -501,10 +551,7 @@ char* VM_PopulateDumpSharedSpace::dump_read_only_tables() { // Write lambform lines into archive LambdaFormInvokers::dump_static_archive_invokers(); - // Write module name into archive - CDS_JAVA_HEAP_ONLY(Modules::dump_main_module_name();) - // Write module names from --add-modules into archive - CDS_JAVA_HEAP_ONLY(Modules::dump_addmods_names();) + // Write the other data to the output array. DumpRegion* ro_region = ArchiveBuilder::current()->ro_region(); char* start = ro_region->top(); @@ -543,6 +590,7 @@ void VM_PopulateDumpSharedSpace::doit() { log_info(cds)("Make classes shareable"); _builder.make_klasses_shareable(); + char* early_serialized_data = dump_early_read_only_tables(); char* serialized_data = dump_read_only_tables(); SystemDictionaryShared::adjust_lambda_proxy_class_dictionary(); @@ -556,6 +604,7 @@ void VM_PopulateDumpSharedSpace::doit() { assert(static_archive != nullptr, "SharedArchiveFile not set?"); _map_info = new FileMapInfo(static_archive, true); _map_info->populate_header(MetaspaceShared::core_region_alignment()); + _map_info->set_early_serialized_data(early_serialized_data); _map_info->set_serialized_data(serialized_data); _map_info->set_cloned_vtables(CppVtables::vtables_serialized_base()); } @@ -1199,19 +1248,25 @@ MapArchiveResult MetaspaceShared::map_archives(FileMapInfo* static_mapinfo, File address cds_base = (address)static_mapinfo->mapped_base(); address ccs_end = (address)class_space_rs.end(); assert(ccs_end > cds_base, "Sanity check"); -#if INCLUDE_CDS_JAVA_HEAP - // We archived objects with pre-computed narrow Klass id. Set up encoding such that these Ids stay valid. - address precomputed_narrow_klass_base = cds_base; - const int precomputed_narrow_klass_shift = ArchiveHeapWriter::precomputed_narrow_klass_shift; - CompressedKlassPointers::initialize_for_given_encoding( - cds_base, ccs_end - cds_base, // Klass range - precomputed_narrow_klass_base, precomputed_narrow_klass_shift // precomputed encoding, see ArchiveHeapWriter - ); -#else - CompressedKlassPointers::initialize ( - cds_base, ccs_end - cds_base // Klass range + if (INCLUDE_CDS_JAVA_HEAP || UseCompactObjectHeaders) { + // The CDS archive may contain narrow Klass IDs that were precomputed at archive generation time: + // - every archived java object header (only if INCLUDE_CDS_JAVA_HEAP) + // - every archived Klass' prototype (only if +UseCompactObjectHeaders) + // + // In order for those IDs to still be valid, we need to dictate base and shift: base should be the + // mapping start, shift the shift used at archive generation time. + address precomputed_narrow_klass_base = cds_base; + const int precomputed_narrow_klass_shift = ArchiveBuilder::precomputed_narrow_klass_shift(); + CompressedKlassPointers::initialize_for_given_encoding( + cds_base, ccs_end - cds_base, // Klass range + precomputed_narrow_klass_base, precomputed_narrow_klass_shift // precomputed encoding, see ArchiveBuilder ); -#endif // INCLUDE_CDS_JAVA_HEAP + } else { + // Let JVM freely chose encoding base and shift + CompressedKlassPointers::initialize ( + cds_base, ccs_end - cds_base // Klass range + ); + } // map_or_load_heap_region() compares the current narrow oop and klass encodings // with the archived ones, so it must be done after all encodings are determined. static_mapinfo->map_or_load_heap_region(); @@ -1266,7 +1321,7 @@ MapArchiveResult MetaspaceShared::map_archives(FileMapInfo* static_mapinfo, File // // If UseCompressedClassPointers=1, the range encompassing both spaces will be // suitable to en/decode narrow Klass pointers: the base will be valid for -// encoding, the range [Base, End) not surpass KlassEncodingMetaspaceMax. +// encoding, the range [Base, End) and not surpass the max. range for that encoding. // // Return: // @@ -1387,7 +1442,7 @@ char* MetaspaceShared::reserve_address_space_for_archives(FileMapInfo* static_ma } else { // We did not manage to reserve at the preferred address, or were instructed to relocate. In that // case we reserve wherever possible, but the start address needs to be encodable as narrow Klass - // encoding base since the archived heap objects contain nKlass IDs pre-calculated toward the start + // encoding base since the archived heap objects contain narrow Klass IDs pre-calculated toward the start // of the shared Metaspace. That prevents us from using zero-based encoding and therefore we won't // try allocating in low-address regions. total_space_rs = Metaspace::reserve_address_space_for_compressed_classes(total_range_size, false /* optimize_for_zero_base */); @@ -1473,6 +1528,14 @@ MapArchiveResult MetaspaceShared::map_archive(FileMapInfo* mapinfo, char* mapped return MAP_ARCHIVE_OTHER_FAILURE; } + if (mapinfo->is_static()) { + // Currently, only static archive uses early serialized data. + char* buffer = mapinfo->early_serialized_data(); + intptr_t* array = (intptr_t*)buffer; + ReadClosure rc(&array, (intptr_t)mapped_base_address); + early_serialize(&rc); + } + mapinfo->set_is_mapped(true); return MAP_ARCHIVE_SUCCESS; } @@ -1509,7 +1572,7 @@ void MetaspaceShared::initialize_shared_spaces() { // shared string/symbol tables. char* buffer = static_mapinfo->serialized_data(); intptr_t* array = (intptr_t*)buffer; - ReadClosure rc(&array); + ReadClosure rc(&array, (intptr_t)SharedBaseAddress); serialize(&rc); // Finish up archived heap initialization. These must be @@ -1526,7 +1589,7 @@ void MetaspaceShared::initialize_shared_spaces() { FileMapInfo *dynamic_mapinfo = FileMapInfo::dynamic_info(); if (dynamic_mapinfo != nullptr) { intptr_t* buffer = (intptr_t*)dynamic_mapinfo->serialized_data(); - ReadClosure rc(&buffer); + ReadClosure rc(&buffer, (intptr_t)SharedBaseAddress); ArchiveBuilder::serialize_dynamic_archivable_items(&rc); DynamicArchive::setup_array_klasses(); dynamic_mapinfo->close(); diff --git a/src/hotspot/share/cds/metaspaceShared.hpp b/src/hotspot/share/cds/metaspaceShared.hpp index ecc158cc3ef..0b4f6f2ecd4 100644 --- a/src/hotspot/share/cds/metaspaceShared.hpp +++ b/src/hotspot/share/cds/metaspaceShared.hpp @@ -111,6 +111,7 @@ class MetaspaceShared : AllStatic { static void unrecoverable_writing_error(const char* message = nullptr); static void writing_error(const char* message = nullptr); + static void early_serialize(SerializeClosure* sc) NOT_CDS_RETURN; static void serialize(SerializeClosure* sc) NOT_CDS_RETURN; // JVM/TI RedefineClasses() support: diff --git a/src/hotspot/share/cds/runTimeClassInfo.cpp b/src/hotspot/share/cds/runTimeClassInfo.cpp index 2536b533086..e1329d9d2f9 100644 --- a/src/hotspot/share/cds/runTimeClassInfo.cpp +++ b/src/hotspot/share/cds/runTimeClassInfo.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2021, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -30,9 +30,10 @@ void RunTimeClassInfo::init(DumpTimeClassInfo& info) { ArchiveBuilder* builder = ArchiveBuilder::current(); - builder->write_pointer_in_buffer(&_klass, info._klass); + InstanceKlass* k = info._klass; + _klass_offset = builder->any_to_offset_u4(k); - if (!SystemDictionaryShared::is_builtin(_klass)) { + if (!SystemDictionaryShared::is_builtin(k)) { CrcInfo* c = crc(); c->_clsfile_size = info._clsfile_size; c->_clsfile_crc32 = info._clsfile_crc32; @@ -61,10 +62,10 @@ void RunTimeClassInfo::init(DumpTimeClassInfo& info) { } } - if (_klass->is_hidden()) { - builder->write_pointer_in_buffer(nest_host_addr(), info.nest_host()); + if (k->is_hidden()) { + _nest_host_offset = builder->any_to_offset_u4(info.nest_host()); } - if (_klass->has_archived_enum_objs()) { + if (k->has_archived_enum_objs()) { int num = info.num_enum_klass_static_fields(); set_num_enum_klass_static_fields(num); for (int i = 0; i < num; i++) { @@ -74,6 +75,14 @@ void RunTimeClassInfo::init(DumpTimeClassInfo& info) { } } +InstanceKlass* RunTimeClassInfo::klass() const { + if (ArchiveBuilder::is_active() && ArchiveBuilder::current()->is_in_buffer_space((address)this)) { + return ArchiveBuilder::current()->offset_to_buffered(_klass_offset); + } else { + return ArchiveUtils::from_offset(_klass_offset); + } +} + size_t RunTimeClassInfo::crc_size(InstanceKlass* klass) { if (!SystemDictionaryShared::is_builtin(klass)) { return sizeof(CrcInfo); diff --git a/src/hotspot/share/cds/runTimeClassInfo.hpp b/src/hotspot/share/cds/runTimeClassInfo.hpp index 50cf3d528b9..d0f02d1c095 100644 --- a/src/hotspot/share/cds/runTimeClassInfo.hpp +++ b/src/hotspot/share/cds/runTimeClassInfo.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2021, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -52,24 +52,24 @@ class RunTimeClassInfo { struct RTVerifierConstraint { u4 _name; u4 _from_name; - Symbol* name() { return (Symbol*)(SharedBaseAddress + _name);} - Symbol* from_name() { return (Symbol*)(SharedBaseAddress + _from_name); } + Symbol* name() { return ArchiveUtils::from_offset(_name); } + Symbol* from_name() { return ArchiveUtils::from_offset(_from_name); } }; struct RTLoaderConstraint { u4 _name; char _loader_type1; char _loader_type2; - Symbol* constraint_name() { - return (Symbol*)(SharedBaseAddress + _name); - } + Symbol* constraint_name() { return ArchiveUtils::from_offset(_name); } }; struct RTEnumKlassStaticFields { int _num; int _root_indices[1]; }; - InstanceKlass* _klass; +private: + u4 _klass_offset; + u4 _nest_host_offset; int _num_verifier_constraints; int _num_loader_constraints; @@ -80,7 +80,6 @@ class RunTimeClassInfo { // optional char _verifier_constraint_flags[_num_verifier_constraints] // optional RTEnumKlassStaticFields _enum_klass_static_fields; -private: static size_t header_size_size() { return align_up(sizeof(RunTimeClassInfo), wordSize); } @@ -108,6 +107,9 @@ class RunTimeClassInfo { static size_t crc_size(InstanceKlass* klass); public: + InstanceKlass* klass() const; + int num_verifier_constraints() const { return _num_verifier_constraints; } + int num_loader_constraints() const { return _num_loader_constraints; } static size_t byte_size(InstanceKlass* klass, int num_verifier_constraints, int num_loader_constraints, int num_enum_klass_static_fields) { return header_size_size() + @@ -125,11 +127,11 @@ class RunTimeClassInfo { } size_t nest_host_offset() const { - return crc_offset() + crc_size(_klass); + return crc_offset() + crc_size(klass()); } size_t loader_constraints_offset() const { - return nest_host_offset() + nest_host_size(_klass); + return nest_host_offset() + nest_host_size(klass()); } size_t verifier_constraints_offset() const { return loader_constraints_offset() + loader_constraints_size(_num_loader_constraints); @@ -150,13 +152,13 @@ class RunTimeClassInfo { } RTEnumKlassStaticFields* enum_klass_static_fields_addr() const { - assert(_klass->has_archived_enum_objs(), "sanity"); + assert(klass()->has_archived_enum_objs(), "sanity"); return (RTEnumKlassStaticFields*)(address(this) + enum_klass_static_fields_offset()); } public: CrcInfo* crc() const { - assert(crc_size(_klass) > 0, "must be"); + assert(crc_size(klass()) > 0, "must be"); return (CrcInfo*)(address(this) + crc_offset()); } RTVerifierConstraint* verifier_constraints() { @@ -173,12 +175,9 @@ class RunTimeClassInfo { return (char*)(address(this) + verifier_constraint_flags_offset()); } - InstanceKlass** nest_host_addr() { - assert(_klass->is_hidden(), "sanity"); - return (InstanceKlass**)(address(this) + nest_host_offset()); - } InstanceKlass* nest_host() { - return *nest_host_addr(); + assert(!ArchiveBuilder::is_active(), "not called when dumping archive"); + return ArchiveUtils::from_offset(_nest_host_offset); } RTLoaderConstraint* loader_constraints() { @@ -248,7 +247,7 @@ class RunTimeClassInfo { // Used by RunTimeSharedDictionary to implement OffsetCompactHashtable::EQUALS static inline bool EQUALS( const RunTimeClassInfo* value, Symbol* key, int len_unused) { - return (value->_klass->name() == key); + return (value->klass()->name() == key); } }; diff --git a/src/hotspot/share/cds/unregisteredClasses.cpp b/src/hotspot/share/cds/unregisteredClasses.cpp index 91346a20085..d23ca8903e9 100644 --- a/src/hotspot/share/cds/unregisteredClasses.cpp +++ b/src/hotspot/share/cds/unregisteredClasses.cpp @@ -51,6 +51,7 @@ InstanceKlass* UnregisteredClasses::load_class(Symbol* name, const char* path, T PerfClassTraceTime::CLASS_LOAD); Symbol* path_symbol = SymbolTable::new_symbol(path); + Symbol* findClass = SymbolTable::new_symbol("findClass"); Handle url_classloader = get_url_classloader(path_symbol, CHECK_NULL); Handle ext_class_name = java_lang_String::externalize_classname(name, CHECK_NULL); @@ -58,11 +59,10 @@ InstanceKlass* UnregisteredClasses::load_class(Symbol* name, const char* path, T JavaCallArguments args(2); args.set_receiver(url_classloader); args.push_oop(ext_class_name); - args.push_int(JNI_FALSE); JavaCalls::call_virtual(&result, vmClasses::URLClassLoader_klass(), - vmSymbols::loadClass_name(), - vmSymbols::string_boolean_class_signature(), + findClass, + vmSymbols::string_class_signature(), &args, CHECK_NULL); assert(result.get_type() == T_OBJECT, "just checking"); diff --git a/src/hotspot/share/ci/ciKlass.cpp b/src/hotspot/share/ci/ciKlass.cpp index efdd2512f90..f65d4a0311c 100644 --- a/src/hotspot/share/ci/ciKlass.cpp +++ b/src/hotspot/share/ci/ciKlass.cpp @@ -258,3 +258,23 @@ const char* ciKlass::external_name() const { return get_Klass()->external_name(); ) } + +// ------------------------------------------------------------------ +// ciKlass::prototype_header_offset +juint ciKlass::prototype_header_offset() { + assert(is_loaded(), "must be loaded"); + + VM_ENTRY_MARK; + Klass* this_klass = get_Klass(); + return in_bytes(this_klass->prototype_header_offset()); +} + +// ------------------------------------------------------------------ +// ciKlass::prototype_header +uintptr_t ciKlass::prototype_header() { + assert(is_loaded(), "must be loaded"); + + VM_ENTRY_MARK; + Klass* this_klass = get_Klass(); + return (uintptr_t)this_klass->prototype_header().to_pointer(); +} diff --git a/src/hotspot/share/ci/ciKlass.hpp b/src/hotspot/share/ci/ciKlass.hpp index 10d8395ed7f..7b8d871eb56 100644 --- a/src/hotspot/share/ci/ciKlass.hpp +++ b/src/hotspot/share/ci/ciKlass.hpp @@ -139,6 +139,9 @@ class ciKlass : public ciType { void print_name_on(outputStream* st); const char* external_name() const; + + juint prototype_header_offset(); + uintptr_t prototype_header(); }; #endif // SHARE_CI_CIKLASS_HPP diff --git a/src/hotspot/share/classfile/classFileParser.cpp b/src/hotspot/share/classfile/classFileParser.cpp index 93dd0af65e7..eb2d1b684d2 100644 --- a/src/hotspot/share/classfile/classFileParser.cpp +++ b/src/hotspot/share/classfile/classFileParser.cpp @@ -5836,6 +5836,15 @@ bool ClassFileParser::is_java_lang_ref_Reference_subclass() const { return _super_klass->reference_type() != REF_NONE; } +// Returns true if the future Klass will need to be addressable with a narrow Klass ID. +bool ClassFileParser::klass_needs_narrow_id() const { + // Classes that are never instantiated need no narrow Klass Id, since the + // only point of having a narrow id is to put it into an object header. Keeping + // never instantiated classes out of class space lessens the class space pressure. + // For more details, see JDK-8338526. + return !is_interface() && !is_abstract(); +} + // ---------------------------------------------------------------------------- // debugging diff --git a/src/hotspot/share/classfile/classFileParser.hpp b/src/hotspot/share/classfile/classFileParser.hpp index a8a388f8ded..18cdeec7d8e 100644 --- a/src/hotspot/share/classfile/classFileParser.hpp +++ b/src/hotspot/share/classfile/classFileParser.hpp @@ -511,6 +511,10 @@ class ClassFileParser { bool is_interface() const { return _access_flags.is_interface(); } bool is_abstract() const { return _access_flags.is_abstract(); } + // Returns true if the Klass to be generated will need to be addressable + // with a narrow Klass ID. + bool klass_needs_narrow_id() const; + ClassLoaderData* loader_data() const { return _loader_data; } const Symbol* class_name() const { return _class_name; } const InstanceKlass* super_klass() const { return _super_klass; } diff --git a/src/hotspot/share/classfile/classLoader.cpp b/src/hotspot/share/classfile/classLoader.cpp index 9a68e264044..cb155bdeb19 100644 --- a/src/hotspot/share/classfile/classLoader.cpp +++ b/src/hotspot/share/classfile/classLoader.cpp @@ -1242,7 +1242,7 @@ static char decode_percent_encoded(const char *str, size_t& index) { hex[1] = str[index + 2]; hex[2] = '\0'; index += 2; - return (char) strtol(hex, NULL, 16); + return (char) strtol(hex, nullptr, 16); } return str[index]; } diff --git a/src/hotspot/share/classfile/dictionary.cpp b/src/hotspot/share/classfile/dictionary.cpp index f25c453582f..eecfc9e88a0 100644 --- a/src/hotspot/share/classfile/dictionary.cpp +++ b/src/hotspot/share/classfile/dictionary.cpp @@ -347,68 +347,6 @@ void Dictionary::check_package_access(InstanceKlass* klass, assert(class_loader() != nullptr, "Should not call this"); assert(protection_domain() != nullptr, "Should not call this"); - - if (!java_lang_System::allow_security_manager()) { - // No need for any further checking. Package access always allowed. - return; - } - - if (is_in_package_access_cache(THREAD, klass->name(), protection_domain)) { - // No need to check again. - return; - } - - // We only have to call checkPackageAccess if there's a security manager installed. - if (java_lang_System::has_security_manager()) { - - // This handle and the class_loader handle passed in keeps this class from - // being unloaded through several GC points. - // The class_loader handle passed in is the initiating loader. - Handle mirror(THREAD, klass->java_mirror()); - - // Now we have to call back to java to check if the initating class has access - InstanceKlass* system_loader = vmClasses::ClassLoader_klass(); - JavaValue result(T_VOID); - JavaCalls::call_special(&result, - class_loader, - system_loader, - vmSymbols::checkPackageAccess_name(), - vmSymbols::class_protectiondomain_signature(), - mirror, - protection_domain, - THREAD); - - LogTarget(Debug, protectiondomain) lt; - if (lt.is_enabled()) { - ResourceMark rm(THREAD); - // Print out trace information - LogStream ls(lt); - ls.print_cr("Checking package access"); - ls.print("class loader: "); - class_loader()->print_value_on(&ls); - ls.print(" protection domain: "); - protection_domain()->print_value_on(&ls); - ls.print(" loading: "); klass->print_value_on(&ls); - if (HAS_PENDING_EXCEPTION) { - ls.print_cr(" DENIED !!!!!!!!!!!!!!!!!!!!!"); - } else { - ls.print_cr(" granted"); - } - } - - if (HAS_PENDING_EXCEPTION) return; - } - - // If no exception has been thrown, we have checked that the protection_domain can access - // this klass. Always add it to the cache (even if no SecurityManager is installed yet). - // - // This ensures that subsequent calls to Dictionary::find(THREAD, klass->name(), protection_domain) - // will always succeed. I.e., a new SecurityManager installed in the future cannot retroactively - // revoke the granted access. - { - MutexLocker mu(THREAD, SystemDictionary_lock); - add_to_package_access_cache(THREAD, klass, protection_domain); - } } // During class loading we may have cached a protection domain that has diff --git a/src/hotspot/share/classfile/javaClasses.cpp b/src/hotspot/share/classfile/javaClasses.cpp index 3dce1a343d8..0dd183e06d4 100644 --- a/src/hotspot/share/classfile/javaClasses.cpp +++ b/src/hotspot/share/classfile/javaClasses.cpp @@ -412,12 +412,6 @@ Handle java_lang_String::create_from_platform_dependent_str(const char* str, TRA if (_to_java_string_fn == nullptr) { void *lib_handle = os::native_java_library(); _to_java_string_fn = CAST_TO_FN_PTR(to_java_string_fn_t, os::dll_lookup(lib_handle, "JNU_NewStringPlatform")); -#if defined(_WIN32) && !defined(_WIN64) - if (_to_java_string_fn == nullptr) { - // On 32 bit Windows, also try __stdcall decorated name - _to_java_string_fn = CAST_TO_FN_PTR(to_java_string_fn_t, os::dll_lookup(lib_handle, "_JNU_NewStringPlatform@8")); - } -#endif if (_to_java_string_fn == nullptr) { fatal("JNU_NewStringPlatform missing"); } @@ -1599,7 +1593,6 @@ oop java_lang_Thread_Constants::get_VTHREAD_GROUP() { int java_lang_Thread::_holder_offset; int java_lang_Thread::_name_offset; int java_lang_Thread::_contextClassLoader_offset; -int java_lang_Thread::_inheritedAccessControlContext_offset; int java_lang_Thread::_eetop_offset; int java_lang_Thread::_jvmti_thread_state_offset; int java_lang_Thread::_jvmti_VTMS_transition_disable_count_offset; @@ -1616,7 +1609,6 @@ JFR_ONLY(int java_lang_Thread::_jfr_epoch_offset;) macro(_holder_offset, k, "holder", thread_fieldholder_signature, false); \ macro(_name_offset, k, vmSymbols::name_name(), string_signature, false); \ macro(_contextClassLoader_offset, k, vmSymbols::contextClassLoader_name(), classloader_signature, false); \ - macro(_inheritedAccessControlContext_offset, k, vmSymbols::inheritedAccessControlContext_name(), accesscontrolcontext_signature, false); \ macro(_eetop_offset, k, "eetop", long_signature, false); \ macro(_interrupted_offset, k, "interrupted", bool_signature, false); \ macro(_interruptLock_offset, k, "interruptLock", object_signature, false); \ @@ -1686,6 +1678,7 @@ bool java_lang_Thread::is_in_VTMS_transition(oop java_thread) { } void java_lang_Thread::set_is_in_VTMS_transition(oop java_thread, bool val) { + assert(is_in_VTMS_transition(java_thread) != val, "already %s transition", val ? "inside" : "outside"); java_thread->bool_field_put_volatile(_jvmti_is_in_VTMS_transition_offset, val); } @@ -1793,10 +1786,6 @@ oop java_lang_Thread::context_class_loader(oop java_thread) { return java_thread->obj_field(_contextClassLoader_offset); } -oop java_lang_Thread::inherited_access_control_context(oop java_thread) { - return java_thread->obj_field(_inheritedAccessControlContext_offset); -} - jlong java_lang_Thread::stackSize(oop java_thread) { GET_FIELDHOLDER_FIELD(java_thread, stackSize, 0); @@ -2021,12 +2010,20 @@ int java_lang_VirtualThread::static_vthread_scope_offset; int java_lang_VirtualThread::_carrierThread_offset; int java_lang_VirtualThread::_continuation_offset; int java_lang_VirtualThread::_state_offset; +int java_lang_VirtualThread::_next_offset; +int java_lang_VirtualThread::_onWaitingList_offset; +int java_lang_VirtualThread::_notified_offset; +int java_lang_VirtualThread::_timeout_offset; #define VTHREAD_FIELDS_DO(macro) \ macro(static_vthread_scope_offset, k, "VTHREAD_SCOPE", continuationscope_signature, true); \ macro(_carrierThread_offset, k, "carrierThread", thread_signature, false); \ macro(_continuation_offset, k, "cont", continuation_signature, false); \ - macro(_state_offset, k, "state", int_signature, false) + macro(_state_offset, k, "state", int_signature, false); \ + macro(_next_offset, k, "next", vthread_signature, false); \ + macro(_onWaitingList_offset, k, "onWaitingList", bool_signature, false); \ + macro(_notified_offset, k, "notified", bool_signature, false); \ + macro(_timeout_offset, k, "timeout", long_signature, false); void java_lang_VirtualThread::compute_offsets() { @@ -2052,6 +2049,56 @@ int java_lang_VirtualThread::state(oop vthread) { return vthread->int_field_acquire(_state_offset); } +void java_lang_VirtualThread::set_state(oop vthread, int state) { + vthread->release_int_field_put(_state_offset, state); +} + +int java_lang_VirtualThread::cmpxchg_state(oop vthread, int old_state, int new_state) { + jint* addr = vthread->field_addr(_state_offset); + int res = Atomic::cmpxchg(addr, old_state, new_state); + return res; +} + +oop java_lang_VirtualThread::next(oop vthread) { + return vthread->obj_field(_next_offset); +} + +void java_lang_VirtualThread::set_next(oop vthread, oop next_vthread) { + vthread->obj_field_put(_next_offset, next_vthread); +} + +// Add vthread to the waiting list if it's not already in it. Multiple threads +// could be trying to add vthread to the list at the same time, so we control +// access with a cmpxchg on onWaitingList. The winner adds vthread to the list. +// Method returns true if we added vthread to the list, false otherwise. +bool java_lang_VirtualThread::set_onWaitingList(oop vthread, OopHandle& list_head) { + jboolean* addr = vthread->field_addr(_onWaitingList_offset); + jboolean vthread_on_list = Atomic::load(addr); + if (!vthread_on_list) { + vthread_on_list = Atomic::cmpxchg(addr, (jboolean)JNI_FALSE, (jboolean)JNI_TRUE); + if (!vthread_on_list) { + for (;;) { + oop head = list_head.resolve(); + java_lang_VirtualThread::set_next(vthread, head); + if (list_head.cmpxchg(head, vthread) == head) return true; + } + } + } + return false; // already on waiting list +} + +void java_lang_VirtualThread::set_notified(oop vthread, jboolean value) { + vthread->bool_field_put_volatile(_notified_offset, value); +} + +jlong java_lang_VirtualThread::timeout(oop vthread) { + return vthread->long_field(_timeout_offset); +} + +void java_lang_VirtualThread::set_timeout(oop vthread, jlong value) { + vthread->long_field_put(_timeout_offset, value); +} + JavaThreadStatus java_lang_VirtualThread::map_state_to_thread_status(int state) { JavaThreadStatus status = JavaThreadStatus::NEW; switch (state & ~SUSPENDED) { @@ -2065,6 +2112,9 @@ JavaThreadStatus java_lang_VirtualThread::map_state_to_thread_status(int state) case UNPARKED: case YIELDING: case YIELDED: + case UNBLOCKED: + case WAITING: + case TIMED_WAITING: status = JavaThreadStatus::RUNNABLE; break; case PARKED: @@ -2075,6 +2125,16 @@ JavaThreadStatus java_lang_VirtualThread::map_state_to_thread_status(int state) case TIMED_PINNED: status = JavaThreadStatus::PARKED_TIMED; break; + case BLOCKING: + case BLOCKED: + status = JavaThreadStatus::BLOCKED_ON_MONITOR_ENTER; + break; + case WAIT: + status = JavaThreadStatus::IN_OBJECT_WAIT; + break; + case TIMED_WAIT: + status = JavaThreadStatus::IN_OBJECT_WAIT_TIMED; + break; case TERMINATED: status = JavaThreadStatus::TERMINATED; break; @@ -2084,6 +2144,13 @@ JavaThreadStatus java_lang_VirtualThread::map_state_to_thread_status(int state) return status; } +bool java_lang_VirtualThread::is_preempted(oop vthread) { + oop continuation = java_lang_VirtualThread::continuation(vthread); + assert(continuation != nullptr, "vthread with no continuation"); + stackChunkOop chunk = jdk_internal_vm_Continuation::tail(continuation); + return chunk != nullptr && chunk->preempted(); +} + #if INCLUDE_CDS void java_lang_VirtualThread::serialize_offsets(SerializeClosure* f) { VTHREAD_FIELDS_DO(FIELD_SERIALIZE_OFFSET); @@ -4782,17 +4849,11 @@ oop java_lang_ClassLoader::unnamedModule(oop loader) { int java_lang_System::_static_in_offset; int java_lang_System::_static_out_offset; int java_lang_System::_static_err_offset; -int java_lang_System::_static_security_offset; -int java_lang_System::_static_allow_security_offset; -int java_lang_System::_static_never_offset; #define SYSTEM_FIELDS_DO(macro) \ macro(_static_in_offset, k, "in", input_stream_signature, true); \ macro(_static_out_offset, k, "out", print_stream_signature, true); \ - macro(_static_err_offset, k, "err", print_stream_signature, true); \ - macro(_static_security_offset, k, "security", security_manager_signature, true); \ - macro(_static_allow_security_offset, k, "allowSecurityManager", int_signature, true); \ - macro(_static_never_offset, k, "NEVER", int_signature, true) + macro(_static_err_offset, k, "err", print_stream_signature, true); void java_lang_System::compute_offsets() { InstanceKlass* k = vmClasses::System_klass(); @@ -4802,21 +4863,12 @@ void java_lang_System::compute_offsets() { // This field tells us that a security manager can never be installed so we // can completely skip populating the ProtectionDomainCacheTable. bool java_lang_System::allow_security_manager() { - static int initialized = false; - static bool allowed = true; // default - if (!initialized) { - oop base = vmClasses::System_klass()->static_field_base_raw(); - int never = base->int_field(_static_never_offset); - allowed = (base->int_field(_static_allow_security_offset) != never); - initialized = true; - } - return allowed; + return false; } // This field tells us that a security manager is installed. bool java_lang_System::has_security_manager() { - oop base = vmClasses::System_klass()->static_field_base_raw(); - return base->obj_field(_static_security_offset) != nullptr; + return false; } #if INCLUDE_CDS diff --git a/src/hotspot/share/classfile/javaClasses.hpp b/src/hotspot/share/classfile/javaClasses.hpp index 6ab73b161f4..e0123595810 100644 --- a/src/hotspot/share/classfile/javaClasses.hpp +++ b/src/hotspot/share/classfile/javaClasses.hpp @@ -358,7 +358,6 @@ class java_lang_Thread : AllStatic { static int _holder_offset; static int _name_offset; static int _contextClassLoader_offset; - static int _inheritedAccessControlContext_offset; static int _eetop_offset; static int _jvmti_thread_state_offset; static int _jvmti_VTMS_transition_disable_count_offset; @@ -405,8 +404,6 @@ class java_lang_Thread : AllStatic { static void set_daemon(oop java_thread); // Context ClassLoader static oop context_class_loader(oop java_thread); - // Control context - static oop inherited_access_control_context(oop java_thread); // Stack size hint static jlong stackSize(oop java_thread); // Thread ID @@ -530,6 +527,11 @@ class java_lang_VirtualThread : AllStatic { static int _carrierThread_offset; static int _continuation_offset; static int _state_offset; + static int _next_offset; + static int _onWaitingList_offset; + static int _notified_offset; + static int _recheckInterval_offset; + static int _timeout_offset; JFR_ONLY(static int _jfr_epoch_offset;) public: enum { @@ -545,6 +547,13 @@ class java_lang_VirtualThread : AllStatic { UNPARKED = 9, YIELDING = 10, YIELDED = 11, + BLOCKING = 12, + BLOCKED = 13, + UNBLOCKED = 14, + WAITING = 15, + WAIT = 16, // waiting in Object.wait + TIMED_WAITING = 17, + TIMED_WAIT = 18, // waiting in timed-Object.wait TERMINATED = 99, // additional state bits @@ -564,6 +573,15 @@ class java_lang_VirtualThread : AllStatic { static oop carrier_thread(oop vthread); static oop continuation(oop vthread); static int state(oop vthread); + static void set_state(oop vthread, int state); + static int cmpxchg_state(oop vthread, int old_state, int new_state); + static oop next(oop vthread); + static void set_next(oop vthread, oop next_vthread); + static bool set_onWaitingList(oop vthread, OopHandle& list_head); + static jlong timeout(oop vthread); + static void set_timeout(oop vthread, jlong value); + static void set_notified(oop vthread, jboolean value); + static bool is_preempted(oop vthread); static JavaThreadStatus map_state_to_thread_status(int state); }; diff --git a/src/hotspot/share/classfile/modules.cpp b/src/hotspot/share/classfile/modules.cpp index 94e407d045f..2a8f8bd2424 100644 --- a/src/hotspot/share/classfile/modules.cpp +++ b/src/hotspot/share/classfile/modules.cpp @@ -599,6 +599,9 @@ void Modules::serialize(SerializeClosure* soc) { } log_info(cds)("optimized module handling: %s", CDSConfig::is_using_optimized_module_handling() ? "enabled" : "disabled"); log_info(cds)("full module graph: %s", CDSConfig::is_using_full_module_graph() ? "enabled" : "disabled"); + + // Don't hold onto the pointer, in case we might decide to unmap the archive. + _archived_main_module_name = nullptr; } } @@ -641,6 +644,9 @@ void Modules::serialize_addmods_names(SerializeClosure* soc) { } log_info(cds)("optimized module handling: %s", CDSConfig::is_using_optimized_module_handling() ? "enabled" : "disabled"); log_info(cds)("full module graph: %s", CDSConfig::is_using_full_module_graph() ? "enabled" : "disabled"); + + // Don't hold onto the pointer, in case we might decide to unmap the archive. + _archived_addmods_names = nullptr; } } diff --git a/src/hotspot/share/classfile/systemDictionaryShared.cpp b/src/hotspot/share/classfile/systemDictionaryShared.cpp index 05f6ace9b8a..08f224d969f 100644 --- a/src/hotspot/share/classfile/systemDictionaryShared.cpp +++ b/src/hotspot/share/classfile/systemDictionaryShared.cpp @@ -59,6 +59,7 @@ #include "memory/oopFactory.hpp" #include "memory/resourceArea.hpp" #include "memory/universe.hpp" +#include "oops/compressedKlass.inline.hpp" #include "oops/instanceKlass.hpp" #include "oops/klass.inline.hpp" #include "oops/objArrayKlass.hpp" @@ -83,6 +84,16 @@ DumpTimeLambdaProxyClassDictionary* SystemDictionaryShared::_dumptime_lambda_pro // Used by NoClassLoadingMark DEBUG_ONLY(bool SystemDictionaryShared::_class_loading_may_happen = true;) +#ifdef ASSERT +static void check_klass_after_loading(const Klass* k) { +#ifdef _LP64 + if (k != nullptr && UseCompressedClassPointers && k->needs_narrow_id()) { + CompressedKlassPointers::check_encodable(k); + } +#endif +} +#endif + InstanceKlass* SystemDictionaryShared::load_shared_class_for_builtin_loader( Symbol* class_name, Handle class_loader, TRAPS) { assert(CDSConfig::is_using_archive(), "must be"); @@ -134,7 +145,7 @@ InstanceKlass* SystemDictionaryShared::lookup_from_stream(Symbol* class_name, return nullptr; } - return acquire_class_for_current_thread(record->_klass, class_loader, + return acquire_class_for_current_thread(record->klass(), class_loader, protection_domain, cfs, THREAD); } @@ -425,6 +436,9 @@ InstanceKlass* SystemDictionaryShared::find_or_load_shared_class( } } } + + DEBUG_ONLY(check_klass_after_loading(k);) + return k; } @@ -789,9 +803,20 @@ InstanceKlass* SystemDictionaryShared::get_shared_lambda_proxy_class(InstanceKla Symbol* method_type, Method* member_method, Symbol* instantiated_method_type) { + if (!caller_ik->is_shared() || + !invoked_name->is_shared() || + !invoked_type->is_shared() || + !method_type->is_shared() || + !member_method->is_shared() || + !instantiated_method_type->is_shared()) { + // These can't be represented as u4 offset, but we wouldn't have archived a lambda proxy in this case anyway. + return nullptr; + } + MutexLocker ml(CDSLambda_lock, Mutex::_no_safepoint_check_flag); - LambdaProxyClassKey key(caller_ik, invoked_name, invoked_type, - method_type, member_method, instantiated_method_type); + RunTimeLambdaProxyClassKey key = + RunTimeLambdaProxyClassKey::init_for_runtime(caller_ik, invoked_name, invoked_type, + method_type, member_method, instantiated_method_type); // Try to retrieve the lambda proxy class from static archive. const RunTimeLambdaProxyClassInfo* info = _static_archive.lookup_lambda_proxy_class(&key); @@ -899,7 +924,7 @@ void SystemDictionaryShared::check_verification_constraints(InstanceKlass* klass assert(!CDSConfig::is_dumping_static_archive() && CDSConfig::is_using_archive(), "called at run time with CDS enabled only"); RunTimeClassInfo* record = RunTimeClassInfo::get_for(klass); - int length = record->_num_verifier_constraints; + int length = record->num_verifier_constraints(); if (length > 0) { for (int i = 0; i < length; i++) { RunTimeClassInfo::RTVerifierConstraint* vc = record->verifier_constraint_at(i); @@ -1015,9 +1040,9 @@ bool SystemDictionaryShared::check_linking_constraints(Thread* current, Instance if (klass->is_shared_platform_class() || klass->is_shared_app_class()) { RunTimeClassInfo* info = RunTimeClassInfo::get_for(klass); assert(info != nullptr, "Sanity"); - if (info->_num_loader_constraints > 0) { + if (info->num_loader_constraints() > 0) { HandleMark hm(current); - for (int i = 0; i < info->_num_loader_constraints; i++) { + for (int i = 0; i < info->num_loader_constraints(); i++) { RunTimeClassInfo::RTLoaderConstraint* lc = info->loader_constraint_at(i); Symbol* name = lc->constraint_name(); Handle loader1(current, get_class_loader_by(lc->_loader_type1)); @@ -1333,14 +1358,14 @@ InstanceKlass* SystemDictionaryShared::find_builtin_class(Symbol* name) { &_dynamic_archive._builtin_dictionary, name); if (record != nullptr) { - assert(!record->_klass->is_hidden(), "hidden class cannot be looked up by name"); - assert(check_alignment(record->_klass), "Address not aligned"); + assert(!record->klass()->is_hidden(), "hidden class cannot be looked up by name"); + DEBUG_ONLY(check_klass_after_loading(record->klass());) // We did not save the classfile data of the generated LambdaForm invoker classes, // so we cannot support CLFH for such classes. - if (record->_klass->is_generated_shared_class() && JvmtiExport::should_post_class_file_load_hook()) { + if (record->klass()->is_generated_shared_class() && JvmtiExport::should_post_class_file_load_hook()) { return nullptr; } - return record->_klass; + return record->klass(); } else { return nullptr; } @@ -1378,10 +1403,10 @@ class SharedDictionaryPrinter : StackObj { void do_value(const RunTimeClassInfo* record) { ResourceMark rm; - _st->print_cr("%4d: %s %s", _index++, record->_klass->external_name(), - class_loader_name_for_shared(record->_klass)); - if (record->_klass->array_klasses() != nullptr) { - record->_klass->array_klasses()->cds_print_value_on(_st); + _st->print_cr("%4d: %s %s", _index++, record->klass()->external_name(), + class_loader_name_for_shared(record->klass())); + if (record->klass()->array_klasses() != nullptr) { + record->klass()->array_klasses()->cds_print_value_on(_st); _st->cr(); } } diff --git a/src/hotspot/share/classfile/systemDictionaryShared.hpp b/src/hotspot/share/classfile/systemDictionaryShared.hpp index 070f671e7a7..c79821036e0 100644 --- a/src/hotspot/share/classfile/systemDictionaryShared.hpp +++ b/src/hotspot/share/classfile/systemDictionaryShared.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2014, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -145,7 +145,7 @@ class SystemDictionaryShared: public SystemDictionary { RunTimeSharedDictionary _unregistered_dictionary; LambdaProxyClassDictionary _lambda_proxy_class_dictionary; - const RunTimeLambdaProxyClassInfo* lookup_lambda_proxy_class(LambdaProxyClassKey* key) { + const RunTimeLambdaProxyClassInfo* lookup_lambda_proxy_class(RunTimeLambdaProxyClassKey* key) { return _lambda_proxy_class_dictionary.lookup(key, key->hash(), 0); } diff --git a/src/hotspot/share/classfile/vmSymbols.hpp b/src/hotspot/share/classfile/vmSymbols.hpp index d8018cd0c8a..34861b52d54 100644 --- a/src/hotspot/share/classfile/vmSymbols.hpp +++ b/src/hotspot/share/classfile/vmSymbols.hpp @@ -439,10 +439,8 @@ class SerializeClosure; template(getProperty_name, "getProperty") \ template(context_name, "context") \ template(contextClassLoader_name, "contextClassLoader") \ - template(inheritedAccessControlContext_name, "inheritedAccessControlContext") \ template(getClassContext_name, "getClassContext") \ template(wait_name, "wait0") \ - template(checkPackageAccess_name, "checkPackageAccess") \ template(forName_name, "forName") \ template(forName0_name, "forName0") \ template(isJavaIdentifierStart_name, "isJavaIdentifierStart") \ @@ -516,6 +514,8 @@ class SerializeClosure; template(checkIndex_name, "checkIndex") \ template(jfr_epoch_name, "jfr_epoch") \ template(maxThawingSize_name, "maxThawingSize") \ + template(lockStackSize_name, "lockStackSize") \ + template(objectWaiter_name, "objectWaiter") \ \ /* name symbols needed by intrinsics */ \ VM_INTRINSICS_DO(VM_INTRINSIC_IGNORE, VM_SYMBOL_IGNORE, template, VM_SYMBOL_IGNORE, VM_ALIAS_IGNORE) \ @@ -564,6 +564,7 @@ class SerializeClosure; template(continuation_signature, "Ljdk/internal/vm/Continuation;") \ template(continuationscope_signature, "Ljdk/internal/vm/ContinuationScope;") \ template(stackchunk_signature, "Ljdk/internal/vm/StackChunk;") \ + template(vthread_signature, "Ljava/lang/VirtualThread;") \ template(object_void_signature, "(Ljava/lang/Object;)V") \ template(object_int_signature, "(Ljava/lang/Object;)I") \ template(long_object_long_signature, "(JLjava/lang/Object;)J") \ diff --git a/src/hotspot/share/code/debugInfo.cpp b/src/hotspot/share/code/debugInfo.cpp index 5376e8ade9b..ccee142c938 100644 --- a/src/hotspot/share/code/debugInfo.cpp +++ b/src/hotspot/share/code/debugInfo.cpp @@ -382,7 +382,7 @@ void ConstantOopReadValue::print_on(outputStream* st) const { if (value()() != nullptr) { value()()->print_value_on(st); } else { - st->print("nullptr"); + st->print("null"); } } diff --git a/src/hotspot/share/code/nmethod.cpp b/src/hotspot/share/code/nmethod.cpp index 7fb72997749..754a7786605 100644 --- a/src/hotspot/share/code/nmethod.cpp +++ b/src/hotspot/share/code/nmethod.cpp @@ -708,7 +708,8 @@ void nmethod::preserve_callee_argument_oops(frame fr, const RegisterMap *reg_map // handle the case of an anchor explicitly set in continuation code that doesn't have a callee JavaThread* thread = reg_map->thread(); - if (thread->has_last_Java_frame() && fr.sp() == thread->last_Java_sp()) { + if ((thread->has_last_Java_frame() && fr.sp() == thread->last_Java_sp()) + JVMTI_ONLY(|| (method()->is_continuation_enter_intrinsic() && thread->on_monitor_waited_event()))) { return; } @@ -1298,7 +1299,7 @@ nmethod::nmethod( _comp_level = CompLevel_none; _compiler_type = type; _orig_pc_offset = 0; - _num_stack_arg_slots = _method->constMethod()->num_stack_arg_slots(); + _num_stack_arg_slots = 0; if (offsets->value(CodeOffsets::Exceptions) != -1) { // Continuation enter intrinsic diff --git a/src/hotspot/share/compiler/compilerDefinitions.cpp b/src/hotspot/share/compiler/compilerDefinitions.cpp index 7b091d8ade5..ad3d14012ff 100644 --- a/src/hotspot/share/compiler/compilerDefinitions.cpp +++ b/src/hotspot/share/compiler/compilerDefinitions.cpp @@ -560,11 +560,6 @@ void CompilerConfig::ergo_initialize() { if (has_c1()) { if (!is_compilation_mode_selected()) { -#if defined(_WINDOWS) && !defined(_LP64) - if (FLAG_IS_DEFAULT(NeverActAsServerClassMachine)) { - FLAG_SET_ERGO(NeverActAsServerClassMachine, true); - } -#endif if (NeverActAsServerClassMachine) { set_client_emulation_mode_flags(); } @@ -628,4 +623,3 @@ void CompilerConfig::ergo_initialize() { } #endif // COMPILER2 } - diff --git a/src/hotspot/share/gc/g1/g1AnalyticsSequences.hpp b/src/hotspot/share/gc/g1/g1AnalyticsSequences.hpp index 4593c8d934d..dbd13d29b78 100644 --- a/src/hotspot/share/gc/g1/g1AnalyticsSequences.hpp +++ b/src/hotspot/share/gc/g1/g1AnalyticsSequences.hpp @@ -35,6 +35,7 @@ class G1Predictions; // Container for TruncatedSeqs that need separate predictors by GC phase. class G1PhaseDependentSeq { TruncatedSeq _young_only_seq; + double _initial_value; TruncatedSeq _mixed_seq; NONCOPYABLE(G1PhaseDependentSeq); diff --git a/src/hotspot/share/gc/g1/g1AnalyticsSequences.inline.hpp b/src/hotspot/share/gc/g1/g1AnalyticsSequences.inline.hpp index d3520487caa..02a54e2a035 100644 --- a/src/hotspot/share/gc/g1/g1AnalyticsSequences.inline.hpp +++ b/src/hotspot/share/gc/g1/g1AnalyticsSequences.inline.hpp @@ -34,6 +34,7 @@ bool G1PhaseDependentSeq::enough_samples_to_use_mixed_seq() const { G1PhaseDependentSeq::G1PhaseDependentSeq(int length) : _young_only_seq(length), + _initial_value(0.0), _mixed_seq(length) { } @@ -42,7 +43,7 @@ TruncatedSeq* G1PhaseDependentSeq::seq_raw(bool use_young_only_phase_seq) { } void G1PhaseDependentSeq::set_initial(double value) { - _young_only_seq.add(value); + _initial_value = value; } void G1PhaseDependentSeq::add(double value, bool for_young_only_phase) { @@ -51,8 +52,12 @@ void G1PhaseDependentSeq::add(double value, bool for_young_only_phase) { double G1PhaseDependentSeq::predict(const G1Predictions* predictor, bool use_young_only_phase_seq) const { if (use_young_only_phase_seq || !enough_samples_to_use_mixed_seq()) { + if (_young_only_seq.num() == 0) { + return _initial_value; + } return predictor->predict(&_young_only_seq); } else { + assert(_mixed_seq.num() > 0, "must not ask this with no samples"); return predictor->predict(&_mixed_seq); } } diff --git a/src/hotspot/share/gc/g1/g1Arguments.cpp b/src/hotspot/share/gc/g1/g1Arguments.cpp index 3d4ce0d780d..35dfbb7290e 100644 --- a/src/hotspot/share/gc/g1/g1Arguments.cpp +++ b/src/hotspot/share/gc/g1/g1Arguments.cpp @@ -34,6 +34,7 @@ #include "gc/g1/g1HeapRegionRemSet.hpp" #include "gc/g1/g1HeapVerifier.hpp" #include "gc/shared/cardTable.hpp" +#include "gc/shared/fullGCForwarding.hpp" #include "gc/shared/gcArguments.hpp" #include "gc/shared/workerPolicy.hpp" #include "runtime/globals.hpp" @@ -247,6 +248,7 @@ void G1Arguments::initialize() { void G1Arguments::initialize_heap_flags_and_sizes() { GCArguments::initialize_heap_flags_and_sizes(); + FullGCForwarding::initialize_flags(heap_reserved_size_bytes()); } CollectedHeap* G1Arguments::create_heap() { diff --git a/src/hotspot/share/gc/g1/g1CollectedHeap.cpp b/src/hotspot/share/gc/g1/g1CollectedHeap.cpp index 57236d6f6db..daf7eb5371b 100644 --- a/src/hotspot/share/gc/g1/g1CollectedHeap.cpp +++ b/src/hotspot/share/gc/g1/g1CollectedHeap.cpp @@ -77,6 +77,7 @@ #include "gc/g1/g1YoungGCAllocationFailureInjector.hpp" #include "gc/shared/classUnloadingContext.hpp" #include "gc/shared/concurrentGCBreakpoints.hpp" +#include "gc/shared/fullGCForwarding.hpp" #include "gc/shared/gcBehaviours.hpp" #include "gc/shared/gcHeapSummary.hpp" #include "gc/shared/gcId.hpp" @@ -85,7 +86,6 @@ #include "gc/shared/isGCActiveMark.hpp" #include "gc/shared/locationPrinter.inline.hpp" #include "gc/shared/oopStorageParState.hpp" -#include "gc/shared/preservedMarks.inline.hpp" #include "gc/shared/referenceProcessor.inline.hpp" #include "gc/shared/suspendibleThreadSet.hpp" #include "gc/shared/taskqueue.inline.hpp" @@ -1435,6 +1435,8 @@ jint G1CollectedHeap::initialize() { G1InitLogger::print(); + FullGCForwarding::initialize(heap_rs.region()); + return JNI_OK; } diff --git a/src/hotspot/share/gc/g1/g1FullGCCompactTask.cpp b/src/hotspot/share/gc/g1/g1FullGCCompactTask.cpp index 05f66959243..bee3656ead5 100644 --- a/src/hotspot/share/gc/g1/g1FullGCCompactTask.cpp +++ b/src/hotspot/share/gc/g1/g1FullGCCompactTask.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2017, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -29,6 +29,7 @@ #include "gc/g1/g1FullGCCompactionPoint.hpp" #include "gc/g1/g1FullGCCompactTask.hpp" #include "gc/g1/g1HeapRegion.inline.hpp" +#include "gc/shared/fullGCForwarding.inline.hpp" #include "gc/shared/gcTraceTime.inline.hpp" #include "logging/log.hpp" #include "oops/oop.inline.hpp" @@ -41,7 +42,7 @@ void G1FullGCCompactTask::G1CompactRegionClosure::clear_in_bitmap(oop obj) { size_t G1FullGCCompactTask::G1CompactRegionClosure::apply(oop obj) { size_t size = obj->size(); - if (obj->is_forwarded()) { + if (FullGCForwarding::is_forwarded(obj)) { G1FullGCCompactTask::copy_object_to_new_location(obj); } @@ -52,13 +53,13 @@ size_t G1FullGCCompactTask::G1CompactRegionClosure::apply(oop obj) { } void G1FullGCCompactTask::copy_object_to_new_location(oop obj) { - assert(obj->is_forwarded(), "Sanity!"); - assert(obj->forwardee() != obj, "Object must have a new location"); + assert(FullGCForwarding::is_forwarded(obj), "Sanity!"); + assert(FullGCForwarding::forwardee(obj) != obj, "Object must have a new location"); size_t size = obj->size(); // Copy object and reinit its mark. HeapWord* obj_addr = cast_from_oop(obj); - HeapWord* destination = cast_from_oop(obj->forwardee()); + HeapWord* destination = cast_from_oop(FullGCForwarding::forwardee(obj)); Copy::aligned_conjoint_words(obj_addr, destination, size); // There is no need to transform stack chunks - marking already did that. @@ -121,7 +122,7 @@ void G1FullGCCompactTask::compact_humongous_obj(G1HeapRegion* src_hr) { size_t word_size = obj->size(); uint num_regions = (uint)G1CollectedHeap::humongous_obj_size_in_regions(word_size); - HeapWord* destination = cast_from_oop(obj->forwardee()); + HeapWord* destination = cast_from_oop(FullGCForwarding::forwardee(obj)); assert(collector()->mark_bitmap()->is_marked(obj), "Should only compact marked objects"); collector()->mark_bitmap()->clear(obj); diff --git a/src/hotspot/share/gc/g1/g1FullGCCompactionPoint.cpp b/src/hotspot/share/gc/g1/g1FullGCCompactionPoint.cpp index 019484c810a..ddd1b7c0999 100644 --- a/src/hotspot/share/gc/g1/g1FullGCCompactionPoint.cpp +++ b/src/hotspot/share/gc/g1/g1FullGCCompactionPoint.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2017, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -26,6 +26,7 @@ #include "gc/g1/g1FullCollector.inline.hpp" #include "gc/g1/g1FullGCCompactionPoint.hpp" #include "gc/g1/g1HeapRegion.hpp" +#include "gc/shared/fullGCForwarding.inline.hpp" #include "gc/shared/preservedMarks.inline.hpp" #include "oops/oop.inline.hpp" #include "utilities/debug.hpp" @@ -106,10 +107,10 @@ void G1FullGCCompactionPoint::forward(oop object, size_t size) { if (!object->is_forwarded()) { preserved_stack()->push_if_necessary(object, object->mark()); } - object->forward_to(cast_to_oop(_compaction_top)); - assert(object->is_forwarded(), "must be forwarded"); + FullGCForwarding::forward_to(object, cast_to_oop(_compaction_top)); + assert(FullGCForwarding::is_forwarded(object), "must be forwarded"); } else { - assert(!object->is_forwarded(), "must not be forwarded"); + assert(!FullGCForwarding::is_forwarded(object), "must not be forwarded"); } // Update compaction values. @@ -172,8 +173,8 @@ void G1FullGCCompactionPoint::forward_humongous(G1HeapRegion* hr) { preserved_stack()->push_if_necessary(obj, obj->mark()); G1HeapRegion* dest_hr = _compaction_regions->at(range_begin); - obj->forward_to(cast_to_oop(dest_hr->bottom())); - assert(obj->is_forwarded(), "Object must be forwarded!"); + FullGCForwarding::forward_to(obj, cast_to_oop(dest_hr->bottom())); + assert(FullGCForwarding::is_forwarded(obj), "Object must be forwarded!"); // Add the humongous object regions to the compaction point. add_humongous(hr); diff --git a/src/hotspot/share/gc/g1/g1FullGCOopClosures.inline.hpp b/src/hotspot/share/gc/g1/g1FullGCOopClosures.inline.hpp index f10f884b242..b20593ac290 100644 --- a/src/hotspot/share/gc/g1/g1FullGCOopClosures.inline.hpp +++ b/src/hotspot/share/gc/g1/g1FullGCOopClosures.inline.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, 2022, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2017, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -32,6 +32,7 @@ #include "gc/g1/g1FullCollector.inline.hpp" #include "gc/g1/g1FullGCMarker.inline.hpp" #include "gc/g1/g1HeapRegionRemSet.inline.hpp" +#include "gc/shared/fullGCForwarding.inline.hpp" #include "memory/iterator.inline.hpp" #include "memory/universe.hpp" #include "oops/access.inline.hpp" @@ -65,8 +66,8 @@ template inline void G1AdjustClosure::adjust_pointer(T* p) { return; } - if (obj->is_forwarded()) { - oop forwardee = obj->forwardee(); + if (FullGCForwarding::is_forwarded(obj)) { + oop forwardee = FullGCForwarding::forwardee(obj); // Forwarded, just update. assert(G1CollectedHeap::heap()->is_in_reserved(forwardee), "should be in object space"); RawAccess::oop_store(p, forwardee); diff --git a/src/hotspot/share/gc/g1/g1FullGCPrepareTask.inline.hpp b/src/hotspot/share/gc/g1/g1FullGCPrepareTask.inline.hpp index 2fb61cc5934..1d2a023ed3a 100644 --- a/src/hotspot/share/gc/g1/g1FullGCPrepareTask.inline.hpp +++ b/src/hotspot/share/gc/g1/g1FullGCPrepareTask.inline.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2022, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -32,6 +32,7 @@ #include "gc/g1/g1FullGCCompactionPoint.hpp" #include "gc/g1/g1FullGCScope.hpp" #include "gc/g1/g1HeapRegion.inline.hpp" +#include "gc/shared/fullGCForwarding.inline.hpp" void G1DetermineCompactionQueueClosure::free_empty_humongous_region(G1HeapRegion* hr) { _g1h->free_humongous_region(hr, nullptr); @@ -114,10 +115,10 @@ inline bool G1DetermineCompactionQueueClosure::do_heap_region(G1HeapRegion* hr) } inline size_t G1SerialRePrepareClosure::apply(oop obj) { - if (obj->is_forwarded()) { + if (FullGCForwarding::is_forwarded(obj)) { // We skip objects compiled into the first region or // into regions not part of the serial compaction point. - if (cast_from_oop(obj->forwardee()) < _dense_prefix_top) { + if (cast_from_oop(FullGCForwarding::forwardee(obj)) < _dense_prefix_top) { return obj->size(); } } diff --git a/src/hotspot/share/gc/g1/g1GCPhaseTimes.cpp b/src/hotspot/share/gc/g1/g1GCPhaseTimes.cpp index 32a56d71205..9e48a16018e 100644 --- a/src/hotspot/share/gc/g1/g1GCPhaseTimes.cpp +++ b/src/hotspot/share/gc/g1/g1GCPhaseTimes.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2013, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -105,7 +105,6 @@ G1GCPhaseTimes::G1GCPhaseTimes(STWGCTimer* gc_timer, uint max_gc_threads) : _gc_par_phases[UpdateDerivedPointers] = new WorkerDataArray("UpdateDerivedPointers", "Update Derived Pointers (ms):", max_gc_threads); #endif _gc_par_phases[EagerlyReclaimHumongousObjects] = new WorkerDataArray("EagerlyReclaimHumongousObjects", "Eagerly Reclaim Humongous Objects (ms):", max_gc_threads); - _gc_par_phases[RestorePreservedMarks] = new WorkerDataArray("RestorePreservedMarks", "Restore Preserved Marks (ms):", max_gc_threads); _gc_par_phases[ProcessEvacuationFailedRegions] = new WorkerDataArray("ProcessEvacuationFailedRegions", "Process Evacuation Failed Regions (ms):", max_gc_threads); _gc_par_phases[ScanHR]->create_thread_work_items("Scanned Cards:", ScanHRScannedCards); @@ -512,7 +511,6 @@ double G1GCPhaseTimes::print_post_evacuate_collection_set(bool evacuation_failed debug_time("Post Evacuate Cleanup 2", _cur_post_evacuate_cleanup_2_time_ms); if (evacuation_failed) { debug_phase(_gc_par_phases[RecalculateUsed], 1); - debug_phase(_gc_par_phases[RestorePreservedMarks], 1); debug_phase(_gc_par_phases[ProcessEvacuationFailedRegions], 1); } #if COMPILER2_OR_JVMCI diff --git a/src/hotspot/share/gc/g1/g1GCPhaseTimes.hpp b/src/hotspot/share/gc/g1/g1GCPhaseTimes.hpp index 40abfd60533..a54ef431abd 100644 --- a/src/hotspot/share/gc/g1/g1GCPhaseTimes.hpp +++ b/src/hotspot/share/gc/g1/g1GCPhaseTimes.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2013, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -87,7 +87,6 @@ class G1GCPhaseTimes : public CHeapObj { UpdateDerivedPointers, #endif EagerlyReclaimHumongousObjects, - RestorePreservedMarks, ProcessEvacuationFailedRegions, ResetMarkingState, NoteStartOfMark, diff --git a/src/hotspot/share/gc/g1/g1OopClosures.inline.hpp b/src/hotspot/share/gc/g1/g1OopClosures.inline.hpp index 26cc88de0be..5b3bbedfeb2 100644 --- a/src/hotspot/share/gc/g1/g1OopClosures.inline.hpp +++ b/src/hotspot/share/gc/g1/g1OopClosures.inline.hpp @@ -228,7 +228,7 @@ void G1ParCopyClosure::do_oop_work(T* p) { oop forwardee; markWord m = obj->mark(); if (m.is_forwarded()) { - forwardee = m.forwardee(); + forwardee = obj->forwardee(m); } else { forwardee = _par_scan_state->copy_to_survivor_space(state, obj, m); } diff --git a/src/hotspot/share/gc/g1/g1ParScanThreadState.cpp b/src/hotspot/share/gc/g1/g1ParScanThreadState.cpp index f81c3241a1a..ad924b2fad4 100644 --- a/src/hotspot/share/gc/g1/g1ParScanThreadState.cpp +++ b/src/hotspot/share/gc/g1/g1ParScanThreadState.cpp @@ -37,7 +37,6 @@ #include "gc/shared/continuationGCSupport.inline.hpp" #include "gc/shared/partialArrayState.hpp" #include "gc/shared/partialArrayTaskStepper.inline.hpp" -#include "gc/shared/preservedMarks.inline.hpp" #include "gc/shared/stringdedup/stringDedup.hpp" #include "gc/shared/taskqueue.inline.hpp" #include "memory/allocation.inline.hpp" @@ -59,7 +58,6 @@ G1ParScanThreadState::G1ParScanThreadState(G1CollectedHeap* g1h, G1RedirtyCardsQueueSet* rdcqs, - PreservedMarks* preserved_marks, uint worker_id, uint num_workers, G1CollectionSet* collection_set, @@ -90,7 +88,6 @@ G1ParScanThreadState::G1ParScanThreadState(G1CollectedHeap* g1h, _numa(g1h->numa()), _obj_alloc_stat(nullptr), ALLOCATION_FAILURE_INJECTOR_ONLY(_allocation_failure_inject_counter(0) COMMA) - _preserved_marks(preserved_marks), _evacuation_failed_info(), _evac_failure_regions(evac_failure_regions), _evac_failure_enqueued_cards(0) @@ -216,7 +213,7 @@ void G1ParScanThreadState::do_oop_evac(T* p) { markWord m = obj->mark(); if (m.is_forwarded()) { - obj = m.forwardee(); + obj = obj->forwardee(m); } else { obj = do_copy_to_survivor_space(region_attr, obj, m); } @@ -232,7 +229,6 @@ void G1ParScanThreadState::do_partial_array(PartialArrayState* state) { #ifdef ASSERT oop from_obj = state->source(); assert(_g1h->is_in_reserved(from_obj), "must be in heap."); - assert(from_obj->is_objArray(), "must be obj array"); assert(from_obj->is_forwarded(), "must be forwarded"); assert(from_obj != to_obj, "should not be chunking self-forwarded objects"); assert(to_obj->is_objArray(), "must be obj array"); @@ -265,7 +261,6 @@ MAYBE_INLINE_EVACUATION void G1ParScanThreadState::start_partial_objarray(G1HeapRegionAttr dest_attr, oop from_obj, oop to_obj) { - assert(from_obj->is_objArray(), "precondition"); assert(from_obj->is_forwarded(), "precondition"); assert(from_obj->forwardee() == to_obj, "precondition"); assert(to_obj->is_objArray(), "precondition"); @@ -401,22 +396,22 @@ G1HeapRegionAttr G1ParScanThreadState::next_region_attr(G1HeapRegionAttr const r } void G1ParScanThreadState::report_promotion_event(G1HeapRegionAttr const dest_attr, - oop const old, size_t word_sz, uint age, + Klass* klass, size_t word_sz, uint age, HeapWord * const obj_ptr, uint node_index) const { PLAB* alloc_buf = _plab_allocator->alloc_buffer(dest_attr, node_index); if (alloc_buf->contains(obj_ptr)) { - _g1h->gc_tracer_stw()->report_promotion_in_new_plab_event(old->klass(), word_sz * HeapWordSize, age, + _g1h->gc_tracer_stw()->report_promotion_in_new_plab_event(klass, word_sz * HeapWordSize, age, dest_attr.type() == G1HeapRegionAttr::Old, alloc_buf->word_sz() * HeapWordSize); } else { - _g1h->gc_tracer_stw()->report_promotion_outside_plab_event(old->klass(), word_sz * HeapWordSize, age, + _g1h->gc_tracer_stw()->report_promotion_outside_plab_event(klass, word_sz * HeapWordSize, age, dest_attr.type() == G1HeapRegionAttr::Old); } } NOINLINE HeapWord* G1ParScanThreadState::allocate_copy_slow(G1HeapRegionAttr* dest_attr, - oop old, + Klass* klass, size_t word_sz, uint age, uint node_index) { @@ -439,7 +434,7 @@ HeapWord* G1ParScanThreadState::allocate_copy_slow(G1HeapRegionAttr* dest_attr, update_numa_stats(node_index); if (_g1h->gc_tracer_stw()->should_report_promotion_events()) { // The events are checked individually as part of the actual commit - report_promotion_event(*dest_attr, old, word_sz, age, obj_ptr, node_index); + report_promotion_event(*dest_attr, klass, word_sz, age, obj_ptr, node_index); } } return obj_ptr; @@ -474,9 +469,17 @@ oop G1ParScanThreadState::do_copy_to_survivor_space(G1HeapRegionAttr const regio assert(region_attr.is_in_cset(), "Unexpected region attr type: %s", region_attr.get_type_str()); - // Get the klass once. We'll need it again later, and this avoids - // re-decoding when it's compressed. - Klass* klass = old->klass(); + // NOTE: With compact headers, it is not safe to load the Klass* from old, because + // that would access the mark-word, that might change at any time by concurrent + // workers. + // This mark word would refer to a forwardee, which may not yet have completed + // copying. Therefore we must load the Klass* from the mark-word that we already + // loaded. This is safe, because we only enter here if not yet forwarded. + assert(!old_mark.is_forwarded(), "precondition"); + Klass* klass = UseCompactObjectHeaders + ? old_mark.klass() + : old->klass(); + const size_t word_sz = old->size_given_klass(klass); // JNI only allows pinning of typeArrays, so we only need to keep those in place. @@ -494,7 +497,7 @@ oop G1ParScanThreadState::do_copy_to_survivor_space(G1HeapRegionAttr const regio // PLAB allocations should succeed most of the time, so we'll // normally check against null once and that's it. if (obj_ptr == nullptr) { - obj_ptr = allocate_copy_slow(&dest_attr, old, word_sz, age, node_index); + obj_ptr = allocate_copy_slow(&dest_attr, klass, word_sz, age, node_index); if (obj_ptr == nullptr) { // This will either forward-to-self, or detect that someone else has // installed a forwarding pointer. @@ -595,7 +598,6 @@ G1ParScanThreadState* G1ParScanThreadStateSet::state_for_worker(uint worker_id) if (_states[worker_id] == nullptr) { _states[worker_id] = new G1ParScanThreadState(_g1h, rdcqs(), - _preserved_marks_set.get(worker_id), worker_id, _num_workers, _collection_set, @@ -655,7 +657,7 @@ NOINLINE oop G1ParScanThreadState::handle_evacuation_failure_par(oop old, markWord m, size_t word_sz, bool cause_pinned) { assert(_g1h->is_in_cset(old), "Object " PTR_FORMAT " should be in the CSet", p2i(old)); - oop forward_ptr = old->forward_to_atomic(old, m, memory_order_relaxed); + oop forward_ptr = old->forward_to_self_atomic(m, memory_order_relaxed); if (forward_ptr == nullptr) { // Forward-to-self succeeded. We are the "owner" of the object. G1HeapRegion* r = _g1h->heap_region_containing(old); @@ -668,8 +670,6 @@ oop G1ParScanThreadState::handle_evacuation_failure_par(oop old, markWord m, siz // evacuation failure recovery. _g1h->mark_evac_failure_object(_worker_id, old, word_sz); - _preserved_marks->push_if_necessary(old, m); - ContinuationGCSupport::transform_stack_chunk(old); _evacuation_failed_info.register_copy_failure(word_sz); @@ -727,7 +727,6 @@ G1ParScanThreadStateSet::G1ParScanThreadStateSet(G1CollectedHeap* g1h, _g1h(g1h), _collection_set(collection_set), _rdcqs(G1BarrierSet::dirty_card_queue_set().allocator()), - _preserved_marks_set(true /* in_c_heap */), _states(NEW_C_HEAP_ARRAY(G1ParScanThreadState*, num_workers, mtGC)), _rdc_buffers(NEW_C_HEAP_ARRAY(BufferNodeList, num_workers, mtGC)), _surviving_young_words_total(NEW_C_HEAP_ARRAY(size_t, collection_set->young_region_length() + 1, mtGC)), @@ -736,7 +735,6 @@ G1ParScanThreadStateSet::G1ParScanThreadStateSet(G1CollectedHeap* g1h, _evac_failure_regions(evac_failure_regions), _partial_array_state_allocator(num_workers) { - _preserved_marks_set.init(num_workers); for (uint i = 0; i < num_workers; ++i) { _states[i] = nullptr; _rdc_buffers[i] = BufferNodeList(); @@ -749,5 +747,4 @@ G1ParScanThreadStateSet::~G1ParScanThreadStateSet() { FREE_C_HEAP_ARRAY(G1ParScanThreadState*, _states); FREE_C_HEAP_ARRAY(size_t, _surviving_young_words_total); FREE_C_HEAP_ARRAY(BufferNodeList, _rdc_buffers); - _preserved_marks_set.reclaim(); } diff --git a/src/hotspot/share/gc/g1/g1ParScanThreadState.hpp b/src/hotspot/share/gc/g1/g1ParScanThreadState.hpp index 3f7fefd8a07..f61f993f028 100644 --- a/src/hotspot/share/gc/g1/g1ParScanThreadState.hpp +++ b/src/hotspot/share/gc/g1/g1ParScanThreadState.hpp @@ -34,7 +34,6 @@ #include "gc/shared/gc_globals.hpp" #include "gc/shared/partialArrayState.hpp" #include "gc/shared/partialArrayTaskStepper.hpp" -#include "gc/shared/preservedMarks.hpp" #include "gc/shared/stringdedup/stringDedup.hpp" #include "gc/shared/taskqueue.hpp" #include "memory/allocation.hpp" @@ -48,8 +47,6 @@ class G1EvacuationRootClosures; class G1OopStarChunkedList; class G1PLABAllocator; class G1HeapRegion; -class PreservedMarks; -class PreservedMarksSet; class outputStream; class G1ParScanThreadState : public CHeapObj { @@ -106,7 +103,6 @@ class G1ParScanThreadState : public CHeapObj { // Per-thread evacuation failure data structures. ALLOCATION_FAILURE_INJECTOR_ONLY(size_t _allocation_failure_inject_counter;) - PreservedMarks* _preserved_marks; EvacuationFailedInfo _evacuation_failed_info; G1EvacFailureRegions* _evac_failure_regions; // Number of additional cards into evacuation failed regions enqueued into @@ -125,7 +121,6 @@ class G1ParScanThreadState : public CHeapObj { public: G1ParScanThreadState(G1CollectedHeap* g1h, G1RedirtyCardsQueueSet* rdcqs, - PreservedMarks* preserved_marks, uint worker_id, uint num_workers, G1CollectionSet* collection_set, @@ -174,7 +169,7 @@ class G1ParScanThreadState : public CHeapObj { void start_partial_objarray(G1HeapRegionAttr dest_dir, oop from, oop to); HeapWord* allocate_copy_slow(G1HeapRegionAttr* dest_attr, - oop old, + Klass* klass, size_t word_sz, uint age, uint node_index); @@ -209,7 +204,7 @@ class G1ParScanThreadState : public CHeapObj { inline G1HeapRegionAttr next_region_attr(G1HeapRegionAttr const region_attr, markWord const m, uint& age); void report_promotion_event(G1HeapRegionAttr const dest_attr, - oop const old, size_t word_sz, uint age, + Klass* klass, size_t word_sz, uint age, HeapWord * const obj_ptr, uint node_index) const; void trim_queue_to_threshold(uint threshold); @@ -246,7 +241,6 @@ class G1ParScanThreadStateSet : public StackObj { G1CollectedHeap* _g1h; G1CollectionSet* _collection_set; G1RedirtyCardsQueueSet _rdcqs; - PreservedMarksSet _preserved_marks_set; G1ParScanThreadState** _states; BufferNodeList* _rdc_buffers; size_t* _surviving_young_words_total; @@ -264,7 +258,6 @@ class G1ParScanThreadStateSet : public StackObj { G1RedirtyCardsQueueSet* rdcqs() { return &_rdcqs; } BufferNodeList* rdc_buffers() { return _rdc_buffers; } - PreservedMarksSet* preserved_marks_set() { return &_preserved_marks_set; } void flush_stats(); void record_unused_optional_region(G1HeapRegion* hr); diff --git a/src/hotspot/share/gc/g1/g1YoungCollector.cpp b/src/hotspot/share/gc/g1/g1YoungCollector.cpp index f2fe93015c5..f3590aa2ff6 100644 --- a/src/hotspot/share/gc/g1/g1YoungCollector.cpp +++ b/src/hotspot/share/gc/g1/g1YoungCollector.cpp @@ -53,7 +53,6 @@ #include "gc/shared/gcTraceTime.inline.hpp" #include "gc/shared/gcTimer.hpp" #include "gc/shared/gc_globals.hpp" -#include "gc/shared/preservedMarks.hpp" #include "gc/shared/referenceProcessor.hpp" #include "gc/shared/weakProcessor.inline.hpp" #include "gc/shared/workerPolicy.hpp" diff --git a/src/hotspot/share/gc/g1/g1YoungGCPostEvacuateTasks.cpp b/src/hotspot/share/gc/g1/g1YoungGCPostEvacuateTasks.cpp index a0e9a9b1569..1d76a44f8f8 100644 --- a/src/hotspot/share/gc/g1/g1YoungGCPostEvacuateTasks.cpp +++ b/src/hotspot/share/gc/g1/g1YoungGCPostEvacuateTasks.cpp @@ -42,7 +42,6 @@ #include "gc/g1/g1RemSet.hpp" #include "gc/g1/g1YoungGCPostEvacuateTasks.hpp" #include "gc/shared/bufferNode.hpp" -#include "gc/shared/preservedMarks.inline.hpp" #include "jfr/jfrEvents.hpp" #include "oops/access.inline.hpp" #include "oops/compressedOops.inline.hpp" @@ -251,8 +250,8 @@ class G1PostEvacuateCollectionSetCleanupTask1::RestoreEvacFailureRegionsTask : p { // Process marked object. - assert(obj->is_forwarded() && obj->forwardee() == obj, "must be self-forwarded"); - obj->init_mark(); + assert(obj->is_self_forwarded(), "must be self-forwarded"); + obj->unset_self_forwarded(); hr->update_bot_for_block(obj_addr, obj_end_addr); // Statistics @@ -477,27 +476,6 @@ class G1PostEvacuateCollectionSetCleanupTask2::EagerlyReclaimHumongousObjectsTas } }; -class G1PostEvacuateCollectionSetCleanupTask2::RestorePreservedMarksTask : public G1AbstractSubTask { - PreservedMarksSet* _preserved_marks; - WorkerTask* _task; - -public: - RestorePreservedMarksTask(PreservedMarksSet* preserved_marks) : - G1AbstractSubTask(G1GCPhaseTimes::RestorePreservedMarks), - _preserved_marks(preserved_marks), - _task(preserved_marks->create_task()) { } - - virtual ~RestorePreservedMarksTask() { - delete _task; - } - - double worker_cost() const override { - return _preserved_marks->num(); - } - - void do_work(uint worker_id) override { _task->work(worker_id); } -}; - class RedirtyLoggedCardTableEntryClosure : public G1CardTableEntryClosure { size_t _num_dirtied; G1CollectedHeap* _g1h; @@ -979,7 +957,6 @@ G1PostEvacuateCollectionSetCleanupTask2::G1PostEvacuateCollectionSetCleanupTask2 } if (evac_failure_regions->has_regions_evac_failed()) { - add_parallel_task(new RestorePreservedMarksTask(per_thread_states->preserved_marks_set())); add_parallel_task(new ProcessEvacuationFailedRegionsTask(evac_failure_regions)); } add_parallel_task(new RedirtyLoggedCardsTask(evac_failure_regions, diff --git a/src/hotspot/share/gc/g1/g1YoungGCPostEvacuateTasks.hpp b/src/hotspot/share/gc/g1/g1YoungGCPostEvacuateTasks.hpp index 868ab788b53..96eeaf27de1 100644 --- a/src/hotspot/share/gc/g1/g1YoungGCPostEvacuateTasks.hpp +++ b/src/hotspot/share/gc/g1/g1YoungGCPostEvacuateTasks.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2021, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -56,7 +56,6 @@ class G1PostEvacuateCollectionSetCleanupTask1 : public G1BatchedTask { // - Update Derived Pointers (s) // - Clear Retained Region Data (on evacuation failure) // - Redirty Logged Cards -// - Restore Preserved Marks (on evacuation failure) // - Free Collection Set // - Resize TLABs class G1PostEvacuateCollectionSetCleanupTask2 : public G1BatchedTask { @@ -67,7 +66,6 @@ class G1PostEvacuateCollectionSetCleanupTask2 : public G1BatchedTask { class ProcessEvacuationFailedRegionsTask; class RedirtyLoggedCardsTask; - class RestorePreservedMarksTask; class FreeCollectionSetTask; class ResizeTLABsTask; diff --git a/src/hotspot/share/gc/parallel/mutableSpace.cpp b/src/hotspot/share/gc/parallel/mutableSpace.cpp index 01d4e6bb04d..74801f4870b 100644 --- a/src/hotspot/share/gc/parallel/mutableSpace.cpp +++ b/src/hotspot/share/gc/parallel/mutableSpace.cpp @@ -218,15 +218,16 @@ void MutableSpace::object_iterate(ObjectClosure* cl) { // When promotion-failure occurs during Young GC, eden/from space is not cleared, // so we can encounter objects with "forwarded" markword. // They are essentially dead, so skipping them - if (!obj->is_forwarded()) { + if (obj->is_forwarded()) { + assert(!obj->is_self_forwarded(), "must not be self-forwarded"); + // It is safe to use the forwardee here. Parallel GC only uses + // header-based forwarding during promotion. Full GC doesn't + // use the object header for forwarding at all. + p += obj->forwardee()->size(); + } else { cl->do_object(obj); + p += obj->size(); } -#ifdef ASSERT - else { - assert(obj->forwardee() != obj, "must not be self-forwarded"); - } -#endif - p += obj->size(); } } diff --git a/src/hotspot/share/gc/parallel/parallelArguments.cpp b/src/hotspot/share/gc/parallel/parallelArguments.cpp index 313716752d5..4035259e6d6 100644 --- a/src/hotspot/share/gc/parallel/parallelArguments.cpp +++ b/src/hotspot/share/gc/parallel/parallelArguments.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2018, 2024, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2017, Red Hat, Inc. and/or its affiliates. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * @@ -27,6 +27,7 @@ #include "gc/parallel/parallelArguments.hpp" #include "gc/parallel/parallelScavengeHeap.hpp" #include "gc/shared/adaptiveSizePolicy.hpp" +#include "gc/shared/fullGCForwarding.hpp" #include "gc/shared/gcArguments.hpp" #include "gc/shared/genArguments.hpp" #include "gc/shared/workerPolicy.hpp" @@ -127,6 +128,7 @@ void ParallelArguments::initialize_heap_flags_and_sizes() { // Redo everything from the start initialize_heap_flags_and_sizes_one_pass(); } + FullGCForwarding::initialize_flags(heap_reserved_size_bytes()); } size_t ParallelArguments::heap_reserved_size_bytes() { diff --git a/src/hotspot/share/gc/parallel/parallelScavengeHeap.cpp b/src/hotspot/share/gc/parallel/parallelScavengeHeap.cpp index 5883b1cd607..52d031b6dd2 100644 --- a/src/hotspot/share/gc/parallel/parallelScavengeHeap.cpp +++ b/src/hotspot/share/gc/parallel/parallelScavengeHeap.cpp @@ -33,6 +33,7 @@ #include "gc/parallel/psPromotionManager.hpp" #include "gc/parallel/psScavenge.hpp" #include "gc/parallel/psVMOperations.hpp" +#include "gc/shared/fullGCForwarding.inline.hpp" #include "gc/shared/gcHeapSummary.hpp" #include "gc/shared/gcLocker.inline.hpp" #include "gc/shared/gcWhen.hpp" @@ -129,6 +130,8 @@ jint ParallelScavengeHeap::initialize() { ParallelInitLogger::print(); + FullGCForwarding::initialize(heap_rs.region()); + return JNI_OK; } diff --git a/src/hotspot/share/gc/parallel/psParallelCompact.cpp b/src/hotspot/share/gc/parallel/psParallelCompact.cpp index f6ef195abfc..e90404b6502 100644 --- a/src/hotspot/share/gc/parallel/psParallelCompact.cpp +++ b/src/hotspot/share/gc/parallel/psParallelCompact.cpp @@ -44,6 +44,7 @@ #include "gc/parallel/psStringDedup.hpp" #include "gc/parallel/psYoungGen.hpp" #include "gc/shared/classUnloadingContext.hpp" +#include "gc/shared/fullGCForwarding.inline.hpp" #include "gc/shared/gcCause.hpp" #include "gc/shared/gcHeapSummary.hpp" #include "gc/shared/gcId.hpp" @@ -127,73 +128,84 @@ ParallelCompactData::RegionData::dc_claimed = 0x8U << dc_shift; const ParallelCompactData::RegionData::region_sz_t ParallelCompactData::RegionData::dc_completed = 0xcU << dc_shift; +bool ParallelCompactData::RegionData::is_clear() { + return (_destination == nullptr) && + (_source_region == 0) && + (_partial_obj_addr == nullptr) && + (_partial_obj_size == 0) && + (_dc_and_los == 0) && + (_shadow_state == 0); +} + +#ifdef ASSERT +void ParallelCompactData::RegionData::verify_clear() { + assert(_destination == nullptr, "inv"); + assert(_source_region == 0, "inv"); + assert(_partial_obj_addr == nullptr, "inv"); + assert(_partial_obj_size == 0, "inv"); + assert(_dc_and_los == 0, "inv"); + assert(_shadow_state == 0, "inv"); +} +#endif + SpaceInfo PSParallelCompact::_space_info[PSParallelCompact::last_space_id]; SpanSubjectToDiscoveryClosure PSParallelCompact::_span_based_discoverer; ReferenceProcessor* PSParallelCompact::_ref_processor = nullptr; -void SplitInfo::record(size_t src_region_idx, size_t partial_obj_size, - HeapWord* destination) -{ - assert(src_region_idx != 0, "invalid src_region_idx"); - assert(partial_obj_size != 0, "invalid partial_obj_size argument"); - assert(destination != nullptr, "invalid destination argument"); - - _src_region_idx = src_region_idx; - _partial_obj_size = partial_obj_size; - _destination = destination; +void SplitInfo::record(size_t split_region_idx, HeapWord* split_point, size_t preceding_live_words) { + assert(split_region_idx != 0, "precondition"); - // These fields may not be updated below, so make sure they're clear. - assert(_dest_region_addr == nullptr, "should have been cleared"); - assert(_first_src_addr == nullptr, "should have been cleared"); + // Obj denoted by split_point will be deferred to the next space. + assert(split_point != nullptr, "precondition"); - // Determine the number of destination regions for the partial object. - HeapWord* const last_word = destination + partial_obj_size - 1; const ParallelCompactData& sd = PSParallelCompact::summary_data(); - HeapWord* const beg_region_addr = sd.region_align_down(destination); - HeapWord* const end_region_addr = sd.region_align_down(last_word); - - if (beg_region_addr == end_region_addr) { - // One destination region. - _destination_count = 1; - if (end_region_addr == destination) { - // The destination falls on a region boundary, thus the first word of the - // partial object will be the first word copied to the destination region. - _dest_region_addr = end_region_addr; - _first_src_addr = sd.region_to_addr(src_region_idx); - } + + PSParallelCompact::RegionData* split_region_ptr = sd.region(split_region_idx); + assert(preceding_live_words < split_region_ptr->data_size(), "inv"); + + HeapWord* preceding_destination = split_region_ptr->destination(); + assert(preceding_destination != nullptr, "inv"); + + // How many regions does the preceding part occupy + uint preceding_destination_count; + if (preceding_live_words == 0) { + preceding_destination_count = 0; } else { - // Two destination regions. When copied, the partial object will cross a - // destination region boundary, so a word somewhere within the partial - // object will be the first word copied to the second destination region. - _destination_count = 2; - _dest_region_addr = end_region_addr; - const size_t ofs = pointer_delta(end_region_addr, destination); - assert(ofs < _partial_obj_size, "sanity"); - _first_src_addr = sd.region_to_addr(src_region_idx) + ofs; + // -1 so that the ending address doesn't fall on the region-boundary + if (sd.region_align_down(preceding_destination) == + sd.region_align_down(preceding_destination + preceding_live_words - 1)) { + preceding_destination_count = 1; + } else { + preceding_destination_count = 2; + } } + + _split_region_idx = split_region_idx; + _split_point = split_point; + _preceding_live_words = preceding_live_words; + _preceding_destination = preceding_destination; + _preceding_destination_count = preceding_destination_count; } void SplitInfo::clear() { - _src_region_idx = 0; - _partial_obj_size = 0; - _destination = nullptr; - _destination_count = 0; - _dest_region_addr = nullptr; - _first_src_addr = nullptr; + _split_region_idx = 0; + _split_point = nullptr; + _preceding_live_words = 0; + _preceding_destination = nullptr; + _preceding_destination_count = 0; assert(!is_valid(), "sanity"); } #ifdef ASSERT void SplitInfo::verify_clear() { - assert(_src_region_idx == 0, "not clear"); - assert(_partial_obj_size == 0, "not clear"); - assert(_destination == nullptr, "not clear"); - assert(_destination_count == 0, "not clear"); - assert(_dest_region_addr == nullptr, "not clear"); - assert(_first_src_addr == nullptr, "not clear"); + assert(_split_region_idx == 0, "not clear"); + assert(_split_point == nullptr, "not clear"); + assert(_preceding_live_words == 0, "not clear"); + assert(_preceding_destination == nullptr, "not clear"); + assert(_preceding_destination_count == 0, "not clear"); } #endif // #ifdef ASSERT @@ -296,110 +308,105 @@ ParallelCompactData::summarize_dense_prefix(HeapWord* beg, HeapWord* end) } } -// Find the point at which a space can be split and, if necessary, record the -// split point. -// -// If the current src region (which overflowed the destination space) doesn't -// have a partial object, the split point is at the beginning of the current src -// region (an "easy" split, no extra bookkeeping required). -// -// If the current src region has a partial object, the split point is in the -// region where that partial object starts (call it the split_region). If -// split_region has a partial object, then the split point is just after that -// partial object (a "hard" split where we have to record the split data and -// zero the partial_obj_size field). With a "hard" split, we know that the -// partial_obj ends within split_region because the partial object that caused -// the overflow starts in split_region. If split_region doesn't have a partial -// obj, then the split is at the beginning of split_region (another "easy" -// split). -HeapWord* -ParallelCompactData::summarize_split_space(size_t src_region, - SplitInfo& split_info, - HeapWord* destination, - HeapWord* target_end, - HeapWord** target_next) -{ +// The total live words on src_region would overflow the target space, so find +// the overflowing object and record the split point. The invariant is that an +// obj should not cross space boundary. +HeapWord* ParallelCompactData::summarize_split_space(size_t src_region, + SplitInfo& split_info, + HeapWord* const destination, + HeapWord* const target_end, + HeapWord** target_next) { assert(destination <= target_end, "sanity"); assert(destination + _region_data[src_region].data_size() > target_end, "region should not fit into target space"); assert(is_region_aligned(target_end), "sanity"); - size_t split_region = src_region; - HeapWord* split_destination = destination; size_t partial_obj_size = _region_data[src_region].partial_obj_size(); if (destination + partial_obj_size > target_end) { - // The split point is just after the partial object (if any) in the - // src_region that contains the start of the object that overflowed the - // destination space. + assert(partial_obj_size > 0, "inv"); + // The overflowing obj is from a previous region. // - // Find the start of the "overflow" object and set split_region to the - // region containing it. - HeapWord* const overflow_obj = _region_data[src_region].partial_obj_addr(); - split_region = addr_to_region_idx(overflow_obj); - - // Clear the source_region field of all destination regions whose first word - // came from data after the split point (a non-null source_region field - // implies a region must be filled). + // source-regions: // - // An alternative to the simple loop below: clear during post_compact(), - // which uses memcpy instead of individual stores, and is easy to - // parallelize. (The downside is that it clears the entire RegionData - // object as opposed to just one field.) + // *************** + // | A|AA | + // *************** + // ^ + // | split-point // - // post_compact() would have to clear the summary data up to the highest - // address that was written during the summary phase, which would be + // dest-region: // - // max(top, max(new_top, clear_top)) + // ******** + // |~~~~A | + // ******** + // ^^ + // || target-space-end + // | + // | destination // - // where clear_top is a new field in SpaceInfo. Would have to set clear_top - // to target_end. - const RegionData* const sr = region(split_region); - const size_t beg_idx = - addr_to_region_idx(region_align_up(sr->destination() + - sr->partial_obj_size())); - const size_t end_idx = addr_to_region_idx(target_end); + // AAA would overflow target-space. + // + HeapWord* overflowing_obj = _region_data[src_region].partial_obj_addr(); + size_t split_region = addr_to_region_idx(overflowing_obj); - log_develop_trace(gc, compaction)("split: clearing source_region field in [" SIZE_FORMAT ", " SIZE_FORMAT ")", beg_idx, end_idx); - for (size_t idx = beg_idx; idx < end_idx; ++idx) { - _region_data[idx].set_source_region(0); + // The number of live words before the overflowing object on this split region + size_t preceding_live_words; + if (is_region_aligned(overflowing_obj)) { + preceding_live_words = 0; + } else { + // Words accounted by the overflowing object on the split region + size_t overflowing_size = pointer_delta(region_align_up(overflowing_obj), overflowing_obj); + preceding_live_words = region(split_region)->data_size() - overflowing_size; } - // Set split_destination and partial_obj_size to reflect the split region. - split_destination = sr->destination(); - partial_obj_size = sr->partial_obj_size(); - } + split_info.record(split_region, overflowing_obj, preceding_live_words); + + HeapWord* src_region_start = region_to_addr(src_region); + HeapWord* new_top = destination - pointer_delta(src_region_start, overflowing_obj); + + // If the overflowing obj was relocated to its original destination, + // those destination regions would have their source_region set. Now that + // this overflowing obj is relocated somewhere else, reset the + // source_region. + { + size_t range_start = addr_to_region_idx(region_align_up(new_top)); + size_t range_end = addr_to_region_idx(region_align_up(destination)); + for (size_t i = range_start; i < range_end; ++i) { + region(i)->set_source_region(0); + } + } + + // Update new top of target space + *target_next = new_top; - // The split is recorded only if a partial object extends onto the region. - if (partial_obj_size != 0) { - _region_data[split_region].set_partial_obj_size(0); - split_info.record(split_region, partial_obj_size, split_destination); + return overflowing_obj; } - // Setup the continuation addresses. - *target_next = split_destination + partial_obj_size; - HeapWord* const source_next = region_to_addr(split_region) + partial_obj_size; + // Obj-iteration to locate the overflowing obj + HeapWord* region_start = region_to_addr(src_region); + HeapWord* region_end = region_start + RegionSize; + HeapWord* cur_addr = region_start + partial_obj_size; + size_t live_words = partial_obj_size; - if (log_develop_is_enabled(Trace, gc, compaction)) { - const char * split_type = partial_obj_size == 0 ? "easy" : "hard"; - log_develop_trace(gc, compaction)("%s split: src=" PTR_FORMAT " src_c=" SIZE_FORMAT " pos=" SIZE_FORMAT, - split_type, p2i(source_next), split_region, partial_obj_size); - log_develop_trace(gc, compaction)("%s split: dst=" PTR_FORMAT " dst_c=" SIZE_FORMAT " tn=" PTR_FORMAT, - split_type, p2i(split_destination), - addr_to_region_idx(split_destination), - p2i(*target_next)); + while (true) { + assert(cur_addr < region_end, "inv"); + cur_addr = PSParallelCompact::mark_bitmap()->find_obj_beg(cur_addr, region_end); + // There must be an overflowing obj in this region + assert(cur_addr < region_end, "inv"); - if (partial_obj_size != 0) { - HeapWord* const po_beg = split_info.destination(); - HeapWord* const po_end = po_beg + split_info.partial_obj_size(); - log_develop_trace(gc, compaction)("%s split: po_beg=" PTR_FORMAT " " SIZE_FORMAT " po_end=" PTR_FORMAT " " SIZE_FORMAT, - split_type, - p2i(po_beg), addr_to_region_idx(po_beg), - p2i(po_end), addr_to_region_idx(po_end)); + oop obj = cast_to_oop(cur_addr); + size_t obj_size = obj->size(); + if (destination + live_words + obj_size > target_end) { + // Found the overflowing obj + split_info.record(src_region, cur_addr, live_words); + *target_next = destination + live_words; + return cur_addr; } - } - return source_next; + live_words += obj_size; + cur_addr += obj_size; + } } size_t ParallelCompactData::live_words_in_space(const MutableSpace* space, @@ -451,70 +458,57 @@ bool ParallelCompactData::summarize(SplitInfo& split_info, const size_t end_region = addr_to_region_idx(region_align_up(source_end)); HeapWord *dest_addr = target_beg; - while (cur_region < end_region) { - // The destination must be set even if the region has no data. + for (/* empty */; cur_region < end_region; cur_region++) { + size_t words = _region_data[cur_region].data_size(); + + // Skip empty ones + if (words == 0) { + continue; + } + + if (split_info.is_split(cur_region)) { + assert(words > split_info.preceding_live_words(), "inv"); + words -= split_info.preceding_live_words(); + } + _region_data[cur_region].set_destination(dest_addr); - size_t words = _region_data[cur_region].data_size(); - if (words > 0) { - // If cur_region does not fit entirely into the target space, find a point - // at which the source space can be 'split' so that part is copied to the - // target space and the rest is copied elsewhere. - if (dest_addr + words > target_end) { - assert(source_next != nullptr, "source_next is null when splitting"); - *source_next = summarize_split_space(cur_region, split_info, dest_addr, - target_end, target_next); - return false; - } + // If cur_region does not fit entirely into the target space, find a point + // at which the source space can be 'split' so that part is copied to the + // target space and the rest is copied elsewhere. + if (dest_addr + words > target_end) { + assert(source_next != nullptr, "source_next is null when splitting"); + *source_next = summarize_split_space(cur_region, split_info, dest_addr, + target_end, target_next); + return false; + } - // Compute the destination_count for cur_region, and if necessary, update - // source_region for a destination region. The source_region field is - // updated if cur_region is the first (left-most) region to be copied to a - // destination region. - // - // The destination_count calculation is a bit subtle. A region that has - // data that compacts into itself does not count itself as a destination. - // This maintains the invariant that a zero count means the region is - // available and can be claimed and then filled. - uint destination_count = 0; - if (split_info.is_split(cur_region)) { - // The current region has been split: the partial object will be copied - // to one destination space and the remaining data will be copied to - // another destination space. Adjust the initial destination_count and, - // if necessary, set the source_region field if the partial object will - // cross a destination region boundary. - destination_count = split_info.destination_count(); - if (destination_count == 2) { - size_t dest_idx = addr_to_region_idx(split_info.dest_region_addr()); - _region_data[dest_idx].set_source_region(cur_region); - } - } + uint destination_count = split_info.is_split(cur_region) + ? split_info.preceding_destination_count() + : 0; - HeapWord* const last_addr = dest_addr + words - 1; - const size_t dest_region_1 = addr_to_region_idx(dest_addr); - const size_t dest_region_2 = addr_to_region_idx(last_addr); - - // Initially assume that the destination regions will be the same and - // adjust the value below if necessary. Under this assumption, if - // cur_region == dest_region_2, then cur_region will be compacted - // completely into itself. - destination_count += cur_region == dest_region_2 ? 0 : 1; - if (dest_region_1 != dest_region_2) { - // Destination regions differ; adjust destination_count. - destination_count += 1; - // Data from cur_region will be copied to the start of dest_region_2. - _region_data[dest_region_2].set_source_region(cur_region); - } else if (is_region_aligned(dest_addr)) { - // Data from cur_region will be copied to the start of the destination - // region. - _region_data[dest_region_1].set_source_region(cur_region); - } + HeapWord* const last_addr = dest_addr + words - 1; + const size_t dest_region_1 = addr_to_region_idx(dest_addr); + const size_t dest_region_2 = addr_to_region_idx(last_addr); - _region_data[cur_region].set_destination_count(destination_count); - dest_addr += words; + // Initially assume that the destination regions will be the same and + // adjust the value below if necessary. Under this assumption, if + // cur_region == dest_region_2, then cur_region will be compacted + // completely into itself. + destination_count += cur_region == dest_region_2 ? 0 : 1; + if (dest_region_1 != dest_region_2) { + // Destination regions differ; adjust destination_count. + destination_count += 1; + // Data from cur_region will be copied to the start of dest_region_2. + _region_data[dest_region_2].set_source_region(cur_region); + } else if (is_region_aligned(dest_addr)) { + // Data from cur_region will be copied to the start of the destination + // region. + _region_data[dest_region_1].set_source_region(cur_region); } - ++cur_region; + _region_data[cur_region].set_destination_count(destination_count); + dest_addr += words; } *target_next = dest_addr; @@ -522,12 +516,12 @@ bool ParallelCompactData::summarize(SplitInfo& split_info, } #ifdef ASSERT -void ParallelCompactData::verify_clear() -{ - const size_t* const beg = (const size_t*) _region_vspace->committed_low_addr(); - const size_t* const end = (const size_t*) _region_vspace->committed_high_addr(); - for (const size_t* p = beg; p < end; ++p) { - assert(*p == 0, "not zero"); +void ParallelCompactData::verify_clear() { + for (uint cur_idx = 0; cur_idx < region_count(); ++cur_idx) { + if (!region(cur_idx)->is_clear()) { + log_warning(gc)("Uncleared Region: %u", cur_idx); + region(cur_idx)->verify_clear(); + } } } #endif // #ifdef ASSERT @@ -694,6 +688,13 @@ void PSParallelCompact::post_compact() } } +#ifdef ASSERT + { + mark_bitmap()->verify_clear(); + summary_data().verify_clear(); + } +#endif + ParCompactionManager::flush_all_string_dedup_requests(); MutableSpace* const eden_space = _space_info[eden_space_id].space(); @@ -773,6 +774,8 @@ void PSParallelCompact::fill_dense_prefix_end(SpaceId id) { // // The size of the filler (min-obj-size) is 2 heap words with the default // MinObjAlignment, since both markword and klass take 1 heap word. + // With +UseCompactObjectHeaders, the minimum filler size is only one word, + // because the Klass* gets encoded in the mark-word. // // The size of the gap (if any) right before dense-prefix-end is // MinObjAlignment. @@ -780,16 +783,11 @@ void PSParallelCompact::fill_dense_prefix_end(SpaceId id) { // Need to fill in the gap only if it's smaller than min-obj-size, and the // filler obj will extend to next region. - // Note: If min-fill-size decreases to 1, this whole method becomes redundant. - assert(CollectedHeap::min_fill_size() >= 2, "inv"); -#ifndef _LP64 - // In 32-bit system, each heap word is 4 bytes, so MinObjAlignment == 2. - // The gap is always equal to min-fill-size, so nothing to do. - return; -#endif - if (MinObjAlignment > 1) { + if (MinObjAlignment >= checked_cast(CollectedHeap::min_fill_size())) { return; } + + assert(!UseCompactObjectHeaders, "Compact headers can allocate small objects"); assert(CollectedHeap::min_fill_size() == 2, "inv"); HeapWord* const dense_prefix_end = dense_prefix(id); assert(_summary_data.is_region_aligned(dense_prefix_end), "precondition"); @@ -880,10 +878,10 @@ void PSParallelCompact::summary_phase() bool maximum_compaction = check_maximum_compaction(total_live_words, old_space, full_region_prefix_end); - HeapWord* dense_prefix_end = - maximum_compaction ? full_region_prefix_end - : compute_dense_prefix_for_old_space(old_space, - full_region_prefix_end); + HeapWord* dense_prefix_end = maximum_compaction + ? full_region_prefix_end + : compute_dense_prefix_for_old_space(old_space, + full_region_prefix_end); SpaceId id = old_space_id; _space_info[id].set_dense_prefix(dense_prefix_end); @@ -891,6 +889,8 @@ void PSParallelCompact::summary_phase() fill_dense_prefix_end(id); _summary_data.summarize_dense_prefix(old_space->bottom(), dense_prefix_end); } + + // Compacting objs inn [dense_prefix_end, old_space->top()) _summary_data.summarize(_space_info[id].split_info(), dense_prefix_end, old_space->top(), nullptr, dense_prefix_end, old_space->end(), @@ -1550,6 +1550,30 @@ void PSParallelCompact::forward_to_new_addr() { WorkerTask("PSForward task"), _num_workers(num_workers) {} + static void forward_objs_in_range(ParCompactionManager* cm, + HeapWord* start, + HeapWord* end, + HeapWord* destination) { + HeapWord* cur_addr = start; + HeapWord* new_addr = destination; + + while (cur_addr < end) { + cur_addr = mark_bitmap()->find_obj_beg(cur_addr, end); + if (cur_addr >= end) { + return; + } + assert(mark_bitmap()->is_marked(cur_addr), "inv"); + oop obj = cast_to_oop(cur_addr); + if (new_addr != cur_addr) { + cm->preserved_marks()->push_if_necessary(obj, obj->mark()); + FullGCForwarding::forward_to(obj, cast_to_oop(new_addr)); + } + size_t obj_size = obj->size(); + new_addr += obj_size; + cur_addr += obj_size; + } + } + void work(uint worker_id) override { ParCompactionManager* cm = ParCompactionManager::gc_thread_compaction_manager(worker_id); for (uint id = old_space_id; id < last_space_id; ++id) { @@ -1561,6 +1585,8 @@ void PSParallelCompact::forward_to_new_addr() { continue; } + const SplitInfo& split_info = _space_info[SpaceId(id)].split_info(); + size_t dense_prefix_region = _summary_data.addr_to_region_idx(dense_prefix_addr); size_t top_region = _summary_data.addr_to_region_idx(_summary_data.region_align_up(top)); size_t start_region; @@ -1580,24 +1606,19 @@ void PSParallelCompact::forward_to_new_addr() { HeapWord* region_start = _summary_data.region_to_addr(cur_region); HeapWord* region_end = region_start + ParallelCompactData::RegionSize; - HeapWord* cur_addr = region_start + live_words; - - HeapWord* destination = region_ptr->destination(); - while (cur_addr < region_end) { - cur_addr = mark_bitmap()->find_obj_beg(cur_addr, region_end); - if (cur_addr >= region_end) { - break; - } - assert(mark_bitmap()->is_marked(cur_addr), "inv"); - HeapWord* new_addr = destination + live_words; - oop obj = cast_to_oop(cur_addr); - if (new_addr != cur_addr) { - cm->preserved_marks()->push_if_necessary(obj, obj->mark()); - obj->forward_to(cast_to_oop(new_addr)); - } - size_t obj_size = obj->size(); - live_words += obj_size; - cur_addr += obj_size; + + if (split_info.is_split(cur_region)) { + // Part 1: will be relocated to space-1 + HeapWord* preceding_destination = split_info.preceding_destination(); + HeapWord* split_point = split_info.split_point(); + forward_objs_in_range(cm, region_start + live_words, split_point, preceding_destination + live_words); + + // Part 2: will be relocated to space-2 + HeapWord* destination = region_ptr->destination(); + forward_objs_in_range(cm, split_point, region_end, destination); + } else { + HeapWord* destination = region_ptr->destination(); + forward_objs_in_range(cm, region_start + live_words, region_end, destination + live_words); } } } @@ -1629,14 +1650,17 @@ void PSParallelCompact::verify_forward() { break; } assert(mark_bitmap()->is_marked(cur_addr), "inv"); + assert(bump_ptr <= _space_info[bump_ptr_space].new_top(), "inv"); // Move to the space containing cur_addr if (bump_ptr == _space_info[bump_ptr_space].new_top()) { bump_ptr = space(space_id(cur_addr))->bottom(); bump_ptr_space = space_id(bump_ptr); } oop obj = cast_to_oop(cur_addr); - if (cur_addr != bump_ptr) { - assert(obj->forwardee() == cast_to_oop(bump_ptr), "inv"); + if (cur_addr == bump_ptr) { + assert(!FullGCForwarding::is_forwarded(obj), "inv"); + } else { + assert(FullGCForwarding::forwardee(obj) == cast_to_oop(bump_ptr), "inv"); } bump_ptr += obj->size(); cur_addr += obj->size(); @@ -1890,7 +1914,7 @@ void PSParallelCompact::verify_filler_in_dense_prefix() { oop obj = cast_to_oop(cur_addr); oopDesc::verify(obj); if (!mark_bitmap()->is_marked(cur_addr)) { - Klass* k = cast_to_oop(cur_addr)->klass_without_asserts(); + Klass* k = cast_to_oop(cur_addr)->klass(); assert(k == Universe::fillerArrayKlass() || k == vmClasses::FillerObject_klass(), "inv"); } cur_addr += obj->size(); @@ -1941,15 +1965,10 @@ PSParallelCompact::SpaceId PSParallelCompact::space_id(HeapWord* addr) { } // Skip over count live words starting from beg, and return the address of the -// next live word. Unless marked, the word corresponding to beg is assumed to -// be dead. Callers must either ensure beg does not correspond to the middle of -// an object, or account for those live words in some other way. Callers must -// also ensure that there are enough live words in the range [beg, end) to skip. -HeapWord* -PSParallelCompact::skip_live_words(HeapWord* beg, HeapWord* end, size_t count) +// next live word. Callers must also ensure that there are enough live words in +// the range [beg, end) to skip. +HeapWord* PSParallelCompact::skip_live_words(HeapWord* beg, HeapWord* end, size_t count) { - assert(count > 0, "sanity"); - ParMarkBitMap* m = mark_bitmap(); HeapWord* cur_addr = beg; while (true) { @@ -1965,69 +1984,94 @@ PSParallelCompact::skip_live_words(HeapWord* beg, HeapWord* end, size_t count) } } +// On filling a destination region (dest-region), we need to know the location +// of the word that will be at the start of the dest-region after compaction. +// A dest-region can have one or more source regions, but only the first +// source-region contains this location. This location is retrieved by calling +// `first_src_addr` on a dest-region. +// Conversely, a source-region has a dest-region which holds the destination of +// the first live word on this source-region, based on which the destination +// for the rest of live words can be derived. +// +// Note: +// There is some complication due to space-boundary-fragmentation (an obj can't +// cross space-boundary) -- a source-region may be split and behave like two +// distinct regions with their own dest-region, as depicted below. +// +// source-region: region-n +// +// ********************** +// | A|A~~~~B|B | +// ********************** +// n-1 n n+1 +// +// AA, BB denote two live objs. ~~~~ denotes unknown number of live objs. +// +// Assuming the dest-region for region-n is the final region before +// old-space-end and its first-live-word is the middle of AA, the heap content +// will look like the following after compaction: +// +// ************** ************* +// A|A~~~~ | |BB | +// ************** ************* +// ^ ^ +// | old-space-end | eden-space-start +// +// Therefore, in this example, region-n will have two dest-regions, one for +// the final region in old-space and the other for the first region in +// eden-space. +// To handle this special case, we introduce the concept of split-region, whose +// contents are relocated to two spaces. `SplitInfo` captures all necessary +// info about the split, the first part, spliting-point, and the second part. HeapWord* PSParallelCompact::first_src_addr(HeapWord* const dest_addr, SpaceId src_space_id, size_t src_region_idx) { - assert(summary_data().is_region_aligned(dest_addr), "not aligned"); - - const SplitInfo& split_info = _space_info[src_space_id].split_info(); - if (split_info.dest_region_addr() == dest_addr) { - // The partial object ending at the split point contains the first word to - // be copied to dest_addr. - return split_info.first_src_addr(); - } - - const ParallelCompactData& sd = summary_data(); - ParMarkBitMap* const bitmap = mark_bitmap(); const size_t RegionSize = ParallelCompactData::RegionSize; + const ParallelCompactData& sd = summary_data(); + assert(sd.is_region_aligned(dest_addr), "precondition"); - assert(sd.is_region_aligned(dest_addr), "not aligned"); const RegionData* const src_region_ptr = sd.region(src_region_idx); + assert(src_region_ptr->data_size() > 0, "src region cannot be empty"); + const size_t partial_obj_size = src_region_ptr->partial_obj_size(); HeapWord* const src_region_destination = src_region_ptr->destination(); - assert(dest_addr >= src_region_destination, "wrong src region"); - assert(src_region_ptr->data_size() > 0, "src region cannot be empty"); - - HeapWord* const src_region_beg = sd.region_to_addr(src_region_idx); - HeapWord* const src_region_end = src_region_beg + RegionSize; + HeapWord* const region_start = sd.region_to_addr(src_region_idx); + HeapWord* const region_end = sd.region_to_addr(src_region_idx) + RegionSize; - HeapWord* addr = src_region_beg; - if (dest_addr == src_region_destination) { - // Return the first live word in the source region. - if (partial_obj_size == 0) { - addr = bitmap->find_obj_beg(addr, src_region_end); - assert(addr < src_region_end, "no objects start in src region"); + // Identify the actual destination for the first live words on this region, + // taking split-region into account. + HeapWord* region_start_destination; + const SplitInfo& split_info = _space_info[src_space_id].split_info(); + if (split_info.is_split(src_region_idx)) { + // The second part of this split region; use the recorded split point. + if (dest_addr == src_region_destination) { + return split_info.split_point(); } - return addr; + region_start_destination = split_info.preceding_destination(); + } else { + region_start_destination = src_region_destination; } - // Must skip some live data. - size_t words_to_skip = dest_addr - src_region_destination; - assert(src_region_ptr->data_size() > words_to_skip, "wrong src region"); + // Calculate the offset to be skipped + size_t words_to_skip = pointer_delta(dest_addr, region_start_destination); - if (partial_obj_size >= words_to_skip) { - // All the live words to skip are part of the partial object. - addr += words_to_skip; - if (partial_obj_size == words_to_skip) { - // Find the first live word past the partial object. - addr = bitmap->find_obj_beg(addr, src_region_end); - assert(addr < src_region_end, "wrong src region"); - } - return addr; + HeapWord* result; + if (partial_obj_size > words_to_skip) { + result = region_start + words_to_skip; + } else { + words_to_skip -= partial_obj_size; + result = skip_live_words(region_start + partial_obj_size, region_end, words_to_skip); } - // Skip over the partial object (if any). - if (partial_obj_size != 0) { - words_to_skip -= partial_obj_size; - addr += partial_obj_size; + if (split_info.is_split(src_region_idx)) { + assert(result < split_info.split_point(), "postcondition"); + } else { + assert(result < region_end, "postcondition"); } - // Skip over live words due to objects that start in the region. - addr = skip_live_words(addr, src_region_end, words_to_skip); - assert(addr < src_region_end, "wrong src region"); - return addr; + return result; } void PSParallelCompact::decrement_destination_counts(ParCompactionManager* cm, @@ -2078,10 +2122,7 @@ size_t PSParallelCompact::next_src_region(MoveAndUpdateClosure& closure, HeapWord*& src_space_top, HeapWord* end_addr) { - typedef ParallelCompactData::RegionData RegionData; - ParallelCompactData& sd = PSParallelCompact::summary_data(); - const size_t region_size = ParallelCompactData::RegionSize; size_t src_region_idx = 0; @@ -2089,8 +2130,8 @@ size_t PSParallelCompact::next_src_region(MoveAndUpdateClosure& closure, HeapWord* const src_aligned_up = sd.region_align_up(end_addr); RegionData* src_region_ptr = sd.addr_to_region_ptr(src_aligned_up); HeapWord* const top_aligned_up = sd.region_align_up(src_space_top); - const RegionData* const top_region_ptr = - sd.addr_to_region_ptr(top_aligned_up); + const RegionData* const top_region_ptr = sd.addr_to_region_ptr(top_aligned_up); + while (src_region_ptr < top_region_ptr && src_region_ptr->data_size() == 0) { ++src_region_ptr; } @@ -2107,59 +2148,50 @@ size_t PSParallelCompact::next_src_region(MoveAndUpdateClosure& closure, } // Switch to a new source space and find the first non-empty region. - unsigned int space_id = src_space_id + 1; + uint space_id = src_space_id + 1; assert(space_id < last_space_id, "not enough spaces"); - HeapWord* const destination = closure.destination(); - - do { - MutableSpace* space = _space_info[space_id].space(); - HeapWord* const bottom = space->bottom(); - const RegionData* const bottom_cp = sd.addr_to_region_ptr(bottom); - - // Iterate over the spaces that do not compact into themselves. - if (bottom_cp->destination() != bottom) { - HeapWord* const top_aligned_up = sd.region_align_up(space->top()); - const RegionData* const top_cp = sd.addr_to_region_ptr(top_aligned_up); - - for (const RegionData* src_cp = bottom_cp; src_cp < top_cp; ++src_cp) { - if (src_cp->live_obj_size() > 0) { - // Found it. - assert(src_cp->destination() == destination, - "first live obj in the space must match the destination"); - assert(src_cp->partial_obj_size() == 0, - "a space cannot begin with a partial obj"); - - src_space_id = SpaceId(space_id); - src_space_top = space->top(); - const size_t src_region_idx = sd.region(src_cp); - closure.set_source(sd.region_to_addr(src_region_idx)); - return src_region_idx; - } else { - assert(src_cp->data_size() == 0, "sanity"); - } + for (/* empty */; space_id < last_space_id; ++space_id) { + HeapWord* bottom = _space_info[space_id].space()->bottom(); + HeapWord* top = _space_info[space_id].space()->top(); + // Skip empty space + if (bottom == top) { + continue; + } + + // Identify the first region that contains live words in this space + size_t cur_region = sd.addr_to_region_idx(bottom); + size_t end_region = sd.addr_to_region_idx(sd.region_align_up(top)); + + for (/* empty */ ; cur_region < end_region; ++cur_region) { + RegionData* cur = sd.region(cur_region); + if (cur->live_obj_size() > 0) { + HeapWord* region_start_addr = sd.region_to_addr(cur_region); + HeapWord* region_end_addr = region_start_addr + ParallelCompactData::RegionSize; + HeapWord* first_live_word = mark_bitmap()->find_obj_beg(region_start_addr, region_end_addr); + assert(first_live_word < region_end_addr, "inv"); + + src_space_id = SpaceId(space_id); + src_space_top = top; + closure.set_source(first_live_word); + return cur_region; } } - } while (++space_id < last_space_id); + } - assert(false, "no source region was found"); - return 0; + ShouldNotReachHere(); } HeapWord* PSParallelCompact::partial_obj_end(HeapWord* region_start_addr) { ParallelCompactData& sd = summary_data(); assert(sd.is_region_aligned(region_start_addr), "precondition"); - // Use per-region partial_obj_size to locate the end of the obj, that extends to region_start_addr. - SplitInfo& split_info = _space_info[space_id(region_start_addr)].split_info(); + // Use per-region partial_obj_size to locate the end of the obj, that extends + // to region_start_addr. size_t start_region_idx = sd.addr_to_region_idx(region_start_addr); size_t end_region_idx = sd.region_count(); size_t accumulated_size = 0; for (size_t region_idx = start_region_idx; region_idx < end_region_idx; ++region_idx) { - if (split_info.is_split(region_idx)) { - accumulated_size += split_info.partial_obj_size(); - break; - } size_t cur_partial_obj_size = sd.region(region_idx)->partial_obj_size(); accumulated_size += cur_partial_obj_size; if (cur_partial_obj_size != ParallelCompactData::RegionSize) { @@ -2169,6 +2201,8 @@ HeapWord* PSParallelCompact::partial_obj_end(HeapWord* region_start_addr) { return region_start_addr + accumulated_size; } +// Use region_idx as the destination region, and evacuate all live objs on its +// source regions to this destination region. void PSParallelCompact::fill_region(ParCompactionManager* cm, MoveAndUpdateClosure& closure, size_t region_idx) { ParMarkBitMap* const bitmap = mark_bitmap(); @@ -2189,20 +2223,43 @@ void PSParallelCompact::fill_region(ParCompactionManager* cm, MoveAndUpdateClosu src_region_idx += 1; } + // source-region: + // + // ********** + // | ~~~ | + // ********** + // ^ + // |-- closure.source() / first_src_addr + // + // + // ~~~ : live words + // + // destination-region: + // + // ********** + // | | + // ********** + // ^ + // |-- region-start if (bitmap->is_unmarked(closure.source())) { - // The first source word is in the middle of an object; copy the remainder - // of the object or as much as will fit. The fact that pointer updates were - // deferred will be noted when the object header is processed. + // An object overflows the previous destination region, so this + // destination region should copy the remainder of the object or as much as + // will fit. HeapWord* const old_src_addr = closure.source(); { HeapWord* region_start = sd.region_align_down(closure.source()); HeapWord* obj_start = bitmap->find_obj_beg_reverse(region_start, closure.source()); HeapWord* obj_end; - if (bitmap->is_marked(obj_start)) { + if (obj_start != closure.source()) { + assert(bitmap->is_marked(obj_start), "inv"); + // Found the actual obj-start, try to find the obj-end using either + // size() if this obj is completely contained in the current region. HeapWord* next_region_start = region_start + ParallelCompactData::RegionSize; HeapWord* partial_obj_start = (next_region_start >= src_space_top) ? nullptr : sd.addr_to_region_ptr(next_region_start)->partial_obj_addr(); + // This obj extends to next region iff partial_obj_addr of the *next* + // region is the same as obj-start. if (partial_obj_start == obj_start) { // This obj extends to next region. obj_end = partial_obj_end(next_region_start); @@ -2219,39 +2276,41 @@ void PSParallelCompact::fill_region(ParCompactionManager* cm, MoveAndUpdateClosu } if (closure.is_full()) { - decrement_destination_counts(cm, src_space_id, src_region_idx, - closure.source()); + decrement_destination_counts(cm, src_space_id, src_region_idx, closure.source()); closure.complete_region(dest_addr, region_ptr); return; } + // Finished copying without using up the current destination-region HeapWord* const end_addr = sd.region_align_down(closure.source()); if (sd.region_align_down(old_src_addr) != end_addr) { + assert(sd.region_align_up(old_src_addr) == end_addr, "only one region"); // The partial object was copied from more than one source region. decrement_destination_counts(cm, src_space_id, src_region_idx, end_addr); // Move to the next source region, possibly switching spaces as well. All // args except end_addr may be modified. - src_region_idx = next_src_region(closure, src_space_id, src_space_top, - end_addr); + src_region_idx = next_src_region(closure, src_space_id, src_space_top, end_addr); } } + // Handle the rest obj-by-obj, where we know obj-start. do { HeapWord* cur_addr = closure.source(); HeapWord* const end_addr = MIN2(sd.region_align_up(cur_addr + 1), src_space_top); - HeapWord* partial_obj_start = (end_addr == src_space_top) + // To handle the case where the final obj in source region extends to next region. + HeapWord* final_obj_start = (end_addr == src_space_top) ? nullptr : sd.addr_to_region_ptr(end_addr)->partial_obj_addr(); - // apply closure on objs inside [cur_addr, end_addr) + // Apply closure on objs inside [cur_addr, end_addr) do { cur_addr = bitmap->find_obj_beg(cur_addr, end_addr); if (cur_addr == end_addr) { break; } size_t obj_size; - if (partial_obj_start == cur_addr) { + if (final_obj_start == cur_addr) { obj_size = pointer_delta(partial_obj_end(end_addr), cur_addr); } else { // This obj doesn't extend into next region; size() is safe to use. @@ -2262,8 +2321,7 @@ void PSParallelCompact::fill_region(ParCompactionManager* cm, MoveAndUpdateClosu } while (cur_addr < end_addr && !closure.is_full()); if (closure.is_full()) { - decrement_destination_counts(cm, src_space_id, src_region_idx, - closure.source()); + decrement_destination_counts(cm, src_space_id, src_region_idx, closure.source()); closure.complete_region(dest_addr, region_ptr); return; } @@ -2272,8 +2330,7 @@ void PSParallelCompact::fill_region(ParCompactionManager* cm, MoveAndUpdateClosu // Move to the next source region, possibly switching spaces as well. All // args except end_addr may be modified. - src_region_idx = next_src_region(closure, src_space_id, src_space_top, - end_addr); + src_region_idx = next_src_region(closure, src_space_id, src_space_top, end_addr); } while (true); } @@ -2395,8 +2452,8 @@ void MoveAndUpdateClosure::do_addr(HeapWord* addr, size_t words) { if (copy_destination() != source()) { DEBUG_ONLY(PSParallelCompact::check_new_location(source(), destination());) assert(source() != destination(), "inv"); - assert(cast_to_oop(source())->is_forwarded(), "inv"); - assert(cast_to_oop(source())->forwardee() == cast_to_oop(destination()), "inv"); + assert(FullGCForwarding::is_forwarded(cast_to_oop(source())), "inv"); + assert(FullGCForwarding::forwardee(cast_to_oop(source())) == cast_to_oop(destination()), "inv"); Copy::aligned_conjoint_words(source(), copy_destination(), words); cast_to_oop(copy_destination())->init_mark(); } diff --git a/src/hotspot/share/gc/parallel/psParallelCompact.hpp b/src/hotspot/share/gc/parallel/psParallelCompact.hpp index 3f487ec3ef4..878a09e283b 100644 --- a/src/hotspot/share/gc/parallel/psParallelCompact.hpp +++ b/src/hotspot/share/gc/parallel/psParallelCompact.hpp @@ -116,51 +116,45 @@ class SplitInfo // Return true if this split info is valid (i.e., if a split has been // recorded). The very first region cannot have a partial object and thus is // never split, so 0 is the 'invalid' value. - bool is_valid() const { return _src_region_idx > 0; } + bool is_valid() const { return _split_region_idx > 0; } // Return true if this split holds data for the specified source region. - inline bool is_split(size_t source_region) const; + inline bool is_split(size_t region_idx) const; - // The index of the split region, the size of the partial object on that - // region and the destination of the partial object. - size_t partial_obj_size() const { return _partial_obj_size; } - HeapWord* destination() const { return _destination; } + // Obj at the split point doesn't fit the previous space and will be relocated to the next space. + HeapWord* split_point() const { return _split_point; } - // The destination count of the partial object referenced by this split - // (either 1 or 2). This must be added to the destination count of the - // remainder of the source region. - unsigned int destination_count() const { return _destination_count; } + // Number of live words before the split point on this region. + size_t preceding_live_words() const { return _preceding_live_words; } - // If a word within the partial object will be written to the first word of a - // destination region, this is the address of the destination region; - // otherwise this is null. - HeapWord* dest_region_addr() const { return _dest_region_addr; } + // A split region has two "destinations", living in two spaces. This method + // returns the first one -- destination for the first live word on + // this split region. + HeapWord* preceding_destination() const { + assert(_preceding_destination != nullptr, "inv"); + return _preceding_destination; + } - // If a word within the partial object will be written to the first word of a - // destination region, this is the address of that word within the partial - // object; otherwise this is null. - HeapWord* first_src_addr() const { return _first_src_addr; } + // Number of regions the preceding live words are relocated into. + uint preceding_destination_count() const { return _preceding_destination_count; } - // Record the data necessary to split the region src_region_idx. - void record(size_t src_region_idx, size_t partial_obj_size, - HeapWord* destination); + void record(size_t split_region_idx, HeapWord* split_point, size_t preceding_live_words); void clear(); DEBUG_ONLY(void verify_clear();) private: - size_t _src_region_idx; - size_t _partial_obj_size; - HeapWord* _destination; - unsigned int _destination_count; - HeapWord* _dest_region_addr; - HeapWord* _first_src_addr; + size_t _split_region_idx; + HeapWord* _split_point; + size_t _preceding_live_words; + HeapWord* _preceding_destination; + uint _preceding_destination_count; }; inline bool SplitInfo::is_split(size_t region_idx) const { - return _src_region_idx == region_idx && is_valid(); + return _split_region_idx == region_idx && is_valid(); } class SpaceInfo @@ -215,10 +209,18 @@ class ParallelCompactData class RegionData { public: - // Destination address of the region. + // Destination for the first live word in this region. + // Therefore, the new addr for every live obj on this region can be calculated as: + // + // new_addr := _destination + live_words_offset(old_addr); + // + // where, live_words_offset is the number of live words accumulated from + // region-start to old_addr. HeapWord* destination() const { return _destination; } - // The first region containing data destined for this region. + // A destination region can have multiple source regions; only the first + // one is recorded. Since all live objs are slided down, subsequent source + // regions can be found via plain heap-region iteration. size_t source_region() const { return _source_region; } // Reuse _source_region to store the corresponding shadow region index @@ -313,9 +315,12 @@ class ParallelCompactData // Return to the normal path here inline void shadow_to_normal(); - int shadow_state() { return _shadow_state; } + bool is_clear(); + + void verify_clear() NOT_DEBUG_RETURN; + private: // The type used to represent object sizes within a region. typedef uint region_sz_t; @@ -873,7 +878,10 @@ class MoveAndUpdateClosure: public StackObj { size_t words_remaining() const { return _words_remaining; } bool is_full() const { return _words_remaining == 0; } HeapWord* source() const { return _source; } - void set_source(HeapWord* addr) { _source = addr; } + void set_source(HeapWord* addr) { + assert(addr != nullptr, "precondition"); + _source = addr; + } // If the object will fit (size <= words_remaining()), copy it to the current // destination, update the interior oops and the start array. @@ -902,9 +910,8 @@ inline size_t MoveAndUpdateClosure::calculate_words_remaining(size_t region) { HeapWord* dest_addr = PSParallelCompact::summary_data().region_to_addr(region); PSParallelCompact::SpaceId dest_space_id = PSParallelCompact::space_id(dest_addr); HeapWord* new_top = PSParallelCompact::new_top(dest_space_id); - assert(dest_addr < new_top, "sanity"); - - return MIN2(pointer_delta(new_top, dest_addr), ParallelCompactData::RegionSize); + return MIN2(pointer_delta(new_top, dest_addr), + ParallelCompactData::RegionSize); } inline diff --git a/src/hotspot/share/gc/parallel/psParallelCompact.inline.hpp b/src/hotspot/share/gc/parallel/psParallelCompact.inline.hpp index e5f1b9d30ae..aa62d474a55 100644 --- a/src/hotspot/share/gc/parallel/psParallelCompact.inline.hpp +++ b/src/hotspot/share/gc/parallel/psParallelCompact.inline.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2016, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -31,6 +31,7 @@ #include "gc/parallel/parMarkBitMap.inline.hpp" #include "gc/shared/collectedHeap.hpp" #include "gc/shared/continuationGCSupport.inline.hpp" +#include "gc/shared/fullGCForwarding.inline.hpp" #include "oops/access.inline.hpp" #include "oops/compressedOops.inline.hpp" #include "oops/klass.hpp" @@ -79,7 +80,7 @@ inline void PSParallelCompact::adjust_pointer(T* p) { if (!obj->is_forwarded()) { return; } - oop new_obj = obj->forwardee(); + oop new_obj = FullGCForwarding::forwardee(obj); assert(new_obj != nullptr, "non-null address for live objects"); assert(new_obj != obj, "inv"); assert(ParallelScavengeHeap::heap()->is_in_reserved(new_obj), diff --git a/src/hotspot/share/gc/parallel/psPromotionManager.cpp b/src/hotspot/share/gc/parallel/psPromotionManager.cpp index 5f6629e6cc2..c740b1488d7 100644 --- a/src/hotspot/share/gc/parallel/psPromotionManager.cpp +++ b/src/hotspot/share/gc/parallel/psPromotionManager.cpp @@ -321,7 +321,6 @@ void PSPromotionManager::process_array_chunk(PartialArrayState* state) { } void PSPromotionManager::push_objArray(oop old_obj, oop new_obj) { - assert(old_obj->is_objArray(), "precondition"); assert(old_obj->is_forwarded(), "precondition"); assert(old_obj->forwardee() == new_obj, "precondition"); assert(new_obj->is_objArray(), "precondition"); @@ -357,7 +356,7 @@ oop PSPromotionManager::oop_promotion_failed(oop obj, markWord obj_mark) { // this started. If it is the same (i.e., no forwarding // pointer has been installed), then this thread owns // it. - if (obj->forward_to_atomic(obj, obj_mark) == nullptr) { + if (obj->forward_to_self_atomic(obj_mark) == nullptr) { // We won any races, we "own" this object. assert(obj == obj->forwardee(), "Sanity"); diff --git a/src/hotspot/share/gc/parallel/psPromotionManager.hpp b/src/hotspot/share/gc/parallel/psPromotionManager.hpp index 8d412234614..a69d975956d 100644 --- a/src/hotspot/share/gc/parallel/psPromotionManager.hpp +++ b/src/hotspot/share/gc/parallel/psPromotionManager.hpp @@ -111,7 +111,7 @@ class PSPromotionManager { void push_depth(ScannerTask task); - inline void promotion_trace_event(oop new_obj, oop old_obj, size_t obj_size, + inline void promotion_trace_event(oop new_obj, Klass* klass, size_t obj_size, uint age, bool tenured, const PSPromotionLAB* lab); diff --git a/src/hotspot/share/gc/parallel/psPromotionManager.inline.hpp b/src/hotspot/share/gc/parallel/psPromotionManager.inline.hpp index f38cbaecf43..ed517c06a40 100644 --- a/src/hotspot/share/gc/parallel/psPromotionManager.inline.hpp +++ b/src/hotspot/share/gc/parallel/psPromotionManager.inline.hpp @@ -66,7 +66,7 @@ inline void PSPromotionManager::claim_or_forward_depth(T* p) { } } -inline void PSPromotionManager::promotion_trace_event(oop new_obj, oop old_obj, +inline void PSPromotionManager::promotion_trace_event(oop new_obj, Klass* klass, size_t obj_size, uint age, bool tenured, const PSPromotionLAB* lab) { @@ -79,14 +79,14 @@ inline void PSPromotionManager::promotion_trace_event(oop new_obj, oop old_obj, if (gc_tracer->should_report_promotion_in_new_plab_event()) { size_t obj_bytes = obj_size * HeapWordSize; size_t lab_size = lab->capacity(); - gc_tracer->report_promotion_in_new_plab_event(old_obj->klass(), obj_bytes, + gc_tracer->report_promotion_in_new_plab_event(klass, obj_bytes, age, tenured, lab_size); } } else { // Promotion of object directly to heap if (gc_tracer->should_report_promotion_outside_plab_event()) { size_t obj_bytes = obj_size * HeapWordSize; - gc_tracer->report_promotion_outside_plab_event(old_obj->klass(), obj_bytes, + gc_tracer->report_promotion_outside_plab_event(klass, obj_bytes, age, tenured); } } @@ -149,7 +149,7 @@ inline oop PSPromotionManager::copy_to_survivor_space(oop o) { return copy_unmarked_to_survivor_space(o, m); } else { // Return the already installed forwardee. - return m.forwardee(); + return o->forwardee(m); } } @@ -165,7 +165,19 @@ inline oop PSPromotionManager::copy_unmarked_to_survivor_space(oop o, oop new_obj = nullptr; bool new_obj_is_tenured = false; - size_t new_obj_size = o->size(); + + // NOTE: With compact headers, it is not safe to load the Klass* from old, because + // that would access the mark-word, that might change at any time by concurrent + // workers. + // This mark word would refer to a forwardee, which may not yet have completed + // copying. Therefore we must load the Klass* from the mark-word that we already + // loaded. This is safe, because we only enter here if not yet forwarded. + assert(!test_mark.is_forwarded(), "precondition"); + Klass* klass = UseCompactObjectHeaders + ? test_mark.klass() + : o->klass(); + + size_t new_obj_size = o->size_given_klass(klass); // Find the objects age, MT safe. uint age = (test_mark.has_displaced_mark_helper() /* o->has_displaced_mark() */) ? @@ -180,7 +192,7 @@ inline oop PSPromotionManager::copy_unmarked_to_survivor_space(oop o, if (new_obj_size > (YoungPLABSize / 2)) { // Allocate this object directly new_obj = cast_to_oop(young_space()->cas_allocate(new_obj_size)); - promotion_trace_event(new_obj, o, new_obj_size, age, false, nullptr); + promotion_trace_event(new_obj, klass, new_obj_size, age, false, nullptr); } else { // Flush and fill _young_lab.flush(); @@ -190,7 +202,7 @@ inline oop PSPromotionManager::copy_unmarked_to_survivor_space(oop o, _young_lab.initialize(MemRegion(lab_base, YoungPLABSize)); // Try the young lab allocation again. new_obj = cast_to_oop(_young_lab.allocate(new_obj_size)); - promotion_trace_event(new_obj, o, new_obj_size, age, false, &_young_lab); + promotion_trace_event(new_obj, klass, new_obj_size, age, false, &_young_lab); } else { _young_gen_is_full = true; } @@ -216,7 +228,7 @@ inline oop PSPromotionManager::copy_unmarked_to_survivor_space(oop o, if (new_obj_size > (OldPLABSize / 2)) { // Allocate this object directly new_obj = cast_to_oop(old_gen()->allocate(new_obj_size)); - promotion_trace_event(new_obj, o, new_obj_size, age, true, nullptr); + promotion_trace_event(new_obj, klass, new_obj_size, age, true, nullptr); } else { // Flush and fill _old_lab.flush(); @@ -226,7 +238,7 @@ inline oop PSPromotionManager::copy_unmarked_to_survivor_space(oop o, _old_lab.initialize(MemRegion(lab_base, OldPLABSize)); // Try the old lab allocation again. new_obj = cast_to_oop(_old_lab.allocate(new_obj_size)); - promotion_trace_event(new_obj, o, new_obj_size, age, true, &_old_lab); + promotion_trace_event(new_obj, klass, new_obj_size, age, true, &_old_lab); } } } diff --git a/src/hotspot/share/gc/serial/defNewGeneration.cpp b/src/hotspot/share/gc/serial/defNewGeneration.cpp index f3b3c8952b9..3792bb5a721 100644 --- a/src/hotspot/share/gc/serial/defNewGeneration.cpp +++ b/src/hotspot/share/gc/serial/defNewGeneration.cpp @@ -39,7 +39,6 @@ #include "gc/shared/gcTimer.hpp" #include "gc/shared/gcTrace.hpp" #include "gc/shared/gcTraceTime.inline.hpp" -#include "gc/shared/preservedMarks.inline.hpp" #include "gc/shared/referencePolicy.hpp" #include "gc/shared/referenceProcessorPhaseTimes.hpp" #include "gc/shared/space.hpp" @@ -227,7 +226,6 @@ DefNewGeneration::DefNewGeneration(ReservedSpace rs, const char* policy) : Generation(rs, initial_size), _promotion_failed(false), - _preserved_marks_set(false /* in_c_heap */), _promo_failure_drain_in_progress(false), _string_dedup_requests() { @@ -609,8 +607,6 @@ bool DefNewGeneration::collect(bool clear_all_soft_refs) { age_table()->clear(); to()->clear(SpaceDecorator::Mangle); - // The preserved marks should be empty at the start of the GC. - _preserved_marks_set.init(1); YoungGenScanClosure young_gen_cl(this); OldGenScanClosure old_gen_cl(this); @@ -681,8 +677,6 @@ bool DefNewGeneration::collect(bool clear_all_soft_refs) { // Reset the PromotionFailureALot counters. NOT_PRODUCT(heap->reset_promotion_should_fail();) } - // We should have processed and cleared all the preserved marks. - _preserved_marks_set.reclaim(); heap->trace_heap_after_gc(_gc_tracer); @@ -706,19 +700,17 @@ void DefNewGeneration::remove_forwarding_pointers() { // starts. (The mark word is overloaded: `is_marked()` == `is_forwarded()`.) struct ResetForwardedMarkWord : ObjectClosure { void do_object(oop obj) override { - if (obj->is_forwarded()) { - obj->init_mark(); + if (obj->is_self_forwarded()) { + obj->unset_self_forwarded(); + } else if (obj->is_forwarded()) { + // To restore the klass-bits in the header. + // Needed for object iteration to work properly. + obj->set_mark(obj->forwardee()->prototype_mark()); } } } cl; eden()->object_iterate(&cl); from()->object_iterate(&cl); - - restore_preserved_marks(); -} - -void DefNewGeneration::restore_preserved_marks() { - _preserved_marks_set.restore(nullptr); } void DefNewGeneration::handle_promotion_failure(oop old) { @@ -726,12 +718,11 @@ void DefNewGeneration::handle_promotion_failure(oop old) { _promotion_failed = true; _promotion_failed_info.register_copy_failure(old->size()); - _preserved_marks_set.get()->push_if_necessary(old, old->mark()); ContinuationGCSupport::transform_stack_chunk(old); // forward to self - old->forward_to(old); + old->forward_to_self(); _promo_failure_scan_stack.push(old); diff --git a/src/hotspot/share/gc/serial/defNewGeneration.hpp b/src/hotspot/share/gc/serial/defNewGeneration.hpp index e86ea6b9747..c5e1c2b152e 100644 --- a/src/hotspot/share/gc/serial/defNewGeneration.hpp +++ b/src/hotspot/share/gc/serial/defNewGeneration.hpp @@ -32,7 +32,6 @@ #include "gc/shared/copyFailedInfo.hpp" #include "gc/shared/gc_globals.hpp" #include "gc/shared/generationCounters.hpp" -#include "gc/shared/preservedMarks.hpp" #include "gc/shared/stringdedup/stringDedup.hpp" #include "gc/shared/tlab_globals.hpp" #include "utilities/align.hpp" @@ -99,11 +98,6 @@ class DefNewGeneration: public Generation { // therefore we must remove their forwarding pointers. void remove_forwarding_pointers(); - virtual void restore_preserved_marks(); - - // Preserved marks - PreservedMarksSet _preserved_marks_set; - Stack _promo_failure_scan_stack; void drain_promo_failure_scan_stack(void); bool _promo_failure_drain_in_progress; diff --git a/src/hotspot/share/gc/serial/serialArguments.cpp b/src/hotspot/share/gc/serial/serialArguments.cpp index ac6dd24fdbf..f8efa192807 100644 --- a/src/hotspot/share/gc/serial/serialArguments.cpp +++ b/src/hotspot/share/gc/serial/serialArguments.cpp @@ -23,10 +23,16 @@ */ #include "precompiled.hpp" +#include "gc/shared/fullGCForwarding.hpp" #include "gc/shared/genArguments.hpp" #include "gc/serial/serialArguments.hpp" #include "gc/serial/serialHeap.hpp" +void SerialArguments::initialize_heap_flags_and_sizes() { + GenArguments::initialize_heap_flags_and_sizes(); + FullGCForwarding::initialize_flags(MaxHeapSize); +} + CollectedHeap* SerialArguments::create_heap() { return new SerialHeap(); } diff --git a/src/hotspot/share/gc/serial/serialArguments.hpp b/src/hotspot/share/gc/serial/serialArguments.hpp index 3ed4df5f41b..d12bd7d8e59 100644 --- a/src/hotspot/share/gc/serial/serialArguments.hpp +++ b/src/hotspot/share/gc/serial/serialArguments.hpp @@ -32,6 +32,7 @@ class CollectedHeap; class SerialArguments : public GenArguments { private: virtual CollectedHeap* create_heap(); + virtual void initialize_heap_flags_and_sizes(); }; #endif // SHARE_GC_SERIAL_SERIALARGUMENTS_HPP diff --git a/src/hotspot/share/gc/serial/serialFullGC.cpp b/src/hotspot/share/gc/serial/serialFullGC.cpp index 897437e33c9..0df28fa7bd5 100644 --- a/src/hotspot/share/gc/serial/serialFullGC.cpp +++ b/src/hotspot/share/gc/serial/serialFullGC.cpp @@ -43,6 +43,7 @@ #include "gc/shared/classUnloadingContext.hpp" #include "gc/shared/collectedHeap.inline.hpp" #include "gc/shared/continuationGCSupport.inline.hpp" +#include "gc/shared/fullGCForwarding.inline.hpp" #include "gc/shared/gcHeapSummary.hpp" #include "gc/shared/gcTimer.hpp" #include "gc/shared/gcTrace.hpp" @@ -230,7 +231,7 @@ class Compacter { static void forward_obj(oop obj, HeapWord* new_addr) { prefetch_write_scan(obj); if (cast_from_oop(obj) != new_addr) { - obj->forward_to(cast_to_oop(new_addr)); + FullGCForwarding::forward_to(obj, cast_to_oop(new_addr)); } else { assert(obj->is_gc_marked(), "inv"); // This obj will stay in-place. Fix the markword. @@ -255,7 +256,7 @@ class Compacter { prefetch_read_scan(addr); oop obj = cast_to_oop(addr); - oop new_obj = obj->forwardee(); + oop new_obj = FullGCForwarding::forwardee(obj); HeapWord* new_addr = cast_from_oop(new_obj); assert(addr != new_addr, "inv"); prefetch_write_copy(new_addr); @@ -352,13 +353,13 @@ class Compacter { HeapWord* top = space->top(); // Check if the first obj inside this space is forwarded. - if (!cast_to_oop(cur_addr)->is_forwarded()) { + if (!FullGCForwarding::is_forwarded(cast_to_oop(cur_addr))) { // Jump over consecutive (in-place) live-objs-chunk cur_addr = get_first_dead(i); } while (cur_addr < top) { - if (!cast_to_oop(cur_addr)->is_forwarded()) { + if (!FullGCForwarding::is_forwarded(cast_to_oop(cur_addr))) { cur_addr = *(HeapWord**) cur_addr; continue; } @@ -593,7 +594,7 @@ void SerialFullGC::mark_object(oop obj) { // some marks may contain information we need to preserve so we store them away // and overwrite the mark. We'll restore it at the end of serial full GC. markWord mark = obj->mark(); - obj->set_mark(markWord::prototype().set_marked()); + obj->set_mark(obj->prototype_mark().set_marked()); ContinuationGCSupport::transform_stack_chunk(obj); @@ -624,8 +625,8 @@ template void SerialFullGC::adjust_pointer(T* p) { oop obj = CompressedOops::decode_not_null(heap_oop); assert(Universe::heap()->is_in(obj), "should be in heap"); - if (obj->is_forwarded()) { - oop new_obj = obj->forwardee(); + if (FullGCForwarding::is_forwarded(obj)) { + oop new_obj = FullGCForwarding::forwardee(obj); assert(is_object_aligned(new_obj), "oop must be aligned"); RawAccess::oop_store(p, new_obj); } diff --git a/src/hotspot/share/gc/serial/serialHeap.cpp b/src/hotspot/share/gc/serial/serialHeap.cpp index 9dcfb5b6092..f5e6a46df29 100644 --- a/src/hotspot/share/gc/serial/serialHeap.cpp +++ b/src/hotspot/share/gc/serial/serialHeap.cpp @@ -40,6 +40,7 @@ #include "gc/shared/collectedHeap.inline.hpp" #include "gc/shared/collectorCounters.hpp" #include "gc/shared/continuationGCSupport.inline.hpp" +#include "gc/shared/fullGCForwarding.hpp" #include "gc/shared/gcId.hpp" #include "gc/shared/gcInitLogger.hpp" #include "gc/shared/gcLocker.inline.hpp" @@ -200,6 +201,8 @@ jint SerialHeap::initialize() { GCInitLogger::print(); + FullGCForwarding::initialize(_reserved); + return JNI_OK; } diff --git a/src/hotspot/share/gc/shared/c2/barrierSetC2.cpp b/src/hotspot/share/gc/shared/c2/barrierSetC2.cpp index 643a7936b9b..080af1c9693 100644 --- a/src/hotspot/share/gc/shared/c2/barrierSetC2.cpp +++ b/src/hotspot/share/gc/shared/c2/barrierSetC2.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2018, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -156,8 +156,8 @@ Node* BarrierSetC2::store_at_resolved(C2Access& access, C2AccessValue& val) cons } store = kit->store_to_memory(kit->control(), access.addr().node(), val.node(), bt, - access.addr().type(), mo, requires_atomic_access, unaligned, - mismatched, unsafe, access.barrier_data()); + mo, requires_atomic_access, unaligned, mismatched, + unsafe, access.barrier_data()); } else { assert(access.is_opt_access(), "either parse or opt access"); C2OptAccess& opt_access = static_cast(access); @@ -217,7 +217,7 @@ Node* BarrierSetC2::load_at_resolved(C2Access& access, const Type* val_type) con unaligned, mismatched, unsafe, access.barrier_data()); load = kit->gvn().transform(load); } else { - load = kit->make_load(control, adr, val_type, access.type(), adr_type, mo, + load = kit->make_load(control, adr, val_type, access.type(), mo, dep, requires_atomic_access, unaligned, mismatched, unsafe, access.barrier_data()); } @@ -710,11 +710,12 @@ int BarrierSetC2::arraycopy_payload_base_offset(bool is_array) { int base_off = is_array ? arrayOopDesc::length_offset_in_bytes() : instanceOopDesc::base_offset_in_bytes(); // base_off: - // 8 - 32-bit VM + // 8 - 32-bit VM or 64-bit VM, compact headers // 12 - 64-bit VM, compressed klass // 16 - 64-bit VM, normal klass if (base_off % BytesPerLong != 0) { assert(UseCompressedClassPointers, ""); + assert(!UseCompactObjectHeaders, ""); if (is_array) { // Exclude length to copy by 8 bytes words. base_off += sizeof(int); diff --git a/src/hotspot/share/gc/shared/c2/barrierSetC2.hpp b/src/hotspot/share/gc/shared/c2/barrierSetC2.hpp index 00fbf1f2c9f..a78fd434ad9 100644 --- a/src/hotspot/share/gc/shared/c2/barrierSetC2.hpp +++ b/src/hotspot/share/gc/shared/c2/barrierSetC2.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2018, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -92,9 +92,9 @@ class C2AccessValuePtr: public C2AccessValue { public: C2AccessValuePtr(Node* node, const TypePtr* type) : - C2AccessValue(node, reinterpret_cast(type)) {} + C2AccessValue(node, type) {} - const TypePtr* type() const { return reinterpret_cast(_type); } + const TypePtr* type() const { return _type->is_ptr(); } }; // This class wraps a bunch of context parameters that are passed around in the diff --git a/src/hotspot/share/gc/shared/collectedHeap.cpp b/src/hotspot/share/gc/shared/collectedHeap.cpp index 82eaaf9a396..3f0447b6558 100644 --- a/src/hotspot/share/gc/shared/collectedHeap.cpp +++ b/src/hotspot/share/gc/shared/collectedHeap.cpp @@ -220,6 +220,26 @@ bool CollectedHeap::supports_concurrent_gc_breakpoints() const { return false; } +static bool klass_is_sane(oop object) { + if (UseCompactObjectHeaders) { + // With compact headers, we can't safely access the Klass* when + // the object has been forwarded, because non-full-GC-forwarding + // temporarily overwrites the mark-word, and thus the Klass*, with + // the forwarding pointer, and here we have no way to make a + // distinction between Full-GC and regular GC forwarding. + markWord mark = object->mark(); + if (mark.is_forwarded()) { + // We can't access the Klass*. We optimistically assume that + // it is ok. This happens very rarely. + return true; + } + + return Metaspace::contains(mark.klass_without_asserts()); + } + + return Metaspace::contains(object->klass_without_asserts()); +} + bool CollectedHeap::is_oop(oop object) const { if (!is_object_aligned(object)) { return false; @@ -229,7 +249,7 @@ bool CollectedHeap::is_oop(oop object) const { return false; } - if (!Metaspace::contains(object->klass_without_asserts())) { + if (!klass_is_sane(object)) { return false; } diff --git a/src/hotspot/share/gc/shared/collectedHeap.hpp b/src/hotspot/share/gc/shared/collectedHeap.hpp index b413e3dfb43..036bc0230c8 100644 --- a/src/hotspot/share/gc/shared/collectedHeap.hpp +++ b/src/hotspot/share/gc/shared/collectedHeap.hpp @@ -306,7 +306,7 @@ class CollectedHeap : public CHeapObj { } virtual void fill_with_dummy_object(HeapWord* start, HeapWord* end, bool zap); - static constexpr size_t min_dummy_object_size() { + static size_t min_dummy_object_size() { return oopDesc::header_size(); } diff --git a/src/hotspot/share/gc/shared/fullGCForwarding.cpp b/src/hotspot/share/gc/shared/fullGCForwarding.cpp new file mode 100644 index 00000000000..4880b08887e --- /dev/null +++ b/src/hotspot/share/gc/shared/fullGCForwarding.cpp @@ -0,0 +1,57 @@ +/* + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/shared/fullGCForwarding.hpp" +#include "memory/memRegion.hpp" +#include "runtime/globals_extension.hpp" + +HeapWord* FullGCForwarding::_heap_base = nullptr; +int FullGCForwarding::_num_low_bits = 0; + +void FullGCForwarding::initialize_flags(size_t max_heap_size) { +#ifdef _LP64 + size_t max_narrow_heap_size = right_n_bits(NumLowBitsNarrow - Shift); + if (UseCompactObjectHeaders && max_heap_size > max_narrow_heap_size * HeapWordSize) { + warning("Compact object headers require a java heap size smaller than " SIZE_FORMAT + "%s (given: " SIZE_FORMAT "%s). Disabling compact object headers.", + byte_size_in_proper_unit(max_narrow_heap_size * HeapWordSize), + proper_unit_for_byte_size(max_narrow_heap_size * HeapWordSize), + byte_size_in_proper_unit(max_heap_size), + proper_unit_for_byte_size(max_heap_size)); + FLAG_SET_ERGO(UseCompactObjectHeaders, false); + } +#endif +} + +void FullGCForwarding::initialize(MemRegion heap) { +#ifdef _LP64 + _heap_base = heap.start(); + if (UseCompactObjectHeaders) { + _num_low_bits = NumLowBitsNarrow; + } else { + _num_low_bits = NumLowBitsWide; + } +#endif +} diff --git a/src/hotspot/share/gc/shared/fullGCForwarding.hpp b/src/hotspot/share/gc/shared/fullGCForwarding.hpp new file mode 100644 index 00000000000..a6ca182428b --- /dev/null +++ b/src/hotspot/share/gc/shared/fullGCForwarding.hpp @@ -0,0 +1,59 @@ +/* + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_GC_SHARED_FULLGCFORWARDING_HPP +#define SHARE_GC_SHARED_FULLGCFORWARDING_HPP + +#include "memory/allStatic.hpp" +#include "memory/memRegion.hpp" +#include "oops/markWord.hpp" +#include "oops/oopsHierarchy.hpp" + +/* + * Implements forwarding for the Full GCs of Serial, Parallel, G1 and Shenandoah in + * a way that preserves upper N bits of object mark-words, which contain crucial + * Klass* information when running with compact headers. The encoding is similar to + * compressed-oops encoding: it basically subtracts the forwardee address from the + * heap-base, shifts that difference into the right place, and sets the lowest two + * bits (to indicate 'forwarded' state as usual). + * With compact-headers, we have 40 bits to encode forwarding pointers. This is + * enough to address 8TB of heap. If the heap size exceeds that limit, we turn off + * compact headers. + */ +class FullGCForwarding : public AllStatic { + static const int NumLowBitsNarrow = LP64_ONLY(markWord::klass_shift) NOT_LP64(0 /*unused*/); + static const int NumLowBitsWide = BitsPerWord; + static const int Shift = markWord::lock_bits + markWord::lock_shift; + + static HeapWord* _heap_base; + static int _num_low_bits; +public: + static void initialize_flags(size_t max_heap_size); + static void initialize(MemRegion heap); + static inline void forward_to(oop from, oop to); + static inline oop forwardee(oop from); + static inline bool is_forwarded(oop obj); +}; + +#endif // SHARE_GC_SHARED_FULLGCFORWARDING_HPP diff --git a/src/hotspot/share/gc/shared/fullGCForwarding.inline.hpp b/src/hotspot/share/gc/shared/fullGCForwarding.inline.hpp new file mode 100644 index 00000000000..ebd280a454f --- /dev/null +++ b/src/hotspot/share/gc/shared/fullGCForwarding.inline.hpp @@ -0,0 +1,60 @@ +/* + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef GC_SHARED_FULLGCFORWARDING_INLINE_HPP +#define GC_SHARED_FULLGCFORWARDING_INLINE_HPP + +#include "gc/shared/fullGCForwarding.hpp" + +#include "oops/oop.inline.hpp" +#include "utilities/globalDefinitions.hpp" + +void FullGCForwarding::forward_to(oop from, oop to) { +#ifdef _LP64 + uintptr_t encoded = pointer_delta(cast_from_oop(to), _heap_base) << Shift; + assert(encoded <= static_cast(right_n_bits(_num_low_bits)), "encoded forwardee must fit"); + uintptr_t mark = from->mark().value(); + mark &= ~right_n_bits(_num_low_bits); + mark |= (encoded | markWord::marked_value); + from->set_mark(markWord(mark)); +#else + from->forward_to(to); +#endif +} + +oop FullGCForwarding::forwardee(oop from) { +#ifdef _LP64 + uintptr_t mark = from->mark().value(); + HeapWord* decoded = _heap_base + ((mark & right_n_bits(_num_low_bits)) >> Shift); + return cast_to_oop(decoded); +#else + return from->forwardee(); +#endif +} + +bool FullGCForwarding::is_forwarded(oop obj) { + return obj->mark().is_forwarded(); +} + +#endif // GC_SHARED_FULLGCFORWARDING_INLINE_HPP diff --git a/src/hotspot/share/gc/shared/locationPrinter.inline.hpp b/src/hotspot/share/gc/shared/locationPrinter.inline.hpp index d368aa1c6c6..ae873d52cb5 100644 --- a/src/hotspot/share/gc/shared/locationPrinter.inline.hpp +++ b/src/hotspot/share/gc/shared/locationPrinter.inline.hpp @@ -37,7 +37,7 @@ oop BlockLocationPrinter::base_oop_or_null(void* addr) { return cast_to_oop(addr); } - // Try to find addr using block_start. + // Try to find addr using block_start (not implemented for all GCs/generations). HeapWord* p = CollectedHeapT::heap()->block_start(addr); if (p != nullptr && CollectedHeapT::heap()->block_is_obj(p)) { if (!is_valid_obj(p)) { @@ -52,7 +52,9 @@ oop BlockLocationPrinter::base_oop_or_null(void* addr) { template bool BlockLocationPrinter::print_location(outputStream* st, void* addr) { // Check if addr points into Java heap. - if (CollectedHeapT::heap()->is_in(addr)) { + bool in_heap = CollectedHeapT::heap()->is_in(addr); + if (in_heap) { + // base_oop_or_null() might be unimplemented and return NULL for some GCs/generations oop o = base_oop_or_null(addr); if (o != nullptr) { if ((void*)o == addr) { @@ -83,6 +85,10 @@ bool BlockLocationPrinter::print_location(outputStream* st, void } #endif + if (in_heap) { + st->print_cr(PTR_FORMAT " is an unknown heap location", p2i(addr)); + return true; + } return false; } diff --git a/src/hotspot/share/gc/shared/memAllocator.cpp b/src/hotspot/share/gc/shared/memAllocator.cpp index 318ab00188b..f96ec50e3b0 100644 --- a/src/hotspot/share/gc/shared/memAllocator.cpp +++ b/src/hotspot/share/gc/shared/memAllocator.cpp @@ -361,18 +361,23 @@ void MemAllocator::mem_clear(HeapWord* mem) const { assert(mem != nullptr, "cannot initialize null object"); const size_t hs = oopDesc::header_size(); assert(_word_size >= hs, "unexpected object size"); - oopDesc::set_klass_gap(mem, 0); + if (oopDesc::has_klass_gap()) { + oopDesc::set_klass_gap(mem, 0); + } Copy::fill_to_aligned_words(mem + hs, _word_size - hs); } oop MemAllocator::finish(HeapWord* mem) const { assert(mem != nullptr, "null object pointer"); - // May be bootstrapping - oopDesc::set_mark(mem, markWord::prototype()); // Need a release store to ensure array/class length, mark word, and // object zeroing are visible before setting the klass non-null, for // concurrent collectors. - oopDesc::release_set_klass(mem, _klass); + if (UseCompactObjectHeaders) { + oopDesc::release_set_mark(mem, _klass->prototype_header()); + } else { + oopDesc::set_mark(mem, markWord::prototype()); + oopDesc::release_set_klass(mem, _klass); + } return cast_to_oop(mem); } diff --git a/src/hotspot/share/gc/shared/preservedMarks.cpp b/src/hotspot/share/gc/shared/preservedMarks.cpp index 9889dbc3690..bc241fb5daf 100644 --- a/src/hotspot/share/gc/shared/preservedMarks.cpp +++ b/src/hotspot/share/gc/shared/preservedMarks.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2016, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -23,6 +23,7 @@ */ #include "precompiled.hpp" +#include "gc/shared/fullGCForwarding.inline.hpp" #include "gc/shared/preservedMarks.inline.hpp" #include "gc/shared/workerThread.hpp" #include "gc/shared/workerUtils.hpp" @@ -42,8 +43,8 @@ void PreservedMarks::restore() { void PreservedMarks::adjust_preserved_mark(PreservedMark* elem) { oop obj = elem->get_oop(); - if (obj->is_forwarded()) { - elem->set_oop(obj->forwardee()); + if (FullGCForwarding::is_forwarded(obj)) { + elem->set_oop(FullGCForwarding::forwardee(obj)); } } @@ -142,10 +143,6 @@ void PreservedMarksSet::restore(WorkerThreads* workers) { assert_empty(); } -WorkerTask* PreservedMarksSet::create_task() { - return new RestorePreservedMarksTask(this); -} - void PreservedMarksSet::reclaim() { assert_empty(); diff --git a/src/hotspot/share/gc/shared/preservedMarks.hpp b/src/hotspot/share/gc/shared/preservedMarks.hpp index 3fc8c62bff1..84bf18e41af 100644 --- a/src/hotspot/share/gc/shared/preservedMarks.hpp +++ b/src/hotspot/share/gc/shared/preservedMarks.hpp @@ -114,8 +114,6 @@ class PreservedMarksSet : public CHeapObj { // is null, perform the work serially in the current thread. void restore(WorkerThreads* workers); - WorkerTask* create_task(); - // Reclaim stack array. void reclaim(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahArguments.cpp b/src/hotspot/share/gc/shenandoah/shenandoahArguments.cpp index bcc370eeb31..ca95cde83b1 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahArguments.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahArguments.cpp @@ -23,6 +23,7 @@ */ #include "precompiled.hpp" +#include "gc/shared/fullGCForwarding.hpp" #include "gc/shared/gcArguments.hpp" #include "gc/shared/tlab_globals.hpp" #include "gc/shared/workerPolicy.hpp" @@ -198,6 +199,11 @@ void ShenandoahArguments::initialize_alignments() { HeapAlignment = align; } +void ShenandoahArguments::initialize_heap_flags_and_sizes() { + GCArguments::initialize_heap_flags_and_sizes(); + FullGCForwarding::initialize_flags(MaxHeapSize); +} + CollectedHeap* ShenandoahArguments::create_heap() { return new ShenandoahHeap(new ShenandoahCollectorPolicy()); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahArguments.hpp b/src/hotspot/share/gc/shenandoah/shenandoahArguments.hpp index bc73d9a2d12..ad54b1d235c 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahArguments.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahArguments.hpp @@ -35,6 +35,7 @@ class ShenandoahArguments : public GCArguments { virtual void initialize(); virtual size_t conservative_max_heap_alignment(); + virtual void initialize_heap_flags_and_sizes(); virtual CollectedHeap* create_heap(); }; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahAsserts.cpp b/src/hotspot/share/gc/shenandoah/shenandoahAsserts.cpp index 0bbced2e8d3..22822ad3fec 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahAsserts.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahAsserts.cpp @@ -197,7 +197,7 @@ void ShenandoahAsserts::assert_correct(void* interior_loc, oop obj, const char* file, line); } - Klass* obj_klass = obj->klass_or_null(); + Klass* obj_klass = ShenandoahForwarding::klass(obj); if (obj_klass == nullptr) { print_failure(_safe_unknown, obj, interior_loc, nullptr, "Shenandoah assert_correct failed", "Object klass pointer should not be null", @@ -235,7 +235,7 @@ void ShenandoahAsserts::assert_correct(void* interior_loc, oop obj, const char* file, line); } - if (obj_klass != fwd->klass()) { + if (obj_klass != ShenandoahForwarding::klass(fwd)) { print_failure(_safe_oop, obj, interior_loc, nullptr, "Shenandoah assert_correct failed", "Forwardee klass disagrees with object class", file, line); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahForwarding.hpp b/src/hotspot/share/gc/shenandoah/shenandoahForwarding.hpp index 95152017a69..fb57c55e09a 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahForwarding.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahForwarding.hpp @@ -62,6 +62,8 @@ class ShenandoahForwarding { */ static inline oop try_update_forwardee(oop obj, oop update); + static inline size_t size(oop obj); + static inline Klass* klass(oop obj); }; #endif // SHARE_GC_SHENANDOAH_SHENANDOAHFORWARDING_HPP diff --git a/src/hotspot/share/gc/shenandoah/shenandoahForwarding.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahForwarding.inline.hpp index 01294f9c890..ccdbb81f33b 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahForwarding.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahForwarding.inline.hpp @@ -90,4 +90,21 @@ inline oop ShenandoahForwarding::try_update_forwardee(oop obj, oop update) { } } +inline Klass* ShenandoahForwarding::klass(oop obj) { + if (UseCompactObjectHeaders) { + markWord mark = obj->mark(); + if (mark.is_marked()) { + oop fwd = cast_to_oop(mark.clear_lock_bits().to_pointer()); + mark = fwd->mark(); + } + return mark.klass(); + } else { + return obj->klass(); + } +} + +inline size_t ShenandoahForwarding::size(oop obj) { + return obj->size_given_klass(klass(obj)); +} + #endif // SHARE_GC_SHENANDOAH_SHENANDOAHFORWARDING_INLINE_HPP diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp index c2d94353d54..53cb8e5d20f 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp @@ -26,6 +26,7 @@ #include "compiler/oopMap.hpp" #include "gc/shared/continuationGCSupport.hpp" +#include "gc/shared/fullGCForwarding.inline.hpp" #include "gc/shared/gcTraceTime.inline.hpp" #include "gc/shared/preservedMarks.inline.hpp" #include "gc/shared/tlab_globals.hpp" @@ -369,7 +370,7 @@ class ShenandoahPrepareForCompactionObjectClosure : public ObjectClosure { shenandoah_assert_not_forwarded(nullptr, p); if (_compact_point != cast_from_oop(p)) { _preserved_marks->push_if_necessary(p, p->mark()); - p->forward_to(cast_to_oop(_compact_point)); + FullGCForwarding::forward_to(p, cast_to_oop(_compact_point)); } _compact_point += obj_size; } @@ -492,7 +493,7 @@ void ShenandoahFullGC::calculate_target_humongous_objects() { if (start >= to_begin && start != r->index()) { // Fits into current window, and the move is non-trivial. Record the move then, and continue scan. _preserved_marks->get(0)->push_if_necessary(old_obj, old_obj->mark()); - old_obj->forward_to(cast_to_oop(heap->get_region(start)->bottom())); + FullGCForwarding::forward_to(old_obj, cast_to_oop(heap->get_region(start)->bottom())); to_end = start; continue; } @@ -752,8 +753,8 @@ class ShenandoahAdjustPointersClosure : public MetadataVisitingOopIterateClosure if (!CompressedOops::is_null(o)) { oop obj = CompressedOops::decode_not_null(o); assert(_ctx->is_marked(obj), "must be marked"); - if (obj->is_forwarded()) { - oop forw = obj->forwardee(); + if (FullGCForwarding::is_forwarded(obj)) { + oop forw = FullGCForwarding::forwardee(obj); RawAccess::oop_store(p, forw); } } @@ -863,9 +864,9 @@ class ShenandoahCompactObjectsClosure : public ObjectClosure { void do_object(oop p) { assert(_heap->complete_marking_context()->is_marked(p), "must be marked"); size_t size = p->size(); - if (p->is_forwarded()) { + if (FullGCForwarding::is_forwarded(p)) { HeapWord* compact_from = cast_from_oop(p); - HeapWord* compact_to = cast_from_oop(p->forwardee()); + HeapWord* compact_to = cast_from_oop(FullGCForwarding::forwardee(p)); assert(compact_from != compact_to, "Forwarded object should move"); Copy::aligned_conjoint_words(compact_from, compact_to, size); oop new_obj = cast_to_oop(compact_to); @@ -970,7 +971,7 @@ void ShenandoahFullGC::compact_humongous_objects() { ShenandoahHeapRegion* r = heap->get_region(c - 1); if (r->is_humongous_start()) { oop old_obj = cast_to_oop(r->bottom()); - if (!old_obj->is_forwarded()) { + if (!FullGCForwarding::is_forwarded(old_obj)) { // No need to move the object, it stays at the same slot continue; } @@ -979,7 +980,7 @@ void ShenandoahFullGC::compact_humongous_objects() { size_t old_start = r->index(); size_t old_end = old_start + num_regions - 1; - size_t new_start = heap->heap_region_index_containing(old_obj->forwardee()); + size_t new_start = heap->heap_region_index_containing(FullGCForwarding::forwardee(old_obj)); size_t new_end = new_start + num_regions - 1; assert(old_start != new_start, "must be real move"); assert(r->is_stw_move_allowed(), "Region " SIZE_FORMAT " should be movable", r->index()); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp index 07914947ead..0f7139691a3 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -28,6 +28,7 @@ #include "memory/universe.hpp" #include "gc/shared/classUnloadingContext.hpp" +#include "gc/shared/fullGCForwarding.hpp" #include "gc/shared/gcArguments.hpp" #include "gc/shared/gcTimer.hpp" #include "gc/shared/gcTraceTime.inline.hpp" @@ -421,6 +422,8 @@ jint ShenandoahHeap::initialize() { ShenandoahInitLogger::print(); + FullGCForwarding::initialize(_heap_region); + return JNI_OK; } @@ -1129,7 +1132,7 @@ oop ShenandoahHeap::evacuate_object(oop p, Thread* thread) { assert(ShenandoahThreadLocalData::is_evac_allowed(thread), "must be enclosed in oom-evac scope"); - size_t size = p->size(); + size_t size = ShenandoahForwarding::size(p); assert(!heap_region_containing(p)->is_humongous(), "never evacuate humongous objects"); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp index cd2cc2eb209..6131d079590 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp @@ -434,7 +434,7 @@ inline void ShenandoahHeap::marked_object_iterate(ShenandoahHeapRegion* region, oop obj = cast_to_oop(cs); assert(oopDesc::is_oop(obj), "sanity"); assert(ctx->is_marked(obj), "object expected to be marked"); - size_t size = obj->size(); + size_t size = ShenandoahForwarding::size(obj); cl->do_object(obj); cs += size; } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp b/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp index 51878772889..b653a06b877 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp @@ -102,7 +102,7 @@ class ShenandoahVerifyOopClosure : public BasicOopIterateClosure { T o = RawAccess<>::oop_load(p); if (!CompressedOops::is_null(o)) { oop obj = CompressedOops::decode_not_null(o); - if (is_instance_ref_klass(obj->klass())) { + if (is_instance_ref_klass(ShenandoahForwarding::klass(obj))) { obj = ShenandoahForwarding::get_forwardee(obj); } // Single threaded verification can use faster non-atomic stack and bitmap @@ -129,7 +129,7 @@ class ShenandoahVerifyOopClosure : public BasicOopIterateClosure { "oop must be aligned"); ShenandoahHeapRegion *obj_reg = _heap->heap_region_containing(obj); - Klass* obj_klass = obj->klass_or_null(); + Klass* obj_klass = ShenandoahForwarding::klass(obj); // Verify that obj is not in dead space: { @@ -144,11 +144,11 @@ class ShenandoahVerifyOopClosure : public BasicOopIterateClosure { "Object start should be within the region"); if (!obj_reg->is_humongous()) { - check(ShenandoahAsserts::_safe_unknown, obj, (obj_addr + obj->size()) <= obj_reg->top(), + check(ShenandoahAsserts::_safe_unknown, obj, (obj_addr + ShenandoahForwarding::size(obj)) <= obj_reg->top(), "Object end should be within the region"); } else { size_t humongous_start = obj_reg->index(); - size_t humongous_end = humongous_start + (obj->size() >> ShenandoahHeapRegion::region_size_words_shift()); + size_t humongous_end = humongous_start + (ShenandoahForwarding::size(obj) >> ShenandoahHeapRegion::region_size_words_shift()); for (size_t idx = humongous_start + 1; idx < humongous_end; idx++) { check(ShenandoahAsserts::_safe_unknown, obj, _heap->get_region(idx)->is_humongous_continuation(), "Humongous object is in continuation that fits it"); @@ -165,7 +165,7 @@ class ShenandoahVerifyOopClosure : public BasicOopIterateClosure { // skip break; case ShenandoahVerifier::_verify_liveness_complete: - Atomic::add(&_ld[obj_reg->index()], (uint) obj->size(), memory_order_relaxed); + Atomic::add(&_ld[obj_reg->index()], (uint) ShenandoahForwarding::size(obj), memory_order_relaxed); // fallthrough for fast failure for un-live regions: case ShenandoahVerifier::_verify_liveness_conservative: check(ShenandoahAsserts::_safe_oop, obj, obj_reg->has_live(), @@ -209,7 +209,7 @@ class ShenandoahVerifyOopClosure : public BasicOopIterateClosure { HeapWord *fwd_addr = cast_from_oop(fwd); check(ShenandoahAsserts::_safe_oop, obj, fwd_addr < fwd_reg->top(), "Forwardee start should be within the region"); - check(ShenandoahAsserts::_safe_oop, obj, (fwd_addr + fwd->size()) <= fwd_reg->top(), + check(ShenandoahAsserts::_safe_oop, obj, (fwd_addr + ShenandoahForwarding::size(fwd)) <= fwd_reg->top(), "Forwardee end should be within the region"); oop fwd2 = ShenandoahForwarding::get_forwardee_raw_unchecked(fwd); @@ -327,7 +327,11 @@ class ShenandoahVerifyOopClosure : public BasicOopIterateClosure { */ void verify_oops_from(oop obj) { _loc = obj; - obj->oop_iterate(this); + // oop_iterate() can not deal with forwarded objects, because + // it needs to load klass(), which may be overridden by the + // forwarding pointer. + oop fwd = ShenandoahForwarding::get_forwardee_raw(obj); + fwd->oop_iterate(this); _loc = nullptr; } @@ -591,7 +595,7 @@ class ShenandoahVerifierMarkedRegionTask : public WorkerTask { while (addr < limit) { verify_and_follow(addr, stack, cl, &processed); - addr += cast_to_oop(addr)->size(); + addr += ShenandoahForwarding::size(cast_to_oop(addr)); } } @@ -607,7 +611,7 @@ class ShenandoahVerifierMarkedRegionTask : public WorkerTask { // Verify everything reachable from that object too, hopefully realizing // everything was already marked, and never touching further: - if (!is_instance_ref_klass(obj->klass())) { + if (!is_instance_ref_klass(ShenandoahForwarding::klass(obj))) { cl.verify_oops_from(obj); (*processed)++; } diff --git a/src/hotspot/share/gc/z/c2/zBarrierSetC2.cpp b/src/hotspot/share/gc/z/c2/zBarrierSetC2.cpp index 12ee400eb2d..3f9c09f1c8c 100644 --- a/src/hotspot/share/gc/z/c2/zBarrierSetC2.cpp +++ b/src/hotspot/share/gc/z/c2/zBarrierSetC2.cpp @@ -445,7 +445,7 @@ void ZBarrierSetC2::clone_at_expansion(PhaseMacroExpand* phase, ArrayCopyNode* a assert(src_offset == dest_offset, "should be equal"); const jlong offset = src_offset->get_long(); if (offset != arrayOopDesc::base_offset_in_bytes(T_OBJECT)) { - assert(!UseCompressedClassPointers, "should only happen without compressed class pointers"); + assert(!UseCompressedClassPointers || UseCompactObjectHeaders, "should only happen without compressed class pointers"); assert((arrayOopDesc::base_offset_in_bytes(T_OBJECT) - offset) == BytesPerLong, "unexpected offset"); length = phase->transform_later(new SubLNode(length, phase->longcon(1))); // Size is in longs src_offset = phase->longcon(arrayOopDesc::base_offset_in_bytes(T_OBJECT)); diff --git a/src/hotspot/share/gc/z/zObjArrayAllocator.cpp b/src/hotspot/share/gc/z/zObjArrayAllocator.cpp index ada8351a9f6..c63b989dc4e 100644 --- a/src/hotspot/share/gc/z/zObjArrayAllocator.cpp +++ b/src/hotspot/share/gc/z/zObjArrayAllocator.cpp @@ -63,8 +63,12 @@ oop ZObjArrayAllocator::initialize(HeapWord* mem) const { // Signal to the ZIterator that this is an invisible root, by setting // the mark word to "marked". Reset to prototype() after the clearing. - arrayOopDesc::set_mark(mem, markWord::prototype().set_marked()); - arrayOopDesc::release_set_klass(mem, _klass); + if (UseCompactObjectHeaders) { + oopDesc::release_set_mark(mem, _klass->prototype_header().set_marked()); + } else { + arrayOopDesc::set_mark(mem, markWord::prototype().set_marked()); + arrayOopDesc::release_set_klass(mem, _klass); + } assert(_length >= 0, "length should be non-negative"); arrayOopDesc::set_length(mem, _length); @@ -152,7 +156,11 @@ oop ZObjArrayAllocator::initialize(HeapWord* mem) const { ZThreadLocalData::clear_invisible_root(_thread); // Signal to the ZIterator that this is no longer an invisible root - oopDesc::release_set_mark(mem, markWord::prototype()); + if (UseCompactObjectHeaders) { + oopDesc::release_set_mark(mem, _klass->prototype_header()); + } else { + oopDesc::release_set_mark(mem, markWord::prototype()); + } return cast_to_oop(mem); } diff --git a/src/hotspot/share/include/cds.h b/src/hotspot/share/include/cds.h index af48c1a8c35..4cc309dd991 100644 --- a/src/hotspot/share/include/cds.h +++ b/src/hotspot/share/include/cds.h @@ -69,7 +69,7 @@ typedef struct CDSFileMapRegion { size_t _ptrmap_offset; // Bitmap for relocating native pointer fields in archived heap objects. // (The base address is the bottom of the BM region). size_t _ptrmap_size_in_bits; - char* _mapped_base; // Actually mapped address (NULL if this region is not mapped). + char* _mapped_base; // Actually mapped address (null if this region is not mapped). } CDSFileMapRegion; // This portion of the archive file header must remain unchanged for diff --git a/src/hotspot/share/include/jvm.h b/src/hotspot/share/include/jvm.h index 6634306636b..e50d758335c 100644 --- a/src/hotspot/share/include/jvm.h +++ b/src/hotspot/share/include/jvm.h @@ -419,7 +419,7 @@ JVM_FindPrimitiveClass(JNIEnv *env, const char *utf); /* - * Find a class from a boot class loader. Returns NULL if class not found. + * Find a class from a boot class loader. Returns null if class not found. */ JNIEXPORT jclass JNICALL JVM_FindClassFromBootLoader(JNIEnv *env, const char *name); @@ -723,13 +723,6 @@ JNIEXPORT jbyte JNICALL JVM_ConstantPoolGetTagAt JNIEXPORT jobjectArray JNICALL JVM_GetMethodParameters(JNIEnv *env, jobject method); -/* - * java.security.* - */ - -JNIEXPORT jobject JNICALL -JVM_GetInheritedAccessControlContext(JNIEnv *env, jclass cls); - /* * Ensure that code doing a stackwalk and using javaVFrame::locals() to * get the value will see a materialized value and not a scalar-replaced @@ -741,9 +734,6 @@ JVM_GetInheritedAccessControlContext(JNIEnv *env, jclass cls); JNIEXPORT void JNICALL JVM_EnsureMaterializedForStackWalk_func(JNIEnv* env, jobject vthread, jobject value); -JNIEXPORT jobject JNICALL -JVM_GetStackAccessControlContext(JNIEnv *env, jclass cls); - /* * Signal support, used to implement the shutdown sequence. Every VM must * support JVM_SIGINT and JVM_SIGTERM, raising the former for user interrupts @@ -1145,6 +1135,12 @@ JVM_VirtualThreadUnmount(JNIEnv* env, jobject vthread, jboolean hide); JNIEXPORT void JNICALL JVM_VirtualThreadDisableSuspend(JNIEnv* env, jclass clazz, jboolean enter); +JNIEXPORT void JNICALL +JVM_VirtualThreadPinnedEvent(JNIEnv* env, jclass clazz, jstring op); + +JNIEXPORT jobject JNICALL +JVM_TakeVirtualThreadListToUnblock(JNIEnv* env, jclass ignored); + /* * Core reflection support. */ @@ -1157,6 +1153,13 @@ JVM_GetClassFileVersion(JNIEnv *env, jclass current); JNIEXPORT jboolean JNICALL JVM_PrintWarningAtDynamicAgentLoad(void); +#define JNI_ONLOAD_SYMBOLS {"JNI_OnLoad"} +#define JNI_ONUNLOAD_SYMBOLS {"JNI_OnUnload"} +#define JVM_ONLOAD_SYMBOLS {"JVM_OnLoad"} +#define AGENT_ONLOAD_SYMBOLS {"Agent_OnLoad"} +#define AGENT_ONUNLOAD_SYMBOLS {"Agent_OnUnload"} +#define AGENT_ONATTACH_SYMBOLS {"Agent_OnAttach"} + /* * This structure is used by the launcher to get the default thread * stack size from the VM using JNI_GetDefaultJavaVMInitArgs() with a diff --git a/src/hotspot/share/interpreter/interpreterRuntime.cpp b/src/hotspot/share/interpreter/interpreterRuntime.cpp index 525258b1ebd..1d8268dbda6 100644 --- a/src/hotspot/share/interpreter/interpreterRuntime.cpp +++ b/src/hotspot/share/interpreter/interpreterRuntime.cpp @@ -735,7 +735,7 @@ JRT_ENTRY_NO_ASYNC(void, InterpreterRuntime::monitorenter(JavaThread* current, B assert(Universe::heap()->is_in_or_null(elem->obj()), "must be null or an object"); #ifdef ASSERT - current->last_frame().interpreter_frame_verify_monitor(elem); + if (!current->preempting()) current->last_frame().interpreter_frame_verify_monitor(elem); #endif JRT_END diff --git a/src/hotspot/share/interpreter/oopMapCache.cpp b/src/hotspot/share/interpreter/oopMapCache.cpp index 87b124e9d79..3406c85df7f 100644 --- a/src/hotspot/share/interpreter/oopMapCache.cpp +++ b/src/hotspot/share/interpreter/oopMapCache.cpp @@ -234,8 +234,10 @@ class MaskFillerForNative: public NativeSignatureIterator { private: uintptr_t * _mask; // the bit mask to be filled int _size; // the mask size in bits + int _num_oops; void set_one(int i) { + _num_oops++; i *= InterpreterOopMap::bits_per_entry; assert(0 <= i && i < _size, "offset out of bounds"); _mask[i / BitsPerWord] |= (((uintptr_t) 1 << InterpreterOopMap::oop_bit_number) << (i % BitsPerWord)); @@ -253,6 +255,7 @@ class MaskFillerForNative: public NativeSignatureIterator { MaskFillerForNative(const methodHandle& method, uintptr_t* mask, int size) : NativeSignatureIterator(method) { _mask = mask; _size = size; + _num_oops = 0; // initialize with 0 int i = (size + BitsPerWord - 1) / BitsPerWord; while (i-- > 0) _mask[i] = 0; @@ -261,6 +264,8 @@ class MaskFillerForNative: public NativeSignatureIterator { void generate() { iterate(); } + + int num_oops() { return _num_oops; } }; bool OopMapCacheEntry::verify_mask(CellTypeState* vars, CellTypeState* stack, int max_locals, int stack_top) { @@ -319,6 +324,7 @@ void OopMapCacheEntry::fill_for_native(const methodHandle& mh) { // fill mask for parameters MaskFillerForNative mf(mh, bit_mask(), mask_size()); mf.generate(); + _num_oops = mf.num_oops(); } diff --git a/src/hotspot/share/interpreter/templateInterpreter.cpp b/src/hotspot/share/interpreter/templateInterpreter.cpp index 58559481e5d..dc3e4b67b33 100644 --- a/src/hotspot/share/interpreter/templateInterpreter.cpp +++ b/src/hotspot/share/interpreter/templateInterpreter.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -212,6 +212,7 @@ address TemplateInterpreter::_throw_ClassCastException_entry = nu address TemplateInterpreter::_throw_NullPointerException_entry = nullptr; address TemplateInterpreter::_throw_StackOverflowError_entry = nullptr; address TemplateInterpreter::_throw_exception_entry = nullptr; +address TemplateInterpreter::_cont_resume_interpreter_adapter = nullptr; #ifndef PRODUCT EntryPoint TemplateInterpreter::_trace_code; diff --git a/src/hotspot/share/interpreter/templateInterpreter.hpp b/src/hotspot/share/interpreter/templateInterpreter.hpp index 9d3c5545fa0..224d9b973ed 100644 --- a/src/hotspot/share/interpreter/templateInterpreter.hpp +++ b/src/hotspot/share/interpreter/templateInterpreter.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -111,6 +111,8 @@ class TemplateInterpreter: public AbstractInterpreter { static address _throw_StackOverflowError_entry; + static address _cont_resume_interpreter_adapter; + static address _remove_activation_entry; // continuation address if an exception is not handled by current frame static address _remove_activation_preserving_args_entry; // continuation address when current frame is being popped @@ -154,6 +156,8 @@ class TemplateInterpreter: public AbstractInterpreter { static address throw_NullPointerException_entry() { return _throw_NullPointerException_entry; } static address throw_StackOverflowError_entry() { return _throw_StackOverflowError_entry; } + static address cont_resume_interpreter_adapter() { return _cont_resume_interpreter_adapter; } + // Code generation #ifndef PRODUCT static address trace_code (TosState state) { return _trace_code.entry(state); } diff --git a/src/hotspot/share/interpreter/templateInterpreterGenerator.cpp b/src/hotspot/share/interpreter/templateInterpreterGenerator.cpp index 3f497c3360b..91c30288843 100644 --- a/src/hotspot/share/interpreter/templateInterpreterGenerator.cpp +++ b/src/hotspot/share/interpreter/templateInterpreterGenerator.cpp @@ -175,7 +175,9 @@ void TemplateInterpreterGenerator::generate_all() { Interpreter::_throw_StackOverflowError_entry = generate_StackOverflowError_handler(); } - + { CodeletMark cm(_masm, "preemption resume adapter"); + Interpreter::_cont_resume_interpreter_adapter = generate_cont_resume_interpreter_adapter(); + } #define method_entry(kind) \ { CodeletMark cm(_masm, "method entry point (kind = " #kind ")"); \ diff --git a/src/hotspot/share/interpreter/templateInterpreterGenerator.hpp b/src/hotspot/share/interpreter/templateInterpreterGenerator.hpp index b0afcb52795..d3ddf2a830f 100644 --- a/src/hotspot/share/interpreter/templateInterpreterGenerator.hpp +++ b/src/hotspot/share/interpreter/templateInterpreterGenerator.hpp @@ -56,6 +56,7 @@ class TemplateInterpreterGenerator: public AbstractInterpreterGenerator { address generate_earlyret_entry_for(TosState state); address generate_deopt_entry_for(TosState state, int step, address continuation = nullptr); address generate_safept_entry_for(TosState state, address runtime_entry); + address generate_cont_resume_interpreter_adapter(); void generate_throw_exception(); void lock_method(); diff --git a/src/hotspot/share/interpreter/zero/bytecodeInterpreter.cpp b/src/hotspot/share/interpreter/zero/bytecodeInterpreter.cpp index fbdf8f9ca71..fa572537c2f 100644 --- a/src/hotspot/share/interpreter/zero/bytecodeInterpreter.cpp +++ b/src/hotspot/share/interpreter/zero/bytecodeInterpreter.cpp @@ -641,9 +641,6 @@ void BytecodeInterpreter::run(interpreterState istate) { success = false; } } - if (success) { - THREAD->inc_held_monitor_count(); - } } if (!success) { CALL_VM(InterpreterRuntime::monitorenter(THREAD, mon), handle_exception); @@ -745,9 +742,6 @@ void BytecodeInterpreter::run(interpreterState istate) { success = false; } } - if (success) { - THREAD->inc_held_monitor_count(); - } } if (!success) { CALL_VM(InterpreterRuntime::monitorenter(THREAD, entry), handle_exception); @@ -1680,9 +1674,6 @@ void BytecodeInterpreter::run(interpreterState istate) { success = false; } } - if (success) { - THREAD->inc_held_monitor_count(); - } } if (!success) { CALL_VM(InterpreterRuntime::monitorenter(THREAD, entry), handle_exception); @@ -1720,9 +1711,6 @@ void BytecodeInterpreter::run(interpreterState istate) { success = false; } } - if (success) { - THREAD->dec_held_monitor_count(); - } } if (!success) { InterpreterRuntime::monitorexit(most_recent); @@ -2021,10 +2009,13 @@ void BytecodeInterpreter::run(interpreterState istate) { } // Initialize header, mirrors MemAllocator. - oopDesc::set_mark(result, markWord::prototype()); - oopDesc::set_klass_gap(result, 0); - oopDesc::release_set_klass(result, ik); - + if (UseCompactObjectHeaders) { + oopDesc::release_set_mark(result, ik->prototype_header()); + } else { + oopDesc::set_mark(result, markWord::prototype()); + oopDesc::set_klass_gap(result, 0); + oopDesc::release_set_klass(result, ik); + } oop obj = cast_to_oop(result); // Must prevent reordering of stores for object initialization @@ -3163,9 +3154,6 @@ void BytecodeInterpreter::run(interpreterState istate) { success = false; } } - if (success) { - THREAD->dec_held_monitor_count(); - } } if (!success) { InterpreterRuntime::monitorexit(end); @@ -3242,9 +3230,6 @@ void BytecodeInterpreter::run(interpreterState istate) { } } } - if (dec_monitor_count) { - THREAD->dec_held_monitor_count(); - } } } } diff --git a/src/hotspot/share/interpreter/zero/zeroInterpreter.hpp b/src/hotspot/share/interpreter/zero/zeroInterpreter.hpp index 04df0012d67..699d50e778f 100644 --- a/src/hotspot/share/interpreter/zero/zeroInterpreter.hpp +++ b/src/hotspot/share/interpreter/zero/zeroInterpreter.hpp @@ -71,6 +71,7 @@ class ZeroInterpreter: public AbstractInterpreter { static address throw_NullPointerException_entry() { return nullptr; } static address throw_ArithmeticException_entry() { return nullptr; } static address throw_StackOverflowError_entry() { return nullptr; } + static address cont_resume_interpreter_adapter() { return nullptr; } # include "zeroInterpreter_zero.hpp" }; diff --git a/src/hotspot/share/jfr/jni/jfrJavaSupport.cpp b/src/hotspot/share/jfr/jni/jfrJavaSupport.cpp index 4e2493fc251..777c4ced8a3 100644 --- a/src/hotspot/share/jfr/jni/jfrJavaSupport.cpp +++ b/src/hotspot/share/jfr/jni/jfrJavaSupport.cpp @@ -299,6 +299,24 @@ void JfrJavaSupport::set_array_element(jobjectArray arr, jobject element, int in /* * Field access */ +static void write_bool_field(const Handle& h_oop, fieldDescriptor* fd, jboolean value) { + assert(h_oop.not_null(), "invariant"); + assert(fd != nullptr, "invariant"); + h_oop->bool_field_put(fd->offset(), value); +} + +static void write_char_field(const Handle& h_oop, fieldDescriptor* fd, jchar value) { + assert(h_oop.not_null(), "invariant"); + assert(fd != nullptr, "invariant"); + h_oop->char_field_put(fd->offset(), value); +} + +static void write_short_field(const Handle& h_oop, fieldDescriptor* fd, jshort value) { + assert(h_oop.not_null(), "invariant"); + assert(fd != nullptr, "invariant"); + h_oop->short_field_put(fd->offset(), value); +} + static void write_int_field(const Handle& h_oop, fieldDescriptor* fd, jint value) { assert(h_oop.not_null(), "invariant"); assert(fd != nullptr, "invariant"); @@ -341,8 +359,14 @@ static void write_specialized_field(JfrJavaArguments* args, const Handle& h_oop, switch(fd->field_type()) { case T_BOOLEAN: + write_bool_field(h_oop, fd, args->param(1).get_jboolean()); + break; case T_CHAR: + write_char_field(h_oop, fd, args->param(1).get_jchar()); + break; case T_SHORT: + write_short_field(h_oop, fd, args->param(1).get_jshort()); + break; case T_INT: write_int_field(h_oop, fd, args->param(1).get_jint()); break; @@ -374,8 +398,14 @@ static void read_specialized_field(JavaValue* result, const Handle& h_oop, field switch(fd->field_type()) { case T_BOOLEAN: + result->set_jint(h_oop->bool_field(fd->offset())); + break; case T_CHAR: + result->set_jint(h_oop->char_field(fd->offset())); + break; case T_SHORT: + result->set_jint(h_oop->short_field(fd->offset())); + break; case T_INT: result->set_jint(h_oop->int_field(fd->offset())); break; diff --git a/src/hotspot/share/jfr/jni/jfrJniMethod.hpp b/src/hotspot/share/jfr/jni/jfrJniMethod.hpp index 680e6bee1ee..ece4425782d 100644 --- a/src/hotspot/share/jfr/jni/jfrJniMethod.hpp +++ b/src/hotspot/share/jfr/jni/jfrJniMethod.hpp @@ -163,7 +163,7 @@ void JNICALL jfr_emit_data_loss(JNIEnv* env, jclass jvm, jlong bytes); jlong JNICALL jfr_register_stack_filter(JNIEnv* env, jclass jvm, jobjectArray classes, jobjectArray methods); -jlong JNICALL jfr_unregister_stack_filter(JNIEnv* env, jclass jvm, jlong id); +void JNICALL jfr_unregister_stack_filter(JNIEnv* env, jclass jvm, jlong id); jlong JNICALL jfr_nanos_now(JNIEnv* env, jclass jvm); diff --git a/src/hotspot/share/jfr/leakprofiler/chains/objectSampleMarker.hpp b/src/hotspot/share/jfr/leakprofiler/chains/objectSampleMarker.hpp index 13b55c34e23..2eca6b5eb71 100644 --- a/src/hotspot/share/jfr/leakprofiler/chains/objectSampleMarker.hpp +++ b/src/hotspot/share/jfr/leakprofiler/chains/objectSampleMarker.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2017, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -70,7 +70,7 @@ class ObjectSampleMarker : public StackObj { // now we will set the mark word to "marked" in order to quickly // identify sample objects during the reachability search from gc roots. assert(!obj->mark().is_marked(), "should only mark an object once"); - obj->set_mark(markWord::prototype().set_marked()); + obj->set_mark(obj->prototype_mark().set_marked()); assert(obj->mark().is_marked(), "invariant"); } }; diff --git a/src/hotspot/share/jfr/metadata/metadata.xml b/src/hotspot/share/jfr/metadata/metadata.xml index babb24f28ea..f8b792f3094 100644 --- a/src/hotspot/share/jfr/metadata/metadata.xml +++ b/src/hotspot/share/jfr/metadata/metadata.xml @@ -156,6 +156,12 @@ + + + + + + diff --git a/src/hotspot/share/jfr/recorder/repository/jfrEmergencyDump.cpp b/src/hotspot/share/jfr/recorder/repository/jfrEmergencyDump.cpp index 0759d02cb7c..eaee79497bb 100644 --- a/src/hotspot/share/jfr/recorder/repository/jfrEmergencyDump.cpp +++ b/src/hotspot/share/jfr/recorder/repository/jfrEmergencyDump.cpp @@ -109,8 +109,6 @@ static void close_emergency_dump_file() { } static const char* create_emergency_dump_path() { - assert(is_path_empty(), "invariant"); - const size_t path_len = get_dump_directory(); if (path_len == 0) { return nullptr; diff --git a/src/hotspot/share/jfr/support/jfrDeprecationManager.cpp b/src/hotspot/share/jfr/support/jfrDeprecationManager.cpp index 8969b787a1c..79bcb37e5c0 100644 --- a/src/hotspot/share/jfr/support/jfrDeprecationManager.cpp +++ b/src/hotspot/share/jfr/support/jfrDeprecationManager.cpp @@ -24,6 +24,7 @@ #include "precompiled.hpp" #include "classfile/moduleEntry.hpp" +#include "interpreter/bytecodes.hpp" #include "jfrfiles/jfrEventIds.hpp" #include "jfr/jni/jfrJavaSupport.hpp" #include "jfr/recorder/jfrRecorder.hpp" @@ -216,6 +217,25 @@ static bool should_record(const Method* method, const Method* sender, JavaThread return is_not_jdk_module(sender_module, jt) && max_limit_not_reached(); } +static inline bool is_invoke_bytecode(const Method* sender, int bci) { + assert(sender != nullptr, "invariant"); + assert(sender->validate_bci(bci) >= 0, "invariant"); + const Bytecodes::Code bc = (Bytecodes::Code)*sender->bcp_from(bci); + switch (bc) { + case Bytecodes::_invokevirtual: + case Bytecodes::_invokestatic: + case Bytecodes::_invokeinterface: + case Bytecodes::_invokespecial: + case Bytecodes::_invokedynamic: { + return true; + } + default: { + return false; + } + } + return false; +} + // This is the entry point for newly discovered edges in JfrResolution.cpp. void JfrDeprecationManager::on_link(const Method* method, Method* sender, int bci, u1 frame_type, JavaThread* jt) { assert(method != nullptr, "invariant"); @@ -224,10 +244,13 @@ void JfrDeprecationManager::on_link(const Method* method, Method* sender, int bc assert(!sender->is_native(), "invariant"); assert(jt != nullptr, "invariant"); assert(JfrRecorder::is_started_on_commandline(), "invariant"); - if (JfrMethodData::mark_deprecated_call_site(sender, bci, jt)) { - if (should_record(method, sender, jt)) { - create_edge(method, sender, bci, frame_type, jt); + if (should_record(method, sender, jt)) { + if (is_invoke_bytecode(sender, bci)) { + if (!JfrMethodData::mark_deprecated_call_site(sender, bci, jt)) { + return; + } } + create_edge(method, sender, bci, frame_type, jt); } } diff --git a/src/hotspot/share/jfr/support/jfrJdkJfrEvent.cpp b/src/hotspot/share/jfr/support/jfrJdkJfrEvent.cpp index 0c2fb0206ec..a6cff1609a4 100644 --- a/src/hotspot/share/jfr/support/jfrJdkJfrEvent.cpp +++ b/src/hotspot/share/jfr/support/jfrJdkJfrEvent.cpp @@ -35,6 +35,7 @@ #include "oops/klass.inline.hpp" #include "runtime/handles.inline.hpp" #include "runtime/javaThread.hpp" +#include "runtime/safepointVerifiers.hpp" #include "utilities/stack.inline.hpp" static jobject empty_java_util_arraylist = nullptr; @@ -80,30 +81,25 @@ static bool is_allowed(const Klass* k) { return !(k->is_abstract() || k->should_be_initialized()); } -static void fill_klasses(GrowableArray& event_subklasses, const InstanceKlass* event_klass, JavaThread* thread) { +static void fill_klasses(GrowableArray& event_subklasses, const InstanceKlass* event_klass, JavaThread* thread) { assert(event_subklasses.length() == 0, "invariant"); assert(event_klass != nullptr, "invariant"); DEBUG_ONLY(JfrJavaSupport::check_java_thread_in_vm(thread)); + // Do not safepoint while walking the ClassHierarchy, keeping klasses alive and storing their mirrors in JNI handles. + NoSafepointVerifier nsv; for (ClassHierarchyIterator iter(const_cast(event_klass)); !iter.done(); iter.next()) { Klass* subk = iter.klass(); if (is_allowed(subk)) { - event_subklasses.append(subk); + // We are walking the class hierarchy and saving the relevant klasses in JNI handles. + // To be allowed to store the java mirror, we must ensure that the klass and its oops are kept alive, + // and perform the store before the next safepoint. + subk->keep_alive(); + event_subklasses.append((jclass)JfrJavaSupport::local_jni_handle(subk->java_mirror(), thread)); } } } -static void transform_klasses_to_local_jni_handles(GrowableArray& event_subklasses, JavaThread* thread) { - assert(event_subklasses.is_nonempty(), "invariant"); - DEBUG_ONLY(JfrJavaSupport::check_java_thread_in_vm(thread)); - - for (int i = 0; i < event_subklasses.length(); ++i) { - const InstanceKlass* k = static_cast(event_subklasses.at(i)); - assert(is_allowed(k), "invariant"); - event_subklasses.at_put(i, JfrJavaSupport::local_jni_handle(k->java_mirror(), thread)); - } -} - jobject JdkJfrEvent::get_all_klasses(TRAPS) { DEBUG_ONLY(JfrJavaSupport::check_java_thread_in_vm(THREAD)); initialize(THREAD); @@ -126,15 +122,13 @@ jobject JdkJfrEvent::get_all_klasses(TRAPS) { } ResourceMark rm(THREAD); - GrowableArray event_subklasses(initial_array_size); + GrowableArray event_subklasses(initial_array_size); fill_klasses(event_subklasses, InstanceKlass::cast(klass), THREAD); if (event_subklasses.is_empty()) { return empty_java_util_arraylist; } - transform_klasses_to_local_jni_handles(event_subklasses, THREAD); - Handle h_array_list(THREAD, new_java_util_arraylist(THREAD)); if (h_array_list.is_null()) { return empty_java_util_arraylist; @@ -152,7 +146,7 @@ jobject JdkJfrEvent::get_all_klasses(TRAPS) { JavaValue result(T_BOOLEAN); for (int i = 0; i < event_subklasses.length(); ++i) { - const jclass clazz = (jclass)event_subklasses.at(i); + const jclass clazz = event_subklasses.at(i); assert(JdkJfrEvent::is_subklass(clazz), "invariant"); JfrJavaArguments args(&result, array_list_klass, add_method_sym, add_method_sig_sym); args.set_receiver(h_array_list()); diff --git a/src/hotspot/share/jfr/support/jfrObjectAllocationSample.cpp b/src/hotspot/share/jfr/support/jfrObjectAllocationSample.cpp index e31a96153a4..55ea5225e24 100644 --- a/src/hotspot/share/jfr/support/jfrObjectAllocationSample.cpp +++ b/src/hotspot/share/jfr/support/jfrObjectAllocationSample.cpp @@ -1,5 +1,5 @@ /* -* Copyright (c) 2020, 2021, Oracle and/or its affiliates. All rights reserved. +* Copyright (c) 2020, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -47,7 +47,7 @@ inline bool send_allocation_sample(const Klass* klass, int64_t allocated_bytes, inline int64_t estimate_tlab_size_bytes(Thread* thread) { const size_t desired_tlab_size_bytes = thread->tlab().desired_size() * HeapWordSize; const size_t alignment_reserve_bytes = thread->tlab().alignment_reserve_in_bytes(); - assert(desired_tlab_size_bytes > alignment_reserve_bytes, "invariant"); + assert(desired_tlab_size_bytes >= alignment_reserve_bytes, "invariant"); return static_cast(desired_tlab_size_bytes - alignment_reserve_bytes); } @@ -66,6 +66,10 @@ static void normalize_as_tlab_and_send_allocation_samples(const Klass* klass, in return; } const int64_t tlab_size_bytes = estimate_tlab_size_bytes(thread); + if (tlab_size_bytes <= 0) { + // We don't get a TLAB, avoid endless loop below. + return; + } if (allocated_bytes - tl->last_allocated_bytes() < tlab_size_bytes) { return; } diff --git a/src/hotspot/share/jfr/support/jfrThreadLocal.cpp b/src/hotspot/share/jfr/support/jfrThreadLocal.cpp index 35e50bd6318..9fc6be3d4c8 100644 --- a/src/hotspot/share/jfr/support/jfrThreadLocal.cpp +++ b/src/hotspot/share/jfr/support/jfrThreadLocal.cpp @@ -104,7 +104,7 @@ void JfrThreadLocal::initialize_main_thread(JavaThread* jt) { assert(Thread::is_starting_thread(jt), "invariant"); assert(jt->threadObj() == nullptr, "invariant"); assert(jt->jfr_thread_local()->_jvm_thread_id == 0, "invariant"); - jt->jfr_thread_local()->_jvm_thread_id = 1; + jt->jfr_thread_local()->_jvm_thread_id = ThreadIdentifier::initial(); } static void send_java_thread_start_event(JavaThread* jt) { diff --git a/src/hotspot/share/jvmci/jvmciCompilerToVM.cpp b/src/hotspot/share/jvmci/jvmciCompilerToVM.cpp index 7922a681639..3599df18544 100644 --- a/src/hotspot/share/jvmci/jvmciCompilerToVM.cpp +++ b/src/hotspot/share/jvmci/jvmciCompilerToVM.cpp @@ -1382,7 +1382,8 @@ C2V_VMENTRY(void, reprofile, (JNIEnv* env, jobject, ARGUMENT_PAIR(method))) if (method_data == nullptr) { method_data = get_profiling_method_data(method, CHECK); } else { - method_data->initialize(); + CompilerThreadCanCallJava canCallJava(THREAD, true); + method_data->reinitialize(); } C2V_END @@ -2577,9 +2578,7 @@ C2V_VMENTRY_NULL(jlongArray, registerNativeMethods, (JNIEnv* env, jobject, jclas stringStream st; char* pure_name = NativeLookup::pure_jni_name(method); guarantee(pure_name != nullptr, "Illegal native method name encountered"); - os::print_jni_name_prefix_on(&st, args_size); st.print_raw(pure_name); - os::print_jni_name_suffix_on(&st, args_size); char* jni_name = st.as_string(); address entry = (address) os::dll_lookup(sl_handle, jni_name); @@ -2588,10 +2587,8 @@ C2V_VMENTRY_NULL(jlongArray, registerNativeMethods, (JNIEnv* env, jobject, jclas st.reset(); char* long_name = NativeLookup::long_jni_name(method); guarantee(long_name != nullptr, "Illegal native method name encountered"); - os::print_jni_name_prefix_on(&st, args_size); st.print_raw(pure_name); st.print_raw(long_name); - os::print_jni_name_suffix_on(&st, args_size); char* jni_long_name = st.as_string(); entry = (address) os::dll_lookup(sl_handle, jni_long_name); if (entry == nullptr) { diff --git a/src/hotspot/share/jvmci/jvmciCompilerToVM.hpp b/src/hotspot/share/jvmci/jvmciCompilerToVM.hpp index 0773de6ddba..b44b4bb9116 100644 --- a/src/hotspot/share/jvmci/jvmciCompilerToVM.hpp +++ b/src/hotspot/share/jvmci/jvmciCompilerToVM.hpp @@ -39,6 +39,9 @@ class CompilerToVM { friend class JVMCIVMStructs; private: + static int oopDesc_klass_offset_in_bytes; + static int arrayOopDesc_length_offset_in_bytes; + static int Klass_vtable_start_offset; static int Klass_vtable_length_offset; diff --git a/src/hotspot/share/jvmci/jvmciCompilerToVMInit.cpp b/src/hotspot/share/jvmci/jvmciCompilerToVMInit.cpp index ab536a81e6e..faa23a84178 100644 --- a/src/hotspot/share/jvmci/jvmciCompilerToVMInit.cpp +++ b/src/hotspot/share/jvmci/jvmciCompilerToVMInit.cpp @@ -54,6 +54,8 @@ #include "runtime/stubRoutines.hpp" #include "utilities/resourceHash.hpp" +int CompilerToVM::Data::oopDesc_klass_offset_in_bytes; +int CompilerToVM::Data::arrayOopDesc_length_offset_in_bytes; int CompilerToVM::Data::Klass_vtable_start_offset; int CompilerToVM::Data::Klass_vtable_length_offset; @@ -147,6 +149,9 @@ int CompilerToVM::Data::data_section_item_alignment; JVMTI_ONLY( int* CompilerToVM::Data::_should_notify_object_alloc; ) void CompilerToVM::Data::initialize(JVMCI_TRAPS) { + oopDesc_klass_offset_in_bytes = oopDesc::klass_offset_in_bytes(); + arrayOopDesc_length_offset_in_bytes = arrayOopDesc::length_offset_in_bytes(); + Klass_vtable_start_offset = in_bytes(Klass::vtable_start_offset()); Klass_vtable_length_offset = in_bytes(Klass::vtable_length_offset()); diff --git a/src/hotspot/share/jvmci/vmStructs_jvmci.cpp b/src/hotspot/share/jvmci/vmStructs_jvmci.cpp index da50a524243..bde157a34e1 100644 --- a/src/hotspot/share/jvmci/vmStructs_jvmci.cpp +++ b/src/hotspot/share/jvmci/vmStructs_jvmci.cpp @@ -56,6 +56,9 @@ #endif #define VM_STRUCTS(nonstatic_field, static_field, unchecked_nonstatic_field, volatile_nonstatic_field) \ + static_field(CompilerToVM::Data, oopDesc_klass_offset_in_bytes, int) \ + static_field(CompilerToVM::Data, arrayOopDesc_length_offset_in_bytes, int) \ + \ static_field(CompilerToVM::Data, Klass_vtable_start_offset, int) \ static_field(CompilerToVM::Data, Klass_vtable_length_offset, int) \ \ @@ -224,6 +227,7 @@ nonstatic_field(JavaThread, _vthread, OopHandle) \ nonstatic_field(JavaThread, _scopedValueCache, OopHandle) \ nonstatic_field(JavaThread, _anchor, JavaFrameAnchor) \ + nonstatic_field(JavaThread, _lock_id, int64_t) \ nonstatic_field(JavaThread, _vm_result, oop) \ nonstatic_field(JavaThread, _stack_overflow_state._stack_overflow_limit, address) \ volatile_nonstatic_field(JavaThread, _exception_oop, oop) \ @@ -275,6 +279,7 @@ nonstatic_field(Klass, _secondary_supers_bitmap, uintx) \ nonstatic_field(Klass, _hash_slot, uint8_t) \ nonstatic_field(Klass, _misc_flags._flags, u1) \ + nonstatic_field(Klass, _prototype_header, markWord) \ \ nonstatic_field(LocalVariableTableElement, start_bci, u2) \ nonstatic_field(LocalVariableTableElement, length, u2) \ @@ -323,11 +328,12 @@ \ nonstatic_field(ObjArrayKlass, _element_klass, Klass*) \ \ - unchecked_nonstatic_field(ObjectMonitor, _owner, sizeof(void *)) /* NOTE: no type */ \ + volatile_nonstatic_field(ObjectMonitor, _owner, int64_t) \ volatile_nonstatic_field(ObjectMonitor, _recursions, intptr_t) \ volatile_nonstatic_field(ObjectMonitor, _cxq, ObjectWaiter*) \ volatile_nonstatic_field(ObjectMonitor, _EntryList, ObjectWaiter*) \ - volatile_nonstatic_field(ObjectMonitor, _succ, JavaThread*) \ + volatile_nonstatic_field(ObjectMonitor, _succ, int64_t) \ + volatile_nonstatic_field(ObjectMonitor, _stack_locker, BasicLock*) \ \ volatile_nonstatic_field(oopDesc, _mark, markWord) \ volatile_nonstatic_field(oopDesc, _metadata._klass, Klass*) \ @@ -480,7 +486,6 @@ declare_constant(CompLevel_full_optimization) \ declare_constant(HeapWordSize) \ declare_constant(InvocationEntryBci) \ - declare_constant(LogKlassAlignmentInBytes) \ declare_constant(JVMCINMethodData::SPECULATION_LENGTH_BITS) \ \ declare_constant(JVM_ACC_WRITTEN_FLAGS) \ @@ -776,7 +781,9 @@ AARCH64_ONLY(declare_constant(NMethodPatchingType::conc_instruction_and_data_patch)) \ AARCH64_ONLY(declare_constant(NMethodPatchingType::conc_data_patch)) \ \ + declare_constant(ObjectMonitor::NO_OWNER) \ declare_constant(ObjectMonitor::ANONYMOUS_OWNER) \ + declare_constant(ObjectMonitor::DEFLATER_MARKER) \ \ declare_constant(ReceiverTypeData::receiver_type_row_cell_count) \ declare_constant(ReceiverTypeData::receiver0_offset) \ @@ -796,6 +803,7 @@ declare_constant(InvocationCounter::count_increment) \ declare_constant(InvocationCounter::count_shift) \ \ + declare_constant(markWord::klass_shift) \ declare_constant(markWord::hash_shift) \ declare_constant(markWord::monitor_value) \ \ @@ -805,6 +813,7 @@ declare_constant(markWord::hash_mask_in_place) \ \ declare_constant(markWord::unlocked_value) \ + declare_constant(markWord::marked_value) \ \ declare_constant(markWord::no_hash_in_place) \ declare_constant(markWord::no_lock_in_place) \ diff --git a/src/hotspot/share/memory/classLoaderMetaspace.cpp b/src/hotspot/share/memory/classLoaderMetaspace.cpp index 08bf42da8e3..4bcbb862a5a 100644 --- a/src/hotspot/share/memory/classLoaderMetaspace.cpp +++ b/src/hotspot/share/memory/classLoaderMetaspace.cpp @@ -30,17 +30,23 @@ #include "memory/metaspaceUtils.hpp" #include "memory/metaspace/chunkManager.hpp" #include "memory/metaspace/internalStats.hpp" +#include "memory/metaspace/metablock.hpp" #include "memory/metaspace/metaspaceArena.hpp" #include "memory/metaspace/metaspaceArenaGrowthPolicy.hpp" +#include "memory/metaspace/metaspaceCommon.hpp" +#include "memory/metaspace/metaspaceContext.hpp" #include "memory/metaspace/metaspaceSettings.hpp" #include "memory/metaspace/metaspaceStatistics.hpp" #include "memory/metaspace/runningCounters.hpp" #include "memory/metaspaceTracer.hpp" +#include "oops/klass.hpp" #include "runtime/mutexLocker.hpp" #include "utilities/debug.hpp" using metaspace::ChunkManager; +using metaspace::MetaBlock; using metaspace::MetaspaceArena; +using metaspace::MetaspaceContext; using metaspace::ArenaGrowthPolicy; using metaspace::RunningCounters; using metaspace::InternalStats; @@ -49,30 +55,35 @@ using metaspace::InternalStats; #define LOGFMT_ARGS p2i(this) ClassLoaderMetaspace::ClassLoaderMetaspace(Mutex* lock, Metaspace::MetaspaceType space_type) : + ClassLoaderMetaspace(lock, space_type, + MetaspaceContext::context_nonclass(), + MetaspaceContext::context_class(), + CompressedKlassPointers::klass_alignment_in_words()) +{} + +ClassLoaderMetaspace::ClassLoaderMetaspace(Mutex* lock, Metaspace::MetaspaceType space_type, + MetaspaceContext* non_class_context, + MetaspaceContext* class_context, + size_t klass_alignment_words) : _lock(lock), _space_type(space_type), _non_class_space_arena(nullptr), _class_space_arena(nullptr) { - ChunkManager* const non_class_cm = - ChunkManager::chunkmanager_nonclass(); - // Initialize non-class Arena _non_class_space_arena = new MetaspaceArena( - non_class_cm, + non_class_context, ArenaGrowthPolicy::policy_for_space_type(space_type, false), - RunningCounters::used_nonclass_counter(), - "non-class sm"); + Metaspace::min_allocation_alignment_words, + "non-class arena"); // If needed, initialize class arena - if (Metaspace::using_class_space()) { - ChunkManager* const class_cm = - ChunkManager::chunkmanager_class(); + if (class_context != nullptr) { _class_space_arena = new MetaspaceArena( - class_cm, + class_context, ArenaGrowthPolicy::policy_for_space_type(space_type, true), - RunningCounters::used_class_counter(), - "class sm"); + klass_alignment_words, + "class arena"); } UL2(debug, "born (nonclass arena: " PTR_FORMAT ", class arena: " PTR_FORMAT ".", @@ -89,12 +100,28 @@ ClassLoaderMetaspace::~ClassLoaderMetaspace() { // Allocate word_size words from Metaspace. MetaWord* ClassLoaderMetaspace::allocate(size_t word_size, Metaspace::MetadataType mdType) { + word_size = align_up(word_size, Metaspace::min_allocation_word_size); MutexLocker fcl(lock(), Mutex::_no_safepoint_check_flag); - if (Metaspace::is_class_space_allocation(mdType)) { - return class_space_arena()->allocate(word_size); + MetaBlock result, wastage; + const bool is_class = have_class_space_arena() && mdType == Metaspace::ClassType; + if (is_class) { + assert(word_size >= (sizeof(Klass)/BytesPerWord), "weird size for klass: %zu", word_size); + result = class_space_arena()->allocate(word_size, wastage); } else { - return non_class_space_arena()->allocate(word_size); + result = non_class_space_arena()->allocate(word_size, wastage); } + if (wastage.is_nonempty()) { + non_class_space_arena()->deallocate(wastage); + } +#ifdef ASSERT + if (result.is_nonempty()) { + const bool in_class_arena = class_space_arena() != nullptr ? class_space_arena()->contains(result) : false; + const bool in_nonclass_arena = non_class_space_arena()->contains(result); + assert((is_class && in_class_arena) || (!is_class && in_class_arena != in_nonclass_arena), + "block from neither arena " METABLOCKFORMAT "?", METABLOCKFORMATARGS(result)); + } +#endif + return result.base(); } // Attempt to expand the GC threshold to be good for at least another word_size words @@ -132,12 +159,15 @@ MetaWord* ClassLoaderMetaspace::expand_and_allocate(size_t word_size, Metaspace: // because it is not needed anymore. void ClassLoaderMetaspace::deallocate(MetaWord* ptr, size_t word_size) { MutexLocker fcl(lock(), Mutex::_no_safepoint_check_flag); - const bool is_class = Metaspace::using_class_space() && Metaspace::is_in_class_space(ptr); - if (is_class) { - class_space_arena()->deallocate(ptr, word_size); - } else { - non_class_space_arena()->deallocate(ptr, word_size); + NOT_LP64(word_size = align_down(word_size, Metaspace::min_allocation_word_size);) + MetaBlock bl(ptr, word_size); + // Add to class arena only if block is usable for encodable Klass storage. + MetaspaceArena* receiving_arena = non_class_space_arena(); + if (Metaspace::using_class_space() && Metaspace::is_in_class_space(ptr) && + is_aligned(ptr, class_space_arena()->allocation_alignment_bytes())) { + receiving_arena = class_space_arena(); } + receiving_arena->deallocate(bl); DEBUG_ONLY(InternalStats::inc_num_deallocs();) } @@ -180,7 +210,7 @@ void ClassLoaderMetaspace::usage_numbers(size_t* p_used_words, size_t* p_committ { MutexLocker fcl(lock(), Mutex::_no_safepoint_check_flag); usage_numbers(Metaspace::MetadataType::NonClassType, &used_nc, &comm_nc, &cap_nc); - if (Metaspace::using_class_space()) { + if (have_class_space_arena()) { usage_numbers(Metaspace::MetadataType::ClassType, &used_c, &comm_c, &cap_c); } } diff --git a/src/hotspot/share/memory/classLoaderMetaspace.hpp b/src/hotspot/share/memory/classLoaderMetaspace.hpp index 176b8c5082a..aa43e171708 100644 --- a/src/hotspot/share/memory/classLoaderMetaspace.hpp +++ b/src/hotspot/share/memory/classLoaderMetaspace.hpp @@ -33,7 +33,9 @@ class outputStream; namespace metaspace { struct ClmsStats; + class ClmsTester; class MetaspaceArena; + class MetaspaceContext; } // A ClassLoaderMetaspace manages MetaspaceArena(s) for a CLD. @@ -57,6 +59,7 @@ namespace metaspace { // alloc top // class ClassLoaderMetaspace : public CHeapObj { + friend class metaspace::ClmsTester; // for gtests // A reference to an outside lock, held by the CLD. Mutex* const _lock; @@ -75,8 +78,14 @@ class ClassLoaderMetaspace : public CHeapObj { metaspace::MetaspaceArena* non_class_space_arena() const { return _non_class_space_arena; } metaspace::MetaspaceArena* class_space_arena() const { return _class_space_arena; } -public: + bool have_class_space_arena() const { return _class_space_arena != nullptr; } + + ClassLoaderMetaspace(Mutex* lock, Metaspace::MetaspaceType space_type, + metaspace::MetaspaceContext* non_class_context, + metaspace::MetaspaceContext* class_context, + size_t klass_alignment_words); +public: ClassLoaderMetaspace(Mutex* lock, Metaspace::MetaspaceType space_type); ~ClassLoaderMetaspace(); diff --git a/src/hotspot/share/memory/metaspace.cpp b/src/hotspot/share/memory/metaspace.cpp index f86be4774d5..aa592a3a6aa 100644 --- a/src/hotspot/share/memory/metaspace.cpp +++ b/src/hotspot/share/memory/metaspace.cpp @@ -651,23 +651,41 @@ void Metaspace::ergo_initialize() { MaxMetaspaceSize = MAX2(MaxMetaspaceSize, commit_alignment()); if (UseCompressedClassPointers) { - // Let CCS size not be larger than 80% of MaxMetaspaceSize. Note that is + // Let Class Space not be larger than 80% of MaxMetaspaceSize. Note that is // grossly over-dimensioned for most usage scenarios; typical ratio of // class space : non class space usage is about 1:6. With many small classes, // it can get as low as 1:2. It is not a big deal though since ccs is only // reserved and will be committed on demand only. - size_t max_ccs_size = 8 * (MaxMetaspaceSize / 10); - size_t adjusted_ccs_size = MIN2(CompressedClassSpaceSize, max_ccs_size); + const size_t max_ccs_size = 8 * (MaxMetaspaceSize / 10); + + // Sanity check. + const size_t max_klass_range = CompressedKlassPointers::max_klass_range_size(); + assert(max_klass_range >= reserve_alignment(), + "Klass range (%zu) must cover at least a full root chunk (%zu)", + max_klass_range, reserve_alignment()); + + size_t adjusted_ccs_size = MIN3(CompressedClassSpaceSize, max_ccs_size, max_klass_range); // CCS must be aligned to root chunk size, and be at least the size of one // root chunk. adjusted_ccs_size = align_up(adjusted_ccs_size, reserve_alignment()); adjusted_ccs_size = MAX2(adjusted_ccs_size, reserve_alignment()); + // Print a warning if the adjusted size differs from the users input + if (CompressedClassSpaceSize != adjusted_ccs_size) { + #define X "CompressedClassSpaceSize adjusted from user input " \ + "%zu bytes to %zu bytes", CompressedClassSpaceSize, adjusted_ccs_size + if (FLAG_IS_CMDLINE(CompressedClassSpaceSize)) { + log_warning(metaspace)(X); + } else { + log_info(metaspace)(X); + } + #undef X + } + // Note: re-adjusting may have us left with a CompressedClassSpaceSize // larger than MaxMetaspaceSize for very small values of MaxMetaspaceSize. // Lets just live with that, its not a big deal. - if (adjusted_ccs_size != CompressedClassSpaceSize) { FLAG_SET_ERGO(CompressedClassSpaceSize, adjusted_ccs_size); log_info(metaspace)("Setting CompressedClassSpaceSize to " SIZE_FORMAT ".", @@ -778,6 +796,7 @@ void Metaspace::global_initialize() { Metaspace::initialize_class_space(rs); // Set up compressed class pointer encoding. + // In CDS=off mode, we give the JVM some leeway to choose a favorable base/shift combination. CompressedKlassPointers::initialize((address)rs.base(), rs.size()); } @@ -846,9 +865,17 @@ MetaWord* Metaspace::allocate(ClassLoaderData* loader_data, size_t word_size, MetaWord* result = loader_data->metaspace_non_null()->allocate(word_size, mdtype); if (result != nullptr) { +#ifdef ASSERT + if (using_class_space() && mdtype == ClassType) { + assert(is_in_class_space(result) && + is_aligned(result, CompressedKlassPointers::klass_alignment_in_bytes()), "Sanity"); + } else { + assert((is_in_class_space(result) || is_in_nonclass_metaspace(result)) && + is_aligned(result, Metaspace::min_allocation_alignment_bytes), "Sanity"); + } +#endif // Zero initialize. Copy::fill_to_words((HeapWord*)result, word_size, 0); - log_trace(metaspace)("Metaspace::allocate: type %d return " PTR_FORMAT ".", (int)type, p2i(result)); } diff --git a/src/hotspot/share/memory/metaspace.hpp b/src/hotspot/share/memory/metaspace.hpp index 7076a6a09bb..7adebfd826b 100644 --- a/src/hotspot/share/memory/metaspace.hpp +++ b/src/hotspot/share/memory/metaspace.hpp @@ -26,6 +26,7 @@ #define SHARE_MEMORY_METASPACE_HPP #include "memory/allocation.hpp" +#include "memory/virtualspace.hpp" #include "runtime/globals.hpp" #include "utilities/exceptions.hpp" #include "utilities/globalDefinitions.hpp" @@ -35,7 +36,6 @@ class MetaspaceShared; class MetaspaceTracer; class Mutex; class outputStream; -class ReservedSpace; ////////////////// Metaspace /////////////////////// @@ -108,6 +108,17 @@ class Metaspace : public AllStatic { // The largest possible single allocation static size_t max_allocation_word_size(); + // Minimum allocation alignment, in bytes. All MetaData shall be aligned correctly + // to be able to hold 64-bit data types. Unlike malloc, we don't care for larger + // data types. + static constexpr size_t min_allocation_alignment_bytes = sizeof(uint64_t); + + // Minimum allocation alignment, in words, Metaspace observes. + static constexpr size_t min_allocation_alignment_words = min_allocation_alignment_bytes / BytesPerWord; + + // Every allocation will get rounded up to the minimum word size. + static constexpr size_t min_allocation_word_size = min_allocation_alignment_words; + static MetaWord* allocate(ClassLoaderData* loader_data, size_t word_size, MetaspaceObj::Type type, bool use_class_space, TRAPS); diff --git a/src/hotspot/share/memory/metaspace/binList.hpp b/src/hotspot/share/memory/metaspace/binList.hpp index 76cd9e155aa..9442ea3cd52 100644 --- a/src/hotspot/share/memory/metaspace/binList.hpp +++ b/src/hotspot/share/memory/metaspace/binList.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2024, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2020 SAP SE. All rights reserved. * Copyright (c) 2023 Red Hat, Inc. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. @@ -28,6 +28,7 @@ #define SHARE_MEMORY_METASPACE_BINLIST_HPP #include "memory/metaspace/counters.hpp" +#include "memory/metaspace/metablock.hpp" #include "memory/metaspace/metaspaceCommon.hpp" #include "utilities/align.hpp" #include "utilities/debug.hpp" @@ -36,8 +37,7 @@ namespace metaspace { // BinList is a data structure to manage small to very small memory blocks -// (only a few words). It is used to manage deallocated blocks - see -// class FreeBlocks. +// (only a few words). It is used to manage deallocated small blocks. // Memory blocks are kept in a vector of linked lists of equi-sized blocks: // @@ -143,7 +143,10 @@ class BinListImpl { } } - void add_block(MetaWord* p, size_t word_size) { + void add_block(MetaBlock mb) { + assert(!mb.is_empty(), "Don't add empty blocks"); + const size_t word_size = mb.word_size(); + MetaWord* const p = mb.base(); assert(word_size >= MinWordSize && word_size <= MaxWordSize, "bad block size"); DEBUG_ONLY(write_canary(p, word_size);) @@ -155,10 +158,11 @@ class BinListImpl { } // Given a word_size, searches and returns a block of at least that size. - // Block may be larger. Real block size is returned in *p_real_word_size. - MetaWord* remove_block(size_t word_size, size_t* p_real_word_size) { + // Block may be larger. + MetaBlock remove_block(size_t word_size) { assert(word_size >= MinWordSize && word_size <= MaxWordSize, "bad block size " SIZE_FORMAT ".", word_size); + MetaBlock result; int index = index_for_word_size(word_size); index = index_for_next_non_empty_list(index); if (index != -1) { @@ -169,12 +173,9 @@ class BinListImpl { "bad block in list[%d] (" BLOCK_FORMAT ")", index, BLOCK_FORMAT_ARGS(b, real_word_size)); _blocks[index] = b->_next; _counter.sub(real_word_size); - *p_real_word_size = real_word_size; - return (MetaWord*)b; - } else { - *p_real_word_size = 0; - return nullptr; + result = MetaBlock((MetaWord*)b, real_word_size); } + return result; } // Returns number of blocks in this structure @@ -191,9 +192,12 @@ class BinListImpl { for (int i = 0; i < num_lists; i++) { const size_t s = word_size_for_index(i); int pos = 0; + Block* b_last = nullptr; // catch simple circularities for (Block* b = _blocks[i]; b != nullptr; b = b->_next, pos++) { assert(check_canary(b, s), ""); + assert(b != b_last, "Circle"); local_counter.add(s); + b_last = b; } } local_counter.check(_counter); diff --git a/src/hotspot/share/memory/metaspace/blockTree.cpp b/src/hotspot/share/memory/metaspace/blockTree.cpp index 1f1e54f4a46..85e77508836 100644 --- a/src/hotspot/share/memory/metaspace/blockTree.cpp +++ b/src/hotspot/share/memory/metaspace/blockTree.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2024, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2020, 2022 SAP SE. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * @@ -180,8 +180,8 @@ void BlockTree::verify() const { _counter.check(counter); } -void BlockTree::zap_range(MetaWord* p, size_t word_size) { - memset(p, 0xF3, word_size * sizeof(MetaWord)); +void BlockTree::zap_block(MetaBlock bl) { + memset(bl.base(), 0xF3, bl.word_size() * sizeof(MetaWord)); } void BlockTree::print_tree(outputStream* st) const { @@ -224,6 +224,12 @@ void BlockTree::print_tree(outputStream* st) const { } } + // Handle simple circularities + if (n == n->_right || n == n->_left || n == n->_next) { + st->print_cr("@" PTR_FORMAT ": circularity detected.", p2i(n)); + return; // stop printing + } + // Handle children. if (n->_right != nullptr) { walkinfo info2; diff --git a/src/hotspot/share/memory/metaspace/blockTree.hpp b/src/hotspot/share/memory/metaspace/blockTree.hpp index c5dd48926ac..8bcdd30919a 100644 --- a/src/hotspot/share/memory/metaspace/blockTree.hpp +++ b/src/hotspot/share/memory/metaspace/blockTree.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2024, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2020 SAP SE. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * @@ -29,13 +29,14 @@ #include "memory/allocation.hpp" #include "memory/metaspace/chunklevel.hpp" #include "memory/metaspace/counters.hpp" +#include "memory/metaspace/metablock.hpp" #include "utilities/debug.hpp" #include "utilities/globalDefinitions.hpp" namespace metaspace { // BlockTree is a rather simple binary search tree. It is used to -// manage small to medium free memory blocks (see class FreeBlocks). +// manage medium to large free memory blocks. // // There is no separation between payload (managed blocks) and nodes: the // memory blocks themselves are the nodes, with the block size being the key. @@ -80,8 +81,7 @@ class BlockTree: public CHeapObj { NOT_LP64(0x4e4f4445) LP64_ONLY(0x4e4f44454e4f4445ULL); // "NODE" resp "NODENODE" // Note: we afford us the luxury of an always-there canary value. - // The space for that is there (these nodes are only used to manage larger blocks, - // see FreeBlocks::MaxSmallBlocksWordSize). + // The space for that is there (these nodes are only used to manage larger blocks). // It is initialized in debug and release, but only automatically tested // in debug. const intptr_t _canary; @@ -335,7 +335,7 @@ class BlockTree: public CHeapObj { } #ifdef ASSERT - void zap_range(MetaWord* p, size_t word_size); + void zap_block(MetaBlock block); // Helper for verify() void verify_node_pointer(const Node* n) const; #endif // ASSERT @@ -345,10 +345,11 @@ class BlockTree: public CHeapObj { BlockTree() : _root(nullptr) {} // Add a memory block to the tree. Its content will be overwritten. - void add_block(MetaWord* p, size_t word_size) { - DEBUG_ONLY(zap_range(p, word_size)); + void add_block(MetaBlock block) { + DEBUG_ONLY(zap_block(block);) + const size_t word_size = block.word_size(); assert(word_size >= MinWordSize, "invalid block size " SIZE_FORMAT, word_size); - Node* n = new(p) Node(word_size); + Node* n = new(block.base()) Node(word_size); if (_root == nullptr) { _root = n; } else { @@ -358,11 +359,11 @@ class BlockTree: public CHeapObj { } // Given a word_size, search and return the smallest block that is equal or - // larger than that size. Upon return, *p_real_word_size contains the actual - // block size. - MetaWord* remove_block(size_t word_size, size_t* p_real_word_size) { + // larger than that size. + MetaBlock remove_block(size_t word_size) { assert(word_size >= MinWordSize, "invalid block size " SIZE_FORMAT, word_size); + MetaBlock result; Node* n = find_closest_fit(word_size); if (n != nullptr) { @@ -379,15 +380,13 @@ class BlockTree: public CHeapObj { remove_node_from_tree(n); } - MetaWord* p = (MetaWord*)n; - *p_real_word_size = n->_word_size; + result = MetaBlock((MetaWord*)n, n->_word_size); _counter.sub(n->_word_size); - DEBUG_ONLY(zap_range(p, n->_word_size)); - return p; + DEBUG_ONLY(zap_block(result);) } - return nullptr; + return result; } // Returns number of blocks in this structure diff --git a/src/hotspot/share/memory/metaspace/freeBlocks.cpp b/src/hotspot/share/memory/metaspace/freeBlocks.cpp index b6a281f0db7..ab65387043a 100644 --- a/src/hotspot/share/memory/metaspace/freeBlocks.cpp +++ b/src/hotspot/share/memory/metaspace/freeBlocks.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2024, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2020 SAP SE. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * @@ -30,32 +30,23 @@ namespace metaspace { -void FreeBlocks::add_block(MetaWord* p, size_t word_size) { - if (word_size > MaxSmallBlocksWordSize) { - _tree.add_block(p, word_size); +void FreeBlocks::add_block(MetaBlock bl) { + if (bl.word_size() > _small_blocks.MaxWordSize) { + _tree.add_block(bl); } else { - _small_blocks.add_block(p, word_size); + _small_blocks.add_block(bl); } } -MetaWord* FreeBlocks::remove_block(size_t requested_word_size) { +MetaBlock FreeBlocks::remove_block(size_t requested_word_size) { size_t real_size = 0; - MetaWord* p = nullptr; - if (requested_word_size > MaxSmallBlocksWordSize) { - p = _tree.remove_block(requested_word_size, &real_size); + MetaBlock bl; + if (requested_word_size > _small_blocks.MaxWordSize) { + bl = _tree.remove_block(requested_word_size); } else { - p = _small_blocks.remove_block(requested_word_size, &real_size); + bl = _small_blocks.remove_block(requested_word_size); } - if (p != nullptr) { - // Blocks which are larger than a certain threshold are split and - // the remainder is handed back to the manager. - const size_t waste = real_size - requested_word_size; - if (waste >= MinWordSize) { - add_block(p + requested_word_size, waste); - } - } - return p; + return bl; } } // namespace metaspace - diff --git a/src/hotspot/share/memory/metaspace/freeBlocks.hpp b/src/hotspot/share/memory/metaspace/freeBlocks.hpp index d5a3c29a9d7..77ab48e17ea 100644 --- a/src/hotspot/share/memory/metaspace/freeBlocks.hpp +++ b/src/hotspot/share/memory/metaspace/freeBlocks.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2024, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2020 SAP SE. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * @@ -73,20 +73,16 @@ class FreeBlocks : public CHeapObj { // kept in the blocktree. STATIC_ASSERT(BinList32::MaxWordSize >= BlockTree::MinWordSize); - // Cutoff point: blocks larger than this size are kept in the - // tree, blocks smaller than or equal to this size in the bin list. - const size_t MaxSmallBlocksWordSize = BinList32::MaxWordSize; - public: // Smallest blocks we can keep in this structure. const static size_t MinWordSize = BinList32::MinWordSize; // Add a block to the deallocation management. - void add_block(MetaWord* p, size_t word_size); + void add_block(MetaBlock bl); - // Retrieve a block of at least requested_word_size. - MetaWord* remove_block(size_t requested_word_size); + // Retrieve a block of at least requested_word_size. May be larger. + MetaBlock remove_block(size_t requested_word_size); #ifdef ASSERT void verify() const { diff --git a/src/hotspot/share/memory/metaspace/metablock.hpp b/src/hotspot/share/memory/metaspace/metablock.hpp new file mode 100644 index 00000000000..96e27ff8702 --- /dev/null +++ b/src/hotspot/share/memory/metaspace/metablock.hpp @@ -0,0 +1,76 @@ +/* + * Copyright (c) 2023 Red Hat, Inc. All rights reserved. + * Copyright (c) 2023, 2024, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_MEMORY_METASPACE_METABLOCK_HPP +#define SHARE_MEMORY_METASPACE_METABLOCK_HPP + +#include "utilities/globalDefinitions.hpp" + +class outputStream; + +namespace metaspace { + +// Tiny structure to be passed by value +class MetaBlock { + + MetaWord* _base; + size_t _word_size; + +public: + + MetaBlock(MetaWord* p, size_t word_size) : + _base(word_size == 0 ? nullptr : p), _word_size(word_size) {} + MetaBlock() : MetaBlock(nullptr, 0) {} + + MetaWord* base() const { return _base; } + const MetaWord* end() const { return _base + _word_size; } + size_t word_size() const { return _word_size; } + bool is_empty() const { return _base == nullptr; } + bool is_nonempty() const { return _base != nullptr; } + void reset() { _base = nullptr; _word_size = 0; } + + bool operator==(const MetaBlock& rhs) const { + return base() == rhs.base() && + word_size() == rhs.word_size(); + } + + // Split off tail block. + inline MetaBlock split_off_tail(size_t tailsize); + + DEBUG_ONLY(inline void verify() const;) + + // Convenience functions + inline bool is_aligned_base(size_t alignment_words) const; + inline bool is_aligned_size(size_t alignment_words) const; + + void print_on(outputStream* st) const; +}; + +#define METABLOCKFORMAT "block (@" PTR_FORMAT " word size " SIZE_FORMAT ")" +#define METABLOCKFORMATARGS(__block__) p2i((__block__).base()), (__block__).word_size() + +} // namespace metaspace + +#endif // SHARE_MEMORY_METASPACE_METABLOCK_HPP diff --git a/src/hotspot/share/memory/metaspace/metablock.inline.hpp b/src/hotspot/share/memory/metaspace/metablock.inline.hpp new file mode 100644 index 00000000000..04eb6c22277 --- /dev/null +++ b/src/hotspot/share/memory/metaspace/metablock.inline.hpp @@ -0,0 +1,87 @@ +/* + * Copyright (c) 2023 Red Hat, Inc. All rights reserved. + * Copyright (c) 2023, 2024, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_MEMORY_METASPACE_METABLOCK_INLINE_HPP +#define SHARE_MEMORY_METASPACE_METABLOCK_INLINE_HPP + +#include "memory/metaspace/metablock.hpp" +#include "utilities/globalDefinitions.hpp" +#include "utilities/align.hpp" +#include "utilities/ostream.hpp" +#include "utilities/debug.hpp" + +class outputStream; + +namespace metaspace { + +inline MetaBlock MetaBlock::split_off_tail(size_t tailsize) { + if (is_empty() || tailsize == 0) { + return MetaBlock(); + } + assert(tailsize <= _word_size, "invalid split point for block " + METABLOCKFORMAT ": %zu", METABLOCKFORMATARGS(*this), tailsize); + const size_t new_size = _word_size - tailsize; + MetaBlock tail(_base + new_size, tailsize); + _word_size = new_size; + if (_word_size == 0) { + _base = nullptr; + } + return tail; +} + +inline void MetaBlock::print_on(outputStream* st) const { + st->print(METABLOCKFORMAT, METABLOCKFORMATARGS(*this)); +} + +// Convenience functions +inline bool MetaBlock::is_aligned_base(size_t alignment_words) const { + return is_aligned(_base, alignment_words * BytesPerWord); +} + +inline bool MetaBlock::is_aligned_size(size_t alignment_words) const { + return is_aligned(_word_size, alignment_words); +} + +// some convenience asserts +#define assert_block_base_aligned(block, alignment_words) \ + assert(block.is_aligned_base(alignment_words), "Block wrong base alignment " METABLOCKFORMAT, METABLOCKFORMATARGS(block)); + +#define assert_block_size_aligned(block, alignment_words) \ + assert(block.is_aligned_size(alignment_words), "Block wrong size alignment " METABLOCKFORMAT, METABLOCKFORMATARGS(block)); + +#define assert_block_larger_or_equal(block, x) \ + assert(block.word_size() >= x, "Block too small " METABLOCKFORMAT, METABLOCKFORMATARGS(block)); + +#ifdef ASSERT +inline void MetaBlock::verify() const { + assert( (_base == nullptr && _word_size == 0) || + (_base != nullptr && _word_size > 0), + "block invalid " METABLOCKFORMAT, METABLOCKFORMATARGS(*this)); +} +#endif + +} // namespace metaspace + +#endif // SHARE_MEMORY_METASPACE_METABLOCK_INLINE_HPP diff --git a/src/hotspot/share/memory/metaspace/metaspaceArena.cpp b/src/hotspot/share/memory/metaspace/metaspaceArena.cpp index 92d7d0ea7eb..33f1bfc46a3 100644 --- a/src/hotspot/share/memory/metaspace/metaspaceArena.cpp +++ b/src/hotspot/share/memory/metaspace/metaspaceArena.cpp @@ -30,10 +30,12 @@ #include "memory/metaspace/counters.hpp" #include "memory/metaspace/freeBlocks.hpp" #include "memory/metaspace/internalStats.hpp" +#include "memory/metaspace/metablock.inline.hpp" #include "memory/metaspace/metachunk.hpp" #include "memory/metaspace/metaspaceArena.hpp" #include "memory/metaspace/metaspaceArenaGrowthPolicy.hpp" #include "memory/metaspace/metaspaceCommon.hpp" +#include "memory/metaspace/metaspaceContext.hpp" #include "memory/metaspace/metaspaceSettings.hpp" #include "memory/metaspace/metaspaceStatistics.hpp" #include "memory/metaspace/virtualSpaceList.hpp" @@ -56,24 +58,25 @@ chunklevel_t MetaspaceArena::next_chunk_level() const { return _growth_policy->get_level_at_step(growth_step); } -// Given a chunk, add its remaining free committed space to the free block list. -void MetaspaceArena::salvage_chunk(Metachunk* c) { - size_t remaining_words = c->free_below_committed_words(); +// Given a chunk, return the committed remainder of this chunk. +MetaBlock MetaspaceArena::salvage_chunk(Metachunk* c) { + MetaBlock result; + const size_t remaining_words = c->free_below_committed_words(); if (remaining_words >= FreeBlocks::MinWordSize) { UL2(trace, "salvaging chunk " METACHUNK_FULL_FORMAT ".", METACHUNK_FULL_FORMAT_ARGS(c)); MetaWord* ptr = c->allocate(remaining_words); assert(ptr != nullptr, "Should have worked"); - _total_used_words_counter->increment_by(remaining_words); - add_allocation_to_fbl(ptr, remaining_words); + result = MetaBlock(ptr, remaining_words); // After this operation: the chunk should have no free committed space left. assert(c->free_below_committed_words() == 0, "Salvaging chunk failed (chunk " METACHUNK_FULL_FORMAT ").", METACHUNK_FULL_FORMAT_ARGS(c)); } + return result; } // Allocate a new chunk from the underlying chunk manager able to hold at least @@ -97,28 +100,35 @@ Metachunk* MetaspaceArena::allocate_new_chunk(size_t requested_word_size) { return c; } -void MetaspaceArena::add_allocation_to_fbl(MetaWord* p, size_t word_size) { - assert(p != nullptr, "p is null"); - assert_is_aligned_metaspace_pointer(p); - assert(word_size > 0, "zero sized"); - +void MetaspaceArena::add_allocation_to_fbl(MetaBlock bl) { + assert(bl.is_nonempty(), "Sanity"); + assert_block_base_aligned(bl, allocation_alignment_words()); + assert_block_size_aligned(bl, Metaspace::min_allocation_alignment_words); if (_fbl == nullptr) { _fbl = new FreeBlocks(); // Create only on demand } - _fbl->add_block(p, word_size); + _fbl->add_block(bl); } -MetaspaceArena::MetaspaceArena(ChunkManager* chunk_manager, const ArenaGrowthPolicy* growth_policy, - SizeAtomicCounter* total_used_words_counter, - const char* name) : - _chunk_manager(chunk_manager), +MetaspaceArena::MetaspaceArena(MetaspaceContext* context, + const ArenaGrowthPolicy* growth_policy, + size_t allocation_alignment_words, + const char* name) : + _allocation_alignment_words(allocation_alignment_words), + _chunk_manager(context->cm()), _growth_policy(growth_policy), _chunks(), _fbl(nullptr), - _total_used_words_counter(total_used_words_counter), + _total_used_words_counter(context->used_words_counter()), _name(name) { - UL(debug, ": born."); + // Check arena allocation alignment + assert(is_power_of_2(_allocation_alignment_words) && + _allocation_alignment_words >= Metaspace::min_allocation_alignment_words && + _allocation_alignment_words <= chunklevel::MIN_CHUNK_WORD_SIZE, + "Invalid alignment: %zu", _allocation_alignment_words); + + UL(debug, "born."); // Update statistics InternalStats::inc_num_arena_births(); @@ -140,7 +150,7 @@ MetaspaceArena::~MetaspaceArena() { c = c2; } - UL2(info, "returned %d chunks, total capacity " SIZE_FORMAT " words.", + UL2(debug, "returned %d chunks, total capacity " SIZE_FORMAT " words.", return_counter.count(), return_counter.total_size()); _total_used_words_counter->decrement_by(return_counter.total_size()); @@ -205,49 +215,93 @@ bool MetaspaceArena::attempt_enlarge_current_chunk(size_t requested_word_size) { // 3) Attempt to enlarge the current chunk in place if it is too small. // 4) Attempt to get a new chunk and allocate from that chunk. // At any point, if we hit a commit limit, we return null. -MetaWord* MetaspaceArena::allocate(size_t requested_word_size) { +MetaBlock MetaspaceArena::allocate(size_t requested_word_size, MetaBlock& wastage) { UL2(trace, "requested " SIZE_FORMAT " words.", requested_word_size); - MetaWord* p = nullptr; const size_t aligned_word_size = get_raw_word_size_for_requested_word_size(requested_word_size); + MetaBlock result; + bool taken_from_fbl = false; + // Before bothering the arena proper, attempt to re-use a block from the free blocks list if (_fbl != nullptr && !_fbl->is_empty()) { - p = _fbl->remove_block(aligned_word_size); - if (p != nullptr) { + result = _fbl->remove_block(aligned_word_size); + if (result.is_nonempty()) { + assert_block_larger_or_equal(result, aligned_word_size); + assert_block_base_aligned(result, allocation_alignment_words()); + assert_block_size_aligned(result, Metaspace::min_allocation_alignment_words); + // Split off wastage + wastage = result.split_off_tail(result.word_size() - aligned_word_size); + // Stats, logging DEBUG_ONLY(InternalStats::inc_num_allocs_from_deallocated_blocks();) - UL2(trace, "returning " PTR_FORMAT " - taken from fbl (now: %d, " SIZE_FORMAT ").", - p2i(p), _fbl->count(), _fbl->total_size()); - assert_is_aligned_metaspace_pointer(p); + UL2(trace, "returning " METABLOCKFORMAT " with wastage " METABLOCKFORMAT " - taken from fbl (now: %d, " SIZE_FORMAT ").", + METABLOCKFORMATARGS(result), METABLOCKFORMATARGS(wastage), _fbl->count(), _fbl->total_size()); // Note: free blocks in freeblock dictionary still count as "used" as far as statistics go; - // therefore we have no need to adjust any usage counters (see epilogue of allocate_inner()) - // and can just return here. - return p; + // therefore we don't need to adjust any usage counters (see epilogue of allocate_inner()). + taken_from_fbl = true; } } - // Primary allocation - p = allocate_inner(aligned_word_size); + if (result.is_empty()) { + // Free-block allocation failed; we allocate from the arena. + result = allocate_inner(aligned_word_size, wastage); + } - return p; + // Logging + if (result.is_nonempty()) { + LogTarget(Trace, metaspace) lt; + if (lt.is_enabled()) { + LogStream ls(lt); + ls.print(LOGFMT ": returning " METABLOCKFORMAT " taken from %s, ", LOGFMT_ARGS, + METABLOCKFORMATARGS(result), (taken_from_fbl ? "fbl" : "arena")); + if (wastage.is_empty()) { + ls.print("no wastage"); + } else { + ls.print("wastage " METABLOCKFORMAT, METABLOCKFORMATARGS(wastage)); + } + } + } else { + UL(info, "allocation failed, returned null."); + } + + // Final sanity checks +#ifdef ASSERT + result.verify(); + wastage.verify(); + if (result.is_nonempty()) { + assert(result.word_size() == aligned_word_size && + is_aligned(result.base(), _allocation_alignment_words * BytesPerWord), + "result bad or unaligned: " METABLOCKFORMAT ".", METABLOCKFORMATARGS(result)); + } + if (wastage.is_nonempty()) { + assert(wastage.is_empty() || + (wastage.is_aligned_base(Metaspace::min_allocation_alignment_words) && + wastage.is_aligned_size(Metaspace::min_allocation_alignment_words)), + "Misaligned wastage: " METABLOCKFORMAT".", METABLOCKFORMATARGS(wastage)); + } +#endif // ASSERT + return result; } // Allocate from the arena proper, once dictionary allocations and fencing are sorted out. -MetaWord* MetaspaceArena::allocate_inner(size_t word_size) { - assert_is_aligned(word_size, metaspace::AllocationAlignmentWordSize); +MetaBlock MetaspaceArena::allocate_inner(size_t word_size, MetaBlock& wastage) { - MetaWord* p = nullptr; + MetaBlock result; bool current_chunk_too_small = false; bool commit_failure = false; + size_t alignment_gap_size = 0; if (current_chunk() != nullptr) { - // Attempt to satisfy the allocation from the current chunk. + const MetaWord* const chunk_top = current_chunk()->top(); + alignment_gap_size = align_up(chunk_top, _allocation_alignment_words * BytesPerWord) - chunk_top; + const size_t word_size_plus_alignment = word_size + alignment_gap_size; + // If the current chunk is too small to hold the requested size, attempt to enlarge it. // If that fails, retire the chunk. - if (current_chunk()->free_words() < word_size) { - if (!attempt_enlarge_current_chunk(word_size)) { + if (current_chunk()->free_words() < word_size_plus_alignment) { + if (!attempt_enlarge_current_chunk(word_size_plus_alignment)) { current_chunk_too_small = true; } else { DEBUG_ONLY(InternalStats::inc_num_chunks_enlarged();) @@ -259,20 +313,26 @@ MetaWord* MetaspaceArena::allocate_inner(size_t word_size) { // hit a limit (either GC threshold or MaxMetaspaceSize). In that case retire the // chunk. if (!current_chunk_too_small) { - if (!current_chunk()->ensure_committed_additional(word_size)) { - UL2(info, "commit failure (requested size: " SIZE_FORMAT ")", word_size); + if (!current_chunk()->ensure_committed_additional(word_size_plus_alignment)) { + UL2(info, "commit failure (requested size: " SIZE_FORMAT ")", word_size_plus_alignment); commit_failure = true; } } // Allocate from the current chunk. This should work now. if (!current_chunk_too_small && !commit_failure) { - p = current_chunk()->allocate(word_size); - assert(p != nullptr, "Allocation from chunk failed."); + MetaWord* const p_gap = current_chunk()->allocate(word_size_plus_alignment); + assert(p_gap != nullptr, "Allocation from chunk failed."); + MetaWord* const p_user_allocation = p_gap + alignment_gap_size; + result = MetaBlock(p_user_allocation, word_size); + if (alignment_gap_size > 0) { + NOT_LP64(assert(alignment_gap_size >= AllocationAlignmentWordSize, "Sanity")); + wastage = MetaBlock(p_gap, alignment_gap_size); + } } } - if (p == nullptr) { + if (result.is_empty()) { // If we are here, we either had no current chunk to begin with or it was deemed insufficient. assert(current_chunk() == nullptr || current_chunk_too_small || commit_failure, "Sanity"); @@ -286,63 +346,69 @@ MetaWord* MetaspaceArena::allocate_inner(size_t word_size) { // We have a new chunk. Before making it the current chunk, retire the old one. if (current_chunk() != nullptr) { - salvage_chunk(current_chunk()); + wastage = salvage_chunk(current_chunk()); DEBUG_ONLY(InternalStats::inc_num_chunks_retired();) } _chunks.add(new_chunk); - // Now, allocate from that chunk. That should work. - p = current_chunk()->allocate(word_size); + // Now, allocate from that chunk. That should work. Note that the resulting allocation + // is guaranteed to be aligned to arena alignment, since arena alignment cannot be larger + // than smallest chunk size, and chunk starts are aligned by their size (buddy allocation). + MetaWord* const p = current_chunk()->allocate(word_size); assert(p != nullptr, "Allocation from chunk failed."); + result = MetaBlock(p, word_size); } else { UL2(info, "failed to allocate new chunk for requested word size " SIZE_FORMAT ".", word_size); } } - if (p == nullptr) { + if (result.is_empty()) { InternalStats::inc_num_allocs_failed_limit(); } else { DEBUG_ONLY(InternalStats::inc_num_allocs();) - _total_used_words_counter->increment_by(word_size); + _total_used_words_counter->increment_by(word_size + wastage.word_size()); } SOMETIMES(verify();) - if (p == nullptr) { - UL(info, "allocation failed, returned null."); - } else { + if (result.is_nonempty()) { UL2(trace, "after allocation: %u chunk(s), current:" METACHUNK_FULL_FORMAT, _chunks.count(), METACHUNK_FULL_FORMAT_ARGS(current_chunk())); - UL2(trace, "returning " PTR_FORMAT ".", p2i(p)); } - assert_is_aligned_metaspace_pointer(p); +#ifdef ASSERT + if (wastage.is_nonempty()) { + // Wastage from arena allocations only occurs if either or both are true: + // - it is too small to hold the requested allocation words + // - it is misaligned + assert(!wastage.is_aligned_base(allocation_alignment_words()) || + wastage.word_size() < word_size, + "Unexpected wastage: " METABLOCKFORMAT ", arena alignment: %zu, allocation word size: %zu", + METABLOCKFORMATARGS(wastage), allocation_alignment_words(), word_size); + wastage.verify(); + } +#endif // ASSERT - return p; + return result; } // Prematurely returns a metaspace allocation to the _block_freelists // because it is not needed anymore (requires CLD lock to be active). -void MetaspaceArena::deallocate(MetaWord* p, size_t word_size) { - // At this point a current chunk must exist since we only deallocate if we did allocate before. - assert(current_chunk() != nullptr, "stray deallocation?"); - assert(is_valid_area(p, word_size), - "Pointer range not part of this Arena and cannot be deallocated: (" PTR_FORMAT ".." PTR_FORMAT ").", - p2i(p), p2i(p + word_size)); - - UL2(trace, "deallocating " PTR_FORMAT ", word size: " SIZE_FORMAT ".", - p2i(p), word_size); - - // Only blocks that had been allocated via MetaspaceArena::allocate(size) must be handed in - // to MetaspaceArena::deallocate(), and only with the same size that had been original used for allocation. - // Therefore the pointer must be aligned correctly, and size can be alignment-adjusted (the latter - // only matters on 32-bit): - assert_is_aligned_metaspace_pointer(p); - size_t raw_word_size = get_raw_word_size_for_requested_word_size(word_size); - - add_allocation_to_fbl(p, raw_word_size); - +void MetaspaceArena::deallocate(MetaBlock block) { + // Note that we may receive blocks that don't originate from this + // arena, and that is okay. + DEBUG_ONLY(block.verify();) + // This only matters on 32-bit: + // Since we always align up allocations from arena, we align up here, too. +#ifndef _LP64 + MetaBlock raw_block(block.base(), get_raw_word_size_for_requested_word_size(block.word_size())); + add_allocation_to_fbl(raw_block); +#else + add_allocation_to_fbl(block); +#endif + UL2(trace, "added to fbl: " METABLOCKFORMAT ", (now: %d, " SIZE_FORMAT ").", + METABLOCKFORMATARGS(block), _fbl->count(), _fbl->total_size()); SOMETIMES(verify();) } @@ -400,15 +466,15 @@ void MetaspaceArena::verify() const { } } -// Returns true if the area indicated by pointer and size have actually been allocated -// from this arena. -bool MetaspaceArena::is_valid_area(MetaWord* p, size_t word_size) const { - assert(p != nullptr && word_size > 0, "Sanity"); +// Returns true if the given block is contained in this arena +bool MetaspaceArena::contains(MetaBlock bl) const { + DEBUG_ONLY(bl.verify();) + assert(bl.is_nonempty(), "Sanity"); bool found = false; for (const Metachunk* c = _chunks.first(); c != nullptr && !found; c = c->next()) { - assert(c->is_valid_committed_pointer(p) == - c->is_valid_committed_pointer(p + word_size - 1), "range intersects"); - found = c->is_valid_committed_pointer(p); + assert(c->is_valid_committed_pointer(bl.base()) == + c->is_valid_committed_pointer(bl.end() - 1), "range intersects"); + found = c->is_valid_committed_pointer(bl.base()); } return found; } diff --git a/src/hotspot/share/memory/metaspace/metaspaceArena.hpp b/src/hotspot/share/memory/metaspace/metaspaceArena.hpp index 77eb939c6b4..75d1ec5318c 100644 --- a/src/hotspot/share/memory/metaspace/metaspaceArena.hpp +++ b/src/hotspot/share/memory/metaspace/metaspaceArena.hpp @@ -29,6 +29,7 @@ #include "memory/allocation.hpp" #include "memory/metaspace.hpp" #include "memory/metaspace/counters.hpp" +#include "memory/metaspace/metablock.hpp" #include "memory/metaspace/metachunkList.hpp" class outputStream; @@ -37,11 +38,12 @@ class Mutex; namespace metaspace { class ArenaGrowthPolicy; +struct ArenaStats; class ChunkManager; -class Metachunk; class FreeBlocks; +class Metachunk; +class MetaspaceContext; -struct ArenaStats; // The MetaspaceArena is a growable metaspace memory pool belonging to a CLD; // internally it consists of a list of metaspace chunks, of which the head chunk @@ -74,10 +76,14 @@ struct ArenaStats; // class MetaspaceArena : public CHeapObj { + friend class MetaspaceArenaTestFriend; // Please note that access to a metaspace arena may be shared // between threads and needs to be synchronized in CLMS. + // Allocation alignment specific to this arena + const size_t _allocation_alignment_words; + // Reference to the chunk manager to allocate chunks from. ChunkManager* const _chunk_manager; @@ -104,10 +110,10 @@ class MetaspaceArena : public CHeapObj { // free block list FreeBlocks* fbl() const { return _fbl; } - void add_allocation_to_fbl(MetaWord* p, size_t word_size); + void add_allocation_to_fbl(MetaBlock bl); - // Given a chunk, add its remaining free committed space to the free block list. - void salvage_chunk(Metachunk* c); + // Given a chunk, return the committed remainder of this chunk. + MetaBlock salvage_chunk(Metachunk* c); // Allocate a new chunk from the underlying chunk manager able to hold at least // requested word size. @@ -122,32 +128,31 @@ class MetaspaceArena : public CHeapObj { // On success, true is returned, false otherwise. bool attempt_enlarge_current_chunk(size_t requested_word_size); - // Returns true if the area indicated by pointer and size have actually been allocated - // from this arena. - DEBUG_ONLY(bool is_valid_area(MetaWord* p, size_t word_size) const;) - // Allocate from the arena proper, once dictionary allocations and fencing are sorted out. - MetaWord* allocate_inner(size_t word_size); + MetaBlock allocate_inner(size_t word_size, MetaBlock& wastage); public: - MetaspaceArena(ChunkManager* chunk_manager, const ArenaGrowthPolicy* growth_policy, - SizeAtomicCounter* total_used_words_counter, + MetaspaceArena(MetaspaceContext* context, + const ArenaGrowthPolicy* growth_policy, + size_t allocation_alignment_words, const char* name); ~MetaspaceArena(); + size_t allocation_alignment_words() const { return _allocation_alignment_words; } + size_t allocation_alignment_bytes() const { return allocation_alignment_words() * BytesPerWord; } + // Allocate memory from Metaspace. - // 1) Attempt to allocate from the dictionary of deallocated blocks. - // 2) Attempt to allocate from the current chunk. - // 3) Attempt to enlarge the current chunk in place if it is too small. - // 4) Attempt to get a new chunk and allocate from that chunk. - // At any point, if we hit a commit limit, we return null. - MetaWord* allocate(size_t word_size); + // On success, returns non-empty block of the specified word size, and + // possibly a wastage block that is the result of alignment operations. + // On failure, returns an empty block. Failure may happen if we hit a + // commit limit. + MetaBlock allocate(size_t word_size, MetaBlock& wastage); // Prematurely returns a metaspace allocation to the _block_freelists because it is not // needed anymore. - void deallocate(MetaWord* p, size_t word_size); + void deallocate(MetaBlock bl); // Update statistics. This walks all in-use chunks. void add_to_statistics(ArenaStats* out) const; @@ -161,6 +166,8 @@ class MetaspaceArena : public CHeapObj { void print_on(outputStream* st) const; + // Returns true if the given block is contained in this arena + DEBUG_ONLY(bool contains(MetaBlock bl) const;) }; } // namespace metaspace diff --git a/src/hotspot/share/memory/metaspace/metaspaceCommon.hpp b/src/hotspot/share/memory/metaspace/metaspaceCommon.hpp index 511f4e6e092..d296ffd6cd7 100644 --- a/src/hotspot/share/memory/metaspace/metaspaceCommon.hpp +++ b/src/hotspot/share/memory/metaspace/metaspaceCommon.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2018, 2024, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2018, 2020 SAP SE. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * @@ -42,13 +42,11 @@ namespace metaspace { // correctly. We currently don't hold members with a larger alignment requirement // than 64-bit inside MetaData, so 8-byte alignment is enough. // -// Klass* structures need to be aligned to KlassAlignmentInBytes, but since that is -// 64-bit, we don't need special handling for allocating Klass*. +// Klass* structures need to be aligned to Klass* alignment, // // On 64-bit platforms, we align to word size; on 32-bit, we align to two words. static const size_t AllocationAlignmentByteSize = 8; -STATIC_ASSERT(AllocationAlignmentByteSize == (size_t)KlassAlignmentInBytes); static const size_t AllocationAlignmentWordSize = AllocationAlignmentByteSize / BytesPerWord; diff --git a/src/hotspot/share/memory/metaspace/metaspaceContext.cpp b/src/hotspot/share/memory/metaspace/metaspaceContext.cpp index e2a2d752d64..b43f4cd3b15 100644 --- a/src/hotspot/share/memory/metaspace/metaspaceContext.cpp +++ b/src/hotspot/share/memory/metaspace/metaspaceContext.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2024, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2020 SAP SE. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * @@ -75,6 +75,18 @@ void MetaspaceContext::print_on(outputStream* st) const { _cm->print_on(st); } +size_t MetaspaceContext::used_words() const { + return _used_words_counter.get(); +} + +size_t MetaspaceContext::committed_words() const { + return _vslist->committed_words(); +} + +size_t MetaspaceContext::reserved_words() const { + return _vslist->reserved_words(); +} + #ifdef ASSERT void MetaspaceContext::verify() const { _vslist->verify(); diff --git a/src/hotspot/share/memory/metaspace/metaspaceContext.hpp b/src/hotspot/share/memory/metaspace/metaspaceContext.hpp index be1d6f356ae..c773c6385b3 100644 --- a/src/hotspot/share/memory/metaspace/metaspaceContext.hpp +++ b/src/hotspot/share/memory/metaspace/metaspaceContext.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2024, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2020 SAP SE. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * @@ -27,6 +27,7 @@ #define SHARE_MEMORY_METASPACE_METASPACECONTEXT_HPP #include "memory/allocation.hpp" +#include "memory/metaspace/counters.hpp" #include "memory/virtualspace.hpp" #include "utilities/debug.hpp" @@ -61,6 +62,7 @@ class MetaspaceContext : public CHeapObj { const char* const _name; VirtualSpaceList* const _vslist; ChunkManager* const _cm; + SizeAtomicCounter _used_words_counter; MetaspaceContext(const char* name, VirtualSpaceList* vslist, ChunkManager* cm) : _name(name), @@ -78,8 +80,9 @@ class MetaspaceContext : public CHeapObj { // untouched, otherwise all memory is unmapped. ~MetaspaceContext(); - VirtualSpaceList* vslist() { return _vslist; } - ChunkManager* cm() { return _cm; } + VirtualSpaceList* vslist() { return _vslist; } + ChunkManager* cm() { return _cm; } + SizeAtomicCounter* used_words_counter() { return &_used_words_counter; } // Create a new, empty, expandable metaspace context. static MetaspaceContext* create_expandable_context(const char* name, CommitLimiter* commit_limiter); @@ -103,6 +106,9 @@ class MetaspaceContext : public CHeapObj { // null otherwise. static MetaspaceContext* context_class() { return _class_space_context; } + size_t used_words() const; + size_t committed_words() const; + size_t reserved_words() const; }; } // end namespace diff --git a/src/hotspot/share/memory/metaspace/metaspaceReporter.cpp b/src/hotspot/share/memory/metaspace/metaspaceReporter.cpp index 3cd9ba5ab87..cbd2400444f 100644 --- a/src/hotspot/share/memory/metaspace/metaspaceReporter.cpp +++ b/src/hotspot/share/memory/metaspace/metaspaceReporter.cpp @@ -39,6 +39,7 @@ #include "memory/metaspace/runningCounters.hpp" #include "memory/metaspace/virtualSpaceList.hpp" #include "memory/metaspaceUtils.hpp" +#include "oops/compressedKlass.hpp" #include "runtime/os.hpp" namespace metaspace { @@ -117,6 +118,9 @@ static void print_settings(outputStream* out, size_t scale) { out->cr(); out->print_cr("CDS: %s", (CDSConfig::is_using_archive() ? "on" : (CDSConfig::is_dumping_static_archive() ? "dump" : "off"))); Settings::print_on(out); +#ifdef _LP64 + CompressedKlassPointers::print_mode(out); +#endif } // This will print out a basic metaspace usage report but @@ -324,7 +328,7 @@ void MetaspaceReporter::print_report(outputStream* out, size_t scale, int flags) // For all wastages, print percentages from total. As total use the total size of memory committed for metaspace. const size_t committed_words = RunningCounters::committed_words(); - out->print("(percentages refer to total committed size "); + out->print(" (percentages refer to total committed size "); print_scaled_words(out, committed_words, scale); out->print_cr("):"); diff --git a/src/hotspot/share/memory/metaspace/metaspaceStatistics.cpp b/src/hotspot/share/memory/metaspace/metaspaceStatistics.cpp index 2c32cbe38b1..32329831e7c 100644 --- a/src/hotspot/share/memory/metaspace/metaspaceStatistics.cpp +++ b/src/hotspot/share/memory/metaspace/metaspaceStatistics.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2018, 2024, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2018, 2020 SAP SE. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * @@ -186,16 +186,12 @@ void ArenaStats::print_on(outputStream* st, size_t scale, bool detailed) const } #ifdef ASSERT - void ArenaStats::verify() const { size_t total_used = 0; for (chunklevel_t l = chunklevel::LOWEST_CHUNK_LEVEL; l <= chunklevel::HIGHEST_CHUNK_LEVEL; l++) { _stats[l].verify(); total_used += _stats[l]._used_words; } - // Deallocated allocations still count as used - assert(total_used >= _free_blocks_word_size, - "Sanity"); } #endif diff --git a/src/hotspot/share/memory/metaspace/runningCounters.cpp b/src/hotspot/share/memory/metaspace/runningCounters.cpp index 323862ac02f..75fc4b9792c 100644 --- a/src/hotspot/share/memory/metaspace/runningCounters.cpp +++ b/src/hotspot/share/memory/metaspace/runningCounters.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2024, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2020 SAP SE. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * @@ -25,15 +25,12 @@ #include "precompiled.hpp" #include "memory/metaspace/chunkManager.hpp" -#include "memory/metaspace/counters.hpp" +#include "memory/metaspace/metaspaceContext.hpp" #include "memory/metaspace/runningCounters.hpp" #include "memory/metaspace/virtualSpaceList.hpp" namespace metaspace { -SizeAtomicCounter RunningCounters::_used_class_counter; -SizeAtomicCounter RunningCounters::_used_nonclass_counter; - // Return reserved size, in words, for Metaspace size_t RunningCounters::reserved_words() { return reserved_words_class() + reserved_words_nonclass(); @@ -72,11 +69,12 @@ size_t RunningCounters::used_words() { } size_t RunningCounters::used_words_class() { - return _used_class_counter.get(); + const MetaspaceContext* context = MetaspaceContext::context_class(); + return context != nullptr ? context->used_words() : 0; } size_t RunningCounters::used_words_nonclass() { - return _used_nonclass_counter.get(); + return MetaspaceContext::context_nonclass()->used_words(); } // ---- free chunks ----- diff --git a/src/hotspot/share/memory/metaspace/runningCounters.hpp b/src/hotspot/share/memory/metaspace/runningCounters.hpp index ff24ead7ebd..e898a099818 100644 --- a/src/hotspot/share/memory/metaspace/runningCounters.hpp +++ b/src/hotspot/share/memory/metaspace/runningCounters.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020, 2022, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2024, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2020, 2022 SAP SE. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * @@ -27,17 +27,11 @@ #define SHARE_MEMORY_METASPACE_RUNNINGCOUNTERS_HPP #include "memory/allStatic.hpp" -#include "memory/metaspace/counters.hpp" namespace metaspace { // This class is a convenience interface for accessing global metaspace counters. -class RunningCounters : public AllStatic { - - static SizeAtomicCounter _used_class_counter; - static SizeAtomicCounter _used_nonclass_counter; - -public: +struct RunningCounters : public AllStatic { // ---- virtual memory ----- @@ -65,10 +59,6 @@ class RunningCounters : public AllStatic { static size_t free_chunks_words_class(); static size_t free_chunks_words_nonclass(); - // Direct access to the counters. - static SizeAtomicCounter* used_nonclass_counter() { return &_used_nonclass_counter; } - static SizeAtomicCounter* used_class_counter() { return &_used_class_counter; } - }; } // namespace metaspace diff --git a/src/hotspot/share/memory/metaspace/testHelpers.cpp b/src/hotspot/share/memory/metaspace/testHelpers.cpp index 2d1071d77dd..b974f06f243 100644 --- a/src/hotspot/share/memory/metaspace/testHelpers.cpp +++ b/src/hotspot/share/memory/metaspace/testHelpers.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2024, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2020, 2021 SAP SE. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * @@ -54,12 +54,17 @@ MetaspaceTestArena::~MetaspaceTestArena() { MetaWord* MetaspaceTestArena::allocate(size_t word_size) { MutexLocker fcl(_lock, Mutex::_no_safepoint_check_flag); - return _arena->allocate(word_size); + MetaBlock result, wastage; + result = _arena->allocate(word_size, wastage); + if (wastage.is_nonempty()) { + _arena->deallocate(wastage); + } + return result.base(); } void MetaspaceTestArena::deallocate(MetaWord* p, size_t word_size) { MutexLocker fcl(_lock, Mutex::_no_safepoint_check_flag); - return _arena->deallocate(p, word_size); + _arena->deallocate(MetaBlock(p, word_size)); } ///// MetaspaceTestArea ////// @@ -70,7 +75,6 @@ MetaspaceTestContext::MetaspaceTestContext(const char* name, size_t commit_limit _commit_limit(commit_limit), _context(nullptr), _commit_limiter(commit_limit == 0 ? max_uintx : commit_limit), // commit_limit == 0 -> no limit - _used_words_counter(), _rs() { assert(is_aligned(reserve_limit, Metaspace::reserve_alignment_words()), "reserve_limit (" SIZE_FORMAT ") " @@ -103,7 +107,7 @@ MetaspaceTestArena* MetaspaceTestContext::create_arena(Metaspace::MetaspaceType MetaspaceArena* arena = nullptr; { MutexLocker ml(lock, Mutex::_no_safepoint_check_flag); - arena = new MetaspaceArena(_context->cm(), growth_policy, &_used_words_counter, _name); + arena = new MetaspaceArena(_context, growth_policy, Metaspace::min_allocation_alignment_words, _name); } return new MetaspaceTestArena(lock, arena); } @@ -124,5 +128,18 @@ void MetaspaceTestContext::print_on(outputStream* st) const { _context->print_on(st); } +size_t MetaspaceTestContext::used_words() const { + return _context->used_words_counter()->get(); +} + +size_t MetaspaceTestContext::committed_words() const { + assert(_commit_limiter.committed_words() == _context->committed_words(), "Sanity"); + return _context->committed_words(); +} + +size_t MetaspaceTestContext::reserved_words() const { + return _context->reserved_words(); +} + } // namespace metaspace diff --git a/src/hotspot/share/memory/metaspace/testHelpers.hpp b/src/hotspot/share/memory/metaspace/testHelpers.hpp index 669a3f61226..65d5ee47512 100644 --- a/src/hotspot/share/memory/metaspace/testHelpers.hpp +++ b/src/hotspot/share/memory/metaspace/testHelpers.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020, 2022, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2024, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2020 SAP SE. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * @@ -75,7 +75,6 @@ class MetaspaceTestContext : public CHeapObj { MetaspaceContext* _context; CommitLimiter _commit_limiter; - SizeAtomicCounter _used_words_counter; // For non-expandable contexts we keep track of the space // and delete it at destruction time. @@ -98,15 +97,16 @@ class MetaspaceTestContext : public CHeapObj { const CommitLimiter& commit_limiter() const { return _commit_limiter; } const VirtualSpaceList& vslist() const { return *(_context->vslist()); } ChunkManager& cm() { return *(_context->cm()); } + MetaspaceContext* context() const { return _context; } // Returns reserve- and commit limit we run the test with (in the real world, // these would be equivalent to CompressedClassSpaceSize resp MaxMetaspaceSize) size_t reserve_limit() const { return _reserve_limit == 0 ? max_uintx : 0; } size_t commit_limit() const { return _commit_limit == 0 ? max_uintx : 0; } - // Convenience function to retrieve total committed/used words - size_t used_words() const { return _used_words_counter.get(); } - size_t committed_words() const { return _commit_limiter.committed_words(); } + size_t used_words() const; + size_t committed_words() const; + size_t reserved_words() const; DEBUG_ONLY(void verify() const;) diff --git a/src/hotspot/share/memory/universe.cpp b/src/hotspot/share/memory/universe.cpp index 4af9c8b2ef2..6daeb2f1045 100644 --- a/src/hotspot/share/memory/universe.cpp +++ b/src/hotspot/share/memory/universe.cpp @@ -397,8 +397,13 @@ void Universe::genesis(TRAPS) { HandleMark hm(THREAD); // Explicit null checks are needed if these offsets are not smaller than the page size - assert(oopDesc::klass_offset_in_bytes() < static_cast(os::vm_page_size()), - "Klass offset is expected to be less than the page size"); + if (UseCompactObjectHeaders) { + assert(oopDesc::mark_offset_in_bytes() < static_cast(os::vm_page_size()), + "Mark offset is expected to be less than the page size"); + } else { + assert(oopDesc::klass_offset_in_bytes() < static_cast(os::vm_page_size()), + "Klass offset is expected to be less than the page size"); + } assert(arrayOopDesc::length_offset_in_bytes() < static_cast(os::vm_page_size()), "Array length offset is expected to be less than the page size"); diff --git a/src/hotspot/share/nmt/memBaseline.cpp b/src/hotspot/share/nmt/memBaseline.cpp index 270601e9c05..6f82b2de9f1 100644 --- a/src/hotspot/share/nmt/memBaseline.cpp +++ b/src/hotspot/share/nmt/memBaseline.cpp @@ -141,7 +141,7 @@ void MemBaseline::baseline_summary() { MallocMemorySummary::snapshot(&_malloc_memory_snapshot); VirtualMemorySummary::snapshot(&_virtual_memory_snapshot); { - NmtVirtualMemoryLocker ml; + MemoryFileTracker::Instance::Locker lock; MemoryFileTracker::Instance::summary_snapshot(&_virtual_memory_snapshot); } diff --git a/src/hotspot/share/nmt/memReporter.cpp b/src/hotspot/share/nmt/memReporter.cpp index 15767da276c..6ce6206ebcc 100644 --- a/src/hotspot/share/nmt/memReporter.cpp +++ b/src/hotspot/share/nmt/memReporter.cpp @@ -465,7 +465,7 @@ void MemDetailReporter::report_virtual_memory_region(const ReservedMemoryRegion* void MemDetailReporter::report_memory_file_allocations() { stringStream st; { - NmtVirtualMemoryLocker ml; + MemoryFileTracker::Instance::Locker lock; MemoryFileTracker::Instance::print_all_reports_on(&st, scale()); } output()->print_raw(st.freeze()); diff --git a/src/hotspot/share/nmt/memTracker.hpp b/src/hotspot/share/nmt/memTracker.hpp index 92640430e1c..6ba1db2e7ff 100644 --- a/src/hotspot/share/nmt/memTracker.hpp +++ b/src/hotspot/share/nmt/memTracker.hpp @@ -31,6 +31,7 @@ #include "nmt/threadStackTracker.hpp" #include "nmt/virtualMemoryTracker.hpp" #include "runtime/mutexLocker.hpp" +#include "runtime/threadCritical.hpp" #include "utilities/debug.hpp" #include "utilities/nativeCallStack.hpp" @@ -124,7 +125,7 @@ class MemTracker : AllStatic { assert_post_init(); if (!enabled()) return; if (addr != nullptr) { - NmtVirtualMemoryLocker ml; + ThreadCritical tc; VirtualMemoryTracker::add_reserved_region((address)addr, size, stack, mem_tag); } } @@ -150,7 +151,7 @@ class MemTracker : AllStatic { assert_post_init(); if (!enabled()) return; if (addr != nullptr) { - NmtVirtualMemoryLocker ml; + ThreadCritical tc; VirtualMemoryTracker::add_reserved_region((address)addr, size, stack, mem_tag); VirtualMemoryTracker::add_committed_region((address)addr, size, stack); } @@ -161,7 +162,7 @@ class MemTracker : AllStatic { assert_post_init(); if (!enabled()) return; if (addr != nullptr) { - NmtVirtualMemoryLocker ml; + ThreadCritical tc; VirtualMemoryTracker::add_committed_region((address)addr, size, stack); } } @@ -169,7 +170,7 @@ class MemTracker : AllStatic { static inline MemoryFileTracker::MemoryFile* register_file(const char* descriptive_name) { assert_post_init(); if (!enabled()) return nullptr; - NmtVirtualMemoryLocker ml; + MemoryFileTracker::Instance::Locker lock; return MemoryFileTracker::Instance::make_file(descriptive_name); } @@ -177,7 +178,7 @@ class MemTracker : AllStatic { assert_post_init(); if (!enabled()) return; assert(file != nullptr, "must be"); - NmtVirtualMemoryLocker ml; + MemoryFileTracker::Instance::Locker lock; MemoryFileTracker::Instance::free_file(file); } @@ -186,7 +187,7 @@ class MemTracker : AllStatic { assert_post_init(); if (!enabled()) return; assert(file != nullptr, "must be"); - NmtVirtualMemoryLocker ml; + MemoryFileTracker::Instance::Locker lock; MemoryFileTracker::Instance::allocate_memory(file, offset, size, stack, mem_tag); } @@ -195,7 +196,7 @@ class MemTracker : AllStatic { assert_post_init(); if (!enabled()) return; assert(file != nullptr, "must be"); - NmtVirtualMemoryLocker ml; + MemoryFileTracker::Instance::Locker lock; MemoryFileTracker::Instance::free_memory(file, offset, size); } @@ -209,7 +210,7 @@ class MemTracker : AllStatic { assert_post_init(); if (!enabled()) return; if (addr != nullptr) { - NmtVirtualMemoryLocker ml; + ThreadCritical tc; VirtualMemoryTracker::split_reserved_region((address)addr, size, split, mem_tag, split_tag); } } @@ -218,7 +219,7 @@ class MemTracker : AllStatic { assert_post_init(); if (!enabled()) return; if (addr != nullptr) { - NmtVirtualMemoryLocker ml; + ThreadCritical tc; VirtualMemoryTracker::set_reserved_region_type((address)addr, mem_tag); } } diff --git a/src/hotspot/share/nmt/memoryFileTracker.cpp b/src/hotspot/share/nmt/memoryFileTracker.cpp index ab4f8b6d1f3..ede483ed337 100644 --- a/src/hotspot/share/nmt/memoryFileTracker.cpp +++ b/src/hotspot/share/nmt/memoryFileTracker.cpp @@ -29,11 +29,13 @@ #include "nmt/nmtCommon.hpp" #include "nmt/nmtNativeCallStackStorage.hpp" #include "nmt/vmatree.hpp" +#include "runtime/mutex.hpp" #include "utilities/growableArray.hpp" #include "utilities/nativeCallStack.hpp" #include "utilities/ostream.hpp" MemoryFileTracker* MemoryFileTracker::Instance::_tracker = nullptr; +PlatformMutex* MemoryFileTracker::Instance::_mutex = nullptr; MemoryFileTracker::MemoryFileTracker(bool is_detailed_mode) : _stack_storage(is_detailed_mode), _files() {} @@ -130,6 +132,7 @@ bool MemoryFileTracker::Instance::initialize(NMT_TrackingLevel tracking_level) { _tracker = static_cast(os::malloc(sizeof(MemoryFileTracker), mtNMT)); if (_tracker == nullptr) return false; new (_tracker) MemoryFileTracker(tracking_level == NMT_TrackingLevel::NMT_detail); + _mutex = new PlatformMutex(); return true; } @@ -190,3 +193,11 @@ void MemoryFileTracker::summary_snapshot(VirtualMemorySnapshot* snapshot) const void MemoryFileTracker::Instance::summary_snapshot(VirtualMemorySnapshot* snapshot) { _tracker->summary_snapshot(snapshot); } + +MemoryFileTracker::Instance::Locker::Locker() { + MemoryFileTracker::Instance::_mutex->lock(); +} + +MemoryFileTracker::Instance::Locker::~Locker() { + MemoryFileTracker::Instance::_mutex->unlock(); +} diff --git a/src/hotspot/share/nmt/memoryFileTracker.hpp b/src/hotspot/share/nmt/memoryFileTracker.hpp index 911f10baf98..42902701a16 100644 --- a/src/hotspot/share/nmt/memoryFileTracker.hpp +++ b/src/hotspot/share/nmt/memoryFileTracker.hpp @@ -30,6 +30,7 @@ #include "nmt/nmtNativeCallStackStorage.hpp" #include "nmt/virtualMemoryTracker.hpp" #include "nmt/vmatree.hpp" +#include "runtime/mutex.hpp" #include "runtime/os.inline.hpp" #include "utilities/growableArray.hpp" #include "utilities/nativeCallStack.hpp" @@ -80,8 +81,14 @@ class MemoryFileTracker { class Instance : public AllStatic { static MemoryFileTracker* _tracker; + static PlatformMutex* _mutex; public: + class Locker : public StackObj { + public: + Locker(); + ~Locker(); + }; static bool initialize(NMT_TrackingLevel tracking_level); diff --git a/src/hotspot/share/nmt/nmtCommon.hpp b/src/hotspot/share/nmt/nmtCommon.hpp index a6fd29e83f2..3f72960f21f 100644 --- a/src/hotspot/share/nmt/nmtCommon.hpp +++ b/src/hotspot/share/nmt/nmtCommon.hpp @@ -29,8 +29,6 @@ #include "memory/allStatic.hpp" #include "nmt/memTag.hpp" -#include "runtime/mutexLocker.hpp" -#include "runtime/thread.hpp" #include "utilities/align.hpp" #include "utilities/globalDefinitions.hpp" @@ -139,13 +137,5 @@ class NMTUtil : AllStatic { static S _strings[mt_number_of_tags]; }; -// Same as MutexLocker but can be used during VM init. -// Performs no action if given a null mutex or with detached threads. -class NmtVirtualMemoryLocker: public ConditionalMutexLocker { -public: - NmtVirtualMemoryLocker() : - ConditionalMutexLocker(NmtVirtualMemory_lock, Thread::current_or_null_safe() != nullptr, Mutex::_no_safepoint_check_flag) { - } -}; #endif // SHARE_NMT_NMTCOMMON_HPP diff --git a/src/hotspot/share/nmt/threadStackTracker.cpp b/src/hotspot/share/nmt/threadStackTracker.cpp index 42af67d6464..6f112fa8fc5 100644 --- a/src/hotspot/share/nmt/threadStackTracker.cpp +++ b/src/hotspot/share/nmt/threadStackTracker.cpp @@ -29,6 +29,7 @@ #include "nmt/threadStackTracker.hpp" #include "nmt/virtualMemoryTracker.hpp" #include "runtime/os.hpp" +#include "runtime/threadCritical.hpp" #include "utilities/align.hpp" #include "utilities/debug.hpp" #include "utilities/globalDefinitions.hpp" @@ -52,7 +53,7 @@ void ThreadStackTracker::new_thread_stack(void* base, size_t size, const NativeC assert(base != nullptr, "Should have been filtered"); align_thread_stack_boundaries_inward(base, size); - NmtVirtualMemoryLocker ml; + ThreadCritical tc; VirtualMemoryTracker::add_reserved_region((address)base, size, stack, mtThreadStack); _thread_count++; } @@ -62,7 +63,7 @@ void ThreadStackTracker::delete_thread_stack(void* base, size_t size) { assert(base != nullptr, "Should have been filtered"); align_thread_stack_boundaries_inward(base, size); - NmtVirtualMemoryLocker ml; + ThreadCritical tc; VirtualMemoryTracker::remove_released_region((address)base, size); _thread_count--; } diff --git a/src/hotspot/share/nmt/virtualMemoryTracker.cpp b/src/hotspot/share/nmt/virtualMemoryTracker.cpp index 8b0f2f4d7a4..d298381f103 100644 --- a/src/hotspot/share/nmt/virtualMemoryTracker.cpp +++ b/src/hotspot/share/nmt/virtualMemoryTracker.cpp @@ -30,6 +30,7 @@ #include "nmt/threadStackTracker.hpp" #include "nmt/virtualMemoryTracker.hpp" #include "runtime/os.hpp" +#include "runtime/threadCritical.hpp" #include "utilities/ostream.hpp" VirtualMemorySnapshot VirtualMemorySummary::_snapshot; @@ -620,7 +621,6 @@ class SnapshotThreadStackWalker : public VirtualMemoryWalker { SnapshotThreadStackWalker() {} bool do_allocation_site(const ReservedMemoryRegion* rgn) { - assert_lock_strong(NmtVirtualMemory_lock); if (rgn->mem_tag() == mtThreadStack) { address stack_bottom = rgn->thread_stack_uncommitted_bottom(); address committed_start; @@ -661,7 +661,7 @@ void VirtualMemoryTracker::snapshot_thread_stacks() { bool VirtualMemoryTracker::walk_virtual_memory(VirtualMemoryWalker* walker) { assert(_reserved_regions != nullptr, "Sanity check"); - NmtVirtualMemoryLocker ml; + ThreadCritical tc; // Check that the _reserved_regions haven't been deleted. if (_reserved_regions != nullptr) { LinkedListNode* head = _reserved_regions->head(); diff --git a/src/hotspot/share/oops/arrayOop.hpp b/src/hotspot/share/oops/arrayOop.hpp index 1ca8a9530a4..f0c476a2486 100644 --- a/src/hotspot/share/oops/arrayOop.hpp +++ b/src/hotspot/share/oops/arrayOop.hpp @@ -79,11 +79,11 @@ class arrayOopDesc : public oopDesc { } // The _length field is not declared in C++. It is allocated after the - // declared nonstatic fields in arrayOopDesc if not compressed, otherwise - // it occupies the second half of the _klass field in oopDesc. + // mark-word when using compact headers (+UseCompactObjectHeaders), otherwise + // after the compressed Klass* when running with compressed class-pointers + // (+UseCompressedClassPointers), or else after the full Klass*. static int length_offset_in_bytes() { - return UseCompressedClassPointers ? klass_gap_offset_in_bytes() : - (int)sizeof(arrayOopDesc); + return oopDesc::base_offset_in_bytes(); } // Returns the offset of the first element. diff --git a/src/hotspot/share/oops/compressedKlass.cpp b/src/hotspot/share/oops/compressedKlass.cpp index 02483d25a9a..f3c6fe92897 100644 --- a/src/hotspot/share/oops/compressedKlass.cpp +++ b/src/hotspot/share/oops/compressedKlass.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2019, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -27,39 +27,145 @@ #include "memory/metaspace.hpp" #include "oops/compressedKlass.inline.hpp" #include "runtime/globals.hpp" +#include "runtime/java.hpp" #include "runtime/os.hpp" #include "utilities/debug.hpp" #include "utilities/globalDefinitions.hpp" #include "utilities/ostream.hpp" -address CompressedKlassPointers::_base = nullptr; -int CompressedKlassPointers::_shift = 0; +int CompressedKlassPointers::_narrow_klass_pointer_bits = -1; +int CompressedKlassPointers::_max_shift = -1; + +address CompressedKlassPointers::_base = (address)-1; +int CompressedKlassPointers::_shift = -1; address CompressedKlassPointers::_klass_range_start = nullptr; address CompressedKlassPointers::_klass_range_end = nullptr; +narrowKlass CompressedKlassPointers::_lowest_valid_narrow_klass_id = (narrowKlass)-1; +narrowKlass CompressedKlassPointers::_highest_valid_narrow_klass_id = (narrowKlass)-1; #ifdef _LP64 -#ifdef ASSERT -void CompressedKlassPointers::assert_is_valid_encoding(address addr, size_t len, address base, int shift) { - assert(base + nth_bit(32 + shift) >= addr + len, "Encoding (base=" PTR_FORMAT ", shift=%d) does not " - "fully cover the class range " PTR_FORMAT "-" PTR_FORMAT, p2i(base), shift, p2i(addr), p2i(addr + len)); +size_t CompressedKlassPointers::max_klass_range_size() { + // We disallow klass range sizes larger than 4GB even if the encoding + // range would allow for a larger Klass range (e.g. Base=zero, shift=3 -> 32GB). + // That is because many CPU-specific compiler decodings do not want the + // shifted narrow Klass to spill over into the third quadrant of the 64-bit target + // address, e.g. to use a 16-bit move for a simplified base addition. + return MIN2(4 * G, max_encoding_range_size()); +} + +void CompressedKlassPointers::pre_initialize() { + if (UseCompactObjectHeaders) { + _narrow_klass_pointer_bits = narrow_klass_pointer_bits_coh; + _max_shift = max_shift_coh; + } else { + _narrow_klass_pointer_bits = narrow_klass_pointer_bits_noncoh; + _max_shift = max_shift_noncoh; + } } + +#ifdef ASSERT +void CompressedKlassPointers::sanity_check_after_initialization() { + // In expectation of an assert, prepare condensed info to be printed with the assert. + char tmp[256]; + os::snprintf(tmp, sizeof(tmp), "klass range: " RANGE2FMT "," + " base " PTR_FORMAT ", shift %d, lowest/highest valid narrowKlass %u/%u", + RANGE2FMTARGS(_klass_range_start, _klass_range_end), + p2i(_base), _shift, _lowest_valid_narrow_klass_id, _highest_valid_narrow_klass_id); +#define ASSERT_HERE(cond) assert(cond, " (%s)", tmp); +#define ASSERT_HERE_2(cond, msg) assert(cond, msg " (%s)", tmp); + + // All values must be inited + ASSERT_HERE(_max_shift != -1); + ASSERT_HERE(_klass_range_start != (address)-1); + ASSERT_HERE(_klass_range_end != (address)-1); + ASSERT_HERE(_lowest_valid_narrow_klass_id != (narrowKlass)-1); + ASSERT_HERE(_base != (address)-1); + ASSERT_HERE(_shift != -1); + + const size_t klass_align = klass_alignment_in_bytes(); + + // must be aligned enough hold 64-bit data + ASSERT_HERE(is_aligned(klass_align, sizeof(uint64_t))); + + // should be smaller than the minimum metaspace chunk size (soft requirement) + ASSERT_HERE(klass_align <= K); + + ASSERT_HERE(_klass_range_end > _klass_range_start); + + // Check that Klass range is fully engulfed in the encoding range + const address encoding_start = _base; + const address encoding_end = _base + nth_bit(narrow_klass_pointer_bits() + _shift); + ASSERT_HERE_2(_klass_range_start >= _base && _klass_range_end <= encoding_end, + "Resulting encoding range does not fully cover the class range"); + + // Check that Klass range is aligned to Klass alignment. Note that this should never be + // an issue since the Klass range is handed in by either CDS- or Metaspace-initialization, and + // it should be the result of an mmap operation that operates on page sizes. So as long as + // the Klass alignment is <= page size, we are fine. + ASSERT_HERE_2(is_aligned(_klass_range_start, klass_align) && + is_aligned(_klass_range_end, klass_align), + "Klass range must start and end at a properly aligned address"); + + // Check _lowest_valid_narrow_klass_id and _highest_valid_narrow_klass_id + ASSERT_HERE_2(_lowest_valid_narrow_klass_id > 0, "Null is not a valid narrowKlass"); + ASSERT_HERE(_highest_valid_narrow_klass_id > _lowest_valid_narrow_klass_id); + + Klass* const k1 = decode_not_null_without_asserts(_lowest_valid_narrow_klass_id, _base, _shift); + if (encoding_start == _klass_range_start) { + ASSERT_HERE_2((address)k1 == _klass_range_start + klass_align, "Not lowest"); + } else { + ASSERT_HERE_2((address)k1 == _klass_range_start, "Not lowest"); + } + narrowKlass nk1 = encode_not_null_without_asserts(k1, _base, _shift); + ASSERT_HERE_2(nk1 == _lowest_valid_narrow_klass_id, "not reversible"); + + Klass* const k2 = decode_not_null_without_asserts(_highest_valid_narrow_klass_id, _base, _shift); + ASSERT_HERE((address)k2 == _klass_range_end - klass_align); + narrowKlass nk2 = encode_not_null_without_asserts(k2, _base, _shift); + ASSERT_HERE_2(nk2 == _highest_valid_narrow_klass_id, "not reversible"); + +#ifdef AARCH64 + // On aarch64, we never expect a shift value > 0 in standard (non-coh) mode + ASSERT_HERE_2(UseCompactObjectHeaders || _shift == 0, "Shift > 0 in non-coh mode?"); #endif +#undef ASSERT_HERE +#undef ASSERT_HERE_2 +} +#endif // ASSERT + +// Helper function: given current Klass Range, Base and Shift, calculate the lowest and highest values +// of narrowKlass we can expect. +void CompressedKlassPointers::calc_lowest_highest_narrow_klass_id() { + address lowest_possible_klass_location = _klass_range_start; + + // A Klass will never be placed at the Encoding range start, since that would translate to a narrowKlass=0, which + // is disallowed. Note that both Metaspace and CDS prvent allocation at the first address for this reason. + if (lowest_possible_klass_location == _base) { + lowest_possible_klass_location += klass_alignment_in_bytes(); + } + _lowest_valid_narrow_klass_id = (narrowKlass) ((uintptr_t)(lowest_possible_klass_location - _base) >> _shift); + + address highest_possible_klass_location = _klass_range_end - klass_alignment_in_bytes(); + _highest_valid_narrow_klass_id = (narrowKlass) ((uintptr_t)(highest_possible_klass_location - _base) >> _shift); +} // Given a klass range [addr, addr+len) and a given encoding scheme, assert that this scheme covers the range, then // set this encoding scheme. Used by CDS at runtime to re-instate the scheme used to pre-compute klass ids for // archived heap objects. void CompressedKlassPointers::initialize_for_given_encoding(address addr, size_t len, address requested_base, int requested_shift) { - address const end = addr + len; - - const int narrow_klasspointer_bits = sizeof(narrowKlass) * 8; - const size_t encoding_range_size = nth_bit(narrow_klasspointer_bits + requested_shift); - address encoding_range_end = requested_base + encoding_range_size; + if (len > max_klass_range_size()) { + stringStream ss; + ss.print("Class space size and CDS archive size combined (%zu) " + "exceed the maximum possible size (%zu)", + len, max_klass_range_size()); + vm_exit_during_initialization(ss.base()); + } - // Note: it would be technically valid for the encoding base to precede the start of the Klass range. But we only call - // this function from CDS, and therefore know this to be true. + // Note: While it would be technically valid for the encoding base to precede the start of the Klass range, + // we never do this here. This is used at CDS runtime to re-instate the scheme used to precompute the + // narrow Klass IDs in the archive, and the requested base should point to the start of the Klass range. assert(requested_base == addr, "Invalid requested base"); - assert(encoding_range_end >= end, "Encoding does not cover the full Klass range"); // Remember Klass range: _klass_range_start = addr; @@ -68,7 +174,9 @@ void CompressedKlassPointers::initialize_for_given_encoding(address addr, size_t _base = requested_base; _shift = requested_shift; - DEBUG_ONLY(assert_is_valid_encoding(addr, len, _base, _shift);) + calc_lowest_highest_narrow_klass_id(); + + DEBUG_ONLY(sanity_check_after_initialization();) } char* CompressedKlassPointers::reserve_address_space_X(uintptr_t from, uintptr_t to, size_t size, size_t alignment, bool aslr) { @@ -77,60 +185,105 @@ char* CompressedKlassPointers::reserve_address_space_X(uintptr_t from, uintptr_t } char* CompressedKlassPointers::reserve_address_space_for_unscaled_encoding(size_t size, bool aslr) { - return reserve_address_space_X(0, nth_bit(32), size, Metaspace::reserve_alignment(), aslr); + const size_t unscaled_max = nth_bit(narrow_klass_pointer_bits()); + return reserve_address_space_X(0, unscaled_max, size, Metaspace::reserve_alignment(), aslr); } char* CompressedKlassPointers::reserve_address_space_for_zerobased_encoding(size_t size, bool aslr) { - return reserve_address_space_X(nth_bit(32), nth_bit(32 + LogKlassAlignmentInBytes), size, Metaspace::reserve_alignment(), aslr); + const size_t unscaled_max = nth_bit(narrow_klass_pointer_bits()); + const size_t zerobased_max = nth_bit(narrow_klass_pointer_bits() + max_shift()); + return reserve_address_space_X(unscaled_max, zerobased_max, size, Metaspace::reserve_alignment(), aslr); } char* CompressedKlassPointers::reserve_address_space_for_16bit_move(size_t size, bool aslr) { return reserve_address_space_X(nth_bit(32), nth_bit(48), size, nth_bit(32), aslr); } -#if !defined(AARCH64) || defined(ZERO) -// On aarch64 we have an own version; all other platforms use the default version void CompressedKlassPointers::initialize(address addr, size_t len) { + if (len > max_klass_range_size()) { + stringStream ss; + ss.print("Class space size (%zu) exceeds the maximum possible size (%zu)", + len, max_klass_range_size()); + vm_exit_during_initialization(ss.base()); + } + // Remember the Klass range: _klass_range_start = addr; _klass_range_end = addr + len; - // The default version of this code tries, in order of preference: - // -unscaled (base=0 shift=0) - // -zero-based (base=0 shift>0) - // -nonzero-base (base>0 shift=0) - // Note that base>0 shift>0 should never be needed, since the klass range will - // never exceed 4GB. - constexpr uintptr_t unscaled_max = nth_bit(32); - assert(len <= unscaled_max, "Klass range larger than 32 bits?"); + // Calculate Base and Shift: - constexpr uintptr_t zerobased_max = nth_bit(32 + LogKlassAlignmentInBytes); + if (UseCompactObjectHeaders) { + + // In compact object header mode, with 22-bit narrowKlass, we don't attempt for + // zero-based mode. Instead, we set the base to the start of the klass range and + // then try for the smallest shift possible that still covers the whole range. + // The reason is that we want to avoid, if possible, shifts larger than + // a cacheline size. + _base = addr; + + const int log_cacheline = exact_log2(DEFAULT_CACHE_LINE_SIZE); + int s = max_shift(); + while (s > log_cacheline && ((size_t)nth_bit(narrow_klass_pointer_bits() + s - 1) > len)) { + s--; + } + _shift = s; - address const end = addr + len; - if (end <= (address)unscaled_max) { - _base = nullptr; - _shift = 0; } else { - if (end <= (address)zerobased_max) { + + // Traditional (non-compact) header mode + const uintptr_t unscaled_max = nth_bit(narrow_klass_pointer_bits()); + const uintptr_t zerobased_max = nth_bit(narrow_klass_pointer_bits() + max_shift()); + +#ifdef AARCH64 + // Aarch64 avoids zero-base shifted mode (_base=0 _shift>0), instead prefers + // non-zero-based mode with a zero shift. + _shift = 0; + address const end = addr + len; + _base = (end <= (address)unscaled_max) ? nullptr : addr; +#else + // We try, in order of preference: + // -unscaled (base=0 shift=0) + // -zero-based (base=0 shift>0) + // -nonzero-base (base>0 shift=0) + // Note that base>0 shift>0 should never be needed, since the klass range will + // never exceed 4GB. + address const end = addr + len; + if (end <= (address)unscaled_max) { _base = nullptr; - _shift = LogKlassAlignmentInBytes; - } else { - _base = addr; _shift = 0; + } else { + if (end <= (address)zerobased_max) { + _base = nullptr; + _shift = max_shift(); + } else { + _base = addr; + _shift = 0; + } } +#endif // AARCH64 } - DEBUG_ONLY(assert_is_valid_encoding(addr, len, _base, _shift);) + calc_lowest_highest_narrow_klass_id(); + +#ifdef ASSERT + sanity_check_after_initialization(); +#endif } -#endif // !AARCH64 || ZERO void CompressedKlassPointers::print_mode(outputStream* st) { + st->print_cr("UseCompressedClassPointers %d, UseCompactObjectHeaders %d", + UseCompressedClassPointers, UseCompactObjectHeaders); if (UseCompressedClassPointers) { + st->print_cr("Narrow klass pointer bits %d, Max shift %d", + _narrow_klass_pointer_bits, _max_shift); st->print_cr("Narrow klass base: " PTR_FORMAT ", Narrow klass shift: %d", p2i(base()), shift()); st->print_cr("Encoding Range: " RANGE2FMT, RANGE2FMTARGS(_base, encoding_range_end())); st->print_cr("Klass Range: " RANGE2FMT, RANGE2FMTARGS(_klass_range_start, _klass_range_end)); + st->print_cr("Klass ID Range: [%u - %u) (%u)", _lowest_valid_narrow_klass_id, _highest_valid_narrow_klass_id + 1, + _highest_valid_narrow_klass_id + 1 - _lowest_valid_narrow_klass_id); } else { st->print_cr("UseCompressedClassPointers off"); } diff --git a/src/hotspot/share/oops/compressedKlass.hpp b/src/hotspot/share/oops/compressedKlass.hpp index 8f89b0550ff..9e3a09d73b9 100644 --- a/src/hotspot/share/oops/compressedKlass.hpp +++ b/src/hotspot/share/oops/compressedKlass.hpp @@ -26,6 +26,7 @@ #define SHARE_OOPS_COMPRESSEDKLASS_HPP #include "memory/allStatic.hpp" +#include "utilities/align.hpp" #include "utilities/globalDefinitions.hpp" class outputStream; @@ -97,42 +98,98 @@ class Klass; // If compressed klass pointers then use narrowKlass. typedef juint narrowKlass; -const int LogKlassAlignmentInBytes = 3; -const int KlassAlignmentInBytes = 1 << LogKlassAlignmentInBytes; - -// Maximal size of compressed class space. Above this limit compression is not possible. -// Also upper bound for placement of zero based class space. (Class space is further limited -// to be < 3G, see arguments.cpp.) -const uint64_t KlassEncodingMetaspaceMax = (uint64_t(max_juint) + 1) << LogKlassAlignmentInBytes; - // For UseCompressedClassPointers. class CompressedKlassPointers : public AllStatic { friend class VMStructs; friend class ArchiveBuilder; + // We use a different narrow Klass pointer geometry depending on + // whether we run in standard mode or in compact-object-header-mode. + + // Narrow klass pointer bits for an unshifted narrow Klass pointer. + static constexpr int narrow_klass_pointer_bits_noncoh = 32; + static constexpr int narrow_klass_pointer_bits_coh = 22; + + // Bit size of a narrowKlass + static int _narrow_klass_pointer_bits; + + // The maximum shift values we can use depending on UseCompactObjectHeaders + static constexpr int max_shift_noncoh = 3; + static constexpr int max_shift_coh = 10; + + // Maximum shift usable + static int _max_shift; + + // Encoding Base, Encoding Shift static address _base; static int _shift; // Start and end of the Klass Range. - // Note: guaranteed to be aligned to KlassAlignmentInBytes + // Note: guaranteed to be aligned to 1< + static inline void check_init(T var) { + assert(var != (T)-1, "Not yet initialized"); + } - static inline narrowKlass encode_not_null(Klass* v, address base, int shift); + static inline Klass* decode_not_null_without_asserts(narrowKlass v, address base, int shift); public: + // Initialization sequence: + // 1) Parse arguments. The following arguments take a role: + // - UseCompressedClassPointers + // - UseCompactObjectHeaders + // - Xshare on off dump + // - CompressedClassSpaceSize + // 2) call pre_initialize(): depending on UseCompactObjectHeaders, defines the limits of narrow Klass pointer + // geometry (how many bits, the max. possible shift) + // 3) .. from here on, narrow_klass_pointer_bits() and max_shift() can be used + // 4) call reserve_address_space_for_compressed_classes() either from CDS initialization or, if CDS is off, + // from metaspace initialization. Reserves space for class space + CDS, attempts to reserve such that + // we later can use a "good" encoding scheme. Reservation is highly CPU-specific. + // 5) Initialize the narrow Klass encoding scheme by determining encoding base and shift: + // 5a) if CDS=on: Calls initialize_for_given_encoding() with the reservation base from step (4) and the + // CDS-intrinsic setting for shift; here, we don't have any freedom to deviate from the base. + // 5b) if CDS=off: Calls initialize() - here, we have more freedom and, if we want, can choose an encoding + // base that differs from the reservation base from step (4). That allows us, e.g., to later use + // zero-based encoding. + // 6) ... from now on, we can use base() and shift(). + + // Called right after argument parsing; defines narrow klass pointer geometry limits + static void pre_initialize(); + + // The number of bits a narrow Klass pointer has; + static int narrow_klass_pointer_bits() { check_init(_narrow_klass_pointer_bits); return _narrow_klass_pointer_bits; } + + // The maximum possible shift; the actual shift employed later can be smaller (see initialize()) + static int max_shift() { check_init(_max_shift); return _max_shift; } + + // Returns the maximum encoding range, given the current geometry (narrow klass bit size and shift) + static size_t max_encoding_range_size() { return nth_bit(narrow_klass_pointer_bits() + max_shift()); } + + // Returns the maximum allowed klass range size. + static size_t max_klass_range_size(); + // Reserve a range of memory that is to contain Klass strucutures which are referenced by narrow Klass IDs. // If optimize_for_zero_base is true, the implementation will attempt to reserve optimized for zero-based encoding. static char* reserve_address_space_for_compressed_classes(size_t size, bool aslr, bool optimize_for_zero_base); @@ -152,33 +209,55 @@ class CompressedKlassPointers : public AllStatic { static void print_mode(outputStream* st); - static address base() { return _base; } - static int shift() { return _shift; } + // Can only be used after initialization + static address base() { check_init(_base); return _base; } + static int shift() { check_init(_shift); return _shift; } static address klass_range_start() { return _klass_range_start; } static address klass_range_end() { return _klass_range_end; } static inline address encoding_range_end(); + // Returns the alignment a Klass* is guaranteed to have. + // Note: *Not* the same as 1 << shift ! Klass are always guaranteed to be at least 64-bit aligned, + // so this will return 8 even if shift is 0. + static int klass_alignment_in_bytes() { return nth_bit(MAX2(3, _shift)); } + static int klass_alignment_in_words() { return klass_alignment_in_bytes() / BytesPerWord; } + + // Returns the highest possible narrowKlass value given the current Klass range + static narrowKlass highest_valid_narrow_klass_id() { return _highest_valid_narrow_klass_id; } + static bool is_null(Klass* v) { return v == nullptr; } static bool is_null(narrowKlass v) { return v == 0; } // Versions without asserts - static inline Klass* decode_not_null_without_asserts(narrowKlass v); static inline Klass* decode_without_asserts(narrowKlass v); - static inline Klass* decode_not_null(narrowKlass v); static inline Klass* decode(narrowKlass v); + static inline narrowKlass encode_not_null_without_asserts(Klass* k, address narrow_base, int shift); static inline narrowKlass encode_not_null(Klass* v); static inline narrowKlass encode(Klass* v); +#ifdef ASSERT + // Given an address, check that it can be encoded with the current encoding + inline static void check_encodable(const void* addr); + // Given a narrow Klass ID, check that it is valid according to current encoding + inline static void check_valid_narrow_klass_id(narrowKlass nk); +#endif + // Returns whether the pointer is in the memory region used for encoding compressed // class pointers. This includes CDS. - static inline bool is_encodable(const void* p) { - return (address) p >= _klass_range_start && - (address) p < _klass_range_end; + static inline bool is_encodable(const void* addr) { + // An address can only be encoded if: + // + // 1) the address lies within the klass range. + // 2) It is suitably aligned to 2^encoding_shift. This only really matters for + // +UseCompactObjectHeaders, since the encoding shift can be large (max 10 bits -> 1KB). + return (address)addr >= _klass_range_start && (address)addr < _klass_range_end && + is_aligned(addr, klass_alignment_in_bytes()); } + }; #endif // SHARE_OOPS_COMPRESSEDKLASS_HPP diff --git a/src/hotspot/share/oops/compressedKlass.inline.hpp b/src/hotspot/share/oops/compressedKlass.inline.hpp index c9c9af24ad8..7c5da48a494 100644 --- a/src/hotspot/share/oops/compressedKlass.inline.hpp +++ b/src/hotspot/share/oops/compressedKlass.inline.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2017, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -32,42 +32,24 @@ #include "utilities/align.hpp" #include "utilities/globalDefinitions.hpp" -static inline bool check_alignment(Klass* v) { - return (intptr_t)v % KlassAlignmentInBytes == 0; +inline Klass* CompressedKlassPointers::decode_not_null_without_asserts(narrowKlass v, address narrow_base_base, int shift) { + return (Klass*)((uintptr_t)narrow_base_base +((uintptr_t)v << shift)); } -inline Klass* CompressedKlassPointers::decode_not_null_without_asserts(narrowKlass v, address narrow_base, int shift) { - return (Klass*)((uintptr_t)narrow_base +((uintptr_t)v << shift)); -} - -inline Klass* CompressedKlassPointers::decode_not_null(narrowKlass v, address narrow_base, int shift) { - assert(!is_null(v), "narrow klass value can never be zero"); - Klass* result = decode_not_null_without_asserts(v, narrow_base, shift); - assert(check_alignment(result), "address not aligned: " PTR_FORMAT, p2i(result)); - return result; -} - -inline narrowKlass CompressedKlassPointers::encode_not_null(Klass* v, address narrow_base, int shift) { - assert(!is_null(v), "klass value can never be zero"); - assert(check_alignment(v), "Address not aligned"); - uint64_t pd = (uint64_t)(pointer_delta(v, narrow_base, 1)); - assert(KlassEncodingMetaspaceMax > pd, "change encoding max if new encoding (Klass " PTR_FORMAT ", Base " PTR_FORMAT ")", p2i(v), p2i(narrow_base)); - uint64_t result = pd >> shift; - assert((result & CONST64(0xffffffff00000000)) == 0, "narrow klass pointer overflow"); - assert(decode_not_null((narrowKlass)result, narrow_base, shift) == v, "reversibility"); - return (narrowKlass)result; -} - -inline Klass* CompressedKlassPointers::decode_not_null_without_asserts(narrowKlass v) { - return decode_not_null_without_asserts(v, base(), shift()); +inline narrowKlass CompressedKlassPointers::encode_not_null_without_asserts(Klass* k, address narrow_base, int shift) { + return (narrowKlass)(pointer_delta(k, narrow_base, 1) >> shift); } inline Klass* CompressedKlassPointers::decode_without_asserts(narrowKlass v) { - return is_null(v) ? nullptr : decode_not_null_without_asserts(v); + return is_null(v) ? nullptr : decode_not_null_without_asserts(v, base(), shift()); } inline Klass* CompressedKlassPointers::decode_not_null(narrowKlass v) { - return decode_not_null(v, base(), shift()); + assert(!is_null(v), "narrow klass value can never be zero"); + DEBUG_ONLY(check_valid_narrow_klass_id(v);) + Klass* const k = decode_not_null_without_asserts(v, base(), shift()); + DEBUG_ONLY(check_encodable(k)); + return k; } inline Klass* CompressedKlassPointers::decode(narrowKlass v) { @@ -75,15 +57,40 @@ inline Klass* CompressedKlassPointers::decode(narrowKlass v) { } inline narrowKlass CompressedKlassPointers::encode_not_null(Klass* v) { - return encode_not_null(v, base(), shift()); + assert(!is_null(v), "klass value can never be zero"); + DEBUG_ONLY(check_encodable(v);) + const narrowKlass nk = encode_not_null_without_asserts(v, base(), shift()); + assert(decode_not_null_without_asserts(nk, base(), shift()) == v, "reversibility"); + DEBUG_ONLY(check_valid_narrow_klass_id(nk);) + return nk; } inline narrowKlass CompressedKlassPointers::encode(Klass* v) { return is_null(v) ? (narrowKlass)0 : encode_not_null(v); } +#ifdef ASSERT +inline void CompressedKlassPointers::check_encodable(const void* addr) { + assert(UseCompressedClassPointers, "Only call for +UseCCP"); + assert(addr != nullptr, "Null Klass?"); + assert(is_encodable(addr), + "Address " PTR_FORMAT " is not encodable (Klass range: " RANGEFMT ", klass alignment: %d)", + p2i(addr), RANGE2FMTARGS(_klass_range_start, _klass_range_end), klass_alignment_in_bytes()); +} + +inline void CompressedKlassPointers::check_valid_narrow_klass_id(narrowKlass nk) { + check_init(_base); + assert(UseCompressedClassPointers, "Only call for +UseCCP"); + assert(nk > 0, "narrow Klass ID is 0"); + const uint64_t nk_mask = ~right_n_bits(narrow_klass_pointer_bits()); + assert(((uint64_t)nk & nk_mask) == 0, "narrow klass id bit spillover (%u)", nk); + assert(nk >= _lowest_valid_narrow_klass_id && + nk <= _highest_valid_narrow_klass_id, "narrowKlass ID out of range (%u)", nk); +} +#endif // ASSERT + inline address CompressedKlassPointers::encoding_range_end() { - const int max_bits = (sizeof(narrowKlass) * BitsPerByte) + _shift; // narrowKlass are 32 bit + const int max_bits = narrow_klass_pointer_bits() + _shift; return _base + nth_bit(max_bits); } diff --git a/src/hotspot/share/oops/constantPool.cpp b/src/hotspot/share/oops/constantPool.cpp index 1c295e10b21..7147e2588c9 100644 --- a/src/hotspot/share/oops/constantPool.cpp +++ b/src/hotspot/share/oops/constantPool.cpp @@ -477,7 +477,7 @@ void ConstantPool::remove_resolved_klass_if_non_deterministic(int cp_index) { if (k == nullptr) { // We'd come here if the referenced class has been excluded via // SystemDictionaryShared::is_excluded_class(). As a result, ArchiveBuilder - // has cleared the resolved_klasses()->at(...) pointer to NULL. Thus, we + // has cleared the resolved_klasses()->at(...) pointer to null. Thus, we // need to revert the tag to JVM_CONSTANT_UnresolvedClass. can_archive = false; } else { diff --git a/src/hotspot/share/oops/instanceKlass.cpp b/src/hotspot/share/oops/instanceKlass.cpp index e7cc0a00766..a9024bf4a9f 100644 --- a/src/hotspot/share/oops/instanceKlass.cpp +++ b/src/hotspot/share/oops/instanceKlass.cpp @@ -455,7 +455,7 @@ InstanceKlass* InstanceKlass::allocate_instance_klass(const ClassFileParser& par assert(loader_data != nullptr, "invariant"); InstanceKlass* ik; - const bool use_class_space = !parser.is_interface() && !parser.is_abstract(); + const bool use_class_space = parser.klass_needs_narrow_id(); // Allocation if (parser.is_instance_ref_klass()) { @@ -475,6 +475,11 @@ InstanceKlass* InstanceKlass::allocate_instance_klass(const ClassFileParser& par ik = new (loader_data, size, use_class_space, THREAD) InstanceKlass(parser); } + if (ik != nullptr && UseCompressedClassPointers && use_class_space) { + assert(CompressedKlassPointers::is_encodable(ik), + "Klass " PTR_FORMAT "needs a narrow Klass ID, but is not encodable", p2i(ik)); + } + // Check for pending exception before adding to the loader data and incrementing // class count. Can get OOM here. if (HAS_PENDING_EXCEPTION) { @@ -1595,6 +1600,7 @@ void InstanceKlass::call_class_initializer(TRAPS) { THREAD->name()); } if (h_method() != nullptr) { + ThreadInClassInitializer ticl(THREAD, this); // Track class being initialized JavaCallArguments args; // No arguments JavaValue result(T_VOID); JavaCalls::call(&result, h_method, &args, CHECK); // Static call (no args) @@ -2639,10 +2645,15 @@ void InstanceKlass::restore_unshareable_info(ClassLoaderData* loader_data, Handl // have been redefined. bool trace_name_printed = false; adjust_default_methods(&trace_name_printed); - vtable().initialize_vtable(); - itable().initialize_itable(); + if (verified_at_dump_time()) { + // Initialize vtable and itable for classes which can be verified at dump time. + // Unlinked classes such as old classes with major version < 50 cannot be verified + // at dump time. + vtable().initialize_vtable(); + itable().initialize_itable(); + } } -#endif +#endif // INCLUDE_JVMTI // restore constant pool resolved references constants()->restore_unshareable_info(CHECK); diff --git a/src/hotspot/share/oops/instanceOop.hpp b/src/hotspot/share/oops/instanceOop.hpp index 8de3b1a742c..e97cd00f79f 100644 --- a/src/hotspot/share/oops/instanceOop.hpp +++ b/src/hotspot/share/oops/instanceOop.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -32,17 +32,6 @@ // Evaluating "new HashTable()" will create an instanceOop. class instanceOopDesc : public oopDesc { - public: - // aligned header size. - static int header_size() { return sizeof(instanceOopDesc)/HeapWordSize; } - - // If compressed, the offset of the fields of the instance may not be aligned. - static int base_offset_in_bytes() { - return (UseCompressedClassPointers) ? - klass_gap_offset_in_bytes() : - sizeof(instanceOopDesc); - - } }; // See similar requirement for oopDesc. diff --git a/src/hotspot/share/oops/instanceStackChunkKlass.cpp b/src/hotspot/share/oops/instanceStackChunkKlass.cpp index a2227b0a76a..2b819b27f64 100644 --- a/src/hotspot/share/oops/instanceStackChunkKlass.cpp +++ b/src/hotspot/share/oops/instanceStackChunkKlass.cpp @@ -163,6 +163,23 @@ void InstanceStackChunkKlass::oop_oop_iterate_stack_slow(stackChunkOop chunk, Oo chunk->iterate_stack(&frame_closure); } +template +void InstanceStackChunkKlass::oop_oop_iterate_lockstack(stackChunkOop chunk, OopIterateClosure* closure, MemRegion mr) { + if (LockingMode != LM_LIGHTWEIGHT) { + return; + } + + StackChunkOopIterateFilterClosure cl(closure, mr); + if (chunk->has_bitmap()) { + chunk->iterate_lockstack(&cl); + } else { + chunk->iterate_lockstack(&cl); + } +} + +template void InstanceStackChunkKlass::oop_oop_iterate_lockstack(stackChunkOop chunk, OopIterateClosure* closure, MemRegion mr); +template void InstanceStackChunkKlass::oop_oop_iterate_lockstack(stackChunkOop chunk, OopIterateClosure* closure, MemRegion mr); + #ifdef ASSERT class DescribeStackChunkClosure { @@ -224,7 +241,7 @@ class PrintStackChunkClosure { frame f = fs.to_frame(); _st->print_cr("-- frame sp: " PTR_FORMAT " interpreted: %d size: %d argsize: %d", p2i(fs.sp()), fs.is_interpreted(), f.frame_size(), - fs.is_interpreted() ? 0 : f.compiled_frame_stack_argsize()); + fs.is_interpreted() || fs.is_stub() ? 0 : f.compiled_frame_stack_argsize()); #ifdef ASSERT f.print_value_on(_st); #else diff --git a/src/hotspot/share/oops/instanceStackChunkKlass.hpp b/src/hotspot/share/oops/instanceStackChunkKlass.hpp index aa59b46f42c..19ce37c4ddd 100644 --- a/src/hotspot/share/oops/instanceStackChunkKlass.hpp +++ b/src/hotspot/share/oops/instanceStackChunkKlass.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -177,6 +177,9 @@ class InstanceStackChunkKlass: public InstanceKlass { template inline void oop_oop_iterate_stack_with_bitmap(stackChunkOop chunk, OopClosureType* closure, intptr_t* start, intptr_t* end); + template + void oop_oop_iterate_lockstack(stackChunkOop chunk, OopIterateClosure* closure, MemRegion mr); + void do_methods(stackChunkOop chunk, OopIterateClosure* cl); void oop_oop_iterate_stack_slow(stackChunkOop chunk, OopIterateClosure* closure, MemRegion mr); diff --git a/src/hotspot/share/oops/instanceStackChunkKlass.inline.hpp b/src/hotspot/share/oops/instanceStackChunkKlass.inline.hpp index 38aab2c78a7..3374895c6a1 100644 --- a/src/hotspot/share/oops/instanceStackChunkKlass.inline.hpp +++ b/src/hotspot/share/oops/instanceStackChunkKlass.inline.hpp @@ -1,4 +1,4 @@ -/* Copyright (c) 2019, 2022, Oracle and/or its affiliates. All rights reserved. +/* Copyright (c) 2019, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -65,6 +65,7 @@ void InstanceStackChunkKlass::oop_oop_iterate(oop obj, OopClosureType* closure) } oop_oop_iterate_stack(chunk, closure); oop_oop_iterate_header(chunk, closure); + oop_oop_iterate_lockstack(chunk, closure, chunk->range()); } template @@ -73,6 +74,7 @@ void InstanceStackChunkKlass::oop_oop_iterate_reverse(oop obj, OopClosureType* c stackChunkOop chunk = stackChunkOopDesc::cast(obj); oop_oop_iterate_stack(chunk, closure); oop_oop_iterate_header(chunk, closure); + oop_oop_iterate_lockstack(chunk, closure, chunk->range()); } template @@ -85,6 +87,7 @@ void InstanceStackChunkKlass::oop_oop_iterate_bounded(oop obj, OopClosureType* c } oop_oop_iterate_stack_bounded(chunk, closure, mr); oop_oop_iterate_header_bounded(chunk, closure, mr); + oop_oop_iterate_lockstack(chunk, closure, mr); } template diff --git a/src/hotspot/share/oops/klass.cpp b/src/hotspot/share/oops/klass.cpp index b87f341ce48..884816764a0 100644 --- a/src/hotspot/share/oops/klass.cpp +++ b/src/hotspot/share/oops/klass.cpp @@ -43,6 +43,7 @@ #include "memory/oopFactory.hpp" #include "memory/resourceArea.hpp" #include "memory/universe.hpp" +#include "oops/compressedKlass.inline.hpp" #include "oops/compressedOops.inline.hpp" #include "oops/instanceKlass.hpp" #include "oops/klass.inline.hpp" @@ -274,6 +275,23 @@ Method* Klass::uncached_lookup_method(const Symbol* name, const Symbol* signatur return nullptr; } +static markWord make_prototype(const Klass* kls) { + markWord prototype = markWord::prototype(); +#ifdef _LP64 + if (UseCompactObjectHeaders) { + // With compact object headers, the narrow Klass ID is part of the mark word. + // We therfore seed the mark word with the narrow Klass ID. + // Note that only those Klass that can be instantiated have a narrow Klass ID. + // For those who don't, we leave the klass bits empty and assert if someone + // tries to use those. + const narrowKlass nk = CompressedKlassPointers::is_encodable(kls) ? + CompressedKlassPointers::encode(const_cast(kls)) : 0; + prototype = prototype.set_narrow_klass(nk); + } +#endif + return prototype; +} + Klass::Klass() : _kind(UnknownKlassKind) { assert(CDSConfig::is_dumping_static_archive() || CDSConfig::is_using_archive(), "only for cds"); } @@ -283,6 +301,7 @@ Klass::Klass() : _kind(UnknownKlassKind) { // The constructor is also used from CppVtableCloner, // which doesn't zero out the memory before calling the constructor. Klass::Klass(KlassKind kind) : _kind(kind), + _prototype_header(make_prototype(this)), _shared_class_path_index(-1) { CDS_ONLY(_shared_class_flags = 0;) CDS_JAVA_HEAP_ONLY(_archived_mirror_index = -1;) @@ -985,6 +1004,10 @@ void Klass::oop_print_on(oop obj, outputStream* st) { // print header obj->mark().print_on(st); st->cr(); + if (UseCompactObjectHeaders) { + st->print(BULLET"prototype_header: " INTPTR_FORMAT, _prototype_header.value()); + st->cr(); + } } // print class @@ -1007,7 +1030,14 @@ void Klass::verify_on(outputStream* st) { // This can be expensive, but it is worth checking that this klass is actually // in the CLD graph but not in production. - assert(Metaspace::contains((address)this), "Should be"); +#ifdef ASSERT + if (UseCompressedClassPointers && needs_narrow_id()) { + // Stricter checks for both correct alignment and placement + CompressedKlassPointers::check_encodable(this); + } else { + assert(Metaspace::contains((address)this), "Should be"); + } +#endif // ASSERT guarantee(this->is_klass(),"should be klass"); @@ -1035,6 +1065,8 @@ void Klass::oop_verify_on(oop obj, outputStream* st) { guarantee(obj->klass()->is_klass(), "klass field is not a klass"); } +// Note: this function is called with an address that may or may not be a Klass. +// The point is not to assert it is but to check if it could be. bool Klass::is_valid(Klass* k) { if (!is_aligned(k, sizeof(MetaWord))) return false; if ((size_t)k < os::min_page_size()) return false; diff --git a/src/hotspot/share/oops/klass.hpp b/src/hotspot/share/oops/klass.hpp index 4fc670d85f1..2629be64bea 100644 --- a/src/hotspot/share/oops/klass.hpp +++ b/src/hotspot/share/oops/klass.hpp @@ -165,6 +165,8 @@ class Klass : public Metadata { uintx _secondary_supers_bitmap; uint8_t _hash_slot; + markWord _prototype_header; // Used to initialize objects' header + int _vtable_len; // vtable length. This field may be read very often when we // have lots of itable dispatches (e.g., lambdas and streams). // Keep it away from the beginning of a Klass to avoid cacheline @@ -581,6 +583,8 @@ class Klass : public Metadata { inline oop klass_holder() const; + inline void keep_alive() const; + protected: // Error handling when length > max_length or length < 0 @@ -705,6 +709,10 @@ class Klass : public Metadata { bool is_cloneable() const; void set_is_cloneable(); + inline markWord prototype_header() const; + inline void set_prototype_header(markWord header); + static ByteSize prototype_header_offset() { return in_ByteSize(offset_of(Klass, _prototype_header)); } + JFR_ONLY(DEFINE_TRACE_ID_METHODS;) virtual void metaspace_pointers_do(MetaspaceClosure* iter); @@ -759,6 +767,10 @@ class Klass : public Metadata { static bool is_valid(Klass* k); static void on_secondary_supers_verification_failure(Klass* super, Klass* sub, bool linear_result, bool table_result, const char* msg); + + // Returns true if this Klass needs to be addressable via narrow Klass ID. + inline bool needs_narrow_id() const; + }; #endif // SHARE_OOPS_KLASS_HPP diff --git a/src/hotspot/share/oops/klass.inline.hpp b/src/hotspot/share/oops/klass.inline.hpp index ea9af6d6928..4f9401f1709 100644 --- a/src/hotspot/share/oops/klass.inline.hpp +++ b/src/hotspot/share/oops/klass.inline.hpp @@ -37,6 +37,13 @@ inline oop Klass::klass_holder() const { return class_loader_data()->holder(); } +inline void Klass::keep_alive() const { + // Resolving the holder (a WeakHandle) will keep the klass alive until the next safepoint. + // Making the klass's CLD handle oops (e.g. the java_mirror), safe to store in the object + // graph and its roots (e.g. Handles). + static_cast(klass_holder()); +} + inline bool Klass::is_non_strong_hidden() const { return is_hidden() && class_loader_data()->has_class_mirror_holder(); } @@ -52,6 +59,25 @@ inline bool Klass::is_loader_alive() const { return class_loader_data()->is_alive(); } +inline markWord Klass::prototype_header() const { + assert(UseCompactObjectHeaders, "only use with compact object headers"); +#ifdef _LP64 + // You only need prototypes for allocating objects. If the class is not instantiable, it won't live in + // class space and have no narrow Klass ID. But in that case we should not need the prototype. + assert(_prototype_header.narrow_klass() > 0, "Klass " PTR_FORMAT ": invalid prototype (" PTR_FORMAT ")", + p2i(this), _prototype_header.value()); +#endif + return _prototype_header; +} + +// This is only used when dumping the archive. In other cases, +// the _prototype_header is already initialized to the right thing. +inline void Klass::set_prototype_header(markWord header) { + assert(UseCompactObjectHeaders, "only with compact headers"); + _prototype_header = header; +} + +// Loading the java_mirror does not keep its holder alive. See Klass::keep_alive(). inline oop Klass::java_mirror() const { return _java_mirror.resolve(); } @@ -144,4 +170,13 @@ inline bool Klass::search_secondary_supers(Klass *k) const { return result; } +// Returns true if this Klass needs to be addressable via narrow Klass ID. +inline bool Klass::needs_narrow_id() const { + // Classes that are never instantiated need no narrow Klass Id, since the + // only point of having a narrow id is to put it into an object header. Keeping + // never instantiated classes out of class space lessens the class space pressure. + // For more details, see JDK-8338526. + // Note: don't call this function before access flags are initialized. + return !is_abstract() && !is_interface(); +} #endif // SHARE_OOPS_KLASS_INLINE_HPP diff --git a/src/hotspot/share/oops/markWord.cpp b/src/hotspot/share/oops/markWord.cpp index 2bbec570fa8..a9b1a7b026a 100644 --- a/src/hotspot/share/oops/markWord.cpp +++ b/src/hotspot/share/oops/markWord.cpp @@ -29,6 +29,12 @@ #include "runtime/objectMonitor.inline.hpp" #include "utilities/ostream.hpp" +#ifdef _LP64 +STATIC_ASSERT(markWord::klass_shift + markWord::klass_bits == 64); +// The hash (preceding klass bits) shall be a direct neighbor but not interleave +STATIC_ASSERT(markWord::klass_shift == markWord::hash_bits + markWord::hash_shift); +#endif + markWord markWord::displaced_mark_helper() const { assert(has_displaced_mark_helper(), "check"); if (has_monitor()) { diff --git a/src/hotspot/share/oops/markWord.hpp b/src/hotspot/share/oops/markWord.hpp index bbd80d02cbd..7d2bff1efc0 100644 --- a/src/hotspot/share/oops/markWord.hpp +++ b/src/hotspot/share/oops/markWord.hpp @@ -26,6 +26,7 @@ #define SHARE_OOPS_MARKWORD_HPP #include "metaprogramming/primitiveConversions.hpp" +#include "oops/compressedKlass.hpp" #include "oops/oopsHierarchy.hpp" #include "runtime/globals.hpp" @@ -37,11 +38,15 @@ // // 32 bits: // -------- -// hash:25 ------------>| age:4 unused_gap:1 lock:2 (normal object) +// hash:25 ------------>| age:4 self-fwd:1 lock:2 (normal object) // // 64 bits: // -------- -// unused:25 hash:31 -->| unused_gap:1 age:4 unused_gap:1 lock:2 (normal object) +// unused:22 hash:31 -->| unused_gap:4 age:4 self-fwd:1 lock:2 (normal object) +// +// 64 bits (with compact headers): +// ------------------------------- +// klass:22 hash:31 -->| unused_gap:4 age:4 self-fwd:1 lock:2 (normal object) // // - hash contains the identity hash value: largest value is // 31 bits, see os::random(). Also, 64-bit vm's require @@ -104,22 +109,37 @@ class markWord { // Constants static const int age_bits = 4; static const int lock_bits = 2; - static const int first_unused_gap_bits = 1; - static const int max_hash_bits = BitsPerWord - age_bits - lock_bits - first_unused_gap_bits; + static const int self_fwd_bits = 1; + static const int max_hash_bits = BitsPerWord - age_bits - lock_bits - self_fwd_bits; static const int hash_bits = max_hash_bits > 31 ? 31 : max_hash_bits; - static const int second_unused_gap_bits = LP64_ONLY(1) NOT_LP64(0); + static const int unused_gap_bits = LP64_ONLY(4) NOT_LP64(0); // Reserved for Valhalla. static const int lock_shift = 0; - static const int age_shift = lock_bits + first_unused_gap_bits; - static const int hash_shift = age_shift + age_bits + second_unused_gap_bits; + static const int self_fwd_shift = lock_shift + lock_bits; + static const int age_shift = self_fwd_shift + self_fwd_bits; + static const int hash_shift = age_shift + age_bits + unused_gap_bits; static const uintptr_t lock_mask = right_n_bits(lock_bits); static const uintptr_t lock_mask_in_place = lock_mask << lock_shift; + static const uintptr_t self_fwd_mask = right_n_bits(self_fwd_bits); + static const uintptr_t self_fwd_mask_in_place = self_fwd_mask << self_fwd_shift; static const uintptr_t age_mask = right_n_bits(age_bits); static const uintptr_t age_mask_in_place = age_mask << age_shift; static const uintptr_t hash_mask = right_n_bits(hash_bits); static const uintptr_t hash_mask_in_place = hash_mask << hash_shift; +#ifdef _LP64 + // Used only with compact headers: + // We store the (narrow) Klass* in the bits 43 to 64. + + // These are for bit-precise extraction of the narrow Klass* from the 64-bit Markword + static constexpr int klass_shift = hash_shift + hash_bits; + static constexpr int klass_bits = 22; + static constexpr uintptr_t klass_mask = right_n_bits(klass_bits); + static constexpr uintptr_t klass_mask_in_place = klass_mask << klass_shift; +#endif + + static const uintptr_t locked_value = 0; static const uintptr_t unlocked_value = 1; static const uintptr_t monitor_value = 2; @@ -144,8 +164,9 @@ class markWord { bool is_marked() const { return (mask_bits(value(), lock_mask_in_place) == marked_value); } - bool is_forwarded() const { - return (mask_bits(value(), lock_mask_in_place) == marked_value); + bool is_forwarded() const { + // Returns true for normal forwarded (0b011) and self-forwarded (0b1xx). + return mask_bits(value(), lock_mask_in_place | self_fwd_mask_in_place) >= static_cast(marked_value); } bool is_neutral() const { // Not locked, or marked - a "clean" neutral state return (mask_bits(value(), lock_mask_in_place) == unlocked_value); @@ -260,6 +281,12 @@ class markWord { return hash() == no_hash; } + inline Klass* klass() const; + inline Klass* klass_or_null() const; + inline Klass* klass_without_asserts() const; + inline narrowKlass narrow_klass() const; + inline markWord set_narrow_klass(narrowKlass narrow_klass) const; + // Prototype mark for initialization static markWord prototype() { return markWord( no_hash_in_place | no_lock_in_place ); @@ -274,6 +301,21 @@ class markWord { // Recover address of oop from encoded form used in mark inline void* decode_pointer() const { return (void*)clear_lock_bits().value(); } + inline bool is_self_forwarded() const { + NOT_LP64(assert(LockingMode != LM_LEGACY, "incorrect with LM_LEGACY on 32 bit");) + return mask_bits(value(), self_fwd_mask_in_place) != 0; + } + + inline markWord set_self_forwarded() const { + NOT_LP64(assert(LockingMode != LM_LEGACY, "incorrect with LM_LEGACY on 32 bit");) + return markWord(value() | self_fwd_mask_in_place); + } + + inline markWord unset_self_forwarded() const { + NOT_LP64(assert(LockingMode != LM_LEGACY, "incorrect with LM_LEGACY on 32 bit");) + return markWord(value() & ~self_fwd_mask_in_place); + } + inline oop forwardee() const { return cast_to_oop(decode_pointer()); } diff --git a/src/hotspot/share/oops/markWord.inline.hpp b/src/hotspot/share/oops/markWord.inline.hpp new file mode 100644 index 00000000000..27c8cfdeaef --- /dev/null +++ b/src/hotspot/share/oops/markWord.inline.hpp @@ -0,0 +1,81 @@ +/* + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_OOPS_MARKWORD_INLINE_HPP +#define SHARE_OOPS_MARKWORD_INLINE_HPP + +#include "oops/compressedOops.inline.hpp" +#include "oops/markWord.hpp" + +narrowKlass markWord::narrow_klass() const { +#ifdef _LP64 + assert(UseCompactObjectHeaders, "only used with compact object headers"); + return narrowKlass(value() >> klass_shift); +#else + ShouldNotReachHere(); + return 0; +#endif +} + +markWord markWord::set_narrow_klass(narrowKlass narrow_klass) const { +#ifdef _LP64 + assert(UseCompactObjectHeaders, "only used with compact object headers"); + return markWord((value() & ~klass_mask_in_place) | ((uintptr_t) narrow_klass << klass_shift)); +#else + ShouldNotReachHere(); + return markWord(0); +#endif +} + +Klass* markWord::klass() const { +#ifdef _LP64 + assert(UseCompactObjectHeaders, "only used with compact object headers"); + return CompressedKlassPointers::decode_not_null(narrow_klass()); +#else + ShouldNotReachHere(); + return nullptr; +#endif +} + +Klass* markWord::klass_or_null() const { +#ifdef _LP64 + assert(UseCompactObjectHeaders, "only used with compact object headers"); + return CompressedKlassPointers::decode(narrow_klass()); +#else + ShouldNotReachHere(); + return nullptr; +#endif +} + +Klass* markWord::klass_without_asserts() const { +#ifdef _LP64 + assert(UseCompactObjectHeaders, "only used with compact object headers"); + return CompressedKlassPointers::decode_without_asserts(narrow_klass()); +#else + ShouldNotReachHere(); + return nullptr; +#endif +} + +#endif // SHARE_OOPS_MARKWORD_INLINE_HPP diff --git a/src/hotspot/share/oops/method.cpp b/src/hotspot/share/oops/method.cpp index a1b380d3646..26a047449ed 100644 --- a/src/hotspot/share/oops/method.cpp +++ b/src/hotspot/share/oops/method.cpp @@ -867,6 +867,11 @@ bool Method::needs_clinit_barrier() const { return is_static() && !method_holder()->is_initialized(); } +bool Method::is_object_wait0() const { + return klass_name() == vmSymbols::java_lang_Object() + && name() == vmSymbols::wait_name(); +} + objArrayHandle Method::resolved_checked_exceptions_impl(Method* method, TRAPS) { int length = method->checked_exceptions_length(); if (length == 0) { // common case diff --git a/src/hotspot/share/oops/method.hpp b/src/hotspot/share/oops/method.hpp index d42089d3a5c..232e5fb577e 100644 --- a/src/hotspot/share/oops/method.hpp +++ b/src/hotspot/share/oops/method.hpp @@ -586,6 +586,9 @@ class Method : public Metadata { // returns true if the method name is bool is_object_initializer() const; + // returns true if the method name is wait0 + bool is_object_wait0() const; + // compiled code support // NOTE: code() is inherently racy as deopt can be clearing code // simultaneously. Use with caution. diff --git a/src/hotspot/share/oops/methodData.cpp b/src/hotspot/share/oops/methodData.cpp index af19d59f281..bc2649ba0ed 100644 --- a/src/hotspot/share/oops/methodData.cpp +++ b/src/hotspot/share/oops/methodData.cpp @@ -59,9 +59,14 @@ bool DataLayout::needs_array_len(u1 tag) { // Perform generic initialization of the data. More specific // initialization occurs in overrides of ProfileData::post_initialize. void DataLayout::initialize(u1 tag, u2 bci, int cell_count) { - _header._bits = (intptr_t)0; - _header._struct._tag = tag; - _header._struct._bci = bci; + DataLayout temp; + temp._header._bits = (intptr_t)0; + temp._header._struct._tag = tag; + temp._header._struct._bci = bci; + // Write the header using a single intptr_t write. This ensures that if the layout is + // reinitialized readers will never see the transient state where the header is 0. + _header = temp._header; + for (int i = 0; i < cell_count; i++) { set_cell_at(i, (intptr_t)0); } @@ -1224,6 +1229,28 @@ MethodData::MethodData(const methodHandle& method) initialize(); } +// Reinitialize the storage of an existing MDO at a safepoint. Doing it this way will ensure it's +// not being accessed while the contents are being rewritten. +class VM_ReinitializeMDO: public VM_Operation { + private: + MethodData* _mdo; + public: + VM_ReinitializeMDO(MethodData* mdo): _mdo(mdo) {} + VMOp_Type type() const { return VMOp_ReinitializeMDO; } + void doit() { + // The extra data is being zero'd, we'd like to acquire the extra_data_lock but it can't be held + // over a safepoint. This means that we don't actually need to acquire the lock. + _mdo->initialize(); + } + bool allow_nested_vm_operations() const { return true; } +}; + +void MethodData::reinitialize() { + VM_ReinitializeMDO op(this); + VMThread::execute(&op); +} + + void MethodData::initialize() { Thread* thread = Thread::current(); NoSafepointVerifier no_safepoint; // init function atomic wrt GC diff --git a/src/hotspot/share/oops/methodData.hpp b/src/hotspot/share/oops/methodData.hpp index 5071c29f1d7..36fcf8ce5fd 100644 --- a/src/hotspot/share/oops/methodData.hpp +++ b/src/hotspot/share/oops/methodData.hpp @@ -1949,6 +1949,7 @@ class MethodData : public Metadata { friend class ProfileData; friend class TypeEntriesAtCall; friend class ciMethodData; + friend class VM_ReinitializeMDO; // If you add a new field that points to any metaspace object, you // must add this field to MethodData::metaspace_pointers_do(). @@ -1965,11 +1966,18 @@ class MethodData : public Metadata { Mutex _extra_data_lock; MethodData(const methodHandle& method); + + void initialize(); + public: static MethodData* allocate(ClassLoaderData* loader_data, const methodHandle& method, TRAPS); virtual bool is_methodData() const { return true; } - void initialize(); + + // Safely reinitialize the data in the MDO. This is intended as a testing facility as the + // reinitialization is performed at a safepoint so it's isn't cheap and it doesn't ensure that all + // readers will see consistent profile data. + void reinitialize(); // Whole-method sticky bits and flags enum { diff --git a/src/hotspot/share/oops/objArrayKlass.cpp b/src/hotspot/share/oops/objArrayKlass.cpp index bee010b6d72..0697901d174 100644 --- a/src/hotspot/share/oops/objArrayKlass.cpp +++ b/src/hotspot/share/oops/objArrayKlass.cpp @@ -143,7 +143,10 @@ ObjArrayKlass::ObjArrayKlass(int n, Klass* element_klass, Symbol* name) : ArrayK } size_t ObjArrayKlass::oop_size(oop obj) const { - assert(obj->is_objArray(), "must be object array"); + // In this assert, we cannot safely access the Klass* with compact headers, + // because size_given_klass() calls oop_size() on objects that might be + // concurrently forwarded, which would overwrite the Klass*. + assert(UseCompactObjectHeaders || obj->is_objArray(), "must be object array"); return objArrayOop(obj)->object_size(); } diff --git a/src/hotspot/share/oops/objArrayKlass.inline.hpp b/src/hotspot/share/oops/objArrayKlass.inline.hpp index 6c9165509c7..a92c42d21a8 100644 --- a/src/hotspot/share/oops/objArrayKlass.inline.hpp +++ b/src/hotspot/share/oops/objArrayKlass.inline.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2010, 2022, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2010, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -70,7 +70,7 @@ void ObjArrayKlass::oop_oop_iterate_elements_bounded( template void ObjArrayKlass::oop_oop_iterate(oop obj, OopClosureType* closure) { - assert (obj->is_array(), "obj must be array"); + assert(obj->is_array(), "obj must be array"); objArrayOop a = objArrayOop(obj); if (Devirtualizer::do_metadata(closure)) { diff --git a/src/hotspot/share/oops/oop.cpp b/src/hotspot/share/oops/oop.cpp index acb47d4c7cf..9385379a617 100644 --- a/src/hotspot/share/oops/oop.cpp +++ b/src/hotspot/share/oops/oop.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -153,8 +153,7 @@ bool oopDesc::is_objArray_noinline() const { return is_objArray(); } bool oopDesc::is_typeArray_noinline() const { return is_typeArray(); } bool oopDesc::has_klass_gap() { - // Only has a klass gap when compressed class pointers are used. - return UseCompressedClassPointers; + return UseCompressedClassPointers && !UseCompactObjectHeaders; } #if INCLUDE_CDS_JAVA_HEAP diff --git a/src/hotspot/share/oops/oop.hpp b/src/hotspot/share/oops/oop.hpp index a330b3df529..dcf42c7343b 100644 --- a/src/hotspot/share/oops/oop.hpp +++ b/src/hotspot/share/oops/oop.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -62,6 +62,8 @@ class oopDesc { // make use of the C++ copy/assign incorrect. NONCOPYABLE(oopDesc); + inline oop cas_set_forwardee(markWord new_mark, markWord old_mark, atomic_memory_order order); + public: // Must be trivial; see verifying static assert after the class. oopDesc() = default; @@ -78,6 +80,9 @@ class oopDesc { inline markWord cas_set_mark(markWord new_mark, markWord old_mark); inline markWord cas_set_mark(markWord new_mark, markWord old_mark, atomic_memory_order order); + // Returns the prototype mark that should be used for this object. + inline markWord prototype_mark() const; + // Used only to re-initialize the mark word (e.g., of promoted // objects during a GC) -- requires a valid klass pointer inline void init_mark(); @@ -95,8 +100,14 @@ class oopDesc { // For klass field compression static inline void set_klass_gap(HeapWord* mem, int z); - // size of object header, aligned to platform wordSize - static constexpr int header_size() { return sizeof(oopDesc)/HeapWordSize; } + // Size of object header, aligned to platform wordSize + static int header_size() { + if (UseCompactObjectHeaders) { + return sizeof(markWord) / HeapWordSize; + } else { + return sizeof(oopDesc) / HeapWordSize; + } + } // Returns whether this is an instance of k or an instance of a subclass of k inline bool is_a(Klass* k) const; @@ -258,16 +269,22 @@ class oopDesc { // Forward pointer operations for scavenge inline bool is_forwarded() const; + inline bool is_self_forwarded() const; inline void forward_to(oop p); + inline void forward_to_self(); // Like "forward_to", but inserts the forwarding pointer atomically. // Exactly one thread succeeds in inserting the forwarding pointer, and // this call returns null for that thread; any other thread has the // value of the forwarding pointer returned and does not modify "this". inline oop forward_to_atomic(oop p, markWord compare, atomic_memory_order order = memory_order_conservative); + inline oop forward_to_self_atomic(markWord compare, atomic_memory_order order = memory_order_conservative); inline oop forwardee() const; + inline oop forwardee(markWord header) const; + + inline void unset_self_forwarded(); // Age of object during scavenge inline uint age() const; @@ -311,12 +328,39 @@ class oopDesc { // for code generation static int mark_offset_in_bytes() { return (int)offset_of(oopDesc, _mark); } - static int klass_offset_in_bytes() { return (int)offset_of(oopDesc, _metadata._klass); } + static int klass_offset_in_bytes() { +#ifdef _LP64 + if (UseCompactObjectHeaders) { + // NOTE: The only places where this is used with compact headers are the C2 + // compiler and JVMCI, and even there we don't use it to access the (narrow)Klass* + // directly. It is used only as a placeholder to identify the special memory slice + // containing Klass* info. This value could be any value that is not a valid + // field offset. Use an offset halfway into the markWord, as the markWord is never + // partially loaded from C2 and JVMCI. + return mark_offset_in_bytes() + 4; + } else +#endif + { + return (int)offset_of(oopDesc, _metadata._klass); + } + } static int klass_gap_offset_in_bytes() { assert(has_klass_gap(), "only applicable to compressed klass pointers"); return klass_offset_in_bytes() + sizeof(narrowKlass); } + static int base_offset_in_bytes() { + if (UseCompactObjectHeaders) { + // With compact headers, the Klass* field is not used for the Klass* + // and is used for the object fields instead. + return sizeof(markWord); + } else if (UseCompressedClassPointers) { + return sizeof(markWord) + sizeof(narrowKlass); + } else { + return sizeof(markWord) + sizeof(Klass*); + } + } + // for error reporting static void* load_oop_raw(oop obj, int offset); }; diff --git a/src/hotspot/share/oops/oop.inline.hpp b/src/hotspot/share/oops/oop.inline.hpp index 95b99b215fa..098e6bfa90f 100644 --- a/src/hotspot/share/oops/oop.inline.hpp +++ b/src/hotspot/share/oops/oop.inline.hpp @@ -34,7 +34,7 @@ #include "oops/arrayOop.hpp" #include "oops/compressedKlass.inline.hpp" #include "oops/instanceKlass.hpp" -#include "oops/markWord.hpp" +#include "oops/markWord.inline.hpp" #include "oops/oopsHierarchy.hpp" #include "runtime/atomic.hpp" #include "runtime/globals.hpp" @@ -82,20 +82,32 @@ markWord oopDesc::cas_set_mark(markWord new_mark, markWord old_mark, atomic_memo return Atomic::cmpxchg(&_mark, old_mark, new_mark, order); } +markWord oopDesc::prototype_mark() const { + if (UseCompactObjectHeaders) { + return klass()->prototype_header(); + } else { + return markWord::prototype(); + } +} + void oopDesc::init_mark() { - set_mark(markWord::prototype()); + set_mark(prototype_mark()); } Klass* oopDesc::klass() const { - if (UseCompressedClassPointers) { - return CompressedKlassPointers::decode_not_null(_metadata._compressed_klass); + if (UseCompactObjectHeaders) { + return mark().klass(); + } else if (UseCompressedClassPointers) { + return CompressedKlassPointers::decode_not_null(_metadata._compressed_klass); } else { return _metadata._klass; } } Klass* oopDesc::klass_or_null() const { - if (UseCompressedClassPointers) { + if (UseCompactObjectHeaders) { + return mark().klass_or_null(); + } else if (UseCompressedClassPointers) { return CompressedKlassPointers::decode(_metadata._compressed_klass); } else { return _metadata._klass; @@ -103,16 +115,20 @@ Klass* oopDesc::klass_or_null() const { } Klass* oopDesc::klass_or_null_acquire() const { - if (UseCompressedClassPointers) { - narrowKlass nklass = Atomic::load_acquire(&_metadata._compressed_klass); - return CompressedKlassPointers::decode(nklass); + if (UseCompactObjectHeaders) { + return mark_acquire().klass(); + } else if (UseCompressedClassPointers) { + narrowKlass narrow_klass = Atomic::load_acquire(&_metadata._compressed_klass); + return CompressedKlassPointers::decode(narrow_klass); } else { return Atomic::load_acquire(&_metadata._klass); } } Klass* oopDesc::klass_without_asserts() const { - if (UseCompressedClassPointers) { + if (UseCompactObjectHeaders) { + return mark().klass_without_asserts(); + } else if (UseCompressedClassPointers) { return CompressedKlassPointers::decode_without_asserts(_metadata._compressed_klass); } else { return _metadata._klass; @@ -121,6 +137,7 @@ Klass* oopDesc::klass_without_asserts() const { void oopDesc::set_klass(Klass* k) { assert(Universe::is_bootstrapping() || (k != nullptr && k->is_klass()), "incorrect Klass"); + assert(!UseCompactObjectHeaders, "don't set Klass* with compact headers"); if (UseCompressedClassPointers) { _metadata._compressed_klass = CompressedKlassPointers::encode_not_null(k); } else { @@ -130,6 +147,7 @@ void oopDesc::set_klass(Klass* k) { void oopDesc::release_set_klass(HeapWord* mem, Klass* k) { assert(Universe::is_bootstrapping() || (k != nullptr && k->is_klass()), "incorrect Klass"); + assert(!UseCompactObjectHeaders, "don't set Klass* with compact headers"); char* raw_mem = ((char*)mem + klass_offset_in_bytes()); if (UseCompressedClassPointers) { Atomic::release_store((narrowKlass*)raw_mem, @@ -140,9 +158,8 @@ void oopDesc::release_set_klass(HeapWord* mem, Klass* k) { } void oopDesc::set_klass_gap(HeapWord* mem, int v) { - if (UseCompressedClassPointers) { - *(int*)(((char*)mem) + klass_gap_offset_in_bytes()) = v; - } + assert(has_klass_gap(), "precondition"); + *(int*)(((char*)mem) + klass_gap_offset_in_bytes()) = v; } bool oopDesc::is_a(Klass* k) const { @@ -267,21 +284,53 @@ bool oopDesc::is_forwarded() const { return mark().is_forwarded(); } +bool oopDesc::is_self_forwarded() const { + return mark().is_self_forwarded(); +} + // Used by scavengers void oopDesc::forward_to(oop p) { + assert(cast_from_oop(p) != this, + "must not be used for self-forwarding, use forward_to_self() instead"); markWord m = markWord::encode_pointer_as_mark(p); assert(m.decode_pointer() == p, "encoding must be reversible"); set_mark(m); } -oop oopDesc::forward_to_atomic(oop p, markWord compare, atomic_memory_order order) { - markWord m = markWord::encode_pointer_as_mark(p); - assert(m.decode_pointer() == p, "encoding must be reversible"); - markWord old_mark = cas_set_mark(m, compare, order); +void oopDesc::forward_to_self() { + set_mark(mark().set_self_forwarded()); +} + +oop oopDesc::cas_set_forwardee(markWord new_mark, markWord compare, atomic_memory_order order) { + markWord old_mark = cas_set_mark(new_mark, compare, order); if (old_mark == compare) { return nullptr; } else { - return cast_to_oop(old_mark.decode_pointer()); + assert(old_mark.is_forwarded(), "must be forwarded here"); + return forwardee(old_mark); + } +} + +oop oopDesc::forward_to_atomic(oop p, markWord compare, atomic_memory_order order) { + assert(cast_from_oop(p) != this, + "must not be used for self-forwarding, use forward_to_self_atomic() instead"); + markWord m = markWord::encode_pointer_as_mark(p); + assert(forwardee(m) == p, "encoding must be reversible"); + return cas_set_forwardee(m, compare, order); +} + +oop oopDesc::forward_to_self_atomic(markWord old_mark, atomic_memory_order order) { + markWord new_mark = old_mark.set_self_forwarded(); + assert(forwardee(new_mark) == cast_to_oop(this), "encoding must be reversible"); + return cas_set_forwardee(new_mark, old_mark, order); +} + +oop oopDesc::forwardee(markWord mark) const { + assert(mark.is_forwarded(), "only decode when actually forwarded"); + if (mark.is_self_forwarded()) { + return cast_to_oop(this); + } else { + return mark.forwardee(); } } @@ -289,7 +338,11 @@ oop oopDesc::forward_to_atomic(oop p, markWord compare, atomic_memory_order orde // The forwardee is used when copying during scavenge and mark-sweep. // It does need to clear the low two locking- and GC-related bits. oop oopDesc::forwardee() const { - return mark().forwardee(); + return forwardee(mark()); +} + +void oopDesc::unset_self_forwarded() { + set_mark(mark().unset_self_forwarded()); } // The following method needs to be MT safe. @@ -346,6 +399,7 @@ void oopDesc::oop_iterate_backwards(OopClosureType* cl) { template void oopDesc::oop_iterate_backwards(OopClosureType* cl, Klass* k) { + // In this assert, we cannot safely access the Klass* with compact headers. assert(k == klass(), "wrong klass"); OopIteratorClosureDispatch::oop_oop_iterate_backwards(cl, this, k); } diff --git a/src/hotspot/share/oops/oopHandle.hpp b/src/hotspot/share/oops/oopHandle.hpp index 983ddf9ae5c..4f0532534a8 100644 --- a/src/hotspot/share/oops/oopHandle.hpp +++ b/src/hotspot/share/oops/oopHandle.hpp @@ -71,6 +71,7 @@ class OopHandle { inline void replace(oop obj); inline oop xchg(oop new_value); + inline oop cmpxchg(oop old_value, oop new_value); oop* ptr_raw() const { return _obj; } }; diff --git a/src/hotspot/share/oops/oopHandle.inline.hpp b/src/hotspot/share/oops/oopHandle.inline.hpp index 23a13cb35e3..06acd291274 100644 --- a/src/hotspot/share/oops/oopHandle.inline.hpp +++ b/src/hotspot/share/oops/oopHandle.inline.hpp @@ -65,4 +65,8 @@ inline oop OopHandle::xchg(oop new_value) { return NativeAccess::oop_atomic_xchg(_obj, new_value); } +inline oop OopHandle::cmpxchg(oop old_value, oop new_value) { + return NativeAccess::oop_atomic_cmpxchg(_obj, old_value, new_value); +} + #endif // SHARE_OOPS_OOPHANDLE_INLINE_HPP diff --git a/src/hotspot/share/oops/stackChunkOop.cpp b/src/hotspot/share/oops/stackChunkOop.cpp index 344fef15308..7319e93b667 100644 --- a/src/hotspot/share/oops/stackChunkOop.cpp +++ b/src/hotspot/share/oops/stackChunkOop.cpp @@ -65,6 +65,21 @@ class FrameOopIterator : public OopIterator { } }; +class LockStackOopIterator : public OopIterator { +private: + const stackChunkOop _chunk; +public: + LockStackOopIterator(const stackChunkOop chunk) : _chunk(chunk) {} + + virtual void oops_do(OopClosure* cl) override { + int cnt = _chunk->lockstack_size(); + oop* lockstack_start = (oop*)_chunk->start_address(); + for (int i = 0; i < cnt; i++) { + cl->do_oop(&lockstack_start[i]); + } + } +}; + frame stackChunkOopDesc::top_frame(RegisterMap* map) { assert(!is_empty(), ""); StackChunkFrameStream fs(this); @@ -224,6 +239,14 @@ class EncodeGCModeConcurrentFrameClosure { return true; } + + bool do_lockstack() { + BarrierSetStackChunk* bs_chunk = BarrierSet::barrier_set()->barrier_set_stack_chunk(); + LockStackOopIterator iterator(_chunk); + bs_chunk->encode_gc_mode(_chunk, &iterator); + + return true; + } }; bool stackChunkOopDesc::try_acquire_relativization() { @@ -298,6 +321,7 @@ void stackChunkOopDesc::relativize_derived_pointers_concurrently() { DerivedPointersSupport::RelativizeClosure derived_cl; EncodeGCModeConcurrentFrameClosure frame_cl(this, &derived_cl); iterate_stack(&frame_cl); + frame_cl.do_lockstack(); release_relativization(); } @@ -320,6 +344,14 @@ class TransformStackChunkClosure { return true; } + + bool do_lockstack() { + BarrierSetStackChunk* bs_chunk = BarrierSet::barrier_set()->barrier_set_stack_chunk(); + LockStackOopIterator iterator(_chunk); + bs_chunk->encode_gc_mode(_chunk, &iterator); + + return true; + } }; void stackChunkOopDesc::transform() { @@ -332,6 +364,7 @@ void stackChunkOopDesc::transform() { TransformStackChunkClosure closure(this); iterate_stack(&closure); + closure.do_lockstack(); } template @@ -408,6 +441,35 @@ void stackChunkOopDesc::fix_thawed_frame(const frame& f, const RegisterMapT* map template void stackChunkOopDesc::fix_thawed_frame(const frame& f, const RegisterMap* map); template void stackChunkOopDesc::fix_thawed_frame(const frame& f, const SmallRegisterMap* map); +void stackChunkOopDesc::transfer_lockstack(oop* dst, bool requires_barriers) { + const bool requires_gc_barriers = is_gc_mode() || requires_barriers; + const bool requires_uncompress = has_bitmap() && UseCompressedOops; + const auto load_and_clear_obj = [&](intptr_t* at) -> oop { + if (requires_gc_barriers) { + if (requires_uncompress) { + oop value = HeapAccess<>::oop_load(reinterpret_cast(at)); + HeapAccess<>::oop_store(reinterpret_cast(at), nullptr); + return value; + } else { + oop value = HeapAccess<>::oop_load(reinterpret_cast(at)); + HeapAccess<>::oop_store(reinterpret_cast(at), nullptr); + return value; + } + } else { + oop value = *reinterpret_cast(at); + return value; + } + }; + + const int cnt = lockstack_size(); + intptr_t* lockstack_start = start_address(); + for (int i = 0; i < cnt; i++) { + oop mon_owner = load_and_clear_obj(&lockstack_start[i]); + assert(oopDesc::is_oop(mon_owner), "not an oop"); + dst[i] = mon_owner; + } +} + void stackChunkOopDesc::print_on(bool verbose, outputStream* st) const { if (*((juint*)this) == badHeapWordVal) { st->print_cr("BAD WORD"); @@ -456,9 +518,9 @@ class VerifyStackChunkFrameClosure { int _num_interpreted_frames; int _num_i2c; - VerifyStackChunkFrameClosure(stackChunkOop chunk, int num_frames, int size) + VerifyStackChunkFrameClosure(stackChunkOop chunk) : _chunk(chunk), _sp(nullptr), _cb(nullptr), _callee_interpreted(false), - _size(size), _argsize(0), _num_oops(0), _num_frames(num_frames), _num_interpreted_frames(0), _num_i2c(0) {} + _size(0), _argsize(0), _num_oops(0), _num_frames(0), _num_interpreted_frames(0), _num_i2c(0) {} template bool do_frame(const StackChunkFrameStream& f, const RegisterMapT* map) { @@ -552,11 +614,8 @@ bool stackChunkOopDesc::verify(size_t* out_size, int* out_oops, int* out_frames, assert((size == 0) == is_empty(), ""); const StackChunkFrameStream first(this); - const bool has_safepoint_stub_frame = first.is_stub(); - VerifyStackChunkFrameClosure closure(this, - has_safepoint_stub_frame ? 1 : 0, // Iterate_stack skips the safepoint stub - has_safepoint_stub_frame ? first.frame_size() : 0); + VerifyStackChunkFrameClosure closure(this); iterate_stack(&closure); assert(!is_empty() || closure._cb == nullptr, ""); diff --git a/src/hotspot/share/oops/stackChunkOop.hpp b/src/hotspot/share/oops/stackChunkOop.hpp index 250acc8d165..38f26d091f0 100644 --- a/src/hotspot/share/oops/stackChunkOop.hpp +++ b/src/hotspot/share/oops/stackChunkOop.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2021, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -36,6 +36,8 @@ class frame; class MemRegion; class RegisterMap; class VMRegImpl; +class ObjectMonitor; +class ObjectWaiter; typedef VMRegImpl* VMReg; // A continuation stack-chunk oop. @@ -58,6 +60,8 @@ class stackChunkOopDesc : public instanceOopDesc { static const uint8_t FLAG_NOTIFY_RELATIVIZE = 1 << 2; // Someone is waiting for relativization to complete static const uint8_t FLAG_GC_MODE = 1 << 3; // Once true it and FLAG_HAS_INTERPRETED_FRAMES can't change static const uint8_t FLAG_HAS_BITMAP = 1 << 4; // Can only be true if FLAG_GC_MODE is true + static const uint8_t FLAG_HAS_LOCKSTACK = 1 << 5; // LockStack was copied into stackChunk + static const uint8_t FLAG_PREEMPTED = 1 << 6; // Continuation was unmounted from inside VM bool try_acquire_relativization(); void release_relativization(); @@ -91,6 +95,15 @@ class stackChunkOopDesc : public instanceOopDesc { inline int max_thawing_size() const; inline void set_max_thawing_size(int value); + inline uint8_t lockstack_size() const; + inline void set_lockstack_size(uint8_t value); + + inline ObjectWaiter* object_waiter() const; + inline void set_object_waiter(ObjectWaiter* obj_waiter); + + inline ObjectMonitor* current_pending_monitor() const; + inline ObjectMonitor* current_waiting_monitor() const; + inline oop cont() const; template inline oop cont() const; @@ -127,6 +140,12 @@ class stackChunkOopDesc : public instanceOopDesc { inline bool has_mixed_frames() const; inline void set_has_mixed_frames(bool value); + inline bool preempted() const; + inline void set_preempted(bool value); + + inline bool has_lockstack() const; + inline void set_has_lockstack(bool value); + inline bool is_gc_mode() const; inline bool is_gc_mode_acquire() const; inline void set_gc_mode(bool value); @@ -147,6 +166,11 @@ class stackChunkOopDesc : public instanceOopDesc { template void fix_thawed_frame(const frame& f, const RegisterMapT* map); + void transfer_lockstack(oop* start, bool requires_barriers); + + template + inline void iterate_lockstack(StackChunkLockStackClosureType* closure); + template inline void iterate_stack(StackChunkFrameClosureType* closure); diff --git a/src/hotspot/share/oops/stackChunkOop.inline.hpp b/src/hotspot/share/oops/stackChunkOop.inline.hpp index 7b955d551d7..d647904bf5b 100644 --- a/src/hotspot/share/oops/stackChunkOop.inline.hpp +++ b/src/hotspot/share/oops/stackChunkOop.inline.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2021, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -36,9 +36,10 @@ #include "oops/access.inline.hpp" #include "oops/instanceStackChunkKlass.inline.hpp" #include "runtime/continuationJavaClasses.inline.hpp" -#include "runtime/frame.inline.hpp" +#include "runtime/frame.hpp" #include "runtime/globals.hpp" #include "runtime/handles.inline.hpp" +#include "runtime/objectMonitor.hpp" #include "runtime/registerMap.hpp" #include "runtime/smallRegisterMap.inline.hpp" #include "utilities/macros.hpp" @@ -88,6 +89,12 @@ inline void stackChunkOopDesc::set_max_thawing_size(int value) { jdk_internal_vm_StackChunk::set_maxThawingSize(this, (jint)value); } +inline uint8_t stackChunkOopDesc::lockstack_size() const { return jdk_internal_vm_StackChunk::lockStackSize(as_oop()); } +inline void stackChunkOopDesc::set_lockstack_size(uint8_t value) { jdk_internal_vm_StackChunk::set_lockStackSize(this, value); } + +inline ObjectWaiter* stackChunkOopDesc::object_waiter() const { return (ObjectWaiter*)jdk_internal_vm_StackChunk::objectWaiter(as_oop()); } +inline void stackChunkOopDesc::set_object_waiter(ObjectWaiter* obj) { jdk_internal_vm_StackChunk::set_objectWaiter(this, (address)obj); } + inline oop stackChunkOopDesc::cont() const { return jdk_internal_vm_StackChunk::cont(as_oop()); } inline void stackChunkOopDesc::set_cont(oop value) { jdk_internal_vm_StackChunk::set_cont(this, value); } template @@ -154,10 +161,35 @@ inline void stackChunkOopDesc::clear_flags() { inline bool stackChunkOopDesc::has_mixed_frames() const { return is_flag(FLAG_HAS_INTERPRETED_FRAMES); } inline void stackChunkOopDesc::set_has_mixed_frames(bool value) { - assert((flags() & ~FLAG_HAS_INTERPRETED_FRAMES) == 0, "other flags should not be set"); + assert((flags() & ~(FLAG_HAS_INTERPRETED_FRAMES | FLAG_PREEMPTED)) == 0, "other flags should not be set"); set_flag(FLAG_HAS_INTERPRETED_FRAMES, value); } +inline bool stackChunkOopDesc::preempted() const { return is_flag(FLAG_PREEMPTED); } +inline void stackChunkOopDesc::set_preempted(bool value) { + assert(preempted() != value, ""); + set_flag(FLAG_PREEMPTED, value); +} + +inline ObjectMonitor* stackChunkOopDesc::current_pending_monitor() const { + ObjectWaiter* waiter = object_waiter(); + if (waiter != nullptr && waiter->at_monitorenter()) { + return waiter->monitor(); + } + return nullptr; +} + +inline ObjectMonitor* stackChunkOopDesc::current_waiting_monitor() const { + ObjectWaiter* waiter = object_waiter(); + if (waiter != nullptr && waiter->is_wait()) { + return waiter->monitor(); + } + return nullptr; +} + +inline bool stackChunkOopDesc::has_lockstack() const { return is_flag(FLAG_HAS_LOCKSTACK); } +inline void stackChunkOopDesc::set_has_lockstack(bool value) { set_flag(FLAG_HAS_LOCKSTACK, value); } + inline bool stackChunkOopDesc::is_gc_mode() const { return is_flag(FLAG_GC_MODE); } inline bool stackChunkOopDesc::is_gc_mode_acquire() const { return is_flag_acquire(FLAG_GC_MODE); } inline void stackChunkOopDesc::set_gc_mode(bool value) { set_flag(FLAG_GC_MODE, value); } @@ -180,6 +212,16 @@ void stackChunkOopDesc::do_barriers(const StackChunkFrameStream& f, do_barriers0(f, map); } +template +inline void stackChunkOopDesc::iterate_lockstack(StackChunkLockStackClosureType* closure) { + assert(LockingMode == LM_LIGHTWEIGHT, ""); + int cnt = lockstack_size(); + intptr_t* lockstart_addr = start_address(); + for (int i = 0; i < cnt; i++) { + closure->do_oop((OopT*)&lockstart_addr[i]); + } +} + template inline void stackChunkOopDesc::iterate_stack(StackChunkFrameClosureType* closure) { has_mixed_frames() ? iterate_stack(closure) @@ -200,15 +242,14 @@ inline void stackChunkOopDesc::iterate_stack(StackChunkFrameClosureType* closure RegisterMap::ProcessFrames::skip, RegisterMap::WalkContinuation::include); full_map.set_include_argument_oops(false); + closure->do_frame(f, map); f.next(&full_map); - assert(!f.is_done(), ""); assert(f.is_compiled(), ""); should_continue = closure->do_frame(f, &full_map); f.next(map); - f.handle_deopted(); // the stub caller might be deoptimized (as it's not at a call) } assert(!f.is_stub(), ""); @@ -269,7 +310,7 @@ inline MemRegion stackChunkOopDesc::range() { } inline int stackChunkOopDesc::relativize_usp_offset(const frame& fr, const int usp_offset_in_bytes) const { - assert(fr.is_compiled_frame() || fr.cb()->is_safepoint_stub(), ""); + assert(fr.is_compiled_frame() || fr.cb()->is_runtime_stub(), ""); assert(is_in_chunk(fr.unextended_sp()), ""); intptr_t* base = fr.real_fp(); // equal to the caller's sp diff --git a/src/hotspot/share/oops/typeArrayKlass.cpp b/src/hotspot/share/oops/typeArrayKlass.cpp index 38e28edd157..ddf60d4382e 100644 --- a/src/hotspot/share/oops/typeArrayKlass.cpp +++ b/src/hotspot/share/oops/typeArrayKlass.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -171,7 +171,8 @@ void TypeArrayKlass::copy_array(arrayOop s, int src_pos, arrayOop d, int dst_pos } size_t TypeArrayKlass::oop_size(oop obj) const { - assert(obj->is_typeArray(),"must be a type array"); + // In this assert, we cannot safely access the Klass* with compact headers. + assert(UseCompactObjectHeaders || obj->is_typeArray(),"must be a type array"); typeArrayOop t = typeArrayOop(obj); return t->object_size(this); } diff --git a/src/hotspot/share/opto/addnode.cpp b/src/hotspot/share/opto/addnode.cpp index 802af20adae..4215124b0d4 100644 --- a/src/hotspot/share/opto/addnode.cpp +++ b/src/hotspot/share/opto/addnode.cpp @@ -1546,7 +1546,7 @@ const Type* MinLNode::add_ring(const Type* t0, const Type* t1) const { const TypeLong* r0 = t0->is_long(); const TypeLong* r1 = t1->is_long(); - return TypeLong::make(MIN2(r0->_lo, r1->_lo), MIN2(r0->_hi, r1->_hi), MIN2(r0->_widen, r1->_widen)); + return TypeLong::make(MIN2(r0->_lo, r1->_lo), MIN2(r0->_hi, r1->_hi), MAX2(r0->_widen, r1->_widen)); } Node* MinLNode::Identity(PhaseGVN* phase) { diff --git a/src/hotspot/share/opto/arraycopynode.cpp b/src/hotspot/share/opto/arraycopynode.cpp index 4e2b159bf8e..0b4c88f0b5c 100644 --- a/src/hotspot/share/opto/arraycopynode.cpp +++ b/src/hotspot/share/opto/arraycopynode.cpp @@ -220,6 +220,10 @@ Node* ArrayCopyNode::try_clone_instance(PhaseGVN *phase, bool can_reshape, int c Node* off = phase->MakeConX(field->offset_in_bytes()); Node* next_src = phase->transform(new AddPNode(base_src,base_src,off)); Node* next_dest = phase->transform(new AddPNode(base_dest,base_dest,off)); + assert(phase->C->get_alias_index(adr_type) == phase->C->get_alias_index(phase->type(next_src)->isa_ptr()), + "slice of address and input slice don't match"); + assert(phase->C->get_alias_index(adr_type) == phase->C->get_alias_index(phase->type(next_dest)->isa_ptr()), + "slice of address and input slice don't match"); BasicType bt = field->layout_type(); const Type *type; diff --git a/src/hotspot/share/opto/buildOopMap.cpp b/src/hotspot/share/opto/buildOopMap.cpp index b553cc6ea69..c2f771bedb7 100644 --- a/src/hotspot/share/opto/buildOopMap.cpp +++ b/src/hotspot/share/opto/buildOopMap.cpp @@ -601,7 +601,7 @@ static void do_liveness(PhaseRegAlloc* regalloc, PhaseCFG* cfg, Block_List* work // Collect GC mask info - where are all the OOPs? void PhaseOutput::BuildOopMaps() { - Compile::TracePhase tp("bldOopMaps", &timers[_t_buildOopMaps]); + Compile::TracePhase tp(_t_buildOopMaps); // Can't resource-mark because I need to leave all those OopMaps around, // or else I need to resource-mark some arena other than the default. // ResourceMark rm; // Reclaim all OopFlows when done diff --git a/src/hotspot/share/opto/c2_globals.hpp b/src/hotspot/share/opto/c2_globals.hpp index 7f6b7c2dceb..45a067a830b 100644 --- a/src/hotspot/share/opto/c2_globals.hpp +++ b/src/hotspot/share/opto/c2_globals.hpp @@ -419,7 +419,7 @@ \ product(intx, LoopOptsCount, 43, \ "Set level of loop optimization for tier 1 compiles") \ - range(5, 43) \ + range(5, max_jint) \ \ product(bool, OptimizeUnstableIf, true, DIAGNOSTIC, \ "Optimize UnstableIf traps") \ diff --git a/src/hotspot/share/opto/callnode.cpp b/src/hotspot/share/opto/callnode.cpp index e800b3c736b..83ea2eea1a8 100644 --- a/src/hotspot/share/opto/callnode.cpp +++ b/src/hotspot/share/opto/callnode.cpp @@ -1615,8 +1615,14 @@ void AllocateNode::compute_MemBar_redundancy(ciMethod* initializer) } Node *AllocateNode::make_ideal_mark(PhaseGVN *phase, Node* obj, Node* control, Node* mem) { Node* mark_node = nullptr; - // For now only enable fast locking for non-array types - mark_node = phase->MakeConX(markWord::prototype().value()); + if (UseCompactObjectHeaders) { + Node* klass_node = in(AllocateNode::KlassNode); + Node* proto_adr = phase->transform(new AddPNode(klass_node, klass_node, phase->MakeConX(in_bytes(Klass::prototype_header_offset())))); + mark_node = LoadNode::make(*phase, control, mem, proto_adr, TypeRawPtr::BOTTOM, TypeX_X, TypeX_X->basic_type(), MemNode::unordered); + } else { + // For now only enable fast locking for non-array types + mark_node = phase->MakeConX(markWord::prototype().value()); + } return mark_node; } diff --git a/src/hotspot/share/opto/cfgnode.hpp b/src/hotspot/share/opto/cfgnode.hpp index ac8896705de..1d287c2f43d 100644 --- a/src/hotspot/share/opto/cfgnode.hpp +++ b/src/hotspot/share/opto/cfgnode.hpp @@ -324,7 +324,7 @@ class IfNode : public MultiBranchNode { float _fcnt; // Frequency counter private: - NOT_PRODUCT(AssertionPredicateType _assertion_predicate_type;) + AssertionPredicateType _assertion_predicate_type; void init_node(Node* control, Node* bol) { init_class_id(Class_If); @@ -426,7 +426,7 @@ class IfNode : public MultiBranchNode { // gen_subtype_check() and catch_inline_exceptions(). IfNode(Node* control, Node* bol, float p, float fcnt); - NOT_PRODUCT(IfNode(Node* control, Node* bol, float p, float fcnt, AssertionPredicateType assertion_predicate_type);) + IfNode(Node* control, Node* bol, float p, float fcnt, AssertionPredicateType assertion_predicate_type); static IfNode* make_with_same_profile(IfNode* if_node_profile, Node* ctrl, BoolNode* bol); @@ -448,11 +448,11 @@ class IfNode : public MultiBranchNode { // Returns null is it couldn't improve the type. static const TypeInt* filtered_int_type(PhaseGVN* phase, Node* val, Node* if_proj); -#ifndef PRODUCT AssertionPredicateType assertion_predicate_type() const { return _assertion_predicate_type; } +#ifndef PRODUCT virtual void dump_spec(outputStream *st) const; #endif @@ -468,12 +468,10 @@ class RangeCheckNode : public IfNode { init_class_id(Class_RangeCheck); } -#ifndef PRODUCT RangeCheckNode(Node* control, Node* bol, float p, float fcnt, AssertionPredicateType assertion_predicate_type) : IfNode(control, bol, p, fcnt, assertion_predicate_type) { init_class_id(Class_RangeCheck); } -#endif // NOT PRODUCT virtual int Opcode() const; virtual Node* Ideal(PhaseGVN *phase, bool can_reshape); diff --git a/src/hotspot/share/opto/chaitin.cpp b/src/hotspot/share/opto/chaitin.cpp index be0aadacbc2..bef4cfce4d5 100644 --- a/src/hotspot/share/opto/chaitin.cpp +++ b/src/hotspot/share/opto/chaitin.cpp @@ -216,7 +216,7 @@ PhaseChaitin::PhaseChaitin(uint unique, PhaseCFG &cfg, Matcher &matcher, bool sc , _scratch_int_pressure(0, Matcher::int_pressure_limit()) , _scratch_float_pressure(0, Matcher::float_pressure_limit()) { - Compile::TracePhase tp("ctorChaitin", &timers[_t_ctorChaitin]); + Compile::TracePhase tp(_t_ctorChaitin); _high_frequency_lrg = MIN2(double(OPTO_LRG_HIGH_FREQ), _cfg.get_outer_loop_frequency()); @@ -321,7 +321,7 @@ int PhaseChaitin::clone_projs(Block* b, uint idx, Node* orig, Node* copy, uint& // Renumber the live ranges to compact them. Makes the IFG smaller. void PhaseChaitin::compact() { - Compile::TracePhase tp("chaitinCompact", &timers[_t_chaitinCompact]); + Compile::TracePhase tp(_t_chaitinCompact); // Current the _uf_map contains a series of short chains which are headed // by a self-cycle. All the chains run from big numbers to little numbers. @@ -397,7 +397,7 @@ void PhaseChaitin::Register_Allocate() { #endif { - Compile::TracePhase tp("computeLive", &timers[_t_computeLive]); + Compile::TracePhase tp(_t_computeLive); _live = nullptr; // Mark live as being not available rm.reset_to_mark(); // Reclaim working storage IndexSet::reset_memory(C, &live_arena); @@ -414,7 +414,7 @@ void PhaseChaitin::Register_Allocate() { // at all the GC points, and "stretches" the live range of any base pointer // to the GC point. if (stretch_base_pointer_live_ranges(&live_arena)) { - Compile::TracePhase tp("computeLive (sbplr)", &timers[_t_computeLive]); + Compile::TracePhase tp("computeLive (sbplr)", _t_computeLive); // Since some live range stretched, I need to recompute live _live = nullptr; rm.reset_to_mark(); // Reclaim working storage @@ -439,7 +439,7 @@ void PhaseChaitin::Register_Allocate() { // This pass works on virtual copies. Any virtual copies which are not // coalesced get manifested as actual copies { - Compile::TracePhase tp("chaitinCoalesce1", &timers[_t_chaitinCoalesce1]); + Compile::TracePhase tp(_t_chaitinCoalesce1); PhaseAggressiveCoalesce coalesce(*this); coalesce.coalesce_driver(); @@ -454,7 +454,7 @@ void PhaseChaitin::Register_Allocate() { // After aggressive coalesce, attempt a first cut at coloring. // To color, we need the IFG and for that we need LIVE. { - Compile::TracePhase tp("computeLive", &timers[_t_computeLive]); + Compile::TracePhase tp(_t_computeLive); _live = nullptr; rm.reset_to_mark(); // Reclaim working storage IndexSet::reset_memory(C, &live_arena); @@ -495,7 +495,7 @@ void PhaseChaitin::Register_Allocate() { compact(); // Compact LRGs; return new lower max lrg { - Compile::TracePhase tp("computeLive", &timers[_t_computeLive]); + Compile::TracePhase tp(_t_computeLive); _live = nullptr; rm.reset_to_mark(); // Reclaim working storage IndexSet::reset_memory(C, &live_arena); @@ -509,7 +509,7 @@ void PhaseChaitin::Register_Allocate() { _ifg->Compute_Effective_Degree(); // Only do conservative coalescing if requested if (OptoCoalesce) { - Compile::TracePhase tp("chaitinCoalesce2", &timers[_t_chaitinCoalesce2]); + Compile::TracePhase tp(_t_chaitinCoalesce2); // Conservative (and pessimistic) copy coalescing of those spills PhaseConservativeCoalesce coalesce(*this); // If max live ranges greater than cutoff, don't color the stack. @@ -568,7 +568,7 @@ void PhaseChaitin::Register_Allocate() { // Nuke the live-ness and interference graph and LiveRanGe info { - Compile::TracePhase tp("computeLive", &timers[_t_computeLive]); + Compile::TracePhase tp(_t_computeLive); _live = nullptr; rm.reset_to_mark(); // Reclaim working storage IndexSet::reset_memory(C, &live_arena); @@ -586,7 +586,7 @@ void PhaseChaitin::Register_Allocate() { // Only do conservative coalescing if requested if (OptoCoalesce) { - Compile::TracePhase tp("chaitinCoalesce3", &timers[_t_chaitinCoalesce3]); + Compile::TracePhase tp(_t_chaitinCoalesce3); // Conservative (and pessimistic) copy coalescing PhaseConservativeCoalesce coalesce(*this); // Check for few live ranges determines how aggressive coalesce is. @@ -1183,7 +1183,7 @@ void PhaseChaitin::set_was_low() { // Compute cost/area ratio, in case we spill. Build the lo-degree list. void PhaseChaitin::cache_lrg_info( ) { - Compile::TracePhase tp("chaitinCacheLRG", &timers[_t_chaitinCacheLRG]); + Compile::TracePhase tp(_t_chaitinCacheLRG); for (uint i = 1; i < _lrg_map.max_lrg_id(); i++) { LRG &lrg = lrgs(i); @@ -1217,7 +1217,7 @@ void PhaseChaitin::cache_lrg_info( ) { // Simplify the IFG by removing LRGs of low degree. void PhaseChaitin::Simplify( ) { - Compile::TracePhase tp("chaitinSimplify", &timers[_t_chaitinSimplify]); + Compile::TracePhase tp(_t_chaitinSimplify); while( 1 ) { // Repeat till simplified it all // May want to explore simplifying lo_degree before _lo_stk_degree. @@ -1516,7 +1516,7 @@ OptoReg::Name PhaseChaitin::choose_color( LRG &lrg, int chunk ) { // everything going back is guaranteed a color. Select that color. If some // hi-degree LRG cannot get a color then we record that we must spill. uint PhaseChaitin::Select( ) { - Compile::TracePhase tp("chaitinSelect", &timers[_t_chaitinSelect]); + Compile::TracePhase tp(_t_chaitinSelect); uint spill_reg = LRG::SPILL_REG; _max_reg = OptoReg::Name(0); // Past max register used @@ -1704,7 +1704,7 @@ void PhaseChaitin::fixup_spills() { // This function does only cisc spill work. if( !UseCISCSpill ) return; - Compile::TracePhase tp("fixupSpills", &timers[_t_fixupSpills]); + Compile::TracePhase tp(_t_fixupSpills); // Grab the Frame Pointer Node *fp = _cfg.get_root_block()->head()->in(1)->in(TypeFunc::FramePtr); diff --git a/src/hotspot/share/opto/compile.cpp b/src/hotspot/share/opto/compile.cpp index a2428625d4d..1b148db20e0 100644 --- a/src/hotspot/share/opto/compile.cpp +++ b/src/hotspot/share/opto/compile.cpp @@ -746,7 +746,7 @@ Compile::Compile( ciEnv* ci_env, ciMethod* target, int osr_bci, print_inlining_init(); { // Scope for timing the parser - TracePhase tp("parse", &timers[_t_parser]); + TracePhase tp(_t_parser); // Put top into the hash table ASAP. initial_gvn()->transform(top()); @@ -1701,6 +1701,10 @@ Compile::AliasType* Compile::find_alias_type(const TypePtr* adr_type, bool no_cr } } if (flat->isa_klassptr()) { + if (UseCompactObjectHeaders) { + if (flat->offset() == in_bytes(Klass::prototype_header_offset())) + alias_type(idx)->set_rewritable(false); + } if (flat->offset() == in_bytes(Klass::super_check_offset_offset())) alias_type(idx)->set_rewritable(false); if (flat->offset() == in_bytes(Klass::modifier_flags_offset())) @@ -2027,7 +2031,7 @@ void Compile::inline_boxing_calls(PhaseIterGVN& igvn) { bool Compile::inline_incrementally_one() { assert(IncrementalInline, "incremental inlining should be on"); - TracePhase tp("incrementalInline_inline", &timers[_t_incrInline_inline]); + TracePhase tp(_t_incrInline_inline); set_inlining_progress(false); set_do_cleanup(false); @@ -2068,12 +2072,12 @@ bool Compile::inline_incrementally_one() { void Compile::inline_incrementally_cleanup(PhaseIterGVN& igvn) { { - TracePhase tp("incrementalInline_pru", &timers[_t_incrInline_pru]); + TracePhase tp(_t_incrInline_pru); ResourceMark rm; PhaseRemoveUseless pru(initial_gvn(), *igvn_worklist()); } { - TracePhase tp("incrementalInline_igvn", &timers[_t_incrInline_igvn]); + TracePhase tp(_t_incrInline_igvn); igvn.reset_from_gvn(initial_gvn()); igvn.optimize(); if (failing()) return; @@ -2083,7 +2087,7 @@ void Compile::inline_incrementally_cleanup(PhaseIterGVN& igvn) { // Perform incremental inlining until bound on number of live nodes is reached void Compile::inline_incrementally(PhaseIterGVN& igvn) { - TracePhase tp("incrementalInline", &timers[_t_incrInline]); + TracePhase tp(_t_incrInline); set_inlining_incrementally(true); uint low_live_nodes = 0; @@ -2091,7 +2095,7 @@ void Compile::inline_incrementally(PhaseIterGVN& igvn) { while (_late_inlines.length() > 0) { if (live_nodes() > (uint)LiveNodeCountInliningCutoff) { if (low_live_nodes < (uint)LiveNodeCountInliningCutoff * 8 / 10) { - TracePhase tp("incrementalInline_ideal", &timers[_t_incrInline_ideal]); + TracePhase tp(_t_incrInline_ideal); // PhaseIdealLoop is expensive so we only try it once we are // out of live nodes and we only try it again if the previous // helped got the number of nodes down significantly @@ -2175,7 +2179,7 @@ void Compile::process_late_inline_calls_no_inline(PhaseIterGVN& igvn) { bool Compile::optimize_loops(PhaseIterGVN& igvn, LoopOptsMode mode) { if (_loop_opts_cnt > 0) { while (major_progress() && (_loop_opts_cnt > 0)) { - TracePhase tp("idealLoop", &timers[_t_idealLoop]); + TracePhase tp(_t_idealLoop); PhaseIdealLoop::optimize(igvn, mode); _loop_opts_cnt--; if (failing()) return false; @@ -2213,7 +2217,7 @@ void Compile::remove_root_to_sfpts_edges(PhaseIterGVN& igvn) { //------------------------------Optimize--------------------------------------- // Given a graph, optimize it. void Compile::Optimize() { - TracePhase tp("optimizer", &timers[_t_optimizer]); + TracePhase tp(_t_optimizer); #ifndef PRODUCT if (env()->break_at_compile()) { @@ -2243,7 +2247,7 @@ void Compile::Optimize() { _modified_nodes = new (comp_arena()) Unique_Node_List(comp_arena()); #endif { - TracePhase tp("iterGVN", &timers[_t_iterGVN]); + TracePhase tp(_t_iterGVN); igvn.optimize(); } @@ -2292,7 +2296,7 @@ void Compile::Optimize() { assert(EnableVectorSupport || !has_vbox_nodes(), "sanity"); if (EnableVectorSupport && has_vbox_nodes()) { - TracePhase tp("", &timers[_t_vector]); + TracePhase tp(_t_vector); PhaseVector pv(igvn); pv.optimize_vector_boxes(); if (failing()) return; @@ -2301,7 +2305,7 @@ void Compile::Optimize() { assert(!has_vbox_nodes(), "sanity"); if (!failing() && RenumberLiveNodes && live_nodes() + NodeLimitFudgeFactor < unique()) { - Compile::TracePhase tp("", &timers[_t_renumberLive]); + Compile::TracePhase tp(_t_renumberLive); igvn_worklist()->ensure_empty(); // should be done with igvn { ResourceMark rm; @@ -2322,7 +2326,7 @@ void Compile::Optimize() { if (do_escape_analysis() && ConnectionGraph::has_candidates(this)) { if (has_loops()) { // Cleanup graph (remove dead nodes). - TracePhase tp("idealLoop", &timers[_t_idealLoop]); + TracePhase tp(_t_idealLoop); PhaseIdealLoop::optimize(igvn, LoopOptsMaxUnroll); if (failing()) return; } @@ -2342,7 +2346,7 @@ void Compile::Optimize() { if (failing()) return; if (congraph() != nullptr && macro_count() > 0) { - TracePhase tp("macroEliminate", &timers[_t_macroEliminate]); + TracePhase tp(_t_macroEliminate); PhaseMacroExpand mexp(igvn); mexp.eliminate_macro_nodes(); if (failing()) return; @@ -2371,7 +2375,7 @@ void Compile::Optimize() { // Set loop opts counter if((_loop_opts_cnt > 0) && (has_loops() || has_split_ifs())) { { - TracePhase tp("idealLoop", &timers[_t_idealLoop]); + TracePhase tp(_t_idealLoop); PhaseIdealLoop::optimize(igvn, LoopOptsDefault); _loop_opts_cnt--; if (major_progress()) print_method(PHASE_PHASEIDEALLOOP1, 2); @@ -2379,7 +2383,7 @@ void Compile::Optimize() { } // Loop opts pass if partial peeling occurred in previous pass if(PartialPeelLoop && major_progress() && (_loop_opts_cnt > 0)) { - TracePhase tp("idealLoop", &timers[_t_idealLoop]); + TracePhase tp(_t_idealLoop); PhaseIdealLoop::optimize(igvn, LoopOptsSkipSplitIf); _loop_opts_cnt--; if (major_progress()) print_method(PHASE_PHASEIDEALLOOP2, 2); @@ -2387,7 +2391,7 @@ void Compile::Optimize() { } // Loop opts pass for loop-unrolling before CCP if(major_progress() && (_loop_opts_cnt > 0)) { - TracePhase tp("idealLoop", &timers[_t_idealLoop]); + TracePhase tp(_t_idealLoop); PhaseIdealLoop::optimize(igvn, LoopOptsSkipSplitIf); _loop_opts_cnt--; if (major_progress()) print_method(PHASE_PHASEIDEALLOOP3, 2); @@ -2404,7 +2408,7 @@ void Compile::Optimize() { PhaseCCP ccp( &igvn ); assert( true, "Break here to ccp.dump_nodes_and_types(_root,999,1)"); { - TracePhase tp("ccp", &timers[_t_ccp]); + TracePhase tp(_t_ccp); ccp.do_transform(); } print_method(PHASE_CCP1, 2); @@ -2413,7 +2417,7 @@ void Compile::Optimize() { // Iterative Global Value Numbering, including ideal transforms { - TracePhase tp("iterGVN2", &timers[_t_iterGVN2]); + TracePhase tp(_t_iterGVN2); igvn.reset_from_igvn(&ccp); igvn.optimize(); } @@ -2440,7 +2444,7 @@ void Compile::Optimize() { #endif { - TracePhase tp("macroExpand", &timers[_t_macroExpand]); + TracePhase tp(_t_macroExpand); print_method(PHASE_BEFORE_MACRO_EXPANSION, 3); PhaseMacroExpand mex(igvn); if (mex.expand_macro_nodes()) { @@ -2451,7 +2455,7 @@ void Compile::Optimize() { } { - TracePhase tp("barrierExpand", &timers[_t_barrierExpand]); + TracePhase tp(_t_barrierExpand); if (bs->expand_barriers(this, igvn)) { assert(failing(), "must bail out w/ explicit message"); return; @@ -2489,7 +2493,7 @@ void Compile::Optimize() { // A method with only infinite loops has no edges entering loops from root { - TracePhase tp("graphReshape", &timers[_t_graphReshaping]); + TracePhase tp(_t_graphReshaping); if (final_graph_reshaping()) { assert(failing(), "must bail out w/ explicit message"); return; @@ -2931,7 +2935,7 @@ void Compile::Code_Gen() { Matcher matcher; _matcher = &matcher; { - TracePhase tp("matcher", &timers[_t_matcher]); + TracePhase tp(_t_matcher); matcher.match(); if (failing()) { return; @@ -2956,7 +2960,7 @@ void Compile::Code_Gen() { } _cfg = &cfg; { - TracePhase tp("scheduler", &timers[_t_scheduler]); + TracePhase tp(_t_scheduler); bool success = cfg.do_global_code_motion(); if (!success) { return; @@ -2970,7 +2974,7 @@ void Compile::Code_Gen() { PhaseChaitin regalloc(unique(), cfg, matcher, false); _regalloc = ®alloc; { - TracePhase tp("regalloc", &timers[_t_registerAllocation]); + TracePhase tp(_t_registerAllocation); // Perform register allocation. After Chaitin, use-def chains are // no longer accurate (at spill code) and so must be ignored. // Node->LRG->reg mappings are still accurate. @@ -2989,7 +2993,7 @@ void Compile::Code_Gen() { // are not adding any new instructions. If any basic block is empty, we // can now safely remove it. { - TracePhase tp("blockOrdering", &timers[_t_blockOrdering]); + TracePhase tp(_t_blockOrdering); cfg.remove_empty_blocks(); if (do_freq_based_layout()) { PhaseBlockLayout layout(cfg); @@ -3004,7 +3008,7 @@ void Compile::Code_Gen() { // Apply peephole optimizations if( OptoPeephole ) { - TracePhase tp("peephole", &timers[_t_peephole]); + TracePhase tp(_t_peephole); PhasePeephole peep( _regalloc, cfg); peep.do_transform(); print_method(PHASE_PEEPHOLE, 3); @@ -3012,14 +3016,14 @@ void Compile::Code_Gen() { // Do late expand if CPU requires this. if (Matcher::require_postalloc_expand) { - TracePhase tp("postalloc_expand", &timers[_t_postalloc_expand]); + TracePhase tp(_t_postalloc_expand); cfg.postalloc_expand(_regalloc); print_method(PHASE_POSTALLOC_EXPAND, 3); } // Convert Nodes to instruction bits in a buffer { - TracePhase tp("output", &timers[_t_output]); + TracePhase tp(_t_output); PhaseOutput output; output.Output(); if (failing()) return; @@ -4309,11 +4313,10 @@ void Compile::record_failure(const char* reason DEBUG_ONLY(COMMA bool allow_mult _root = nullptr; // flush the graph, too } -Compile::TracePhase::TracePhase(const char* name, elapsedTimer* accumulator) - : TraceTime(name, accumulator, CITime, CITimeVerbose), +Compile::TracePhase::TracePhase(const char* name, PhaseTraceId id) + : TraceTime(name, &Phase::timers[id], CITime, CITimeVerbose), _compile(Compile::current()), _log(nullptr), - _phase_name(name), _dolog(CITimeVerbose) { assert(_compile != nullptr, "sanity check"); @@ -4321,12 +4324,15 @@ Compile::TracePhase::TracePhase(const char* name, elapsedTimer* accumulator) _log = _compile->log(); } if (_log != nullptr) { - _log->begin_head("phase name='%s' nodes='%d' live='%d'", _phase_name, _compile->unique(), _compile->live_nodes()); + _log->begin_head("phase name='%s' nodes='%d' live='%d'", phase_name(), _compile->unique(), _compile->live_nodes()); _log->stamp(); _log->end_head(); } } +Compile::TracePhase::TracePhase(PhaseTraceId id) + : TracePhase(Phase::get_phase_trace_id_text(id), id) {} + Compile::TracePhase::~TracePhase() { if (_compile->failing_internal()) { return; // timing code, not stressing bailouts. @@ -4334,7 +4340,7 @@ Compile::TracePhase::~TracePhase() { #ifdef ASSERT if (PrintIdealNodeCount) { tty->print_cr("phase name='%s' nodes='%d' live='%d' live_graph_walk='%d'", - _phase_name, _compile->unique(), _compile->live_nodes(), _compile->count_live_nodes_by_graph_walk()); + phase_name(), _compile->unique(), _compile->live_nodes(), _compile->count_live_nodes_by_graph_walk()); } if (VerifyIdealNodeCount) { @@ -4343,7 +4349,7 @@ Compile::TracePhase::~TracePhase() { #endif if (_log != nullptr) { - _log->done("phase name='%s' nodes='%d' live='%d'", _phase_name, _compile->unique(), _compile->live_nodes()); + _log->done("phase name='%s' nodes='%d' live='%d'", phase_name(), _compile->unique(), _compile->live_nodes()); } } diff --git a/src/hotspot/share/opto/compile.hpp b/src/hotspot/share/opto/compile.hpp index 05e24bf3f6e..6285d4a5548 100644 --- a/src/hotspot/share/opto/compile.hpp +++ b/src/hotspot/share/opto/compile.hpp @@ -239,11 +239,12 @@ class Compile : public Phase { private: Compile* _compile; CompileLog* _log; - const char* _phase_name; bool _dolog; public: - TracePhase(const char* name, elapsedTimer* accumulator); + TracePhase(PhaseTraceId phaseTraceId); + TracePhase(const char* name, PhaseTraceId phaseTraceId); ~TracePhase(); + const char* phase_name() const { return title(); } }; // Information per category of alias (memory slice) diff --git a/src/hotspot/share/opto/escape.cpp b/src/hotspot/share/opto/escape.cpp index f8a0209d9ad..6d845c86a5f 100644 --- a/src/hotspot/share/opto/escape.cpp +++ b/src/hotspot/share/opto/escape.cpp @@ -103,7 +103,7 @@ bool ConnectionGraph::has_candidates(Compile *C) { } void ConnectionGraph::do_analysis(Compile *C, PhaseIterGVN *igvn) { - Compile::TracePhase tp("escapeAnalysis", &Phase::timers[Phase::_t_escapeAnalysis]); + Compile::TracePhase tp(Phase::_t_escapeAnalysis); ResourceMark rm; // Add ConP and ConN null oop nodes before ConnectionGraph construction @@ -148,7 +148,7 @@ bool ConnectionGraph::compute_escape() { GrowableArray mergemem_worklist; DEBUG_ONLY( GrowableArray addp_worklist; ) - { Compile::TracePhase tp("connectionGraph", &Phase::timers[Phase::_t_connectionGraph]); + { Compile::TracePhase tp(Phase::_t_connectionGraph); // 1. Populate Connection Graph (CG) with PointsTo nodes. ideal_nodes.map(C->live_nodes(), nullptr); // preallocate space @@ -644,7 +644,7 @@ bool ConnectionGraph::can_reduce_phi(PhiNode* ophi) const { // // 'curr_ctrl' is the control of the CastPP that we want to split through phi. // If the CastPP currently doesn't have a control then the CmpP/N will be -// against the NULL constant, otherwise it will be against the constant input of +// against the null constant, otherwise it will be against the constant input of // the existing CmpP/N. It's guaranteed that there will be a CmpP/N in the later // case because we have constraints on it and because the CastPP has a control // input. @@ -672,7 +672,7 @@ Node* ConnectionGraph::specialize_cmp(Node* base, Node* curr_ctrl) { // means that the CastPP now will be specific for a given base instead of a Phi. // An If-Then-Else-Region block is inserted to control the CastPP. The control // of the CastPP is a copy of the current one (if there is one) or a check -// against NULL. +// against null. // // Before: // @@ -806,10 +806,10 @@ Node* ConnectionGraph::split_castpp_load_through_phi(Node* curr_addp, Node* curr // After splitting the CastPP we'll put it under an If-Then-Else-Region control // flow. If the CastPP originally had an IfTrue/False control input then we'll // use a similar CmpP/N to control the new If-Then-Else-Region. Otherwise, we'll -// juse use a CmpP/N against the NULL constant. +// juse use a CmpP/N against the null constant. // // The If-Then-Else-Region isn't always needed. For instance, if input to -// splitted cast was not nullable (or if it was the NULL constant) then we don't +// splitted cast was not nullable (or if it was the null constant) then we don't // need (shouldn't) use a CastPP at all. // // After the casts are splitted we'll split the AddP->Loads through the Phi and @@ -837,7 +837,7 @@ Node* ConnectionGraph::split_castpp_load_through_phi(Node* curr_addp, Node* curr // // After (Very much simplified): // -// Call NULL +// Call Null // \ / // CmpP // | @@ -873,7 +873,7 @@ void ConnectionGraph::reduce_phi_on_castpp_field_load(Node* curr_castpp, Growabl // array, depending on the nullability status of the corresponding input in // ophi. // - // - nullptr: Meaning that the base is actually the NULL constant and therefore + // - nullptr: Meaning that the base is actually the null constant and therefore // we won't try to load from it. // // - CFG Node: Meaning that the base is a CastPP that was specialized for @@ -890,7 +890,7 @@ void ConnectionGraph::reduce_phi_on_castpp_field_load(Node* curr_castpp, Growabl if (base_t->maybe_null()) { if (base->is_Con()) { - // Nothing todo as bases_for_loads[i] is already nullptr + // Nothing todo as bases_for_loads[i] is already null } else { Node* new_castpp = specialize_castpp(curr_castpp, base, ophi->in(0)->in(i)); bases_for_loads.at_put(i, new_castpp->in(0)); // Use the ctrl of the new node just as a flag diff --git a/src/hotspot/share/opto/gcm.cpp b/src/hotspot/share/opto/gcm.cpp index c46d69058e9..c9eb418b4f0 100644 --- a/src/hotspot/share/opto/gcm.cpp +++ b/src/hotspot/share/opto/gcm.cpp @@ -1681,7 +1681,7 @@ void PhaseCFG::global_code_motion() { PhaseIFG ifg(&live_arena); if (OptoRegScheduling && block_size_threshold_ok) { regalloc.mark_ssa(); - Compile::TracePhase tp("computeLive", &timers[_t_computeLive]); + Compile::TracePhase tp(_t_computeLive); rm_live.reset_to_mark(); // Reclaim working storage IndexSet::reset_memory(C, &live_arena); uint node_size = regalloc._lrg_map.max_lrg_id(); diff --git a/src/hotspot/share/opto/generateOptoStub.cpp b/src/hotspot/share/opto/generateOptoStub.cpp index 2f940439045..2e3dc878c84 100644 --- a/src/hotspot/share/opto/generateOptoStub.cpp +++ b/src/hotspot/share/opto/generateOptoStub.cpp @@ -101,7 +101,7 @@ void GraphKit::gen_stub(address C_function, // Node *adr_sp = basic_plus_adr(top(), thread, in_bytes(JavaThread::last_Java_sp_offset())); Node *last_sp = frameptr(); - store_to_memory(control(), adr_sp, last_sp, T_ADDRESS, NoAlias, MemNode::unordered); + store_to_memory(control(), adr_sp, last_sp, T_ADDRESS, MemNode::unordered); // Set _thread_in_native // The order of stores into TLS is critical! Setting _thread_in_native MUST @@ -221,12 +221,12 @@ void GraphKit::gen_stub(address C_function, //----------------------------- // Clear last_Java_sp - store_to_memory(control(), adr_sp, null(), T_ADDRESS, NoAlias, MemNode::unordered); + store_to_memory(control(), adr_sp, null(), T_ADDRESS, MemNode::unordered); // Clear last_Java_pc - store_to_memory(control(), adr_last_Java_pc, null(), T_ADDRESS, NoAlias, MemNode::unordered); + store_to_memory(control(), adr_last_Java_pc, null(), T_ADDRESS, MemNode::unordered); #if (defined(IA64) && !defined(AIX)) Node* adr_last_Java_fp = basic_plus_adr(top(), thread, in_bytes(JavaThread::last_Java_fp_offset())); - store_to_memory(control(), adr_last_Java_fp, null(), T_ADDRESS, NoAlias, MemNode::unordered); + store_to_memory(control(), adr_last_Java_fp, null(), T_ADDRESS, MemNode::unordered); #endif // For is-fancy-jump, the C-return value is also the branch target @@ -234,16 +234,16 @@ void GraphKit::gen_stub(address C_function, // Runtime call returning oop in TLS? Fetch it out if( pass_tls ) { Node* adr = basic_plus_adr(top(), thread, in_bytes(JavaThread::vm_result_offset())); - Node* vm_result = make_load(nullptr, adr, TypeOopPtr::BOTTOM, T_OBJECT, NoAlias, MemNode::unordered); + Node* vm_result = make_load(nullptr, adr, TypeOopPtr::BOTTOM, T_OBJECT, MemNode::unordered); map()->set_req(TypeFunc::Parms, vm_result); // vm_result passed as result // clear thread-local-storage(tls) - store_to_memory(control(), adr, null(), T_ADDRESS, NoAlias, MemNode::unordered); + store_to_memory(control(), adr, null(), T_ADDRESS, MemNode::unordered); } //----------------------------- // check exception Node* adr = basic_plus_adr(top(), thread, in_bytes(Thread::pending_exception_offset())); - Node* pending = make_load(nullptr, adr, TypeOopPtr::BOTTOM, T_OBJECT, NoAlias, MemNode::unordered); + Node* pending = make_load(nullptr, adr, TypeOopPtr::BOTTOM, T_OBJECT, MemNode::unordered); Node* exit_memory = reset_memory(); diff --git a/src/hotspot/share/opto/graphKit.cpp b/src/hotspot/share/opto/graphKit.cpp index ac74abd7e55..0cc68c42639 100644 --- a/src/hotspot/share/opto/graphKit.cpp +++ b/src/hotspot/share/opto/graphKit.cpp @@ -510,7 +510,7 @@ void GraphKit::uncommon_trap_if_should_post_on_exceptions(Deoptimization::DeoptR // first must access the should_post_on_exceptions_flag in this thread's JavaThread Node* jthread = _gvn.transform(new ThreadLocalNode()); Node* adr = basic_plus_adr(top(), jthread, in_bytes(JavaThread::should_post_on_exceptions_flag_offset())); - Node* should_post_flag = make_load(control(), adr, TypeInt::INT, T_INT, Compile::AliasIdxRaw, MemNode::unordered); + Node* should_post_flag = make_load(control(), adr, TypeInt::INT, T_INT, MemNode::unordered); // Test the should_post_on_exceptions_flag vs. 0 Node* chk = _gvn.transform( new CmpINode(should_post_flag, intcon(0)) ); @@ -1550,7 +1550,6 @@ void GraphKit::set_all_memory_call(Node* call, bool separate_io_proj) { // factory methods in "int adr_idx" Node* GraphKit::make_load(Node* ctl, Node* adr, const Type* t, BasicType bt, - int adr_idx, MemNode::MemOrd mo, LoadNode::ControlDependency control_dependency, bool require_atomic_access, @@ -1558,7 +1557,7 @@ Node* GraphKit::make_load(Node* ctl, Node* adr, const Type* t, BasicType bt, bool mismatched, bool unsafe, uint8_t barrier_data) { - assert(adr_idx == C->get_alias_index(_gvn.type(adr)->isa_ptr()), "slice of address and input slice don't match"); + int adr_idx = C->get_alias_index(_gvn.type(adr)->isa_ptr()); assert(adr_idx != Compile::AliasIdxTop, "use other make_load factory" ); const TypePtr* adr_type = nullptr; // debug-mode-only argument debug_only(adr_type = C->get_adr_type(adr_idx)); @@ -1580,15 +1579,14 @@ Node* GraphKit::make_load(Node* ctl, Node* adr, const Type* t, BasicType bt, } Node* GraphKit::store_to_memory(Node* ctl, Node* adr, Node *val, BasicType bt, - int adr_idx, MemNode::MemOrd mo, bool require_atomic_access, bool unaligned, bool mismatched, bool unsafe, int barrier_data) { + int adr_idx = C->get_alias_index(_gvn.type(adr)->isa_ptr()); assert(adr_idx != Compile::AliasIdxTop, "use other store_to_memory factory" ); - assert(adr_idx == C->get_alias_index(_gvn.type(adr)->isa_ptr()), "slice of address and input slice don't match"); const TypePtr* adr_type = nullptr; debug_only(adr_type = C->get_adr_type(adr_idx)); Node *mem = memory(adr_idx); @@ -2044,11 +2042,10 @@ void GraphKit::increment_counter(address counter_addr) { } void GraphKit::increment_counter(Node* counter_addr) { - int adr_type = Compile::AliasIdxRaw; Node* ctrl = control(); - Node* cnt = make_load(ctrl, counter_addr, TypeLong::LONG, T_LONG, adr_type, MemNode::unordered); + Node* cnt = make_load(ctrl, counter_addr, TypeLong::LONG, T_LONG, MemNode::unordered); Node* incr = _gvn.transform(new AddLNode(cnt, _gvn.longcon(1))); - store_to_memory(ctrl, counter_addr, incr, T_LONG, adr_type, MemNode::unordered); + store_to_memory(ctrl, counter_addr, incr, T_LONG, MemNode::unordered); } @@ -4240,8 +4237,8 @@ void GraphKit::inflate_string_slow(Node* src, Node* dst, Node* start, Node* coun set_memory(mem, TypeAryPtr::BYTES); Node* ch = load_array_element(src, i_byte, TypeAryPtr::BYTES, /* set_ctrl */ true); Node* st = store_to_memory(control(), array_element_address(dst, i_char, T_BYTE), - AndI(ch, intcon(0xff)), T_CHAR, TypeAryPtr::BYTES, MemNode::unordered, - false, false, true /* mismatched */); + AndI(ch, intcon(0xff)), T_CHAR, MemNode::unordered, false, + false, true /* mismatched */); IfNode* iff = create_and_map_if(head, Bool(CmpI(i_byte, count), BoolTest::lt), PROB_FAIR, COUNT_UNKNOWN); head->init_req(2, IfTrue(iff)); diff --git a/src/hotspot/share/opto/graphKit.hpp b/src/hotspot/share/opto/graphKit.hpp index 3333b7d1bd9..e91ae769895 100644 --- a/src/hotspot/share/opto/graphKit.hpp +++ b/src/hotspot/share/opto/graphKit.hpp @@ -540,26 +540,6 @@ class GraphKit : public Phase { // adapted the `do_put_xxx' and `do_get_xxx' procedures for the case // of volatile fields. Node* make_load(Node* ctl, Node* adr, const Type* t, BasicType bt, - MemNode::MemOrd mo, LoadNode::ControlDependency control_dependency = LoadNode::DependsOnlyOnTest, - bool require_atomic_access = false, bool unaligned = false, - bool mismatched = false, bool unsafe = false, uint8_t barrier_data = 0) { - // This version computes alias_index from bottom_type - return make_load(ctl, adr, t, bt, adr->bottom_type()->is_ptr(), - mo, control_dependency, require_atomic_access, - unaligned, mismatched, unsafe, barrier_data); - } - Node* make_load(Node* ctl, Node* adr, const Type* t, BasicType bt, const TypePtr* adr_type, - MemNode::MemOrd mo, LoadNode::ControlDependency control_dependency = LoadNode::DependsOnlyOnTest, - bool require_atomic_access = false, bool unaligned = false, - bool mismatched = false, bool unsafe = false, uint8_t barrier_data = 0) { - // This version computes alias_index from an address type - assert(adr_type != nullptr, "use other make_load factory"); - return make_load(ctl, adr, t, bt, C->get_alias_index(adr_type), - mo, control_dependency, require_atomic_access, - unaligned, mismatched, unsafe, barrier_data); - } - // This is the base version which is given an alias index. - Node* make_load(Node* ctl, Node* adr, const Type* t, BasicType bt, int adr_idx, MemNode::MemOrd mo, LoadNode::ControlDependency control_dependency = LoadNode::DependsOnlyOnTest, bool require_atomic_access = false, bool unaligned = false, bool mismatched = false, bool unsafe = false, uint8_t barrier_data = 0); @@ -572,26 +552,8 @@ class GraphKit : public Phase { // procedure must indicate that the store requires `release' // semantics, if the stored value is an object reference that might // point to a new object and may become externally visible. - Node* store_to_memory(Node* ctl, Node* adr, Node* val, BasicType bt, - const TypePtr* adr_type, - MemNode::MemOrd mo, - bool require_atomic_access = false, - bool unaligned = false, - bool mismatched = false, - bool unsafe = false, - int barrier_data = 0) { - // This version computes alias_index from an address type - assert(adr_type != nullptr, "use other store_to_memory factory"); - return store_to_memory(ctl, adr, val, bt, - C->get_alias_index(adr_type), - mo, require_atomic_access, - unaligned, mismatched, unsafe, - barrier_data); - } - // This is the base version which is given alias index // Return the new StoreXNode Node* store_to_memory(Node* ctl, Node* adr, Node* val, BasicType bt, - int adr_idx, MemNode::MemOrd, bool require_atomic_access = false, bool unaligned = false, diff --git a/src/hotspot/share/opto/ifg.cpp b/src/hotspot/share/opto/ifg.cpp index b11fb3deb44..d12698121b9 100644 --- a/src/hotspot/share/opto/ifg.cpp +++ b/src/hotspot/share/opto/ifg.cpp @@ -313,7 +313,7 @@ void PhaseChaitin::interfere_with_live(uint lid, IndexSet* liveout) { // at this point can end up in bad places). Copies I re-insert later I have // more opportunity to insert them in low-frequency locations. void PhaseChaitin::build_ifg_virtual( ) { - Compile::TracePhase tp("buildIFG_virt", &timers[_t_buildIFGvirtual]); + Compile::TracePhase tp(_t_buildIFGvirtual); // For all blocks (in any order) do... for (uint i = 0; i < _cfg.number_of_blocks(); i++) { @@ -843,7 +843,7 @@ void PhaseChaitin::print_pressure_info(Pressure& pressure, const char *str) { * low to high register pressure transition within the block (if any). */ uint PhaseChaitin::build_ifg_physical( ResourceArea *a ) { - Compile::TracePhase tp("buildIFG", &timers[_t_buildIFGphysical]); + Compile::TracePhase tp(_t_buildIFGphysical); uint must_spill = 0; for (uint i = 0; i < _cfg.number_of_blocks(); i++) { diff --git a/src/hotspot/share/opto/ifnode.cpp b/src/hotspot/share/opto/ifnode.cpp index 093cafd1574..c0ed3f7e5d4 100644 --- a/src/hotspot/share/opto/ifnode.cpp +++ b/src/hotspot/share/opto/ifnode.cpp @@ -50,12 +50,11 @@ extern uint explicit_null_checks_elided; IfNode::IfNode(Node* control, Node* bol, float p, float fcnt) : MultiBranchNode(2), _prob(p), - _fcnt(fcnt) - NOT_PRODUCT(COMMA _assertion_predicate_type(AssertionPredicateType::None)) { + _fcnt(fcnt), + _assertion_predicate_type(AssertionPredicateType::None) { init_node(control, bol); } -#ifndef PRODUCT IfNode::IfNode(Node* control, Node* bol, float p, float fcnt, AssertionPredicateType assertion_predicate_type) : MultiBranchNode(2), _prob(p), @@ -63,7 +62,6 @@ IfNode::IfNode(Node* control, Node* bol, float p, float fcnt, AssertionPredicate _assertion_predicate_type(assertion_predicate_type) { init_node(control, bol); } -#endif // NOT_PRODUCT //============================================================================= //------------------------------Value------------------------------------------ @@ -1524,6 +1522,14 @@ Node* IfNode::Ideal(PhaseGVN *phase, bool can_reshape) { Node* prev_dom = search_identical(dist, igvn); if (prev_dom != nullptr) { + // Dominating CountedLoopEnd (left over from some now dead loop) will become the new loop exit. Outer strip mined + // loop will go away. Mark this loop as no longer strip mined. + if (is_CountedLoopEnd()) { + CountedLoopNode* counted_loop_node = as_CountedLoopEnd()->loopnode(); + if (counted_loop_node != nullptr) { + counted_loop_node->clear_strip_mined(); + } + } // Replace dominated IfNode return dominated_by(prev_dom, igvn, false); } diff --git a/src/hotspot/share/opto/lcm.cpp b/src/hotspot/share/opto/lcm.cpp index 2a40cf000d8..88dc00ab463 100644 --- a/src/hotspot/share/opto/lcm.cpp +++ b/src/hotspot/share/opto/lcm.cpp @@ -275,8 +275,8 @@ void PhaseCFG::implicit_null_check(Block* block, Node *proj, Node *val, int allo // cannot reason about it; is probably not implicit null exception } else { const TypePtr* tptr; - if ((UseCompressedOops || UseCompressedClassPointers) && - (CompressedOops::shift() == 0 || CompressedKlassPointers::shift() == 0)) { + if ((UseCompressedOops && CompressedOops::shift() == 0) || + (UseCompressedClassPointers && CompressedKlassPointers::shift() == 0)) { // 32-bits narrow oop can be the base of address expressions tptr = base->get_ptr_type(); } else { diff --git a/src/hotspot/share/opto/library_call.cpp b/src/hotspot/share/opto/library_call.cpp index 523b1efd0c2..226fddce29b 100644 --- a/src/hotspot/share/opto/library_call.cpp +++ b/src/hotspot/share/opto/library_call.cpp @@ -1288,7 +1288,6 @@ bool LibraryCallKit::inline_string_indexOfI(StrIntrinsicNode::ArgEnc ae) { bool call_opt_stub = (StubRoutines::_string_indexof_array[ae] != nullptr); if (call_opt_stub) { - assert(arrayOopDesc::base_offset_in_bytes(T_BYTE) >= 16, "Needed for indexOf"); Node* call = make_runtime_call(RC_LEAF, OptoRuntime::string_IndexOf_Type(), StubRoutines::_string_indexof_array[ae], "stringIndexOf", TypePtr::BOTTOM, src_start, @@ -3110,7 +3109,7 @@ bool LibraryCallKit::inline_native_jvm_commit() { set_control(is_notified); // Reset notified state. - Node* notified_reset_memory = store_to_memory(control(), notified_offset, _gvn.intcon(0), T_BOOLEAN, Compile::AliasIdxRaw, MemNode::unordered); + Node* notified_reset_memory = store_to_memory(control(), notified_offset, _gvn.intcon(0), T_BOOLEAN, MemNode::unordered); // Iff notified, the return address of the commit method is the current position of the backing java buffer. This is used to reset the event writer. Node* current_pos_X = _gvn.transform(new LoadXNode(control(), input_memory_state, java_buffer_pos_offset, TypeRawPtr::NOTNULL, TypeX_X, MemNode::unordered)); @@ -3130,9 +3129,9 @@ bool LibraryCallKit::inline_native_jvm_commit() { // Store the next_position to the underlying jfr java buffer. Node* commit_memory; #ifdef _LP64 - commit_memory = store_to_memory(control(), java_buffer_pos_offset, next_pos_X, T_LONG, Compile::AliasIdxRaw, MemNode::release); + commit_memory = store_to_memory(control(), java_buffer_pos_offset, next_pos_X, T_LONG, MemNode::release); #else - commit_memory = store_to_memory(control(), java_buffer_pos_offset, next_pos_X, T_INT, Compile::AliasIdxRaw, MemNode::release); + commit_memory = store_to_memory(control(), java_buffer_pos_offset, next_pos_X, T_INT, MemNode::release); #endif // Now load the flags from off the java buffer and decide if the buffer is a lease. If so, it needs to be returned post-commit. @@ -3449,13 +3448,10 @@ bool LibraryCallKit::inline_native_getEventWriter() { Node* const event_writer_tid = load_field_from_object(event_writer, "threadID", "J"); // Get the field offset to, conditionally, store an updated tid value later. Node* const event_writer_tid_field = field_address_from_object(event_writer, "threadID", "J", false); - const TypePtr* event_writer_tid_field_type = _gvn.type(event_writer_tid_field)->isa_ptr(); // Get the field offset to, conditionally, store an updated exclusion value later. Node* const event_writer_excluded_field = field_address_from_object(event_writer, "excluded", "Z", false); - const TypePtr* event_writer_excluded_field_type = _gvn.type(event_writer_excluded_field)->isa_ptr(); // Get the field offset to, conditionally, store an updated pinVirtualThread value later. Node* const event_writer_pin_field = field_address_from_object(event_writer, "pinVirtualThread", "Z", false); - const TypePtr* event_writer_pin_field_type = _gvn.type(event_writer_pin_field)->isa_ptr(); RegionNode* event_writer_tid_compare_rgn = new RegionNode(PATH_LIMIT); record_for_igvn(event_writer_tid_compare_rgn); @@ -3477,13 +3473,13 @@ bool LibraryCallKit::inline_native_getEventWriter() { record_for_igvn(tid_is_not_equal); // Store the pin state to the event writer. - store_to_memory(tid_is_not_equal, event_writer_pin_field, _gvn.transform(pinVirtualThread), T_BOOLEAN, event_writer_pin_field_type, MemNode::unordered); + store_to_memory(tid_is_not_equal, event_writer_pin_field, _gvn.transform(pinVirtualThread), T_BOOLEAN, MemNode::unordered); // Store the exclusion state to the event writer. - store_to_memory(tid_is_not_equal, event_writer_excluded_field, _gvn.transform(exclusion), T_BOOLEAN, event_writer_excluded_field_type, MemNode::unordered); + store_to_memory(tid_is_not_equal, event_writer_excluded_field, _gvn.transform(exclusion), T_BOOLEAN, MemNode::unordered); // Store the tid to the event writer. - store_to_memory(tid_is_not_equal, event_writer_tid_field, tid, T_LONG, event_writer_tid_field_type, MemNode::unordered); + store_to_memory(tid_is_not_equal, event_writer_tid_field, tid, T_LONG, MemNode::unordered); // Update control and phi nodes. event_writer_tid_compare_rgn->init_req(_true_path, tid_is_not_equal); @@ -3562,7 +3558,7 @@ void LibraryCallKit::extend_setCurrentThread(Node* jt, Node* thread) { // False branch, is carrierThread. Node* thread_equal_carrierThread = _gvn.transform(new IfFalseNode(iff_thread_not_equal_carrierThread)); // Store release - Node* vthread_false_memory = store_to_memory(thread_equal_carrierThread, vthread_offset, _gvn.intcon(0), T_BOOLEAN, Compile::AliasIdxRaw, MemNode::release, true); + Node* vthread_false_memory = store_to_memory(thread_equal_carrierThread, vthread_offset, _gvn.intcon(0), T_BOOLEAN, MemNode::release, true); set_all_memory(input_memory_state); @@ -3583,7 +3579,7 @@ void LibraryCallKit::extend_setCurrentThread(Node* jt, Node* thread) { // Store the vthread tid to the jfr thread local. Node* thread_id_offset = basic_plus_adr(jt, in_bytes(THREAD_LOCAL_OFFSET_JFR + VTHREAD_ID_OFFSET_JFR)); - Node* tid_memory = store_to_memory(control(), thread_id_offset, tid, T_LONG, Compile::AliasIdxRaw, MemNode::unordered, true); + Node* tid_memory = store_to_memory(control(), thread_id_offset, tid, T_LONG, MemNode::unordered, true); // Branch is_excluded to conditionalize updating the epoch . Node* excluded_cmp = _gvn.transform(new CmpINode(is_excluded, _gvn.transform(excluded_mask))); @@ -3605,7 +3601,7 @@ void LibraryCallKit::extend_setCurrentThread(Node* jt, Node* thread) { // Store the vthread epoch to the jfr thread local. Node* vthread_epoch_offset = basic_plus_adr(jt, in_bytes(THREAD_LOCAL_OFFSET_JFR + VTHREAD_EPOCH_OFFSET_JFR)); - Node* included_memory = store_to_memory(control(), vthread_epoch_offset, epoch, T_CHAR, Compile::AliasIdxRaw, MemNode::unordered, true); + Node* included_memory = store_to_memory(control(), vthread_epoch_offset, epoch, T_CHAR, MemNode::unordered, true); RegionNode* excluded_rgn = new RegionNode(PATH_LIMIT); record_for_igvn(excluded_rgn); @@ -3628,10 +3624,10 @@ void LibraryCallKit::extend_setCurrentThread(Node* jt, Node* thread) { // Store the vthread exclusion state to the jfr thread local. Node* thread_local_excluded_offset = basic_plus_adr(jt, in_bytes(THREAD_LOCAL_OFFSET_JFR + VTHREAD_EXCLUDED_OFFSET_JFR)); - store_to_memory(control(), thread_local_excluded_offset, _gvn.transform(exclusion), T_BOOLEAN, Compile::AliasIdxRaw, MemNode::unordered, true); + store_to_memory(control(), thread_local_excluded_offset, _gvn.transform(exclusion), T_BOOLEAN, MemNode::unordered, true); // Store release - Node * vthread_true_memory = store_to_memory(control(), vthread_offset, _gvn.intcon(1), T_BOOLEAN, Compile::AliasIdxRaw, MemNode::release, true); + Node * vthread_true_memory = store_to_memory(control(), vthread_offset, _gvn.intcon(1), T_BOOLEAN, MemNode::release, true); RegionNode* thread_compare_rgn = new RegionNode(PATH_LIMIT); record_for_igvn(thread_compare_rgn); @@ -3679,6 +3675,12 @@ bool LibraryCallKit::inline_native_setCurrentThread() { thread_obj_handle = _gvn.transform(thread_obj_handle); const TypePtr *adr_type = _gvn.type(thread_obj_handle)->isa_ptr(); access_store_at(nullptr, thread_obj_handle, adr_type, arr, _gvn.type(arr), T_OBJECT, IN_NATIVE | MO_UNORDERED); + + // Change the lock_id of the JavaThread + Node* tid = load_field_from_object(arr, "tid", "J"); + Node* thread_id_offset = basic_plus_adr(thread, in_bytes(JavaThread::lock_id_offset())); + Node* tid_memory = store_to_memory(control(), thread_id_offset, tid, T_LONG, MemNode::unordered, true); + JFR_ONLY(extend_setCurrentThread(thread, arr);) return true; } @@ -3781,7 +3783,7 @@ bool LibraryCallKit::inline_native_Continuation_pinning(bool unpin) { next_pin_count = _gvn.transform(new AddINode(pin_count, _gvn.intcon(1))); } - Node* updated_pin_count_memory = store_to_memory(control(), pin_count_offset, next_pin_count, T_INT, Compile::AliasIdxRaw, MemNode::unordered); + Node* updated_pin_count_memory = store_to_memory(control(), pin_count_offset, next_pin_count, T_INT, MemNode::unordered); // True branch, pin count over/underflow. Node* pin_count_over_underflow = _gvn.transform(new IfTrueNode(iff_pin_count_over_underflow)); @@ -5058,7 +5060,7 @@ bool LibraryCallKit::inline_unsafe_copyMemory() { assert((sizeof(bool) * CHAR_BIT) == 8, "not implemented"); // update volatile field - store_to_memory(control(), doing_unsafe_access_addr, intcon(1), doing_unsafe_access_bt, Compile::AliasIdxRaw, MemNode::unordered); + store_to_memory(control(), doing_unsafe_access_addr, intcon(1), doing_unsafe_access_bt, MemNode::unordered); int flags = RC_LEAF | RC_NO_FP; @@ -5083,7 +5085,7 @@ bool LibraryCallKit::inline_unsafe_copyMemory() { dst_type, src_addr, dst_addr, size XTOP); - store_to_memory(control(), doing_unsafe_access_addr, intcon(0), doing_unsafe_access_bt, Compile::AliasIdxRaw, MemNode::unordered); + store_to_memory(control(), doing_unsafe_access_addr, intcon(0), doing_unsafe_access_bt, MemNode::unordered); return true; } @@ -5113,7 +5115,7 @@ bool LibraryCallKit::inline_unsafe_setMemory() { assert((sizeof(bool) * CHAR_BIT) == 8, "not implemented"); // update volatile field - store_to_memory(control(), doing_unsafe_access_addr, intcon(1), doing_unsafe_access_bt, Compile::AliasIdxRaw, MemNode::unordered); + store_to_memory(control(), doing_unsafe_access_addr, intcon(1), doing_unsafe_access_bt, MemNode::unordered); int flags = RC_LEAF | RC_NO_FP; @@ -5134,7 +5136,7 @@ bool LibraryCallKit::inline_unsafe_setMemory() { dst_type, dst_addr, size XTOP, byte); - store_to_memory(control(), doing_unsafe_access_addr, intcon(0), doing_unsafe_access_bt, Compile::AliasIdxRaw, MemNode::unordered); + store_to_memory(control(), doing_unsafe_access_addr, intcon(0), doing_unsafe_access_bt, MemNode::unordered); return true; } @@ -7017,6 +7019,8 @@ Node* LibraryCallKit::load_field_from_object(Node* fromObj, const char* fieldNam assert(field_klass->is_loaded(), "should be loaded"); const TypePtr* adr_type = C->alias_type(field)->adr_type(); Node *adr = basic_plus_adr(fromObj, fromObj, offset); + assert(C->get_alias_index(adr_type) == C->get_alias_index(_gvn.type(adr)->isa_ptr()), + "slice of address and input slice don't match"); BasicType bt = field->layout_type(); // Build the resultant type of the load @@ -7750,11 +7754,11 @@ bool LibraryCallKit::inline_intpoly_montgomeryMult_P256() { r = must_be_not_null(r, true); Node* a_start = array_element_address(a, intcon(0), T_LONG); - assert(a_start, "a array is NULL"); + assert(a_start, "a array is null"); Node* b_start = array_element_address(b, intcon(0), T_LONG); - assert(b_start, "b array is NULL"); + assert(b_start, "b array is null"); Node* r_start = array_element_address(r, intcon(0), T_LONG); - assert(r_start, "r array is NULL"); + assert(r_start, "r array is null"); Node* call = make_runtime_call(RC_LEAF | RC_NO_FP, OptoRuntime::intpoly_montgomeryMult_P256_Type(), @@ -7779,9 +7783,9 @@ bool LibraryCallKit::inline_intpoly_assign() { b = must_be_not_null(b, true); Node* a_start = array_element_address(a, intcon(0), T_LONG); - assert(a_start, "a array is NULL"); + assert(a_start, "a array is null"); Node* b_start = array_element_address(b, intcon(0), T_LONG); - assert(b_start, "b array is NULL"); + assert(b_start, "b array is null"); Node* call = make_runtime_call(RC_LEAF | RC_NO_FP, OptoRuntime::intpoly_assign_Type(), diff --git a/src/hotspot/share/opto/loopPredicate.cpp b/src/hotspot/share/opto/loopPredicate.cpp index 9a639b1f9a1..a4bec870b18 100644 --- a/src/hotspot/share/opto/loopPredicate.cpp +++ b/src/hotspot/share/opto/loopPredicate.cpp @@ -100,8 +100,8 @@ void PhaseIdealLoop::register_control(Node* n, IdealLoopTree *loop, Node* pred, // is an IfTrue projection. This code is also used to clone predicates to cloned loops. IfTrueNode* PhaseIdealLoop::create_new_if_for_predicate(ParsePredicateSuccessProj* parse_predicate_success_proj, Node* new_entry, const Deoptimization::DeoptReason reason, - const int opcode, const bool rewire_uncommon_proj_phi_inputs - NOT_PRODUCT (COMMA AssertionPredicateType assertion_predicate_type)) { + const int opcode, const bool rewire_uncommon_proj_phi_inputs, + AssertionPredicateType assertion_predicate_type) { assert(parse_predicate_success_proj->is_uncommon_trap_if_pattern(reason), "must be a uct if pattern!"); ParsePredicateNode* parse_predicate = parse_predicate_success_proj->in(0)->as_ParsePredicate(); ParsePredicateUncommonProj* uncommon_proj = parse_predicate->uncommon_proj(); @@ -288,7 +288,7 @@ IfProjNode* PhaseIdealLoop::clone_parse_predicate_to_unswitched_loop(ParsePredic // cloned predicates. void PhaseIdealLoop::clone_assertion_predicates_to_unswitched_loop(IdealLoopTree* loop, const Node_List& old_new, Deoptimization::DeoptReason reason, - IfProjNode* old_predicate_proj, + ParsePredicateSuccessProj* old_parse_predicate_proj, ParsePredicateSuccessProj* fast_loop_parse_predicate_proj, ParsePredicateSuccessProj* slow_loop_parse_predicate_proj) { assert(fast_loop_parse_predicate_proj->in(0)->is_ParsePredicate() && @@ -297,17 +297,15 @@ void PhaseIdealLoop::clone_assertion_predicates_to_unswitched_loop(IdealLoopTree // and doing loop unrolling. Push the original predicates on a list to later process them in reverse order to keep the // original predicate order. Unique_Node_List list; - get_assertion_predicates(old_predicate_proj, list); + get_assertion_predicates(old_parse_predicate_proj, list); Node_List to_process; - IfNode* iff = old_predicate_proj->in(0)->as_If(); - IfProjNode* uncommon_proj = iff->proj_out(1 - old_predicate_proj->as_Proj()->_con)->as_IfProj(); // Process in reverse order such that 'create_new_if_for_predicate' can be used in // 'clone_assertion_predicate_for_unswitched_loops' and the original order is maintained. for (int i = list.size() - 1; i >= 0; i--) { Node* predicate = list.at(i); assert(predicate->in(0)->is_If(), "must be If node"); - iff = predicate->in(0)->as_If(); + IfNode* iff = predicate->in(0)->as_If(); assert(predicate->is_Proj() && predicate->as_Proj()->is_IfProj(), "predicate must be a projection of an if node"); IfProjNode* predicate_proj = predicate->as_IfProj(); @@ -337,34 +335,14 @@ void PhaseIdealLoop::clone_assertion_predicates_to_unswitched_loop(IdealLoopTree } // Put all Assertion Predicate projections on a list, starting at 'predicate' and going up in the tree. If 'get_opaque' -// is set, then the OpaqueTemplateAssertionPredicate nodes of the Assertion Predicates are put on the list instead of -// the projections. -void PhaseIdealLoop::get_assertion_predicates(Node* predicate, Unique_Node_List& list, bool get_opaque) { - ParsePredicateNode* parse_predicate = predicate->in(0)->as_ParsePredicate(); - ProjNode* uncommon_proj = parse_predicate->proj_out(1 - predicate->as_Proj()->_con); - Node* rgn = uncommon_proj->unique_ctrl_out(); - assert(rgn->is_Region() || rgn->is_Call(), "must be a region or call uct"); - predicate = parse_predicate->in(0); - while (predicate != nullptr && predicate->is_Proj() && predicate->in(0)->is_If()) { - IfNode* iff = predicate->in(0)->as_If(); - uncommon_proj = iff->proj_out(1 - predicate->as_Proj()->_con); - if (uncommon_proj->unique_ctrl_out() != rgn) { - break; - } - Node* bol = iff->in(1); - assert(!bol->is_OpaqueInitializedAssertionPredicate(), "should not find an Initialized Assertion Predicate"); - if (bol->is_OpaqueTemplateAssertionPredicate()) { - assert(assertion_predicate_has_loop_opaque_node(iff), "must find OpaqueLoop* nodes"); - if (get_opaque) { - // Collect the OpaqueTemplateAssertionPredicateNode. - list.push(bol); - } else { - // Collect the predicate projection. - list.push(predicate); - } - } - predicate = predicate->in(0)->in(0); - } +// is set, then the OpaqueTemplateAssertionPredicateNode nodes of the Assertion Predicates are put on the list instead +// of the projections. +void PhaseIdealLoop::get_assertion_predicates(ParsePredicateSuccessProj* parse_predicate_proj, Unique_Node_List& list, + const bool get_opaque) { + Deoptimization::DeoptReason deopt_reason = parse_predicate_proj->in(0)->as_ParsePredicate()->deopt_reason(); + PredicateBlockIterator predicate_iterator(parse_predicate_proj, deopt_reason); + TemplateAssertionPredicateCollector template_assertion_predicate_collector(list, get_opaque); + predicate_iterator.for_each(template_assertion_predicate_collector); } // Clone an Assertion Predicate for an unswitched loop. OpaqueLoopInit and OpaqueLoopStride nodes are cloned and uncommon diff --git a/src/hotspot/share/opto/loopTransform.cpp b/src/hotspot/share/opto/loopTransform.cpp index 0ad60c80c2d..9175d79e6dd 100644 --- a/src/hotspot/share/opto/loopTransform.cpp +++ b/src/hotspot/share/opto/loopTransform.cpp @@ -758,7 +758,6 @@ void PhaseIdealLoop::do_peeling(IdealLoopTree *loop, Node_List &old_new) { #endif } } - Node* entry = head->in(LoopNode::EntryControl); // Step 1: Clone the loop body. The clone becomes the peeled iteration. // The pre-loop illegally has 2 control users (old & new loops). @@ -1109,7 +1108,7 @@ void IdealLoopTree::policy_unroll_slp_analysis(CountedLoopNode *cl, PhaseIdealLo // Enable this functionality target by target as needed if (SuperWordLoopUnrollAnalysis) { if (!cl->was_slp_analyzed()) { - Compile::TracePhase tp("autoVectorize", &Phase::timers[Phase::_t_autoVectorize]); + Compile::TracePhase tp(Phase::_t_autoVectorize); VLoop vloop(this, true); if (vloop.check_preconditions()) { @@ -1780,47 +1779,19 @@ bool IdealLoopTree::is_invariant(Node* n) const { // Search the Assertion Predicates added by loop predication and/or range check elimination and update them according // to the new stride. -void PhaseIdealLoop::update_main_loop_assertion_predicates(Node* ctrl, CountedLoopNode* loop_head, Node* init, - const int stride_con) { - Node* entry = ctrl; - Node* prev_proj = ctrl; - LoopNode* outer_loop_head = loop_head->skip_strip_mined(); - IdealLoopTree* outer_loop = get_loop(outer_loop_head); +void PhaseIdealLoop::update_main_loop_assertion_predicates(CountedLoopNode* main_loop_head) { + Node* init = main_loop_head->init_trip(); // Compute the value of the loop induction variable at the end of the // first iteration of the unrolled loop: init + new_stride_con - init_inc - int new_stride_con = stride_con * 2; - Node* max_value = _igvn.intcon(new_stride_con); - set_ctrl(max_value, C->root()); - - while (entry != nullptr && entry->is_Proj() && entry->in(0)->is_If()) { - IfNode* iff = entry->in(0)->as_If(); - ProjNode* proj = iff->proj_out(1 - entry->as_Proj()->_con); - if (!proj->unique_ctrl_out()->is_Halt()) { - break; - } - Node* bol = iff->in(1); - if (bol->is_OpaqueTemplateAssertionPredicate()) { - assert(assertion_predicate_has_loop_opaque_node(iff), "must find OpaqueLoop* nodes"); - // This is a Template Assertion Predicate for the initial or last access. - // Create an Initialized Assertion Predicates for it accordingly: - // - For the initial access a[init] (same as before) - // - For the last access a[init+new_stride-orig_stride] (with the new unroll stride) - prev_proj = create_initialized_assertion_predicate(iff, init, max_value, prev_proj); - } else if (bol->is_OpaqueInitializedAssertionPredicate()) { - // This is one of the two Initialized Assertion Predicates: - // - For the initial access a[init] - // - For the last access a[init+old_stride-orig_stride] - // We could keep the one for the initial access but we do not know which one we currently have here. Just kill both. - _igvn.replace_input_of(iff, 1, _igvn.intcon(1)); - } - assert(!bol->is_OpaqueNotNull() || !loop_head->is_main_loop(), "OpaqueNotNull should not be at main loop"); - entry = entry->in(0)->in(0); - } - if (prev_proj != ctrl) { - _igvn.replace_input_of(outer_loop_head, LoopNode::EntryControl, prev_proj); - set_idom(outer_loop_head, prev_proj, dom_depth(outer_loop_head)); - } + int unrolled_stride_con = main_loop_head->stride_con() * 2; + Node* unrolled_stride = _igvn.intcon(unrolled_stride_con); + set_ctrl(unrolled_stride, C->root()); + + Node* loop_entry = main_loop_head->skip_strip_mined()->in(LoopNode::EntryControl); + PredicateIterator predicate_iterator(loop_entry); + UpdateStrideForAssertionPredicates update_stride_for_assertion_predicates(unrolled_stride, this); + predicate_iterator.for_each(update_stride_for_assertion_predicates); } // Source Loop: Cloned - peeled_loop_head @@ -1937,7 +1908,7 @@ void PhaseIdealLoop::do_unroll(IdealLoopTree *loop, Node_List &old_new, bool adj assert(old_trip_count > 1 && (!adjust_min_trip || stride_p <= MIN2(max_jint / 2 - 2, MAX2(1<<3, Matcher::max_vector_size(T_BYTE)) * loop_head->unrolled_count())), "sanity"); - update_main_loop_assertion_predicates(ctrl, loop_head, init, stride_con); + update_main_loop_assertion_predicates(loop_head); // Adjust loop limit to keep valid iterations number after unroll. // Use (limit - stride) instead of (((limit - init)/stride) & (-2))*stride @@ -2788,8 +2759,8 @@ void PhaseIdealLoop::do_range_check(IdealLoopTree *loop, Node_List &old_new) { // cannot remove an empty loop with a constant limit when init is not a constant as well. We will use // a LoopLimitCheck node that can only be folded if the zero grip guard is also foldable. loop_entry = initialized_assertion_predicate_creator.create(final_iv_placeholder, loop_entry, stride_con, - scale_con, int_offset, int_limit NOT_PRODUCT( - COMMA AssertionPredicateType::FinalIv)); + scale_con, int_offset, int_limit, + AssertionPredicateType::FinalIv); assert(!assertion_predicate_has_loop_opaque_node(loop_entry->in(0)->as_If()), "unexpected"); } @@ -2802,8 +2773,8 @@ void PhaseIdealLoop::do_range_check(IdealLoopTree *loop, Node_List &old_new) { // Initialized Assertion Predicate for the value of the initial main-loop. loop_entry = initialized_assertion_predicate_creator.create(init, loop_entry, stride_con, scale_con, - int_offset, int_limit NOT_PRODUCT(COMMA - AssertionPredicateType::InitValue)); + int_offset, int_limit, + AssertionPredicateType::InitValue); assert(!assertion_predicate_has_loop_opaque_node(loop_entry->in(0)->as_If()), "unexpected"); } else { diff --git a/src/hotspot/share/opto/loopnode.cpp b/src/hotspot/share/opto/loopnode.cpp index e2db179676d..44a72fda644 100644 --- a/src/hotspot/share/opto/loopnode.cpp +++ b/src/hotspot/share/opto/loopnode.cpp @@ -4469,7 +4469,7 @@ void PhaseIdealLoop::collect_useful_template_assertion_predicates_for_loop(Ideal if (UseProfiledLoopPredicate) { const PredicateBlock* profiled_loop_predicate_block = predicates.profiled_loop_predicate_block(); if (profiled_loop_predicate_block->has_parse_predicate()) { - IfProjNode* parse_predicate_proj = profiled_loop_predicate_block->parse_predicate_success_proj(); + ParsePredicateSuccessProj* parse_predicate_proj = profiled_loop_predicate_block->parse_predicate_success_proj(); get_assertion_predicates(parse_predicate_proj, useful_predicates, true); } } @@ -4477,7 +4477,7 @@ void PhaseIdealLoop::collect_useful_template_assertion_predicates_for_loop(Ideal if (UseLoopPredicate) { const PredicateBlock* loop_predicate_block = predicates.loop_predicate_block(); if (loop_predicate_block->has_parse_predicate()) { - IfProjNode* parse_predicate_proj = loop_predicate_block->parse_predicate_success_proj(); + ParsePredicateSuccessProj* parse_predicate_proj = loop_predicate_block->parse_predicate_success_proj(); get_assertion_predicates(parse_predicate_proj, useful_predicates, true); } } @@ -4971,7 +4971,7 @@ void PhaseIdealLoop::build_and_optimize() { // Auto-vectorize main-loop if (C->do_superword() && C->has_loops() && !C->major_progress()) { - Compile::TracePhase tp("autoVectorize", &timers[_t_autoVectorize]); + Compile::TracePhase tp(_t_autoVectorize); // Shared data structures for all AutoVectorizations, to reduce allocations // of large arrays. diff --git a/src/hotspot/share/opto/loopnode.hpp b/src/hotspot/share/opto/loopnode.hpp index 487c4473280..cc67240975d 100644 --- a/src/hotspot/share/opto/loopnode.hpp +++ b/src/hotspot/share/opto/loopnode.hpp @@ -947,8 +947,8 @@ class PhaseIdealLoop : public PhaseTransform { DEBUG_ONLY(static bool assertion_predicate_has_loop_opaque_node(IfNode* iff);) private: DEBUG_ONLY(static void count_opaque_loop_nodes(Node* n, uint& init, uint& stride);) - static void get_assertion_predicates(Node* predicate, Unique_Node_List& list, bool get_opaque = false); - void update_main_loop_assertion_predicates(Node* ctrl, CountedLoopNode* loop_head, Node* init, int stride_con); + static void get_assertion_predicates(ParsePredicateSuccessProj* parse_predicate_proj, Unique_Node_List& list, bool get_opaque = false); + void update_main_loop_assertion_predicates(CountedLoopNode* main_loop_head); void initialize_assertion_predicates_for_peeled_loop(CountedLoopNode* peeled_loop_head, CountedLoopNode* remaining_loop_head, uint first_node_index_in_cloned_loop_body, @@ -1196,7 +1196,7 @@ class PhaseIdealLoop : public PhaseTransform { static void verify(PhaseIterGVN& igvn) { #ifdef ASSERT ResourceMark rm; - Compile::TracePhase tp("idealLoopVerify", &timers[_t_idealLoopVerify]); + Compile::TracePhase tp(_t_idealLoopVerify); PhaseIdealLoop v(igvn); #endif } @@ -1352,8 +1352,8 @@ class PhaseIdealLoop : public PhaseTransform { // Create a new if above the uncommon_trap_if_pattern for the predicate to be promoted IfTrueNode* create_new_if_for_predicate( ParsePredicateSuccessProj* parse_predicate_proj, Node* new_entry, Deoptimization::DeoptReason reason, int opcode, - bool rewire_uncommon_proj_phi_inputs = false - NOT_PRODUCT (COMMA AssertionPredicateType assertion_predicate_type = AssertionPredicateType::None)); + bool rewire_uncommon_proj_phi_inputs = false, + AssertionPredicateType assertion_predicate_type = AssertionPredicateType::None); private: // Helper functions for create_new_if_for_predicate() @@ -1672,7 +1672,7 @@ class PhaseIdealLoop : public PhaseTransform { IfProjNode* clone_parse_predicate_to_unswitched_loop(ParsePredicateSuccessProj* parse_predicate_proj, Node* new_entry, Deoptimization::DeoptReason reason, bool slow_loop); void clone_assertion_predicates_to_unswitched_loop(IdealLoopTree* loop, const Node_List& old_new, - Deoptimization::DeoptReason reason, IfProjNode* old_predicate_proj, + Deoptimization::DeoptReason reason, ParsePredicateSuccessProj* old_parse_predicate_proj, ParsePredicateSuccessProj* fast_loop_parse_predicate_proj, ParsePredicateSuccessProj* slow_loop_parse_predicate_proj); IfProjNode* clone_assertion_predicate_for_unswitched_loops(IfNode* template_assertion_predicate, IfProjNode* predicate, diff --git a/src/hotspot/share/opto/macro.cpp b/src/hotspot/share/opto/macro.cpp index 31a453c9093..289ea30a633 100644 --- a/src/hotspot/share/opto/macro.cpp +++ b/src/hotspot/share/opto/macro.cpp @@ -1706,7 +1706,9 @@ PhaseMacroExpand::initialize_object(AllocateNode* alloc, } rawmem = make_store(control, rawmem, object, oopDesc::mark_offset_in_bytes(), mark_node, TypeX_X->basic_type()); - rawmem = make_store(control, rawmem, object, oopDesc::klass_offset_in_bytes(), klass_node, T_METADATA); + if (!UseCompactObjectHeaders) { + rawmem = make_store(control, rawmem, object, oopDesc::klass_offset_in_bytes(), klass_node, T_METADATA); + } int header_size = alloc->minimum_header_size(); // conservatively small // Array length diff --git a/src/hotspot/share/opto/matcher.cpp b/src/hotspot/share/opto/matcher.cpp index 0d29412dc29..abfbddee21a 100644 --- a/src/hotspot/share/opto/matcher.cpp +++ b/src/hotspot/share/opto/matcher.cpp @@ -434,7 +434,7 @@ void Matcher::match( ) { Fixup_Save_On_Entry( ); { // Cleanup mach IR after selection phase is over. - Compile::TracePhase tp("postselect_cleanup", &timers[_t_postselect_cleanup]); + Compile::TracePhase tp(_t_postselect_cleanup); do_postselect_cleanup(); if (C->failing()) return; assert(verify_after_postselect_cleanup(), ""); diff --git a/src/hotspot/share/opto/memnode.cpp b/src/hotspot/share/opto/memnode.cpp index 919d23fea8d..ad351fb81ad 100644 --- a/src/hotspot/share/opto/memnode.cpp +++ b/src/hotspot/share/opto/memnode.cpp @@ -1971,6 +1971,8 @@ Node *LoadNode::Ideal(PhaseGVN *phase, bool can_reshape) { const Type* LoadNode::load_array_final_field(const TypeKlassPtr *tkls, ciKlass* klass) const { + assert(!UseCompactObjectHeaders || tkls->offset() != in_bytes(Klass::prototype_header_offset()), + "must not happen"); if (tkls->offset() == in_bytes(Klass::modifier_flags_offset())) { // The field is Klass::_modifier_flags. Return its (constant) value. // (Folds up the 2nd indirection in aClassConstant.getModifiers().) @@ -2149,6 +2151,13 @@ const Type* LoadNode::Value(PhaseGVN* phase) const { assert(Opcode() == Op_LoadI, "must load an int from _super_check_offset"); return TypeInt::make(klass->super_check_offset()); } + if (UseCompactObjectHeaders) { + if (tkls->offset() == in_bytes(Klass::prototype_header_offset())) { + // The field is Klass::_prototype_header. Return its (constant) value. + assert(this->Opcode() == Op_LoadX, "must load a proper type from _prototype_header"); + return TypeX::make(klass->prototype_header()); + } + } // Compute index into primary_supers array juint depth = (tkls->offset() - in_bytes(Klass::primary_supers_offset())) / sizeof(Klass*); // Check for overflowing; use unsigned compare to handle the negative case. @@ -2238,9 +2247,11 @@ const Type* LoadNode::Value(PhaseGVN* phase) const { } } - Node* alloc = is_new_object_mark_load(); - if (alloc != nullptr) { - return TypeX::make(markWord::prototype().value()); + if (!UseCompactObjectHeaders) { + Node* alloc = is_new_object_mark_load(); + if (alloc != nullptr) { + return TypeX::make(markWord::prototype().value()); + } } return _type; diff --git a/src/hotspot/share/opto/output.cpp b/src/hotspot/share/opto/output.cpp index ea1175bce14..fa781bf23a6 100644 --- a/src/hotspot/share/opto/output.cpp +++ b/src/hotspot/share/opto/output.cpp @@ -437,7 +437,7 @@ void PhaseOutput::compute_loop_first_inst_sizes() { // branch instructions. Replace eligible long branches with short branches. void PhaseOutput::shorten_branches(uint* blk_starts) { - Compile::TracePhase tp("shorten branches", &timers[_t_shortenBranches]); + Compile::TracePhase tp(_t_shortenBranches); // Compute size of each block, method size, and relocation information size uint nblocks = C->cfg()->number_of_blocks(); @@ -1418,7 +1418,7 @@ void PhaseOutput::fill_buffer(C2_MacroAssembler* masm, uint* blk_starts) { // Compute the size of first NumberOfLoopInstrToAlign instructions at head // of a loop. It is used to determine the padding for loop alignment. - Compile::TracePhase tp("fill buffer", &timers[_t_fillBuffer]); + Compile::TracePhase tp(_t_fillBuffer); compute_loop_first_inst_sizes(); @@ -2162,7 +2162,7 @@ void PhaseOutput::ScheduleAndBundle() { return; } - Compile::TracePhase tp("isched", &timers[_t_instrSched]); + Compile::TracePhase tp(_t_instrSched); // Create a data structure for all the scheduling information Scheduling scheduling(Thread::current()->resource_area(), *C); @@ -3428,7 +3428,7 @@ void PhaseOutput::install_code(ciMethod* target, return; } #endif - Compile::TracePhase tp("install_code", &timers[_t_registerMethod]); + Compile::TracePhase tp(_t_registerMethod); if (C->is_osr_compilation()) { _code_offsets.set_value(CodeOffsets::Verified_Entry, 0); diff --git a/src/hotspot/share/opto/parse1.cpp b/src/hotspot/share/opto/parse1.cpp index d4a6a9ce5b7..68b87d858a5 100644 --- a/src/hotspot/share/opto/parse1.cpp +++ b/src/hotspot/share/opto/parse1.cpp @@ -247,7 +247,7 @@ void Parse::load_interpreter_state(Node* osr_buf) { Node *displaced_hdr = fetch_interpreter_state((index*2) + 1, T_ADDRESS, monitors_addr, osr_buf); - store_to_memory(control(), box, displaced_hdr, T_ADDRESS, Compile::AliasIdxRaw, MemNode::unordered); + store_to_memory(control(), box, displaced_hdr, T_ADDRESS, MemNode::unordered); // Build a bogus FastLockNode (no code will be generated) and push the // monitor into our debug info. @@ -2272,7 +2272,7 @@ void Parse::add_safepoint() { Node *polladr; Node *thread = _gvn.transform(new ThreadLocalNode()); Node *polling_page_load_addr = _gvn.transform(basic_plus_adr(top(), thread, in_bytes(JavaThread::polling_page_offset()))); - polladr = make_load(control(), polling_page_load_addr, TypeRawPtr::BOTTOM, T_ADDRESS, Compile::AliasIdxRaw, MemNode::unordered); + polladr = make_load(control(), polling_page_load_addr, TypeRawPtr::BOTTOM, T_ADDRESS, MemNode::unordered); sfpnt->init_req(TypeFunc::Parms+0, _gvn.transform(polladr)); // Fix up the JVM State edges diff --git a/src/hotspot/share/opto/parse2.cpp b/src/hotspot/share/opto/parse2.cpp index d428024e53e..f7b17361913 100644 --- a/src/hotspot/share/opto/parse2.cpp +++ b/src/hotspot/share/opto/parse2.cpp @@ -1376,9 +1376,9 @@ static volatile int _trap_stress_counter = 0; void Parse::increment_trap_stress_counter(Node*& counter, Node*& incr_store) { Node* counter_addr = makecon(TypeRawPtr::make((address)&_trap_stress_counter)); - counter = make_load(control(), counter_addr, TypeInt::INT, T_INT, Compile::AliasIdxRaw, MemNode::unordered); + counter = make_load(control(), counter_addr, TypeInt::INT, T_INT, MemNode::unordered); counter = _gvn.transform(new AddINode(counter, intcon(1))); - incr_store = store_to_memory(control(), counter_addr, counter, T_INT, Compile::AliasIdxRaw, MemNode::unordered); + incr_store = store_to_memory(control(), counter_addr, counter, T_INT, MemNode::unordered); } //----------------------------------do_ifnull---------------------------------- diff --git a/src/hotspot/share/opto/parse3.cpp b/src/hotspot/share/opto/parse3.cpp index a9fad4e3633..0a65d904c64 100644 --- a/src/hotspot/share/opto/parse3.cpp +++ b/src/hotspot/share/opto/parse3.cpp @@ -131,6 +131,8 @@ void Parse::do_get_xxx(Node* obj, ciField* field, bool is_field) { int offset = field->offset_in_bytes(); const TypePtr* adr_type = C->alias_type(field)->adr_type(); Node *adr = basic_plus_adr(obj, obj, offset); + assert(C->get_alias_index(adr_type) == C->get_alias_index(_gvn.type(adr)->isa_ptr()), + "slice of address and input slice don't match"); // Build the resultant type of the load const Type *type; @@ -204,6 +206,8 @@ void Parse::do_put_xxx(Node* obj, ciField* field, bool is_field) { int offset = field->offset_in_bytes(); const TypePtr* adr_type = C->alias_type(field)->adr_type(); Node* adr = basic_plus_adr(obj, obj, offset); + assert(C->get_alias_index(adr_type) == C->get_alias_index(_gvn.type(adr)->isa_ptr()), + "slice of address and input slice don't match"); BasicType bt = field->layout_type(); // Value to be stored Node* val = type2size[bt] == 1 ? pop() : pop_pair(); @@ -406,7 +410,7 @@ void Parse::do_multianewarray() { // Fill-in it with values for (j = 0; j < ndimensions; j++) { Node *dims_elem = array_element_address(dims, intcon(j), T_INT); - store_to_memory(control(), dims_elem, length[j], T_INT, TypeAryPtr::INTS, MemNode::unordered); + store_to_memory(control(), dims_elem, length[j], T_INT, MemNode::unordered); } } diff --git a/src/hotspot/share/opto/phase.cpp b/src/hotspot/share/opto/phase.cpp index d3e34a4dc69..8ab98f8a3f3 100644 --- a/src/hotspot/share/opto/phase.cpp +++ b/src/hotspot/share/opto/phase.cpp @@ -39,6 +39,16 @@ elapsedTimer Phase::_t_stubCompilation; // The counters to use for LogCompilation elapsedTimer Phase::timers[max_phase_timers]; +const char* Phase::get_phase_trace_id_text(PhaseTraceId id) { + static const char* const texts[] = { + #define DEF_TEXT(name, text) text, + ALL_PHASE_TRACE_IDS(DEF_TEXT) + #undef DEF_TEXT + nullptr + }; + return texts[(int)id]; +} + //------------------------------Phase------------------------------------------ Phase::Phase( PhaseNumber pnum ) : _pnum(pnum), C( pnum == Compiler ? nullptr : Compile::current()) { // Poll for requests from shutdown mechanism to quiesce compiler (4448539, 4448544). diff --git a/src/hotspot/share/opto/phase.hpp b/src/hotspot/share/opto/phase.hpp index db2327f378f..5d86618fc32 100644 --- a/src/hotspot/share/opto/phase.hpp +++ b/src/hotspot/share/opto/phase.hpp @@ -64,64 +64,71 @@ class Phase : public StackObj { last_phase }; +#define ALL_PHASE_TRACE_IDS(f) \ + f( _t_parser, "parse") \ + f( _t_optimizer, "optimizer") \ + f( _t_escapeAnalysis, "escapeAnalysis") \ + f( _t_connectionGraph, "connectionGraph") \ + f( _t_macroEliminate, "macroEliminate") \ + f( _t_iterGVN, "iterGVN") \ + f( _t_incrInline, "incrementalInline") \ + f( _t_incrInline_ideal, "incrementalInline_ideal") \ + f( _t_incrInline_igvn, "incrementalInline_igvn") \ + f( _t_incrInline_pru, "incrementalInline_pru") \ + f( _t_incrInline_inline, "incrementalInline_inline") \ + f( _t_vector, "") \ + f( _t_vector_elimination, "vector_elimination") \ + f( _t_vector_igvn, "incrementalInline_igvn") \ + f( _t_vector_pru, "vector_pru") \ + f( _t_renumberLive, "") \ + f( _t_idealLoop, "idealLoop") \ + f( _t_autoVectorize, "autoVectorize") \ + f( _t_idealLoopVerify, "idealLoopVerify") \ + f( _t_ccp, "ccp") \ + f( _t_iterGVN2, "iterGVN2") \ + f( _t_macroExpand, "macroExpand") \ + f( _t_barrierExpand, "barrierExpand") \ + f( _t_graphReshaping, "graphReshape") \ + f( _t_matcher, "matcher") \ + f( _t_postselect_cleanup, "postselect_cleanup") \ + f( _t_scheduler, "scheduler") \ + f( _t_registerAllocation, "regalloc") \ + f( _t_ctorChaitin, "ctorChaitin") \ + f( _t_buildIFGvirtual, "buildIFG_virt") \ + f( _t_buildIFGphysical, "buildIFG") \ + f( _t_computeLive, "computeLive") \ + f( _t_regAllocSplit, "regAllocSplit") \ + f( _t_postAllocCopyRemoval, "postAllocCopyRemoval") \ + f( _t_mergeMultidefs, "mergeMultidefs") \ + f( _t_fixupSpills, "fixupSpills") \ + f( _t_chaitinCompact, "chaitinCompact") \ + f( _t_chaitinCoalesce1, "chaitinCoalesce1") \ + f( _t_chaitinCoalesce2, "chaitinCoalesce2") \ + f( _t_chaitinCoalesce3, "chaitinCoalesce3") \ + f( _t_chaitinCacheLRG, "chaitinCacheLRG") \ + f( _t_chaitinSimplify, "chaitinSimplify") \ + f( _t_chaitinSelect, "chaitinSelect") \ + f( _t_blockOrdering, "blockOrdering") \ + f( _t_peephole, "peephole") \ + f( _t_postalloc_expand, "postalloc_expand") \ + f( _t_output, "output") \ + f( _t_instrSched, "isched") \ + f( _t_shortenBranches, "shorten branches") \ + f( _t_buildOopMaps, "bldOopMaps") \ + f( _t_fillBuffer, "fill buffer") \ + f( _t_registerMethod, "install_code") \ + f( _t_temporaryTimer1, "tempTimer1") \ + f( _t_temporaryTimer2, "tempTimer2") + enum PhaseTraceId { - _t_parser, - _t_optimizer, - _t_escapeAnalysis, - _t_connectionGraph, - _t_macroEliminate, - _t_iterGVN, - _t_incrInline, - _t_incrInline_ideal, - _t_incrInline_igvn, - _t_incrInline_pru, - _t_incrInline_inline, - _t_vector, - _t_vector_elimination, - _t_vector_igvn, - _t_vector_pru, - _t_renumberLive, - _t_idealLoop, - _t_autoVectorize, - _t_idealLoopVerify, - _t_ccp, - _t_iterGVN2, - _t_macroExpand, - _t_barrierExpand, - _t_graphReshaping, - _t_matcher, - _t_postselect_cleanup, - _t_scheduler, - _t_registerAllocation, - _t_ctorChaitin, - _t_buildIFGvirtual, - _t_buildIFGphysical, - _t_computeLive, - _t_regAllocSplit, - _t_postAllocCopyRemoval, - _t_mergeMultidefs, - _t_fixupSpills, - _t_chaitinCompact, - _t_chaitinCoalesce1, - _t_chaitinCoalesce2, - _t_chaitinCoalesce3, - _t_chaitinCacheLRG, - _t_chaitinSimplify, - _t_chaitinSelect, - _t_blockOrdering, - _t_peephole, - _t_postalloc_expand, - _t_output, - _t_instrSched, - _t_shortenBranches, - _t_buildOopMaps, - _t_fillBuffer, - _t_registerMethod, - _t_temporaryTimer1, - _t_temporaryTimer2, +#define DEFID(name, text) name, + ALL_PHASE_TRACE_IDS(DEFID) +#undef DEFID max_phase_timers }; + static const char* get_phase_trace_id_text(PhaseTraceId id); + static elapsedTimer timers[max_phase_timers]; protected: diff --git a/src/hotspot/share/opto/phaseX.cpp b/src/hotspot/share/opto/phaseX.cpp index f766146a894..c91a7bba078 100644 --- a/src/hotspot/share/opto/phaseX.cpp +++ b/src/hotspot/share/opto/phaseX.cpp @@ -27,6 +27,7 @@ #include "gc/shared/c2/barrierSetC2.hpp" #include "memory/allocation.inline.hpp" #include "memory/resourceArea.hpp" +#include "opto/addnode.hpp" #include "opto/block.hpp" #include "opto/callnode.hpp" #include "opto/castnode.hpp" @@ -1634,12 +1635,19 @@ void PhaseIterGVN::add_users_of_use_to_worklist(Node* n, Node* use, Unique_Node_ } } } - // If changed AddP inputs, check Stores for loop invariant - if( use_op == Op_AddP ) { + // If changed AddP inputs: + // - check Stores for loop invariant, and + // - if the changed input is the offset, check constant-offset AddP users for + // address expression flattening. + if (use_op == Op_AddP) { + bool offset_changed = n == use->in(AddPNode::Offset); for (DUIterator_Fast i2max, i2 = use->fast_outs(i2max); i2 < i2max; i2++) { Node* u = use->fast_out(i2); - if (u->is_Mem()) + if (u->is_Mem()) { worklist.push(u); + } else if (offset_changed && u->is_AddP() && u->in(AddPNode::Offset)->is_Con()) { + worklist.push(u); + } } } // If changed initialization activity, check dependent Stores diff --git a/src/hotspot/share/opto/postaloc.cpp b/src/hotspot/share/opto/postaloc.cpp index bc336e1e0a9..ac91d8a775e 100644 --- a/src/hotspot/share/opto/postaloc.cpp +++ b/src/hotspot/share/opto/postaloc.cpp @@ -401,7 +401,7 @@ bool PhaseChaitin::eliminate_copy_of_constant(Node* val, Node* n, // all the uses that we've seen so far to use the merge. After that we keep replacing the new defs in the same lrg // as they get encountered with the merge node and keep adding these defs to the merge inputs. void PhaseChaitin::merge_multidefs() { - Compile::TracePhase tp("mergeMultidefs", &timers[_t_mergeMultidefs]); + Compile::TracePhase tp(_t_mergeMultidefs); ResourceMark rm; // Keep track of the defs seen in registers and collect their uses in the block. RegToDefUseMap reg2defuse(_max_reg, _max_reg, RegDefUse()); @@ -501,7 +501,7 @@ int PhaseChaitin::possibly_merge_multidef(Node *n, uint k, Block *block, RegToDe // When we see a use from a reg-reg Copy, we will attempt to use the copy's // source directly and make the copy go dead. void PhaseChaitin::post_allocate_copy_removal() { - Compile::TracePhase tp("postAllocCopyRemoval", &timers[_t_postAllocCopyRemoval]); + Compile::TracePhase tp(_t_postAllocCopyRemoval); ResourceMark rm; // Need a mapping from basic block Node_Lists. We need a Node_List to diff --git a/src/hotspot/share/opto/predicates.cpp b/src/hotspot/share/opto/predicates.cpp index ee218063567..5e600eab2f0 100644 --- a/src/hotspot/share/opto/predicates.cpp +++ b/src/hotspot/share/opto/predicates.cpp @@ -152,7 +152,6 @@ void TemplateAssertionPredicate::rewire_loop_data_dependencies(IfTrueNode* targe } } - // Template Assertion Predicates always have the dedicated OpaqueTemplateAssertionPredicate to identify them. bool TemplateAssertionPredicate::is_predicate(Node* node) { if (!may_be_assertion_predicate_if(node)) { @@ -172,13 +171,31 @@ IfTrueNode* TemplateAssertionPredicate::clone_and_replace_init(Node* new_control template_assertion_expression.clone_and_replace_init(new_opaque_init, new_control, phase); AssertionPredicateIfCreator assertion_predicate_if_creator(phase); IfTrueNode* success_proj = assertion_predicate_if_creator.create_for_template(new_control, _if_node->Opcode(), - new_opaque_node NOT_PRODUCT(COMMA - _if_node->assertion_predicate_type())); + new_opaque_node, + _if_node->assertion_predicate_type()); assert(PhaseIdealLoop::assertion_predicate_has_loop_opaque_node(success_proj->in(0)->as_If()), "Template Assertion Predicates must have OpaqueLoop* nodes in the bool expression"); return success_proj; } +// Replace the input to OpaqueLoopStrideNode with 'new_stride' and leave the other nodes unchanged. +void TemplateAssertionPredicate::replace_opaque_stride_input(Node* new_stride, PhaseIterGVN& igvn) const { + TemplateAssertionExpression expression(opaque_node()); + expression.replace_opaque_stride_input(new_stride, igvn); +} + +// Create a new Initialized Assertion Predicate from this template at 'new_control' and return the success projection +// of the newly created Initialized Assertion Predicate. +IfTrueNode* TemplateAssertionPredicate::initialize(PhaseIdealLoop* phase, Node* new_control) const { + assert(phase->assertion_predicate_has_loop_opaque_node(head()), + "must find OpaqueLoop* nodes for Template Assertion Predicate"); + InitializedAssertionPredicateCreator initialized_assertion_predicate(phase); + IfTrueNode* success_proj = initialized_assertion_predicate.create_from_template(head(), new_control); + assert(!phase->assertion_predicate_has_loop_opaque_node(success_proj->in(0)->as_If()), + "Initialized Assertion Predicates do not have OpaqueLoop* nodes in the bool expression anymore"); + return success_proj; +} + // Initialized Assertion Predicates always have the dedicated OpaqueInitiailizedAssertionPredicate node to identify // them. bool InitializedAssertionPredicate::is_predicate(Node* node) { @@ -189,6 +206,12 @@ bool InitializedAssertionPredicate::is_predicate(Node* node) { return if_node->in(1)->is_OpaqueInitializedAssertionPredicate(); } +void InitializedAssertionPredicate::kill(PhaseIdealLoop* phase) const { + Node* true_con = phase->igvn().intcon(1); + phase->set_ctrl(true_con, phase->C->root()); + phase->igvn().replace_input_of(_if_node, 1, true_con); +} + #ifdef ASSERT // Check that the block has at most one Parse Predicate and that we only find Regular Predicate nodes (i.e. IfProj, // If, or RangeCheck nodes). @@ -388,6 +411,63 @@ TemplateAssertionExpression::clone(const TransformStrategyForOpaqueLoopNodes& tr return opaque_node_clone->as_OpaqueTemplateAssertionPredicate(); } +// This class is used to replace the input to OpaqueLoopStrideNode with a new node while leaving the other nodes +// unchanged. +class ReplaceOpaqueStrideInput : public StackObj { + PhaseIterGVN& _igvn; + Unique_Node_List _nodes_to_visit; + + public: + ReplaceOpaqueStrideInput(OpaqueTemplateAssertionPredicateNode* start_node, PhaseIterGVN& igvn) : _igvn(igvn) { + _nodes_to_visit.push(start_node); + } + NONCOPYABLE(ReplaceOpaqueStrideInput); + + void replace(Node* new_opaque_stride_input) { + for (uint i = 0; i < _nodes_to_visit.size(); i++) { + Node* next = _nodes_to_visit[i]; + for (uint j = 1; j < next->req(); j++) { + Node* input = next->in(j); + if (input->is_OpaqueLoopStride()) { + assert(TemplateAssertionExpressionNode::is_maybe_in_expression(input), "must also pass node filter"); + _igvn.replace_input_of(input, 1, new_opaque_stride_input); + } else if (TemplateAssertionExpressionNode::is_maybe_in_expression(input)) { + _nodes_to_visit.push(input); + } + } + } + } +}; + +// Replace the input to OpaqueLoopStrideNode with 'new_stride' and leave the other nodes unchanged. +void TemplateAssertionExpression::replace_opaque_stride_input(Node* new_stride, PhaseIterGVN& igvn) { + ReplaceOpaqueStrideInput replace_opaque_stride_input(_opaque_node, igvn); + replace_opaque_stride_input.replace(new_stride); +} + +// The transformations of this class fold the OpaqueLoop* nodes by returning their inputs. +class RemoveOpaqueLoopNodesStrategy : public TransformStrategyForOpaqueLoopNodes { + public: + Node* transform_opaque_init(OpaqueLoopInitNode* opaque_init) const override { + return opaque_init->in(1); + } + + Node* transform_opaque_stride(OpaqueLoopStrideNode* opaque_stride) const override { + return opaque_stride->in(1); + } +}; + +OpaqueInitializedAssertionPredicateNode* +TemplateAssertionExpression::clone_and_fold_opaque_loop_nodes(Node* new_control, PhaseIdealLoop* phase) { + RemoveOpaqueLoopNodesStrategy remove_opaque_loop_nodes_strategy; + OpaqueTemplateAssertionPredicateNode* cloned_template_opaque = clone(remove_opaque_loop_nodes_strategy, new_control, + phase); + OpaqueInitializedAssertionPredicateNode* opaque_initialized_opaque = + new OpaqueInitializedAssertionPredicateNode(cloned_template_opaque->in(1)->as_Bool(), phase->C); + phase->register_new_node(opaque_initialized_opaque, new_control); + return opaque_initialized_opaque; +} + // Check if this node belongs a Template Assertion Expression (including OpaqueLoop* nodes). bool TemplateAssertionExpressionNode::is_in_expression(Node* node) { if (is_maybe_in_expression(node)) { @@ -466,19 +546,19 @@ class AssertionPredicateExpressionCreator : public StackObj { // Creates an If with a success and a fail path with the given assertion_expression. The only difference to // create_for_initialized() is that we use a template specific Halt message on the fail path. IfTrueNode* AssertionPredicateIfCreator::create_for_template(Node* new_control, const int if_opcode, - Node* assertion_expression NOT_PRODUCT(COMMA - const AssertionPredicateType assertion_predicate_type)) { + Node* assertion_expression, + const AssertionPredicateType assertion_predicate_type) { const char* halt_message = "Template Assertion Predicates are always removed before code generation"; - return create(new_control, if_opcode, assertion_expression, halt_message NOT_PRODUCT(COMMA assertion_predicate_type)); + return create(new_control, if_opcode, assertion_expression, halt_message, assertion_predicate_type); } // Creates an If with a success and a fail path with the given assertion_expression. The only difference to // create_for_template() is that we use a initialized specific Halt message on the fail path. IfTrueNode* AssertionPredicateIfCreator::create_for_initialized(Node* new_control, const int if_opcode, - Node* assertion_expression NOT_PRODUCT(COMMA - const AssertionPredicateType assertion_predicate_type)) { + Node* assertion_expression, + const AssertionPredicateType assertion_predicate_type) { const char* halt_message = "Initialized Assertion Predicate cannot fail"; - return create(new_control, if_opcode, assertion_expression, halt_message NOT_PRODUCT(COMMA assertion_predicate_type)); + return create(new_control, if_opcode, assertion_expression, halt_message, assertion_predicate_type); } // Creates the If node for an Assertion Predicate with a success path and a fail path having a Halt node: @@ -491,28 +571,25 @@ IfTrueNode* AssertionPredicateIfCreator::create_for_initialized(Node* new_contro // proj with Halt // IfTrueNode* AssertionPredicateIfCreator::create(Node* new_control, const int if_opcode, Node* assertion_expression, - const char* halt_message NOT_PRODUCT(COMMA - const AssertionPredicateType assertion_predicate_type)) { + const char* halt_message, + const AssertionPredicateType assertion_predicate_type) { assert(assertion_expression->is_OpaqueTemplateAssertionPredicate() || assertion_expression->is_OpaqueInitializedAssertionPredicate(), "not a valid assertion expression"); IdealLoopTree* loop = _phase->get_loop(new_control); - IfNode* if_node = create_if_node(new_control, if_opcode, assertion_expression, loop - NOT_PRODUCT(COMMA assertion_predicate_type)); + IfNode* if_node = create_if_node(new_control, if_opcode, assertion_expression, loop, assertion_predicate_type); create_fail_path(if_node, loop, halt_message); return create_success_path(if_node, loop); } IfNode* AssertionPredicateIfCreator::create_if_node(Node* new_control, const int if_opcode, Node* assertion_expression, - IdealLoopTree* loop NOT_PRODUCT(COMMA - const AssertionPredicateType assertion_predicate_type)) { + IdealLoopTree* loop, + const AssertionPredicateType assertion_predicate_type) { IfNode* if_node; if (if_opcode == Op_If) { - if_node = new IfNode(new_control, assertion_expression, PROB_MAX, COUNT_UNKNOWN - NOT_PRODUCT(COMMA assertion_predicate_type)); + if_node = new IfNode(new_control, assertion_expression, PROB_MAX, COUNT_UNKNOWN, assertion_predicate_type); } else { assert(if_opcode == Op_RangeCheck, "must be range check"); - if_node = new RangeCheckNode(new_control, assertion_expression, PROB_MAX, COUNT_UNKNOWN - NOT_PRODUCT(COMMA assertion_predicate_type)); + if_node = new RangeCheckNode(new_control, assertion_expression, PROB_MAX, COUNT_UNKNOWN, assertion_predicate_type); } _phase->register_control(if_node, loop, new_control); return if_node; @@ -551,13 +628,13 @@ IfTrueNode* TemplateAssertionPredicateCreator::create_with_uncommon_trap( create_for_init_value(new_control, opaque_init, does_overflow); IfTrueNode* template_predicate_success_proj = create_if_node_with_uncommon_trap(template_assertion_predicate_expression, parse_predicate_success_proj, - deopt_reason, if_opcode, does_overflow - NOT_PRODUCT(COMMA AssertionPredicateType::InitValue)); + deopt_reason, if_opcode, does_overflow, + AssertionPredicateType::InitValue); template_assertion_predicate_expression = create_for_last_value(template_predicate_success_proj, opaque_init, does_overflow); return create_if_node_with_uncommon_trap(template_assertion_predicate_expression, parse_predicate_success_proj, - deopt_reason, if_opcode, does_overflow - NOT_PRODUCT(COMMA AssertionPredicateType::LastValue)); + deopt_reason, if_opcode, does_overflow, + AssertionPredicateType::LastValue); } OpaqueLoopInitNode* TemplateAssertionPredicateCreator::create_opaque_init(Node* new_control) { @@ -576,11 +653,10 @@ TemplateAssertionPredicateCreator::create_for_init_value(Node* new_control, Opaq IfTrueNode* TemplateAssertionPredicateCreator::create_if_node_with_uncommon_trap( OpaqueTemplateAssertionPredicateNode* template_assertion_predicate_expression, ParsePredicateSuccessProj* parse_predicate_success_proj, const Deoptimization::DeoptReason deopt_reason, - const int if_opcode, const bool does_overflow - NOT_PRODUCT(COMMA AssertionPredicateType assertion_predicate_type)) { + const int if_opcode, const bool does_overflow, const AssertionPredicateType assertion_predicate_type) { IfTrueNode* success_proj = _phase->create_new_if_for_predicate(parse_predicate_success_proj, nullptr, deopt_reason, - does_overflow ? Op_If : if_opcode, false - NOT_PRODUCT(COMMA assertion_predicate_type)); + does_overflow ? Op_If : if_opcode, false, + assertion_predicate_type); _phase->igvn().replace_input_of(success_proj->in(0), 1, template_assertion_predicate_expression); return success_proj; } @@ -608,12 +684,12 @@ Node* TemplateAssertionPredicateCreator::create_last_value(Node* new_control, Op } IfTrueNode* TemplateAssertionPredicateCreator::create_if_node_with_halt( - Node* new_control, OpaqueTemplateAssertionPredicateNode* template_assertion_predicate_expression, bool does_overflow - NOT_PRODUCT(COMMA AssertionPredicateType assertion_predicate_type)) { + Node* new_control, OpaqueTemplateAssertionPredicateNode* template_assertion_predicate_expression, + const bool does_overflow, const AssertionPredicateType assertion_predicate_type) { AssertionPredicateIfCreator assertion_predicate_if_creator(_phase); return assertion_predicate_if_creator.create_for_template(new_control, does_overflow ? Op_If : Op_RangeCheck, - template_assertion_predicate_expression - NOT_PRODUCT(COMMA assertion_predicate_type)); + template_assertion_predicate_expression, + assertion_predicate_type); } // Creates an init and last value Template Assertion Predicate connected together with a Halt node on the failing path. @@ -624,12 +700,12 @@ IfTrueNode* TemplateAssertionPredicateCreator::create_with_halt(Node* new_contro OpaqueTemplateAssertionPredicateNode* template_assertion_predicate_expression = create_for_init_value(new_control, opaque_init, does_overflow); IfTrueNode* template_predicate_success_proj = - create_if_node_with_halt(new_control, template_assertion_predicate_expression, does_overflow - NOT_PRODUCT(COMMA AssertionPredicateType::InitValue)); + create_if_node_with_halt(new_control, template_assertion_predicate_expression, does_overflow, + AssertionPredicateType::InitValue); template_assertion_predicate_expression = create_for_last_value(template_predicate_success_proj, opaque_init, does_overflow); return create_if_node_with_halt(template_predicate_success_proj, template_assertion_predicate_expression, - does_overflow NOT_PRODUCT(COMMA AssertionPredicateType::LastValue)); + does_overflow, AssertionPredicateType::LastValue); } InitializedAssertionPredicateCreator::InitializedAssertionPredicateCreator(PhaseIdealLoop* phase) @@ -660,29 +736,42 @@ IfTrueNode* InitializedAssertionPredicateCreator::create_from_template(IfNode* t Node* new_stride) { OpaqueInitializedAssertionPredicateNode* assertion_expression = create_assertion_expression_from_template(template_assertion_predicate, new_control, new_init, new_stride); - return create_control_nodes(new_control, template_assertion_predicate->Opcode(), assertion_expression - NOT_PRODUCT(COMMA template_assertion_predicate->assertion_predicate_type())); + return create_control_nodes(new_control, template_assertion_predicate->Opcode(), assertion_expression, + template_assertion_predicate->assertion_predicate_type()); +} + +// Create a new Initialized Assertion Predicate from 'template_assertion_predicate' by cloning it but omitting the +// OpaqueLoop*Notes (i.e. taking their inputs instead). +IfTrueNode* InitializedAssertionPredicateCreator::create_from_template(IfNode* template_assertion_predicate, + Node* new_control) { + OpaqueTemplateAssertionPredicateNode* template_opaque = + template_assertion_predicate->in(1)->as_OpaqueTemplateAssertionPredicate(); + TemplateAssertionExpression template_assertion_expression(template_opaque); + OpaqueInitializedAssertionPredicateNode* assertion_expression = + template_assertion_expression.clone_and_fold_opaque_loop_nodes(new_control, _phase); + return create_control_nodes(new_control, template_assertion_predicate->Opcode(), assertion_expression, + template_assertion_predicate->assertion_predicate_type()); } // Create a new Initialized Assertion Predicate directly without a template. IfTrueNode* InitializedAssertionPredicateCreator::create(Node* operand, Node* new_control, const jint stride, - const int scale, Node* offset, Node* range NOT_PRODUCT(COMMA - AssertionPredicateType assertion_predicate_type)) { + const int scale, Node* offset, Node* range, + const AssertionPredicateType assertion_predicate_type) { AssertionPredicateExpressionCreator expression_creator(stride, scale, offset, range, _phase); bool does_overflow; OpaqueInitializedAssertionPredicateNode* assertion_expression = expression_creator.create_for_initialized(new_control, operand, does_overflow); - return create_control_nodes(new_control, does_overflow ? Op_If : Op_RangeCheck, assertion_expression - NOT_PRODUCT(COMMA assertion_predicate_type)); + return create_control_nodes(new_control, does_overflow ? Op_If : Op_RangeCheck, assertion_expression, + assertion_predicate_type); } // Creates the CFG nodes for the Initialized Assertion Predicate. IfTrueNode* InitializedAssertionPredicateCreator::create_control_nodes( - Node* new_control, const int if_opcode, OpaqueInitializedAssertionPredicateNode* assertion_expression - NOT_PRODUCT(COMMA AssertionPredicateType assertion_predicate_type)) { + Node* new_control, const int if_opcode, OpaqueInitializedAssertionPredicateNode* assertion_expression, + const AssertionPredicateType assertion_predicate_type) { AssertionPredicateIfCreator assertion_predicate_if_creator(_phase); - return assertion_predicate_if_creator.create_for_initialized(new_control, if_opcode, assertion_expression - NOT_PRODUCT(COMMA assertion_predicate_type)); + return assertion_predicate_if_creator.create_for_initialized(new_control, if_opcode, assertion_expression, + assertion_predicate_type); } // Create a new Assertion Expression based from the given template to be used as bool input for the Initialized @@ -768,7 +857,7 @@ void CreateAssertionPredicatesVisitor::visit(const TemplateAssertionPredicate& t // Create an Initialized Assertion Predicate from the provided Template Assertion Predicate. IfTrueNode* CreateAssertionPredicatesVisitor::initialize_from_template( -const TemplateAssertionPredicate& template_assertion_predicate) const { + const TemplateAssertionPredicate& template_assertion_predicate) const { IfNode* template_head = template_assertion_predicate.head(); IfTrueNode* initialized_predicate = _phase->create_initialized_assertion_predicate(template_head, _init, _stride, _new_control); @@ -783,3 +872,40 @@ IfTrueNode* CreateAssertionPredicatesVisitor::clone_template_and_replace_init_in _phase->register_new_node(opaque_init, _new_control); return template_assertion_predicate.clone_and_replace_init(_new_control, opaque_init, _phase); } + +// Clone the Template Assertion Predicate and set a new input for the OpaqueLoopStrideNode. +void UpdateStrideForAssertionPredicates::visit(const TemplateAssertionPredicate& template_assertion_predicate) { + if (!template_assertion_predicate.is_last_value()) { + // Only Last Value Assertion Predicates have an OpaqueLoopStrideNode. + return; + } + replace_opaque_stride_input(template_assertion_predicate); + Node* template_tail_control_out = template_assertion_predicate.tail()->unique_ctrl_out(); + IfTrueNode* initialized_success_proj = initialize_from_updated_template(template_assertion_predicate); + connect_initialized_assertion_predicate(template_tail_control_out, initialized_success_proj); +} + +// Replace the input to OpaqueLoopStrideNode with 'new_stride' and leave the other nodes unchanged. +void UpdateStrideForAssertionPredicates::replace_opaque_stride_input( + const TemplateAssertionPredicate& template_assertion_predicate) const { + template_assertion_predicate.replace_opaque_stride_input(_new_stride, _phase->igvn()); +} + +IfTrueNode* UpdateStrideForAssertionPredicates::initialize_from_updated_template( + const TemplateAssertionPredicate& template_assertion_predicate) const { + IfTrueNode* initialized_success_proj = template_assertion_predicate.initialize(_phase, template_assertion_predicate.tail()); + return initialized_success_proj; +} + +// The newly created Initialized Assertion Predicate can safely be inserted because this visitor is already visiting +// the Template Assertion Predicate above this. So, we will not accidentally visit this again and kill it with the +// visit() method for Initialized Assertion Predicates. +void UpdateStrideForAssertionPredicates::connect_initialized_assertion_predicate( + Node* new_control_out, IfTrueNode* initialized_success_proj) const { + if (new_control_out->is_Loop()) { + _phase->igvn().replace_input_of(new_control_out, LoopNode::EntryControl, initialized_success_proj); + } else { + _phase->igvn().replace_input_of(new_control_out, 0, initialized_success_proj); + } + _phase->set_idom(new_control_out, initialized_success_proj, _phase->dom_depth(new_control_out)); +} diff --git a/src/hotspot/share/opto/predicates.hpp b/src/hotspot/share/opto/predicates.hpp index f57948f8ab9..342650cd966 100644 --- a/src/hotspot/share/opto/predicates.hpp +++ b/src/hotspot/share/opto/predicates.hpp @@ -201,7 +201,6 @@ class TemplateAssertionPredicate; * Main Loop Head */ -#ifndef PRODUCT // Assertion Predicates are either emitted to check the initial value of a range check in the first iteration or the last // value of a range check in the last iteration of a loop. enum class AssertionPredicateType { @@ -211,7 +210,6 @@ enum class AssertionPredicateType { // Used for the Initialized Assertion Predicate emitted during Range Check Elimination for the final IV value. FinalIv }; -#endif // NOT PRODUCT // Interface to represent a C2 predicate. A predicate is always represented by two CFG nodes: // - An If node (head) @@ -377,8 +375,8 @@ class RuntimePredicate : public Predicate { // Class to represent a Template Assertion Predicate. class TemplateAssertionPredicate : public Predicate { - IfTrueNode* _success_proj; - IfNode* _if_node; + IfTrueNode* const _success_proj; + IfNode* const _if_node; public: explicit TemplateAssertionPredicate(IfTrueNode* success_proj) @@ -403,7 +401,13 @@ class TemplateAssertionPredicate : public Predicate { return _success_proj; } + bool is_last_value() const { + return _if_node->assertion_predicate_type() == AssertionPredicateType::LastValue; + } + IfTrueNode* clone_and_replace_init(Node* new_control, OpaqueLoopInitNode* new_opaque_init, PhaseIdealLoop* phase) const; + void replace_opaque_stride_input(Node* new_stride, PhaseIterGVN& igvn) const; + IfTrueNode* initialize(PhaseIdealLoop* phase, Node* new_control) const; void rewire_loop_data_dependencies(IfTrueNode* target_predicate, const NodeInLoopBody& data_in_loop_body, PhaseIdealLoop* phase) const; static bool is_predicate(Node* node); @@ -412,8 +416,8 @@ class TemplateAssertionPredicate : public Predicate { // Class to represent an Initialized Assertion Predicate which always has a halt node on the failing path. // This predicate should never fail at runtime by design. class InitializedAssertionPredicate : public Predicate { - IfTrueNode* _success_proj; - IfNode* _if_node; + IfTrueNode* const _success_proj; + IfNode* const _if_node; public: explicit InitializedAssertionPredicate(IfTrueNode* success_proj) @@ -434,6 +438,11 @@ class InitializedAssertionPredicate : public Predicate { return _success_proj; } + bool is_last_value() const { + return _if_node->assertion_predicate_type() == AssertionPredicateType::LastValue; + } + + void kill(PhaseIdealLoop* phase) const; static bool is_predicate(Node* node); }; @@ -461,6 +470,8 @@ class TemplateAssertionExpression : public StackObj { OpaqueTemplateAssertionPredicateNode* clone_and_replace_init(Node* new_init, Node* new_ctrl, PhaseIdealLoop* phase); OpaqueTemplateAssertionPredicateNode* clone_and_replace_init_and_stride(Node* new_control, Node* new_init, Node* new_stride, PhaseIdealLoop* phase); + void replace_opaque_stride_input(Node* new_stride, PhaseIterGVN& igvn); + OpaqueInitializedAssertionPredicateNode* clone_and_fold_opaque_loop_nodes(Node* new_ctrl, PhaseIdealLoop* phase); }; // Class to represent a node being part of a Template Assertion Expression. Note that this is not an IR node. @@ -545,15 +556,15 @@ class AssertionPredicateIfCreator : public StackObj { explicit AssertionPredicateIfCreator(PhaseIdealLoop* const phase) : _phase(phase) {} NONCOPYABLE(AssertionPredicateIfCreator); - IfTrueNode* create_for_initialized(Node* new_control, int if_opcode, Node* assertion_expression - NOT_PRODUCT(COMMA const AssertionPredicateType assertion_predicate_type)); - IfTrueNode* create_for_template(Node* new_control, int if_opcode, Node* assertion_expression - NOT_PRODUCT(COMMA const AssertionPredicateType assertion_predicate_type)); + IfTrueNode* create_for_initialized(Node* new_control, int if_opcode, Node* assertion_expression, + AssertionPredicateType assertion_predicate_type); + IfTrueNode* create_for_template(Node* new_control, int if_opcode, Node* assertion_expression, + AssertionPredicateType assertion_predicate_type); private: - IfTrueNode* create(Node* new_control, int if_opcode, Node* assertion_expression, const char* halt_message - NOT_PRODUCT(COMMA const AssertionPredicateType assertion_predicate_type)); - IfNode* create_if_node(Node* new_control, int if_opcode, Node* assertion_expression, IdealLoopTree* loop - NOT_PRODUCT(COMMA const AssertionPredicateType assertion_predicate_type)); + IfTrueNode* create(Node* new_control, int if_opcode, Node* assertion_expression, const char* halt_message, + AssertionPredicateType assertion_predicate_type); + IfNode* create_if_node(Node* new_control, int if_opcode, Node* assertion_expression, IdealLoopTree* loop, + AssertionPredicateType assertion_predicate_type); IfTrueNode* create_success_path(IfNode* if_node, IdealLoopTree* loop); void create_fail_path(IfNode* if_node, IdealLoopTree* loop, const char* halt_message); void create_halt_node(IfFalseNode* fail_proj, IdealLoopTree* loop, const char* halt_message); @@ -576,12 +587,10 @@ class TemplateAssertionPredicateCreator : public StackObj { IfTrueNode* create_if_node_with_uncommon_trap(OpaqueTemplateAssertionPredicateNode* template_assertion_predicate_expression, ParsePredicateSuccessProj* parse_predicate_success_proj, Deoptimization::DeoptReason deopt_reason, int if_opcode, - bool does_overflow - NOT_PRODUCT(COMMA AssertionPredicateType assertion_predicate_type)); + bool does_overflow, AssertionPredicateType assertion_predicate_type); IfTrueNode* create_if_node_with_halt(Node* new_control, OpaqueTemplateAssertionPredicateNode* template_assertion_predicate_expression, - bool does_overflow - NOT_PRODUCT(COMMA AssertionPredicateType assertion_predicate_type)); + bool does_overflow, AssertionPredicateType assertion_predicate_type); public: TemplateAssertionPredicateCreator(CountedLoopNode* loop_head, int scale, Node* offset, Node* range, @@ -608,16 +617,17 @@ class InitializedAssertionPredicateCreator : public StackObj { IfTrueNode* create_from_template(IfNode* template_assertion_predicate, Node* new_control, Node* new_init, Node* new_stride); - IfTrueNode* create(Node* operand, Node* new_control, jint stride, int scale, Node* offset, Node* range - NOT_PRODUCT(COMMA AssertionPredicateType assertion_predicate_type)); + IfTrueNode* create_from_template(IfNode* template_assertion_predicate, Node* new_control); + IfTrueNode* create(Node* operand, Node* new_control, jint stride, int scale, Node* offset, Node* range, + AssertionPredicateType assertion_predicate_type); private: OpaqueInitializedAssertionPredicateNode* create_assertion_expression_from_template(IfNode* template_assertion_predicate, Node* new_control, Node* new_init, Node* new_stride); IfTrueNode* create_control_nodes(Node* new_control, int if_opcode, - OpaqueInitializedAssertionPredicateNode* assertion_expression - NOT_PRODUCT(COMMA AssertionPredicateType assertion_predicate_type)); + OpaqueInitializedAssertionPredicateNode* assertion_expression, + AssertionPredicateType assertion_predicate_type); }; // This class iterates through all predicates of a Regular Predicate Block and applies the given visitor to each. @@ -1010,4 +1020,57 @@ class CreateAssertionPredicatesVisitor : public PredicateVisitor { } }; +// This visitor collects all Template Assertion Predicates If nodes or the corresponding Opaque nodes, depending on the +// provided 'get_opaque' flag, to the provided list. +class TemplateAssertionPredicateCollector : public PredicateVisitor { + Unique_Node_List& _list; + const bool _get_opaque; + + public: + TemplateAssertionPredicateCollector(Unique_Node_List& list, const bool get_opaque) + : _list(list), + _get_opaque(get_opaque) {} + + using PredicateVisitor::visit; + + void visit(const TemplateAssertionPredicate& template_assertion_predicate) override { + if (_get_opaque) { + _list.push(template_assertion_predicate.opaque_node()); + } else { + _list.push(template_assertion_predicate.tail()); + } + } +}; + +// This visitor updates the stride for an Assertion Predicate during Loop Unrolling. The inputs to the OpaqueLoopStride +// nodes Template of Template Assertion Predicates are updated and new Initialized Assertion Predicates are created +// from the updated templates. The old Initialized Assertion Predicates are killed. +class UpdateStrideForAssertionPredicates : public PredicateVisitor { + Node* const _new_stride; + PhaseIdealLoop* const _phase; + + void replace_opaque_stride_input(const TemplateAssertionPredicate& template_assertion_predicate) const; + IfTrueNode* initialize_from_updated_template(const TemplateAssertionPredicate& template_assertion_predicate) const; + void connect_initialized_assertion_predicate(Node* new_control_out, IfTrueNode* initialized_success_proj) const; + + public: + UpdateStrideForAssertionPredicates(Node* const new_stride, PhaseIdealLoop* phase) + : _new_stride(new_stride), + _phase(phase) {} + NONCOPYABLE(UpdateStrideForAssertionPredicates); + + using PredicateVisitor::visit; + + void visit(const TemplateAssertionPredicate& template_assertion_predicate) override; + + // Kill the old Initialized Assertion Predicates with old strides before unrolling. The new Initialized Assertion + // Predicates are inserted after the Template Assertion Predicate which ensures that we are not accidentally visiting + // and killing a newly created Initialized Assertion Predicate here. + void visit(const InitializedAssertionPredicate& initialized_assertion_predicate) override { + if (initialized_assertion_predicate.is_last_value()) { + // Only Last Value Initialized Assertion Predicates need to be killed and updated. + initialized_assertion_predicate.kill(_phase); + } + } +}; #endif // SHARE_OPTO_PREDICATES_HPP diff --git a/src/hotspot/share/opto/reg_split.cpp b/src/hotspot/share/opto/reg_split.cpp index 6d948aff011..9e41c397643 100644 --- a/src/hotspot/share/opto/reg_split.cpp +++ b/src/hotspot/share/opto/reg_split.cpp @@ -495,7 +495,7 @@ bool PhaseChaitin::prompt_use( Block *b, uint lidx ) { // Else, hoist LRG back up to register only (ie - split is also DEF) // We will compute a new maxlrg as we go uint PhaseChaitin::Split(uint maxlrg, ResourceArea* split_arena) { - Compile::TracePhase tp("regAllocSplit", &timers[_t_regAllocSplit]); + Compile::TracePhase tp(_t_regAllocSplit); // Free thread local resources used by this method on exit. ResourceMark rm(split_arena); diff --git a/src/hotspot/share/opto/stringopts.cpp b/src/hotspot/share/opto/stringopts.cpp index 7bf75c93055..7eb5f5da168 100644 --- a/src/hotspot/share/opto/stringopts.cpp +++ b/src/hotspot/share/opto/stringopts.cpp @@ -1325,7 +1325,7 @@ void PhaseStringOpts::getChars(GraphKit& kit, Node* arg, Node* dst_array, BasicT Node* index = __ SubI(charPos, __ intcon((bt == T_BYTE) ? 1 : 2)); Node* ch = __ AddI(r, __ intcon('0')); Node* st = __ store_to_memory(kit.control(), kit.array_element_address(dst_array, index, T_BYTE), - ch, bt, byte_adr_idx, MemNode::unordered, false /* require_atomic_access */, + ch, bt, MemNode::unordered, false /* require_atomic_access */, false /* unaligned */, (bt != T_BYTE) /* mismatched */); iff = kit.create_and_map_if(head, __ Bool(__ CmpI(q, __ intcon(0)), BoolTest::ne), @@ -1364,8 +1364,8 @@ void PhaseStringOpts::getChars(GraphKit& kit, Node* arg, Node* dst_array, BasicT } else { Node* index = __ SubI(charPos, __ intcon((bt == T_BYTE) ? 1 : 2)); st = __ store_to_memory(kit.control(), kit.array_element_address(dst_array, index, T_BYTE), - sign, bt, byte_adr_idx, MemNode::unordered, false /* require_atomic_access */, - false /* unaligned */, (bt != T_BYTE) /* mismatched */); + sign, bt, MemNode::unordered, false /* require_atomic_access */, false /* unaligned */, + (bt != T_BYTE) /* mismatched */); final_merge->init_req(merge_index + 1, kit.control()); final_mem->init_req(merge_index + 1, st); diff --git a/src/hotspot/share/opto/vector.cpp b/src/hotspot/share/opto/vector.cpp index 9ab62c282d1..99b4c62fec7 100644 --- a/src/hotspot/share/opto/vector.cpp +++ b/src/hotspot/share/opto/vector.cpp @@ -42,7 +42,7 @@ static bool is_vector_shuffle(ciKlass* klass) { void PhaseVector::optimize_vector_boxes() { - Compile::TracePhase tp("vector_elimination", &timers[_t_vector_elimination]); + Compile::TracePhase tp(_t_vector_elimination); // Signal GraphKit it's post-parse phase. assert(C->inlining_incrementally() == false, "sanity"); @@ -66,13 +66,13 @@ void PhaseVector::optimize_vector_boxes() { void PhaseVector::do_cleanup() { if (C->failing()) return; { - Compile::TracePhase tp("vector_pru", &timers[_t_vector_pru]); + Compile::TracePhase tp(_t_vector_pru); ResourceMark rm; PhaseRemoveUseless pru(C->initial_gvn(), *C->igvn_worklist()); if (C->failing()) return; } { - Compile::TracePhase tp("incrementalInline_igvn", &timers[_t_vector_igvn]); + Compile::TracePhase tp(_t_vector_igvn); _igvn.reset_from_gvn(C->initial_gvn()); _igvn.optimize(); if (C->failing()) return; diff --git a/src/hotspot/share/prims/jniFastGetField.hpp b/src/hotspot/share/prims/jniFastGetField.hpp index 525ff0aba87..0e4c4c849e0 100644 --- a/src/hotspot/share/prims/jniFastGetField.hpp +++ b/src/hotspot/share/prims/jniFastGetField.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2004, 2022, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2004, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -71,17 +71,6 @@ class JNI_FastGetField : AllStatic { #endif // AARCH64 public: -#if defined(_WINDOWS) && !defined(_WIN64) - static GetBooleanField_t jni_fast_GetBooleanField_fp; - static GetByteField_t jni_fast_GetByteField_fp; - static GetCharField_t jni_fast_GetCharField_fp; - static GetShortField_t jni_fast_GetShortField_fp; - static GetIntField_t jni_fast_GetIntField_fp; - static GetLongField_t jni_fast_GetLongField_fp; - static GetFloatField_t jni_fast_GetFloatField_fp; - static GetDoubleField_t jni_fast_GetDoubleField_fp; -#endif - static address generate_fast_get_boolean_field(); static address generate_fast_get_byte_field(); static address generate_fast_get_char_field(); diff --git a/src/hotspot/share/prims/jvm.cpp b/src/hotspot/share/prims/jvm.cpp index f0f14a05031..a027351ff7b 100644 --- a/src/hotspot/share/prims/jvm.cpp +++ b/src/hotspot/share/prims/jvm.cpp @@ -378,7 +378,7 @@ JVM_ENTRY(jobjectArray, JVM_GetProperties(JNIEnv *env)) // Add the sun.management.compiler property for the compiler's name { #undef CSIZE -#if defined(_LP64) || defined(_WIN64) +#if defined(_LP64) #define CSIZE "64-Bit " #else #define CSIZE @@ -1269,87 +1269,6 @@ JVM_ENTRY(jobject, JVM_GetProtectionDomain(JNIEnv *env, jclass cls)) JVM_END -// Returns the inherited_access_control_context field of the running thread. -JVM_ENTRY(jobject, JVM_GetInheritedAccessControlContext(JNIEnv *env, jclass cls)) - oop result = java_lang_Thread::inherited_access_control_context(thread->threadObj()); - return JNIHandles::make_local(THREAD, result); -JVM_END - -JVM_ENTRY(jobject, JVM_GetStackAccessControlContext(JNIEnv *env, jclass cls)) - if (!UsePrivilegedStack) return nullptr; - - ResourceMark rm(THREAD); - GrowableArray* local_array = new GrowableArray(12); - JvmtiVMObjectAllocEventCollector oam; - - // count the protection domains on the execution stack. We collapse - // duplicate consecutive protection domains into a single one, as - // well as stopping when we hit a privileged frame. - - oop previous_protection_domain = nullptr; - Handle privileged_context(thread, nullptr); - bool is_privileged = false; - oop protection_domain = nullptr; - - // Iterate through Java frames - vframeStream vfst(thread); - for(; !vfst.at_end(); vfst.next()) { - // get method of frame - Method* method = vfst.method(); - - // stop at the first privileged frame - if (method->method_holder() == vmClasses::AccessController_klass() && - method->name() == vmSymbols::executePrivileged_name()) - { - // this frame is privileged - is_privileged = true; - - javaVFrame *priv = vfst.asJavaVFrame(); // executePrivileged - - StackValueCollection* locals = priv->locals(); - StackValue* ctx_sv = locals->at(1); // AccessControlContext context - StackValue* clr_sv = locals->at(2); // Class caller - assert(!ctx_sv->obj_is_scalar_replaced(), "found scalar-replaced object"); - assert(!clr_sv->obj_is_scalar_replaced(), "found scalar-replaced object"); - privileged_context = ctx_sv->get_obj(); - Handle caller = clr_sv->get_obj(); - - Klass *caller_klass = java_lang_Class::as_Klass(caller()); - protection_domain = caller_klass->protection_domain(); - } else { - protection_domain = method->method_holder()->protection_domain(); - } - - if ((previous_protection_domain != protection_domain) && (protection_domain != nullptr)) { - local_array->push(Handle(thread, protection_domain)); - previous_protection_domain = protection_domain; - } - - if (is_privileged) break; - } - - - // either all the domains on the stack were system domains, or - // we had a privileged system domain - if (local_array->is_empty()) { - if (is_privileged && privileged_context.is_null()) return nullptr; - - oop result = java_security_AccessControlContext::create(objArrayHandle(), is_privileged, privileged_context, CHECK_NULL); - return JNIHandles::make_local(THREAD, result); - } - - objArrayOop context = oopFactory::new_objArray(vmClasses::ProtectionDomain_klass(), - local_array->length(), CHECK_NULL); - objArrayHandle h_context(thread, context); - for (int index = 0; index < local_array->length(); index++) { - h_context->obj_at_put(index, local_array->at(index)()); - } - - oop result = java_security_AccessControlContext::create(h_context, is_privileged, privileged_context, CHECK_NULL); - - return JNIHandles::make_local(THREAD, result); -JVM_END - class ScopedValueBindingsResolver { public: InstanceKlass* Carrier_klass; @@ -3089,6 +3008,10 @@ JVM_ENTRY(void, JVM_SetCurrentThread(JNIEnv* env, jobject thisThread, jobject theThread)) oop threadObj = JNIHandles::resolve(theThread); thread->set_vthread(threadObj); + + // Set lock id of new current Thread + thread->set_lock_id(java_lang_Thread::thread_id(threadObj)); + JFR_ONLY(Jfr::on_set_current_thread(thread, threadObj);) JVM_END @@ -3955,6 +3878,39 @@ JVM_ENTRY(void, JVM_VirtualThreadDisableSuspend(JNIEnv* env, jclass clazz, jbool #endif JVM_END +JVM_ENTRY(void, JVM_VirtualThreadPinnedEvent(JNIEnv* env, jclass ignored, jstring op)) +#if INCLUDE_JFR + freeze_result result = THREAD->last_freeze_fail_result(); + assert(result != freeze_ok, "sanity check"); + EventVirtualThreadPinned event(UNTIMED); + event.set_starttime(THREAD->last_freeze_fail_time()); + if (event.should_commit()) { + ResourceMark rm(THREAD); + const char *str = java_lang_String::as_utf8_string(JNIHandles::resolve_non_null(op)); + THREAD->post_vthread_pinned_event(&event, str, result); + } +#endif +JVM_END + +JVM_ENTRY(jobject, JVM_TakeVirtualThreadListToUnblock(JNIEnv* env, jclass ignored)) + ParkEvent* parkEvent = ObjectMonitor::vthread_unparker_ParkEvent(); + assert(parkEvent != nullptr, "not initialized"); + + OopHandle& list_head = ObjectMonitor::vthread_cxq_head(); + oop vthread_head = nullptr; + while (true) { + if (list_head.peek() != nullptr) { + for (;;) { + oop head = list_head.resolve(); + if (list_head.cmpxchg(head, nullptr) == head) { + return JNIHandles::make_local(THREAD, head); + } + } + } + ThreadBlockInVM tbivm(THREAD); + parkEvent->park(); + } +JVM_END /* * Return the current class's class file version. The low order 16 bits of the * returned jint contain the class's major version. The high order 16 bits diff --git a/src/hotspot/share/prims/jvmtiAgent.cpp b/src/hotspot/share/prims/jvmtiAgent.cpp index 1231fd5019f..6dd205082a0 100644 --- a/src/hotspot/share/prims/jvmtiAgent.cpp +++ b/src/hotspot/share/prims/jvmtiAgent.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2023, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -27,6 +27,7 @@ #include "cds/cds_globals.hpp" #include "cds/cdsConfig.hpp" #include "jni.h" +#include "jvm.h" #include "jvm_io.h" #include "jvmtifiles/jvmtiEnv.hpp" #include "prims/jvmtiEnvBase.hpp" diff --git a/src/hotspot/share/prims/jvmtiEnv.cpp b/src/hotspot/share/prims/jvmtiEnv.cpp index 097ce331540..ff3dad67960 100644 --- a/src/hotspot/share/prims/jvmtiEnv.cpp +++ b/src/hotspot/share/prims/jvmtiEnv.cpp @@ -1350,10 +1350,6 @@ JvmtiEnv::GetOwnedMonitorInfo(jthread thread, jint* owned_monitor_count_ptr, job JavaThread* calling_thread = JavaThread::current(); HandleMark hm(calling_thread); - // growable array of jvmti monitors info on the C-heap - GrowableArray *owned_monitors_list = - new (mtServiceability) GrowableArray(1, mtServiceability); - JvmtiVTMSTransitionDisabler disabler(thread); ThreadsListHandle tlh(calling_thread); @@ -1361,23 +1357,29 @@ JvmtiEnv::GetOwnedMonitorInfo(jthread thread, jint* owned_monitor_count_ptr, job oop thread_oop = nullptr; jvmtiError err = get_threadOop_and_JavaThread(tlh.list(), thread, calling_thread, &java_thread, &thread_oop); if (err != JVMTI_ERROR_NONE) { - delete owned_monitors_list; return err; } - if (java_thread != nullptr) { - Handle thread_handle(calling_thread, thread_oop); - EscapeBarrier eb(true, calling_thread, java_thread); - if (!eb.deoptimize_objects(MaxJavaStackTraceDepth)) { - delete owned_monitors_list; - return JVMTI_ERROR_OUT_OF_MEMORY; - } - // get owned monitors info with handshake - GetOwnedMonitorInfoClosure op(this, calling_thread, owned_monitors_list); - JvmtiHandshake::execute(&op, &tlh, java_thread, thread_handle); - err = op.result(); + if (LockingMode == LM_LEGACY && java_thread == nullptr) { + *owned_monitor_count_ptr = 0; + return JVMTI_ERROR_NONE; } + // growable array of jvmti monitors info on the C-heap + GrowableArray *owned_monitors_list = + new (mtServiceability) GrowableArray(1, mtServiceability); + + Handle thread_handle(calling_thread, thread_oop); + EscapeBarrier eb(java_thread != nullptr, calling_thread, java_thread); + if (!eb.deoptimize_objects(MaxJavaStackTraceDepth)) { + delete owned_monitors_list; + return JVMTI_ERROR_OUT_OF_MEMORY; + } + // get owned monitors info with handshake + GetOwnedMonitorInfoClosure op(this, calling_thread, owned_monitors_list); + JvmtiHandshake::execute(&op, &tlh, java_thread, thread_handle); + err = op.result(); + jint owned_monitor_count = owned_monitors_list->length(); if (err == JVMTI_ERROR_NONE) { if ((err = allocate(owned_monitor_count * sizeof(jobject *), @@ -1408,10 +1410,6 @@ JvmtiEnv::GetOwnedMonitorStackDepthInfo(jthread thread, jint* monitor_info_count JavaThread* calling_thread = JavaThread::current(); HandleMark hm(calling_thread); - // growable array of jvmti monitors info on the C-heap - GrowableArray *owned_monitors_list = - new (mtServiceability) GrowableArray(1, mtServiceability); - JvmtiVTMSTransitionDisabler disabler(thread); ThreadsListHandle tlh(calling_thread); @@ -1419,23 +1417,29 @@ JvmtiEnv::GetOwnedMonitorStackDepthInfo(jthread thread, jint* monitor_info_count oop thread_oop = nullptr; jvmtiError err = get_threadOop_and_JavaThread(tlh.list(), thread, calling_thread, &java_thread, &thread_oop); if (err != JVMTI_ERROR_NONE) { - delete owned_monitors_list; return err; } - if (java_thread != nullptr) { - Handle thread_handle(calling_thread, thread_oop); - EscapeBarrier eb(true, calling_thread, java_thread); - if (!eb.deoptimize_objects(MaxJavaStackTraceDepth)) { - delete owned_monitors_list; - return JVMTI_ERROR_OUT_OF_MEMORY; - } - // get owned monitors info with handshake - GetOwnedMonitorInfoClosure op(this, calling_thread, owned_monitors_list); - JvmtiHandshake::execute(&op, &tlh, java_thread, thread_handle); - err = op.result(); + if (LockingMode == LM_LEGACY && java_thread == nullptr) { + *monitor_info_count_ptr = 0; + return JVMTI_ERROR_NONE; } + // growable array of jvmti monitors info on the C-heap + GrowableArray *owned_monitors_list = + new (mtServiceability) GrowableArray(1, mtServiceability); + + Handle thread_handle(calling_thread, thread_oop); + EscapeBarrier eb(java_thread != nullptr, calling_thread, java_thread); + if (!eb.deoptimize_objects(MaxJavaStackTraceDepth)) { + delete owned_monitors_list; + return JVMTI_ERROR_OUT_OF_MEMORY; + } + // get owned monitors info with handshake + GetOwnedMonitorInfoClosure op(this, calling_thread, owned_monitors_list); + JvmtiHandshake::execute(&op, &tlh, java_thread, thread_handle); + err = op.result(); + jint owned_monitor_count = owned_monitors_list->length(); if (err == JVMTI_ERROR_NONE) { if ((err = allocate(owned_monitor_count * sizeof(jvmtiMonitorStackDepthInfo), diff --git a/src/hotspot/share/prims/jvmtiEnvBase.cpp b/src/hotspot/share/prims/jvmtiEnvBase.cpp index c28afbb1c51..422bbced802 100644 --- a/src/hotspot/share/prims/jvmtiEnvBase.cpp +++ b/src/hotspot/share/prims/jvmtiEnvBase.cpp @@ -652,13 +652,19 @@ JavaThread* JvmtiEnvBase::get_JavaThread_or_null(oop vthread) { } // An unmounted vthread may have an empty stack. -// Otherwise, it always has the yield0() and yield() frames we need to hide. -// The methods yield0() and yield() are annotated with the @JvmtiHideEvents. +// If unmounted from Java, it always has the yield0() and yield() frames we +// need to hide. The methods yield0() and yield() are annotated with the @JvmtiHideEvents. javaVFrame* -JvmtiEnvBase::skip_yield_frames_for_unmounted_vthread(javaVFrame* jvf) { +JvmtiEnvBase::skip_yield_frames_for_unmounted_vthread(oop vthread, javaVFrame* jvf) { if (jvf == nullptr) { return jvf; // empty stack is possible } + if (java_lang_VirtualThread::is_preempted(vthread)) { + // Top method should not be from Continuation class. + assert(jvf->method()->method_holder() != vmClasses::Continuation_klass(), ""); + return jvf; + } + assert(jvf->method()->jvmti_hide_events(), "sanity check"); assert(jvf->method()->method_holder() == vmClasses::Continuation_klass(), "expected Continuation class"); jvf = jvf->java_sender(); // skip yield0 frame @@ -721,7 +727,7 @@ JvmtiEnvBase::get_vthread_jvf(oop vthread) { } else { vframeStream vfs(cont); jvf = vfs.at_end() ? nullptr : vfs.asJavaVFrame(); - jvf = skip_yield_frames_for_unmounted_vthread(jvf); + jvf = skip_yield_frames_for_unmounted_vthread(vthread, jvf); } return jvf; } @@ -1023,18 +1029,18 @@ JvmtiEnvBase::get_owned_monitors(JavaThread *calling_thread, JavaThread* java_th } jvmtiError -JvmtiEnvBase::get_owned_monitors(JavaThread* calling_thread, JavaThread* java_thread, javaVFrame* jvf, - GrowableArray *owned_monitors_list) { +JvmtiEnvBase::get_owned_monitors(JavaThread* calling_thread, JavaThread* carrier, javaVFrame* jvf, + GrowableArray *owned_monitors_list, oop vthread) { jvmtiError err = JVMTI_ERROR_NONE; Thread *current_thread = Thread::current(); - assert(java_thread->is_handshake_safe_for(current_thread), + assert(carrier == nullptr || carrier->is_handshake_safe_for(current_thread), "call by myself or at handshake"); int depth = 0; for ( ; jvf != nullptr; jvf = jvf->java_sender()) { if (MaxJavaStackTraceDepth == 0 || depth++ < MaxJavaStackTraceDepth) { // check for stack too deep // Add locked objects for this frame into list. - err = get_locked_objects_in_frame(calling_thread, java_thread, jvf, owned_monitors_list, depth - 1); + err = get_locked_objects_in_frame(calling_thread, carrier, jvf, owned_monitors_list, depth - 1, vthread); if (err != JVMTI_ERROR_NONE) { return err; } @@ -1043,7 +1049,7 @@ JvmtiEnvBase::get_owned_monitors(JavaThread* calling_thread, JavaThread* java_th // Get off stack monitors. (e.g. acquired via jni MonitorEnter). JvmtiMonitorClosure jmc(calling_thread, owned_monitors_list, this); - ObjectSynchronizer::owned_monitors_iterate(&jmc, java_thread); + ObjectSynchronizer::owned_monitors_iterate(&jmc, carrier != nullptr ? carrier->threadObj() : vthread); err = jmc.error(); return err; @@ -1051,8 +1057,9 @@ JvmtiEnvBase::get_owned_monitors(JavaThread* calling_thread, JavaThread* java_th // Save JNI local handles for any objects that this frame owns. jvmtiError -JvmtiEnvBase::get_locked_objects_in_frame(JavaThread* calling_thread, JavaThread* java_thread, - javaVFrame *jvf, GrowableArray* owned_monitors_list, jint stack_depth) { +JvmtiEnvBase::get_locked_objects_in_frame(JavaThread* calling_thread, JavaThread* target, + javaVFrame *jvf, GrowableArray* owned_monitors_list, + jint stack_depth, oop vthread) { jvmtiError err = JVMTI_ERROR_NONE; Thread* current_thread = Thread::current(); ResourceMark rm(current_thread); @@ -1069,9 +1076,17 @@ JvmtiEnvBase::get_locked_objects_in_frame(JavaThread* calling_thread, JavaThread // at a safepoint or the calling thread is operating on itself so // it cannot leave the underlying wait() call. // Save object of current wait() call (if any) for later comparison. - ObjectMonitor *mon = java_thread->current_waiting_monitor(); - if (mon != nullptr) { - wait_obj = mon->object(); + if (target != nullptr) { + ObjectMonitor *mon = target->current_waiting_monitor(); + if (mon != nullptr) wait_obj = mon->object(); + } else { + assert(vthread != nullptr, "no vthread oop"); + oop cont = java_lang_VirtualThread::continuation(vthread); + assert(cont != nullptr, "vthread with no continuation"); + stackChunkOop chunk = jdk_internal_vm_Continuation::tail(cont); + assert(chunk != nullptr, "unmounted vthread should have a chunk"); + ObjectMonitor *mon = chunk->current_waiting_monitor(); + if (mon != nullptr) wait_obj = mon->object(); } } oop pending_obj = nullptr; @@ -1080,9 +1095,17 @@ JvmtiEnvBase::get_locked_objects_in_frame(JavaThread* calling_thread, JavaThread // at a safepoint or the calling thread is operating on itself so // it cannot leave the underlying enter() call. // Save object of current enter() call (if any) for later comparison. - ObjectMonitor *mon = java_thread->current_pending_monitor(); - if (mon != nullptr) { - pending_obj = mon->object(); + if (target != nullptr) { + ObjectMonitor *mon = target->current_pending_monitor(); + if (mon != nullptr) pending_obj = mon->object(); + } else { + assert(vthread != nullptr, "no vthread oop"); + oop cont = java_lang_VirtualThread::continuation(vthread); + assert(cont != nullptr, "vthread with no continuation"); + stackChunkOop chunk = jdk_internal_vm_Continuation::tail(cont); + assert(chunk != nullptr, "unmounted vthread should have a chunk"); + ObjectMonitor *mon = chunk->current_pending_monitor(); + if (mon != nullptr) pending_obj = mon->object(); } } @@ -1533,9 +1556,13 @@ JvmtiEnvBase::get_object_monitor_usage(JavaThread* calling_thread, jobject objec waiter != nullptr && (nWait == 0 || waiter != mon->first_waiter()); waiter = mon->next_waiter(waiter)) { JavaThread *w = mon->thread_of_waiter(waiter); - oop thread_oop = get_vthread_or_thread_oop(w); - if (thread_oop->is_a(vmClasses::BaseVirtualThread_klass())) { + if (w == nullptr) { skipped++; + } else { + oop thread_oop = get_vthread_or_thread_oop(w); + if (thread_oop->is_a(vmClasses::BaseVirtualThread_klass())) { + skipped++; + } } nWait++; } @@ -1583,15 +1610,19 @@ JvmtiEnvBase::get_object_monitor_usage(JavaThread* calling_thread, jobject objec jint skipped = 0; for (int i = 0; i < nWait; i++) { JavaThread *w = mon->thread_of_waiter(waiter); - oop thread_oop = get_vthread_or_thread_oop(w); - bool is_virtual = thread_oop->is_a(vmClasses::BaseVirtualThread_klass()); - assert(w != nullptr, "sanity check"); + bool is_virtual; + if (w == nullptr) { + is_virtual = true; + } else { + oop thread_oop = get_vthread_or_thread_oop(w); + is_virtual = thread_oop->is_a(vmClasses::BaseVirtualThread_klass()); + } if (is_virtual) { skipped++; } else { // If the thread was found on the ObjectWaiter list, then // it has not been notified. - Handle th(current_thread, get_vthread_or_thread_oop(w)); + Handle th(current_thread, w->threadObj()); ret.notify_waiters[i - skipped] = (jthread)jni_reference(calling_thread, th); } waiter = mon->next_waiter(waiter); @@ -2509,18 +2540,18 @@ GetOwnedMonitorInfoClosure::do_thread(Thread *target) { void GetOwnedMonitorInfoClosure::do_vthread(Handle target_h) { - assert(_target_jt != nullptr, "sanity check"); Thread* current = Thread::current(); ResourceMark rm(current); // vframes are resource allocated HandleMark hm(current); javaVFrame *jvf = JvmtiEnvBase::get_vthread_jvf(target_h()); - if (!_target_jt->is_exiting() && _target_jt->threadObj() != nullptr) { + if (_target_jt == nullptr || (!_target_jt->is_exiting() && _target_jt->threadObj() != nullptr)) { _result = ((JvmtiEnvBase *)_env)->get_owned_monitors(_calling_thread, _target_jt, jvf, - _owned_monitors_list); + _owned_monitors_list, + target_h()); } } @@ -2538,6 +2569,13 @@ GetCurrentContendedMonitorClosure::do_thread(Thread *target) { void GetCurrentContendedMonitorClosure::do_vthread(Handle target_h) { if (_target_jt == nullptr) { + oop cont = java_lang_VirtualThread::continuation(target_h()); + assert(cont != nullptr, "vthread with no continuation"); + stackChunkOop chunk = jdk_internal_vm_Continuation::tail(cont); + assert(chunk != nullptr, "unmounted vthread should have a chunk"); + if (chunk->current_pending_monitor() != nullptr) { + *_owned_monitor_ptr = JNIHandles::make_local(_calling_thread, chunk->current_pending_monitor()->object()); + } _result = JVMTI_ERROR_NONE; // target virtual thread is unmounted return; } diff --git a/src/hotspot/share/prims/jvmtiEnvBase.hpp b/src/hotspot/share/prims/jvmtiEnvBase.hpp index e8769d423c5..217731cd8f0 100644 --- a/src/hotspot/share/prims/jvmtiEnvBase.hpp +++ b/src/hotspot/share/prims/jvmtiEnvBase.hpp @@ -358,7 +358,7 @@ class JvmtiEnvBase : public CHeapObj { JavaThread* java_thread, javaVFrame *jvf, GrowableArray* owned_monitors_list, - jint depth); + jint depth, oop vthread = nullptr); public: static javaVFrame* jvf_for_thread_and_depth(JavaThread* java_thread, jint depth); @@ -366,7 +366,7 @@ class JvmtiEnvBase : public CHeapObj { static bool get_field_descriptor(Klass* k, jfieldID field, fieldDescriptor* fd); // check and skip frames hidden in mount/unmount transitions - static javaVFrame* skip_yield_frames_for_unmounted_vthread(javaVFrame* jvf); + static javaVFrame* skip_yield_frames_for_unmounted_vthread(oop vthread, javaVFrame* jvf); static javaVFrame* check_and_skip_hidden_frames(bool is_in_VTMS_transition, javaVFrame* jvf); static javaVFrame* check_and_skip_hidden_frames(JavaThread* jt, javaVFrame* jvf); @@ -422,8 +422,8 @@ class JvmtiEnvBase : public CHeapObj { jobject* monitor_ptr, bool is_virtual); jvmtiError get_owned_monitors(JavaThread* calling_thread, JavaThread* java_thread, GrowableArray *owned_monitors_list); - jvmtiError get_owned_monitors(JavaThread* calling_thread, JavaThread* java_thread, javaVFrame* jvf, - GrowableArray *owned_monitors_list); + jvmtiError get_owned_monitors(JavaThread* calling_thread, JavaThread* carrier, javaVFrame* jvf, + GrowableArray *owned_monitors_list, oop vthread); static jvmtiError check_top_frame(Thread* current_thread, JavaThread* java_thread, jvalue value, TosState tos, Handle* ret_ob_h); jvmtiError force_early_return(jthread thread, jvalue value, TosState tos); diff --git a/src/hotspot/share/prims/jvmtiExport.cpp b/src/hotspot/share/prims/jvmtiExport.cpp index b2e9fb9a4ab..087f426c537 100644 --- a/src/hotspot/share/prims/jvmtiExport.cpp +++ b/src/hotspot/share/prims/jvmtiExport.cpp @@ -2870,6 +2870,19 @@ void JvmtiExport::post_monitor_waited(JavaThread *thread, ObjectMonitor *obj_mnt } } +void JvmtiExport::vthread_post_monitor_waited(JavaThread *current, ObjectMonitor *obj_mntr, jboolean timed_out) { + Handle vthread(current, current->vthread()); + + // Finish the VTMS transition temporarily to post the event. + JvmtiVTMSTransitionDisabler::VTMS_vthread_mount((jthread)vthread.raw_value(), false); + + // Post event. + JvmtiExport::post_monitor_waited(current, obj_mntr, timed_out); + + // Go back to VTMS transition state. + JvmtiVTMSTransitionDisabler::VTMS_vthread_unmount((jthread)vthread.raw_value(), true); +} + void JvmtiExport::post_vm_object_alloc(JavaThread *thread, oop object) { if (object == nullptr) { return; diff --git a/src/hotspot/share/prims/jvmtiExport.hpp b/src/hotspot/share/prims/jvmtiExport.hpp index e98020aef1d..e0fb84f2c03 100644 --- a/src/hotspot/share/prims/jvmtiExport.hpp +++ b/src/hotspot/share/prims/jvmtiExport.hpp @@ -397,6 +397,7 @@ class JvmtiExport : public AllStatic { static void post_monitor_contended_entered(JavaThread *thread, ObjectMonitor *obj_mntr) NOT_JVMTI_RETURN; static void post_monitor_wait(JavaThread *thread, oop obj, jlong timeout) NOT_JVMTI_RETURN; static void post_monitor_waited(JavaThread *thread, ObjectMonitor *obj_mntr, jboolean timed_out) NOT_JVMTI_RETURN; + static void vthread_post_monitor_waited(JavaThread *current, ObjectMonitor *obj_mntr, jboolean timed_out) NOT_JVMTI_RETURN; static void post_object_free(JvmtiEnv* env, GrowableArray* objects) NOT_JVMTI_RETURN; static void post_resource_exhausted(jint resource_exhausted_flags, const char* detail) NOT_JVMTI_RETURN; static void record_vm_internal_object_allocation(oop object) NOT_JVMTI_RETURN; diff --git a/src/hotspot/share/prims/jvmtiTagMap.cpp b/src/hotspot/share/prims/jvmtiTagMap.cpp index bc91c105073..97a4ae10970 100644 --- a/src/hotspot/share/prims/jvmtiTagMap.cpp +++ b/src/hotspot/share/prims/jvmtiTagMap.cpp @@ -2306,10 +2306,11 @@ bool StackRefCollector::report_native_stack_refs(jmethodID method) { _blk->set_context(_thread_tag, _tid, _depth, method); if (_is_top_frame) { // JNI locals for the top frame. - assert(_java_thread != nullptr, "sanity"); - _java_thread->active_handles()->oops_do(_blk); - if (_blk->stopped()) { - return false; + if (_java_thread != nullptr) { + _java_thread->active_handles()->oops_do(_blk); + if (_blk->stopped()) { + return false; + } } } else { if (_last_entry_frame != nullptr) { diff --git a/src/hotspot/share/prims/nativeLookup.cpp b/src/hotspot/share/prims/nativeLookup.cpp index 368d5eb264e..7de782c807b 100644 --- a/src/hotspot/share/prims/nativeLookup.cpp +++ b/src/hotspot/share/prims/nativeLookup.cpp @@ -250,9 +250,9 @@ static address lookup_special_native(const char* jni_name) { return nullptr; } -address NativeLookup::lookup_style(const methodHandle& method, char* pure_name, const char* long_name, int args_size, bool os_style, TRAPS) { +address NativeLookup::lookup_style(const methodHandle& method, char* pure_name, const char* long_name, int args_size, TRAPS) { address entry; - const char* jni_name = compute_complete_jni_name(pure_name, long_name, args_size, os_style); + const char* jni_name = compute_complete_jni_name(pure_name, long_name, args_size); // If the loader is null we have a system class, so we attempt a lookup in @@ -306,17 +306,10 @@ address NativeLookup::lookup_style(const methodHandle& method, char* pure_name, return entry; } -const char* NativeLookup::compute_complete_jni_name(const char* pure_name, const char* long_name, int args_size, bool os_style) { +const char* NativeLookup::compute_complete_jni_name(const char* pure_name, const char* long_name, int args_size) { stringStream st; - if (os_style) { - os::print_jni_name_prefix_on(&st, args_size); - } - st.print_raw(pure_name); st.print_raw(long_name); - if (os_style) { - os::print_jni_name_suffix_on(&st, args_size); - } return st.as_string(); } @@ -339,7 +332,7 @@ address NativeLookup::lookup_entry(const methodHandle& method, TRAPS) { + method->size_of_parameters(); // actual parameters // 1) Try JNI short style - entry = lookup_style(method, pure_name, "", args_size, true, CHECK_NULL); + entry = lookup_style(method, pure_name, "", args_size, CHECK_NULL); if (entry != nullptr) return entry; // Compute long name @@ -351,15 +344,7 @@ address NativeLookup::lookup_entry(const methodHandle& method, TRAPS) { } // 2) Try JNI long style - entry = lookup_style(method, pure_name, long_name, args_size, true, CHECK_NULL); - if (entry != nullptr) return entry; - - // 3) Try JNI short style without os prefix/suffix - entry = lookup_style(method, pure_name, "", args_size, false, CHECK_NULL); - if (entry != nullptr) return entry; - - // 4) Try JNI long style without os prefix/suffix - entry = lookup_style(method, pure_name, long_name, args_size, false, CHECK_NULL); + entry = lookup_style(method, pure_name, long_name, args_size, CHECK_NULL); return entry; // null indicates not found } diff --git a/src/hotspot/share/prims/nativeLookup.hpp b/src/hotspot/share/prims/nativeLookup.hpp index a24768024f7..b9615377e30 100644 --- a/src/hotspot/share/prims/nativeLookup.hpp +++ b/src/hotspot/share/prims/nativeLookup.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2022, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -34,12 +34,12 @@ class NativeLookup : AllStatic { private: // Style specific lookup - static address lookup_style(const methodHandle& method, char* pure_name, const char* long_name, int args_size, bool os_style, TRAPS); + static address lookup_style(const methodHandle& method, char* pure_name, const char* long_name, int args_size, TRAPS); static address lookup_base (const methodHandle& method, TRAPS); static address lookup_entry(const methodHandle& method, TRAPS); static address lookup_entry_prefixed(const methodHandle& method, TRAPS); - static const char* compute_complete_jni_name(const char* pure_name, const char* long_name, int args_size, bool os_style); + static const char* compute_complete_jni_name(const char* pure_name, const char* long_name, int args_size); public: // JNI name computation static char* pure_jni_name(const methodHandle& method); diff --git a/src/hotspot/share/prims/whitebox.cpp b/src/hotspot/share/prims/whitebox.cpp index e19da95e075..d4b5e293c09 100644 --- a/src/hotspot/share/prims/whitebox.cpp +++ b/src/hotspot/share/prims/whitebox.cpp @@ -1721,8 +1721,13 @@ int WhiteBox::array_bytes_to_length(size_t bytes) { /////////////// // MetaspaceTestContext and MetaspaceTestArena WB_ENTRY(jlong, WB_CreateMetaspaceTestContext(JNIEnv* env, jobject wb, jlong commit_limit, jlong reserve_limit)) + assert(is_aligned(commit_limit, BytesPerWord), + "WB_CreateMetaspaceTestContext: commit_limit is not a multiple of the system word byte size"); + assert(is_aligned(reserve_limit, BytesPerWord), + "WB_CreateMetaspaceTestContext: reserve_limit is not a multiple of the system word byte size"); metaspace::MetaspaceTestContext* context = - new metaspace::MetaspaceTestContext("whitebox-metaspace-context", (size_t) commit_limit, (size_t) reserve_limit); + new metaspace::MetaspaceTestContext("whitebox-metaspace-context", (size_t) commit_limit / BytesPerWord, + (size_t) reserve_limit / BytesPerWord); return (jlong)p2i(context); WB_END @@ -1740,14 +1745,14 @@ WB_ENTRY(void, WB_PrintMetaspaceTestContext(JNIEnv* env, jobject wb, jlong conte context0->print_on(tty); WB_END -WB_ENTRY(jlong, WB_GetTotalCommittedWordsInMetaspaceTestContext(JNIEnv* env, jobject wb, jlong context)) +WB_ENTRY(jlong, WB_GetTotalCommittedBytesInMetaspaceTestContext(JNIEnv* env, jobject wb, jlong context)) metaspace::MetaspaceTestContext* context0 = (metaspace::MetaspaceTestContext*) context; - return context0->committed_words(); + return (jlong)context0->committed_words() * BytesPerWord; WB_END -WB_ENTRY(jlong, WB_GetTotalUsedWordsInMetaspaceTestContext(JNIEnv* env, jobject wb, jlong context)) +WB_ENTRY(jlong, WB_GetTotalUsedBytesInMetaspaceTestContext(JNIEnv* env, jobject wb, jlong context)) metaspace::MetaspaceTestContext* context0 = (metaspace::MetaspaceTestContext*) context; - return context0->used_words(); + return (jlong)context0->used_words() * BytesPerWord; WB_END WB_ENTRY(jlong, WB_CreateArenaInTestContext(JNIEnv* env, jobject wb, jlong context, jboolean is_micro)) @@ -1760,21 +1765,33 @@ WB_ENTRY(void, WB_DestroyMetaspaceTestArena(JNIEnv* env, jobject wb, jlong arena delete (metaspace::MetaspaceTestArena*) arena; WB_END -WB_ENTRY(jlong, WB_AllocateFromMetaspaceTestArena(JNIEnv* env, jobject wb, jlong arena, jlong word_size)) - metaspace::MetaspaceTestArena* arena0 = (metaspace::MetaspaceTestArena*) arena; - MetaWord* p = arena0->allocate((size_t) word_size); +WB_ENTRY(jlong, WB_AllocateFromMetaspaceTestArena(JNIEnv* env, jobject wb, jlong arena, jlong size)) + assert(is_aligned(size, BytesPerWord), + "WB_AllocateFromMetaspaceTestArena: size is not a multiple of the system word byte size"); + metaspace::MetaspaceTestArena *arena0 = (metaspace::MetaspaceTestArena *)arena; + MetaWord *p = arena0->allocate((size_t) size / BytesPerWord); return (jlong)p2i(p); WB_END -WB_ENTRY(void, WB_DeallocateToMetaspaceTestArena(JNIEnv* env, jobject wb, jlong arena, jlong p, jlong word_size)) +WB_ENTRY(void, WB_DeallocateToMetaspaceTestArena(JNIEnv* env, jobject wb, jlong arena, jlong p, jlong size)) + assert(is_aligned(size, BytesPerWord), + "WB_DeallocateToMetaspaceTestArena: size is not a multiple of the system word byte size"); metaspace::MetaspaceTestArena* arena0 = (metaspace::MetaspaceTestArena*) arena; - arena0->deallocate((MetaWord*)p, (size_t) word_size); + arena0->deallocate((MetaWord*)p, (size_t) size / BytesPerWord); WB_END WB_ENTRY(jlong, WB_GetMaxMetaspaceAllocationSize(JNIEnv* env, jobject wb)) return (jlong) Metaspace::max_allocation_word_size() * BytesPerWord; WB_END +WB_ENTRY(jlong, WB_WordSize(JNIEnv* env)) + return (jlong)BytesPerWord; +WB_END + +WB_ENTRY(jlong, WB_RootChunkWordSize(JNIEnv* env)) + return (jlong)Metaspace::reserve_alignment_words(); +WB_END + ////////////// WB_ENTRY(jlong, WB_AllocateMetaspace(JNIEnv* env, jobject wb, jobject class_loader, jlong size)) @@ -2956,8 +2973,8 @@ static JNINativeMethod methods[] = { {CC"destroyMetaspaceTestContext", CC"(J)V", (void*)&WB_DestroyMetaspaceTestContext}, {CC"purgeMetaspaceTestContext", CC"(J)V", (void*)&WB_PurgeMetaspaceTestContext}, {CC"printMetaspaceTestContext", CC"(J)V", (void*)&WB_PrintMetaspaceTestContext}, - {CC"getTotalCommittedWordsInMetaspaceTestContext", CC"(J)J",(void*)&WB_GetTotalCommittedWordsInMetaspaceTestContext}, - {CC"getTotalUsedWordsInMetaspaceTestContext", CC"(J)J", (void*)&WB_GetTotalUsedWordsInMetaspaceTestContext}, + {CC"getTotalCommittedBytesInMetaspaceTestContext", CC"(J)J",(void*)&WB_GetTotalCommittedBytesInMetaspaceTestContext}, + {CC"getTotalUsedBytesInMetaspaceTestContext", CC"(J)J", (void*)&WB_GetTotalUsedBytesInMetaspaceTestContext}, {CC"createArenaInTestContext", CC"(JZ)J", (void*)&WB_CreateArenaInTestContext}, {CC"destroyMetaspaceTestArena", CC"(J)V", (void*)&WB_DestroyMetaspaceTestArena}, {CC"allocateFromMetaspaceTestArena", CC"(JJ)J", (void*)&WB_AllocateFromMetaspaceTestArena}, @@ -2977,6 +2994,8 @@ static JNINativeMethod methods[] = { {CC"cleanMetaspaces", CC"()V", (void*)&WB_CleanMetaspaces}, {CC"rss", CC"()J", (void*)&WB_Rss}, {CC"printString", CC"(Ljava/lang/String;I)Ljava/lang/String;", (void*)&WB_PrintString}, + {CC"wordSize", CC"()J", (void*)&WB_WordSize}, + {CC"rootChunkWordSize", CC"()J", (void*)&WB_RootChunkWordSize} }; diff --git a/src/hotspot/share/runtime/arguments.cpp b/src/hotspot/share/runtime/arguments.cpp index bcb6b919023..e9262a48086 100644 --- a/src/hotspot/share/runtime/arguments.cpp +++ b/src/hotspot/share/runtime/arguments.cpp @@ -1429,13 +1429,6 @@ void Arguments::set_use_compressed_oops() { #endif // _LP64 } -void Arguments::set_use_compressed_klass_ptrs() { -#ifdef _LP64 - assert(!UseCompressedClassPointers || CompressedClassSpaceSize <= KlassEncodingMetaspaceMax, - "CompressedClassSpaceSize is too large for UseCompressedClassPointers"); -#endif // _LP64 -} - void Arguments::set_conservative_max_heap_alignment() { // The conservative maximum required alignment for the heap is the maximum of // the alignments imposed by several sources: any requirements from the heap @@ -1454,7 +1447,6 @@ jint Arguments::set_ergonomics_flags() { #ifdef _LP64 set_use_compressed_oops(); - set_use_compressed_klass_ptrs(); // Also checks that certain machines are slower with compressed oops // in vm_version initialization code. @@ -1825,6 +1817,15 @@ bool Arguments::check_vm_args_consistency() { } #endif +#ifndef _LP64 + if (LockingMode == LM_LEGACY) { + FLAG_SET_CMDLINE(LockingMode, LM_LIGHTWEIGHT); + // Self-forwarding in bit 3 of the mark-word conflicts + // with 4-byte-aligned stack-locks. + warning("Legacy locking not supported on this platform"); + } +#endif + if (UseObjectMonitorTable && LockingMode != LM_LIGHTWEIGHT) { // ObjectMonitorTable requires lightweight locking. FLAG_SET_CMDLINE(UseObjectMonitorTable, false); @@ -3648,6 +3649,32 @@ jint Arguments::parse(const JavaVMInitArgs* initial_cmd_args) { Arguments::print_on(&st); } +#ifdef _LP64 + if (UseCompactObjectHeaders && FLAG_IS_CMDLINE(UseCompressedClassPointers) && !UseCompressedClassPointers) { + warning("Compact object headers require compressed class pointers. Disabling compact object headers."); + FLAG_SET_DEFAULT(UseCompactObjectHeaders, false); + } + if (UseCompactObjectHeaders && LockingMode != LM_LIGHTWEIGHT) { + FLAG_SET_DEFAULT(LockingMode, LM_LIGHTWEIGHT); + } + if (UseCompactObjectHeaders && !UseObjectMonitorTable) { + // If UseCompactObjectHeaders is on the command line, turn on UseObjectMonitorTable. + if (FLAG_IS_CMDLINE(UseCompactObjectHeaders)) { + FLAG_SET_DEFAULT(UseObjectMonitorTable, true); + + // If UseObjectMonitorTable is on the command line, turn off UseCompactObjectHeaders. + } else if (FLAG_IS_CMDLINE(UseObjectMonitorTable)) { + FLAG_SET_DEFAULT(UseCompactObjectHeaders, false); + // If neither on the command line, the defaults are incompatible, but turn on UseObjectMonitorTable. + } else { + FLAG_SET_DEFAULT(UseObjectMonitorTable, true); + } + } + if (UseCompactObjectHeaders && !UseCompressedClassPointers) { + FLAG_SET_DEFAULT(UseCompressedClassPointers, true); + } +#endif + return JNI_OK; } @@ -3661,6 +3688,10 @@ jint Arguments::apply_ergo() { GCConfig::arguments()->initialize(); + if (UseCompressedClassPointers) { + CompressedKlassPointers::pre_initialize(); + } + CDSConfig::initialize(); // Initialize Metaspace flags and alignments diff --git a/src/hotspot/share/runtime/arguments.hpp b/src/hotspot/share/runtime/arguments.hpp index ac842285fd8..2105d21430a 100644 --- a/src/hotspot/share/runtime/arguments.hpp +++ b/src/hotspot/share/runtime/arguments.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -263,7 +263,6 @@ class Arguments : AllStatic { // GC ergonomics static void set_conservative_max_heap_alignment(); static void set_use_compressed_oops(); - static void set_use_compressed_klass_ptrs(); static jint set_ergonomics_flags(); // Limits the given heap size by the maximum amount of virtual // memory this process is currently allowed to use. It also takes diff --git a/src/hotspot/share/runtime/basicLock.hpp b/src/hotspot/share/runtime/basicLock.hpp index 193aa63cf8f..cbdb65c8ec6 100644 --- a/src/hotspot/share/runtime/basicLock.hpp +++ b/src/hotspot/share/runtime/basicLock.hpp @@ -88,6 +88,7 @@ class BasicObjectLock { public: // Manipulation oop obj() const { return _obj; } + oop* obj_adr() { return &_obj; } void set_obj(oop obj) { _obj = obj; } BasicLock* lock() { return &_lock; } diff --git a/src/hotspot/share/runtime/continuation.cpp b/src/hotspot/share/runtime/continuation.cpp index fb697dd4920..fd9e4615498 100644 --- a/src/hotspot/share/runtime/continuation.cpp +++ b/src/hotspot/share/runtime/continuation.cpp @@ -26,6 +26,8 @@ #include "classfile/vmSymbols.hpp" #include "gc/shared/barrierSetNMethod.hpp" #include "oops/method.inline.hpp" +#include "oops/oop.inline.hpp" +#include "prims/jvmtiThreadState.inline.hpp" #include "runtime/continuation.hpp" #include "runtime/continuationEntry.inline.hpp" #include "runtime/continuationHelper.inline.hpp" @@ -33,6 +35,7 @@ #include "runtime/continuationWrapper.inline.hpp" #include "runtime/interfaceSupport.inline.hpp" #include "runtime/javaThread.inline.hpp" +#include "runtime/jniHandles.inline.hpp" #include "runtime/osThread.hpp" #include "runtime/vframe.inline.hpp" #include "runtime/vframe_hp.hpp" @@ -54,6 +57,112 @@ JVM_ENTRY(void, CONT_unpin(JNIEnv* env, jclass cls)) { } JVM_END +#if INCLUDE_JVMTI +class JvmtiUnmountBeginMark : public StackObj { + Handle _vthread; + JavaThread* _target; + freeze_result _result; + bool _failed; + + public: + JvmtiUnmountBeginMark(JavaThread* t) : + _vthread(t, t->vthread()), _target(t), _result(freeze_pinned_native), _failed(false) { + assert(!_target->is_in_VTMS_transition(), "must be"); + + if (JvmtiVTMSTransitionDisabler::VTMS_notify_jvmti_events()) { + JvmtiVTMSTransitionDisabler::VTMS_vthread_unmount((jthread)_vthread.raw_value(), true); + + // Don't preempt if there is a pending popframe or earlyret operation. This can + // be installed in start_VTMS_transition() so we need to check it here. + if (JvmtiExport::can_pop_frame() || JvmtiExport::can_force_early_return()) { + JvmtiThreadState* state = _target->jvmti_thread_state(); + if (_target->has_pending_popframe() || (state != nullptr && state->is_earlyret_pending())) { + _failed = true; + } + } + + // Don't preempt in case there is an async exception installed since + // we would incorrectly throw it during the unmount logic in the carrier. + if (_target->has_async_exception_condition()) { + _failed = true; + } + } else { + _target->set_is_in_VTMS_transition(true); + java_lang_Thread::set_is_in_VTMS_transition(_vthread(), true); + } + } + ~JvmtiUnmountBeginMark() { + assert(!_target->is_suspended(), "must be"); + + assert(_target->is_in_VTMS_transition(), "must be"); + assert(java_lang_Thread::is_in_VTMS_transition(_vthread()), "must be"); + + // Read it again since for late binding agents the flag could have + // been set while blocked in the allocation path during freeze. + bool jvmti_present = JvmtiVTMSTransitionDisabler::VTMS_notify_jvmti_events(); + + if (_result != freeze_ok) { + // Undo transition + if (jvmti_present) { + JvmtiVTMSTransitionDisabler::VTMS_vthread_mount((jthread)_vthread.raw_value(), false); + } else { + _target->set_is_in_VTMS_transition(false); + java_lang_Thread::set_is_in_VTMS_transition(_vthread(), false); + } + } + } + void set_result(freeze_result res) { _result = res; } + bool failed() { return _failed; } +}; + +static bool is_vthread_safe_to_preempt_for_jvmti(JavaThread* target) { + if (target->is_in_VTMS_transition()) { + // We caught target at the end of a mount transition. + return false; + } + return true; +} +#endif // INCLUDE_JVMTI + +static bool is_vthread_safe_to_preempt(JavaThread* target, oop vthread) { + assert(java_lang_VirtualThread::is_instance(vthread), ""); + if (java_lang_VirtualThread::state(vthread) != java_lang_VirtualThread::RUNNING) { // inside transition + return false; + } + return JVMTI_ONLY(is_vthread_safe_to_preempt_for_jvmti(target)) NOT_JVMTI(true); +} + +typedef freeze_result (*FreezeContFnT)(JavaThread*, intptr_t*); + +static void verify_preempt_preconditions(JavaThread* target, oop continuation) { + assert(target == JavaThread::current(), "no support for external preemption"); + assert(target->has_last_Java_frame(), ""); + assert(!target->preempting(), ""); + assert(target->last_continuation() != nullptr, ""); + assert(target->last_continuation()->cont_oop(target) == continuation, ""); + assert(Continuation::continuation_scope(continuation) == java_lang_VirtualThread::vthread_scope(), ""); + assert(!target->has_pending_exception(), ""); +} + +freeze_result Continuation::try_preempt(JavaThread* target, oop continuation) { + verify_preempt_preconditions(target, continuation); + + if (LockingMode == LM_LEGACY) { + return freeze_unsupported; + } + + if (!is_vthread_safe_to_preempt(target, target->vthread())) { + return freeze_pinned_native; + } + + JVMTI_ONLY(JvmtiUnmountBeginMark jubm(target);) + JVMTI_ONLY(if (jubm.failed()) return freeze_pinned_native;) + freeze_result res = CAST_TO_FN_PTR(FreezeContFnT, freeze_preempt_entry())(target, target->last_Java_sp()); + log_trace(continuations, preempt)("try_preempt: %d", res); + JVMTI_ONLY(jubm.set_result(res);) + return res; +} + #ifndef PRODUCT static jlong java_tid(JavaThread* thread) { return java_lang_Thread::thread_id(thread->threadObj()); diff --git a/src/hotspot/share/runtime/continuation.hpp b/src/hotspot/share/runtime/continuation.hpp index e579e993f07..e678e0bd42b 100644 --- a/src/hotspot/share/runtime/continuation.hpp +++ b/src/hotspot/share/runtime/continuation.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2018, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -47,9 +47,26 @@ void continuations_init(); class javaVFrame; class JavaThread; +// should match Continuation.pinnedReason() in Continuation.java +enum freeze_result { + freeze_ok = 0, + freeze_ok_bottom = 1, + freeze_pinned_cs = 2, + freeze_pinned_native = 3, + freeze_pinned_monitor = 4, + freeze_exception = 5, + freeze_not_mounted = 6, + freeze_unsupported = 7 +}; + class Continuation : AllStatic { public: + enum preempt_kind { + freeze_on_monitorenter, + freeze_on_wait + }; + enum thaw_kind { thaw_top = 0, thaw_return_barrier = 1, @@ -69,9 +86,12 @@ class Continuation : AllStatic { static void init(); static address freeze_entry(); + static address freeze_preempt_entry(); static int prepare_thaw(JavaThread* thread, bool return_barrier); static address thaw_entry(); + static freeze_result try_preempt(JavaThread* target, oop continuation); + static ContinuationEntry* get_continuation_entry_for_continuation(JavaThread* thread, oop continuation); static ContinuationEntry* get_continuation_entry_for_sp(JavaThread* thread, intptr_t* const sp); static ContinuationEntry* get_continuation_entry_for_entry_frame(JavaThread* thread, const frame& f); diff --git a/src/hotspot/share/runtime/continuationEntry.cpp b/src/hotspot/share/runtime/continuationEntry.cpp index 31b062292f0..254287df18d 100644 --- a/src/hotspot/share/runtime/continuationEntry.cpp +++ b/src/hotspot/share/runtime/continuationEntry.cpp @@ -36,16 +36,23 @@ #include "runtime/stubRoutines.hpp" int ContinuationEntry::_return_pc_offset = 0; +int ContinuationEntry::_thaw_call_pc_offset = 0; +int ContinuationEntry::_cleanup_offset = 0; address ContinuationEntry::_return_pc = nullptr; +address ContinuationEntry::_thaw_call_pc = nullptr; +address ContinuationEntry::_cleanup_pc = nullptr; nmethod* ContinuationEntry::_enter_special = nullptr; int ContinuationEntry::_interpreted_entry_offset = 0; void ContinuationEntry::set_enter_code(nmethod* nm, int interpreted_entry_offset) { assert(_return_pc_offset != 0, ""); _return_pc = nm->code_begin() + _return_pc_offset; + _thaw_call_pc = nm->code_begin() + _thaw_call_pc_offset; + _cleanup_pc = nm->code_begin() + _cleanup_offset; _enter_special = nm; _interpreted_entry_offset = interpreted_entry_offset; + assert(_enter_special->code_contains(compiled_entry()), "entry not in enterSpecial"); assert(_enter_special->code_contains(interpreted_entry()), "entry not in enterSpecial"); assert(interpreted_entry() < compiled_entry(), "unexpected code layout"); diff --git a/src/hotspot/share/runtime/continuationEntry.hpp b/src/hotspot/share/runtime/continuationEntry.hpp index ac76cd6f088..990ee6c6a42 100644 --- a/src/hotspot/share/runtime/continuationEntry.hpp +++ b/src/hotspot/share/runtime/continuationEntry.hpp @@ -57,11 +57,16 @@ class ContinuationEntry { public: static int _return_pc_offset; // friend gen_continuation_enter + static int _thaw_call_pc_offset; + static int _cleanup_offset; + static void set_enter_code(nmethod* nm, int interpreted_entry_offset); static bool is_interpreted_call(address call_address); private: static address _return_pc; + static address _thaw_call_pc; + static address _cleanup_pc; static nmethod* _enter_special; static int _interpreted_entry_offset; @@ -101,6 +106,9 @@ class ContinuationEntry { intptr_t* entry_sp() const { return (intptr_t*)this; } intptr_t* entry_fp() const; + static address thaw_call_pc_address() { return (address)&_thaw_call_pc; } + static address cleanup_pc() { return _cleanup_pc; } + static address compiled_entry(); static address interpreted_entry(); diff --git a/src/hotspot/share/runtime/continuationFreezeThaw.cpp b/src/hotspot/share/runtime/continuationFreezeThaw.cpp index ce2e2bdb9ff..be4fef5255c 100644 --- a/src/hotspot/share/runtime/continuationFreezeThaw.cpp +++ b/src/hotspot/share/runtime/continuationFreezeThaw.cpp @@ -55,6 +55,7 @@ #include "runtime/javaThread.inline.hpp" #include "runtime/jniHandles.inline.hpp" #include "runtime/keepStackGCProcessed.hpp" +#include "runtime/objectMonitor.inline.hpp" #include "runtime/orderAccess.hpp" #include "runtime/prefetch.inline.hpp" #include "runtime/smallRegisterMap.inline.hpp" @@ -66,6 +67,7 @@ #include "utilities/debug.hpp" #include "utilities/exceptions.hpp" #include "utilities/macros.hpp" +#include "utilities/vmError.hpp" #if INCLUDE_ZGC #include "gc/z/zStackChunkGCData.inline.hpp" #endif @@ -179,6 +181,7 @@ static void verify_continuation(oop continuation) { Continuation::debug_verify_c static void do_deopt_after_thaw(JavaThread* thread); static bool do_verify_after_thaw(JavaThread* thread, stackChunkOop chunk, outputStream* st); static void log_frames(JavaThread* thread); +static void log_frames_after_thaw(JavaThread* thread, ContinuationWrapper& cont, intptr_t* sp, bool preempted); static void print_frame_layout(const frame& f, bool callee_complete, outputStream* st = tty); #define assert_pfl(p, ...) \ @@ -198,27 +201,8 @@ static void verify_continuation(oop continuation) { } #define assert_pfl(p, ...) #endif -// should match Continuation.preemptStatus() in Continuation.java -enum freeze_result { - freeze_ok = 0, - freeze_ok_bottom = 1, - freeze_pinned_cs = 2, - freeze_pinned_native = 3, - freeze_pinned_monitor = 4, - freeze_exception = 5 -}; - -const char* freeze_result_names[6] = { - "freeze_ok", - "freeze_ok_bottom", - "freeze_pinned_cs", - "freeze_pinned_native", - "freeze_pinned_monitor", - "freeze_exception" -}; - static freeze_result is_pinned0(JavaThread* thread, oop cont_scope, bool safepoint); -template static inline int freeze_internal(JavaThread* current, intptr_t* const sp); +template static inline freeze_result freeze_internal(JavaThread* current, intptr_t* const sp); static inline int prepare_thaw_internal(JavaThread* thread, bool return_barrier); template static inline intptr_t* thaw_internal(JavaThread* thread, const Continuation::thaw_kind kind); @@ -234,7 +218,7 @@ static JRT_BLOCK_ENTRY(int, freeze(JavaThread* current, intptr_t* sp)) current->set_cont_fastpath(nullptr); } - return ConfigT::freeze(current, sp); + return checked_cast(ConfigT::freeze(current, sp)); JRT_END JRT_LEAF(int, Continuation::prepare_thaw(JavaThread* thread, bool return_barrier)) @@ -245,8 +229,11 @@ template static JRT_LEAF(intptr_t*, thaw(JavaThread* thread, int kind)) // TODO: JRT_LEAF and NoHandleMark is problematic for JFR events. // vFrameStreamCommon allocates Handles in RegisterMap for continuations. + // Also the preemption case with JVMTI events enabled might safepoint so + // undo the NoSafepointVerifier here and rely on handling by ContinuationWrapper. // JRT_ENTRY instead? ResetNoHandleMark rnhm; + DEBUG_ONLY(PauseNoSafepointVerifier pnsv(&__nsv);) // we might modify the code cache via BarrierSetNMethod::nmethod_entry_barrier MACOS_AARCH64_ONLY(ThreadWXEnable __wx(WXWrite, thread)); @@ -268,8 +255,12 @@ class Config { typedef Config SelfT; using OopT = std::conditional_t; - static int freeze(JavaThread* thread, intptr_t* const sp) { - return freeze_internal(thread, sp); + static freeze_result freeze(JavaThread* thread, intptr_t* const sp) { + return freeze_internal(thread, sp); + } + + static freeze_result freeze_preempt(JavaThread* thread, intptr_t* const sp) { + return freeze_internal(thread, sp); } static intptr_t* thaw(JavaThread* thread, Continuation::thaw_kind kind) { @@ -312,14 +303,13 @@ static oop get_continuation(JavaThread* thread) { assert(thread->threadObj() != nullptr, ""); return java_lang_Thread::continuation(thread->threadObj()); } +#endif // ASSERT inline void clear_anchor(JavaThread* thread) { thread->frame_anchor()->clear(); } -static void set_anchor(JavaThread* thread, intptr_t* sp) { - address pc = ContinuationHelper::return_address_at( - sp - frame::sender_sp_ret_address_offset()); +static void set_anchor(JavaThread* thread, intptr_t* sp, address pc) { assert(pc != nullptr, ""); JavaFrameAnchor* anchor = thread->frame_anchor(); @@ -330,7 +320,12 @@ static void set_anchor(JavaThread* thread, intptr_t* sp) { assert(thread->has_last_Java_frame(), ""); assert(thread->last_frame().cb() != nullptr, ""); } -#endif // ASSERT + +static void set_anchor(JavaThread* thread, intptr_t* sp) { + address pc = ContinuationHelper::return_address_at( + sp - frame::sender_sp_ret_address_offset()); + set_anchor(thread, sp, pc); +} static void set_anchor_to_entry(JavaThread* thread, ContinuationEntry* entry) { JavaFrameAnchor* anchor = thread->frame_anchor(); @@ -373,11 +368,16 @@ class FreezeBase : public StackObj { JavaThread* const _thread; ContinuationWrapper& _cont; bool _barriers; // only set when we allocate a chunk - const bool _preempt; // used only on the slow path - const intptr_t * const _frame_sp; // Top frame sp for this freeze intptr_t* _bottom_address; + // Used for preemption only + const bool _preempt; + frame _last_frame; + + // Used to support freezing with held monitors + int _monitors_in_lockstack; + int _freeze_size; // total size of all frames plus metadata in words. int _total_align_size; @@ -397,7 +397,7 @@ class FreezeBase : public StackObj { NOT_PRODUCT(int _frames;) DEBUG_ONLY(intptr_t* _last_write;) - inline FreezeBase(JavaThread* thread, ContinuationWrapper& cont, intptr_t* sp); + inline FreezeBase(JavaThread* thread, ContinuationWrapper& cont, intptr_t* sp, bool preempt); public: NOINLINE freeze_result freeze_slow(); @@ -408,6 +408,8 @@ class FreezeBase : public StackObj { inline int size_if_fast_freeze_available(); + inline frame& last_frame() { return _last_frame; } + #ifdef ASSERT bool check_valid_fast_path(); #endif @@ -429,9 +431,9 @@ class FreezeBase : public StackObj { private: // slow path frame freeze_start_frame(); - frame freeze_start_frame_safepoint_stub(frame f); + frame freeze_start_frame_on_preempt(); NOINLINE freeze_result recurse_freeze(frame& f, frame& caller, int callee_argsize, bool callee_interpreted, bool top); - inline frame freeze_start_frame_yield_stub(frame f); + inline frame freeze_start_frame_yield_stub(); template inline freeze_result recurse_freeze_java_frame(const frame& f, frame& caller, int fsize, int argsize); inline void before_freeze_java_frame(const frame& f, const frame& caller, int fsize, int argsize, bool is_bottom_frame); @@ -441,8 +443,11 @@ class FreezeBase : public StackObj { NOINLINE freeze_result recurse_freeze_interpreted_frame(frame& f, frame& caller, int callee_argsize, bool callee_interpreted); freeze_result recurse_freeze_compiled_frame(frame& f, frame& caller, int callee_argsize, bool callee_interpreted); NOINLINE freeze_result recurse_freeze_stub_frame(frame& f, frame& caller); + NOINLINE freeze_result recurse_freeze_native_frame(frame& f, frame& caller); NOINLINE void finish_freeze(const frame& f, const frame& top); + void freeze_lockstack(stackChunkOop chunk); + inline bool stack_overflow(); static frame sender(const frame& f) { return f.is_interpreted_frame() ? sender(f) @@ -452,6 +457,7 @@ class FreezeBase : public StackObj { inline void set_top_frame_metadata_pd(const frame& hf); inline void patch_pd(frame& callee, const frame& caller); void adjust_interpreted_frame_unextended_sp(frame& f); + static inline void prepare_freeze_interpreted_top_frame(frame& f); static inline void relativize_interpreted_frame_metadata(const frame& f, const frame& hf); protected: @@ -465,8 +471,8 @@ class Freeze : public FreezeBase { stackChunkOop allocate_chunk(size_t stack_size, int argsize_md); public: - inline Freeze(JavaThread* thread, ContinuationWrapper& cont, intptr_t* frame_sp) - : FreezeBase(thread, cont, frame_sp) {} + inline Freeze(JavaThread* thread, ContinuationWrapper& cont, intptr_t* frame_sp, bool preempt) + : FreezeBase(thread, cont, frame_sp, preempt) {} freeze_result try_freeze_fast(); @@ -474,8 +480,8 @@ class Freeze : public FreezeBase { virtual stackChunkOop allocate_chunk_slow(size_t stack_size, int argsize_md) override { return allocate_chunk(stack_size, argsize_md); } }; -FreezeBase::FreezeBase(JavaThread* thread, ContinuationWrapper& cont, intptr_t* frame_sp) : - _thread(thread), _cont(cont), _barriers(false), _preempt(false), _frame_sp(frame_sp) { +FreezeBase::FreezeBase(JavaThread* thread, ContinuationWrapper& cont, intptr_t* frame_sp, bool preempt) : + _thread(thread), _cont(cont), _barriers(false), _preempt(preempt), _last_frame(false /* no initialization */) { DEBUG_ONLY(_jvmti_event_collector = nullptr;) assert(_thread != nullptr, ""); @@ -506,16 +512,27 @@ FreezeBase::FreezeBase(JavaThread* thread, ContinuationWrapper& cont, intptr_t* #else static const int doYield_stub_frame_size = frame::native_abi_reg_args_size >> LogBytesPerWord; #endif - assert(SharedRuntime::cont_doYield_stub()->frame_size() == doYield_stub_frame_size, ""); + // With preemption doYield() might not have been resolved yet + assert(_preempt || SharedRuntime::cont_doYield_stub()->frame_size() == doYield_stub_frame_size, ""); + + if (preempt) { + _last_frame = _thread->last_frame(); + } // properties of the continuation on the stack; all sizes are in words - _cont_stack_top = frame_sp + doYield_stub_frame_size; // we don't freeze the doYield stub frame + _cont_stack_top = frame_sp + (!preempt ? doYield_stub_frame_size : 0); // we don't freeze the doYield stub frame _cont_stack_bottom = _cont.entrySP() + (_cont.argsize() == 0 ? frame::metadata_words_at_top : 0) - ContinuationHelper::frame_align_words(_cont.argsize()); // see alignment in thaw log_develop_trace(continuations)("freeze size: %d argsize: %d top: " INTPTR_FORMAT " bottom: " INTPTR_FORMAT, cont_size(), _cont.argsize(), p2i(_cont_stack_top), p2i(_cont_stack_bottom)); assert(cont_size() > 0, ""); + + if (LockingMode != LM_LIGHTWEIGHT) { + _monitors_in_lockstack = 0; + } else { + _monitors_in_lockstack = _thread->lock_stack().monitor_count(); + } } void FreezeBase::init_rest() { // we want to postpone some initialization after chunk handling @@ -524,6 +541,14 @@ void FreezeBase::init_rest() { // we want to postpone some initialization after NOT_PRODUCT(_frames = 0;) } +void FreezeBase::freeze_lockstack(stackChunkOop chunk) { + assert(chunk->sp_address() - chunk->start_address() >= _monitors_in_lockstack, "no room for lockstack"); + + _thread->lock_stack().move_to_address((oop*)chunk->start_address()); + chunk->set_lockstack_size(checked_cast(_monitors_in_lockstack)); + chunk->set_has_lockstack(true); +} + void FreezeBase::copy_to_chunk(intptr_t* from, intptr_t* to, int size) { stackChunkOop chunk = _cont.tail(); chunk->copy_from_stack_to_chunk(from, to, size); @@ -554,7 +579,7 @@ freeze_result Freeze::try_freeze_fast() { DEBUG_ONLY(_fast_freeze_size = size_if_fast_freeze_available();) assert(_fast_freeze_size == 0, ""); - stackChunkOop chunk = allocate_chunk(cont_size() + frame::metadata_words, _cont.argsize() + frame::metadata_words_at_top); + stackChunkOop chunk = allocate_chunk(cont_size() + frame::metadata_words + _monitors_in_lockstack, _cont.argsize() + frame::metadata_words_at_top); if (freeze_fast_new_chunk(chunk)) { return freeze_ok; } @@ -587,6 +612,8 @@ int FreezeBase::size_if_fast_freeze_available() { total_size_needed -= _cont.argsize() + frame::metadata_words_at_top; } + total_size_needed += _monitors_in_lockstack; + int chunk_free_room = chunk_sp - frame::metadata_words_at_bottom; bool available = chunk_free_room >= total_size_needed; log_develop_trace(continuations)("chunk available: %s size: %d argsize: %d top: " INTPTR_FORMAT " bottom: " INTPTR_FORMAT, @@ -668,7 +695,7 @@ bool FreezeBase::freeze_fast_new_chunk(stackChunkOop chunk) { // in a fresh chunk, we freeze *with* the bottom-most frame's stack arguments. // They'll then be stored twice: in the chunk and in the parent chunk's top frame - const int chunk_start_sp = cont_size() + frame::metadata_words; + const int chunk_start_sp = cont_size() + frame::metadata_words + _monitors_in_lockstack; assert(chunk_start_sp == chunk->stack_size(), ""); DEBUG_ONLY(_orig_chunk_sp = chunk->start_address() + chunk_start_sp;) @@ -697,7 +724,7 @@ void FreezeBase::freeze_fast_copy(stackChunkOop chunk, int chunk_start_sp CONT_J assert(chunk_start_sp >= cont_size(), "no room in the chunk"); const int chunk_new_sp = chunk_start_sp - cont_size(); // the chunk's new sp, after freeze - assert(!(_fast_freeze_size > 0) || _orig_chunk_sp - (chunk->start_address() + chunk_new_sp) == _fast_freeze_size, ""); + assert(!(_fast_freeze_size > 0) || (_orig_chunk_sp - (chunk->start_address() + chunk_new_sp)) == (_fast_freeze_size - _monitors_in_lockstack), ""); intptr_t* chunk_top = chunk->start_address() + chunk_new_sp; #ifdef ASSERT @@ -734,9 +761,24 @@ void FreezeBase::freeze_fast_copy(stackChunkOop chunk, int chunk_start_sp CONT_J // We're always writing to a young chunk, so the GC can't see it until the next safepoint. chunk->set_sp(chunk_new_sp); + // set chunk->pc to the return address of the topmost frame in the chunk - chunk->set_pc(ContinuationHelper::return_address_at( + if (_preempt) { + // On aarch64/riscv64, the return pc of the top frame won't necessarily be at sp[-1]. + // Also, on x64, if the top frame is the native wrapper frame, sp[-1] will not + // be the pc we used when creating the oopmap. Get the top's frame last pc from + // the anchor instead. + address last_pc = _last_frame.pc(); + ContinuationHelper::patch_return_address_at(chunk_top - frame::sender_sp_ret_address_offset(), last_pc); + chunk->set_pc(last_pc); + } else { + chunk->set_pc(ContinuationHelper::return_address_at( _cont_stack_top - frame::sender_sp_ret_address_offset())); + } + + if (_monitors_in_lockstack > 0) { + freeze_lockstack(chunk); + } _cont.write(); @@ -802,43 +844,32 @@ NOINLINE freeze_result FreezeBase::freeze_slow() { } frame FreezeBase::freeze_start_frame() { - frame f = _thread->last_frame(); if (LIKELY(!_preempt)) { - return freeze_start_frame_yield_stub(f); + return freeze_start_frame_yield_stub(); } else { - return freeze_start_frame_safepoint_stub(f); + return freeze_start_frame_on_preempt(); } } -frame FreezeBase::freeze_start_frame_yield_stub(frame f) { +frame FreezeBase::freeze_start_frame_yield_stub() { + frame f = _thread->last_frame(); assert(SharedRuntime::cont_doYield_stub()->contains(f.pc()), "must be"); f = sender(f); assert(Continuation::is_frame_in_continuation(_thread->last_continuation(), f), ""); return f; } -frame FreezeBase::freeze_start_frame_safepoint_stub(frame f) { -#if (defined(X86) || defined(AARCH64) || defined(RISCV64)) && !defined(ZERO) - f.set_fp(f.real_fp()); // f.set_fp(*Frame::callee_link_address(f)); // ???? -#else - Unimplemented(); -#endif - if (!Interpreter::contains(f.pc())) { - assert(ContinuationHelper::Frame::is_stub(f.cb()), "must be"); - assert(f.oop_map() != nullptr, "must be"); - - if (Interpreter::contains(ContinuationHelper::StubFrame::return_pc(f))) { - f = sender(f); // Safepoint stub in interpreter - } - } - assert(Continuation::is_frame_in_continuation(_thread->last_continuation(), f), ""); - return f; +frame FreezeBase::freeze_start_frame_on_preempt() { + assert(_last_frame.sp() == _thread->last_frame().sp(), "_last_frame should be already initialized"); + assert(Continuation::is_frame_in_continuation(_thread->last_continuation(), _last_frame), ""); + return _last_frame; } // The parameter callee_argsize includes metadata that has to be part of caller/callee overlap. NOINLINE freeze_result FreezeBase::recurse_freeze(frame& f, frame& caller, int callee_argsize, bool callee_interpreted, bool top) { assert(f.unextended_sp() < _bottom_address, ""); // see recurse_freeze_java_frame - assert(f.is_interpreted_frame() || ((top && _preempt) == ContinuationHelper::Frame::is_stub(f.cb())), ""); + assert(f.is_interpreted_frame() || ((top && _preempt) == ContinuationHelper::Frame::is_stub(f.cb())) + || ((top && _preempt) == f.is_native_frame()), ""); if (stack_overflow()) { return freeze_exception; @@ -851,16 +882,14 @@ NOINLINE freeze_result FreezeBase::recurse_freeze(frame& f, frame& caller, int c } return recurse_freeze_compiled_frame(f, caller, callee_argsize, callee_interpreted); } else if (f.is_interpreted_frame()) { - assert((_preempt && top) || !f.interpreter_frame_method()->is_native(), ""); - if (_preempt && top && f.interpreter_frame_method()->is_native()) { - // int native entry - return freeze_pinned_native; - } - + assert(!f.interpreter_frame_method()->is_native() || (top && _preempt), ""); return recurse_freeze_interpreted_frame(f, caller, callee_argsize, callee_interpreted); - } else if (_preempt && top && ContinuationHelper::Frame::is_stub(f.cb())) { - return recurse_freeze_stub_frame(f, caller); + } else if (top && _preempt) { + assert(f.is_native_frame() || f.is_runtime_frame(), ""); + return f.is_native_frame() ? recurse_freeze_native_frame(f, caller) : recurse_freeze_stub_frame(f, caller); } else { + // Frame can't be frozen. Most likely the call_stub or upcall_stub + // which indicates there are further natives frames up the stack. return freeze_pinned_native; } } @@ -920,6 +949,7 @@ inline void FreezeBase::after_freeze_java_frame(const frame& hf, bool is_bottom_ freeze_result FreezeBase::finalize_freeze(const frame& callee, frame& caller, int argsize_md) { int argsize = argsize_md - frame::metadata_words_at_top; assert(callee.is_interpreted_frame() + || ContinuationHelper::Frame::is_stub(callee.cb()) || callee.cb()->as_nmethod()->is_osr_method() || argsize == _cont.argsize(), "argsize: %d cont.argsize: %d", argsize, _cont.argsize()); log_develop_trace(continuations)("bottom: " INTPTR_FORMAT " count %d size: %d argsize: %d", @@ -962,6 +992,8 @@ freeze_result FreezeBase::finalize_freeze(const frame& callee, frame& caller, in || unextended_sp == chunk->to_offset(StackChunkFrameStream(chunk).unextended_sp()), ""); assert(chunk != nullptr || unextended_sp < _freeze_size, ""); + _freeze_size += _monitors_in_lockstack; + // _barriers can be set to true by an allocation in freeze_fast, in which case the chunk is available bool allocated_old_in_freeze_fast = _barriers; assert(!allocated_old_in_freeze_fast || (unextended_sp >= _freeze_size && chunk->is_empty()), @@ -1016,12 +1048,23 @@ freeze_result FreezeBase::finalize_freeze(const frame& callee, frame& caller, in assert(!chunk->is_empty() || StackChunkFrameStream(chunk).is_done(), ""); assert(!chunk->is_empty() || StackChunkFrameStream(chunk).to_frame().is_empty(), ""); + if (_preempt) { + frame f = _thread->last_frame(); + if (f.is_interpreted_frame()) { + // Some platforms do not save the last_sp in the top interpreter frame on VM calls. + // We need it so that on resume we can restore the sp to the right place, since + // thawing might add an alignment word to the expression stack (see finish_thaw()). + // We do it now that we know freezing will be successful. + prepare_freeze_interpreted_top_frame(f); + } + } + // We unwind frames after the last safepoint so that the GC will have found the oops in the frames, but before // writing into the chunk. This is so that an asynchronous stack walk (not at a safepoint) that suspends us here // will either see no continuation or a consistent chunk. unwind_frames(); - chunk->set_max_thawing_size(chunk->max_thawing_size() + _freeze_size - frame::metadata_words); + chunk->set_max_thawing_size(chunk->max_thawing_size() + _freeze_size - _monitors_in_lockstack - frame::metadata_words); if (lt.develop_is_enabled()) { LogStream ls(lt); @@ -1029,6 +1072,10 @@ freeze_result FreezeBase::finalize_freeze(const frame& callee, frame& caller, in chunk->print_on(&ls); } + if (_monitors_in_lockstack > 0) { + freeze_lockstack(chunk); + } + // The topmost existing frame in the chunk; or an empty frame if the chunk is empty caller = StackChunkFrameStream(chunk).to_frame(); @@ -1048,13 +1095,14 @@ freeze_result FreezeBase::finalize_freeze(const frame& callee, frame& caller, in frame entry = sender(callee); - assert(Continuation::is_return_barrier_entry(entry.pc()) || Continuation::is_continuation_enterSpecial(entry), ""); + assert((!empty && Continuation::is_return_barrier_entry(entry.pc())) || (empty && Continuation::is_continuation_enterSpecial(entry)), ""); assert(callee.is_interpreted_frame() || entry.sp() == entry.unextended_sp(), ""); #endif return freeze_ok_bottom; } +// After freezing a frame we need to possibly adjust some values related to the caller frame. void FreezeBase::patch(const frame& f, frame& hf, const frame& caller, bool is_bottom_frame) { if (is_bottom_frame) { // If we're the bottom frame, we need to replace the return barrier with the real @@ -1113,8 +1161,8 @@ NOINLINE freeze_result FreezeBase::recurse_freeze_interpreted_frame(frame& f, fr // including metadata between f and its args const int argsize = ContinuationHelper::InterpretedFrame::stack_argsize(f) + frame::metadata_words_at_top; - log_develop_trace(continuations)("recurse_freeze_interpreted_frame %s _size: %d fsize: %d argsize: %d", - frame_method->name_and_sig_as_C_string(), _freeze_size, fsize, argsize); + log_develop_trace(continuations)("recurse_freeze_interpreted_frame %s _size: %d fsize: %d argsize: %d callee_interpreted: %d", + frame_method->name_and_sig_as_C_string(), _freeze_size, fsize, argsize, callee_interpreted); // we'd rather not yield inside methods annotated with @JvmtiMountTransition assert(!ContinuationHelper::Frame::frame_method(f)->jvmti_mount_transition(), ""); @@ -1191,7 +1239,8 @@ freeze_result FreezeBase::recurse_freeze_compiled_frame(frame& f, frame& caller, assert(!is_bottom_frame || !caller.is_compiled_frame() || (heap_frame_top + fsize) == (caller.unextended_sp() + argsize), ""); if (caller.is_interpreted_frame()) { - _total_align_size += frame::align_wiggle; // See Thaw::align + // When thawing the frame we might need to add alignment (see Thaw::align) + _total_align_size += frame::align_wiggle; } patch(f, hf, caller, is_bottom_frame); @@ -1204,44 +1253,77 @@ freeze_result FreezeBase::recurse_freeze_compiled_frame(frame& f, frame& caller, } NOINLINE freeze_result FreezeBase::recurse_freeze_stub_frame(frame& f, frame& caller) { - intptr_t* const stack_frame_top = ContinuationHelper::StubFrame::frame_top(f, 0, 0); + DEBUG_ONLY(frame fsender = sender(f);) + assert(fsender.is_compiled_frame(), "sender should be compiled frame"); + + intptr_t* const stack_frame_top = ContinuationHelper::StubFrame::frame_top(f); const int fsize = f.cb()->frame_size(); log_develop_trace(continuations)("recurse_freeze_stub_frame %s _size: %d fsize: %d :: " INTPTR_FORMAT " - " INTPTR_FORMAT, f.cb()->name(), _freeze_size, fsize, p2i(stack_frame_top), p2i(stack_frame_top+fsize)); - // recurse_freeze_java_frame and freeze inlined here because we need to use a full RegisterMap for lock ownership - NOT_PRODUCT(_frames++;) - _freeze_size += fsize; + freeze_result result = recurse_freeze_java_frame(f, caller, fsize, 0); + if (UNLIKELY(result > freeze_ok_bottom)) { + return result; + } - RegisterMap map(_cont.thread(), - RegisterMap::UpdateMap::include, - RegisterMap::ProcessFrames::skip, - RegisterMap::WalkContinuation::skip); - map.set_include_argument_oops(false); - ContinuationHelper::update_register_map(f, &map); - f.oop_map()->update_register_map(&f, &map); // we have callee-save registers in this case - frame senderf = sender(f); - assert(senderf.unextended_sp() < _bottom_address - 1, ""); - assert(senderf.is_compiled_frame(), ""); - - if (UNLIKELY(senderf.oop_map() == nullptr)) { - // native frame + assert(result == freeze_ok, "should have caller"); + DEBUG_ONLY(before_freeze_java_frame(f, caller, fsize, 0, false /*is_bottom_frame*/);) + + frame hf = new_heap_frame(f, caller); + intptr_t* heap_frame_top = ContinuationHelper::StubFrame::frame_top(hf); + + copy_to_chunk(stack_frame_top, heap_frame_top, fsize); + + patch(f, hf, caller, false /*is_bottom_frame*/); + + DEBUG_ONLY(after_freeze_java_frame(hf, false /*is_bottom_frame*/);) + + caller = hf; + return freeze_ok; +} + +NOINLINE freeze_result FreezeBase::recurse_freeze_native_frame(frame& f, frame& caller) { + if (!f.cb()->as_nmethod()->method()->is_object_wait0()) { + assert(f.cb()->as_nmethod()->method()->is_synchronized(), ""); + // Synchronized native method case. Unlike the interpreter native wrapper, the compiled + // native wrapper tries to acquire the monitor after marshalling the arguments from the + // caller into the native convention. This is so that we have a valid oopMap in case of + // having to block in the slow path. But that would require freezing those registers too + // and then fixing them back on thaw in case of oops. To avoid complicating things and + // given that this would be a rare case anyways just pin the vthread to the carrier. return freeze_pinned_native; } - freeze_result result = recurse_freeze_compiled_frame(senderf, caller, 0, 0); // This might be deoptimized + intptr_t* const stack_frame_top = ContinuationHelper::NativeFrame::frame_top(f); + // There are no stackargs but argsize must include the metadata + const int argsize = frame::metadata_words_at_top; + const int fsize = f.cb()->frame_size() + argsize; + + log_develop_trace(continuations)("recurse_freeze_native_frame %s _size: %d fsize: %d :: " INTPTR_FORMAT " - " INTPTR_FORMAT, + f.cb()->name(), _freeze_size, fsize, p2i(stack_frame_top), p2i(stack_frame_top+fsize)); + + freeze_result result = recurse_freeze_java_frame(f, caller, fsize, argsize); if (UNLIKELY(result > freeze_ok_bottom)) { return result; } - assert(result != freeze_ok_bottom, ""); - assert(!caller.is_interpreted_frame(), ""); - DEBUG_ONLY(before_freeze_java_frame(f, caller, fsize, 0, false);) - frame hf = new_heap_frame(f, caller); - intptr_t* heap_frame_top = ContinuationHelper::StubFrame::frame_top(hf, 0, 0); + assert(result == freeze_ok, "should have caller frame"); + DEBUG_ONLY(before_freeze_java_frame(f, caller, fsize, argsize, false /* is_bottom_frame */);) + + frame hf = new_heap_frame(f, caller); + intptr_t* heap_frame_top = ContinuationHelper::NativeFrame::frame_top(hf); + copy_to_chunk(stack_frame_top, heap_frame_top, fsize); - DEBUG_ONLY(after_freeze_java_frame(hf, false);) + + if (caller.is_interpreted_frame()) { + // When thawing the frame we might need to add alignment (see Thaw::align) + _total_align_size += frame::align_wiggle; + } + + patch(f, hf, caller, false /* is_bottom_frame */); + + DEBUG_ONLY(after_freeze_java_frame(hf, false /* is_bottom_frame */);) caller = hf; return freeze_ok; @@ -1264,6 +1346,8 @@ NOINLINE void FreezeBase::finish_freeze(const frame& f, const frame& top) { chunk->set_max_thawing_size(chunk->max_thawing_size() + _total_align_size); + assert(chunk->sp_address() - chunk->start_address() >= _monitors_in_lockstack, "clash with lockstack"); + // At this point the chunk is consistent if (UNLIKELY(_barriers)) { @@ -1293,6 +1377,7 @@ NOINLINE void FreezeBase::finish_freeze(const frame& f, const frame& top) { ls.print_cr("top hframe after (freeze):"); assert(_cont.last_frame().is_heap_frame(), "should be"); _cont.last_frame().print_on(&ls); + DEBUG_ONLY(print_frame_layout(top, false, &ls);) } assert(_cont.chunk_invariant(), ""); @@ -1327,6 +1412,9 @@ class StackChunkAllocator : public MemAllocator { // zero out fields (but not the stack) const size_t hs = oopDesc::header_size(); + if (oopDesc::has_klass_gap()) { + oopDesc::set_klass_gap(mem, 0); + } Copy::fill_to_aligned_words(mem + hs, vmClasses::StackChunk_klass()->size_helper() - hs); int bottom = (int)_stack_size - _argsize_md; @@ -1435,6 +1523,8 @@ stackChunkOop Freeze::allocate_chunk(size_t stack_size, int argsize_md) assert(chunk->is_empty(), ""); assert(chunk->flags() == 0, ""); assert(chunk->is_gc_mode() == false, ""); + assert(chunk->lockstack_size() == 0, ""); + assert(chunk->object_waiter() == nullptr, ""); // fields are uninitialized chunk->set_parent_access(_cont.last_nonempty_chunk()); @@ -1505,6 +1595,29 @@ static void jvmti_yield_cleanup(JavaThread* thread, ContinuationWrapper& cont) { } invalidate_jvmti_stack(thread); } + +static void jvmti_mount_end(JavaThread* current, ContinuationWrapper& cont, frame top) { + assert(current->vthread() != nullptr, "must be"); + + HandleMarkCleaner hm(current); + Handle vth(current, current->vthread()); + + ContinuationWrapper::SafepointOp so(current, cont); + + // Since we might safepoint set the anchor so that the stack can be walked. + set_anchor(current, top.sp()); + + JRT_BLOCK + JvmtiVTMSTransitionDisabler::VTMS_vthread_mount((jthread)vth.raw_value(), false); + + if (current->pending_contended_entered_event()) { + JvmtiExport::post_monitor_contended_entered(current, current->contended_entered_monitor()); + current->set_contended_entered_monitor(nullptr); + } + JRT_BLOCK_END + + clear_anchor(current); +} #endif // INCLUDE_JVMTI #ifdef ASSERT @@ -1517,7 +1630,8 @@ static bool monitors_on_stack(JavaThread* thread) { map.set_include_argument_oops(false); for (frame f = thread->last_frame(); Continuation::is_frame_in_continuation(ce, f); f = f.sender(&map)) { if ((f.is_interpreted_frame() && ContinuationHelper::InterpretedFrame::is_owning_locks(f)) || - (f.is_compiled_frame() && ContinuationHelper::CompiledFrame::is_owning_locks(map.thread(), &map, f))) { + (f.is_compiled_frame() && ContinuationHelper::CompiledFrame::is_owning_locks(map.thread(), &map, f)) || + (f.is_native_frame() && ContinuationHelper::NativeFrame::is_owning_locks(map.thread(), f))) { return true; } } @@ -1534,8 +1648,9 @@ bool FreezeBase::check_valid_fast_path() { RegisterMap::ProcessFrames::skip, RegisterMap::WalkContinuation::skip); map.set_include_argument_oops(false); - for (frame f = freeze_start_frame(); Continuation::is_frame_in_continuation(ce, f); f = f.sender(&map)) { - if (!f.is_compiled_frame() || f.is_deoptimized_frame()) { + bool is_top_frame = true; + for (frame f = freeze_start_frame(); Continuation::is_frame_in_continuation(ce, f); f = f.sender(&map), is_top_frame = false) { + if (!((f.is_compiled_frame() && !f.is_deoptimized_frame()) || (is_top_frame && (f.is_runtime_frame() || f.is_native_frame())))) { return false; } } @@ -1543,34 +1658,45 @@ bool FreezeBase::check_valid_fast_path() { } #endif // ASSERT -static inline int freeze_epilog(JavaThread* thread, ContinuationWrapper& cont) { +static inline freeze_result freeze_epilog(ContinuationWrapper& cont) { verify_continuation(cont.continuation()); assert(!cont.is_empty(), ""); - // This is done for the sake of the enterSpecial frame - StackWatermarkSet::after_unwind(thread); log_develop_debug(continuations)("=== End of freeze cont ### #" INTPTR_FORMAT, cont.hash()); - - return 0; + return freeze_ok; } -static int freeze_epilog(JavaThread* thread, ContinuationWrapper& cont, freeze_result res) { +static freeze_result freeze_epilog(JavaThread* thread, ContinuationWrapper& cont, freeze_result res) { if (UNLIKELY(res != freeze_ok)) { + JFR_ONLY(thread->set_last_freeze_fail_result(res);) verify_continuation(cont.continuation()); log_develop_trace(continuations)("=== end of freeze (fail %d)", res); return res; } JVMTI_ONLY(jvmti_yield_cleanup(thread, cont)); // can safepoint - return freeze_epilog(thread, cont); + return freeze_epilog(cont); } -template -static inline int freeze_internal(JavaThread* current, intptr_t* const sp) { +static freeze_result preempt_epilog(ContinuationWrapper& cont, freeze_result res, frame& old_last_frame) { + if (UNLIKELY(res != freeze_ok)) { + verify_continuation(cont.continuation()); + log_develop_trace(continuations)("=== end of freeze (fail %d)", res); + return res; + } + + patch_return_pc_with_preempt_stub(old_last_frame); + cont.tail()->set_preempted(true); + + return freeze_epilog(cont); +} + +template +static inline freeze_result freeze_internal(JavaThread* current, intptr_t* const sp) { assert(!current->has_pending_exception(), ""); #ifdef ASSERT - log_trace(continuations)("~~~~ freeze sp: " INTPTR_FORMAT, p2i(current->last_continuation()->entry_sp())); + log_trace(continuations)("~~~~ freeze sp: " INTPTR_FORMAT "JavaThread: " INTPTR_FORMAT, p2i(current->last_continuation()->entry_sp()), p2i(current)); log_frames(current); #endif @@ -1588,8 +1714,10 @@ static inline int freeze_internal(JavaThread* current, intptr_t* const sp) { assert(entry->is_virtual_thread() == (entry->scope(current) == java_lang_VirtualThread::vthread_scope()), ""); - assert(monitors_on_stack(current) == ((current->held_monitor_count() - current->jni_monitor_count()) > 0), + assert(LockingMode != LM_LEGACY || (monitors_on_stack(current) == ((current->held_monitor_count() - current->jni_monitor_count()) > 0)), "Held monitor count and locks on stack invariant: " INT64_FORMAT " JNI: " INT64_FORMAT, (int64_t)current->held_monitor_count(), (int64_t)current->jni_monitor_count()); + assert(LockingMode == LM_LEGACY || (current->held_monitor_count() == 0 && current->jni_monitor_count() == 0), + "Held monitor count should only be used for LM_LEGACY: " INT64_FORMAT " JNI: " INT64_FORMAT, (int64_t)current->held_monitor_count(), (int64_t)current->jni_monitor_count()); if (entry->is_pinned() || current->held_monitor_count() > 0) { log_develop_debug(continuations)("PINNED due to critical section/hold monitor"); @@ -1597,26 +1725,36 @@ static inline int freeze_internal(JavaThread* current, intptr_t* const sp) { freeze_result res = entry->is_pinned() ? freeze_pinned_cs : freeze_pinned_monitor; log_develop_trace(continuations)("=== end of freeze (fail %d)", res); // Avoid Thread.yield() loops without safepoint polls. - if (SafepointMechanism::should_process(current)) { + if (SafepointMechanism::should_process(current) && !preempt) { cont.done(); // allow safepoint ThreadInVMfromJava tivmfj(current); } return res; } - Freeze freeze(current, cont, sp); + Freeze freeze(current, cont, sp, preempt); assert(!current->cont_fastpath() || freeze.check_valid_fast_path(), ""); bool fast = UseContinuationFastPath && current->cont_fastpath(); if (fast && freeze.size_if_fast_freeze_available() > 0) { freeze.freeze_fast_existing_chunk(); CONT_JFR_ONLY(freeze.jfr_info().post_jfr_event(&event, oopCont, current);) - freeze_epilog(current, cont); - return 0; + return !preempt ? freeze_epilog(cont) : preempt_epilog(cont, freeze_ok, freeze.last_frame()); + } + + if (preempt) { + JvmtiSampledObjectAllocEventCollector jsoaec(false); + freeze.set_jvmti_event_collector(&jsoaec); + + freeze_result res = fast ? freeze.try_freeze_fast() : freeze.freeze_slow(); + + CONT_JFR_ONLY(freeze.jfr_info().post_jfr_event(&event, oopCont, current);) + preempt_epilog(cont, res, freeze.last_frame()); + return res; } log_develop_trace(continuations)("chunk unavailable; transitioning to VM"); - assert(current == JavaThread::current(), "must be current thread except for preempt"); + assert(current == JavaThread::current(), "must be current thread"); JRT_BLOCK // delays a possible JvmtiSampledObjectAllocEventCollector in alloc_chunk JvmtiSampledObjectAllocEventCollector jsoaec(false); @@ -1751,6 +1889,7 @@ class ThawBase : public StackObj { intptr_t* _fastpath; bool _barriers; + bool _preempted_case; intptr_t* _top_unextended_sp_before_thaw; int _align_size; DEBUG_ONLY(intptr_t* _top_stack_address); @@ -1769,18 +1908,24 @@ class ThawBase : public StackObj { } void clear_chunk(stackChunkOop chunk); + template int remove_top_compiled_frame_from_chunk(stackChunkOop chunk, int &argsize); void copy_from_chunk(intptr_t* from, intptr_t* to, int size); + void thaw_lockstack(stackChunkOop chunk); + // fast path inline void prefetch_chunk_pd(void* start, int size_words); void patch_return(intptr_t* sp, bool is_last); - // slow path - NOINLINE intptr_t* thaw_slow(stackChunkOop chunk, bool return_barrier); + intptr_t* handle_preempted_continuation(intptr_t* sp, Continuation::preempt_kind preempt_kind, bool fast_case); + inline intptr_t* push_cleanup_continuation(); + void throw_interrupted_exception(JavaThread* current, frame& top); + + void recurse_thaw(const frame& heap_frame, frame& caller, int num_frames, bool top_on_preempt_case); + void finish_thaw(frame& f); private: - void recurse_thaw(const frame& heap_frame, frame& caller, int num_frames, bool top); template bool recurse_thaw_java_frame(frame& caller, int num_frames); void finalize_thaw(frame& entry, int argsize); @@ -1794,12 +1939,13 @@ class ThawBase : public StackObj { NOINLINE void recurse_thaw_interpreted_frame(const frame& hf, frame& caller, int num_frames); void recurse_thaw_compiled_frame(const frame& hf, frame& caller, int num_frames, bool stub_caller); void recurse_thaw_stub_frame(const frame& hf, frame& caller, int num_frames); - void finish_thaw(frame& f); + void recurse_thaw_native_frame(const frame& hf, frame& caller, int num_frames); void push_return_frame(frame& f); inline frame new_entry_frame(); template frame new_stack_frame(const frame& hf, frame& caller, bool bottom); inline void patch_pd(frame& f, const frame& sender); + inline void patch_pd(frame& f, intptr_t* caller_sp); inline intptr_t* align(const frame& hf, intptr_t* frame_sp, frame& caller, bool bottom); void maybe_set_fastpath(intptr_t* sp) { if (sp > _fastpath) _fastpath = sp; } @@ -1823,7 +1969,9 @@ class Thaw : public ThawBase { } inline intptr_t* thaw(Continuation::thaw_kind kind); + template NOINLINE intptr_t* thaw_fast(stackChunkOop chunk); + NOINLINE intptr_t* thaw_slow(stackChunkOop chunk, Continuation::thaw_kind kind); inline void patch_caller_links(intptr_t* sp, intptr_t* bottom); }; @@ -1839,7 +1987,7 @@ inline intptr_t* Thaw::thaw(Continuation::thaw_kind kind) { _barriers = chunk->requires_barriers(); return (LIKELY(can_thaw_fast(chunk))) ? thaw_fast(chunk) - : thaw_slow(chunk, kind != Continuation::thaw_top); + : thaw_slow(chunk, kind); } class ReconstructedStack : public StackObj { @@ -1872,6 +2020,7 @@ inline void ThawBase::clear_chunk(stackChunkOop chunk) { chunk->set_max_thawing_size(0); } +template int ThawBase::remove_top_compiled_frame_from_chunk(stackChunkOop chunk, int &argsize) { bool empty = false; StackChunkFrameStream f(chunk); @@ -1879,9 +2028,30 @@ int ThawBase::remove_top_compiled_frame_from_chunk(stackChunkOop chunk, int &arg assert(chunk_sp == f.sp(), ""); assert(chunk_sp == f.unextended_sp(), ""); - const int frame_size = f.cb()->frame_size(); + int frame_size = f.cb()->frame_size(); argsize = f.stack_argsize(); + assert(!f.is_stub() || check_stub, ""); + if (check_stub && f.is_stub()) { + // If we don't thaw the top compiled frame too, after restoring the saved + // registers back in Java, we would hit the return barrier to thaw one more + // frame effectively overwriting the restored registers during that call. + f.next(SmallRegisterMap::instance(), true /* stop */); + assert(!f.is_done(), ""); + + f.get_cb(); + assert(f.is_compiled(), ""); + frame_size += f.cb()->frame_size(); + argsize = f.stack_argsize(); + + if (f.cb()->as_nmethod()->is_marked_for_deoptimization()) { + // The caller of the runtime stub when the continuation is preempted is not at a + // Java call instruction, and so cannot rely on nmethod patching for deopt. + log_develop_trace(continuations)("Deoptimizing runtime stub caller"); + f.to_frame().deoptimize(nullptr); // the null thread simply avoids the assertion in deoptimize which we're not set up for + } + } + f.next(SmallRegisterMap::instance(), true /* stop */); empty = f.is_done(); assert(!empty || argsize == chunk->argsize(), ""); @@ -1909,6 +2079,18 @@ int ThawBase::remove_top_compiled_frame_from_chunk(stackChunkOop chunk, int &arg return frame_size + argsize + frame::metadata_words_at_top; } +void ThawBase::thaw_lockstack(stackChunkOop chunk) { + int lockStackSize = chunk->lockstack_size(); + assert(lockStackSize > 0 && lockStackSize <= LockStack::CAPACITY, ""); + + oop tmp_lockstack[LockStack::CAPACITY]; + chunk->transfer_lockstack(tmp_lockstack, _barriers); + _thread->lock_stack().move_from_address(tmp_lockstack, lockStackSize); + + chunk->set_lockstack_size(0); + chunk->set_has_lockstack(false); +} + void ThawBase::copy_from_chunk(intptr_t* from, intptr_t* to, int size) { assert(to >= _top_stack_address, "overwrote past thawing space" " to: " INTPTR_FORMAT " top_address: " INTPTR_FORMAT, p2i(to), p2i(_top_stack_address)); @@ -1927,6 +2109,7 @@ void ThawBase::patch_return(intptr_t* sp, bool is_last) { } template +template NOINLINE intptr_t* Thaw::thaw_fast(stackChunkOop chunk) { assert(chunk == _cont.tail(), ""); assert(!chunk->has_mixed_frames(), ""); @@ -1960,7 +2143,7 @@ NOINLINE intptr_t* Thaw::thaw_fast(stackChunkOop chunk) { empty = true; } else { // thaw a single frame partial = true; - thaw_size = remove_top_compiled_frame_from_chunk(chunk, argsize); + thaw_size = remove_top_compiled_frame_from_chunk(chunk, argsize); empty = chunk->is_empty(); } @@ -2016,11 +2199,70 @@ inline bool ThawBase::seen_by_gc() { return _barriers || _cont.tail()->is_gc_mode(); } -NOINLINE intptr_t* ThawBase::thaw_slow(stackChunkOop chunk, bool return_barrier) { +static inline void relativize_chunk_concurrently(stackChunkOop chunk) { +#if INCLUDE_ZGC || INCLUDE_SHENANDOAHGC + if (UseZGC || UseShenandoahGC) { + chunk->relativize_derived_pointers_concurrently(); + } +#endif +} + +template +NOINLINE intptr_t* Thaw::thaw_slow(stackChunkOop chunk, Continuation::thaw_kind kind) { + Continuation::preempt_kind preempt_kind; + bool retry_fast_path = false; + + _preempted_case = chunk->preempted(); + if (_preempted_case) { + if (chunk->object_waiter() != nullptr) { + // Mounted again after preemption. Resume the pending monitor operation, + // which will be either a monitorenter or Object.wait() call. + assert(chunk->current_pending_monitor() != nullptr || chunk->current_waiting_monitor() != nullptr, ""); + ObjectWaiter* waiter = chunk->object_waiter(); + ObjectMonitor* mon = waiter->monitor(); + preempt_kind = waiter->is_wait() ? Continuation::freeze_on_wait : Continuation::freeze_on_monitorenter; + + bool mon_acquired = mon->resume_operation(_thread, waiter, _cont); + assert(!mon_acquired || mon->has_owner(_thread), "invariant"); + if (!mon_acquired) { + // Failed to acquire monitor. Return to enterSpecial to unmount again. + return push_cleanup_continuation(); + } + chunk = _cont.tail(); // reload oop in case of safepoint in resume_operation (if posting JVMTI events). + } else { + // Preemption cancelled in moniterenter case. We actually acquired + // the monitor after freezing all frames so nothing to do. + preempt_kind = Continuation::freeze_on_monitorenter; + } + // Call this first to avoid racing with GC threads later when modifying the chunk flags. + relativize_chunk_concurrently(chunk); + chunk->set_preempted(false); + retry_fast_path = true; + } else { + relativize_chunk_concurrently(chunk); + } + + // On first thaw after freeze restore oops to the lockstack if any. + assert(chunk->lockstack_size() == 0 || kind == Continuation::thaw_top, ""); + if (kind == Continuation::thaw_top && chunk->lockstack_size() > 0) { + thaw_lockstack(chunk); + retry_fast_path = true; + } + + // Retry the fast path now that we possibly cleared the FLAG_HAS_LOCKSTACK + // and FLAG_PREEMPTED flags from the stackChunk. + if (retry_fast_path && can_thaw_fast(chunk)) { + intptr_t* sp = thaw_fast(chunk); + if (_preempted_case) { + return handle_preempted_continuation(sp, preempt_kind, true /* fast_case */); + } + return sp; + } + LogTarget(Trace, continuations) lt; if (lt.develop_is_enabled()) { LogStream ls(lt); - ls.print_cr("thaw slow return_barrier: %d " INTPTR_FORMAT, return_barrier, p2i(chunk)); + ls.print_cr("thaw slow return_barrier: %d " INTPTR_FORMAT, kind, p2i(chunk)); chunk->print_on(true, &ls); } @@ -2034,7 +2276,7 @@ NOINLINE intptr_t* ThawBase::thaw_slow(stackChunkOop chunk, bool return_barrier) DEBUG_ONLY(_frames = 0;) _align_size = 0; - int num_frames = (return_barrier ? 1 : 2); + int num_frames = kind == Continuation::thaw_top ? 2 : 1; _stream = StackChunkFrameStream(chunk); _top_unextended_sp_before_thaw = _stream.unextended_sp(); @@ -2047,14 +2289,8 @@ NOINLINE intptr_t* ThawBase::thaw_slow(stackChunkOop chunk, bool return_barrier) heap_frame.print_value_on(&ls); } -#if INCLUDE_ZGC || INCLUDE_SHENANDOAHGC - if (UseZGC || UseShenandoahGC) { - _cont.tail()->relativize_derived_pointers_concurrently(); - } -#endif - frame caller; // the thawed caller on the stack - recurse_thaw(heap_frame, caller, num_frames, true); + recurse_thaw(heap_frame, caller, num_frames, _preempted_case); finish_thaw(caller); // caller is now the topmost thawed frame _cont.write(); @@ -2065,18 +2301,21 @@ NOINLINE intptr_t* ThawBase::thaw_slow(stackChunkOop chunk, bool return_barrier) _thread->set_cont_fastpath(_fastpath); intptr_t* sp = caller.sp(); + + if (_preempted_case) { + return handle_preempted_continuation(sp, preempt_kind, false /* fast_case */); + } return sp; } -void ThawBase::recurse_thaw(const frame& heap_frame, frame& caller, int num_frames, bool top) { +void ThawBase::recurse_thaw(const frame& heap_frame, frame& caller, int num_frames, bool top_on_preempt_case) { log_develop_debug(continuations)("thaw num_frames: %d", num_frames); assert(!_cont.is_empty(), "no more frames"); assert(num_frames > 0, ""); assert(!heap_frame.is_empty(), ""); - if (top && heap_frame.is_safepoint_blob_frame()) { - assert(ContinuationHelper::Frame::is_stub(heap_frame.cb()), "cb: %s", heap_frame.cb()->name()); - recurse_thaw_stub_frame(heap_frame, caller, num_frames); + if (top_on_preempt_case && (heap_frame.is_native_frame() || heap_frame.is_runtime_frame())) { + heap_frame.is_native_frame() ? recurse_thaw_native_frame(heap_frame, caller, 2) : recurse_thaw_stub_frame(heap_frame, caller, 2); } else if (!heap_frame.is_interpreted_frame()) { recurse_thaw_compiled_frame(heap_frame, caller, num_frames, false); } else { @@ -2106,7 +2345,7 @@ bool ThawBase::recurse_thaw_java_frame(frame& caller, int num_frames) { finalize_thaw(caller, FKind::interpreted ? 0 : argsize); return true; // bottom } else { // recurse - recurse_thaw(_stream.to_frame(), caller, num_frames - 1, false); + recurse_thaw(_stream.to_frame(), caller, num_frames - 1, false /* top_on_preempt_case */); return false; } } @@ -2196,6 +2435,59 @@ void ThawBase::clear_bitmap_bits(address start, address end) { assert(effective_end == end || !chunk->bitmap().at(chunk->bit_index_for(effective_end)), "bit should not be set"); } +intptr_t* ThawBase::handle_preempted_continuation(intptr_t* sp, Continuation::preempt_kind preempt_kind, bool fast_case) { + assert(preempt_kind == Continuation::freeze_on_wait || preempt_kind == Continuation::freeze_on_monitorenter, ""); + frame top(sp); + assert(top.pc() == *(address*)(sp - frame::sender_sp_ret_address_offset()), ""); + +#if INCLUDE_JVMTI + // Finish the VTMS transition. + assert(_thread->is_in_VTMS_transition(), "must be"); + bool is_vthread = Continuation::continuation_scope(_cont.continuation()) == java_lang_VirtualThread::vthread_scope(); + if (is_vthread) { + if (JvmtiVTMSTransitionDisabler::VTMS_notify_jvmti_events()) { + jvmti_mount_end(_thread, _cont, top); + } else { + _thread->set_is_in_VTMS_transition(false); + java_lang_Thread::set_is_in_VTMS_transition(_thread->vthread(), false); + } + } +#endif + + if (fast_case) { + // If we thawed in the slow path the runtime stub/native wrapper frame already + // has the correct fp (see ThawBase::new_stack_frame). On the fast path though, + // we copied the original fp at the time of freeze which now will have to be fixed. + assert(top.is_runtime_frame() || top.is_native_frame(), ""); + int fsize = top.cb()->frame_size(); + patch_pd(top, sp + fsize); + } + + if (preempt_kind == Continuation::freeze_on_wait) { + // Check now if we need to throw IE exception. + if (_thread->pending_interrupted_exception()) { + throw_interrupted_exception(_thread, top); + _thread->set_pending_interrupted_exception(false); + } + } else if (top.is_runtime_frame()) { + // The continuation might now run on a different platform thread than the previous time so + // we need to adjust the current thread saved in the stub frame before restoring registers. + JavaThread** thread_addr = frame::saved_thread_address(top); + if (thread_addr != nullptr) *thread_addr = _thread; + } + return sp; +} + +void ThawBase::throw_interrupted_exception(JavaThread* current, frame& top) { + ContinuationWrapper::SafepointOp so(current, _cont); + // Since we might safepoint set the anchor so that the stack can be walked. + set_anchor(current, top.sp()); + JRT_BLOCK + THROW(vmSymbols::java_lang_InterruptedException()); + JRT_BLOCK_END + clear_anchor(current); +} + NOINLINE void ThawBase::recurse_thaw_interpreted_frame(const frame& hf, frame& caller, int num_frames) { assert(hf.is_interpreted_frame(), ""); @@ -2239,7 +2531,9 @@ NOINLINE void ThawBase::recurse_thaw_interpreted_frame(const frame& hf, frame& c maybe_set_fastpath(f.sp()); - const int locals = hf.interpreter_frame_method()->max_locals(); + Method* m = hf.interpreter_frame_method(); + assert(!m->is_native() || !is_bottom_frame, "should be top frame of thaw_top case; missing caller frame"); + const int locals = m->max_locals(); if (!is_bottom_frame) { // can only fix caller once this frame is thawed (due to callee saved regs) @@ -2256,8 +2550,8 @@ NOINLINE void ThawBase::recurse_thaw_interpreted_frame(const frame& hf, frame& c } void ThawBase::recurse_thaw_compiled_frame(const frame& hf, frame& caller, int num_frames, bool stub_caller) { - assert(!hf.is_interpreted_frame(), ""); - assert(_cont.is_preempted() || !stub_caller, "stub caller not at preemption"); + assert(hf.is_compiled_frame(), ""); + assert(_preempted_case || !stub_caller, "stub caller not at preemption"); if (!stub_caller && UNLIKELY(seen_by_gc())) { // recurse_thaw_stub_frame already invoked our barriers with a full regmap _cont.tail()->do_barriers(_stream, SmallRegisterMap::instance()); @@ -2270,7 +2564,7 @@ void ThawBase::recurse_thaw_compiled_frame(const frame& hf, frame& caller, int n assert(caller.sp() == caller.unextended_sp(), ""); if ((!is_bottom_frame && caller.is_interpreted_frame()) || (is_bottom_frame && Interpreter::contains(_cont.tail()->pc()))) { - _align_size += frame::align_wiggle; // we add one whether or not we've aligned because we add it in freeze_interpreted_frame + _align_size += frame::align_wiggle; // we add one whether or not we've aligned because we add it in recurse_freeze_compiled_frame } // new_stack_frame must construct the resulting frame using hf.pc() rather than hf.raw_pc() because the frame is not @@ -2303,10 +2597,9 @@ void ThawBase::recurse_thaw_compiled_frame(const frame& hf, frame& caller, int n if (hf.is_deoptimized_frame()) { maybe_set_fastpath(f.sp()); } else if (_thread->is_interp_only_mode() - || (_cont.is_preempted() && f.cb()->as_nmethod()->is_marked_for_deoptimization())) { + || (stub_caller && f.cb()->as_nmethod()->is_marked_for_deoptimization())) { // The caller of the safepoint stub when the continuation is preempted is not at a call instruction, and so // cannot rely on nmethod patching for deopt. - assert(_thread->is_interp_only_mode() || stub_caller, "expected a stub-caller"); log_develop_trace(continuations)("Deoptimizing thawed frame"); DEBUG_ONLY(ContinuationHelper::Frame::patch_pc(f, nullptr)); @@ -2334,7 +2627,8 @@ void ThawBase::recurse_thaw_compiled_frame(const frame& hf, frame& caller, int n void ThawBase::recurse_thaw_stub_frame(const frame& hf, frame& caller, int num_frames) { DEBUG_ONLY(_frames++;) - { + if (UNLIKELY(seen_by_gc())) { + // Process the stub's caller here since we might need the full map. RegisterMap map(nullptr, RegisterMap::UpdateMap::include, RegisterMap::ProcessFrames::skip, @@ -2342,41 +2636,89 @@ void ThawBase::recurse_thaw_stub_frame(const frame& hf, frame& caller, int num_f map.set_include_argument_oops(false); _stream.next(&map); assert(!_stream.is_done(), ""); - if (UNLIKELY(seen_by_gc())) { // we're now doing this on the stub's caller - _cont.tail()->do_barriers(_stream, &map); - } + _cont.tail()->do_barriers(_stream, &map); + } else { + _stream.next(SmallRegisterMap::instance()); assert(!_stream.is_done(), ""); } - recurse_thaw_compiled_frame(_stream.to_frame(), caller, num_frames, true); // this could be deoptimized + recurse_thaw_compiled_frame(_stream.to_frame(), caller, num_frames, true); - DEBUG_ONLY(before_thaw_java_frame(hf, caller, false, num_frames);) - - assert(ContinuationHelper::Frame::is_stub(hf.cb()), ""); + assert(caller.is_compiled_frame(), ""); assert(caller.sp() == caller.unextended_sp(), ""); - assert(!caller.is_interpreted_frame(), ""); - int fsize = ContinuationHelper::StubFrame::size(hf); + DEBUG_ONLY(before_thaw_java_frame(hf, caller, false /*is_bottom_frame*/, num_frames);) frame f = new_stack_frame(hf, caller, false); intptr_t* stack_frame_top = f.sp(); intptr_t* heap_frame_top = hf.sp(); + int fsize = ContinuationHelper::StubFrame::size(hf); copy_from_chunk(heap_frame_top - frame::metadata_words, stack_frame_top - frame::metadata_words, fsize + frame::metadata_words); - { // can only fix caller once this frame is thawed (due to callee saved regs) - RegisterMap map(nullptr, - RegisterMap::UpdateMap::include, - RegisterMap::ProcessFrames::skip, - RegisterMap::WalkContinuation::skip); // map.clear(); - map.set_include_argument_oops(false); - f.oop_map()->update_register_map(&f, &map); - ContinuationHelper::update_register_map_with_callee(caller, &map); - _cont.tail()->fix_thawed_frame(caller, &map); + patch(f, caller, false /*is_bottom_frame*/); + + // can only fix caller once this frame is thawed (due to callee saved regs) + RegisterMap map(nullptr, + RegisterMap::UpdateMap::include, + RegisterMap::ProcessFrames::skip, + RegisterMap::WalkContinuation::skip); + map.set_include_argument_oops(false); + f.oop_map()->update_register_map(&f, &map); + ContinuationHelper::update_register_map_with_callee(caller, &map); + _cont.tail()->fix_thawed_frame(caller, &map); + + DEBUG_ONLY(after_thaw_java_frame(f, false /*is_bottom_frame*/);) + caller = f; +} + +void ThawBase::recurse_thaw_native_frame(const frame& hf, frame& caller, int num_frames) { + assert(hf.is_native_frame(), ""); + assert(_preempted_case && hf.cb()->as_nmethod()->method()->is_object_wait0(), ""); + + if (UNLIKELY(seen_by_gc())) { // recurse_thaw_stub_frame already invoked our barriers with a full regmap + _cont.tail()->do_barriers(_stream, SmallRegisterMap::instance()); } - DEBUG_ONLY(after_thaw_java_frame(f, false);) + const bool is_bottom_frame = recurse_thaw_java_frame(caller, num_frames); + assert(!is_bottom_frame, ""); + + DEBUG_ONLY(before_thaw_java_frame(hf, caller, is_bottom_frame, num_frames);) + + assert(caller.sp() == caller.unextended_sp(), ""); + + if (caller.is_interpreted_frame()) { + _align_size += frame::align_wiggle; // we add one whether or not we've aligned because we add it in recurse_freeze_native_frame + } + + // new_stack_frame must construct the resulting frame using hf.pc() rather than hf.raw_pc() because the frame is not + // yet laid out in the stack, and so the original_pc is not stored in it. + // As a result, f.is_deoptimized_frame() is always false and we must test hf to know if the frame is deoptimized. + frame f = new_stack_frame(hf, caller, false /* bottom */); + intptr_t* const stack_frame_top = f.sp(); + intptr_t* const heap_frame_top = hf.unextended_sp(); + + int fsize = ContinuationHelper::NativeFrame::size(hf); + assert(fsize <= (int)(caller.unextended_sp() - f.unextended_sp()), ""); + + intptr_t* from = heap_frame_top - frame::metadata_words_at_bottom; + intptr_t* to = stack_frame_top - frame::metadata_words_at_bottom; + int sz = fsize + frame::metadata_words_at_bottom; + + copy_from_chunk(from, to, sz); // copying good oops because we invoked barriers above + + patch(f, caller, false /* bottom */); + + // f.is_deoptimized_frame() is always false and we must test hf.is_deoptimized_frame() (see comment above) + assert(!f.is_deoptimized_frame(), ""); + assert(!hf.is_deoptimized_frame(), ""); + assert(!f.cb()->as_nmethod()->is_marked_for_deoptimization(), ""); + + // can only fix caller once this frame is thawed (due to callee saved regs); this happens on the stack + _cont.tail()->fix_thawed_frame(caller, SmallRegisterMap::instance()); + + DEBUG_ONLY(after_thaw_java_frame(f, false /* bottom */);) caller = f; } @@ -2463,30 +2805,11 @@ static inline intptr_t* thaw_internal(JavaThread* thread, const Continuation::th clear_anchor(thread); #endif + DEBUG_ONLY(bool preempted = cont.tail()->preempted();) Thaw thw(thread, cont); intptr_t* const sp = thw.thaw(kind); assert(is_aligned(sp, frame::frame_alignment), ""); - - // All or part of the frames have been thawed so we know they don't hold any monitors except JNI monitors. - assert(thread->held_monitor_count() == thread->jni_monitor_count(), "Must be"); - -#ifdef ASSERT - intptr_t* sp0 = sp; - set_anchor(thread, sp0); - log_frames(thread); - if (LoomVerifyAfterThaw) { - assert(do_verify_after_thaw(thread, cont.tail(), tty), ""); - } - assert(ContinuationEntry::assert_entry_frame_laid_out(thread), ""); - clear_anchor(thread); - - LogTarget(Trace, continuations) lt; - if (lt.develop_is_enabled()) { - LogStream ls(lt); - ls.print_cr("Jumping to frame (thaw):"); - frame(sp).print_value_on(&ls); - } -#endif + DEBUG_ONLY(log_frames_after_thaw(thread, cont, sp, preempted);) CONT_JFR_ONLY(thw.jfr_info().post_jfr_event(&event, cont.continuation(), thread);) @@ -2591,7 +2914,7 @@ static void log_frames(JavaThread* thread) { } LogStream ls(lt); - ls.print_cr("------- frames ---------"); + ls.print_cr("------- frames --------- for thread " INTPTR_FORMAT, p2i(thread)); if (!thread->has_last_Java_frame()) { ls.print_cr("NO ANCHOR!"); } @@ -2615,8 +2938,8 @@ static void log_frames(JavaThread* thread) { int i = 0; int post_entry = -1; - for (frame f = thread->last_frame(); !f.is_entry_frame(); f = f.sender(&map)) { - f.describe(values, i++, &map); + for (frame f = thread->last_frame(); !f.is_first_frame(); f = f.sender(&map), i++) { + f.describe(values, i, &map, i == 0); if (post_entry >= 0 || Continuation::is_continuation_enterSpecial(f)) post_entry++; if (post_entry >= show_entry_callers) @@ -2627,6 +2950,33 @@ static void log_frames(JavaThread* thread) { ls.print_cr("======= end frames ========="); } + +static void log_frames_after_thaw(JavaThread* thread, ContinuationWrapper& cont, intptr_t* sp, bool preempted) { + intptr_t* sp0 = sp; + address pc0 = *(address*)(sp - frame::sender_sp_ret_address_offset()); + + if (preempted && sp0 == cont.entrySP()) { + // Still preempted (monitor not acquired) so no frames were thawed. + assert(cont.tail()->preempted(), ""); + set_anchor(thread, cont.entrySP(), cont.entryPC()); + } else { + set_anchor(thread, sp0); + } + + log_frames(thread); + if (LoomVerifyAfterThaw) { + assert(do_verify_after_thaw(thread, cont.tail(), tty), ""); + } + assert(ContinuationEntry::assert_entry_frame_laid_out(thread), ""); + clear_anchor(thread); + + LogTarget(Trace, continuations) lt; + if (lt.develop_is_enabled()) { + LogStream ls(lt); + ls.print_cr("Jumping to frame (thaw):"); + frame(sp).print_value_on(&ls); + } +} #endif // ASSERT #include CPU_HEADER_INLINE(continuationFreezeThaw) @@ -2647,13 +2997,14 @@ static void print_frame_layout(const frame& f, bool callee_complete, outputStrea if (callee_complete) { frame::update_map_with_saved_link(&map, ContinuationHelper::Frame::callee_link_address(f)); } - const_cast(f).describe(values, 0, &map); + const_cast(f).describe(values, 0, &map, true); values.print_on(static_cast(nullptr), st); } #endif static address thaw_entry = nullptr; static address freeze_entry = nullptr; +static address freeze_preempt_entry = nullptr; address Continuation::thaw_entry() { return ::thaw_entry; @@ -2663,6 +3014,10 @@ address Continuation::freeze_entry() { return ::freeze_entry; } +address Continuation::freeze_preempt_entry() { + return ::freeze_preempt_entry; +} + class ConfigResolve { public: static void resolve() { resolve_compressed(); } @@ -2696,6 +3051,7 @@ class ConfigResolve { typedef Config SelectedConfigT; freeze_entry = (address)freeze; + freeze_preempt_entry = (address)SelectedConfigT::freeze_preempt; // If we wanted, we could templatize by kind and have three different thaw entries thaw_entry = (address)thaw; diff --git a/src/hotspot/share/runtime/continuationHelper.hpp b/src/hotspot/share/runtime/continuationHelper.hpp index 6d8474e1844..0b3e6c72452 100644 --- a/src/hotspot/share/runtime/continuationHelper.hpp +++ b/src/hotspot/share/runtime/continuationHelper.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2022, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -58,6 +58,7 @@ class ContinuationHelper { class InterpretedFrame; class NonInterpretedUnknownFrame; class CompiledFrame; + class NativeFrame; class StubFrame; }; @@ -65,6 +66,7 @@ class ContinuationHelper::Frame : public AllStatic { public: static const bool interpreted = false; static const bool stub = false; + static const bool native = false; static inline intptr_t** callee_link_address(const frame& f); static Method* frame_method(const frame& f); @@ -133,11 +135,25 @@ class ContinuationHelper::CompiledFrame : public ContinuationHelper::NonInterpre #endif }; +class ContinuationHelper::NativeFrame : public ContinuationHelper::NonInterpretedFrame { +public: + static const bool native = true; + + static bool is_instance(const frame& f); + +#ifdef ASSERT + static bool is_owning_locks(JavaThread* current, const frame& f); +#endif + + static int stack_argsize(const frame& f) { return 0; } +}; + class ContinuationHelper::StubFrame : public ContinuationHelper::NonInterpretedFrame { public: static const bool stub = true; static bool is_instance(const frame& f); + static int stack_argsize(const frame& f) { return 0; } }; #endif // SHARE_VM_RUNTIME_CONTINUATIONHELPER_HPP diff --git a/src/hotspot/share/runtime/continuationHelper.inline.hpp b/src/hotspot/share/runtime/continuationHelper.inline.hpp index d6b18a75815..279e2a5ffd5 100644 --- a/src/hotspot/share/runtime/continuationHelper.inline.hpp +++ b/src/hotspot/share/runtime/continuationHelper.inline.hpp @@ -32,7 +32,9 @@ #include "compiler/oopMap.inline.hpp" #include "interpreter/oopMapCache.hpp" #include "runtime/frame.inline.hpp" +#include "runtime/objectMonitor.hpp" #include "runtime/stackValue.hpp" +#include "runtime/synchronizer.hpp" #include "utilities/macros.hpp" #include CPU_HEADER_INLINE(continuationHelper) @@ -53,7 +55,7 @@ inline bool ContinuationHelper::NonInterpretedUnknownFrame::is_instance(const fr } inline bool ContinuationHelper::Frame::is_stub(CodeBlob* cb) { - return cb != nullptr && (cb->is_safepoint_stub() || cb->is_runtime_stub()); + return cb != nullptr && cb->is_runtime_stub(); } inline Method* ContinuationHelper::Frame::frame_method(const frame& f) { @@ -195,6 +197,26 @@ bool ContinuationHelper::CompiledFrame::is_owning_locks(JavaThread* thread, Regi } #endif +inline bool ContinuationHelper::NativeFrame::is_instance(const frame& f) { + return f.is_native_frame(); +} + +#ifdef ASSERT +inline bool ContinuationHelper::NativeFrame::is_owning_locks(JavaThread* thread, const frame& f) { + assert(NativeFrame::is_instance(f), ""); + + Method* method = f.cb()->as_nmethod()->method(); + if (!method->is_synchronized()) { + return false; + } else { + // Just verify we are actually the owner + oop synced_obj = f.get_native_receiver(); + assert(ObjectSynchronizer::current_thread_holds_lock(thread, Handle(thread, synced_obj)), "must be owner"); + return true; + } +} +#endif + inline bool ContinuationHelper::StubFrame::is_instance(const frame& f) { return !f.is_interpreted_frame() && is_stub(f.cb()); } diff --git a/src/hotspot/share/runtime/continuationJavaClasses.cpp b/src/hotspot/share/runtime/continuationJavaClasses.cpp index d9f2d05b7e9..5ab96f9cf4f 100644 --- a/src/hotspot/share/runtime/continuationJavaClasses.cpp +++ b/src/hotspot/share/runtime/continuationJavaClasses.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2022, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -87,6 +87,8 @@ int jdk_internal_vm_StackChunk::_pc_offset; int jdk_internal_vm_StackChunk::_bottom_offset; int jdk_internal_vm_StackChunk::_flags_offset; int jdk_internal_vm_StackChunk::_maxThawingSize_offset; +int jdk_internal_vm_StackChunk::_lockStackSize_offset; +int jdk_internal_vm_StackChunk::_objectWaiter_offset; int jdk_internal_vm_StackChunk::_cont_offset; #define STACKCHUNK_FIELDS_DO(macro) \ diff --git a/src/hotspot/share/runtime/continuationJavaClasses.hpp b/src/hotspot/share/runtime/continuationJavaClasses.hpp index 05cba90a0f5..d100ef17871 100644 --- a/src/hotspot/share/runtime/continuationJavaClasses.hpp +++ b/src/hotspot/share/runtime/continuationJavaClasses.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2022, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -71,10 +71,12 @@ class jdk_internal_vm_Continuation: AllStatic { // Interface to jdk.internal.vm.StackChunk objects #define STACKCHUNK_INJECTED_FIELDS(macro) \ - macro(jdk_internal_vm_StackChunk, cont, continuation_signature, false) \ - macro(jdk_internal_vm_StackChunk, flags, byte_signature, false) \ - macro(jdk_internal_vm_StackChunk, pc, intptr_signature, false) \ - macro(jdk_internal_vm_StackChunk, maxThawingSize, int_signature, false) \ + macro(jdk_internal_vm_StackChunk, cont, continuation_signature, false) \ + macro(jdk_internal_vm_StackChunk, flags, byte_signature, false) \ + macro(jdk_internal_vm_StackChunk, pc, intptr_signature, false) \ + macro(jdk_internal_vm_StackChunk, maxThawingSize, int_signature, false) \ + macro(jdk_internal_vm_StackChunk, lockStackSize, byte_signature, false) \ + macro(jdk_internal_vm_StackChunk, objectWaiter, intptr_signature, false) \ class jdk_internal_vm_StackChunk: AllStatic { friend class JavaClasses; @@ -86,6 +88,8 @@ class jdk_internal_vm_StackChunk: AllStatic { static int _bottom_offset; static int _flags_offset; static int _maxThawingSize_offset; + static int _lockStackSize_offset; + static int _objectWaiter_offset; static int _cont_offset; @@ -124,6 +128,12 @@ class jdk_internal_vm_StackChunk: AllStatic { static inline int maxThawingSize(oop chunk); static inline void set_maxThawingSize(oop chunk, int value); + static inline uint8_t lockStackSize(oop chunk); + static inline void set_lockStackSize(oop chunk, uint8_t value); + + static inline address objectWaiter(oop chunk); + static inline void set_objectWaiter(oop chunk, address mon); + // cont oop's processing is essential for the chunk's GC protocol static inline oop cont(oop chunk); template diff --git a/src/hotspot/share/runtime/continuationJavaClasses.inline.hpp b/src/hotspot/share/runtime/continuationJavaClasses.inline.hpp index b2dea2d704a..b4400c59c68 100644 --- a/src/hotspot/share/runtime/continuationJavaClasses.inline.hpp +++ b/src/hotspot/share/runtime/continuationJavaClasses.inline.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -185,4 +185,21 @@ inline void jdk_internal_vm_StackChunk::set_maxThawingSize(oop chunk, int value) chunk->int_field_put(_maxThawingSize_offset, value); } +// lockStackSize is read concurrently by GC threads so we use Atomic. +inline uint8_t jdk_internal_vm_StackChunk::lockStackSize(oop chunk) { + return Atomic::load(chunk->field_addr(_lockStackSize_offset)); +} + +inline void jdk_internal_vm_StackChunk::set_lockStackSize(oop chunk, uint8_t value) { + Atomic::store(chunk->field_addr(_lockStackSize_offset), value); +} + +inline address jdk_internal_vm_StackChunk::objectWaiter(oop chunk) { + return chunk->address_field(_objectWaiter_offset); +} + +inline void jdk_internal_vm_StackChunk::set_objectWaiter(oop chunk, address value) { + chunk->address_field_put(_objectWaiter_offset, value); +} + #endif // SHARE_RUNTIME_CONTINUATIONJAVACLASSES_INLINE_HPP diff --git a/src/hotspot/share/runtime/deoptimization.cpp b/src/hotspot/share/runtime/deoptimization.cpp index 2006f340450..9992d42622f 100644 --- a/src/hotspot/share/runtime/deoptimization.cpp +++ b/src/hotspot/share/runtime/deoptimization.cpp @@ -122,6 +122,10 @@ DeoptimizationScope::~DeoptimizationScope() { } void DeoptimizationScope::mark(nmethod* nm, bool inc_recompile_counts) { + if (!nm->can_be_deoptimized()) { + return; + } + ConditionalMutexLocker ml(NMethodState_lock, !NMethodState_lock->owned_by_self(), Mutex::_no_safepoint_check_flag); // If it's already marked but we still need it to be deopted. @@ -1666,7 +1670,7 @@ bool Deoptimization::relock_objects(JavaThread* thread, GrowableArrayowner()->is_locked(), "object must be locked now"); assert(obj->mark().has_monitor(), "must be"); assert(!deoptee_thread->lock_stack().contains(obj()), "must be"); - assert(ObjectSynchronizer::read_monitor(thread, obj(), obj->mark())->owner() == deoptee_thread, "must be"); + assert(ObjectSynchronizer::read_monitor(thread, obj(), obj->mark())->has_owner(deoptee_thread), "must be"); } else { ObjectSynchronizer::enter_for(obj, lock, deoptee_thread); assert(mon_info->owner()->is_locked(), "object must be locked now"); diff --git a/src/hotspot/share/runtime/frame.cpp b/src/hotspot/share/runtime/frame.cpp index 17078a69ab9..738b89e77bd 100644 --- a/src/hotspot/share/runtime/frame.cpp +++ b/src/hotspot/share/runtime/frame.cpp @@ -497,7 +497,6 @@ jint frame::interpreter_frame_expression_stack_size() const { return (jint)stack_size; } - // (frame::interpreter_frame_sender_sp accessor is in frame_.cpp) const char* frame::print_name() const { @@ -577,10 +576,21 @@ void frame::interpreter_frame_print_on(outputStream* st) const { current < interpreter_frame_monitor_begin(); current = next_monitor_in_interpreter_frame(current)) { st->print(" - obj [%s", current->obj() == nullptr ? "null" : ""); - if (current->obj() != nullptr) current->obj()->print_value_on(st); + oop obj = current->obj(); + if (obj != nullptr) { + if (!is_heap_frame()) { + obj->print_value_on(st); + } else { + // Might be an invalid oop. We don't have the + // stackChunk to correct it so just print address. + st->print(INTPTR_FORMAT, p2i(obj)); + } + } st->print_cr("]"); st->print(" - lock ["); - current->lock()->print_on(st, current->obj()); + if (!is_heap_frame()) { + current->lock()->print_on(st, obj); + } st->print_cr("]"); } // monitor @@ -1085,7 +1095,7 @@ oop frame::retrieve_receiver(RegisterMap* reg_map) { } -BasicLock* frame::get_native_monitor() { +BasicLock* frame::get_native_monitor() const { nmethod* nm = (nmethod*)_cb; assert(_cb != nullptr && _cb->is_nmethod() && nm->method()->is_native(), "Should not call this unless it's a native nmethod"); @@ -1094,7 +1104,7 @@ BasicLock* frame::get_native_monitor() { return (BasicLock*) &sp()[byte_offset / wordSize]; } -oop frame::get_native_receiver() { +oop frame::get_native_receiver() const { nmethod* nm = (nmethod*)_cb; assert(_cb != nullptr && _cb->is_nmethod() && nm->method()->is_native(), "Should not call this unless it's a native nmethod"); @@ -1340,9 +1350,14 @@ class FrameValuesOopMapClosure: public OopMapClosure { // callers need a ResourceMark because of name_and_sig_as_C_string() usage, // RA allocated string is returned to the caller -void frame::describe(FrameValues& values, int frame_no, const RegisterMap* reg_map) { +void frame::describe(FrameValues& values, int frame_no, const RegisterMap* reg_map, bool top) { // boundaries: sp and the 'real' frame pointer values.describe(-1, sp(), err_msg("sp for #%d", frame_no), 0); + if (top) { + values.describe(-1, sp() - 1, err_msg("sp[-1] for #%d", frame_no), 0); + values.describe(-1, sp() - 2, err_msg("sp[-2] for #%d", frame_no), 0); + } + intptr_t* frame_pointer = real_fp(); // Note: may differ from fp() // print frame info at the highest boundary @@ -1414,7 +1429,7 @@ void frame::describe(FrameValues& values, int frame_no, const RegisterMap* reg_m } else if (is_entry_frame()) { // For now just label the frame values.describe(-1, info_address, err_msg("#%d entry frame", frame_no), 2); - } else if (cb()->is_nmethod()) { + } else if (is_compiled_frame()) { // For now just label the frame nmethod* nm = cb()->as_nmethod(); values.describe(-1, info_address, @@ -1517,17 +1532,16 @@ void frame::describe(FrameValues& values, int frame_no, const RegisterMap* reg_m oop_map()->all_type_do(this, OopMapValue::callee_saved_value, &valuesFn); } } - - if (nm->method()->is_continuation_enter_intrinsic()) { - ContinuationEntry* ce = Continuation::get_continuation_entry_for_entry_frame(reg_map->thread(), *this); // (ContinuationEntry*)unextended_sp(); - ce->describe(values, frame_no); - } } else if (is_native_frame()) { // For now just label the frame nmethod* nm = cb()->as_nmethod_or_null(); values.describe(-1, info_address, FormatBuffer<1024>("#%d nmethod " INTPTR_FORMAT " for native method %s", frame_no, p2i(nm), nm->method()->name_and_sig_as_C_string()), 2); + if (nm->method()->is_continuation_enter_intrinsic()) { + ContinuationEntry* ce = Continuation::get_continuation_entry_for_entry_frame(reg_map->thread(), *this); // (ContinuationEntry*)unextended_sp(); + ce->describe(values, frame_no); + } } else { // provide default info if not handled before char *info = (char *) "special frame"; diff --git a/src/hotspot/share/runtime/frame.hpp b/src/hotspot/share/runtime/frame.hpp index 50aafce3837..fe531cff8a6 100644 --- a/src/hotspot/share/runtime/frame.hpp +++ b/src/hotspot/share/runtime/frame.hpp @@ -337,8 +337,8 @@ class frame { // Return the monitor owner and BasicLock for compiled synchronized // native methods. Used by JVMTI's GetLocalInstance method // (via VM_GetReceiver) to retrieve the receiver from a native wrapper frame. - BasicLock* get_native_monitor(); - oop get_native_receiver(); + BasicLock* get_native_monitor() const; + oop get_native_receiver() const; // Find receiver for an invoke when arguments are just pushed on stack (i.e., callee stack-frame is // not setup) @@ -426,6 +426,8 @@ class frame { oop saved_oop_result(RegisterMap* map) const; void set_saved_oop_result(RegisterMap* map, oop obj); + static JavaThread** saved_thread_address(const frame& f); + // For debugging private: const char* print_name() const; @@ -442,7 +444,7 @@ class frame { #ifndef PRODUCT // Add annotated descriptions of memory locations belonging to this frame to values - void describe(FrameValues& values, int frame_no, const RegisterMap* reg_map=nullptr); + void describe(FrameValues& values, int frame_no, const RegisterMap* reg_map=nullptr, bool top = false); #endif // Conversion from a VMReg to physical stack location diff --git a/src/hotspot/share/runtime/globals.hpp b/src/hotspot/share/runtime/globals.hpp index b568e769304..70b537bf2d7 100644 --- a/src/hotspot/share/runtime/globals.hpp +++ b/src/hotspot/share/runtime/globals.hpp @@ -128,6 +128,9 @@ const size_t minimumSymbolTableSize = 1024; "Use 32-bit class pointers in 64-bit VM. " \ "lp64_product means flag is always constant in 32 bit VM") \ \ + product(bool, UseCompactObjectHeaders, false, EXPERIMENTAL, \ + "Use compact 64-bit object headers in 64-bit VM") \ + \ product(int, ObjectAlignmentInBytes, 8, \ "Default object alignment in bytes, 8 is minimum") \ range(8, 256) \ @@ -144,6 +147,7 @@ const size_t minimumSymbolTableSize = 1024; constraint) const bool UseCompressedOops = false; const bool UseCompressedClassPointers = false; +const bool UseCompactObjectHeaders = false; const int ObjectAlignmentInBytes = 8; #endif // _LP64 @@ -677,9 +681,6 @@ const int ObjectAlignmentInBytes = 8; develop(bool, PrintCodeCacheExtension, false, \ "Print extension of code cache") \ \ - develop(bool, UsePrivilegedStack, true, \ - "Enable the security JVM functions") \ - \ product(bool, ClassUnloading, true, \ "Do unloading of classes") \ \ diff --git a/src/hotspot/share/runtime/javaThread.cpp b/src/hotspot/share/runtime/javaThread.cpp index 285c2de17fa..fd9f75c41b4 100644 --- a/src/hotspot/share/runtime/javaThread.cpp +++ b/src/hotspot/share/runtime/javaThread.cpp @@ -83,6 +83,7 @@ #include "runtime/stackWatermarkSet.hpp" #include "runtime/synchronizer.hpp" #include "runtime/threadCritical.hpp" +#include "runtime/threadIdentifier.hpp" #include "runtime/threadSMR.inline.hpp" #include "runtime/threadStatisticalInfo.hpp" #include "runtime/threadWXSetters.inline.hpp" @@ -235,6 +236,8 @@ void JavaThread::allocate_threadObj(Handle thread_group, const char* thread_name // constructor calls Thread.current(), which must be set here. java_lang_Thread::set_thread(thread_oop(), this); set_threadOopHandles(thread_oop()); + // Set the lock_id to the next thread_id temporarily while initialization runs. + set_lock_id(ThreadIdentifier::next()); JavaValue result(T_VOID); if (thread_name != nullptr) { @@ -260,6 +263,9 @@ void JavaThread::allocate_threadObj(Handle thread_group, const char* thread_name Handle(), CHECK); } + // Update the lock_id with the tid value. + set_lock_id(java_lang_Thread::thread_id(thread_oop())); + os::set_priority(this, NormPriority); if (daemon) { @@ -429,6 +435,7 @@ JavaThread::JavaThread(MemTag mem_tag) : _current_waiting_monitor(nullptr), _active_handles(nullptr), _free_handle_block(nullptr), + _lock_id(0), _suspend_flags(0), @@ -448,6 +455,8 @@ JavaThread::JavaThread(MemTag mem_tag) : _is_in_VTMS_transition(false), _is_disable_suspend(false), _VTMS_transition_mark(false), + _on_monitor_waited_event(false), + _contended_entered_monitor(nullptr), #ifdef ASSERT _is_VTMS_transition_disabler(false), #endif @@ -488,6 +497,10 @@ JavaThread::JavaThread(MemTag mem_tag) : _jni_monitor_count(0), _unlocked_inflated_monitor(nullptr), + _preempt_alternate_return(nullptr), + _preemption_cancelled(false), + _pending_interrupted_exception(false), + _handshake(this), _popframe_preserved_args(nullptr), @@ -501,6 +514,7 @@ JavaThread::JavaThread(MemTag mem_tag) : _parker(), _class_to_be_initialized(nullptr), + _class_being_initialized(nullptr), _SleepEvent(ParkEvent::Allocate(this)), @@ -1162,6 +1176,7 @@ void JavaThread::send_async_exception(JavaThread* target, oop java_throwable) { #if INCLUDE_JVMTI void JavaThread::set_is_in_VTMS_transition(bool val) { + assert(is_in_VTMS_transition() != val, "already %s transition", val ? "inside" : "outside"); _is_in_VTMS_transition = val; } @@ -1525,9 +1540,8 @@ void JavaThread::print_on(outputStream *st, bool print_extended_info) const { st->print_cr("[" INTPTR_FORMAT "]", (intptr_t)last_Java_sp() & ~right_n_bits(12)); if (thread_oop != nullptr) { if (is_vthread_mounted()) { - oop vt = vthread(); - assert(vt != nullptr, ""); - st->print_cr(" Carrying virtual thread #" INT64_FORMAT, (int64_t)java_lang_Thread::thread_id(vt)); + // _lock_id is the thread ID of the mounted virtual thread + st->print_cr(" Carrying virtual thread #" INT64_FORMAT, lock_id()); } else { st->print_cr(" java.lang.Thread.State: %s", java_lang_Thread::thread_status_name(thread_oop)); } @@ -1711,6 +1725,7 @@ void JavaThread::prepare(jobject jni_thread, ThreadPriority prio) { assert(InstanceKlass::cast(thread_oop->klass())->is_linked(), "must be initialized"); set_threadOopHandles(thread_oop()); + set_lock_id(java_lang_Thread::thread_id(thread_oop())); if (prio == NoPriority) { prio = java_lang_Thread::priority(thread_oop()); @@ -1990,6 +2005,14 @@ void JavaThread::trace_stack() { // this slow-path. void JavaThread::inc_held_monitor_count(intx i, bool jni) { #ifdef SUPPORT_MONITOR_COUNT + + if (LockingMode != LM_LEGACY) { + // Nothing to do. Just do some sanity check. + assert(_held_monitor_count == 0, "counter should not be used"); + assert(_jni_monitor_count == 0, "counter should not be used"); + return; + } + assert(_held_monitor_count >= 0, "Must always be non-negative: " INTX_FORMAT, _held_monitor_count); _held_monitor_count += i; if (jni) { @@ -1998,18 +2021,26 @@ void JavaThread::inc_held_monitor_count(intx i, bool jni) { } assert(_held_monitor_count >= _jni_monitor_count, "Monitor count discrepancy detected - held count " INTX_FORMAT " is less than JNI count " INTX_FORMAT, _held_monitor_count, _jni_monitor_count); -#endif +#endif // SUPPORT_MONITOR_COUNT } // Slow-path decrement of the held monitor counts. JNI unlocking is always // this slow-path. void JavaThread::dec_held_monitor_count(intx i, bool jni) { #ifdef SUPPORT_MONITOR_COUNT + + if (LockingMode != LM_LEGACY) { + // Nothing to do. Just do some sanity check. + assert(_held_monitor_count == 0, "counter should not be used"); + assert(_jni_monitor_count == 0, "counter should not be used"); + return; + } + _held_monitor_count -= i; - assert(_held_monitor_count >= 0, "Must always be greater than 0: " INTX_FORMAT, _held_monitor_count); + assert(_held_monitor_count >= 0, "Must always be non-negative: " INTX_FORMAT, _held_monitor_count); if (jni) { _jni_monitor_count -= i; - assert(_jni_monitor_count >= 0, "Must always be greater than 0: " INTX_FORMAT, _jni_monitor_count); + assert(_jni_monitor_count >= 0, "Must always be non-negative: " INTX_FORMAT, _jni_monitor_count); } // When a thread is detaching with still owned JNI monitors, the logic that releases // the monitors doesn't know to set the "jni" flag and so the counts can get out of sync. @@ -2017,7 +2048,7 @@ void JavaThread::dec_held_monitor_count(intx i, bool jni) { // JNI count is directly set to zero. assert(_held_monitor_count >= _jni_monitor_count || is_exiting(), "Monitor count discrepancy detected - held count " INTX_FORMAT " is less than JNI count " INTX_FORMAT, _held_monitor_count, _jni_monitor_count); -#endif +#endif // SUPPORT_MONITOR_COUNT } frame JavaThread::vthread_last_frame() { @@ -2199,6 +2230,7 @@ void JavaThread::start_internal_daemon(JavaThread* current, JavaThread* target, // Now bind the thread_oop to the target JavaThread. target->set_threadOopHandles(thread_oop()); + target->set_lock_id(java_lang_Thread::thread_id(thread_oop())); Threads::add(target); // target is now visible for safepoint/handshake // Publish the JavaThread* in java.lang.Thread after the JavaThread* is @@ -2295,3 +2327,38 @@ void JavaThread::add_oop_handles_for_release() { _oop_handle_list = new_head; Service_lock->notify_all(); } + +#if INCLUDE_JFR +void JavaThread::set_last_freeze_fail_result(freeze_result result) { + assert(result != freeze_ok, "sanity check"); + _last_freeze_fail_result = result; + _last_freeze_fail_time = Ticks::now(); +} + +// Post jdk.VirtualThreadPinned event +void JavaThread::post_vthread_pinned_event(EventVirtualThreadPinned* event, const char* op, freeze_result result) { + assert(result != freeze_ok, "sanity check"); + if (event->should_commit()) { + char reason[256]; + if (class_to_be_initialized() != nullptr) { + ResourceMark rm(this); + jio_snprintf(reason, sizeof reason, "Waited for initialization of %s by another thread", + class_to_be_initialized()->external_name()); + event->set_pinnedReason(reason); + } else if (class_being_initialized() != nullptr) { + ResourceMark rm(this); + jio_snprintf(reason, sizeof(reason), "VM call to %s. on stack", + class_being_initialized()->external_name()); + event->set_pinnedReason(reason); + } else if (result == freeze_pinned_native) { + event->set_pinnedReason("Native or VM frame on stack"); + } else { + jio_snprintf(reason, sizeof(reason), "Freeze or preempt failed (%d)", result); + event->set_pinnedReason(reason); + } + event->set_blockingOperation(op); + event->set_carrierThread(JFR_JVM_THREAD_ID(this)); + event->commit(); + } +} +#endif \ No newline at end of file diff --git a/src/hotspot/share/runtime/javaThread.hpp b/src/hotspot/share/runtime/javaThread.hpp index 249a1e4dc87..e24138583b8 100644 --- a/src/hotspot/share/runtime/javaThread.hpp +++ b/src/hotspot/share/runtime/javaThread.hpp @@ -30,6 +30,7 @@ #include "memory/allocation.hpp" #include "oops/oop.hpp" #include "oops/oopHandle.hpp" +#include "runtime/continuationEntry.hpp" #include "runtime/frame.hpp" #include "runtime/globals.hpp" #include "runtime/handshake.hpp" @@ -41,16 +42,17 @@ #include "runtime/stackOverflow.hpp" #include "runtime/thread.hpp" #include "runtime/threadHeapSampler.hpp" +#include "runtime/threadIdentifier.hpp" #include "runtime/threadStatisticalInfo.hpp" #include "utilities/exceptions.hpp" #include "utilities/globalDefinitions.hpp" #include "utilities/macros.hpp" #if INCLUDE_JFR #include "jfr/support/jfrThreadExtension.hpp" +#include "utilities/ticks.hpp" #endif class AsyncExceptionHandshake; -class ContinuationEntry; class DeoptResourceMark; class InternalOOMEMark; class JNIHandleBlock; @@ -77,6 +79,8 @@ class javaVFrame; class JavaThread; typedef void (*ThreadFunction)(JavaThread*, TRAPS); +class EventVirtualThreadPinned; + class JavaThread: public Thread { friend class VMStructs; friend class JVMCIVMStructs; @@ -159,7 +163,18 @@ class JavaThread: public Thread { // One-element thread local free list JNIHandleBlock* _free_handle_block; + // ID used as owner for inflated monitors. Same as the j.l.Thread.tid of the + // current _vthread object, except during creation of the primordial and JNI + // attached thread cases where this field can have a temporary value. + int64_t _lock_id; + public: + void set_lock_id(int64_t tid) { + assert(tid >= ThreadIdentifier::initial() && tid < ThreadIdentifier::current(), "invalid tid"); + _lock_id = tid; + } + int64_t lock_id() const { return _lock_id; } + // For tracking the heavyweight monitor the thread is pending on. ObjectMonitor* current_pending_monitor() { // Use Atomic::load() to prevent data race between concurrent modification and @@ -313,6 +328,8 @@ class JavaThread: public Thread { bool _is_in_VTMS_transition; // thread is in virtual thread mount state transition bool _is_disable_suspend; // JVMTI suspend is temporarily disabled; used on current thread only bool _VTMS_transition_mark; // used for sync between VTMS transitions and disablers + bool _on_monitor_waited_event; // Avoid callee arg processing for enterSpecial when posting waited event + ObjectMonitor* _contended_entered_monitor; // Monitor for pending monitor_contended_entered callback #ifdef ASSERT bool _is_VTMS_transition_disabler; // thread currently disabled VTMS transitions #endif @@ -454,8 +471,8 @@ class JavaThread: public Thread { int _frames_to_pop_failed_realloc; ContinuationEntry* _cont_entry; - intptr_t* _cont_fastpath; // the sp of the oldest known interpreted/call_stub frame inside the - // continuation that we know about + intptr_t* _cont_fastpath; // the sp of the oldest known interpreted/call_stub/upcall_stub/native_wrapper + // frame inside the continuation that we know about int _cont_fastpath_thread_state; // whether global thread state allows continuation fastpath (JVMTI) // It's signed for error detection. @@ -463,6 +480,28 @@ class JavaThread: public Thread { intx _jni_monitor_count; ObjectMonitor* _unlocked_inflated_monitor; + // This is the field we poke in the interpreter and native + // wrapper (Object.wait) to check for preemption. + address _preempt_alternate_return; + // When preempting on monitorenter we could have acquired the + // monitor after freezing all vthread frames. In that case we + // set this field so that in the preempt stub we call thaw again + // instead of unmounting. + bool _preemption_cancelled; + // For Object.wait() we set this field to know if we need to + // throw IE at the end of thawing before returning to Java. + bool _pending_interrupted_exception; + + public: + bool preemption_cancelled() { return _preemption_cancelled; } + void set_preemption_cancelled(bool b) { _preemption_cancelled = b; } + + bool pending_interrupted_exception() { return _pending_interrupted_exception; } + void set_pending_interrupted_exception(bool b) { _pending_interrupted_exception = b; } + + bool preempting() { return _preempt_alternate_return != nullptr; } + void set_preempt_alternate_return(address val) { _preempt_alternate_return = val; } + private: friend class VMThread; @@ -627,7 +666,7 @@ class JavaThread: public Thread { intx held_monitor_count() { return _held_monitor_count; } intx jni_monitor_count() { return _jni_monitor_count; } - void clear_jni_monitor_count() { _jni_monitor_count = 0; } + void clear_jni_monitor_count() { _jni_monitor_count = 0; } // Support for SharedRuntime::monitor_exit_helper() ObjectMonitor* unlocked_inflated_monitor() const { return _unlocked_inflated_monitor; } @@ -682,12 +721,19 @@ class JavaThread: public Thread { bool VTMS_transition_mark() const { return Atomic::load(&_VTMS_transition_mark); } void set_VTMS_transition_mark(bool val) { Atomic::store(&_VTMS_transition_mark, val); } + bool on_monitor_waited_event() { return _on_monitor_waited_event; } + void set_on_monitor_waited_event(bool val) { _on_monitor_waited_event = val; } + + bool pending_contended_entered_event() { return _contended_entered_monitor != nullptr; } + ObjectMonitor* contended_entered_monitor() { return _contended_entered_monitor; } #ifdef ASSERT bool is_VTMS_transition_disabler() const { return _is_VTMS_transition_disabler; } void set_is_VTMS_transition_disabler(bool val); #endif #endif + void set_contended_entered_monitor(ObjectMonitor* val) NOT_JVMTI_RETURN JVMTI_ONLY({ _contended_entered_monitor = val; }) + // Support for object deoptimization and JFR suspension void handle_special_runtime_exit_condition(); bool has_special_runtime_exit_condition() { @@ -838,10 +884,14 @@ class JavaThread: public Thread { static ByteSize doing_unsafe_access_offset() { return byte_offset_of(JavaThread, _doing_unsafe_access); } NOT_PRODUCT(static ByteSize requires_cross_modify_fence_offset() { return byte_offset_of(JavaThread, _requires_cross_modify_fence); }) + static ByteSize lock_id_offset() { return byte_offset_of(JavaThread, _lock_id); } + static ByteSize cont_entry_offset() { return byte_offset_of(JavaThread, _cont_entry); } static ByteSize cont_fastpath_offset() { return byte_offset_of(JavaThread, _cont_fastpath); } static ByteSize held_monitor_count_offset() { return byte_offset_of(JavaThread, _held_monitor_count); } static ByteSize jni_monitor_count_offset() { return byte_offset_of(JavaThread, _jni_monitor_count); } + static ByteSize preemption_cancelled_offset() { return byte_offset_of(JavaThread, _preemption_cancelled); } + static ByteSize preempt_alternate_return_offset() { return byte_offset_of(JavaThread, _preempt_alternate_return); } static ByteSize unlocked_inflated_monitor_offset() { return byte_offset_of(JavaThread, _unlocked_inflated_monitor); } #if INCLUDE_JVMTI @@ -1161,11 +1211,23 @@ class JavaThread: public Thread { void set_class_to_be_initialized(InstanceKlass* k); InstanceKlass* class_to_be_initialized() const; + // Track executing class initializer, see ThreadInClassInitializer + void set_class_being_initialized(InstanceKlass* k); + InstanceKlass* class_being_initialized() const; + private: InstanceKlass* _class_to_be_initialized; + InstanceKlass* _class_being_initialized; // java.lang.Thread.sleep support ParkEvent * _SleepEvent; + +#if INCLUDE_JFR + // Support for jdk.VirtualThreadPinned event + freeze_result _last_freeze_fail_result; + Ticks _last_freeze_fail_time; +#endif + public: bool sleep(jlong millis); bool sleep_nanos(jlong nanos); @@ -1174,6 +1236,15 @@ class JavaThread: public Thread { void interrupt(); bool is_interrupted(bool clear_interrupted); +#if INCLUDE_JFR + // Support for jdk.VirtualThreadPinned event + freeze_result last_freeze_fail_result() { return _last_freeze_fail_result; } + Ticks& last_freeze_fail_time() { return _last_freeze_fail_time; } + void set_last_freeze_fail_result(freeze_result result); +#endif + void post_vthread_pinned_event(EventVirtualThreadPinned* event, const char* op, freeze_result result) NOT_JFR_RETURN(); + + // This is only for use by JVMTI RawMonitorWait. It emulates the actions of // the Java code in Object::wait which are not present in RawMonitorWait. bool get_and_clear_interrupted(); @@ -1261,4 +1332,36 @@ class JNIHandleMark : public StackObj { ~JNIHandleMark() { _thread->pop_jni_handle_block(); } }; +class NoPreemptMark { + ContinuationEntry* _ce; + bool _unpin; + public: + NoPreemptMark(JavaThread* thread) : _ce(thread->last_continuation()), _unpin(false) { + if (_ce != nullptr) _unpin = _ce->pin(); + } + ~NoPreemptMark() { if (_unpin) _ce->unpin(); } +}; + +class ThreadOnMonitorWaitedEvent { + JavaThread* _thread; + public: + ThreadOnMonitorWaitedEvent(JavaThread* thread) : _thread(thread) { + JVMTI_ONLY(_thread->set_on_monitor_waited_event(true);) + } + ~ThreadOnMonitorWaitedEvent() { JVMTI_ONLY(_thread->set_on_monitor_waited_event(false);) } +}; + +class ThreadInClassInitializer : public StackObj { + JavaThread* _thread; + InstanceKlass* _previous; + public: + ThreadInClassInitializer(JavaThread* thread, InstanceKlass* ik) : _thread(thread) { + _previous = _thread->class_being_initialized(); + _thread->set_class_being_initialized(ik); + } + ~ThreadInClassInitializer() { + _thread->set_class_being_initialized(_previous); + } +}; + #endif // SHARE_RUNTIME_JAVATHREAD_HPP diff --git a/src/hotspot/share/runtime/javaThread.inline.hpp b/src/hotspot/share/runtime/javaThread.inline.hpp index fabb589c2b5..f434fc161bb 100644 --- a/src/hotspot/share/runtime/javaThread.inline.hpp +++ b/src/hotspot/share/runtime/javaThread.inline.hpp @@ -241,10 +241,20 @@ inline InstanceKlass* JavaThread::class_to_be_initialized() const { return _class_to_be_initialized; } +inline void JavaThread::set_class_being_initialized(InstanceKlass* k) { + assert(k != nullptr || _class_being_initialized != nullptr, "incorrect usage"); + assert(this == Thread::current(), "Only the current thread can set this field"); + _class_being_initialized = k; +} + +inline InstanceKlass* JavaThread::class_being_initialized() const { + return _class_being_initialized; +} + inline void JavaThread::om_set_monitor_cache(ObjectMonitor* monitor) { assert(UseObjectMonitorTable, "must be"); assert(monitor != nullptr, "use om_clear_monitor_cache to clear"); - assert(this == current() || monitor->owner_raw() == this, "only add owned monitors for other threads"); + assert(this == current() || monitor->has_owner(this), "only add owned monitors for other threads"); assert(this == current() || is_obj_deopt_suspend(), "thread must not run concurrently"); _om_cache.set_monitor(monitor); diff --git a/src/hotspot/share/runtime/lightweightSynchronizer.cpp b/src/hotspot/share/runtime/lightweightSynchronizer.cpp index 7cf7c201faf..b3bcd5f7029 100644 --- a/src/hotspot/share/runtime/lightweightSynchronizer.cpp +++ b/src/hotspot/share/runtime/lightweightSynchronizer.cpp @@ -344,7 +344,7 @@ ObjectMonitor* LightweightSynchronizer::get_or_insert_monitor_from_table(oop obj } ObjectMonitor* alloced_monitor = new ObjectMonitor(object); - alloced_monitor->set_owner_anonymous(); + alloced_monitor->set_anonymous_owner(); // Try insert monitor monitor = add_monitor(current, alloced_monitor, object); @@ -623,8 +623,6 @@ void LightweightSynchronizer::enter_for(Handle obj, BasicLock* lock, JavaThread* ObjectSynchronizer::handle_sync_on_value_based_class(obj, locking_thread); } - locking_thread->inc_held_monitor_count(); - CacheSetter cache_setter(locking_thread, lock); LockStack& lock_stack = locking_thread->lock_stack(); @@ -654,8 +652,6 @@ void LightweightSynchronizer::enter(Handle obj, BasicLock* lock, JavaThread* cur ObjectSynchronizer::handle_sync_on_value_based_class(obj, current); } - current->inc_held_monitor_count(); - CacheSetter cache_setter(current, lock); // Used when deflation is observed. Progress here requires progress @@ -745,7 +741,7 @@ void LightweightSynchronizer::exit(oop object, JavaThread* current) { assert(mark.has_monitor(), "must be"); // The monitor exists ObjectMonitor* monitor = ObjectSynchronizer::read_monitor(current, object, mark); - if (monitor->is_owner_anonymous()) { + if (monitor->has_anonymous_owner()) { assert(current->lock_stack().contains(object), "current must have object on its lock stack"); monitor->set_owner_from_anonymous(current); monitor->set_recursions(current->lock_stack().remove(object) - 1); @@ -790,7 +786,7 @@ ObjectMonitor* LightweightSynchronizer::inflate_locked_or_imse(oop obj, ObjectSy assert(mark.has_monitor(), "must be"); ObjectMonitor* monitor = ObjectSynchronizer::read_monitor(current, obj, mark); if (monitor != nullptr) { - if (monitor->is_owner_anonymous()) { + if (monitor->has_anonymous_owner()) { LockStack& lock_stack = current->lock_stack(); if (lock_stack.contains(obj)) { // Current thread owns the lock but someone else inflated it. @@ -808,10 +804,10 @@ ObjectMonitor* LightweightSynchronizer::inflate_locked_or_imse(oop obj, ObjectSy } } -ObjectMonitor* LightweightSynchronizer::inflate_into_object_header(oop object, ObjectSynchronizer::InflateCause cause, JavaThread* inflating_thread, Thread* current) { +ObjectMonitor* LightweightSynchronizer::inflate_into_object_header(oop object, ObjectSynchronizer::InflateCause cause, JavaThread* locking_thread, Thread* current) { - // The JavaThread* inflating_thread parameter is only used by LM_LIGHTWEIGHT and requires - // that the inflating_thread == Thread::current() or is suspended throughout the call by + // The JavaThread* locking_thread parameter is only used by LM_LIGHTWEIGHT and requires + // that the locking_thread == Thread::current() or is suspended throughout the call by // some other mechanism. // Even with LM_LIGHTWEIGHT the thread might be nullptr when called from a non // JavaThread. (As may still be the case from FastHashCode). However it is only @@ -826,10 +822,10 @@ ObjectMonitor* LightweightSynchronizer::inflate_into_object_header(oop object, O // The mark can be in one of the following states: // * inflated - Just return if using stack-locking. // If using fast-locking and the ObjectMonitor owner - // is anonymous and the inflating_thread owns the - // object lock, then we make the inflating_thread + // is anonymous and the locking_thread owns the + // object lock, then we make the locking_thread // the ObjectMonitor owner and remove the lock from - // the inflating_thread's lock stack. + // the locking_thread's lock stack. // * fast-locked - Coerce it to inflated from fast-locked. // * unlocked - Aggressively inflate the object. @@ -838,42 +834,42 @@ ObjectMonitor* LightweightSynchronizer::inflate_into_object_header(oop object, O ObjectMonitor* inf = mark.monitor(); markWord dmw = inf->header(); assert(dmw.is_neutral(), "invariant: header=" INTPTR_FORMAT, dmw.value()); - if (inf->is_owner_anonymous() && - inflating_thread != nullptr && inflating_thread->lock_stack().contains(object)) { - inf->set_owner_from_anonymous(inflating_thread); - size_t removed = inflating_thread->lock_stack().remove(object); + if (inf->has_anonymous_owner() && + locking_thread != nullptr && locking_thread->lock_stack().contains(object)) { + inf->set_owner_from_anonymous(locking_thread); + size_t removed = locking_thread->lock_stack().remove(object); inf->set_recursions(removed - 1); } return inf; } // CASE: fast-locked - // Could be fast-locked either by the inflating_thread or by some other thread. + // Could be fast-locked either by the locking_thread or by some other thread. // // Note that we allocate the ObjectMonitor speculatively, _before_ // attempting to set the object's mark to the new ObjectMonitor. If - // the inflating_thread owns the monitor, then we set the ObjectMonitor's - // owner to the inflating_thread. Otherwise, we set the ObjectMonitor's owner + // the locking_thread owns the monitor, then we set the ObjectMonitor's + // owner to the locking_thread. Otherwise, we set the ObjectMonitor's owner // to anonymous. If we lose the race to set the object's mark to the // new ObjectMonitor, then we just delete it and loop around again. // if (mark.is_fast_locked()) { ObjectMonitor* monitor = new ObjectMonitor(object); monitor->set_header(mark.set_unlocked()); - bool own = inflating_thread != nullptr && inflating_thread->lock_stack().contains(object); + bool own = locking_thread != nullptr && locking_thread->lock_stack().contains(object); if (own) { - // Owned by inflating_thread. - monitor->set_owner_from(nullptr, inflating_thread); + // Owned by locking_thread. + monitor->set_owner(locking_thread); } else { // Owned by somebody else. - monitor->set_owner_anonymous(); + monitor->set_anonymous_owner(); } markWord monitor_mark = markWord::encode(monitor); markWord old_mark = object->cas_set_mark(monitor_mark, mark); if (old_mark == mark) { // Success! Return inflated monitor. if (own) { - size_t removed = inflating_thread->lock_stack().remove(object); + size_t removed = locking_thread->lock_stack().remove(object); monitor->set_recursions(removed - 1); } // Once the ObjectMonitor is configured and object is associated @@ -956,7 +952,7 @@ ObjectMonitor* LightweightSynchronizer::inflate_fast_locked_object(oop object, O // ObjectMonitors are always inserted as anonymously owned, this thread is // the current holder of the monitor. So unless the entry is stale and // contains a deflating monitor it must be anonymously owned. - if (monitor->is_owner_anonymous()) { + if (monitor->has_anonymous_owner()) { // The monitor must be anonymously owned if it was added assert(monitor == get_monitor_from_table(current, object), "The monitor must be found"); // New fresh monitor @@ -1077,7 +1073,7 @@ ObjectMonitor* LightweightSynchronizer::inflate_and_enter(oop object, ObjectSync // CASE: inflated if (mark.has_monitor()) { LockStack& lock_stack = locking_thread->lock_stack(); - if (monitor->is_owner_anonymous() && lock_stack.contains(object)) { + if (monitor->has_anonymous_owner() && lock_stack.contains(object)) { // The lock is fast-locked by the locking thread, // convert it to a held monitor with a known owner. monitor->set_owner_from_anonymous(locking_thread); @@ -1189,7 +1185,6 @@ bool LightweightSynchronizer::quick_enter(oop obj, BasicLock* lock, JavaThread* // Only for 32bit which has limited support for fast locking outside the runtime. if (lock_stack.try_recursive_enter(obj)) { // Recursive lock successful. - current->inc_held_monitor_count(); return true; } @@ -1198,7 +1193,6 @@ bool LightweightSynchronizer::quick_enter(oop obj, BasicLock* lock, JavaThread* if (obj->cas_set_mark(locked_mark, mark) == mark) { // Successfully fast-locked, push object to lock-stack and return. lock_stack.push(obj); - current->inc_held_monitor_count(); return true; } } @@ -1216,7 +1210,6 @@ bool LightweightSynchronizer::quick_enter(oop obj, BasicLock* lock, JavaThread* if (monitor->try_enter(current)) { // ObjectMonitor enter successful. cache_setter.set_monitor(monitor); - current->inc_held_monitor_count(); return true; } } diff --git a/src/hotspot/share/runtime/lightweightSynchronizer.hpp b/src/hotspot/share/runtime/lightweightSynchronizer.hpp index c546988f778..fdd753e9b9c 100644 --- a/src/hotspot/share/runtime/lightweightSynchronizer.hpp +++ b/src/hotspot/share/runtime/lightweightSynchronizer.hpp @@ -63,7 +63,7 @@ class LightweightSynchronizer : AllStatic { static void enter(Handle obj, BasicLock* lock, JavaThread* current); static void exit(oop object, JavaThread* current); - static ObjectMonitor* inflate_into_object_header(oop object, ObjectSynchronizer::InflateCause cause, JavaThread* inflating_thread, Thread* current); + static ObjectMonitor* inflate_into_object_header(oop object, ObjectSynchronizer::InflateCause cause, JavaThread* locking_thread, Thread* current); static ObjectMonitor* inflate_locked_or_imse(oop object, ObjectSynchronizer::InflateCause cause, TRAPS); static ObjectMonitor* inflate_fast_locked_object(oop object, ObjectSynchronizer::InflateCause cause, JavaThread* locking_thread, JavaThread* current); static ObjectMonitor* inflate_and_enter(oop object, ObjectSynchronizer::InflateCause cause, JavaThread* locking_thread, JavaThread* current); diff --git a/src/hotspot/share/runtime/lockStack.hpp b/src/hotspot/share/runtime/lockStack.hpp index de32eb41259..dbc958a71e2 100644 --- a/src/hotspot/share/runtime/lockStack.hpp +++ b/src/hotspot/share/runtime/lockStack.hpp @@ -118,6 +118,10 @@ class LockStack { // Tests whether the oop is on this lock-stack. inline bool contains(oop o) const; + inline int monitor_count() const; + inline void move_to_address(oop* start); + inline void move_from_address(oop* start, int count); + // GC support inline void oops_do(OopClosure* cl); diff --git a/src/hotspot/share/runtime/lockStack.inline.hpp b/src/hotspot/share/runtime/lockStack.inline.hpp index 515ca94c741..0516a85356d 100644 --- a/src/hotspot/share/runtime/lockStack.inline.hpp +++ b/src/hotspot/share/runtime/lockStack.inline.hpp @@ -215,8 +215,33 @@ inline bool LockStack::contains(oop o) const { return false; } +inline int LockStack::monitor_count() const { + int end = to_index(_top); + assert(end <= CAPACITY, "invariant"); + return end; +} + +inline void LockStack::move_to_address(oop* start) { + int end = to_index(_top); + for (int i = 0; i < end; i++) { + start[i] = _base[i]; + DEBUG_ONLY(_base[i] = nullptr;) + } + _top = lock_stack_base_offset; +} + +inline void LockStack::move_from_address(oop* start, int count) { + assert(to_index(_top) == 0, "lockstack should be empty"); + for (int i = 0; i < count; i++) { + _base[i] = start[i]; + _top += oopSize; + } +} + inline void LockStack::oops_do(OopClosure* cl) { - verify("pre-oops-do"); + // We don't perform pre oops_do verify here because this function + // is used by the GC to fix the oops. + int end = to_index(_top); for (int i = 0; i < end; i++) { cl->do_oop(&_base[i]); diff --git a/src/hotspot/share/runtime/mutexLocker.cpp b/src/hotspot/share/runtime/mutexLocker.cpp index 76d0674e8c6..a0a6e5626e4 100644 --- a/src/hotspot/share/runtime/mutexLocker.cpp +++ b/src/hotspot/share/runtime/mutexLocker.cpp @@ -135,7 +135,6 @@ Mutex* SharedDecoder_lock = nullptr; Mutex* DCmdFactory_lock = nullptr; Mutex* NMTQuery_lock = nullptr; Mutex* NMTCompilationCostHistory_lock = nullptr; -Mutex* NmtVirtualMemory_lock = nullptr; #if INCLUDE_CDS #if INCLUDE_JVMTI @@ -294,11 +293,10 @@ void mutex_init() { MUTEX_DEFN(CodeHeapStateAnalytics_lock , PaddedMutex , safepoint); MUTEX_DEFN(ThreadsSMRDelete_lock , PaddedMonitor, service-2); // Holds ConcurrentHashTableResize_lock MUTEX_DEFN(ThreadIdTableCreate_lock , PaddedMutex , safepoint); - MUTEX_DEFN(SharedDecoder_lock , PaddedMutex , service-5); // Must be lower than NmtVirtualMemory_lock due to MemTracker::print_containing_region + MUTEX_DEFN(SharedDecoder_lock , PaddedMutex , tty-1); MUTEX_DEFN(DCmdFactory_lock , PaddedMutex , nosafepoint); MUTEX_DEFN(NMTQuery_lock , PaddedMutex , safepoint); MUTEX_DEFN(NMTCompilationCostHistory_lock , PaddedMutex , nosafepoint); - MUTEX_DEFN(NmtVirtualMemory_lock , PaddedMutex , service-4); #if INCLUDE_CDS #if INCLUDE_JVMTI MUTEX_DEFN(CDSClassFileStream_lock , PaddedMutex , safepoint); diff --git a/src/hotspot/share/runtime/mutexLocker.hpp b/src/hotspot/share/runtime/mutexLocker.hpp index cc465bfc191..7cca1d94bf2 100644 --- a/src/hotspot/share/runtime/mutexLocker.hpp +++ b/src/hotspot/share/runtime/mutexLocker.hpp @@ -117,7 +117,6 @@ extern Mutex* SharedDecoder_lock; // serializes access to the dec extern Mutex* DCmdFactory_lock; // serialize access to DCmdFactory information extern Mutex* NMTQuery_lock; // serialize NMT Dcmd queries extern Mutex* NMTCompilationCostHistory_lock; // guards NMT compilation cost history -extern Mutex* NmtVirtualMemory_lock; // guards NMT virtual memory updates #if INCLUDE_CDS #if INCLUDE_JVMTI extern Mutex* CDSClassFileStream_lock; // FileMapInfo::open_stream_for_jvmti diff --git a/src/hotspot/share/runtime/objectMonitor.cpp b/src/hotspot/share/runtime/objectMonitor.cpp index 3ea987801db..7d1998ca849 100644 --- a/src/hotspot/share/runtime/objectMonitor.cpp +++ b/src/hotspot/share/runtime/objectMonitor.cpp @@ -39,6 +39,7 @@ #include "prims/jvmtiDeferredUpdates.hpp" #include "prims/jvmtiExport.hpp" #include "runtime/atomic.hpp" +#include "runtime/continuationWrapper.inline.hpp" #include "runtime/globals.hpp" #include "runtime/handles.inline.hpp" #include "runtime/interfaceSupport.inline.hpp" @@ -53,6 +54,7 @@ #include "runtime/safefetch.hpp" #include "runtime/safepointMechanism.inline.hpp" #include "runtime/sharedRuntime.hpp" +#include "runtime/threads.hpp" #include "services/threadService.hpp" #include "utilities/debug.hpp" #include "utilities/dtrace.hpp" @@ -114,27 +116,41 @@ DEBUG_ONLY(static volatile bool InitDone = false;) OopStorage* ObjectMonitor::_oop_storage = nullptr; +OopHandle ObjectMonitor::_vthread_cxq_head; +ParkEvent* ObjectMonitor::_vthread_unparker_ParkEvent = nullptr; + // ----------------------------------------------------------------------------- // Theory of operations -- Monitors lists, thread residency, etc: // // * A thread acquires ownership of a monitor by successfully -// CAS()ing the _owner field from null to non-null. +// CAS()ing the _owner field from NO_OWNER/DEFLATER_MARKER to +// its tid (return value from owner_from()). // // * Invariant: A thread appears on at most one monitor list -- // cxq, EntryList or WaitSet -- at any one time. // // * Contending threads "push" themselves onto the cxq with CAS // and then spin/park. +// If the thread is a virtual thread it will first attempt to +// unmount itself. The virtual thread will first try to freeze +// all frames in the heap. If the operation fails it will just +// follow the regular path for platform threads. If the operation +// succeeds, it will push itself onto the cxq with CAS and then +// return back to Java to continue the unmount logic. // // * After a contending thread eventually acquires the lock it must // dequeue itself from either the EntryList or the cxq. // // * The exiting thread identifies and unparks an "heir presumptive" -// tentative successor thread on the EntryList. Critically, the -// exiting thread doesn't unlink the successor thread from the EntryList. -// After having been unparked, the wakee will recontend for ownership of -// the monitor. The successor (wakee) will either acquire the lock or -// re-park itself. +// tentative successor thread on the EntryList. In case the successor +// is an unmounted virtual thread, the exiting thread will first try +// to add it to the list of vthreads waiting to be unblocked, and on +// success it will unpark the special unblocker thread instead, which +// will be in charge of submitting the vthread back to the scheduler +// queue. Critically, the exiting thread doesn't unlink the successor +// thread from the EntryList. After having been unparked/re-scheduled, +// the wakee will recontend for ownership of the monitor. The successor +// (wakee) will either acquire the lock or re-park/unmount itself. // // Succession is provided for by a policy of competitive handoff. // The exiting thread does _not_ grant or pass ownership to the @@ -196,20 +212,18 @@ OopStorage* ObjectMonitor::_oop_storage = nullptr; // // * The monitor synchronization subsystem avoids the use of native // synchronization primitives except for the narrow platform-specific -// park-unpark abstraction. See the comments in os_solaris.cpp regarding +// park-unpark abstraction. See the comments in os_posix.cpp regarding // the semantics of park-unpark. Put another way, this monitor implementation -// depends only on atomic operations and park-unpark. The monitor subsystem -// manages all RUNNING->BLOCKED and BLOCKED->READY transitions while the -// underlying OS manages the READY<->RUN transitions. +// depends only on atomic operations and park-unpark. // // * Waiting threads reside on the WaitSet list -- wait() puts // the caller onto the WaitSet. // // * notify() or notifyAll() simply transfers threads from the WaitSet to // either the EntryList or cxq. Subsequent exit() operations will -// unpark the notifyee. Unparking a notifee in notify() is inefficient - -// it's likely the notifyee would simply impale itself on the lock held -// by the notifier. +// unpark/re-schedule the notifyee. Unparking/re-scheduling a notifyee in +// notify() is inefficient - it's likely the notifyee would simply impale +// itself on the lock held by the notifier. // Check that object() and set_object() are called from the right context: static void check_object_context() { @@ -237,18 +251,19 @@ static void check_object_context() { ObjectMonitor::ObjectMonitor(oop object) : _metadata(0), _object(_oop_storage, object), - _owner(nullptr), + _owner(NO_OWNER), _previous_owner_tid(0), _next_om(nullptr), _recursions(0), _EntryList(nullptr), _cxq(nullptr), - _succ(nullptr), + _succ(NO_OWNER), _SpinDuration(ObjectMonitor::Knob_SpinLimit), _contentions(0), _WaitSet(nullptr), _waiters(0), - _WaitSetLock(0) + _WaitSetLock(0), + _stack_locker(nullptr) { } ObjectMonitor::~ObjectMonitor() { @@ -263,7 +278,7 @@ oop ObjectMonitor::object() const { void ObjectMonitor::ExitOnSuspend::operator()(JavaThread* current) { if (current->is_suspended()) { _om->_recursions = 0; - _om->_succ = nullptr; + _om->clear_successor(); // Don't need a full fence after clearing successor here because of the call to exit(). _om->exit(current, false /* not_suspended */); _om_exited = true; @@ -274,8 +289,8 @@ void ObjectMonitor::ExitOnSuspend::operator()(JavaThread* current) { void ObjectMonitor::ClearSuccOnSuspend::operator()(JavaThread* current) { if (current->is_suspended()) { - if (_om->_succ == current) { - _om->_succ = nullptr; + if (_om->has_successor(current)) { + _om->clear_successor(); OrderAccess::fence(); // always do a full fence when successor is cleared } } @@ -310,13 +325,13 @@ bool ObjectMonitor::TryLockWithContentionMark(JavaThread* locking_thread, Object assert(contention_mark._monitor == this, "must be"); assert(!is_being_async_deflated(), "must be"); - void* prev_owner = try_set_owner_from(nullptr, locking_thread); + int64_t prev_owner = try_set_owner_from(NO_OWNER, locking_thread); bool success = false; - if (prev_owner == nullptr) { + if (prev_owner == NO_OWNER) { assert(_recursions == 0, "invariant"); success = true; - } else if (prev_owner == locking_thread) { + } else if (prev_owner == owner_from(locking_thread)) { _recursions++; success = true; } else if (prev_owner == DEFLATER_MARKER) { @@ -334,21 +349,16 @@ bool ObjectMonitor::TryLockWithContentionMark(JavaThread* locking_thread, Object // recognizes that the async deflation was cancelled. contention_mark.extend(); success = true; - } else if (prev_owner == nullptr) { + } else if (prev_owner == NO_OWNER) { // At this point we cannot race with deflation as we have both incremented // contentions, seen contention > 0 and seen a DEFLATER_MARKER. // success will only be false if this races with something other than // deflation. - prev_owner = try_set_owner_from(nullptr, locking_thread); - success = prev_owner == nullptr; + prev_owner = try_set_owner_from(NO_OWNER, locking_thread); + success = prev_owner == NO_OWNER; } - } else if (LockingMode == LM_LEGACY && locking_thread->is_lock_owned((address)prev_owner)) { - assert(_recursions == 0, "must be"); - _recursions = 1; - set_owner_from_BasicLock(prev_owner, locking_thread); - success = true; } - assert(!success || owner_raw() == locking_thread, "must be"); + assert(!success || has_owner(locking_thread), "must be"); return success; } @@ -361,8 +371,8 @@ void ObjectMonitor::enter_for_with_contention_mark(JavaThread* locking_thread, O bool success = TryLockWithContentionMark(locking_thread, contention_mark); assert(success, "Failed to enter_for: locking_thread=" INTPTR_FORMAT - ", this=" INTPTR_FORMAT "{owner=" INTPTR_FORMAT "}", - p2i(locking_thread), p2i(this), p2i(owner_raw())); + ", this=" INTPTR_FORMAT "{owner=" INT64_FORMAT "}", + p2i(locking_thread), p2i(this), owner_raw()); } bool ObjectMonitor::enter_for(JavaThread* locking_thread) { @@ -382,9 +392,9 @@ bool ObjectMonitor::enter_for(JavaThread* locking_thread) { bool success = TryLockWithContentionMark(locking_thread, contention_mark); assert(success, "Failed to enter_for: locking_thread=" INTPTR_FORMAT - ", this=" INTPTR_FORMAT "{owner=" INTPTR_FORMAT "}", - p2i(locking_thread), p2i(this), p2i(owner_raw())); - assert(owner_raw() == locking_thread, "must be"); + ", this=" INTPTR_FORMAT "{owner=" INT64_FORMAT "}", + p2i(locking_thread), p2i(this), owner_raw()); + assert(has_owner(locking_thread), "must be"); return true; } @@ -402,18 +412,11 @@ bool ObjectMonitor::try_enter(JavaThread* current, bool check_for_recursion) { return false; } - if (r == TryLockResult::HasOwner && owner() == current) { + if (r == TryLockResult::HasOwner && has_owner(current)) { _recursions++; return true; } - void* cur = owner_raw(); - if (LockingMode == LM_LEGACY && current->is_lock_owned((address)cur)) { - assert(_recursions == 0, "internal state error"); - _recursions = 1; - set_owner_from_BasicLock(cur, current); // Convert from BasicLock* to Thread*. - return true; - } return false; } @@ -436,7 +439,7 @@ bool ObjectMonitor::spin_enter(JavaThread* current) { // Note that if we acquire the monitor from an initial spin // we forgo posting JVMTI events and firing DTRACE probes. if (TrySpin(current)) { - assert(owner_raw() == current, "must be current: owner=" INTPTR_FORMAT, p2i(owner_raw())); + assert(has_owner(current), "must be current: owner=" INT64_FORMAT, owner_raw()); assert(_recursions == 0, "must be 0: recursions=" INTX_FORMAT, _recursions); assert_mark_word_consistency(); return true; @@ -452,8 +455,8 @@ bool ObjectMonitor::enter(JavaThread* current) { return true; } - assert(owner_raw() != current, "invariant"); - assert(_succ != current, "invariant"); + assert(!has_owner(current), "invariant"); + assert(!has_successor(current), "invariant"); assert(!SafepointSynchronize::is_at_safepoint(), "invariant"); assert(current->thread_state() != _thread_blocked, "invariant"); @@ -472,19 +475,22 @@ bool ObjectMonitor::enter(JavaThread* current) { void ObjectMonitor::enter_with_contention_mark(JavaThread *current, ObjectMonitorContentionMark &cm) { assert(current == JavaThread::current(), "must be"); - assert(owner_raw() != current, "must be"); + assert(!has_owner(current), "must be"); assert(cm._monitor == this, "must be"); assert(!is_being_async_deflated(), "must be"); JFR_ONLY(JfrConditionalFlush flush(current);) - EventJavaMonitorEnter event; - if (event.is_started()) { - event.set_monitorClass(object()->klass()); + EventJavaMonitorEnter enter_event; + if (enter_event.is_started()) { + enter_event.set_monitorClass(object()->klass()); // Set an address that is 'unique enough', such that events close in // time and with the same address are likely (but not guaranteed) to // belong to the same object. - event.set_address((uintptr_t)this); + enter_event.set_address((uintptr_t)this); } + EventVirtualThreadPinned vthread_pinned_event; + + freeze_result result; { // Change java thread status to indicate blocked on monitor enter. JavaThreadBlockedOnMonitorEnterState jtbmes(current, this); @@ -503,6 +509,30 @@ void ObjectMonitor::enter_with_contention_mark(JavaThread *current, ObjectMonito // ParkEvent associated with this ObjectMonitor. } + ContinuationEntry* ce = current->last_continuation(); + if (ce != nullptr && ce->is_virtual_thread()) { + result = Continuation::try_preempt(current, ce->cont_oop(current)); + if (result == freeze_ok) { + bool acquired = VThreadMonitorEnter(current); + if (acquired) { + // We actually acquired the monitor while trying to add the vthread to the + // _cxq so cancel preemption. We will still go through the preempt stub + // but instead of unmounting we will call thaw to continue execution. + current->set_preemption_cancelled(true); + if (JvmtiExport::should_post_monitor_contended_entered()) { + // We are going to call thaw again after this and finish the VMTS + // transition so no need to do it here. We will post the event there. + current->set_contended_entered_monitor(this); + } + } + current->set_current_pending_monitor(nullptr); + DEBUG_ONLY(int state = java_lang_VirtualThread::state(current->vthread())); + assert((acquired && current->preemption_cancelled() && state == java_lang_VirtualThread::RUNNING) || + (!acquired && !current->preemption_cancelled() && state == java_lang_VirtualThread::BLOCKING), "invariant"); + return; + } + } + OSThreadContendState osts(current->osthread()); assert(current->thread_state() == _thread_in_vm, "invariant"); @@ -523,7 +553,7 @@ void ObjectMonitor::enter_with_contention_mark(JavaThread *current, ObjectMonito } if (!eos.exited()) { // ExitOnSuspend did not exit the OM - assert(owner_raw() == current, "invariant"); + assert(has_owner(current), "invariant"); break; } } @@ -536,8 +566,8 @@ void ObjectMonitor::enter_with_contention_mark(JavaThread *current, ObjectMonito // Must either set _recursions = 0 or ASSERT _recursions == 0. assert(_recursions == 0, "invariant"); - assert(owner_raw() == current, "invariant"); - assert(_succ != current, "invariant"); + assert(has_owner(current), "invariant"); + assert(!has_successor(current), "invariant"); assert_mark_word_consistency(); // The thread -- now the owner -- is back in vm mode. @@ -562,10 +592,17 @@ void ObjectMonitor::enter_with_contention_mark(JavaThread *current, ObjectMonito // event handler consumed an unpark() issued by the thread that // just exited the monitor. } - if (event.should_commit()) { - event.set_previousOwner(_previous_owner_tid); - event.commit(); + if (enter_event.should_commit()) { + enter_event.set_previousOwner(_previous_owner_tid); + enter_event.commit(); } + + ContinuationEntry* ce = current->last_continuation(); + if (ce != nullptr && ce->is_virtual_thread()) { + assert(result != freeze_ok, "sanity check"); + current->post_vthread_pinned_event(&vthread_pinned_event, "Contended monitor enter", result); + } + OM_PERFDATA_OP(ContendedLockAttempts, inc()); } @@ -573,8 +610,8 @@ void ObjectMonitor::enter_with_contention_mark(JavaThread *current, ObjectMonito // Callers must compensate as needed. ObjectMonitor::TryLockResult ObjectMonitor::TryLock(JavaThread* current) { - void* own = owner_raw(); - void* first_own = own; + int64_t own = owner_raw(); + int64_t first_own = own; for (;;) { if (own == DEFLATER_MARKER) { @@ -593,9 +630,9 @@ ObjectMonitor::TryLockResult ObjectMonitor::TryLock(JavaThread* current) { // Deflation won or change of owner; dont spin break; } - } else if (own == nullptr) { - void* prev_own = try_set_owner_from(nullptr, current); - if (prev_own == nullptr) { + } else if (own == NO_OWNER) { + int64_t prev_own = try_set_owner_from(NO_OWNER, current); + if (prev_own == NO_OWNER) { assert(_recursions == 0, "invariant"); return TryLockResult::Success; } else { @@ -634,7 +671,7 @@ bool ObjectMonitor::deflate_monitor(Thread* current) { if (obj == nullptr) { // If the object died, we can recycle the monitor without racing with // Java threads. The GC already broke the association with the object. - set_owner_from(nullptr, DEFLATER_MARKER); + set_owner_from_raw(NO_OWNER, DEFLATER_MARKER); assert(contentions() >= 0, "must be non-negative: contentions=%d", contentions()); _contentions = INT_MIN; // minimum negative int } else { @@ -643,7 +680,7 @@ bool ObjectMonitor::deflate_monitor(Thread* current) { // Set a null owner to DEFLATER_MARKER to force any contending thread // through the slow path. This is just the first part of the async // deflation dance. - if (try_set_owner_from(nullptr, DEFLATER_MARKER) != nullptr) { + if (try_set_owner_from_raw(NO_OWNER, DEFLATER_MARKER) != NO_OWNER) { // The owner field is no longer null so we lost the race since the // ObjectMonitor is now busy. return false; @@ -654,7 +691,7 @@ bool ObjectMonitor::deflate_monitor(Thread* current) { // is_busy() above or has already entered and waited on // it which makes it busy so no deflation. Restore owner to // null if it is still DEFLATER_MARKER. - if (try_set_owner_from(DEFLATER_MARKER, nullptr) != DEFLATER_MARKER) { + if (try_set_owner_from_raw(DEFLATER_MARKER, NO_OWNER) != DEFLATER_MARKER) { // Deferred decrement for the JT EnterI() that cancelled the async deflation. add_to_contentions(-1); } @@ -667,7 +704,7 @@ bool ObjectMonitor::deflate_monitor(Thread* current) { // Contentions was no longer 0 so we lost the race since the // ObjectMonitor is now busy. Restore owner to null if it is // still DEFLATER_MARKER: - if (try_set_owner_from(DEFLATER_MARKER, nullptr) != DEFLATER_MARKER) { + if (try_set_owner_from_raw(DEFLATER_MARKER, NO_OWNER) != DEFLATER_MARKER) { // Deferred decrement for the JT EnterI() that cancelled the async deflation. add_to_contentions(-1); } @@ -768,7 +805,7 @@ void ObjectMonitor::install_displaced_markword_in_object(const oop obj) { const char* ObjectMonitor::is_busy_to_string(stringStream* ss) { ss->print("is_busy: waiters=%d" ", contentions=%d" - ", owner=" PTR_FORMAT + ", owner=" INT64_FORMAT ", cxq=" PTR_FORMAT ", EntryList=" PTR_FORMAT, _waiters, @@ -776,8 +813,8 @@ const char* ObjectMonitor::is_busy_to_string(stringStream* ss) { owner_is_DEFLATER_MARKER() // We report null instead of DEFLATER_MARKER here because is_busy() // ignores DEFLATER_MARKER values. - ? p2i(nullptr) - : p2i(owner_raw()), + ? NO_OWNER + : owner_raw(), p2i(_cxq), p2i(_EntryList)); return ss->base(); @@ -788,8 +825,8 @@ void ObjectMonitor::EnterI(JavaThread* current) { // Try the lock - TATAS if (TryLock(current) == TryLockResult::Success) { - assert(_succ != current, "invariant"); - assert(owner_raw() == current, "invariant"); + assert(!has_successor(current), "invariant"); + assert(has_owner(current), "invariant"); return; } @@ -803,14 +840,14 @@ void ObjectMonitor::EnterI(JavaThread* current) { // effects. if (TrySpin(current)) { - assert(owner_raw() == current, "invariant"); - assert(_succ != current, "invariant"); + assert(has_owner(current), "invariant"); + assert(!has_successor(current), "invariant"); return; } // The Spin failed -- Enqueue and park the thread ... - assert(_succ != current, "invariant"); - assert(owner_raw() != current, "invariant"); + assert(!has_successor(current), "invariant"); + assert(!has_owner(current), "invariant"); // Enqueue "current" on ObjectMonitor's _cxq. // @@ -838,8 +875,8 @@ void ObjectMonitor::EnterI(JavaThread* current) { // Interference - the CAS failed because _cxq changed. Just retry. // As an optional optimization we retry the lock. if (TryLock(current) == TryLockResult::Success) { - assert(_succ != current, "invariant"); - assert(owner_raw() == current, "invariant"); + assert(!has_successor(current), "invariant"); + assert(has_owner(current), "invariant"); return; } } @@ -855,15 +892,38 @@ void ObjectMonitor::EnterI(JavaThread* current) { // to defer the state transitions until absolutely necessary, // and in doing so avoid some transitions ... + // For virtual threads that are pinned, do a timed-park instead to + // alleviate some deadlocks cases where the succesor is an unmounted + // virtual thread that cannot run. This can happen in particular when + // this virtual thread is currently loading/initializing a class, and + // all other carriers have a vthread pinned to it waiting for said class + // to be loaded/initialized. + static int MAX_RECHECK_INTERVAL = 1000; + int recheck_interval = 1; + bool do_timed_parked = false; + ContinuationEntry* ce = current->last_continuation(); + if (ce != nullptr && ce->is_virtual_thread()) { + do_timed_parked = true; + } + for (;;) { if (TryLock(current) == TryLockResult::Success) { break; } - assert(owner_raw() != current, "invariant"); + assert(!has_owner(current), "invariant"); // park self - current->_ParkEvent->park(); + if (do_timed_parked) { + current->_ParkEvent->park((jlong) recheck_interval); + // Increase the recheck_interval, but clamp the value. + recheck_interval *= 8; + if (recheck_interval > MAX_RECHECK_INTERVAL) { + recheck_interval = MAX_RECHECK_INTERVAL; + } + } else { + current->_ParkEvent->park(); + } if (TryLock(current) == TryLockResult::Success) { break; @@ -893,7 +953,7 @@ void ObjectMonitor::EnterI(JavaThread* current) { // just spin again. This pattern can repeat, leaving _succ to simply // spin on a CPU. - if (_succ == current) _succ = nullptr; + if (has_successor(current)) clear_successor(); // Invariant: after clearing _succ a thread *must* retry _owner before parking. OrderAccess::fence(); @@ -907,11 +967,11 @@ void ObjectMonitor::EnterI(JavaThread* current) { // The head of cxq is volatile but the interior is stable. // In addition, current.TState is stable. - assert(owner_raw() == current, "invariant"); + assert(has_owner(current), "invariant"); UnlinkAfterAcquire(current, &node); - if (_succ == current) { - _succ = nullptr; + if (has_successor(current)) { + clear_successor(); // Note that we don't need to do OrderAccess::fence() after clearing // _succ here, since we own the lock. } @@ -958,7 +1018,7 @@ void ObjectMonitor::ReenterI(JavaThread* current, ObjectWaiter* currentNode) { for (;;) { ObjectWaiter::TStates v = currentNode->TState; guarantee(v == ObjectWaiter::TS_ENTER || v == ObjectWaiter::TS_CXQ, "invariant"); - assert(owner_raw() != current, "invariant"); + assert(!has_owner(current), "invariant"); // This thread has been notified so try to reacquire the lock. if (TryLock(current) == TryLockResult::Success) { @@ -994,7 +1054,7 @@ void ObjectMonitor::ReenterI(JavaThread* current, ObjectWaiter* currentNode) { // Assuming this is not a spurious wakeup we'll normally // find that _succ == current. - if (_succ == current) _succ = nullptr; + if (has_successor(current)) clear_successor(); // Invariant: after clearing _succ a contending thread // *must* retry _owner before parking. @@ -1017,22 +1077,152 @@ void ObjectMonitor::ReenterI(JavaThread* current, ObjectWaiter* currentNode) { // The head of cxq is volatile but the interior is stable. // In addition, current.TState is stable. - assert(owner_raw() == current, "invariant"); + assert(has_owner(current), "invariant"); assert_mark_word_consistency(); UnlinkAfterAcquire(current, currentNode); - if (_succ == current) _succ = nullptr; - assert(_succ != current, "invariant"); + if (has_successor(current)) clear_successor(); + assert(!has_successor(current), "invariant"); currentNode->TState = ObjectWaiter::TS_RUN; OrderAccess::fence(); // see comments at the end of EnterI() } +// This method is called from two places: +// - On monitorenter contention with a null waiter. +// - After Object.wait() times out or the target is interrupted to reenter the +// monitor, with the existing waiter. +// For the Object.wait() case we do not delete the ObjectWaiter in case we +// succesfully acquire the monitor since we are going to need it on return. +bool ObjectMonitor::VThreadMonitorEnter(JavaThread* current, ObjectWaiter* waiter) { + if (TryLock(current) == TryLockResult::Success) { + assert(has_owner(current), "invariant"); + assert(!has_successor(current), "invariant"); + return true; + } + + oop vthread = current->vthread(); + ObjectWaiter* node = waiter != nullptr ? waiter : new ObjectWaiter(vthread, this); + node->_prev = (ObjectWaiter*) 0xBAD; + node->TState = ObjectWaiter::TS_CXQ; + + // Push node associated with vthread onto the front of the _cxq. + ObjectWaiter* nxt; + for (;;) { + node->_next = nxt = _cxq; + if (Atomic::cmpxchg(&_cxq, nxt, node) == nxt) break; + + // Interference - the CAS failed because _cxq changed. Just retry. + // As an optional optimization we retry the lock. + if (TryLock(current) == TryLockResult::Success) { + assert(has_owner(current), "invariant"); + assert(!has_successor(current), "invariant"); + if (waiter == nullptr) delete node; // for Object.wait() don't delete yet + return true; + } + } + + // We have to try once more since owner could have exited monitor and checked + // _cxq before we added the node to the queue. + if (TryLock(current) == TryLockResult::Success) { + assert(has_owner(current), "invariant"); + UnlinkAfterAcquire(current, node); + if (has_successor(current)) clear_successor(); + if (waiter == nullptr) delete node; // for Object.wait() don't delete yet + return true; + } + + assert(java_lang_VirtualThread::state(vthread) == java_lang_VirtualThread::RUNNING, "wrong state for vthread"); + java_lang_VirtualThread::set_state(vthread, java_lang_VirtualThread::BLOCKING); + + // We didn't succeed in acquiring the monitor so increment _contentions and + // save ObjectWaiter* in the chunk since we will need it when resuming execution. + add_to_contentions(1); + oop cont = java_lang_VirtualThread::continuation(vthread); + stackChunkOop chunk = jdk_internal_vm_Continuation::tail(cont); + chunk->set_object_waiter(node); + return false; +} + +// Called from thaw code to resume the monitor operation that caused the vthread +// to be unmounted. Method returns true if the monitor is successfully acquired, +// which marks the end of the monitor operation, otherwise it returns false. +bool ObjectMonitor::resume_operation(JavaThread* current, ObjectWaiter* node, ContinuationWrapper& cont) { + assert(java_lang_VirtualThread::state(current->vthread()) == java_lang_VirtualThread::RUNNING, "wrong state for vthread"); + assert(!has_owner(current), ""); + + if (node->is_wait() && !node->at_reenter()) { + bool acquired_monitor = VThreadWaitReenter(current, node, cont); + if (acquired_monitor) return true; + } + + // Retry acquiring monitor... + + int state = node->TState; + guarantee(state == ObjectWaiter::TS_ENTER || state == ObjectWaiter::TS_CXQ, "invariant"); + + if (TryLock(current) == TryLockResult::Success) { + VThreadEpilog(current, node); + return true; + } + + oop vthread = current->vthread(); + if (has_successor(current)) clear_successor(); + + // Invariant: after clearing _succ a thread *must* retry acquiring the monitor. + OrderAccess::fence(); + + if (TryLock(current) == TryLockResult::Success) { + VThreadEpilog(current, node); + return true; + } + + // We will return to Continuation.run() and unmount so set the right state. + java_lang_VirtualThread::set_state(vthread, java_lang_VirtualThread::BLOCKING); + + return false; +} + +void ObjectMonitor::VThreadEpilog(JavaThread* current, ObjectWaiter* node) { + assert(has_owner(current), "invariant"); + add_to_contentions(-1); + + if (has_successor(current)) clear_successor(); + + guarantee(_recursions == 0, "invariant"); + + if (node->is_wait()) { + _recursions = node->_recursions; // restore the old recursion count + _waiters--; // decrement the number of waiters + + if (node->_interrupted) { + // We will throw at thaw end after finishing the mount transition. + current->set_pending_interrupted_exception(true); + } + } + + UnlinkAfterAcquire(current, node); + delete node; + + // Remove the ObjectWaiter* from the stackChunk. + oop vthread = current->vthread(); + oop cont = java_lang_VirtualThread::continuation(vthread); + stackChunkOop chunk = jdk_internal_vm_Continuation::tail(cont); + chunk->set_object_waiter(nullptr); + + if (JvmtiExport::should_post_monitor_contended_entered()) { + // We are going to call thaw again after this and finish the VMTS + // transition so no need to do it here. We will post the event there. + current->set_contended_entered_monitor(this); + } +} + // By convention we unlink a contending thread from EntryList|cxq immediately // after the thread acquires the lock in ::enter(). Equally, we could defer // unlinking the thread until ::exit()-time. void ObjectMonitor::UnlinkAfterAcquire(JavaThread* current, ObjectWaiter* currentNode) { - assert(owner_raw() == current, "invariant"); - assert(currentNode->_thread == current, "invariant"); + assert(has_owner(current), "invariant"); + assert((!currentNode->is_vthread() && currentNode->thread() == current) || + (currentNode->is_vthread() && currentNode->vthread() == current->vthread()), "invariant"); if (currentNode->TState == ObjectWaiter::TS_ENTER) { // Normal case: remove current from the DLL EntryList . @@ -1151,32 +1341,25 @@ void ObjectMonitor::UnlinkAfterAcquire(JavaThread* current, ObjectWaiter* curren // of such futile wakups is low. void ObjectMonitor::exit(JavaThread* current, bool not_suspended) { - void* cur = owner_raw(); - if (current != cur) { - if (LockingMode != LM_LIGHTWEIGHT && current->is_lock_owned((address)cur)) { - assert(_recursions == 0, "invariant"); - set_owner_from_BasicLock(cur, current); // Convert from BasicLock* to Thread*. - _recursions = 0; - } else { - // Apparent unbalanced locking ... - // Naively we'd like to throw IllegalMonitorStateException. - // As a practical matter we can neither allocate nor throw an - // exception as ::exit() can be called from leaf routines. - // see x86_32.ad Fast_Unlock() and the I1 and I2 properties. - // Upon deeper reflection, however, in a properly run JVM the only - // way we should encounter this situation is in the presence of - // unbalanced JNI locking. TODO: CheckJNICalls. - // See also: CR4414101 + if (!has_owner(current)) { + // Apparent unbalanced locking ... + // Naively we'd like to throw IllegalMonitorStateException. + // As a practical matter we can neither allocate nor throw an + // exception as ::exit() can be called from leaf routines. + // see x86_32.ad Fast_Unlock() and the I1 and I2 properties. + // Upon deeper reflection, however, in a properly run JVM the only + // way we should encounter this situation is in the presence of + // unbalanced JNI locking. TODO: CheckJNICalls. + // See also: CR4414101 #ifdef ASSERT - LogStreamHandle(Error, monitorinflation) lsh; - lsh.print_cr("ERROR: ObjectMonitor::exit(): thread=" INTPTR_FORMAT - " is exiting an ObjectMonitor it does not own.", p2i(current)); - lsh.print_cr("The imbalance is possibly caused by JNI locking."); - print_debug_style_on(&lsh); - assert(false, "Non-balanced monitor enter/exit!"); + LogStreamHandle(Error, monitorinflation) lsh; + lsh.print_cr("ERROR: ObjectMonitor::exit(): thread=" INTPTR_FORMAT + " is exiting an ObjectMonitor it does not own.", p2i(current)); + lsh.print_cr("The imbalance is possibly caused by JNI locking."); + print_debug_style_on(&lsh); + assert(false, "Non-balanced monitor enter/exit!"); #endif - return; - } + return; } if (_recursions != 0) { @@ -1193,18 +1376,18 @@ void ObjectMonitor::exit(JavaThread* current, bool not_suspended) { #endif for (;;) { - assert(current == owner_raw(), "invariant"); + assert(has_owner(current), "invariant"); // Drop the lock. // release semantics: prior loads and stores from within the critical section // must not float (reorder) past the following store that drops the lock. // Uses a storeload to separate release_store(owner) from the - // successor check. The try_set_owner() below uses cmpxchg() so + // successor check. The try_set_owner_from() below uses cmpxchg() so // we get the fence down there. release_clear_owner(current); OrderAccess::storeload(); - if ((intptr_t(_EntryList)|intptr_t(_cxq)) == 0 || _succ != nullptr) { + if ((intptr_t(_EntryList)|intptr_t(_cxq)) == 0 || has_successor()) { return; } // Other threads are blocked trying to acquire the lock. @@ -1246,7 +1429,7 @@ void ObjectMonitor::exit(JavaThread* current, bool not_suspended) { return; } - guarantee(owner_raw() == current, "invariant"); + guarantee(has_owner(current), "invariant"); ObjectWaiter* w = nullptr; @@ -1311,7 +1494,7 @@ void ObjectMonitor::exit(JavaThread* current, bool not_suspended) { // See if we can abdicate to a spinner instead of waking a thread. // A primary goal of the implementation is to reduce the // context-switch rate. - if (_succ != nullptr) continue; + if (has_successor()) continue; w = _EntryList; if (w != nullptr) { @@ -1323,7 +1506,7 @@ void ObjectMonitor::exit(JavaThread* current, bool not_suspended) { } void ObjectMonitor::ExitEpilog(JavaThread* current, ObjectWaiter* Wakee) { - assert(owner_raw() == current, "invariant"); + assert(has_owner(current), "invariant"); // Exit protocol: // 1. ST _succ = wakee @@ -1331,8 +1514,19 @@ void ObjectMonitor::ExitEpilog(JavaThread* current, ObjectWaiter* Wakee) { // 2. ST _owner = nullptr // 3. unpark(wakee) - _succ = Wakee->_thread; - ParkEvent * Trigger = Wakee->_event; + oop vthread = nullptr; + ParkEvent * Trigger; + if (!Wakee->is_vthread()) { + JavaThread* t = Wakee->thread(); + assert(t != nullptr, ""); + Trigger = t->_ParkEvent; + set_successor(t); + } else { + vthread = Wakee->vthread(); + assert(vthread != nullptr, ""); + Trigger = ObjectMonitor::vthread_unparker_ParkEvent(); + set_successor(vthread); + } // Hygiene -- once we've set _owner = nullptr we can't safely dereference Wakee again. // The thread associated with Wakee may have grabbed the lock and "Wakee" may be @@ -1345,34 +1539,29 @@ void ObjectMonitor::ExitEpilog(JavaThread* current, ObjectWaiter* Wakee) { OrderAccess::fence(); DTRACE_MONITOR_PROBE(contended__exit, this, object(), current); - Trigger->unpark(); + + if (vthread == nullptr) { + // Platform thread case. + Trigger->unpark(); + } else if (java_lang_VirtualThread::set_onWaitingList(vthread, vthread_cxq_head())) { + // Virtual thread case. + Trigger->unpark(); + } // Maintain stats and report events to JVMTI OM_PERFDATA_OP(Parks, inc()); } -// complete_exit exits a lock returning recursion count -// complete_exit requires an inflated monitor -// The _owner field is not always the Thread addr even with an -// inflated monitor, e.g. the monitor can be inflated by a non-owning -// thread due to contention. +// Exits the monitor returning recursion count. _owner should +// be set to current's tid, i.e. no ANONYMOUS_OWNER allowed. intx ObjectMonitor::complete_exit(JavaThread* current) { assert(InitDone, "Unexpectedly not initialized"); + guarantee(has_owner(current), "complete_exit not owner"); - void* cur = owner_raw(); - if (current != cur) { - if (LockingMode != LM_LIGHTWEIGHT && current->is_lock_owned((address)cur)) { - assert(_recursions == 0, "internal state error"); - set_owner_from_BasicLock(cur, current); // Convert from BasicLock* to Thread*. - _recursions = 0; - } - } - - guarantee(current == owner_raw(), "complete_exit not owner"); intx save = _recursions; // record the old recursion count _recursions = 0; // set the recursion level to be 0 exit(current); // exit the monitor - guarantee(owner_raw() != current, "invariant"); + guarantee(!has_owner(current), "invariant"); return save; } @@ -1395,14 +1584,8 @@ intx ObjectMonitor::complete_exit(JavaThread* current) { // is not the owner, that exception will be replaced by the IMSE. bool ObjectMonitor::check_owner(TRAPS) { JavaThread* current = THREAD; - void* cur = owner_raw(); - assert(cur != anon_owner_ptr(), "no anon owner here"); - if (cur == current) { - return true; - } - if (LockingMode != LM_LIGHTWEIGHT && current->is_lock_owned((address)cur)) { - set_owner_from_BasicLock(cur, current); // Convert from BasicLock* to Thread*. - _recursions = 0; + int64_t cur = owner_raw(); + if (cur == owner_from(current)) { return true; } THROW_MSG_(vmSymbols::java_lang_IllegalMonitorStateException(), @@ -1437,6 +1620,32 @@ static void post_monitor_wait_event(EventJavaMonitorWait* event, event->commit(); } +static void vthread_monitor_waited_event(JavaThread *current, ObjectWaiter* node, ContinuationWrapper& cont, EventJavaMonitorWait* event, jboolean timed_out) { + // Since we might safepoint set the anchor so that the stack can we walked. + assert(current->last_continuation() != nullptr, ""); + JavaFrameAnchor* anchor = current->frame_anchor(); + anchor->set_last_Java_sp(current->last_continuation()->entry_sp()); + anchor->set_last_Java_pc(current->last_continuation()->entry_pc()); + + ContinuationWrapper::SafepointOp so(current, cont); + + JRT_BLOCK + if (event->should_commit()) { + long timeout = java_lang_VirtualThread::timeout(current->vthread()); + post_monitor_wait_event(event, node->_monitor, node->_notifier_tid, timeout, timed_out); + } + if (JvmtiExport::should_post_monitor_waited()) { + // We mark this call in case of an upcall to Java while posting the event. + // If somebody walks the stack in that case, processing the enterSpecial + // frame should not include processing callee arguments since there is no + // actual callee (see nmethod::preserve_callee_argument_oops()). + ThreadOnMonitorWaitedEvent tmwe(current); + JvmtiExport::vthread_post_monitor_waited(current, node->_monitor, timed_out); + } + JRT_BLOCK_END + current->frame_anchor()->clear(); +} + // ----------------------------------------------------------------------------- // Wait/Notify/NotifyAll // @@ -1449,7 +1658,8 @@ void ObjectMonitor::wait(jlong millis, bool interruptible, TRAPS) { CHECK_OWNER(); // Throws IMSE if not owner. - EventJavaMonitorWait event; + EventJavaMonitorWait wait_event; + EventVirtualThreadPinned vthread_pinned_event; // check for a pending interrupt if (interruptible && current->is_interrupted(true) && !HAS_PENDING_EXCEPTION) { @@ -1467,8 +1677,8 @@ void ObjectMonitor::wait(jlong millis, bool interruptible, TRAPS) { // consume an unpark() meant for the ParkEvent associated with // this ObjectMonitor. } - if (event.should_commit()) { - post_monitor_wait_event(&event, this, 0, millis, false); + if (wait_event.should_commit()) { + post_monitor_wait_event(&wait_event, this, 0, millis, false); } THROW(vmSymbols::java_lang_InterruptedException()); return; @@ -1476,6 +1686,17 @@ void ObjectMonitor::wait(jlong millis, bool interruptible, TRAPS) { current->set_current_waiting_monitor(this); + freeze_result result; + ContinuationEntry* ce = current->last_continuation(); + if (ce != nullptr && ce->is_virtual_thread()) { + result = Continuation::try_preempt(current, ce->cont_oop(current)); + if (result == freeze_ok) { + VThreadWait(current, millis); + current->set_current_waiting_monitor(nullptr); + return; + } + } + // create a node to be put into the queue // Critically, after we reset() the event but prior to park(), we must check // for a pending interrupt. @@ -1499,7 +1720,7 @@ void ObjectMonitor::wait(jlong millis, bool interruptible, TRAPS) { _waiters++; // increment the number of waiters _recursions = 0; // set the recursion level to be 1 exit(current); // exit the monitor - guarantee(owner_raw() != current, "invariant"); + guarantee(!has_owner(current), "invariant"); // The thread is on the WaitSet list - now park() it. // On MP systems it's conceivable that a brief spin before we park @@ -1525,7 +1746,7 @@ void ObjectMonitor::wait(jlong millis, bool interruptible, TRAPS) { ThreadBlockInVMPreprocess tbivs(current, csos, true /* allow_suspend */); if (interrupted || HAS_PENDING_EXCEPTION) { // Intentionally empty - } else if (node._notified == 0) { + } else if (!node._notified) { if (millis <= 0) { current->_ParkEvent->park(); } else { @@ -1553,7 +1774,7 @@ void ObjectMonitor::wait(jlong millis, bool interruptible, TRAPS) { Thread::SpinAcquire(&_WaitSetLock, "WaitSet - unlink"); if (node.TState == ObjectWaiter::TS_WAIT) { DequeueSpecificWaiter(&node); // unlink from WaitSet - assert(node._notified == 0, "invariant"); + assert(!node._notified, "invariant"); node.TState = ObjectWaiter::TS_RUN; } Thread::SpinRelease(&_WaitSetLock); @@ -1565,7 +1786,7 @@ void ObjectMonitor::wait(jlong millis, bool interruptible, TRAPS) { // No other threads will asynchronously modify TState. guarantee(node.TState != ObjectWaiter::TS_WAIT, "invariant"); OrderAccess::loadload(); - if (_succ == current) _succ = nullptr; + if (has_successor(current)) clear_successor(); WasNotified = node._notified; // Reentry phase -- reacquire the monitor. @@ -1579,7 +1800,7 @@ void ObjectMonitor::wait(jlong millis, bool interruptible, TRAPS) { if (JvmtiExport::should_post_monitor_waited()) { JvmtiExport::post_monitor_waited(current, this, ret == OS_TIMEOUT); - if (node._notified != 0 && _succ == current) { + if (node._notified && has_successor(current)) { // In this part of the monitor wait-notify-reenter protocol it // is possible (and normal) for another thread to do a fastpath // monitor enter-exit while this thread is still trying to get @@ -1595,19 +1816,28 @@ void ObjectMonitor::wait(jlong millis, bool interruptible, TRAPS) { // We redo the unpark() to ensure forward progress, i.e., we // don't want all pending threads hanging (parked) with none // entering the unlocked monitor. - node._event->unpark(); + current->_ParkEvent->unpark(); } } - if (event.should_commit()) { - post_monitor_wait_event(&event, this, node._notifier_tid, millis, ret == OS_TIMEOUT); + if (wait_event.should_commit()) { + post_monitor_wait_event(&wait_event, this, node._notifier_tid, millis, ret == OS_TIMEOUT); + } + + if (ce != nullptr && ce->is_virtual_thread()) { + assert(result != freeze_ok, "sanity check"); + current->post_vthread_pinned_event(&vthread_pinned_event, "Object.wait", result); } OrderAccess::fence(); - assert(owner_raw() != current, "invariant"); + assert(!has_owner(current), "invariant"); ObjectWaiter::TStates v = node.TState; if (v == ObjectWaiter::TS_RUN) { + // We use the NoPreemptMark for the very rare case where the previous + // preempt attempt failed due to OOM. The preempt on monitor contention + // could succeed but we can't unmount now. + NoPreemptMark npm(current); enter(current); } else { guarantee(v == ObjectWaiter::TS_ENTER || v == ObjectWaiter::TS_CXQ, "invariant"); @@ -1620,8 +1850,8 @@ void ObjectMonitor::wait(jlong millis, bool interruptible, TRAPS) { // Node is about to go out-of-scope, but even if it were immortal we wouldn't // want residual elements associated with this thread left on any lists. guarantee(node.TState == ObjectWaiter::TS_RUN, "invariant"); - assert(owner_raw() == current, "invariant"); - assert(_succ != current, "invariant"); + assert(has_owner(current), "invariant"); + assert(!has_successor(current), "invariant"); } // OSThreadWaitState() current->set_current_waiting_monitor(nullptr); @@ -1634,8 +1864,8 @@ void ObjectMonitor::wait(jlong millis, bool interruptible, TRAPS) { _waiters--; // decrement the number of waiters // Verify a few postconditions - assert(owner_raw() == current, "invariant"); - assert(_succ != current, "invariant"); + assert(has_owner(current), "invariant"); + assert(!has_successor(current), "invariant"); assert_mark_word_consistency(); // check if the notification happened @@ -1651,7 +1881,6 @@ void ObjectMonitor::wait(jlong millis, bool interruptible, TRAPS) { // Monitor notify has precedence over thread interrupt. } - // Consider: // If the lock is cool (cxq == null && succ == null) and we're on an MP system // then instead of transferring a thread from the WaitSet to the EntryList @@ -1662,16 +1891,32 @@ void ObjectMonitor::INotify(JavaThread* current) { ObjectWaiter* iterator = DequeueWaiter(); if (iterator != nullptr) { guarantee(iterator->TState == ObjectWaiter::TS_WAIT, "invariant"); - guarantee(iterator->_notified == 0, "invariant"); + guarantee(!iterator->_notified, "invariant"); // Disposition - what might we do with iterator ? // a. add it directly to the EntryList - either tail (policy == 1) // or head (policy == 0). // b. push it onto the front of the _cxq (policy == 2). // For now we use (b). + if (iterator->is_vthread()) { + oop vthread = iterator->vthread(); + java_lang_VirtualThread::set_notified(vthread, true); + int old_state = java_lang_VirtualThread::state(vthread); + // If state is not WAIT/TIMED_WAIT then target could still be on + // unmount transition, or wait could have already timed-out or target + // could have been interrupted. In the first case, the target itself + // will set the state to BLOCKED at the end of the unmount transition. + // In the other cases the target would have been already unblocked so + // there is nothing to do. + if (old_state == java_lang_VirtualThread::WAIT || + old_state == java_lang_VirtualThread::TIMED_WAIT) { + java_lang_VirtualThread::cmpxchg_state(vthread, old_state, java_lang_VirtualThread::BLOCKED); + } + } + iterator->TState = ObjectWaiter::TS_ENTER; - iterator->_notified = 1; + iterator->_notified = true; iterator->_notifier_tid = JFR_THREAD_ID(current); ObjectWaiter* list = _EntryList; @@ -1704,7 +1949,9 @@ void ObjectMonitor::INotify(JavaThread* current) { // on _WaitSetLock so it's not profitable to reduce the length of the // critical section. - iterator->wait_reenter_begin(this); + if (!iterator->is_vthread()) { + iterator->wait_reenter_begin(this); + } } Thread::SpinRelease(&_WaitSetLock); } @@ -1755,6 +2002,94 @@ void ObjectMonitor::notifyAll(TRAPS) { OM_PERFDATA_OP(Notifications, inc(tally)); } +void ObjectMonitor::VThreadWait(JavaThread* current, jlong millis) { + oop vthread = current->vthread(); + ObjectWaiter* node = new ObjectWaiter(vthread, this); + node->_is_wait = true; + node->TState = ObjectWaiter::TS_WAIT; + java_lang_VirtualThread::set_notified(vthread, false); // Reset notified flag + + // Enter the waiting queue, which is a circular doubly linked list in this case + // but it could be a priority queue or any data structure. + // _WaitSetLock protects the wait queue. Normally the wait queue is accessed only + // by the owner of the monitor *except* in the case where park() + // returns because of a timeout or interrupt. Contention is exceptionally rare + // so we use a simple spin-lock instead of a heavier-weight blocking lock. + + Thread::SpinAcquire(&_WaitSetLock, "WaitSet - add"); + AddWaiter(node); + Thread::SpinRelease(&_WaitSetLock); + + node->_recursions = _recursions; // record the old recursion count + _recursions = 0; // set the recursion level to be 0 + _waiters++; // increment the number of waiters + exit(current); // exit the monitor + guarantee(!has_owner(current), "invariant"); + + assert(java_lang_VirtualThread::state(vthread) == java_lang_VirtualThread::RUNNING, "wrong state for vthread"); + java_lang_VirtualThread::set_state(vthread, millis == 0 ? java_lang_VirtualThread::WAITING : java_lang_VirtualThread::TIMED_WAITING); + java_lang_VirtualThread::set_timeout(vthread, millis); + + // Save the ObjectWaiter* in the chunk since we will need it when resuming execution. + oop cont = java_lang_VirtualThread::continuation(vthread); + stackChunkOop chunk = jdk_internal_vm_Continuation::tail(cont); + chunk->set_object_waiter(node); +} + +bool ObjectMonitor::VThreadWaitReenter(JavaThread* current, ObjectWaiter* node, ContinuationWrapper& cont) { + // The first time we run after being preempted on Object.wait() we + // need to check if we were interrupted or the wait timed-out, and + // in that case remove ourselves from the _WaitSet queue. + if (node->TState == ObjectWaiter::TS_WAIT) { + Thread::SpinAcquire(&_WaitSetLock, "WaitSet - unlink"); + if (node->TState == ObjectWaiter::TS_WAIT) { + DequeueSpecificWaiter(node); // unlink from WaitSet + assert(!node->_notified, "invariant"); + node->TState = ObjectWaiter::TS_RUN; + } + Thread::SpinRelease(&_WaitSetLock); + } + + // If this was an interrupted case, set the _interrupted boolean so that + // once we re-acquire the monitor we know if we need to throw IE or not. + ObjectWaiter::TStates state = node->TState; + bool was_notified = state == ObjectWaiter::TS_ENTER || state == ObjectWaiter::TS_CXQ; + assert(was_notified || state == ObjectWaiter::TS_RUN, ""); + node->_interrupted = !was_notified && current->is_interrupted(false); + + // Post JFR and JVMTI events. + EventJavaMonitorWait wait_event; + if (wait_event.should_commit() || JvmtiExport::should_post_monitor_waited()) { + vthread_monitor_waited_event(current, node, cont, &wait_event, !was_notified && !node->_interrupted); + } + + // Mark that we are at reenter so that we don't call this method again. + node->_at_reenter = true; + + if (!was_notified) { + bool acquired = VThreadMonitorEnter(current, node); + if (acquired) { + guarantee(_recursions == 0, "invariant"); + _recursions = node->_recursions; // restore the old recursion count + _waiters--; // decrement the number of waiters + + if (node->_interrupted) { + // We will throw at thaw end after finishing the mount transition. + current->set_pending_interrupted_exception(true); + } + + delete node; + stackChunkOop chunk = cont.tail(); + chunk->set_object_waiter(nullptr); + return true; + } + } else { + // Already moved to _cxq or _EntryList by notifier, so just add to contentions. + add_to_contentions(1); + } + return false; +} + // ----------------------------------------------------------------------------- // Adaptive Spinning Support // @@ -1908,10 +2243,10 @@ bool ObjectMonitor::TrySpin(JavaThread* current) { // CONSIDER: use Prefetch::write() to avoid RTS->RTO upgrades // when preparing to LD...CAS _owner, etc and the CAS is likely // to succeed. - if (_succ == nullptr) { - _succ = current; + if (!has_successor()) { + set_successor(current); } - Thread* prv = nullptr; + int64_t prv = NO_OWNER; // There are three ways to exit the following loop: // 1. A successful spin where this thread has acquired the lock. @@ -1949,14 +2284,14 @@ bool ObjectMonitor::TrySpin(JavaThread* current) { // the spin without prejudice or apply a "penalty" to the // spin count-down variable "ctr", reducing it by 100, say. - JavaThread* ox = static_cast(owner_raw()); - if (ox == nullptr) { - ox = static_cast(try_set_owner_from(nullptr, current)); - if (ox == nullptr) { + int64_t ox = owner_raw(); + if (ox == NO_OWNER) { + ox = try_set_owner_from(NO_OWNER, current); + if (ox == NO_OWNER) { // The CAS succeeded -- this thread acquired ownership // Take care of some bookkeeping to exit spin state. - if (_succ == current) { - _succ = nullptr; + if (has_successor(current)) { + clear_successor(); } // Increase _SpinDuration : @@ -1979,13 +2314,13 @@ bool ObjectMonitor::TrySpin(JavaThread* current) { } // Did lock ownership change hands ? - if (ox != prv && prv != nullptr) { + if (ox != prv && prv != NO_OWNER) { break; } prv = ox; - if (_succ == nullptr) { - _succ = current; + if (!has_successor()) { + set_successor(current); } } @@ -1994,8 +2329,8 @@ bool ObjectMonitor::TrySpin(JavaThread* current) { _SpinDuration = adjust_down(_SpinDuration); } - if (_succ == current) { - _succ = nullptr; + if (has_successor(current)) { + clear_successor(); // Invariant: after setting succ=null a contending thread // must recheck-retry _owner before parking. This usually happens // in the normal usage of TrySpin(), but it's safest @@ -2016,13 +2351,33 @@ bool ObjectMonitor::TrySpin(JavaThread* current) { ObjectWaiter::ObjectWaiter(JavaThread* current) { _next = nullptr; _prev = nullptr; - _notified = 0; + _thread = current; + _monitor = nullptr; _notifier_tid = 0; + _recursions = 0; TState = TS_RUN; - _thread = current; - _event = _thread->_ParkEvent; + _notified = false; + _is_wait = false; + _at_reenter = false; + _interrupted = false; _active = false; - assert(_event != nullptr, "invariant"); +} + +ObjectWaiter::ObjectWaiter(oop vthread, ObjectMonitor* mon) : ObjectWaiter(nullptr) { + assert(oopDesc::is_oop(vthread), ""); + _vthread = OopHandle(JavaThread::thread_oop_storage(), vthread); + _monitor = mon; +} + +ObjectWaiter::~ObjectWaiter() { + if (is_vthread()) { + assert(vthread() != nullptr, ""); + _vthread.release(JavaThread::thread_oop_storage()); + } +} + +oop ObjectWaiter::vthread() const { + return _vthread.resolve(); } void ObjectWaiter::wait_reenter_begin(ObjectMonitor * const mon) { @@ -2140,12 +2495,18 @@ void ObjectMonitor::Initialize() { DEBUG_ONLY(InitDone = true;) } +// We can't call this during Initialize() because BarrierSet needs to be set. +void ObjectMonitor::Initialize2() { + _vthread_cxq_head = OopHandle(JavaThread::thread_oop_storage(), nullptr); + _vthread_unparker_ParkEvent = ParkEvent::Allocate(nullptr); +} + void ObjectMonitor::print_on(outputStream* st) const { // The minimal things to print for markWord printing, more can be added for debugging and logging. st->print("{contentions=0x%08x,waiters=0x%08x" - ",recursions=" INTX_FORMAT ",owner=" INTPTR_FORMAT "}", + ",recursions=" INTX_FORMAT ",owner=" INT64_FORMAT "}", contentions(), waiters(), recursions(), - p2i(owner())); + owner()); } void ObjectMonitor::print() const { print_on(tty); } @@ -2188,7 +2549,7 @@ void ObjectMonitor::print_debug_style_on(outputStream* st) const { st->print_cr(" ..."); st->print_cr(" [%d] = '\\0'", (int)sizeof(_pad_buf0) - 1); st->print_cr(" }"); - st->print_cr(" _owner = " INTPTR_FORMAT, p2i(owner_raw())); + st->print_cr(" _owner = " INT64_FORMAT, owner_raw()); st->print_cr(" _previous_owner_tid = " UINT64_FORMAT, _previous_owner_tid); st->print_cr(" _pad_buf1 = {"); st->print_cr(" [0] = '\\0'"); @@ -2199,7 +2560,7 @@ void ObjectMonitor::print_debug_style_on(outputStream* st) const { st->print_cr(" _recursions = " INTX_FORMAT, _recursions); st->print_cr(" _EntryList = " INTPTR_FORMAT, p2i(_EntryList)); st->print_cr(" _cxq = " INTPTR_FORMAT, p2i(_cxq)); - st->print_cr(" _succ = " INTPTR_FORMAT, p2i(_succ)); + st->print_cr(" _succ = " INT64_FORMAT, successor()); st->print_cr(" _SpinDuration = %d", _SpinDuration); st->print_cr(" _contentions = %d", contentions()); st->print_cr(" _WaitSet = " INTPTR_FORMAT, p2i(_WaitSet)); diff --git a/src/hotspot/share/runtime/objectMonitor.hpp b/src/hotspot/share/runtime/objectMonitor.hpp index 30d2e509416..c205eedf684 100644 --- a/src/hotspot/share/runtime/objectMonitor.hpp +++ b/src/hotspot/share/runtime/objectMonitor.hpp @@ -28,34 +28,48 @@ #include "memory/allocation.hpp" #include "memory/padded.hpp" #include "oops/markWord.hpp" +#include "oops/oopHandle.hpp" #include "oops/weakHandle.hpp" +#include "runtime/javaThread.hpp" #include "runtime/perfDataTypes.hpp" #include "utilities/checkedCast.hpp" class ObjectMonitor; class ObjectMonitorContentionMark; class ParkEvent; +class BasicLock; +class ContinuationWrapper; -// ObjectWaiter serves as a "proxy" or surrogate thread. -// TODO-FIXME: Eliminate ObjectWaiter and use the thread-specific -// ParkEvent instead. Beware, however, that the JVMTI code -// knows about ObjectWaiters, so we'll have to reconcile that code. -// See next_waiter(), first_waiter(), etc. -class ObjectWaiter : public StackObj { +class ObjectWaiter : public CHeapObj { public: - enum TStates { TS_UNDEF, TS_READY, TS_RUN, TS_WAIT, TS_ENTER, TS_CXQ }; + enum TStates : uint8_t { TS_UNDEF, TS_READY, TS_RUN, TS_WAIT, TS_ENTER, TS_CXQ }; ObjectWaiter* volatile _next; ObjectWaiter* volatile _prev; - JavaThread* _thread; - uint64_t _notifier_tid; - ParkEvent * _event; - volatile int _notified; + JavaThread* _thread; + OopHandle _vthread; + ObjectMonitor* _monitor; + uint64_t _notifier_tid; + int _recursions; volatile TStates TState; - bool _active; // Contention monitoring is enabled + volatile bool _notified; + bool _is_wait; + bool _at_reenter; + bool _interrupted; + bool _active; // Contention monitoring is enabled public: ObjectWaiter(JavaThread* current); - + ObjectWaiter(oop vthread, ObjectMonitor* mon); + ~ObjectWaiter(); + JavaThread* thread() const { return _thread; } + bool is_vthread() const { return _thread == nullptr; } + uint8_t state() const { return TState; } + ObjectMonitor* monitor() const { return _monitor; } + bool is_wait() const { return _is_wait; } + bool notified() const { return _notified; } + bool at_reenter() const { return _at_reenter; } + bool at_monitorenter() const { return !_is_wait || _at_reenter || _notified; } + oop vthread() const; void wait_reenter_begin(ObjectMonitor *mon); void wait_reenter_end(ObjectMonitor *mon); }; @@ -132,6 +146,11 @@ class ObjectMonitor : public CHeapObj { static OopStorage* _oop_storage; + // List of j.l.VirtualThread waiting to be unblocked by unblocker thread. + static OopHandle _vthread_cxq_head; + // ParkEvent of unblocker thread. + static ParkEvent* _vthread_unparker_ParkEvent; + // The sync code expects the metadata field to be at offset zero (0). // Enforced by the assert() in metadata_addr(). // * LM_LIGHTWEIGHT with UseObjectMonitorTable: @@ -146,24 +165,12 @@ class ObjectMonitor : public CHeapObj { // its cache line with _metadata. DEFINE_PAD_MINUS_SIZE(0, OM_CACHE_LINE_SIZE, sizeof(_metadata) + sizeof(WeakHandle)); - // Used by async deflation as a marker in the _owner field. - // Note that the choice of the two markers is peculiar: - // - They need to represent values that cannot be pointers. In particular, - // we achieve this by using the lowest two bits. - // - ANONYMOUS_OWNER should be a small value, it is used in generated code - // and small values encode much better. - // - We test for anonymous owner by testing for the lowest bit, therefore - // DEFLATER_MARKER must *not* have that bit set. - static const uintptr_t DEFLATER_MARKER_VALUE = 2; - #define DEFLATER_MARKER reinterpret_cast(DEFLATER_MARKER_VALUE) - public: - // NOTE: Typed as uintptr_t so that we can pick it up in SA, via vmStructs. - static const uintptr_t ANONYMOUS_OWNER = 1; - private: - static void* anon_owner_ptr() { return reinterpret_cast(ANONYMOUS_OWNER); } + static const int64_t NO_OWNER = 0; + static const int64_t ANONYMOUS_OWNER = 1; + static const int64_t DEFLATER_MARKER = 2; - void* volatile _owner; // pointer to owning thread OR BasicLock + int64_t volatile _owner; // Either tid of owner, NO_OWNER, ANONYMOUS_OWNER or DEFLATER_MARKER. volatile uint64_t _previous_owner_tid; // thread id of the previous owner of the monitor // Separate _owner and _next_om on different cache lines since // both can have busy multi-threaded access. _previous_owner_tid is only @@ -178,7 +185,7 @@ class ObjectMonitor : public CHeapObj { // acting as proxies for Threads. ObjectWaiter* volatile _cxq; // LL of recently-arrived threads blocked on entry. - JavaThread* volatile _succ; // Heir presumptive thread - used for futile wakeup throttling + int64_t volatile _succ; // Heir presumptive thread - used for futile wakeup throttling volatile int _SpinDuration; @@ -191,9 +198,16 @@ class ObjectMonitor : public CHeapObj { volatile int _waiters; // number of waiting threads volatile int _WaitSetLock; // protects Wait Queue - simple spinlock + // Used in LM_LEGACY mode to store BasicLock* in case of inflation by contending thread. + BasicLock* volatile _stack_locker; + public: static void Initialize(); + static void Initialize2(); + + static OopHandle& vthread_cxq_head() { return _vthread_cxq_head; } + static ParkEvent* vthread_unparker_ParkEvent() { return _vthread_unparker_ParkEvent; } // Only perform a PerfData operation if the PerfData object has been // allocated and if the PerfDataManager has not freed the PerfData @@ -249,7 +263,7 @@ class ObjectMonitor : public CHeapObj { void set_hash(intptr_t hash); bool is_busy() const { - // TODO-FIXME: assert _owner == null implies _recursions = 0 + // TODO-FIXME: assert _owner == NO_OWNER implies _recursions = 0 intptr_t ret_code = intptr_t(_waiters) | intptr_t(_cxq) | intptr_t(_EntryList); int cnts = contentions(); // read once if (cnts > 0) { @@ -266,48 +280,75 @@ class ObjectMonitor : public CHeapObj { // Returns true if this OM has an owner, false otherwise. bool has_owner() const; - void* owner() const; // Returns null if DEFLATER_MARKER is observed. - void* owner_raw() const; + int64_t owner() const; // Returns NO_OWNER if DEFLATER_MARKER is observed. + int64_t owner_raw() const; + + // These methods return the value we set in _owner when acquiring + // the monitor with the given thread/vthread (tid). + static int64_t owner_from(JavaThread* thread); + static int64_t owner_from(oop vthread); + // Returns true if owner field == DEFLATER_MARKER and false otherwise. bool owner_is_DEFLATER_MARKER() const; // Returns true if 'this' is being async deflated and false otherwise. bool is_being_async_deflated(); - // Clear _owner field; current value must match old_value. - void release_clear_owner(void* old_value); + // Clear _owner field; current value must match thread's tid. + void release_clear_owner(JavaThread* thread); // Simply set _owner field to new_value; current value must match old_value. - void set_owner_from(void* old_value, void* new_value); - // Simply set _owner field to current; current value must match basic_lock_p. - void set_owner_from_BasicLock(void* basic_lock_p, JavaThread* current); + void set_owner_from_raw(int64_t old_value, int64_t new_value); + // Same as above but uses tid of current as new value. + void set_owner_from(int64_t old_value, JavaThread* current); // Try to set _owner field to new_value if the current value matches // old_value, using Atomic::cmpxchg(). Otherwise, does not change the // _owner field. Returns the prior value of the _owner field. - void* try_set_owner_from(void* old_value, void* new_value); - - void set_owner_anonymous() { - set_owner_from(nullptr, anon_owner_ptr()); + int64_t try_set_owner_from_raw(int64_t old_value, int64_t new_value); + // Same as above but uses tid of current as new_value. + int64_t try_set_owner_from(int64_t old_value, JavaThread* current); + + // Methods to check and set _succ. The successor is the thread selected + // from _cxq/_EntryList by the current owner when releasing the monitor, + // to run again and re-try acquiring the monitor. It is used to avoid + // unnecessary wake-ups if there is already a successor set. + bool has_successor() const; + bool has_successor(JavaThread* thread) const; + void set_successor(JavaThread* thread); + void set_successor(oop vthread); + void clear_successor(); + int64_t successor() const; + + // Returns true if _owner field == tid of thread, false otherwise. + bool has_owner(JavaThread* thread) const { return owner() == owner_from(thread); } + // Set _owner field to tid of thread; current value must be NO_OWNER. + void set_owner(JavaThread* thread) { set_owner_from(NO_OWNER, thread); } + // Try to set _owner field from NO_OWNER to tid of thread. + bool try_set_owner(JavaThread* thread) { + return try_set_owner_from(NO_OWNER, thread) == NO_OWNER; } - bool is_owner_anonymous() const { - return owner_raw() == anon_owner_ptr(); + bool has_anonymous_owner() const { return owner_raw() == ANONYMOUS_OWNER; } + void set_anonymous_owner() { + set_owner_from_raw(NO_OWNER, ANONYMOUS_OWNER); } - - void set_owner_from_anonymous(Thread* owner) { - set_owner_from(anon_owner_ptr(), owner); + void set_owner_from_anonymous(JavaThread* owner) { + set_owner_from(ANONYMOUS_OWNER, owner); } + // Get and set _stack_locker. + BasicLock* stack_locker() const; + void set_stack_locker(BasicLock* locker); + // Simply get _next_om field. ObjectMonitor* next_om() const; // Simply set _next_om field to new_value. void set_next_om(ObjectMonitor* new_value); - int waiters() const; - int contentions() const; void add_to_contentions(int value); intx recursions() const { return _recursions; } void set_recursions(size_t recursions); // JVM/TI GetObjectMonitorUsage() needs this: + int waiters() const; ObjectWaiter* first_waiter() { return _WaitSet; } ObjectWaiter* next_waiter(ObjectWaiter* o) { return o->_next; } JavaThread* thread_of_waiter(ObjectWaiter* o) { return o->_thread; } @@ -351,6 +392,7 @@ class ObjectMonitor : public CHeapObj { bool spin_enter(JavaThread* current); void enter_with_contention_mark(JavaThread* current, ObjectMonitorContentionMark& contention_mark); void exit(JavaThread* current, bool not_suspended = true); + bool resume_operation(JavaThread* current, ObjectWaiter* node, ContinuationWrapper& cont); void wait(jlong millis, bool interruptible, TRAPS); void notify(TRAPS); void notifyAll(TRAPS); @@ -373,6 +415,10 @@ class ObjectMonitor : public CHeapObj { void ReenterI(JavaThread* current, ObjectWaiter* current_node); void UnlinkAfterAcquire(JavaThread* current, ObjectWaiter* current_node); + bool VThreadMonitorEnter(JavaThread* current, ObjectWaiter* node = nullptr); + void VThreadWait(JavaThread* current, jlong millis); + bool VThreadWaitReenter(JavaThread* current, ObjectWaiter* node, ContinuationWrapper& cont); + void VThreadEpilog(JavaThread* current, ObjectWaiter* node); enum class TryLockResult { Interference = -1, HasOwner = 0, Success = 1 }; diff --git a/src/hotspot/share/runtime/objectMonitor.inline.hpp b/src/hotspot/share/runtime/objectMonitor.inline.hpp index 6d3c6ff24c3..64e233a1ff5 100644 --- a/src/hotspot/share/runtime/objectMonitor.inline.hpp +++ b/src/hotspot/share/runtime/objectMonitor.inline.hpp @@ -32,23 +32,34 @@ #include "oops/markWord.hpp" #include "runtime/atomic.hpp" #include "runtime/globals.hpp" +#include "runtime/javaThread.inline.hpp" #include "runtime/lockStack.inline.hpp" #include "runtime/synchronizer.hpp" +#include "runtime/threadIdentifier.hpp" #include "utilities/checkedCast.hpp" #include "utilities/globalDefinitions.hpp" +inline int64_t ObjectMonitor::owner_from(JavaThread* thread) { + int64_t tid = thread->lock_id(); + assert(tid >= ThreadIdentifier::initial() && tid < ThreadIdentifier::current(), "must be reasonable"); + return tid; +} + +inline int64_t ObjectMonitor::owner_from(oop vthread) { + int64_t tid = java_lang_Thread::thread_id(vthread); + assert(tid >= ThreadIdentifier::initial() && tid < ThreadIdentifier::current(), "must be reasonable"); + return tid; +} + inline bool ObjectMonitor::is_entered(JavaThread* current) const { - if (LockingMode == LM_LIGHTWEIGHT) { - if (is_owner_anonymous()) { + if (has_anonymous_owner()) { + if (LockingMode == LM_LIGHTWEIGHT) { return current->lock_stack().contains(object()); } else { - return current == owner_raw(); + return current->is_lock_owned((address)stack_locker()); } } else { - void* owner = owner_raw(); - if (current == owner || current->is_lock_owned((address)owner)) { - return true; - } + return has_owner(current); } return false; } @@ -92,23 +103,29 @@ inline int ObjectMonitor::waiters() const { } inline bool ObjectMonitor::has_owner() const { - void* owner = owner_raw(); - return owner != nullptr && owner != DEFLATER_MARKER; + int64_t owner = owner_raw(); + return owner != NO_OWNER && owner != DEFLATER_MARKER; } -// Returns null if DEFLATER_MARKER is observed. -inline void* ObjectMonitor::owner() const { - void* owner = owner_raw(); - return owner != DEFLATER_MARKER ? owner : nullptr; +// Returns NO_OWNER if DEFLATER_MARKER is observed. +inline int64_t ObjectMonitor::owner() const { + int64_t owner = owner_raw(); + return owner != DEFLATER_MARKER ? owner : NO_OWNER; } -inline void* ObjectMonitor::owner_raw() const { +inline int64_t ObjectMonitor::owner_raw() const { return Atomic::load(&_owner); } +inline BasicLock* ObjectMonitor::stack_locker() const { + return Atomic::load(&_stack_locker); +} + +inline void ObjectMonitor::set_stack_locker(BasicLock* locker) { + Atomic::store(&_stack_locker, locker); +} + // Returns true if owner field == DEFLATER_MARKER and false otherwise. -// This accessor is called when we really need to know if the owner -// field == DEFLATER_MARKER and any non-null value won't do the trick. inline bool ObjectMonitor::owner_is_DEFLATER_MARKER() const { return owner_raw() == DEFLATER_MARKER; } @@ -135,63 +152,82 @@ inline void ObjectMonitor::set_recursions(size_t recursions) { } // Clear _owner field; current value must match old_value. -inline void ObjectMonitor::release_clear_owner(void* old_value) { +inline void ObjectMonitor::release_clear_owner(JavaThread* old_owner) { + int64_t old_value = owner_from(old_owner); #ifdef ASSERT - void* prev = Atomic::load(&_owner); - assert(prev == old_value, "unexpected prev owner=" INTPTR_FORMAT - ", expected=" INTPTR_FORMAT, p2i(prev), p2i(old_value)); + int64_t prev = Atomic::load(&_owner); + assert(prev == old_value, "unexpected prev owner=" INT64_FORMAT + ", expected=" INT64_FORMAT, prev, old_value); #endif - Atomic::release_store(&_owner, (void*)nullptr); + Atomic::release_store(&_owner, NO_OWNER); log_trace(monitorinflation, owner)("release_clear_owner(): mid=" - INTPTR_FORMAT ", old_value=" INTPTR_FORMAT, - p2i(this), p2i(old_value)); + INTPTR_FORMAT ", old_value=" INT64_FORMAT, + p2i(this), old_value); } // Simply set _owner field to new_value; current value must match old_value. // (Simple means no memory sync needed.) -inline void ObjectMonitor::set_owner_from(void* old_value, void* new_value) { +inline void ObjectMonitor::set_owner_from_raw(int64_t old_value, int64_t new_value) { #ifdef ASSERT - void* prev = Atomic::load(&_owner); - assert(prev == old_value, "unexpected prev owner=" INTPTR_FORMAT - ", expected=" INTPTR_FORMAT, p2i(prev), p2i(old_value)); + int64_t prev = Atomic::load(&_owner); + assert((int64_t)prev < ThreadIdentifier::current(), "must be reasonable"); + assert(prev == old_value, "unexpected prev owner=" INT64_FORMAT + ", expected=" INT64_FORMAT, prev, old_value); #endif Atomic::store(&_owner, new_value); log_trace(monitorinflation, owner)("set_owner_from(): mid=" - INTPTR_FORMAT ", old_value=" INTPTR_FORMAT - ", new_value=" INTPTR_FORMAT, p2i(this), - p2i(old_value), p2i(new_value)); + INTPTR_FORMAT ", old_value=" INT64_FORMAT + ", new_value=" INT64_FORMAT, p2i(this), + old_value, new_value); } -// Simply set _owner field to self; current value must match basic_lock_p. -inline void ObjectMonitor::set_owner_from_BasicLock(void* basic_lock_p, JavaThread* current) { -#ifdef ASSERT - void* prev = Atomic::load(&_owner); - assert(prev == basic_lock_p, "unexpected prev owner=" INTPTR_FORMAT - ", expected=" INTPTR_FORMAT, p2i(prev), p2i(basic_lock_p)); -#endif - // Non-null owner field to non-null owner field is safe without - // cmpxchg() as long as all readers can tolerate either flavor. - Atomic::store(&_owner, current); - log_trace(monitorinflation, owner)("set_owner_from_BasicLock(): mid=" - INTPTR_FORMAT ", basic_lock_p=" - INTPTR_FORMAT ", new_value=" INTPTR_FORMAT, - p2i(this), p2i(basic_lock_p), p2i(current)); +inline void ObjectMonitor::set_owner_from(int64_t old_value, JavaThread* current) { + set_owner_from_raw(old_value, owner_from(current)); } // Try to set _owner field to new_value if the current value matches // old_value. Otherwise, does not change the _owner field. Returns // the prior value of the _owner field. -inline void* ObjectMonitor::try_set_owner_from(void* old_value, void* new_value) { - void* prev = Atomic::cmpxchg(&_owner, old_value, new_value); +inline int64_t ObjectMonitor::try_set_owner_from_raw(int64_t old_value, int64_t new_value) { + assert((int64_t)new_value < ThreadIdentifier::current(), "must be reasonable"); + int64_t prev = Atomic::cmpxchg(&_owner, old_value, new_value); if (prev == old_value) { log_trace(monitorinflation, owner)("try_set_owner_from(): mid=" - INTPTR_FORMAT ", prev=" INTPTR_FORMAT - ", new=" INTPTR_FORMAT, p2i(this), - p2i(prev), p2i(new_value)); + INTPTR_FORMAT ", prev=" INT64_FORMAT + ", new=" INT64_FORMAT, p2i(this), + prev, new_value); } return prev; } +inline int64_t ObjectMonitor::try_set_owner_from(int64_t old_value, JavaThread* current) { + return try_set_owner_from_raw(old_value, owner_from(current)); +} + +inline bool ObjectMonitor::has_successor() const { + return Atomic::load(&_succ) != NO_OWNER; +} + +inline bool ObjectMonitor::has_successor(JavaThread* thread) const { + return owner_from(thread) == Atomic::load(&_succ); +} + +inline void ObjectMonitor::set_successor(JavaThread* thread) { + Atomic::store(&_succ, owner_from(thread)); +} + +inline void ObjectMonitor::set_successor(oop vthread) { + Atomic::store(&_succ, java_lang_Thread::thread_id(vthread)); +} + +inline void ObjectMonitor::clear_successor() { + Atomic::store(&_succ, NO_OWNER); +} + +inline int64_t ObjectMonitor::successor() const { + return Atomic::load(&_succ); +} + // The _next_om field can be concurrently read and modified so we // use Atomic operations to disable compiler optimizations that // might try to elide loading and/or storing this field. diff --git a/src/hotspot/share/runtime/os.cpp b/src/hotspot/share/runtime/os.cpp index 2395510f27f..d8e539ca115 100644 --- a/src/hotspot/share/runtime/os.cpp +++ b/src/hotspot/share/runtime/os.cpp @@ -1083,6 +1083,26 @@ void os::print_dhm(outputStream* st, const char* startStr, long sec) { st->print_cr("%s %ld days %ld:%02ld hours", startStr, days, hours, minutes); } +void os::print_tos_pc(outputStream* st, const void* context) { + if (context == nullptr) return; + + // First of all, carefully determine sp without inspecting memory near pc. + // See comment below. + intptr_t* sp = nullptr; + fetch_frame_from_context(context, &sp, nullptr); + print_tos(st, (address)sp); + st->cr(); + + // Note: it may be unsafe to inspect memory near pc. For example, pc may + // point to garbage if entry point in an nmethod is corrupted. Leave + // this at the end, and hope for the best. + // This version of fetch_frame_from_context finds the caller pc if the actual + // one is bad. + address pc = fetch_frame_from_context(context).pc(); + print_instructions(st, pc); + st->cr(); +} + void os::print_tos(outputStream* st, address sp) { st->print_cr("Top of Stack: (sp=" PTR_FORMAT ")", p2i(sp)); print_hex_dump(st, sp, sp + 512, sizeof(intptr_t)); @@ -2166,7 +2186,7 @@ bool os::uncommit_memory(char* addr, size_t bytes, bool executable) { assert_nonempty_range(addr, bytes); bool res; if (MemTracker::enabled()) { - NmtVirtualMemoryLocker ml; + ThreadCritical tc; res = pd_uncommit_memory(addr, bytes, executable); if (res) { MemTracker::record_virtual_memory_uncommit((address)addr, bytes); @@ -2188,7 +2208,7 @@ bool os::release_memory(char* addr, size_t bytes) { assert_nonempty_range(addr, bytes); bool res; if (MemTracker::enabled()) { - NmtVirtualMemoryLocker ml; + ThreadCritical tc; res = pd_release_memory(addr, bytes); if (res) { MemTracker::record_virtual_memory_release((address)addr, bytes); @@ -2273,7 +2293,7 @@ char* os::map_memory(int fd, const char* file_name, size_t file_offset, bool os::unmap_memory(char *addr, size_t bytes) { bool result; if (MemTracker::enabled()) { - NmtVirtualMemoryLocker ml; + ThreadCritical tc; result = pd_unmap_memory(addr, bytes); if (result) { MemTracker::record_virtual_memory_release((address)addr, bytes); @@ -2312,7 +2332,7 @@ char* os::reserve_memory_special(size_t size, size_t alignment, size_t page_size bool os::release_memory_special(char* addr, size_t bytes) { bool res; if (MemTracker::enabled()) { - NmtVirtualMemoryLocker ml; + ThreadCritical tc; res = pd_release_memory_special(addr, bytes); if (res) { MemTracker::record_virtual_memory_release((address)addr, bytes); diff --git a/src/hotspot/share/runtime/os.hpp b/src/hotspot/share/runtime/os.hpp index 323b8d2df8d..2f665df5bba 100644 --- a/src/hotspot/share/runtime/os.hpp +++ b/src/hotspot/share/runtime/os.hpp @@ -882,10 +882,6 @@ class os: AllStatic { // Fills in path to jvm.dll/libjvm.so (used by the Disassembler) static void jvm_path(char *buf, jint buflen); - // JNI names - static void print_jni_name_prefix_on(outputStream* st, int args_size); - static void print_jni_name_suffix_on(outputStream* st, int args_size); - // Init os specific system properties values static void init_system_properties_values(); diff --git a/src/hotspot/share/runtime/sharedRuntime.cpp b/src/hotspot/share/runtime/sharedRuntime.cpp index e4d4e6aea0f..a9ad42ec5f2 100644 --- a/src/hotspot/share/runtime/sharedRuntime.cpp +++ b/src/hotspot/share/runtime/sharedRuntime.cpp @@ -1971,7 +1971,7 @@ void SharedRuntime::monitor_exit_helper(oopDesc* obj, BasicLock* lock, JavaThrea // hasn't been deallocated. ObjectMonitor* m = current->unlocked_inflated_monitor(); if (m != nullptr) { - assert(m->owner_raw() != current, "must be"); + assert(!m->has_owner(current), "must be"); current->clear_unlocked_inflated_monitor(); // We need to reacquire the lock before we can call ObjectSynchronizer::exit(). diff --git a/src/hotspot/share/runtime/sharedRuntime.hpp b/src/hotspot/share/runtime/sharedRuntime.hpp index f530c0ad2a8..854d4dcf1ad 100644 --- a/src/hotspot/share/runtime/sharedRuntime.hpp +++ b/src/hotspot/share/runtime/sharedRuntime.hpp @@ -512,6 +512,10 @@ class SharedRuntime: AllStatic { // On PowerPC it includes the 4 words holding the old TOC & LR glue. static uint in_preserve_stack_slots(); + static VMReg thread_register(); + + static void continuation_enter_cleanup(MacroAssembler* masm); + // Is vector's size (in bytes) bigger than a size saved by default? // For example, on x86 16 bytes XMM registers are saved by default. static bool is_wide_vector(int size); diff --git a/src/hotspot/share/runtime/sharedRuntimeTrans.cpp b/src/hotspot/share/runtime/sharedRuntimeTrans.cpp index 5285819ee96..78788d487b7 100644 --- a/src/hotspot/share/runtime/sharedRuntimeTrans.cpp +++ b/src/hotspot/share/runtime/sharedRuntimeTrans.cpp @@ -42,7 +42,7 @@ // Enabling optimizations in this file causes incorrect code to be // generated; can not figure out how to turn down optimization for one // file in the IDE on Windows -#ifdef WIN32 +#ifdef _WINDOWS # pragma warning( disable: 4748 ) // /GS can not protect parameters and local variables from local buffer overrun because optimizations are disabled in function # pragma optimize ( "", off ) #endif @@ -671,6 +671,6 @@ JRT_LEAF(jdouble, SharedRuntime::dpow(jdouble x, jdouble y)) return __ieee754_pow(x, y); JRT_END -#ifdef WIN32 +#ifdef _WINDOWS # pragma optimize ( "", on ) #endif diff --git a/src/hotspot/share/runtime/stackChunkFrameStream.hpp b/src/hotspot/share/runtime/stackChunkFrameStream.hpp index 6937feb1517..207203dbb35 100644 --- a/src/hotspot/share/runtime/stackChunkFrameStream.hpp +++ b/src/hotspot/share/runtime/stackChunkFrameStream.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -75,6 +75,7 @@ class StackChunkFrameStream : public StackObj { inline bool is_stub() const; inline bool is_compiled() const; CodeBlob* cb() const { return _cb; } + inline void get_cb(); const ImmutableOopMap* oopmap() const { if (_oopmap == nullptr) get_oopmap(); return _oopmap; } inline int frame_size() const; inline int stack_argsize() const; @@ -98,7 +99,6 @@ class StackChunkFrameStream : public StackObj { private: inline address get_pc() const; - inline void get_cb(); inline int interpreter_frame_size() const; inline int interpreter_frame_num_oops() const; diff --git a/src/hotspot/share/runtime/stackChunkFrameStream.inline.hpp b/src/hotspot/share/runtime/stackChunkFrameStream.inline.hpp index c56034786c9..734d31d3390 100644 --- a/src/hotspot/share/runtime/stackChunkFrameStream.inline.hpp +++ b/src/hotspot/share/runtime/stackChunkFrameStream.inline.hpp @@ -106,7 +106,7 @@ StackChunkFrameStream::StackChunkFrameStream(stackChunkOop chunk, co template inline bool StackChunkFrameStream::is_stub() const { - return cb() != nullptr && (_cb->is_safepoint_stub() || _cb->is_runtime_stub()); + return cb() != nullptr && _cb->is_runtime_stub(); } template @@ -196,7 +196,14 @@ inline int StackChunkFrameStream::stack_argsize() const { template inline int StackChunkFrameStream::num_oops() const { - return is_interpreted() ? interpreter_frame_num_oops() : oopmap()->num_oops(); + if (is_interpreted()) { + return interpreter_frame_num_oops(); + } else if (is_compiled()) { + return oopmap()->num_oops(); + } else { + assert(is_stub(), "invariant"); + return 0; + } } template @@ -208,7 +215,7 @@ template template inline void StackChunkFrameStream::next(RegisterMapT* map, bool stop) { update_reg_map(map); - bool safepoint = is_stub(); + bool is_runtime_stub = is_stub(); if (frame_kind == ChunkFrames::Mixed) { if (is_interpreted()) { next_for_interpreter_frame(); @@ -232,8 +239,9 @@ inline void StackChunkFrameStream::next(RegisterMapT* map, bool stop get_cb(); update_reg_map_pd(map); - if (safepoint && cb() != nullptr) { // there's no post-call nop and no fast oopmap lookup - _oopmap = cb()->oop_map_for_return_address(pc()); + if (is_runtime_stub && cb() != nullptr) { // there's no post-call nop and no fast oopmap lookup + // caller could have been deoptimized so use orig_pc() + _oopmap = cb()->oop_map_for_return_address(orig_pc()); } } @@ -300,9 +308,8 @@ inline void StackChunkFrameStream::update_reg_map(RegisterMa template<> template<> inline void StackChunkFrameStream::update_reg_map(RegisterMap* map) { - assert(map->in_cont(), ""); - assert(map->stack_chunk()() == _chunk, ""); - if (map->update_map()) { + assert(!map->in_cont() || map->stack_chunk() == _chunk, ""); + if (map->update_map() && is_stub()) { frame f = to_frame(); oopmap()->update_register_map(&f, map); // we have callee-save registers in this case } diff --git a/src/hotspot/share/runtime/stackValue.cpp b/src/hotspot/share/runtime/stackValue.cpp index c0b511ed537..cc6e0bb92d6 100644 --- a/src/hotspot/share/runtime/stackValue.cpp +++ b/src/hotspot/share/runtime/stackValue.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -293,7 +293,7 @@ address StackValue::stack_value_address(const frame* fr, const RegisterMapT* reg return value_addr; } -BasicLock* StackValue::resolve_monitor_lock(const frame* fr, Location location) { +BasicLock* StackValue::resolve_monitor_lock(const frame& fr, Location location) { assert(location.is_stack(), "for now we only look at the stack"); int word_offset = location.stack_offset() / wordSize; // (stack picture) @@ -304,7 +304,7 @@ BasicLock* StackValue::resolve_monitor_lock(const frame* fr, Location location) // the word_offset is the distance from the stack pointer to the lowest address // The frame's original stack pointer, before any extension by its callee // (due to Compiler1 linkage on SPARC), must be used. - return (BasicLock*) (fr->unextended_sp() + word_offset); + return (BasicLock*) (fr.unextended_sp() + word_offset); } diff --git a/src/hotspot/share/runtime/stackValue.hpp b/src/hotspot/share/runtime/stackValue.hpp index 517c4cf9e77..5001675872c 100644 --- a/src/hotspot/share/runtime/stackValue.hpp +++ b/src/hotspot/share/runtime/stackValue.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2022, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -123,7 +123,7 @@ class StackValue : public ResourceObj { static StackValue* create_stack_value_from_oop_location(stackChunkOop chunk, void* addr); static StackValue* create_stack_value_from_narrowOop_location(stackChunkOop chunk, void* addr, bool is_register); - static BasicLock* resolve_monitor_lock(const frame* fr, Location location); + static BasicLock* resolve_monitor_lock(const frame& fr, Location location); template static StackValue* create_stack_value(const frame* fr, const RegisterMapT* reg_map, ScopeValue* sv); diff --git a/src/hotspot/share/runtime/stubRoutines.cpp b/src/hotspot/share/runtime/stubRoutines.cpp index c881b64b592..5253bcd0571 100644 --- a/src/hotspot/share/runtime/stubRoutines.cpp +++ b/src/hotspot/share/runtime/stubRoutines.cpp @@ -186,6 +186,7 @@ address StubRoutines::_array_partition = nullptr; address StubRoutines::_cont_thaw = nullptr; address StubRoutines::_cont_returnBarrier = nullptr; address StubRoutines::_cont_returnBarrierExc = nullptr; +address StubRoutines::_cont_preempt_stub = nullptr; address StubRoutines::_upcall_stub_exception_handler = nullptr; address StubRoutines::_upcall_stub_load_target = nullptr; diff --git a/src/hotspot/share/runtime/stubRoutines.hpp b/src/hotspot/share/runtime/stubRoutines.hpp index f025742b605..4c96738ce4b 100644 --- a/src/hotspot/share/runtime/stubRoutines.hpp +++ b/src/hotspot/share/runtime/stubRoutines.hpp @@ -292,6 +292,7 @@ class StubRoutines: AllStatic { static address _cont_thaw; static address _cont_returnBarrier; static address _cont_returnBarrierExc; + static address _cont_preempt_stub; // Vector Math Routines static address _vector_f_math[VectorSupport::NUM_VEC_SIZES][VectorSupport::NUM_VECTOR_OP_MATH]; @@ -501,6 +502,7 @@ class StubRoutines: AllStatic { static address cont_thaw() { return _cont_thaw; } static address cont_returnBarrier() { return _cont_returnBarrier; } static address cont_returnBarrierExc(){return _cont_returnBarrierExc; } + static address cont_preempt_stub() { return _cont_preempt_stub; } static address upcall_stub_exception_handler() { assert(_upcall_stub_exception_handler != nullptr, "not implemented"); diff --git a/src/hotspot/share/runtime/synchronizer.cpp b/src/hotspot/share/runtime/synchronizer.cpp index d8fb7a4734a..cba1918529c 100644 --- a/src/hotspot/share/runtime/synchronizer.cpp +++ b/src/hotspot/share/runtime/synchronizer.cpp @@ -361,7 +361,7 @@ bool ObjectSynchronizer::quick_notify(oopDesc* obj, JavaThread* current, bool al return false; } assert(mon->object() == oop(obj), "invariant"); - if (mon->owner() != current) return false; // slow-path for IMS exception + if (!mon->has_owner(current)) return false; // slow-path for IMS exception if (mon->first_waiter() != nullptr) { // We have one or more waiters. Since this is an inflated monitor @@ -424,14 +424,13 @@ bool ObjectSynchronizer::quick_enter_legacy(oop obj, BasicLock* lock, JavaThread if (m->object_peek() == nullptr) { return false; } - JavaThread* const owner = static_cast(m->owner_raw()); // Lock contention and Transactional Lock Elision (TLE) diagnostics // and observability // Case: light contention possibly amenable to TLE // Case: TLE inimical operations such as nested/recursive synchronization - if (owner == current) { + if (m->has_owner(current)) { m->_recursions++; current->inc_held_monitor_count(); return true; @@ -448,7 +447,7 @@ bool ObjectSynchronizer::quick_enter_legacy(oop obj, BasicLock* lock, JavaThread // and last are the inflated Java Monitor (ObjectMonitor) checks. lock->set_displaced_header(markWord::unused_mark()); - if (owner == nullptr && m->try_set_owner_from(nullptr, current) == nullptr) { + if (!m->has_owner() && m->try_set_owner(current)) { assert(m->_recursions == 0, "invariant"); current->inc_held_monitor_count(); return true; @@ -658,7 +657,7 @@ void ObjectSynchronizer::exit_legacy(oop object, BasicLock* lock, JavaThread* cu // The ObjectMonitor* can't be async deflated until ownership is // dropped inside exit() and the ObjectMonitor* must be !is_busy(). ObjectMonitor* monitor = inflate(current, object, inflate_cause_vm_internal); - assert(!monitor->is_owner_anonymous(), "must not be"); + assert(!monitor->has_anonymous_owner(), "must not be"); monitor->exit(current); } @@ -666,6 +665,10 @@ void ObjectSynchronizer::exit_legacy(oop object, BasicLock* lock, JavaThread* cu // JNI locks on java objects // NOTE: must use heavy weight monitor to handle jni monitor enter void ObjectSynchronizer::jni_enter(Handle obj, JavaThread* current) { + // Top native frames in the stack will not be seen if we attempt + // preemption, since we start walking from the last Java anchor. + NoPreemptMark npm(current); + if (obj->klass()->is_value_based()) { handle_sync_on_value_based_class(obj, current); } @@ -717,7 +720,7 @@ void ObjectSynchronizer::jni_exit(oop obj, TRAPS) { // ----------------------------------------------------------------------------- // Internal VM locks on java objects // standard constructor, allows locking failures -ObjectLocker::ObjectLocker(Handle obj, JavaThread* thread) { +ObjectLocker::ObjectLocker(Handle obj, JavaThread* thread) : _npm(thread) { _thread = thread; _thread->check_for_valid_safepoint_state(); _obj = obj; @@ -1165,7 +1168,7 @@ JavaThread* ObjectSynchronizer::get_lock_owner(ThreadsList * t_list, Handle h_ob if (LockingMode == LM_LEGACY && mark.has_locker()) { // stack-locked so header points into owner's stack. // owning_thread_from_monitor_owner() may also return null here: - return Threads::owning_thread_from_monitor_owner(t_list, (address) mark.locker()); + return Threads::owning_thread_from_stacklock(t_list, (address) mark.locker()); } if (LockingMode == LM_LIGHTWEIGHT && mark.is_fast_locked()) { @@ -1229,7 +1232,7 @@ void ObjectSynchronizer::owned_monitors_iterate_filtered(MonitorClosure* closure // only interested in an owned ObjectMonitor and ownership // cannot be dropped under the calling contexts so the // ObjectMonitor cannot be async deflated. - if (monitor->has_owner() && filter(monitor->owner_raw())) { + if (monitor->has_owner() && filter(monitor)) { assert(!monitor->is_being_async_deflated(), "Owned monitors should not be deflating"); closure->do_monitor(monitor); @@ -1240,13 +1243,20 @@ void ObjectSynchronizer::owned_monitors_iterate_filtered(MonitorClosure* closure // Iterate ObjectMonitors where the owner == thread; this does NOT include // ObjectMonitors where owner is set to a stack-lock address in thread. void ObjectSynchronizer::owned_monitors_iterate(MonitorClosure* closure, JavaThread* thread) { - auto thread_filter = [&](void* owner) { return owner == thread; }; + int64_t key = ObjectMonitor::owner_from(thread); + auto thread_filter = [&](ObjectMonitor* monitor) { return monitor->owner() == key; }; + return owned_monitors_iterate_filtered(closure, thread_filter); +} + +void ObjectSynchronizer::owned_monitors_iterate(MonitorClosure* closure, oop vthread) { + int64_t key = ObjectMonitor::owner_from(vthread); + auto thread_filter = [&](ObjectMonitor* monitor) { return monitor->owner() == key; }; return owned_monitors_iterate_filtered(closure, thread_filter); } // Iterate ObjectMonitors owned by any thread. void ObjectSynchronizer::owned_monitors_iterate(MonitorClosure* closure) { - auto all_filter = [&](void* owner) { return true; }; + auto all_filter = [&](ObjectMonitor* monitor) { return true; }; return owned_monitors_iterate_filtered(closure, all_filter); } @@ -1418,16 +1428,22 @@ void ObjectSynchronizer::inflate_helper(oop obj) { ObjectMonitor* ObjectSynchronizer::inflate(Thread* current, oop obj, const InflateCause cause) { assert(current == Thread::current(), "must be"); assert(LockingMode != LM_LIGHTWEIGHT, "only inflate through enter"); - return inflate_impl(obj, cause); + return inflate_impl(current->is_Java_thread() ? JavaThread::cast(current) : nullptr, obj, cause); } ObjectMonitor* ObjectSynchronizer::inflate_for(JavaThread* thread, oop obj, const InflateCause cause) { assert(thread == Thread::current() || thread->is_obj_deopt_suspend(), "must be"); assert(LockingMode != LM_LIGHTWEIGHT, "LM_LIGHTWEIGHT cannot use inflate_for"); - return inflate_impl(obj, cause); + return inflate_impl(thread, obj, cause); } -ObjectMonitor* ObjectSynchronizer::inflate_impl(oop object, const InflateCause cause) { +ObjectMonitor* ObjectSynchronizer::inflate_impl(JavaThread* locking_thread, oop object, const InflateCause cause) { + // The JavaThread* locking_thread requires that the locking_thread == Thread::current() or + // is suspended throughout the call by some other mechanism. + // The thread might be nullptr when called from a non JavaThread. (As may still be + // the case from FastHashCode). However it is only important for correctness that the + // thread is set when called from ObjectSynchronizer::enter from the owning thread, + // ObjectSynchronizer::enter_for from any thread, or ObjectSynchronizer::exit. assert(LockingMode != LM_LIGHTWEIGHT, "LM_LIGHTWEIGHT cannot use inflate_impl"); EventJavaMonitorInflate event; @@ -1435,7 +1451,9 @@ ObjectMonitor* ObjectSynchronizer::inflate_impl(oop object, const InflateCause c const markWord mark = object->mark_acquire(); // The mark can be in one of the following states: - // * inflated - Just return it. + // * inflated - If the ObjectMonitor owner is anonymous and the + // locking_thread owns the object lock, then we + // make the locking_thread the ObjectMonitor owner. // * stack-locked - Coerce it to inflated from stack-locked. // * INFLATING - Busy wait for conversion from stack-locked to // inflated. @@ -1446,6 +1464,13 @@ ObjectMonitor* ObjectSynchronizer::inflate_impl(oop object, const InflateCause c ObjectMonitor* inf = mark.monitor(); markWord dmw = inf->header(); assert(dmw.is_neutral(), "invariant: header=" INTPTR_FORMAT, dmw.value()); + if (inf->has_anonymous_owner() && locking_thread != nullptr) { + assert(LockingMode == LM_LEGACY, "invariant"); + if (locking_thread->is_lock_owned((address)inf->stack_locker())) { + inf->set_stack_locker(nullptr); + inf->set_owner_from_anonymous(locking_thread); + } + } return inf; } @@ -1522,12 +1547,17 @@ ObjectMonitor* ObjectSynchronizer::inflate_impl(oop object, const InflateCause c // Setup monitor fields to proper values -- prepare the monitor m->set_header(dmw); - // Optimization: if the mark.locker stack address is associated - // with this thread we could simply set m->_owner = current. // Note that a thread can inflate an object // that it has stack-locked -- as might happen in wait() -- directly // with CAS. That is, we can avoid the xchg-nullptr .... ST idiom. - m->set_owner_from(nullptr, mark.locker()); + if (locking_thread != nullptr && locking_thread->is_lock_owned((address)mark.locker())) { + m->set_owner(locking_thread); + } else { + // Use ANONYMOUS_OWNER to indicate that the owner is the BasicLock on the stack, + // and set the stack locker field in the monitor. + m->set_stack_locker(mark.locker()); + m->set_anonymous_owner(); + } // TODO-FIXME: assert BasicLock->dhw != 0. // Must preserve store ordering. The monitor state must @@ -2050,7 +2080,7 @@ void ObjectSynchronizer::log_in_use_monitor_details(outputStream* out, bool log_ const intptr_t hash = UseObjectMonitorTable ? monitor->hash() : monitor->header().hash(); ResourceMark rm; out->print(INTPTR_FORMAT " %d%d%d " INTPTR_FORMAT " %s", p2i(monitor), - monitor->is_busy(), hash != 0, monitor->owner() != nullptr, + monitor->is_busy(), hash != 0, monitor->has_owner(), p2i(obj), obj == nullptr ? "" : obj->klass()->external_name()); if (monitor->is_busy()) { out->print(" (%s)", monitor->is_busy_to_string(&ss)); diff --git a/src/hotspot/share/runtime/synchronizer.hpp b/src/hotspot/share/runtime/synchronizer.hpp index 5b171560ee1..d3d1f4ee4ce 100644 --- a/src/hotspot/share/runtime/synchronizer.hpp +++ b/src/hotspot/share/runtime/synchronizer.hpp @@ -137,7 +137,7 @@ class ObjectSynchronizer : AllStatic { private: // Shared implementation between the different LockingMode. - static ObjectMonitor* inflate_impl(oop obj, const InflateCause cause); + static ObjectMonitor* inflate_impl(JavaThread* locking_thread, oop obj, const InflateCause cause); public: // This version is only for internal use @@ -168,10 +168,13 @@ class ObjectSynchronizer : AllStatic { template static void owned_monitors_iterate_filtered(MonitorClosure* closure, OwnerFilter filter); - // Iterate ObjectMonitors where the owner == thread; this does NOT include - // ObjectMonitors where owner is set to a stack lock address in thread. + // Iterate ObjectMonitors where the owner is thread; this does NOT include + // ObjectMonitors where the owner is anonymous. static void owned_monitors_iterate(MonitorClosure* m, JavaThread* thread); + // Iterate ObjectMonitors where the owner is vthread. + static void owned_monitors_iterate(MonitorClosure* m, oop vthread); + // Iterate ObjectMonitors owned by any thread. static void owned_monitors_iterate(MonitorClosure* closure); @@ -229,11 +232,15 @@ class ObjectSynchronizer : AllStatic { // have to pass through, and we must also be able to deal with // asynchronous exceptions. The caller is responsible for checking // the thread's pending exception if needed. +// When using ObjectLocker the top native frames in the stack will +// not be seen in case we attempt preemption, since we start walking +// from the last Java anchor, so we disable it with NoPreemptMark. class ObjectLocker : public StackObj { private: JavaThread* _thread; Handle _obj; BasicLock _lock; + NoPreemptMark _npm; public: ObjectLocker(Handle obj, JavaThread* current); ~ObjectLocker(); diff --git a/src/hotspot/share/runtime/threadIdentifier.cpp b/src/hotspot/share/runtime/threadIdentifier.cpp index ab250bbf6c3..d63033e7786 100644 --- a/src/hotspot/share/runtime/threadIdentifier.cpp +++ b/src/hotspot/share/runtime/threadIdentifier.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2022, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -26,12 +26,22 @@ #include "runtime/atomic.hpp" #include "runtime/threadIdentifier.hpp" -static volatile int64_t next_thread_id = 2; // starting at 2, excluding the primordial thread id +// starting at 3, excluding reserved values defined in ObjectMonitor.hpp +static const int64_t INITIAL_TID = 3; +static volatile int64_t next_thread_id = INITIAL_TID; + +int64_t ThreadIdentifier::initial() { + return INITIAL_TID; +} int64_t ThreadIdentifier::unsafe_offset() { return reinterpret_cast(&next_thread_id); } +int64_t ThreadIdentifier::current() { + return Atomic::load(&next_thread_id); +} + int64_t ThreadIdentifier::next() { int64_t next_tid; do { diff --git a/src/hotspot/share/runtime/threadIdentifier.hpp b/src/hotspot/share/runtime/threadIdentifier.hpp index 44d34d88091..b4803ae22f1 100644 --- a/src/hotspot/share/runtime/threadIdentifier.hpp +++ b/src/hotspot/share/runtime/threadIdentifier.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2022, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -36,7 +36,9 @@ class ThreadIdentifier : AllStatic { public: static int64_t next(); + static int64_t current(); static int64_t unsafe_offset(); + static int64_t initial(); }; #endif // SHARE_RUNTIME_THREADIDENTIFIER_HPP diff --git a/src/hotspot/share/runtime/threads.cpp b/src/hotspot/share/runtime/threads.cpp index 76ee237ae5e..29386a0c368 100644 --- a/src/hotspot/share/runtime/threads.cpp +++ b/src/hotspot/share/runtime/threads.cpp @@ -166,8 +166,10 @@ static void create_initial_thread(Handle thread_group, JavaThread* thread, string, CHECK); - JFR_ONLY(assert(JFR_JVM_THREAD_ID(thread) == static_cast(java_lang_Thread::thread_id(thread_oop())), - "initial tid mismatch");) + DEBUG_ONLY(int64_t main_thread_tid = java_lang_Thread::thread_id(thread_oop());) + assert(main_thread_tid == ThreadIdentifier::initial(), ""); + assert(main_thread_tid == thread->lock_id(), ""); + JFR_ONLY(assert(JFR_JVM_THREAD_ID(thread) == static_cast(main_thread_tid), "initial tid mismatch");) // Set thread status to running since main thread has // been started and running. @@ -535,6 +537,10 @@ jint Threads::create_vm(JavaVMInitArgs* args, bool* canTryAgain) { main_thread->set_active_handles(JNIHandleBlock::allocate_block()); MACOS_AARCH64_ONLY(main_thread->init_wx()); + // Set the lock_id now since we will run Java code before the Thread instance + // is even created. The same value will be assigned to the Thread instance on init. + main_thread->set_lock_id(ThreadIdentifier::next()); + if (!Thread::set_as_starting_thread(main_thread)) { vm_shutdown_during_initialization( "Failed necessary internal allocation. Out of swap space"); @@ -586,6 +592,8 @@ jint Threads::create_vm(JavaVMInitArgs* args, bool* canTryAgain) { return status; } + ObjectMonitor::Initialize2(); + JFR_ONLY(Jfr::on_create_vm_1();) // Should be done after the heap is fully created @@ -1231,35 +1239,16 @@ GrowableArray* Threads::get_pending_threads(ThreadsList * t_list, } #endif // INCLUDE_JVMTI -JavaThread *Threads::owning_thread_from_monitor_owner(ThreadsList * t_list, - address owner) { - assert(LockingMode != LM_LIGHTWEIGHT, "Not with new lightweight locking"); - // null owner means not locked so we can skip the search - if (owner == nullptr) return nullptr; - - for (JavaThread* p : *t_list) { - // first, see if owner is the address of a Java thread - if (owner == (address)p) return p; - } +JavaThread *Threads::owning_thread_from_stacklock(ThreadsList * t_list, address basicLock) { + assert(LockingMode == LM_LEGACY, "Not with new lightweight locking"); - // Cannot assert on lack of success here since this function may be - // used by code that is trying to report useful problem information - // like deadlock detection. - if (LockingMode == LM_MONITOR) return nullptr; - - // If we didn't find a matching Java thread and we didn't force use of - // heavyweight monitors, then the owner is the stack address of the - // Lock Word in the owning Java thread's stack. - // JavaThread* the_owner = nullptr; for (JavaThread* q : *t_list) { - if (q->is_lock_owned(owner)) { + if (q->is_lock_owned(basicLock)) { the_owner = q; break; } } - - // cannot assert on lack of success here; see above comment return the_owner; } @@ -1280,17 +1269,22 @@ JavaThread* Threads::owning_thread_from_object(ThreadsList * t_list, oop obj) { } JavaThread* Threads::owning_thread_from_monitor(ThreadsList* t_list, ObjectMonitor* monitor) { - if (LockingMode == LM_LIGHTWEIGHT) { - if (monitor->is_owner_anonymous()) { + if (monitor->has_anonymous_owner()) { + if (LockingMode == LM_LIGHTWEIGHT) { return owning_thread_from_object(t_list, monitor->object()); } else { - Thread* owner = reinterpret_cast(monitor->owner()); - assert(owner == nullptr || owner->is_Java_thread(), "only JavaThreads own monitors"); - return reinterpret_cast(owner); + assert(LockingMode == LM_LEGACY, "invariant"); + return owning_thread_from_stacklock(t_list, (address)monitor->stack_locker()); } } else { - address owner = (address)monitor->owner(); - return owning_thread_from_monitor_owner(t_list, owner); + JavaThread* the_owner = nullptr; + for (JavaThread* q : *t_list) { + if (monitor->has_owner(q)) { + the_owner = q; + break; + } + } + return the_owner; } } @@ -1342,17 +1336,10 @@ void Threads::print_on(outputStream* st, bool print_stacks, p->trace_stack(); } else { p->print_stack_on(st); - const oop thread_oop = p->threadObj(); - if (thread_oop != nullptr) { - if (p->is_vthread_mounted()) { - const oop vt = p->vthread(); - assert(vt != nullptr, "vthread should not be null when vthread is mounted"); - // JavaThread._vthread can refer to the carrier thread. Print only if _vthread refers to a virtual thread. - if (vt != thread_oop) { - st->print_cr(" Mounted virtual thread #" INT64_FORMAT, (int64_t)java_lang_Thread::thread_id(vt)); - p->print_vthread_stack_on(st); - } - } + if (p->is_vthread_mounted()) { + // _lock_id is the thread ID of the mounted virtual thread + st->print_cr(" Mounted virtual thread #" INT64_FORMAT, p->lock_id()); + p->print_vthread_stack_on(st); } } } diff --git a/src/hotspot/share/runtime/threads.hpp b/src/hotspot/share/runtime/threads.hpp index 39e307122c8..979b1dffe26 100644 --- a/src/hotspot/share/runtime/threads.hpp +++ b/src/hotspot/share/runtime/threads.hpp @@ -134,9 +134,8 @@ class Threads: AllStatic { static GrowableArray* get_pending_threads(ThreadsList * t_list, int count, address monitor); - // Get owning Java thread from the monitor's owner field. - static JavaThread *owning_thread_from_monitor_owner(ThreadsList * t_list, - address owner); + // Get owning Java thread from the basicLock address. + static JavaThread *owning_thread_from_stacklock(ThreadsList * t_list, address basicLock); static JavaThread* owning_thread_from_object(ThreadsList* t_list, oop obj); static JavaThread* owning_thread_from_monitor(ThreadsList* t_list, ObjectMonitor* owner); diff --git a/src/hotspot/share/runtime/timerTrace.hpp b/src/hotspot/share/runtime/timerTrace.hpp index 8c7cdc8399b..ae632c0b46d 100644 --- a/src/hotspot/share/runtime/timerTrace.hpp +++ b/src/hotspot/share/runtime/timerTrace.hpp @@ -67,6 +67,8 @@ class TraceTime: public StackObj { TraceTimerLogPrintFunc ttlpf); ~TraceTime(); + + const char* title() const { return _title; } }; diff --git a/src/hotspot/share/runtime/vframe.cpp b/src/hotspot/share/runtime/vframe.cpp index d93b380ce92..41a46c86a70 100644 --- a/src/hotspot/share/runtime/vframe.cpp +++ b/src/hotspot/share/runtime/vframe.cpp @@ -280,13 +280,14 @@ intptr_t* interpretedVFrame::locals_addr_at(int offset) const { } GrowableArray* interpretedVFrame::monitors() const { + bool heap_frame = stack_chunk() != nullptr; + frame f = !heap_frame ? _fr : stack_chunk()->derelativize(_fr); GrowableArray* result = new GrowableArray(5); - if (stack_chunk() == nullptr) { // no monitors in continuations - for (BasicObjectLock* current = (fr().previous_monitor_in_interpreter_frame(fr().interpreter_frame_monitor_begin())); - current >= fr().interpreter_frame_monitor_end(); - current = fr().previous_monitor_in_interpreter_frame(current)) { - result->push(new MonitorInfo(current->obj(), current->lock(), false, false)); - } + for (BasicObjectLock* current = (f.previous_monitor_in_interpreter_frame(f.interpreter_frame_monitor_begin())); + current >= f.interpreter_frame_monitor_end(); + current = f.previous_monitor_in_interpreter_frame(current)) { + oop owner = !heap_frame ? current->obj() : StackValue::create_stack_value_from_oop_location(stack_chunk(), (void*)current->obj_adr())->get_obj()(); + result->push(new MonitorInfo(owner, current->lock(), false, false)); } return result; } diff --git a/src/hotspot/share/runtime/vframe.inline.hpp b/src/hotspot/share/runtime/vframe.inline.hpp index 4630e695ce9..e7f9edbc232 100644 --- a/src/hotspot/share/runtime/vframe.inline.hpp +++ b/src/hotspot/share/runtime/vframe.inline.hpp @@ -122,6 +122,13 @@ inline vframeStream::vframeStream(JavaThread* thread, bool stop_at_java_call_stu if (thread->is_vthread_mounted()) { _frame = vthread_carrier ? _thread->carrier_last_frame(&_reg_map) : _thread->vthread_last_frame(); + if (Continuation::is_continuation_enterSpecial(_frame)) { + // This can happen when calling async_get_stack_trace() and catching the target + // vthread at the JRT_BLOCK_END in freeze_internal() or when posting the Monitor + // Waited event after target vthread was preempted. Since all continuation frames + // are freezed we get the top frame from the stackChunk instead. + _frame = Continuation::last_frame(java_lang_VirtualThread::continuation(_thread->vthread()), &_reg_map); + } } else { _frame = _thread->last_frame(); } diff --git a/src/hotspot/share/runtime/vframe_hp.cpp b/src/hotspot/share/runtime/vframe_hp.cpp index 28b62c60fd9..49ed47835d6 100644 --- a/src/hotspot/share/runtime/vframe_hp.cpp +++ b/src/hotspot/share/runtime/vframe_hp.cpp @@ -235,7 +235,7 @@ StackValue *compiledVFrame::create_stack_value(ScopeValue *sv) const { } BasicLock* compiledVFrame::resolve_monitor_lock(Location location) const { - return StackValue::resolve_monitor_lock(&_fr, location); + return StackValue::resolve_monitor_lock(stack_chunk() == nullptr ? _fr : stack_chunk()->derelativize(_fr), location); } @@ -284,6 +284,7 @@ GrowableArray* compiledVFrame::monitors() const { // Replace the original values with any stores that have been // performed through compiledVFrame::update_monitors. + if (thread() == nullptr) return result; // Unmounted continuations have no thread so nothing to do. GrowableArrayView* list = JvmtiDeferredUpdates::deferred_locals(thread()); if (list != nullptr ) { // In real life this never happens or is typically a single element search diff --git a/src/hotspot/share/runtime/vmOperation.hpp b/src/hotspot/share/runtime/vmOperation.hpp index eede52f00d5..50d85944485 100644 --- a/src/hotspot/share/runtime/vmOperation.hpp +++ b/src/hotspot/share/runtime/vmOperation.hpp @@ -114,7 +114,8 @@ template(GTestStopSafepoint) \ template(JFROldObject) \ template(JvmtiPostObjectFree) \ - template(RendezvousGCThreads) + template(RendezvousGCThreads) \ + template(ReinitializeMDO) class Thread; class outputStream; diff --git a/src/hotspot/share/runtime/vmOperations.cpp b/src/hotspot/share/runtime/vmOperations.cpp index af9911e0fdc..1f5d63a917e 100644 --- a/src/hotspot/share/runtime/vmOperations.cpp +++ b/src/hotspot/share/runtime/vmOperations.cpp @@ -275,13 +275,13 @@ void VM_ThreadDump::doit_epilogue() { } } -// Hash table of void* to a list of ObjectMonitor* owned by the JavaThread. +// Hash table of int64_t to a list of ObjectMonitor* owned by the JavaThread. // The JavaThread's owner key is either a JavaThread* or a stack lock -// address in the JavaThread so we use "void*". +// address in the JavaThread so we use "int64_t". // class ObjectMonitorsDump : public MonitorClosure, public ObjectMonitorsView { private: - static unsigned int ptr_hash(void* const& s1) { + static unsigned int ptr_hash(int64_t const& s1) { // 2654435761 = 2^32 * Phi (golden ratio) return (unsigned int)(((uint32_t)(uintptr_t)s1) * 2654435761u); } @@ -294,24 +294,24 @@ class ObjectMonitorsDump : public MonitorClosure, public ObjectMonitorsView { // ResourceHashtable SIZE is specified at compile time so we // use 1031 which is the first prime after 1024. - typedef ResourceHashtable PtrTable; PtrTable* _ptrs; size_t _key_count; size_t _om_count; - void add_list(void* key, ObjectMonitorLinkedList* list) { + void add_list(int64_t key, ObjectMonitorLinkedList* list) { _ptrs->put(key, list); _key_count++; } - ObjectMonitorLinkedList* get_list(void* key) { + ObjectMonitorLinkedList* get_list(int64_t key) { ObjectMonitorLinkedList** listpp = _ptrs->get(key); return (listpp == nullptr) ? nullptr : *listpp; } void add(ObjectMonitor* monitor) { - void* key = monitor->owner(); + int64_t key = monitor->owner(); ObjectMonitorLinkedList* list = get_list(key); if (list == nullptr) { @@ -335,7 +335,7 @@ class ObjectMonitorsDump : public MonitorClosure, public ObjectMonitorsView { ~ObjectMonitorsDump() { class CleanupObjectMonitorsDump: StackObj { public: - bool do_entry(void*& key, ObjectMonitorLinkedList*& list) { + bool do_entry(int64_t& key, ObjectMonitorLinkedList*& list) { list->clear(); // clear the LinkListNodes delete list; // then delete the LinkedList return true; @@ -350,7 +350,7 @@ class ObjectMonitorsDump : public MonitorClosure, public ObjectMonitorsView { void do_monitor(ObjectMonitor* monitor) override { assert(monitor->has_owner(), "Expects only owned monitors"); - if (monitor->is_owner_anonymous()) { + if (monitor->has_anonymous_owner()) { // There's no need to collect anonymous owned monitors // because the caller of this code is only interested // in JNI owned monitors. @@ -368,7 +368,8 @@ class ObjectMonitorsDump : public MonitorClosure, public ObjectMonitorsView { // Implements the ObjectMonitorsView interface void visit(MonitorClosure* closure, JavaThread* thread) override { - ObjectMonitorLinkedList* list = get_list(thread); + int64_t key = ObjectMonitor::owner_from(thread); + ObjectMonitorLinkedList* list = get_list(key); LinkedListIterator iter(list != nullptr ? list->head() : nullptr); while (!iter.is_empty()) { ObjectMonitor* monitor = *iter.next(); diff --git a/src/hotspot/share/runtime/vmStructs.cpp b/src/hotspot/share/runtime/vmStructs.cpp index 3060e225427..bc941534242 100644 --- a/src/hotspot/share/runtime/vmStructs.cpp +++ b/src/hotspot/share/runtime/vmStructs.cpp @@ -665,6 +665,7 @@ nonstatic_field(JavaThread, _vframe_array_head, vframeArray*) \ nonstatic_field(JavaThread, _vframe_array_last, vframeArray*) \ nonstatic_field(JavaThread, _active_handles, JNIHandleBlock*) \ + nonstatic_field(JavaThread, _lock_id, int64_t) \ volatile_nonstatic_field(JavaThread, _terminated, JavaThread::TerminatedTypes) \ nonstatic_field(Thread, _resource_area, ResourceArea*) \ nonstatic_field(CompilerThread, _env, ciEnv*) \ @@ -785,7 +786,8 @@ \ volatile_nonstatic_field(ObjectMonitor, _metadata, uintptr_t) \ unchecked_nonstatic_field(ObjectMonitor, _object, sizeof(void *)) /* NOTE: no type */ \ - unchecked_nonstatic_field(ObjectMonitor, _owner, sizeof(void *)) /* NOTE: no type */ \ + volatile_nonstatic_field(ObjectMonitor, _owner, int64_t) \ + volatile_nonstatic_field(ObjectMonitor, _stack_locker, BasicLock*) \ volatile_nonstatic_field(ObjectMonitor, _next_om, ObjectMonitor*) \ volatile_nonstatic_field(BasicLock, _metadata, uintptr_t) \ nonstatic_field(ObjectMonitor, _contentions, int) \ @@ -2004,8 +2006,6 @@ declare_constant(BytesPerWord) \ declare_constant(BytesPerLong) \ \ - declare_constant(LogKlassAlignmentInBytes) \ - \ declare_constant(HeapWordSize) \ declare_constant(LogHeapWordSize) \ \ @@ -2510,6 +2510,7 @@ declare_constant(markWord::lock_shift) \ declare_constant(markWord::age_shift) \ declare_constant(markWord::hash_shift) \ + LP64_ONLY(declare_constant(markWord::klass_shift)) \ \ declare_constant(markWord::lock_mask) \ declare_constant(markWord::lock_mask_in_place) \ @@ -2533,7 +2534,9 @@ declare_constant(InvocationCounter::count_shift) \ \ /* ObjectMonitor constants */ \ + declare_constant(ObjectMonitor::NO_OWNER) \ declare_constant(ObjectMonitor::ANONYMOUS_OWNER) \ + declare_constant(ObjectMonitor::DEFLATER_MARKER) \ //-------------------------------------------------------------------------------- // diff --git a/src/hotspot/share/services/attachListener.cpp b/src/hotspot/share/services/attachListener.cpp index 90f5930cce0..8cf5c31b0b3 100644 --- a/src/hotspot/share/services/attachListener.cpp +++ b/src/hotspot/share/services/attachListener.cpp @@ -628,15 +628,16 @@ bool AttachOperation::read_request(RequestReader* reader) { buffer_size = (name_length_max + 1) + arg_count_max * (arg_length_max + 1); min_str_count = 1 /*name*/ + arg_count_max; break; - case ATTACH_API_V2: // 000000 + case ATTACH_API_V2: // 000(0)* (any number of arguments) if (AttachListener::get_supported_version() < 2) { - log_error(attach)("Failed to read request: v2 is unsupported ot disabled"); + log_error(attach)("Failed to read request: v2 is unsupported or disabled"); return false; } // read size of the data buffer_size = reader->read_uint(); if (buffer_size < 0) { + log_error(attach)("Failed to read request: negative request size (%d)", buffer_size); return false; } log_debug(attach)("v2 request, data size = %d", buffer_size); @@ -646,7 +647,7 @@ bool AttachOperation::read_request(RequestReader* reader) { log_error(attach)("Failed to read request: too big"); return false; } - // Must contain exact 'buffer_size' bytes. + // Must contain exactly 'buffer_size' bytes. min_read_size = buffer_size; break; default: diff --git a/src/hotspot/share/services/diagnosticFramework.cpp b/src/hotspot/share/services/diagnosticFramework.cpp index 73aa80a950f..16f074c9cb5 100644 --- a/src/hotspot/share/services/diagnosticFramework.cpp +++ b/src/hotspot/share/services/diagnosticFramework.cpp @@ -424,7 +424,7 @@ bool DCmd::reorder_help_cmd(CmdLine line, stringStream &updated_line) { args.print("%s", line.args_addr()); char* rest = args.as_string(); char* token = strtok_r(rest, " ", &rest); - while (token != NULL) { + while (token != nullptr) { if (strcmp(token, "-h") == 0 || strcmp(token, "--help") == 0 || strcmp(token, "-help") == 0) { updated_line.print("%s", "help "); diff --git a/src/hotspot/share/services/heapDumper.cpp b/src/hotspot/share/services/heapDumper.cpp index 10e1a804ad2..91bcce382f0 100644 --- a/src/hotspot/share/services/heapDumper.cpp +++ b/src/hotspot/share/services/heapDumper.cpp @@ -1875,9 +1875,12 @@ void ThreadDumper::dump_stack_refs(AbstractDumpWriter * writer) { // native frame blk.set_frame_number(depth); if (is_top_frame) { - // JNI locals for the top frame. - assert(_java_thread != nullptr, "impossible for unmounted vthread"); - _java_thread->active_handles()->oops_do(&blk); + // JNI locals for the top frame if mounted + assert(_java_thread != nullptr || jvf->method()->is_synchronized() + || jvf->method()->is_object_wait0(), "impossible for unmounted vthread"); + if (_java_thread != nullptr) { + _java_thread->active_handles()->oops_do(&blk); + } } else { if (last_entry_frame != nullptr) { // JNI locals for the entry frame diff --git a/src/hotspot/share/services/threadService.cpp b/src/hotspot/share/services/threadService.cpp index a1ad0a910f6..8cb8bf19bba 100644 --- a/src/hotspot/share/services/threadService.cpp +++ b/src/hotspot/share/services/threadService.cpp @@ -464,23 +464,14 @@ DeadlockCycle* ThreadService::find_deadlocks_at_safepoint(ThreadsList * t_list, } else if (waitingToLockMonitor != nullptr) { if (waitingToLockMonitor->has_owner()) { currentThread = Threads::owning_thread_from_monitor(t_list, waitingToLockMonitor); - if (currentThread == nullptr) { - // This function is called at a safepoint so the JavaThread - // that owns waitingToLockMonitor should be findable, but - // if it is not findable, then the previous currentThread is - // blocked permanently. We record this as a deadlock. - num_deadlocks++; - - // add this cycle to the deadlocks list - if (deadlocks == nullptr) { - deadlocks = cycle; - } else { - last->set_next(cycle); - } - last = cycle; - cycle = new DeadlockCycle(); - break; - } + // If currentThread is null we would like to know if the owner + // is an unmounted vthread (no JavaThread*), because if it's not, + // it would mean the previous currentThread is blocked permanently + // and we should record this as a deadlock. Since there is currently + // no fast way to determine if the owner is indeed an unmounted + // vthread we never record this as a deadlock. Note: unless there + // is a bug in the VM, or a thread exits without releasing monitors + // acquired through JNI, null should imply an unmounted vthread owner. } } else { if (concurrent_locks) { @@ -1053,8 +1044,8 @@ void DeadlockCycle::print_on_with(ThreadsList * t_list, outputStream* st) const // that owns waitingToLockMonitor should be findable, but // if it is not findable, then the previous currentThread is // blocked permanently. - st->print_cr("%s UNKNOWN_owner_addr=" PTR_FORMAT, owner_desc, - p2i(waitingToLockMonitor->owner())); + st->print_cr("%s UNKNOWN_owner_addr=" INT64_FORMAT, owner_desc, + waitingToLockMonitor->owner()); continue; } } else { diff --git a/src/hotspot/share/utilities/concurrentHashTable.inline.hpp b/src/hotspot/share/utilities/concurrentHashTable.inline.hpp index 5656655275a..74491e39ea9 100644 --- a/src/hotspot/share/utilities/concurrentHashTable.inline.hpp +++ b/src/hotspot/share/utilities/concurrentHashTable.inline.hpp @@ -491,7 +491,7 @@ inline void ConcurrentHashTable:: Node* ndel_stack[StackBufferSize]; InternalTable* table = get_table(); assert(start_idx < stop_idx, "Must be"); - assert(stop_idx <= _table->_size, "Must be"); + assert(stop_idx <= _table->_size, "stop_idx %zu larger than table size %zu", stop_idx, _table->_size); // Here manual do critical section since we don't want to take the cost of // locking the bucket if there is nothing to delete. But we can have // concurrent single deletes. The _invisible_epoch can only be used by the @@ -1185,7 +1185,7 @@ inline bool ConcurrentHashTable:: do_scan_for_range(FUNC& scan_f, size_t start_idx, size_t stop_idx, InternalTable* table) { assert(start_idx < stop_idx, "Must be"); - assert(stop_idx <= table->_size, "Must be"); + assert(stop_idx <= table->_size, "stop_idx %zu larger than table size %zu", stop_idx, table->_size); for (size_t bucket_it = start_idx; bucket_it < stop_idx; ++bucket_it) { Bucket* bucket = table->get_bucket(bucket_it); diff --git a/src/hotspot/share/utilities/globalDefinitions_visCPP.hpp b/src/hotspot/share/utilities/globalDefinitions_visCPP.hpp index 3b28328d7dd..237d12baa6d 100644 --- a/src/hotspot/share/utilities/globalDefinitions_visCPP.hpp +++ b/src/hotspot/share/utilities/globalDefinitions_visCPP.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2022, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -48,6 +48,11 @@ #define _USE_MATH_DEFINES # include +// Only 64-bit Windows is supported +#ifndef _LP64 +#error unsupported platform +#endif + // 4810578: varargs unsafe on 32-bit integer/64-bit pointer architectures // When __cplusplus is defined, NULL is defined as 0 (32-bit constant) in // system header files. On 32-bit architectures, there is no problem. @@ -61,17 +66,11 @@ // pointer when it extracts the argument, then we may have a problem. // // Solution: For 64-bit architectures, redefine NULL as 64-bit constant 0. -#ifdef _LP64 #undef NULL // 64-bit Windows uses a P64 data model (not LP64, although we define _LP64) // Since longs are 32-bit we cannot use 0L here. Use the Visual C++ specific // 64-bit integer-suffix (LL) instead. #define NULL 0LL -#else -#ifndef NULL -#define NULL 0 -#endif -#endif // NULL vs NULL_WORD: // On Linux NULL is defined as a special type '__null'. Assigning __null to @@ -79,11 +78,7 @@ // pointer is stored as integer value. #define NULL_WORD NULL -#ifdef _WIN64 typedef int64_t ssize_t; -#else -typedef int32_t ssize_t; -#endif // Non-standard stdlib-like stuff: inline int strcasecmp(const char *s1, const char *s2) { return _stricmp(s1,s2); } @@ -120,13 +115,8 @@ inline int g_isfinite(jdouble f) { return _finite(f); } #endif #ifndef SSIZE_MAX -#ifdef _LP64 #define SSIZE_MIN LLONG_MIN #define SSIZE_MAX LLONG_MAX -#else -#define SSIZE_MIN INT_MIN -#define SSIZE_MAX INT_MAX -#endif #endif // SSIZE_MAX missing #endif // SHARE_UTILITIES_GLOBALDEFINITIONS_VISCPP_HPP diff --git a/src/hotspot/share/utilities/vmError.cpp b/src/hotspot/share/utilities/vmError.cpp index 0df77689a04..23db59f766a 100644 --- a/src/hotspot/share/utilities/vmError.cpp +++ b/src/hotspot/share/utilities/vmError.cpp @@ -717,10 +717,6 @@ void VMError::report(outputStream* st, bool _verbose) { address lastpc = nullptr; BEGIN - if (MemTracker::enabled() && NmtVirtualMemory_lock != nullptr && NmtVirtualMemory_lock->owned_by_self()) { - // Manually unlock to avoid reentrancy due to mallocs in detailed mode. - NmtVirtualMemory_lock->unlock(); - } STEP("printing fatal error message") st->print_cr("#"); diff --git a/src/java.base/linux/classes/sun/nio/ch/EPollSelectorImpl.java b/src/java.base/linux/classes/sun/nio/ch/EPollSelectorImpl.java index a07a927e635..14f4e87448b 100644 --- a/src/java.base/linux/classes/sun/nio/ch/EPollSelectorImpl.java +++ b/src/java.base/linux/classes/sun/nio/ch/EPollSelectorImpl.java @@ -35,7 +35,6 @@ import java.util.Map; import java.util.concurrent.TimeUnit; import java.util.function.Consumer; -import jdk.internal.misc.Blocker; import static sun.nio.ch.EPoll.EPOLLIN; import static sun.nio.ch.EPoll.EPOLL_CTL_ADD; @@ -105,36 +104,71 @@ protected int doSelect(Consumer action, long timeout) int numEntries; processUpdateQueue(); processDeregisterQueue(); - try { - begin(blocking); - do { - long startTime = timedPoll ? System.nanoTime() : 0; - boolean attempted = Blocker.begin(blocking); - try { + if (Thread.currentThread().isVirtual()) { + numEntries = (timedPoll) + ? timedPoll(TimeUnit.MILLISECONDS.toNanos(to)) + : untimedPoll(blocking); + } else { + try { + begin(blocking); + do { + long startTime = timedPoll ? System.nanoTime() : 0; numEntries = EPoll.wait(epfd, pollArrayAddress, NUM_EPOLLEVENTS, to); - } finally { - Blocker.end(attempted); - } - if (numEntries == IOStatus.INTERRUPTED && timedPoll) { - // timed poll interrupted so need to adjust timeout - long adjust = System.nanoTime() - startTime; - to -= (int) TimeUnit.NANOSECONDS.toMillis(adjust); - if (to <= 0) { - // timeout expired so no retry - numEntries = 0; + if (numEntries == IOStatus.INTERRUPTED && timedPoll) { + // timed poll interrupted so need to adjust timeout + long adjust = System.nanoTime() - startTime; + to -= (int) TimeUnit.NANOSECONDS.toMillis(adjust); + if (to <= 0) { + // timeout expired so no retry + numEntries = 0; + } } - } - } while (numEntries == IOStatus.INTERRUPTED); - assert IOStatus.check(numEntries); - - } finally { - end(blocking); + } while (numEntries == IOStatus.INTERRUPTED); + } finally { + end(blocking); + } } + assert IOStatus.check(numEntries); + processDeregisterQueue(); return processEvents(numEntries, action); } + /** + * If blocking, parks the current virtual thread until a file descriptor is polled + * or the thread is interrupted. + */ + private int untimedPoll(boolean block) throws IOException { + int numEntries = EPoll.wait(epfd, pollArrayAddress, NUM_EPOLLEVENTS, 0); + if (block) { + while (numEntries == 0 && !Thread.currentThread().isInterrupted()) { + Poller.pollSelector(epfd, 0); + numEntries = EPoll.wait(epfd, pollArrayAddress, NUM_EPOLLEVENTS, 0); + } + } + return numEntries; + } + + /** + * Parks the current virtual thread until a file descriptor is polled, or the thread + * is interrupted, for up to the specified waiting time. + */ + private int timedPoll(long nanos) throws IOException { + long startNanos = System.nanoTime(); + int numEntries = EPoll.wait(epfd, pollArrayAddress, NUM_EPOLLEVENTS, 0); + while (numEntries == 0 && !Thread.currentThread().isInterrupted()) { + long remainingNanos = nanos - (System.nanoTime() - startNanos); + if (remainingNanos <= 0) { + // timeout + break; + } + Poller.pollSelector(epfd, remainingNanos); + numEntries = EPoll.wait(epfd, pollArrayAddress, NUM_EPOLLEVENTS, 0); + } + return numEntries; + } + /** * Process changes to the interest ops. */ diff --git a/src/java.base/macosx/classes/sun/nio/ch/KQueueSelectorImpl.java b/src/java.base/macosx/classes/sun/nio/ch/KQueueSelectorImpl.java index 6abc9e31879..b1a781d7144 100644 --- a/src/java.base/macosx/classes/sun/nio/ch/KQueueSelectorImpl.java +++ b/src/java.base/macosx/classes/sun/nio/ch/KQueueSelectorImpl.java @@ -35,7 +35,6 @@ import java.util.Map; import java.util.concurrent.TimeUnit; import java.util.function.Consumer; -import jdk.internal.misc.Blocker; import static sun.nio.ch.KQueue.EVFILT_READ; import static sun.nio.ch.KQueue.EVFILT_WRITE; @@ -109,36 +108,71 @@ protected int doSelect(Consumer action, long timeout) int numEntries; processUpdateQueue(); processDeregisterQueue(); - try { - begin(blocking); - do { - long startTime = timedPoll ? System.nanoTime() : 0; - boolean attempted = Blocker.begin(blocking); - try { + if (Thread.currentThread().isVirtual()) { + numEntries = (timedPoll) + ? timedPoll(TimeUnit.MILLISECONDS.toNanos(to)) + : untimedPoll(blocking); + } else { + try { + begin(blocking); + do { + long startTime = timedPoll ? System.nanoTime() : 0; numEntries = KQueue.poll(kqfd, pollArrayAddress, MAX_KEVENTS, to); - } finally { - Blocker.end(attempted); - } - if (numEntries == IOStatus.INTERRUPTED && timedPoll) { - // timed poll interrupted so need to adjust timeout - long adjust = System.nanoTime() - startTime; - to -= TimeUnit.NANOSECONDS.toMillis(adjust); - if (to <= 0) { - // timeout expired so no retry - numEntries = 0; + if (numEntries == IOStatus.INTERRUPTED && timedPoll) { + // timed poll interrupted so need to adjust timeout + long adjust = System.nanoTime() - startTime; + to -= TimeUnit.NANOSECONDS.toMillis(adjust); + if (to <= 0) { + // timeout expired so no retry + numEntries = 0; + } } - } - } while (numEntries == IOStatus.INTERRUPTED); - assert IOStatus.check(numEntries); - - } finally { - end(blocking); + } while (numEntries == IOStatus.INTERRUPTED); + } finally { + end(blocking); + } } + assert IOStatus.check(numEntries); + processDeregisterQueue(); return processEvents(numEntries, action); } + /** + * If blocking, parks the current virtual thread until a file descriptor is polled + * or the thread is interrupted. + */ + private int untimedPoll(boolean block) throws IOException { + int numEntries = KQueue.poll(kqfd, pollArrayAddress, MAX_KEVENTS, 0); + if (block) { + while (numEntries == 0 && !Thread.currentThread().isInterrupted()) { + Poller.pollSelector(kqfd, 0); + numEntries = KQueue.poll(kqfd, pollArrayAddress, MAX_KEVENTS, 0); + } + } + return numEntries; + } + + /** + * Parks the current virtual thread until a file descriptor is polled, or the thread + * is interrupted, for up to the specified waiting time. + */ + private int timedPoll(long nanos) throws IOException { + long startNanos = System.nanoTime(); + int numEntries = KQueue.poll(kqfd, pollArrayAddress, MAX_KEVENTS, 0); + while (numEntries == 0 && !Thread.currentThread().isInterrupted()) { + long remainingNanos = nanos - (System.nanoTime() - startNanos); + if (remainingNanos <= 0) { + // timeout + break; + } + Poller.pollSelector(kqfd, remainingNanos); + numEntries = KQueue.poll(kqfd, pollArrayAddress, MAX_KEVENTS, 0); + } + return numEntries; + } + /** * Process changes to the interest ops. */ diff --git a/src/java.base/macosx/native/libjava/java_props_macosx.c b/src/java.base/macosx/native/libjava/java_props_macosx.c index f50daac13cb..07bc2be9d54 100644 --- a/src/java.base/macosx/native/libjava/java_props_macosx.c +++ b/src/java.base/macosx/native/libjava/java_props_macosx.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 1998, 2022, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1998, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -222,54 +222,35 @@ char *setupMacOSXLocale(int cat) { } } -// 10.9 SDK does not include the NSOperatingSystemVersion struct. -// For now, create our own -typedef struct { - NSInteger majorVersion; - NSInteger minorVersion; - NSInteger patchVersion; -} OSVerStruct; - void setOSNameAndVersion(java_props_t *sprops) { // Hardcode os_name, and fill in os_version sprops->os_name = strdup("Mac OS X"); NSString *nsVerStr = NULL; char* osVersionCStr = NULL; - // Mac OS 10.9 includes the [NSProcessInfo operatingSystemVersion] function, - // but it's not in the 10.9 SDK. So, call it via NSInvocation. - if ([[NSProcessInfo processInfo] respondsToSelector:@selector(operatingSystemVersion)]) { - OSVerStruct osVer; - NSMethodSignature *sig = [[NSProcessInfo processInfo] methodSignatureForSelector: - @selector(operatingSystemVersion)]; - NSInvocation *invoke = [NSInvocation invocationWithMethodSignature:sig]; - invoke.selector = @selector(operatingSystemVersion); - [invoke invokeWithTarget:[NSProcessInfo processInfo]]; - [invoke getReturnValue:&osVer]; - - // Copy out the char* if running on version other than 10.16 Mac OS (10.16 == 11.x) - // or explicitly requesting version compatibility - if (!((long)osVer.majorVersion == 10 && (long)osVer.minorVersion >= 16) || - (getenv("SYSTEM_VERSION_COMPAT") != NULL)) { - if (osVer.patchVersion == 0) { // Omit trailing ".0" - nsVerStr = [NSString stringWithFormat:@"%ld.%ld", - (long)osVer.majorVersion, (long)osVer.minorVersion]; - } else { - nsVerStr = [NSString stringWithFormat:@"%ld.%ld.%ld", - (long)osVer.majorVersion, (long)osVer.minorVersion, (long)osVer.patchVersion]; - } + NSOperatingSystemVersion osVer = [[NSProcessInfo processInfo] operatingSystemVersion]; + // Copy out the char* if running on version other than 10.16 Mac OS (10.16 == 11.x) + // or explicitly requesting version compatibility + if (!((long)osVer.majorVersion == 10 && (long)osVer.minorVersion >= 16) || + (getenv("SYSTEM_VERSION_COMPAT") != NULL)) { + if (osVer.patchVersion == 0) { // Omit trailing ".0" + nsVerStr = [NSString stringWithFormat:@"%ld.%ld", + (long)osVer.majorVersion, (long)osVer.minorVersion]; } else { - // Version 10.16, without explicit env setting of SYSTEM_VERSION_COMPAT - // AKA 11+ Read the *real* ProductVersion from the hidden link to avoid SYSTEM_VERSION_COMPAT - // If not found, fallback below to the SystemVersion.plist - NSDictionary *version = [NSDictionary dictionaryWithContentsOfFile : - @"/System/Library/CoreServices/.SystemVersionPlatform.plist"]; - if (version != NULL) { - nsVerStr = [version objectForKey : @"ProductVersion"]; - } + nsVerStr = [NSString stringWithFormat:@"%ld.%ld.%ld", + (long)osVer.majorVersion, (long)osVer.minorVersion, (long)osVer.patchVersion]; + } + } else { + // Version 10.16, without explicit env setting of SYSTEM_VERSION_COMPAT + // AKA 11+ Read the *real* ProductVersion from the hidden link to avoid SYSTEM_VERSION_COMPAT + // If not found, fallback below to the SystemVersion.plist + NSDictionary *version = [NSDictionary dictionaryWithContentsOfFile : + @"/System/Library/CoreServices/.SystemVersionPlatform.plist"]; + if (version != NULL) { + nsVerStr = [version objectForKey : @"ProductVersion"]; } } - // Fallback if running on pre-10.9 Mac OS + // Fallback to reading the SystemVersion.plist if (nsVerStr == NULL) { NSDictionary *version = [NSDictionary dictionaryWithContentsOfFile : @"/System/Library/CoreServices/SystemVersion.plist"]; diff --git a/src/java.base/macosx/native/libjli/java_md_macosx.m b/src/java.base/macosx/native/libjli/java_md_macosx.m index 354efc69769..c836ae903e5 100644 --- a/src/java.base/macosx/native/libjli/java_md_macosx.m +++ b/src/java.base/macosx/native/libjli/java_md_macosx.m @@ -415,6 +415,8 @@ static void MacOSXStartup(int argc, char *argv[]) { { char libjava[MAXPATHLEN]; + JLI_TraceLauncher("Attempt to get JDK installation root from launcher executable path\n"); + if (GetApplicationHome(path, pathsize)) { /* Is the JDK co-located with the application? */ if (JLI_IsStaticallyLinked()) { @@ -429,18 +431,6 @@ static void MacOSXStartup(int argc, char *argv[]) { return JNI_TRUE; } } - /* ensure storage for path + /jre + NULL */ - if ((JLI_StrLen(path) + 4 + 1) > (size_t) pathsize) { - JLI_TraceLauncher("Insufficient space to store JRE path\n"); - return JNI_FALSE; - } - /* Does the app ship a private JRE in /jre directory? */ - JLI_Snprintf(libjava, sizeof(libjava), "%s/jre/lib/" JAVA_DLL, path); - if (access(libjava, F_OK) == 0) { - JLI_StrCat(path, "/jre"); - JLI_TraceLauncher("JRE path is %s\n", path); - return JNI_TRUE; - } } /* try to find ourselves instead */ diff --git a/src/java.base/share/classes/java/io/ByteArrayOutputStream.java b/src/java.base/share/classes/java/io/ByteArrayOutputStream.java index 8f5ff41957e..020cd1f878a 100644 --- a/src/java.base/share/classes/java/io/ByteArrayOutputStream.java +++ b/src/java.base/share/classes/java/io/ByteArrayOutputStream.java @@ -159,16 +159,8 @@ public void writeBytes(byte[] b) { * @throws NullPointerException if {@code out} is {@code null}. * @throws IOException if an I/O error occurs. */ - public void writeTo(OutputStream out) throws IOException { - if (Thread.currentThread().isVirtual()) { - byte[] bytes; - synchronized (this) { - bytes = Arrays.copyOf(buf, count); - } - out.write(bytes); - } else synchronized (this) { - out.write(buf, 0, count); - } + public synchronized void writeTo(OutputStream out) throws IOException { + out.write(buf, 0, count); } /** diff --git a/src/java.base/share/classes/java/io/File.java b/src/java.base/share/classes/java/io/File.java index 61c15bdd15c..b8eda9dcf83 100644 --- a/src/java.base/share/classes/java/io/File.java +++ b/src/java.base/share/classes/java/io/File.java @@ -556,9 +556,6 @@ public boolean isAbsolute() { * @return The absolute pathname string denoting the same file or * directory as this abstract pathname * - * @throws SecurityException - * If a required system property value cannot be accessed. - * * @see java.io.File#isAbsolute() */ public String getAbsolutePath() { @@ -572,9 +569,6 @@ public String getAbsolutePath() { * @return The absolute abstract pathname denoting the same file or * directory as this abstract pathname * - * @throws SecurityException - * If a required system property value cannot be accessed. - * * @since 1.2 */ public File getAbsoluteFile() { @@ -614,12 +608,6 @@ public File getAbsoluteFile() { * construction of the canonical pathname may require * filesystem queries * - * @throws SecurityException - * If a required system property value cannot be accessed, or - * if a security manager exists and its {@link - * java.lang.SecurityManager#checkRead} method denies - * read access to the file - * * @since 1.1 * @see Path#toRealPath */ @@ -642,12 +630,6 @@ public String getCanonicalPath() throws IOException { * construction of the canonical pathname may require * filesystem queries * - * @throws SecurityException - * If a required system property value cannot be accessed, or - * if a security manager exists and its {@link - * java.lang.SecurityManager#checkRead} method denies - * read access to the file - * * @since 1.2 * @see Path#toRealPath */ @@ -736,8 +718,6 @@ public URL toURL() throws MalformedURLException { * @return An absolute, hierarchical URI with a scheme equal to * {@code "file"}, a path representing this abstract pathname, * and undefined authority, query, and fragment components - * @throws SecurityException If a required system property value cannot - * be accessed. * * @see #File(java.net.URI) * @see java.net.URI @@ -769,11 +749,6 @@ public URI toURI() { * @return {@code true} if and only if the file specified by this * abstract pathname exists and can be read by the * application; {@code false} otherwise - * - * @throws SecurityException - * If a security manager exists and its {@link - * java.lang.SecurityManager#checkRead(java.lang.String)} - * method denies read access to the file */ public boolean canRead() { @SuppressWarnings("removal") @@ -798,11 +773,6 @@ public boolean canRead() { * contains a file denoted by this abstract pathname and * the application is allowed to write to the file; * {@code false} otherwise. - * - * @throws SecurityException - * If a security manager exists and its {@link - * java.lang.SecurityManager#checkWrite(java.lang.String)} - * method denies write access to the file */ public boolean canWrite() { @SuppressWarnings("removal") @@ -822,11 +792,6 @@ public boolean canWrite() { * * @return {@code true} if and only if the file or directory denoted * by this abstract pathname exists; {@code false} otherwise - * - * @throws SecurityException - * If a security manager exists and its {@link - * java.lang.SecurityManager#checkRead(java.lang.String)} - * method denies read access to the file or directory */ public boolean exists() { @SuppressWarnings("removal") @@ -853,11 +818,6 @@ public boolean exists() { * @return {@code true} if and only if the file denoted by this * abstract pathname exists and is a directory; * {@code false} otherwise - * - * @throws SecurityException - * If a security manager exists and its {@link - * java.lang.SecurityManager#checkRead(java.lang.String)} - * method denies read access to the file */ public boolean isDirectory() { @SuppressWarnings("removal") @@ -886,11 +846,6 @@ public boolean isDirectory() { * @return {@code true} if and only if the file denoted by this * abstract pathname exists and is a normal file; * {@code false} otherwise - * - * @throws SecurityException - * If a security manager exists and its {@link - * java.lang.SecurityManager#checkRead(java.lang.String)} - * method denies read access to the file */ public boolean isFile() { @SuppressWarnings("removal") @@ -923,11 +878,6 @@ public boolean isFile() { * abstract pathname is hidden according to the conventions of the * underlying platform * - * @throws SecurityException - * If a security manager exists and its {@link - * java.lang.SecurityManager#checkRead(java.lang.String)} - * method denies read access to the file - * * @since 1.2 */ public boolean isHidden() { @@ -968,11 +918,6 @@ public boolean isHidden() { * file does not exist or if an I/O error occurs. The value may * be negative indicating the number of milliseconds before the * epoch - * - * @throws SecurityException - * If a security manager exists and its {@link - * java.lang.SecurityManager#checkRead(java.lang.String)} - * method denies read access to the file */ public long lastModified() { @SuppressWarnings("removal") @@ -1000,11 +945,6 @@ public long lastModified() { * pathname, or {@code 0L} if the file does not exist. Some * operating systems may return {@code 0L} for pathnames * denoting system-dependent entities such as devices or pipes. - * - * @throws SecurityException - * If a security manager exists and its {@link - * java.lang.SecurityManager#checkRead(java.lang.String)} - * method denies read access to the file */ public long length() { @SuppressWarnings("removal") @@ -1040,11 +980,6 @@ public long length() { * @throws IOException * If an I/O error occurred * - * @throws SecurityException - * If a security manager exists and its {@link - * java.lang.SecurityManager#checkWrite(java.lang.String)} - * method denies write access to the file - * * @since 1.2 */ public boolean createNewFile() throws IOException { @@ -1070,11 +1005,6 @@ public boolean createNewFile() throws IOException { * * @return {@code true} if and only if the file or directory is * successfully deleted; {@code false} otherwise - * - * @throws SecurityException - * If a security manager exists and its {@link - * java.lang.SecurityManager#checkDelete} method denies - * delete access to the file */ public boolean delete() { @SuppressWarnings("removal") @@ -1108,11 +1038,6 @@ public boolean delete() { * {@link java.nio.channels.FileLock FileLock} * facility should be used instead. * - * @throws SecurityException - * If a security manager exists and its {@link - * java.lang.SecurityManager#checkDelete} method denies - * delete access to the file - * * @see #delete * * @since 1.2 @@ -1155,11 +1080,6 @@ public void deleteOnExit() { * empty if the directory is empty. Returns {@code null} if * this abstract pathname does not denote a directory, or if an * I/O error occurs. - * - * @throws SecurityException - * If a security manager exists and its {@link - * SecurityManager#checkRead(String)} method denies read access to - * the directory */ public String[] list() { return normalizedList(); @@ -1175,11 +1095,6 @@ public String[] list() { * empty if the directory is empty. Returns {@code null} if * this abstract pathname does not denote a directory, or if an * I/O error occurs. - * - * @throws SecurityException - * If a security manager exists and its {@link - * SecurityManager#checkRead(String)} method denies read access to - * the directory */ private final String[] normalizedList() { @SuppressWarnings("removal") @@ -1223,11 +1138,6 @@ private final String[] normalizedList() { * Returns {@code null} if this abstract pathname does not denote * a directory, or if an I/O error occurs. * - * @throws SecurityException - * If a security manager exists and its {@link - * SecurityManager#checkRead(String)} method denies read access to - * the directory - * * @see java.nio.file.Files#newDirectoryStream(Path,String) */ public String[] list(FilenameFilter filter) { @@ -1275,11 +1185,6 @@ public String[] list(FilenameFilter filter) { * {@code null} if this abstract pathname does not denote a * directory, or if an I/O error occurs. * - * @throws SecurityException - * If a security manager exists and its {@link - * SecurityManager#checkRead(String)} method denies read access to - * the directory - * * @since 1.2 */ public File[] listFiles() { @@ -1315,11 +1220,6 @@ public File[] listFiles() { * {@code null} if this abstract pathname does not denote a * directory, or if an I/O error occurs. * - * @throws SecurityException - * If a security manager exists and its {@link - * SecurityManager#checkRead(String)} method denies read access to - * the directory - * * @since 1.2 * @see java.nio.file.Files#newDirectoryStream(Path,String) */ @@ -1353,11 +1253,6 @@ public File[] listFiles(FilenameFilter filter) { * {@code null} if this abstract pathname does not denote a * directory, or if an I/O error occurs. * - * @throws SecurityException - * If a security manager exists and its {@link - * SecurityManager#checkRead(String)} method denies read access to - * the directory - * * @since 1.2 * @see java.nio.file.Files#newDirectoryStream(Path,java.nio.file.DirectoryStream.Filter) */ @@ -1378,11 +1273,6 @@ public File[] listFiles(FileFilter filter) { * * @return {@code true} if and only if the directory was * created; {@code false} otherwise - * - * @throws SecurityException - * If a security manager exists and its {@link - * java.lang.SecurityManager#checkWrite(java.lang.String)} - * method does not permit the named directory to be created */ public boolean mkdir() { @SuppressWarnings("removal") @@ -1405,16 +1295,6 @@ public boolean mkdir() { * @return {@code true} if and only if the directory was created, * along with all necessary parent directories; {@code false} * otherwise - * - * @throws SecurityException - * If a security manager exists and its {@link - * java.lang.SecurityManager#checkRead(java.lang.String)} - * method does not permit verification of the existence of the - * named directory and all necessary parent directories; or if - * the {@link - * java.lang.SecurityManager#checkWrite(java.lang.String)} - * method does not permit the named directory and all necessary - * parent directories to be created */ public boolean mkdirs() { if (exists()) { @@ -1458,11 +1338,6 @@ public boolean mkdirs() { * @return {@code true} if and only if the renaming succeeded; * {@code false} otherwise * - * @throws SecurityException - * If a security manager exists and its {@link - * java.lang.SecurityManager#checkWrite(java.lang.String)} - * method denies write access to either the old or new pathnames - * * @throws NullPointerException * If parameter {@code dest} is {@code null} */ @@ -1501,11 +1376,6 @@ public boolean renameTo(File dest) { * * @throws IllegalArgumentException If the argument is negative * - * @throws SecurityException - * If a security manager exists and its {@link - * java.lang.SecurityManager#checkWrite(java.lang.String)} - * method denies write access to the named file - * * @since 1.2 */ public boolean setLastModified(long time) { @@ -1533,11 +1403,6 @@ public boolean setLastModified(long time) { * @return {@code true} if and only if the operation succeeded; * {@code false} otherwise * - * @throws SecurityException - * If a security manager exists and its {@link - * java.lang.SecurityManager#checkWrite(java.lang.String)} - * method denies write access to the named file - * * @since 1.2 */ public boolean setReadOnly() { @@ -1577,11 +1442,6 @@ public boolean setReadOnly() { * operation will fail if the user does not have permission to change * the access permissions of this abstract pathname. * - * @throws SecurityException - * If a security manager exists and its {@link - * java.lang.SecurityManager#checkWrite(java.lang.String)} - * method denies write access to the named file - * * @since 1.6 */ public boolean setWritable(boolean writable, boolean ownerOnly) { @@ -1617,11 +1477,6 @@ public boolean setWritable(boolean writable, boolean ownerOnly) { * operation will fail if the user does not have permission to * change the access permissions of this abstract pathname. * - * @throws SecurityException - * If a security manager exists and its {@link - * java.lang.SecurityManager#checkWrite(java.lang.String)} - * method denies write access to the file - * * @since 1.6 */ public boolean setWritable(boolean writable) { @@ -1659,11 +1514,6 @@ public boolean setWritable(boolean writable) { * fails, or the value of the {@code readable} parameter if * setting the read permission is not supported. * - * @throws SecurityException - * If a security manager exists and its {@link - * java.lang.SecurityManager#checkWrite(java.lang.String)} - * method denies write access to the file - * * @since 1.6 */ public boolean setReadable(boolean readable, boolean ownerOnly) { @@ -1705,11 +1555,6 @@ public boolean setReadable(boolean readable, boolean ownerOnly) { * fails, or the value of the {@code readable} parameter if * setting the read permission is not supported. * - * @throws SecurityException - * If a security manager exists and its {@link - * java.lang.SecurityManager#checkWrite(java.lang.String)} - * method denies write access to the file - * * @since 1.6 */ public boolean setReadable(boolean readable) { @@ -1747,11 +1592,6 @@ public boolean setReadable(boolean readable) { * fails, or the value of the {@code executable} parameter if * setting the execute permission is not supported. * - * @throws SecurityException - * If a security manager exists and its {@link - * java.lang.SecurityManager#checkWrite(java.lang.String)} - * method denies write access to the file - * * @since 1.6 */ public boolean setExecutable(boolean executable, boolean ownerOnly) { @@ -1793,11 +1633,6 @@ public boolean setExecutable(boolean executable, boolean ownerOnly) { * fails, or the value of the {@code executable} parameter if * setting the execute permission is not supported. * - * @throws SecurityException - * If a security manager exists and its {@link - * java.lang.SecurityManager#checkWrite(java.lang.String)} - * method denies write access to the file - * * @since 1.6 */ public boolean setExecutable(boolean executable) { @@ -1814,11 +1649,6 @@ public boolean setExecutable(boolean executable) { * @return {@code true} if and only if the abstract pathname exists * and the application is allowed to execute the file * - * @throws SecurityException - * If a security manager exists and its {@link - * java.lang.SecurityManager#checkExec(java.lang.String)} - * method denies execute access to the file - * * @since 1.6 */ public boolean canExecute() { @@ -1850,12 +1680,6 @@ public boolean canExecute() { * machine will begin with one of the roots returned by this method. * There is no guarantee that a root directory can be accessed. * - *

Unlike most methods in this class, this method does not throw - * security exceptions. If a security manager exists and its {@link - * SecurityManager#checkRead(String)} method denies read access to a - * particular root directory, then that directory will not appear in the - * result. - * * @implNote * Windows platforms, for example, have a root directory * for each active drive; UNIX platforms have a single root directory, @@ -1898,12 +1722,6 @@ public static File[] listRoots() { * abstract pathname does not name a partition or if the size * cannot be obtained * - * @throws SecurityException - * If a security manager has been installed and it denies - * {@link RuntimePermission}{@code ("getFileSystemAttributes")} - * or its {@link SecurityManager#checkRead(String)} method denies - * read access to the file named by this abstract pathname - * * @since 1.6 * @see FileStore#getTotalSpace */ @@ -1942,12 +1760,6 @@ public long getTotalSpace() { * equal to the total file system size returned by * {@link #getTotalSpace}. * - * @throws SecurityException - * If a security manager has been installed and it denies - * {@link RuntimePermission}{@code ("getFileSystemAttributes")} - * or its {@link SecurityManager#checkRead(String)} method denies - * read access to the file named by this abstract pathname - * * @since 1.6 * @see FileStore#getUnallocatedSpace */ @@ -1989,12 +1801,6 @@ public long getFreeSpace() { * is not available, this method will be equivalent to a call to * {@link #getFreeSpace}. * - * @throws SecurityException - * If a security manager has been installed and it denies - * {@link RuntimePermission}{@code ("getFileSystemAttributes")} - * or its {@link SecurityManager#checkRead(String)} method denies - * read access to the file named by this abstract pathname - * * @since 1.6 * @see FileStore#getUsableSpace */ @@ -2176,11 +1982,6 @@ static File generateFile(String prefix, String suffix, File dir) * @throws IOException * If a file could not be created * - * @throws SecurityException - * If a security manager exists and its {@link - * java.lang.SecurityManager#checkWrite(java.lang.String)} - * method does not allow a file to be created - * * @since 1.2 */ public static File createTempFile(String prefix, String suffix, @@ -2250,11 +2051,6 @@ public static File createTempFile(String prefix, String suffix, * * @throws IOException If a file could not be created * - * @throws SecurityException - * If a security manager exists and its {@link - * java.lang.SecurityManager#checkWrite(java.lang.String)} - * method does not allow a file to be created - * * @since 1.2 * @see java.nio.file.Files#createTempDirectory(String,FileAttribute[]) */ diff --git a/src/java.base/share/classes/java/io/FileInputStream.java b/src/java.base/share/classes/java/io/FileInputStream.java index 4210a0f56b6..e429faec09b 100644 --- a/src/java.base/share/classes/java/io/FileInputStream.java +++ b/src/java.base/share/classes/java/io/FileInputStream.java @@ -95,11 +95,6 @@ public class FileInputStream extends InputStream * object is created to represent this file * connection. *

- * First, if there is a security - * manager, its {@code checkRead} method - * is called with the {@code name} argument - * as its argument. - *

* If the named file does not exist, is a directory rather than a regular * file, or for some other reason cannot be opened for reading then a * {@code FileNotFoundException} is thrown. @@ -109,10 +104,6 @@ public class FileInputStream extends InputStream * is a directory rather than a regular file, * or for some other reason cannot be opened for * reading. - * @throws SecurityException if a security manager exists and its - * {@code checkRead} method denies read access - * to the file. - * @see java.lang.SecurityManager#checkRead(java.lang.String) */ public FileInputStream(String name) throws FileNotFoundException { this(name != null ? new File(name) : null); @@ -126,11 +117,6 @@ public FileInputStream(String name) throws FileNotFoundException { * A new {@code FileDescriptor} object * is created to represent this file connection. *

- * First, if there is a security manager, - * its {@code checkRead} method is called - * with the path represented by the {@code file} - * argument as its argument. - *

* If the named file does not exist, is a directory rather than a regular * file, or for some other reason cannot be opened for reading then a * {@code FileNotFoundException} is thrown. @@ -140,10 +126,7 @@ public FileInputStream(String name) throws FileNotFoundException { * is a directory rather than a regular file, * or for some other reason cannot be opened for * reading. - * @throws SecurityException if a security manager exists and its - * {@code checkRead} method denies read access to the file. * @see java.io.File#getPath() - * @see java.lang.SecurityManager#checkRead(java.lang.String) */ @SuppressWarnings("this-escape") public FileInputStream(File file) throws FileNotFoundException { @@ -171,11 +154,6 @@ public FileInputStream(File file) throws FileNotFoundException { * {@code fdObj}, which represents an existing connection to an * actual file in the file system. *

- * If there is a security manager, its {@code checkRead} method is - * called with the file descriptor {@code fdObj} as its argument to - * see if it's ok to read the file descriptor. If read access is denied - * to the file descriptor a {@code SecurityException} is thrown. - *

* If {@code fdObj} is null then a {@code NullPointerException} * is thrown. *

@@ -185,10 +163,6 @@ public FileInputStream(File file) throws FileNotFoundException { * I/O on the stream, an {@code IOException} is thrown. * * @param fdObj the file descriptor to be opened for reading. - * @throws SecurityException if a security manager exists and its - * {@code checkRead} method denies read access to the - * file descriptor. - * @see SecurityManager#checkRead(java.io.FileDescriptor) */ @SuppressWarnings("this-escape") public FileInputStream(FileDescriptor fdObj) { diff --git a/src/java.base/share/classes/java/io/FileOutputStream.java b/src/java.base/share/classes/java/io/FileOutputStream.java index ff15e831b95..557bea0c3fc 100644 --- a/src/java.base/share/classes/java/io/FileOutputStream.java +++ b/src/java.base/share/classes/java/io/FileOutputStream.java @@ -103,9 +103,6 @@ public class FileOutputStream extends OutputStream * A new {@code FileDescriptor} object is * created to represent this file connection. *

- * First, if there is a security manager, its {@code checkWrite} - * method is called with {@code name} as its argument. - *

* If the file exists but is a directory rather than a regular file, does * not exist but cannot be created, or cannot be opened for any other * reason then a {@code FileNotFoundException} is thrown. @@ -118,10 +115,6 @@ public class FileOutputStream extends OutputStream * @throws FileNotFoundException if the file exists but is a directory * rather than a regular file, does not exist but cannot * be created, or cannot be opened for any other reason - * @throws SecurityException if a security manager exists and its - * {@code checkWrite} method denies write access - * to the file. - * @see java.lang.SecurityManager#checkWrite(java.lang.String) */ public FileOutputStream(String name) throws FileNotFoundException { this(name != null ? new File(name) : null, false); @@ -137,9 +130,6 @@ public FileOutputStream(String name) throws FileNotFoundException { * A new {@code FileDescriptor} object is created to represent this * file connection. *

- * First, if there is a security manager, its {@code checkWrite} - * method is called with {@code name} as its argument. - *

* If the file exists but is a directory rather than a regular file, does * not exist but cannot be created, or cannot be opened for any other * reason then a {@code FileNotFoundException} is thrown. @@ -150,10 +140,6 @@ public FileOutputStream(String name) throws FileNotFoundException { * @throws FileNotFoundException if the file exists but is a directory * rather than a regular file, does not exist but cannot * be created, or cannot be opened for any other reason. - * @throws SecurityException if a security manager exists and its - * {@code checkWrite} method denies write access - * to the file. - * @see java.lang.SecurityManager#checkWrite(java.lang.String) * @since 1.1 */ public FileOutputStream(String name, boolean append) @@ -171,10 +157,6 @@ public FileOutputStream(String name, boolean append) * A new {@code FileDescriptor} object is * created to represent this file connection. *

- * First, if there is a security manager, its {@code checkWrite} - * method is called with the path represented by the {@code file} - * argument as its argument. - *

* If the file exists but is a directory rather than a regular file, does * not exist but cannot be created, or cannot be opened for any other * reason then a {@code FileNotFoundException} is thrown. @@ -183,12 +165,7 @@ public FileOutputStream(String name, boolean append) * @throws FileNotFoundException if the file exists but is a directory * rather than a regular file, does not exist but cannot * be created, or cannot be opened for any other reason - * @throws SecurityException if a security manager exists and its - * {@code checkWrite} method denies write access - * to the file. * @see java.io.File#getPath() - * @see java.lang.SecurityException - * @see java.lang.SecurityManager#checkWrite(java.lang.String) */ public FileOutputStream(File file) throws FileNotFoundException { this(file, false); @@ -205,10 +182,6 @@ public FileOutputStream(File file) throws FileNotFoundException { * A new {@code FileDescriptor} object is created to represent this * file connection. *

- * First, if there is a security manager, its {@code checkWrite} - * method is called with the path represented by the {@code file} - * argument as its argument. - *

* If the file exists but is a directory rather than a regular file, does * not exist but cannot be created, or cannot be opened for any other * reason then a {@code FileNotFoundException} is thrown. @@ -219,12 +192,7 @@ public FileOutputStream(File file) throws FileNotFoundException { * @throws FileNotFoundException if the file exists but is a directory * rather than a regular file, does not exist but cannot * be created, or cannot be opened for any other reason - * @throws SecurityException if a security manager exists and its - * {@code checkWrite} method denies write access - * to the file. * @see java.io.File#getPath() - * @see java.lang.SecurityException - * @see java.lang.SecurityManager#checkWrite(java.lang.String) * @since 1.4 */ @SuppressWarnings("this-escape") @@ -256,10 +224,6 @@ public FileOutputStream(File file, boolean append) * descriptor, which represents an existing connection to an actual * file in the file system. *

- * First, if there is a security manager, its {@code checkWrite} - * method is called with the file descriptor {@code fdObj} - * argument as its argument. - *

* If {@code fdObj} is null then a {@code NullPointerException} * is thrown. *

@@ -269,10 +233,6 @@ public FileOutputStream(File file, boolean append) * I/O on the stream, an {@code IOException} is thrown. * * @param fdObj the file descriptor to be opened for writing - * @throws SecurityException if a security manager exists and its - * {@code checkWrite} method denies - * write access to the file descriptor - * @see java.lang.SecurityManager#checkWrite(java.io.FileDescriptor) */ @SuppressWarnings("this-escape") public FileOutputStream(FileDescriptor fdObj) { diff --git a/src/java.base/share/classes/java/io/FilePermission.java b/src/java.base/share/classes/java/io/FilePermission.java index 808c4293143..b11e0dd25be 100644 --- a/src/java.base/share/classes/java/io/FilePermission.java +++ b/src/java.base/share/classes/java/io/FilePermission.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -61,39 +61,15 @@ * (recursively) all files and subdirectories contained in the current * directory. *

- * The actions to be granted are passed to the constructor in a string containing + * The actions are passed to the constructor in a string containing * a list of one or more comma-separated keywords. The possible keywords are - * "read", "write", "execute", "delete", and "readlink". Their meaning is - * defined as follows: - * - *

- *
read
read permission - *
write
write permission - *
execute - *
execute permission. Allows {@code Runtime.exec} to - * be called. Corresponds to {@code SecurityManager.checkExec}. - *
delete - *
delete permission. Allows {@code File.delete} to - * be called. Corresponds to {@code SecurityManager.checkDelete}. - *
readlink - *
read link permission. Allows the target of a - * symbolic link - * to be read by invoking the {@link java.nio.file.Files#readSymbolicLink - * readSymbolicLink } method. - *
+ * "read", "write", "execute", "delete", and "readlink". *

* The actions string is converted to lowercase before processing. - *

- * Be careful when granting FilePermissions. Think about the implications - * of granting read and especially write access to various files and - * directories. The {@literal "<>"} permission with write action is - * especially dangerous. This grants permission to write to the entire - * file system. One thing this effectively allows is replacement of the - * system binary, including the JVM runtime environment. - *

- * Please note: Code can always read a file from the same - * directory it's in (or a subdirectory of that directory); it does not - * need explicit permission to do so. + * + * @apiNote + * This permission cannot be used for controlling access to resources + * as the Security Manager is no longer supported. * * @see java.security.Permission * @see java.security.Permissions diff --git a/src/java.base/share/classes/java/io/ObjectInputFilter.java b/src/java.base/share/classes/java/io/ObjectInputFilter.java index 879820761c1..9a04e1f58e3 100644 --- a/src/java.base/share/classes/java/io/ObjectInputFilter.java +++ b/src/java.base/share/classes/java/io/ObjectInputFilter.java @@ -728,8 +728,6 @@ public static ObjectInputFilter getSerialFilter() { * Set the static JVM-wide filter if it has not already been configured or set. * * @param filter the deserialization filter to set as the JVM-wide filter; not null - * @throws SecurityException if there is security manager and the - * {@code SerializablePermission("serialFilter")} is not granted * @throws IllegalStateException if the filter has already been set or the initialization * of the filter from the system property {@code jdk.serialFilter} or * the security property {@code jdk.serialFilter} fails. @@ -829,8 +827,6 @@ static BinaryOperator getSerialFilterFactorySingleton() { * @throws IllegalStateException if the builtin deserialization filter factory * has already been replaced or any instance of {@link ObjectInputStream} * has been created. - * @throws SecurityException if there is security manager and the - * {@code SerializablePermission("serialFilter")} is not granted * @since 17 */ public static void setSerialFilterFactory(BinaryOperator filterFactory) { diff --git a/src/java.base/share/classes/java/io/ObjectInputStream.java b/src/java.base/share/classes/java/io/ObjectInputStream.java index 983b80d2bc6..60d8cadf6a3 100644 --- a/src/java.base/share/classes/java/io/ObjectInputStream.java +++ b/src/java.base/share/classes/java/io/ObjectInputStream.java @@ -374,17 +374,9 @@ private static class Logging { * When the filter factory {@code apply} method is invoked it may throw a runtime exception * preventing the {@code ObjectInputStream} from being constructed. * - *

If a security manager is installed, this constructor will check for - * the "enableSubclassImplementation" SerializablePermission when invoked - * directly or indirectly by the constructor of a subclass which overrides - * the ObjectInputStream.readFields or ObjectInputStream.readUnshared - * methods. - * * @param in input stream to read from * @throws StreamCorruptedException if the stream header is incorrect * @throws IOException if an I/O error occurs while reading stream header - * @throws SecurityException if untrusted subclass illegally overrides - * security-sensitive methods * @throws IllegalStateException if the initialization of {@link ObjectInputFilter.Config} * fails due to invalid serial filter or serial filter factory properties. * @throws NullPointerException if {@code in} is {@code null} @@ -419,21 +411,11 @@ public ObjectInputStream(InputStream in) throws IOException { * When the filter factory {@code apply} method is invoked it may throw a runtime exception * preventing the {@code ObjectInputStream} from being constructed. * - *

If there is a security manager installed, this method first calls the - * security manager's {@code checkPermission} method with the - * {@code SerializablePermission("enableSubclassImplementation")} - * permission to ensure it's ok to enable subclassing. - * - * @throws SecurityException if a security manager exists and its - * {@code checkPermission} method denies enabling - * subclassing. * @throws IOException if an I/O error occurs while creating this stream * @throws IllegalStateException if the initialization of {@link ObjectInputFilter.Config} * fails due to invalid serial filter or serial filter factory properties. - * @see SecurityManager#checkPermission - * @see java.io.SerializablePermission */ - protected ObjectInputStream() throws IOException, SecurityException { + protected ObjectInputStream() throws IOException { @SuppressWarnings("removal") SecurityManager sm = System.getSecurityManager(); if (sm != null) { @@ -599,12 +581,6 @@ protected Object readObjectOverride() * each object (regular or class) read to reconstruct the root object. * See {@link #setObjectInputFilter(ObjectInputFilter) setObjectInputFilter} for details. * - *

ObjectInputStream subclasses which override this method can only be - * constructed in security contexts possessing the - * "enableSubclassImplementation" SerializablePermission; any attempt to - * instantiate such a subclass without this permission will cause a - * SecurityException to be thrown. - * * @return reference to deserialized object * @throws ClassNotFoundException if class of an object to deserialize * cannot be found @@ -923,26 +899,11 @@ protected Object resolveObject(Object obj) throws IOException { * enabled, the {@link #resolveObject} method is called for every object being * deserialized. * - *

If object replacement is currently not enabled, and - * {@code enable} is true, and there is a security manager installed, - * this method first calls the security manager's - * {@code checkPermission} method with the - * {@code SerializablePermission("enableSubstitution")} permission to - * ensure that the caller is permitted to enable the stream to do replacement - * of objects read from the stream. - * * @param enable true for enabling use of {@code resolveObject} for * every object being deserialized * @return the previous setting before this method was invoked - * @throws SecurityException if a security manager exists and its - * {@code checkPermission} method denies enabling the stream - * to do replacement of objects read from the stream. - * @see SecurityManager#checkPermission - * @see java.io.SerializablePermission */ - protected boolean enableResolveObject(boolean enable) - throws SecurityException - { + protected boolean enableResolveObject(boolean enable) { if (enable == enableResolve) { return enable; } @@ -1341,8 +1302,6 @@ public final ObjectInputFilter getObjectInputFilter() { * is increased before reading an object. * * @param filter the filter, may be null - * @throws SecurityException if there is security manager and the - * {@code SerializablePermission("serialFilter")} is not granted * @throws IllegalStateException if an object has been read, * if the filter factory returns {@code null} when the * {@linkplain #getObjectInputFilter() current filter} is non-null, or diff --git a/src/java.base/share/classes/java/io/ObjectOutputStream.java b/src/java.base/share/classes/java/io/ObjectOutputStream.java index bde069a1774..429c91ab673 100644 --- a/src/java.base/share/classes/java/io/ObjectOutputStream.java +++ b/src/java.base/share/classes/java/io/ObjectOutputStream.java @@ -237,16 +237,8 @@ protected Boolean computeValue(Class type) { * ensure that constructors for receiving ObjectInputStreams will not block * when reading the header. * - *

If a security manager is installed, this constructor will check for - * the "enableSubclassImplementation" SerializablePermission when invoked - * directly or indirectly by the constructor of a subclass which overrides - * the ObjectOutputStream.putFields or ObjectOutputStream.writeUnshared - * methods. - * * @param out output stream to write to * @throws IOException if an I/O error occurs while writing stream header - * @throws SecurityException if untrusted subclass illegally overrides - * security-sensitive methods * @throws NullPointerException if {@code out} is {@code null} * @since 1.4 * @see ObjectOutputStream#ObjectOutputStream() @@ -274,19 +266,9 @@ public ObjectOutputStream(OutputStream out) throws IOException { * ObjectOutputStream to not have to allocate private data just used by * this implementation of ObjectOutputStream. * - *

If there is a security manager installed, this method first calls the - * security manager's {@code checkPermission} method with a - * {@code SerializablePermission("enableSubclassImplementation")} - * permission to ensure it's ok to enable subclassing. - * - * @throws SecurityException if a security manager exists and its - * {@code checkPermission} method denies enabling - * subclassing. * @throws IOException if an I/O error occurs while creating this stream - * @see SecurityManager#checkPermission - * @see java.io.SerializablePermission */ - protected ObjectOutputStream() throws IOException, SecurityException { + protected ObjectOutputStream() throws IOException { @SuppressWarnings("removal") SecurityManager sm = System.getSecurityManager(); if (sm != null) { @@ -414,12 +396,6 @@ protected void writeObjectOverride(Object obj) throws IOException { * writeUnshared, and not to any transitively referenced sub-objects in the * object graph to be serialized. * - *

ObjectOutputStream subclasses which override this method can only be - * constructed in security contexts possessing the - * "enableSubclassImplementation" SerializablePermission; any attempt to - * instantiate such a subclass without this permission will cause a - * SecurityException to be thrown. - * * @param obj object to write to stream * @throws NotSerializableException if an object in the graph to be * serialized does not implement the Serializable interface @@ -611,26 +587,11 @@ protected Object replaceObject(Object obj) throws IOException { * enabled, the {@link #replaceObject} method is called for every object being * serialized. * - *

If object replacement is currently not enabled, and - * {@code enable} is true, and there is a security manager installed, - * this method first calls the security manager's - * {@code checkPermission} method with the - * {@code SerializablePermission("enableSubstitution")} permission to - * ensure that the caller is permitted to enable the stream to do replacement - * of objects written to the stream. - * * @param enable true for enabling use of {@code replaceObject} for * every object being serialized * @return the previous setting before this method was invoked - * @throws SecurityException if a security manager exists and its - * {@code checkPermission} method denies enabling the stream - * to do replacement of objects written to the stream. - * @see SecurityManager#checkPermission - * @see java.io.SerializablePermission */ - protected boolean enableReplaceObject(boolean enable) - throws SecurityException - { + protected boolean enableReplaceObject(boolean enable) { if (enable == enableReplace) { return enable; } diff --git a/src/java.base/share/classes/java/io/PrintStream.java b/src/java.base/share/classes/java/io/PrintStream.java index 35e2716dbd3..3096f2356f5 100644 --- a/src/java.base/share/classes/java/io/PrintStream.java +++ b/src/java.base/share/classes/java/io/PrintStream.java @@ -249,10 +249,6 @@ public PrintStream(OutputStream out, boolean autoFlush, Charset charset) { * created, or if some other error occurs while opening or * creating the file * - * @throws SecurityException - * If a security manager is present and {@link - * SecurityManager#checkWrite checkWrite(fileName)} denies write - * access to the file * @see Charset#defaultCharset() * * @since 1.5 @@ -284,11 +280,6 @@ public PrintStream(String fileName) throws FileNotFoundException { * created, or if some other error occurs while opening or * creating the file * - * @throws SecurityException - * If a security manager is present and {@link - * SecurityManager#checkWrite checkWrite(fileName)} denies write - * access to the file - * * @throws UnsupportedEncodingException * If the named charset is not supported * @@ -320,11 +311,6 @@ public PrintStream(String fileName, String csn) * @throws IOException * if an I/O error occurs while opening or creating the file * - * @throws SecurityException - * If a security manager is present and {@link - * SecurityManager#checkWrite checkWrite(fileName)} denies write - * access to the file - * * @since 10 */ public PrintStream(String fileName, Charset charset) throws IOException { @@ -351,10 +337,6 @@ public PrintStream(String fileName, Charset charset) throws IOException { * created, or if some other error occurs while opening or * creating the file * - * @throws SecurityException - * If a security manager is present and {@link - * SecurityManager#checkWrite checkWrite(file.getPath())} - * denies write access to the file * @see Charset#defaultCharset() * * @since 1.5 @@ -386,11 +368,6 @@ public PrintStream(File file) throws FileNotFoundException { * created, or if some other error occurs while opening or * creating the file * - * @throws SecurityException - * If a security manager is present and {@link - * SecurityManager#checkWrite checkWrite(file.getPath())} - * denies write access to the file - * * @throws UnsupportedEncodingException * If the named charset is not supported * @@ -423,11 +400,6 @@ public PrintStream(File file, String csn) * @throws IOException * if an I/O error occurs while opening or creating the file * - * @throws SecurityException - * If a security manager is present and {@link - * SecurityManager#checkWrite checkWrite(file.getPath())} - * denies write access to the file - * * @since 10 */ public PrintStream(File file, Charset charset) throws IOException { diff --git a/src/java.base/share/classes/java/io/PrintWriter.java b/src/java.base/share/classes/java/io/PrintWriter.java index 2ed9315ba6a..55baa2e0a57 100644 --- a/src/java.base/share/classes/java/io/PrintWriter.java +++ b/src/java.base/share/classes/java/io/PrintWriter.java @@ -195,10 +195,6 @@ public PrintWriter(OutputStream out, boolean autoFlush, Charset charset) { * created, or if some other error occurs while opening or * creating the file * - * @throws SecurityException - * If a security manager is present and {@link - * SecurityManager#checkWrite checkWrite(fileName)} denies write - * access to the file * @see Charset#defaultCharset() * * @since 1.5 @@ -247,11 +243,6 @@ private PrintWriter(Charset charset, File file) * created, or if some other error occurs while opening or * creating the file * - * @throws SecurityException - * If a security manager is present and {@link - * SecurityManager#checkWrite checkWrite(fileName)} denies write - * access to the file - * * @throws UnsupportedEncodingException * If the named charset is not supported * @@ -282,11 +273,6 @@ public PrintWriter(String fileName, String csn) * @throws IOException * if an I/O error occurs while opening or creating the file * - * @throws SecurityException - * If a security manager is present and {@link - * SecurityManager#checkWrite checkWrite(fileName)} denies write - * access to the file - * * @since 10 */ public PrintWriter(String fileName, Charset charset) throws IOException { @@ -313,10 +299,6 @@ public PrintWriter(String fileName, Charset charset) throws IOException { * created, or if some other error occurs while opening or * creating the file * - * @throws SecurityException - * If a security manager is present and {@link - * SecurityManager#checkWrite checkWrite(file.getPath())} - * denies write access to the file * @see Charset#defaultCharset() * * @since 1.5 @@ -348,11 +330,6 @@ public PrintWriter(File file) throws FileNotFoundException { * created, or if some other error occurs while opening or * creating the file * - * @throws SecurityException - * If a security manager is present and {@link - * SecurityManager#checkWrite checkWrite(file.getPath())} - * denies write access to the file - * * @throws UnsupportedEncodingException * If the named charset is not supported * @@ -383,11 +360,6 @@ public PrintWriter(File file, String csn) * @throws IOException * if an I/O error occurs while opening or creating the file * - * @throws SecurityException - * If a security manager is present and {@link - * SecurityManager#checkWrite checkWrite(file.getPath())} - * denies write access to the file - * * @since 10 */ public PrintWriter(File file, Charset charset) throws IOException { diff --git a/src/java.base/share/classes/java/io/RandomAccessFile.java b/src/java.base/share/classes/java/io/RandomAccessFile.java index cf8ae43dc21..1487764ac44 100644 --- a/src/java.base/share/classes/java/io/RandomAccessFile.java +++ b/src/java.base/share/classes/java/io/RandomAccessFile.java @@ -114,15 +114,6 @@ public class RandomAccessFile implements DataOutput, DataInput, Closeable { * specified for the {@code RandomAccessFile(File,String)} constructor. * - *

- * If there is a security manager, its {@code checkRead} method - * is called with the {@code pathname} argument - * as its argument to see if read access to the file is allowed. - * If the mode allows writing, the security manager's - * {@code checkWrite} method - * is also called with the {@code pathname} argument - * as its argument to see if write access to the file is allowed. - * * @param pathname the system-dependent pathname string * @param mode the access mode * @throws IllegalArgumentException if the mode argument is not equal @@ -135,13 +126,6 @@ public class RandomAccessFile implements DataOutput, DataInput, Closeable { * existing, writable regular file and a new regular file of * that pathname cannot be created, or if some other error * occurs while opening or creating the file - * @throws SecurityException if a security manager exists and its - * {@code checkRead} method denies read access to the file - * or the mode is {@code "rw"} and the security manager's - * {@code checkWrite} method denies write access to the file - * @see java.lang.SecurityException - * @see java.lang.SecurityManager#checkRead(java.lang.String) - * @see java.lang.SecurityManager#checkWrite(java.lang.String) */ public RandomAccessFile(String pathname, String mode) throws FileNotFoundException @@ -205,13 +189,6 @@ public RandomAccessFile(String pathname, String mode) * updates to both the file's content and its metadata to be written, which * generally requires at least one more low-level I/O operation. * - *

If there is a security manager, its {@code checkRead} method is - * called with the pathname of the {@code file} argument as its - * argument to see if read access to the file is allowed. If the mode - * allows writing, the security manager's {@code checkWrite} method is - * also called with the pathname of the {@code file} argument to see if - * write access to the file is allowed. - * * @param file the file object * @param mode the access mode, as described * above @@ -225,12 +202,6 @@ public RandomAccessFile(String pathname, String mode) * an existing, writable regular file and a new regular file of * that pathname cannot be created, or if some other error * occurs while opening or creating the file - * @throws SecurityException if a security manager exists and its - * {@code checkRead} method denies read access to the file - * or the mode is {@code "rw"} and the security manager's - * {@code checkWrite} method denies write access to the file - * @see java.lang.SecurityManager#checkRead(java.lang.String) - * @see java.lang.SecurityManager#checkWrite(java.lang.String) * @see java.nio.channels.FileChannel#force(boolean) */ @SuppressWarnings("this-escape") diff --git a/src/java.base/share/classes/java/io/SerializablePermission.java b/src/java.base/share/classes/java/io/SerializablePermission.java index 914b5b13dc3..1e617e173a7 100644 --- a/src/java.base/share/classes/java/io/SerializablePermission.java +++ b/src/java.base/share/classes/java/io/SerializablePermission.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -36,55 +36,9 @@ * no actions list; you either have the named permission * or you don't. * - *

- * The target name is the name of the Serializable permission (see below). - * - *

- * The following table lists the standard {@code SerializablePermission} target names, - * and for each provides a description of what the permission allows - * and a discussion of the risks of granting code the permission. - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - *
Permission target name, what the permission allows, and associated risks
Permission Target NameWhat the Permission AllowsRisks of Allowing this Permission
enableSubclassImplementationSubclass implementation of ObjectOutputStream or ObjectInputStream - * to override the default serialization or deserialization, respectively, - * of objectsCode can use this to serialize or - * deserialize classes in a purposefully malfeasant manner. For example, - * during serialization, malicious code can use this to - * purposefully store confidential private field data in a way easily accessible - * to attackers. Or, during deserialization it could, for example, deserialize - * a class with all its private fields zeroed out.
enableSubstitutionSubstitution of one object for another during - * serialization or deserializationThis is dangerous because malicious code - * can replace the actual object with one which has incorrect or - * malignant data.
serialFilterSetting a filter for ObjectInputStreams.Code could remove a configured filter and remove protections - * already established.
+ * @apiNote + * This permission cannot be used for controlling access to resources + * as the Security Manager is no longer supported. * * @see java.security.BasicPermission * @see java.security.Permission @@ -92,7 +46,6 @@ * @see java.security.PermissionCollection * @see java.lang.SecurityManager * - * * @author Joe Fialli * @since 1.2 */ diff --git a/src/java.base/share/classes/java/lang/Boolean.java b/src/java.base/share/classes/java/lang/Boolean.java index 31507f60e37..f5c643881a8 100644 --- a/src/java.base/share/classes/java/lang/Boolean.java +++ b/src/java.base/share/classes/java/lang/Boolean.java @@ -271,8 +271,6 @@ public boolean equals(Object obj) { * * @param name the system property name. * @return the {@code boolean} value of the system property. - * @throws SecurityException for the same reasons as - * {@link System#getProperty(String) System.getProperty} * @see java.lang.System#getProperty(java.lang.String) * @see java.lang.System#getProperty(java.lang.String, java.lang.String) */ diff --git a/src/java.base/share/classes/java/lang/Class.java b/src/java.base/share/classes/java/lang/Class.java index 2edf0a9169e..bb091235646 100644 --- a/src/java.base/share/classes/java/lang/Class.java +++ b/src/java.base/share/classes/java/lang/Class.java @@ -530,11 +530,6 @@ private static Class forName(String className, Class caller) * by this method fails * @throws ClassNotFoundException if the class cannot be located by * the specified class loader - * @throws SecurityException - * if a security manager is present, and the {@code loader} is - * {@code null}, and the caller's class loader is not - * {@code null}, and the caller does not have the - * {@link RuntimePermission}{@code ("getClassLoader")} * * @see java.lang.Class#forName(String) * @see java.lang.ClassLoader @@ -610,8 +605,6 @@ private static native Class forName0(String name, boolean initialize, * a binary name. This method returns {@code null} on failure rather than * throwing a {@link ClassNotFoundException}, as is done by * the {@link #forName(String, boolean, ClassLoader)} method. - * The security check is a stack-based permission check if the caller - * loads a class in another module. * * @param module A module * @param name The {@linkplain ClassLoader##binary-name binary name} @@ -623,16 +616,6 @@ private static native Class forName0(String name, boolean initialize, * * @throws LinkageError if the linkage fails * - * @throws SecurityException - *

    - *
  • if the caller is not the specified module and - * {@code RuntimePermission("getClassLoader")} permission is denied; or
  • - *
  • access to the module content is denied. For example, - * permission check will be performed when a class loader calls - * {@link ModuleReader#open(String)} to read the bytes of a class file - * in a module.
  • - *
- * * @jls 12.2 Loading of Classes and Interfaces * @jls 12.3 Linking of Classes and Interfaces * @since 9 @@ -756,13 +739,6 @@ public static Class forPrimitiveName(String primitiveName) { * or if the instantiation fails for some other reason. * @throws ExceptionInInitializerError if the initialization * provoked by this method fails. - * @throws SecurityException - * If a security manager, s, is present and - * the caller's class loader is not the same as or an - * ancestor of the class loader for the current class and - * invocation of {@link SecurityManager#checkPackageAccess - * s.checkPackageAccess()} denies access to the package - * of this class. */ @SuppressWarnings("removal") @CallerSensitive @@ -1057,15 +1033,7 @@ public String getName() { * * @return the class loader that loaded the class or interface * represented by this {@code Class} object. - * @throws SecurityException - * if a security manager is present, and the caller's class loader - * is not {@code null} and is not the same as or an ancestor of the - * class loader for the class whose class loader is requested, - * and the caller does not have the - * {@link RuntimePermission}{@code ("getClassLoader")} * @see java.lang.ClassLoader - * @see SecurityManager#checkPermission - * @see java.lang.RuntimePermission */ @CallerSensitive @ForceInline // to ensure Reflection.getCallerClass optimization @@ -1541,30 +1509,10 @@ void setSigners(Object[] signers) { * @return the immediately enclosing method of the underlying class, if * that class is a local or anonymous class; otherwise {@code null}. * - * @throws SecurityException - * If a security manager, s, is present and any of the - * following conditions is met: - * - *
    - * - *
  • the caller's class loader is not the same as the - * class loader of the enclosing class and invocation of - * {@link SecurityManager#checkPermission - * s.checkPermission} method with - * {@code RuntimePermission("accessDeclaredMembers")} - * denies access to the methods within the enclosing class - * - *
  • the caller's class loader is not the same as or an - * ancestor of the class loader for the enclosing class and - * invocation of {@link SecurityManager#checkPackageAccess - * s.checkPackageAccess()} denies access to the package - * of the enclosing class - * - *
* @since 1.5 */ @CallerSensitive - public Method getEnclosingMethod() throws SecurityException { + public Method getEnclosingMethod() { EnclosingMethodInfo enclosingInfo = getEnclosingMethodInfo(); if (enclosingInfo == null) @@ -1697,30 +1645,11 @@ private static Class toClass(Type o) { * * @return the immediately enclosing constructor of the underlying class, if * that class is a local or anonymous class; otherwise {@code null}. - * @throws SecurityException - * If a security manager, s, is present and any of the - * following conditions is met: - * - *
    - * - *
  • the caller's class loader is not the same as the - * class loader of the enclosing class and invocation of - * {@link SecurityManager#checkPermission - * s.checkPermission} method with - * {@code RuntimePermission("accessDeclaredMembers")} - * denies access to the constructors within the enclosing class - * - *
  • the caller's class loader is not the same as or an - * ancestor of the class loader for the enclosing class and - * invocation of {@link SecurityManager#checkPackageAccess - * s.checkPackageAccess()} denies access to the package - * of the enclosing class - * - *
+ * * @since 1.5 */ @CallerSensitive - public Constructor getEnclosingConstructor() throws SecurityException { + public Constructor getEnclosingConstructor() { EnclosingMethodInfo enclosingInfo = getEnclosingMethodInfo(); if (enclosingInfo == null) @@ -1777,16 +1706,10 @@ public Constructor getEnclosingConstructor() throws SecurityException { * type, or void, then this method returns null. * * @return the declaring class for this class - * @throws SecurityException - * If a security manager, s, is present and the caller's - * class loader is not the same as or an ancestor of the class - * loader for the declaring class and invocation of {@link - * SecurityManager#checkPackageAccess s.checkPackageAccess()} - * denies access to the package of the declaring class * @since 1.1 */ @CallerSensitive - public Class getDeclaringClass() throws SecurityException { + public Class getDeclaringClass() { final Class candidate = getDeclaringClass0(); if (candidate != null) { @@ -1808,16 +1731,10 @@ public Class getDeclaringClass() throws SecurityException { * class. If the underlying class is a top level class this * method returns {@code null}. * @return the immediately enclosing class of the underlying class - * @throws SecurityException - * If a security manager, s, is present and the caller's - * class loader is not the same as or an ancestor of the class - * loader for the enclosing class and invocation of {@link - * SecurityManager#checkPackageAccess s.checkPackageAccess()} - * denies access to the package of the enclosing class * @since 1.5 */ @CallerSensitive - public Class getEnclosingClass() throws SecurityException { + public Class getEnclosingClass() { // There are five kinds of classes (or interfaces): // a) Top level classes // b) Nested classes (static member classes) @@ -2072,14 +1989,6 @@ private boolean hasEnclosingMethodInfo() { * * @return the array of {@code Class} objects representing the public * members of this class - * @throws SecurityException - * If a security manager, s, is present and - * the caller's class loader is not the same as or an - * ancestor of the class loader for the current class and - * invocation of {@link SecurityManager#checkPackageAccess - * s.checkPackageAccess()} denies access to the package - * of this class. - * * @since 1.1 */ @SuppressWarnings("removal") @@ -2140,20 +2049,13 @@ public Class[] run() { * * @return the array of {@code Field} objects representing the * public fields - * @throws SecurityException - * If a security manager, s, is present and - * the caller's class loader is not the same as or an - * ancestor of the class loader for the current class and - * invocation of {@link SecurityManager#checkPackageAccess - * s.checkPackageAccess()} denies access to the package - * of this class. * * @since 1.1 * @jls 8.2 Class Members * @jls 8.3 Field Declarations */ @CallerSensitive - public Field[] getFields() throws SecurityException { + public Field[] getFields() { @SuppressWarnings("removal") SecurityManager sm = System.getSecurityManager(); if (sm != null) { @@ -2231,20 +2133,13 @@ public Field[] getFields() throws SecurityException { * * @return the array of {@code Method} objects representing the * public methods of this class - * @throws SecurityException - * If a security manager, s, is present and - * the caller's class loader is not the same as or an - * ancestor of the class loader for the current class and - * invocation of {@link SecurityManager#checkPackageAccess - * s.checkPackageAccess()} denies access to the package - * of this class. * * @jls 8.2 Class Members * @jls 8.4 Method Declarations * @since 1.1 */ @CallerSensitive - public Method[] getMethods() throws SecurityException { + public Method[] getMethods() { @SuppressWarnings("removal") SecurityManager sm = System.getSecurityManager(); if (sm != null) { @@ -2274,19 +2169,12 @@ public Method[] getMethods() throws SecurityException { * * @return the array of {@code Constructor} objects representing the * public constructors of this class - * @throws SecurityException - * If a security manager, s, is present and - * the caller's class loader is not the same as or an - * ancestor of the class loader for the current class and - * invocation of {@link SecurityManager#checkPackageAccess - * s.checkPackageAccess()} denies access to the package - * of this class. * * @see #getDeclaredConstructors() * @since 1.1 */ @CallerSensitive - public Constructor[] getConstructors() throws SecurityException { + public Constructor[] getConstructors() { @SuppressWarnings("removal") SecurityManager sm = System.getSecurityManager(); if (sm != null) { @@ -2326,21 +2214,13 @@ public Constructor[] getConstructors() throws SecurityException { * @throws NoSuchFieldException if a field with the specified name is * not found. * @throws NullPointerException if {@code name} is {@code null} - * @throws SecurityException - * If a security manager, s, is present and - * the caller's class loader is not the same as or an - * ancestor of the class loader for the current class and - * invocation of {@link SecurityManager#checkPackageAccess - * s.checkPackageAccess()} denies access to the package - * of this class. * * @since 1.1 * @jls 8.2 Class Members * @jls 8.3 Field Declarations */ @CallerSensitive - public Field getField(String name) - throws NoSuchFieldException, SecurityException { + public Field getField(String name) throws NoSuchFieldException { Objects.requireNonNull(name); @SuppressWarnings("removal") SecurityManager sm = System.getSecurityManager(); @@ -2437,13 +2317,6 @@ public Field getField(String name) * or if the name is {@value ConstantDescs#INIT_NAME} or * {@value ConstantDescs#CLASS_INIT_NAME}. * @throws NullPointerException if {@code name} is {@code null} - * @throws SecurityException - * If a security manager, s, is present and - * the caller's class loader is not the same as or an - * ancestor of the class loader for the current class and - * invocation of {@link SecurityManager#checkPackageAccess - * s.checkPackageAccess()} denies access to the package - * of this class. * * @jls 8.2 Class Members * @jls 8.4 Method Declarations @@ -2451,7 +2324,7 @@ public Field getField(String name) */ @CallerSensitive public Method getMethod(String name, Class... parameterTypes) - throws NoSuchMethodException, SecurityException { + throws NoSuchMethodException { Objects.requireNonNull(name); @SuppressWarnings("removal") SecurityManager sm = System.getSecurityManager(); @@ -2486,21 +2359,13 @@ public Method getMethod(String name, Class... parameterTypes) * @throws NoSuchMethodException if a matching constructor is not found, * including when this {@code Class} object represents * an interface, a primitive type, an array class, or void. - * @throws SecurityException - * If a security manager, s, is present and - * the caller's class loader is not the same as or an - * ancestor of the class loader for the current class and - * invocation of {@link SecurityManager#checkPackageAccess - * s.checkPackageAccess()} denies access to the package - * of this class. - * - * @see #getDeclaredConstructor(Class[]) + * + * @see #getDeclaredConstructor(Class[]) * @since 1.1 */ @CallerSensitive public Constructor getConstructor(Class... parameterTypes) - throws NoSuchMethodException, SecurityException - { + throws NoSuchMethodException { @SuppressWarnings("removal") SecurityManager sm = System.getSecurityManager(); if (sm != null) { @@ -2523,32 +2388,12 @@ public Constructor getConstructor(Class... parameterTypes) * * @return the array of {@code Class} objects representing all the * declared members of this class - * @throws SecurityException - * If a security manager, s, is present and any of the - * following conditions is met: - * - *
    - * - *
  • the caller's class loader is not the same as the - * class loader of this class and invocation of - * {@link SecurityManager#checkPermission - * s.checkPermission} method with - * {@code RuntimePermission("accessDeclaredMembers")} - * denies access to the declared classes within this class - * - *
  • the caller's class loader is not the same as or an - * ancestor of the class loader for the current class and - * invocation of {@link SecurityManager#checkPackageAccess - * s.checkPackageAccess()} denies access to the package - * of this class - * - *
* * @since 1.1 * @jls 8.5 Member Class and Interface Declarations */ @CallerSensitive - public Class[] getDeclaredClasses() throws SecurityException { + public Class[] getDeclaredClasses() { @SuppressWarnings("removal") SecurityManager sm = System.getSecurityManager(); if (sm != null) { @@ -2575,33 +2420,13 @@ public Class[] getDeclaredClasses() throws SecurityException { * * @return the array of {@code Field} objects representing all the * declared fields of this class - * @throws SecurityException - * If a security manager, s, is present and any of the - * following conditions is met: - * - *
    - * - *
  • the caller's class loader is not the same as the - * class loader of this class and invocation of - * {@link SecurityManager#checkPermission - * s.checkPermission} method with - * {@code RuntimePermission("accessDeclaredMembers")} - * denies access to the declared fields within this class - * - *
  • the caller's class loader is not the same as or an - * ancestor of the class loader for the current class and - * invocation of {@link SecurityManager#checkPackageAccess - * s.checkPackageAccess()} denies access to the package - * of this class - * - *
* * @since 1.1 * @jls 8.2 Class Members * @jls 8.3 Field Declarations */ @CallerSensitive - public Field[] getDeclaredFields() throws SecurityException { + public Field[] getDeclaredFields() { @SuppressWarnings("removal") SecurityManager sm = System.getSecurityManager(); if (sm != null) { @@ -2638,26 +2463,6 @@ public Field[] getDeclaredFields() throws SecurityException { * @return An array of {@code RecordComponent} objects representing all the * record components of this record class, or {@code null} if this * class is not a record class - * @throws SecurityException - * If a security manager, s, is present and any of the - * following conditions is met: - * - *
    - * - *
  • the caller's class loader is not the same as the - * class loader of this class and invocation of - * {@link SecurityManager#checkPermission - * s.checkPermission} method with - * {@code RuntimePermission("accessDeclaredMembers")} - * denies access to the declared methods within this class - * - *
  • the caller's class loader is not the same as or an - * ancestor of the class loader for the current class and - * invocation of {@link SecurityManager#checkPackageAccess - * s.checkPackageAccess()} denies access to the package - * of this class - * - *
* * @jls 8.10 Record Classes * @since 16 @@ -2706,26 +2511,6 @@ public RecordComponent[] getRecordComponents() { * * @return the array of {@code Method} objects representing all the * declared methods of this class - * @throws SecurityException - * If a security manager, s, is present and any of the - * following conditions is met: - * - *
    - * - *
  • the caller's class loader is not the same as the - * class loader of this class and invocation of - * {@link SecurityManager#checkPermission - * s.checkPermission} method with - * {@code RuntimePermission("accessDeclaredMembers")} - * denies access to the declared methods within this class - * - *
  • the caller's class loader is not the same as or an - * ancestor of the class loader for the current class and - * invocation of {@link SecurityManager#checkPackageAccess - * s.checkPackageAccess()} denies access to the package - * of this class - * - *
* * @jls 8.2 Class Members * @jls 8.4 Method Declarations @@ -2735,7 +2520,7 @@ public RecordComponent[] getRecordComponents() { * @since 1.1 */ @CallerSensitive - public Method[] getDeclaredMethods() throws SecurityException { + public Method[] getDeclaredMethods() { @SuppressWarnings("removal") SecurityManager sm = System.getSecurityManager(); if (sm != null) { @@ -2760,33 +2545,13 @@ public Method[] getDeclaredMethods() throws SecurityException { * * @return the array of {@code Constructor} objects representing all the * declared constructors of this class - * @throws SecurityException - * If a security manager, s, is present and any of the - * following conditions is met: - * - *
    - * - *
  • the caller's class loader is not the same as the - * class loader of this class and invocation of - * {@link SecurityManager#checkPermission - * s.checkPermission} method with - * {@code RuntimePermission("accessDeclaredMembers")} - * denies access to the declared constructors within this class - * - *
  • the caller's class loader is not the same as or an - * ancestor of the class loader for the current class and - * invocation of {@link SecurityManager#checkPackageAccess - * s.checkPackageAccess()} denies access to the package - * of this class - * - *
* * @since 1.1 * @see #getConstructors() * @jls 8.8 Constructor Declarations */ @CallerSensitive - public Constructor[] getDeclaredConstructors() throws SecurityException { + public Constructor[] getDeclaredConstructors() { @SuppressWarnings("removal") SecurityManager sm = System.getSecurityManager(); if (sm != null) { @@ -2811,34 +2576,13 @@ public Constructor[] getDeclaredConstructors() throws SecurityException { * @throws NoSuchFieldException if a field with the specified name is * not found. * @throws NullPointerException if {@code name} is {@code null} - * @throws SecurityException - * If a security manager, s, is present and any of the - * following conditions is met: - * - *
    - * - *
  • the caller's class loader is not the same as the - * class loader of this class and invocation of - * {@link SecurityManager#checkPermission - * s.checkPermission} method with - * {@code RuntimePermission("accessDeclaredMembers")} - * denies access to the declared field - * - *
  • the caller's class loader is not the same as or an - * ancestor of the class loader for the current class and - * invocation of {@link SecurityManager#checkPackageAccess - * s.checkPackageAccess()} denies access to the package - * of this class - * - *
* * @since 1.1 * @jls 8.2 Class Members * @jls 8.3 Field Declarations */ @CallerSensitive - public Field getDeclaredField(String name) - throws NoSuchFieldException, SecurityException { + public Field getDeclaredField(String name) throws NoSuchFieldException { Objects.requireNonNull(name); @SuppressWarnings("removal") SecurityManager sm = System.getSecurityManager(); @@ -2877,26 +2621,6 @@ public Field getDeclaredField(String name) * matching the specified name and parameters * @throws NoSuchMethodException if a matching method is not found. * @throws NullPointerException if {@code name} is {@code null} - * @throws SecurityException - * If a security manager, s, is present and any of the - * following conditions is met: - * - *
    - * - *
  • the caller's class loader is not the same as the - * class loader of this class and invocation of - * {@link SecurityManager#checkPermission - * s.checkPermission} method with - * {@code RuntimePermission("accessDeclaredMembers")} - * denies access to the declared method - * - *
  • the caller's class loader is not the same as or an - * ancestor of the class loader for the current class and - * invocation of {@link SecurityManager#checkPackageAccess - * s.checkPackageAccess()} denies access to the package - * of this class - * - *
* * @jls 8.2 Class Members * @jls 8.4 Method Declarations @@ -2904,7 +2628,7 @@ public Field getDeclaredField(String name) */ @CallerSensitive public Method getDeclaredMethod(String name, Class... parameterTypes) - throws NoSuchMethodException, SecurityException { + throws NoSuchMethodException { Objects.requireNonNull(name); @SuppressWarnings("removal") SecurityManager sm = System.getSecurityManager(); @@ -2975,34 +2699,13 @@ Method findMethod(boolean publicOnly, String name, Class... parameterTypes) { * @throws NoSuchMethodException if a matching constructor is not found, * including when this {@code Class} object represents * an interface, a primitive type, an array class, or void. - * @throws SecurityException - * If a security manager, s, is present and any of the - * following conditions is met: - * - *
    - * - *
  • the caller's class loader is not the same as the - * class loader of this class and invocation of - * {@link SecurityManager#checkPermission - * s.checkPermission} method with - * {@code RuntimePermission("accessDeclaredMembers")} - * denies access to the declared constructor - * - *
  • the caller's class loader is not the same as or an - * ancestor of the class loader for the current class and - * invocation of {@link SecurityManager#checkPackageAccess - * s.checkPackageAccess()} denies access to the package - * of this class * - *
- * - * @see #getConstructor(Class[]) + * @see #getConstructor(Class[]) * @since 1.1 */ @CallerSensitive public Constructor getDeclaredConstructor(Class... parameterTypes) - throws NoSuchMethodException, SecurityException - { + throws NoSuchMethodException { @SuppressWarnings("removal") SecurityManager sm = System.getSecurityManager(); if (sm != null) { @@ -3058,10 +2761,9 @@ public Constructor getDeclaredConstructor(Class... parameterTypes) * * @param name name of the desired resource * @return A {@link java.io.InputStream} object; {@code null} if no - * resource with this name is found, the resource is in a package + * resource with this name is found, or the resource is in a package * that is not {@linkplain Module#isOpen(String, Module) open} to at - * least the caller module, or access to the resource is denied - * by the security manager. + * least the caller module. * @throws NullPointerException If {@code name} is {@code null} * * @see Module#getResourceAsStream(String) @@ -3154,11 +2856,10 @@ public InputStream getResourceAsStream(String name) { * * @param name name of the desired resource * @return A {@link java.net.URL} object; {@code null} if no resource with - * this name is found, the resource cannot be located by a URL, the + * this name is found, the resource cannot be located by a URL, or the * resource is in a package that is not * {@linkplain Module#isOpen(String, Module) open} to at least the caller - * module, or access to the resource is denied by the security - * manager. + * module. * @throws NullPointerException If {@code name} is {@code null} * @since 1.1 */ @@ -3224,23 +2925,11 @@ private boolean isOpenToCaller(String name, Class caller) { } /** - * Returns the {@code ProtectionDomain} of this class. If there is a - * security manager installed, this method first calls the security - * manager's {@code checkPermission} method with a - * {@code RuntimePermission("getProtectionDomain")} permission to - * ensure it's ok to get the - * {@code ProtectionDomain}. + * Returns the {@code ProtectionDomain} of this class. * * @return the ProtectionDomain of this class * - * @throws SecurityException - * if a security manager exists and its - * {@code checkPermission} method doesn't allow - * getting the ProtectionDomain. - * * @see java.security.ProtectionDomain - * @see SecurityManager#checkPermission - * @see java.lang.RuntimePermission * @since 1.2 */ public ProtectionDomain getProtectionDomain() { @@ -4466,13 +4155,6 @@ public AnnotatedType[] getAnnotatedInterfaces() { * * @return the nest host of this class or interface * - * @throws SecurityException - * If the returned class is not the current class, and - * if a security manager, s, is present and the caller's - * class loader is not the same as or an ancestor of the class - * loader for the returned class and invocation of {@link - * SecurityManager#checkPackageAccess s.checkPackageAccess()} - * denies access to the package of the returned class * @since 11 * @jvms 4.7.28 The {@code NestHost} Attribute * @jvms 4.7.29 The {@code NestMembers} Attribute @@ -4557,14 +4239,6 @@ public boolean isNestmateOf(Class c) { * @return an array of all classes and interfaces in the same nest as * this class or interface * - * @throws SecurityException - * If any returned class is not the current class, and - * if a security manager, s, is present and the caller's - * class loader is not the same as or an ancestor of the class - * loader for that returned class and invocation of {@link - * SecurityManager#checkPackageAccess s.checkPackageAccess()} - * denies access to the package of that returned class - * * @since 11 * @see #getNestHost() * @jvms 4.7.28 The {@code NestHost} Attribute @@ -4751,15 +4425,8 @@ public Optional describeConstable() { * cannot be obtained, it is silently ignored, and not included in the result * array. * - * @return an array of {@code Class} objects of the permitted subclasses of this class or interface, - * or {@code null} if this class or interface is not sealed. - * - * @throws SecurityException - * If a security manager, s, is present and the caller's - * class loader is not the same as or an ancestor of the class - * loader for that returned class and invocation of {@link - * SecurityManager#checkPackageAccess s.checkPackageAccess()} - * denies access to the package of any class in the returned array. + * @return an array of {@code Class} objects of the permitted subclasses of this class + * or interface, or {@code null} if this class or interface is not sealed. * * @jls 8.1 Class Declarations * @jls 9.1 Interface Declarations diff --git a/src/java.base/share/classes/java/lang/ClassLoader.java b/src/java.base/share/classes/java/lang/ClassLoader.java index fafa8895ee6..85fc315c767 100644 --- a/src/java.base/share/classes/java/lang/ClassLoader.java +++ b/src/java.base/share/classes/java/lang/ClassLoader.java @@ -33,10 +33,7 @@ import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; import java.net.URL; -import java.security.AccessController; -import java.security.AccessControlContext; import java.security.CodeSource; -import java.security.PrivilegedAction; import java.security.ProtectionDomain; import java.security.cert.Certificate; import java.util.ArrayList; @@ -67,7 +64,6 @@ import jdk.internal.reflect.CallerSensitiveAdapter; import jdk.internal.reflect.Reflection; import jdk.internal.util.StaticProperty; -import sun.reflect.misc.ReflectUtil; import sun.security.util.SecurityConstants; /** @@ -93,9 +89,6 @@ * extend the manner in which the Java virtual machine dynamically loads * classes. * - *

Class loaders may typically be used by security managers to indicate - * security domains. - * *

In addition to loading classes, a class loader is also responsible for * locating resources. A resource is some data (a "{@code .class}" file, * configuration data, or an image for example) that is identified with an @@ -424,11 +417,6 @@ String nameAndId() { * * @throws IllegalArgumentException if the given name is empty. * - * @throws SecurityException - * If a security manager exists and its - * {@link SecurityManager#checkCreateClassLoader()} - * method doesn't allow creation of a new class loader. - * * @since 9 */ @SuppressWarnings("this-escape") @@ -440,10 +428,6 @@ protected ClassLoader(String name, ClassLoader parent) { * Creates a new class loader using the specified parent class loader for * delegation. * - *

If there is a security manager, its {@link - * SecurityManager#checkCreateClassLoader() checkCreateClassLoader} method - * is invoked. This may result in a security exception.

- * * @apiNote If the parent is specified as {@code null} (for the * bootstrap class loader) then there is no guarantee that all platform * classes are visible. @@ -451,11 +435,6 @@ protected ClassLoader(String name, ClassLoader parent) { * @param parent * The parent class loader * - * @throws SecurityException - * If a security manager exists and its - * {@code checkCreateClassLoader} method doesn't allow creation - * of a new class loader. - * * @since 1.2 */ @SuppressWarnings("this-escape") @@ -467,16 +446,6 @@ protected ClassLoader(ClassLoader parent) { * Creates a new class loader using the {@code ClassLoader} returned by * the method {@link #getSystemClassLoader() * getSystemClassLoader()} as the parent class loader. - * - *

If there is a security manager, its {@link - * SecurityManager#checkCreateClassLoader() - * checkCreateClassLoader} method is invoked. This may result in - * a security exception.

- * - * @throws SecurityException - * If a security manager exists and its - * {@code checkCreateClassLoader} method doesn't allow creation - * of a new class loader. */ @SuppressWarnings("this-escape") protected ClassLoader() { @@ -688,30 +657,6 @@ protected Object getClassLoadingLock(String className) { return lock; } - // Invoked by the VM after loading class with this loader. - @SuppressWarnings("removal") - private void checkPackageAccess(Class cls, ProtectionDomain pd) { - final SecurityManager sm = System.getSecurityManager(); - if (sm != null) { - if (ReflectUtil.isNonPublicProxyClass(cls)) { - for (Class intf: cls.getInterfaces()) { - checkPackageAccess(intf, pd); - } - return; - } - - final String packageName = cls.getPackageName(); - if (!packageName.isEmpty()) { - AccessController.doPrivileged(new PrivilegedAction<>() { - public Void run() { - sm.checkPackageAccess(packageName); - return null; - } - }, new AccessControlContext(new ProtectionDomain[] {pd})); - } - } - } - /** * Finds the class with the specified binary name. * This method should be overridden by class loader implementations that @@ -823,12 +768,10 @@ protected final Class defineClass(byte[] b, int off, int len) * Before the {@code Class} can be used it must be resolved. * *

This method assigns a default {@link java.security.ProtectionDomain - * ProtectionDomain} to the newly defined class. The - * {@code ProtectionDomain} is effectively granted the same set of - * permissions returned when {@link - * java.security.Policy#getPermissions(java.security.CodeSource) - * Policy.getPolicy().getPermissions(new CodeSource(null, null))} - * is invoked. The default protection domain is created on the first invocation + * ProtectionDomain} to the newly defined class. The + * {@code getPermissions} method of the {@code ProtectionDomain} always + * returns {@code null}. + * The default protection domain is created on the first invocation * of {@link #defineClass(String, byte[], int, int) defineClass}, * and re-used on subsequent invocations. * @@ -1342,8 +1285,7 @@ protected final void setSigners(Class c, Object[] signers) { * The resource name * * @return A URL to the resource; {@code null} if the resource could not be - * found, a URL could not be constructed to locate the resource, - * access to the resource is denied by the security manager, or + * found, a URL could not be constructed to locate the resource, or * there isn't a module of the given name defined to the class * loader. * @@ -1395,9 +1337,8 @@ protected URL findResource(String moduleName, String name) throws IOException { * * @return {@code URL} object for reading the resource; {@code null} if * the resource could not be found, a {@code URL} could not be - * constructed to locate the resource, the resource is in a package - * that is not opened unconditionally, or access to the resource is - * denied by the security manager. + * constructed to locate the resource, or the resource is in a package + * that is not opened unconditionally. * * @throws NullPointerException If {@code name} is {@code null} * @@ -1457,9 +1398,8 @@ public URL getResource(String name) { * @return An enumeration of {@link java.net.URL URL} objects for the * resource. If no resources could be found, the enumeration will * be empty. Resources for which a {@code URL} cannot be - * constructed, are in a package that is not opened - * unconditionally, or access to the resource is denied by the - * security manager, are not returned in the enumeration. + * constructed, or are in a package that is not opened + * unconditionally, are not returned in the enumeration. * * @throws IOException * If I/O errors occur @@ -1518,9 +1458,8 @@ public Enumeration getResources(String name) throws IOException { * * @return A stream of resource {@link java.net.URL URL} objects. If no * resources could be found, the stream will be empty. Resources - * for which a {@code URL} cannot be constructed, are in a package - * that is not opened unconditionally, or access to the resource - * is denied by the security manager, will not be in the stream. + * for which a {@code URL} cannot be constructed, or are in a package + * that is not opened unconditionally, will not be in the stream. * * @throws NullPointerException If {@code name} is {@code null} * @@ -1558,9 +1497,8 @@ public Stream resources(String name) { * * @return {@code URL} object for reading the resource; {@code null} if * the resource could not be found, a {@code URL} could not be - * constructed to locate the resource, the resource is in a package - * that is not opened unconditionally, or access to the resource is - * denied by the security manager. + * constructed to locate the resource, or the resource is in a package + * that is not opened unconditionally. * * @since 1.2 */ @@ -1589,8 +1527,7 @@ protected URL findResource(String name) { * @return An enumeration of {@link java.net.URL URL} objects for * the resource. If no resources could be found, the enumeration * will be empty. Resources for which a {@code URL} cannot be - * constructed, are in a package that is not opened unconditionally, - * or access to the resource is denied by the security manager, + * constructed, or are in a package that is not opened unconditionally, * are not returned in the enumeration. * * @throws IOException @@ -1676,9 +1613,8 @@ public final boolean isRegisteredAsParallelCapable() { * * @return A {@link java.net.URL URL} to the resource; {@code * null} if the resource could not be found, a URL could not be - * constructed to locate the resource, the resource is in a package - * that is not opened unconditionally or access to the resource is - * denied by the security manager. + * constructed to locate the resource, or the resource is in a package + * that is not opened unconditionally. * * @since 1.1 */ @@ -1708,8 +1644,7 @@ public static URL getSystemResource(String name) { * @return An enumeration of {@link java.net.URL URL} objects for * the resource. If no resources could be found, the enumeration * will be empty. Resources for which a {@code URL} cannot be - * constructed, are in a package that is not opened unconditionally, - * or access to the resource is denied by the security manager, + * constructed, or are in a package that is not opened unconditionally, * are not returned in the enumeration. * * @throws IOException @@ -1740,9 +1675,8 @@ public static Enumeration getSystemResources(String name) * The resource name * * @return An input stream for reading the resource; {@code null} if the - * resource could not be found, the resource is in a package that - * is not opened unconditionally, or access to the resource is - * denied by the security manager. + * resource could not be found, or the resource is in a package that + * is not opened unconditionally. * * @throws NullPointerException If {@code name} is {@code null} * @@ -1774,9 +1708,8 @@ public InputStream getResourceAsStream(String name) { * The resource name * * @return An input stream for reading the resource; {@code null} if the - * resource could not be found, the resource is in a package that - * is not opened unconditionally, or access to the resource is - * denied by the security manager. + * resource could not be found, or the resource is in a package that + * is not opened unconditionally. * * @since 1.1 */ @@ -1800,12 +1733,6 @@ public static InputStream getSystemResourceAsStream(String name) { * * @return The parent {@code ClassLoader} * - * @throws SecurityException - * If a security manager is present, and the caller's class loader - * is not {@code null} and is not an ancestor of this class loader, - * and the caller does not have the - * {@link RuntimePermission}{@code ("getClassLoader")} - * * @since 1.2 */ @CallerSensitive @@ -1845,13 +1772,6 @@ public final Module getUnnamedModule() { * * @return The platform {@code ClassLoader}. * - * @throws SecurityException - * If a security manager is present, and the caller's class loader is - * not {@code null}, and the caller's class loader is not the same - * as or an ancestor of the platform class loader, - * and the caller does not have the - * {@link RuntimePermission}{@code ("getClassLoader")} - * * @since 9 */ @CallerSensitive @@ -1920,12 +1840,6 @@ public static ClassLoader getPlatformClassLoader() { * * @return The system {@code ClassLoader} * - * @throws SecurityException - * If a security manager is present, and the caller's class loader - * is not {@code null} and is not the same as or an ancestor of the - * system class loader, and the caller does not have the - * {@link RuntimePermission}{@code ("getClassLoader")} - * * @throws IllegalStateException * If invoked recursively during the construction of the class * loader specified by the "{@code java.system.class.loader}" diff --git a/src/java.base/share/classes/java/lang/Double.java b/src/java.base/share/classes/java/lang/Double.java index ed23f7d39c9..8f7b6c6aa42 100644 --- a/src/java.base/share/classes/java/lang/Double.java +++ b/src/java.base/share/classes/java/lang/Double.java @@ -375,9 +375,9 @@ public final class Double extends Number public static final double NEGATIVE_INFINITY = -1.0 / 0.0; /** - * A constant holding a Not-a-Number (NaN) value of type - * {@code double}. It is equivalent to the value returned by - * {@code Double.longBitsToDouble(0x7ff8000000000000L)}. + * A constant holding a Not-a-Number (NaN) value of type {@code double}. + * It is {@linkplain Double##equivalenceRelation equivalent} to the + * value returned by {@code Double.longBitsToDouble(0x7ff8000000000000L)}. */ public static final double NaN = 0.0d / 0.0; diff --git a/src/java.base/share/classes/java/lang/Float.java b/src/java.base/share/classes/java/lang/Float.java index 821a05fa00a..85b20e6a2e1 100644 --- a/src/java.base/share/classes/java/lang/Float.java +++ b/src/java.base/share/classes/java/lang/Float.java @@ -93,9 +93,9 @@ public final class Float extends Number public static final float NEGATIVE_INFINITY = -1.0f / 0.0f; /** - * A constant holding a Not-a-Number (NaN) value of type - * {@code float}. It is equivalent to the value returned by - * {@code Float.intBitsToFloat(0x7fc00000)}. + * A constant holding a Not-a-Number (NaN) value of type {@code float}. + * It is {@linkplain Double##equivalenceRelation equivalent} + * to the value returned by{@code Float.intBitsToFloat(0x7fc00000)}. */ public static final float NaN = 0.0f / 0.0f; diff --git a/src/java.base/share/classes/java/lang/Integer.java b/src/java.base/share/classes/java/lang/Integer.java index e666e977c61..8bf573733f2 100644 --- a/src/java.base/share/classes/java/lang/Integer.java +++ b/src/java.base/share/classes/java/lang/Integer.java @@ -1187,8 +1187,6 @@ public boolean equals(Object obj) { * * @param nm property name. * @return the {@code Integer} value of the property. - * @throws SecurityException for the same reasons as - * {@link System#getProperty(String) System.getProperty} * @see java.lang.System#getProperty(java.lang.String) * @see java.lang.System#getProperty(java.lang.String, java.lang.String) */ @@ -1233,8 +1231,6 @@ public static Integer getInteger(String nm) { * @param nm property name. * @param val default value. * @return the {@code Integer} value of the property. - * @throws SecurityException for the same reasons as - * {@link System#getProperty(String) System.getProperty} * @see java.lang.System#getProperty(java.lang.String) * @see java.lang.System#getProperty(java.lang.String, java.lang.String) */ @@ -1275,8 +1271,6 @@ public static Integer getInteger(String nm, int val) { * @param nm property name. * @param val default value. * @return the {@code Integer} value of the property. - * @throws SecurityException for the same reasons as - * {@link System#getProperty(String) System.getProperty} * @see System#getProperty(java.lang.String) * @see System#getProperty(java.lang.String, java.lang.String) */ diff --git a/src/java.base/share/classes/java/lang/Long.java b/src/java.base/share/classes/java/lang/Long.java index 8c083b3ec84..e67b751470e 100644 --- a/src/java.base/share/classes/java/lang/Long.java +++ b/src/java.base/share/classes/java/lang/Long.java @@ -1276,8 +1276,6 @@ public boolean equals(Object obj) { * * @param nm property name. * @return the {@code Long} value of the property. - * @throws SecurityException for the same reasons as - * {@link System#getProperty(String) System.getProperty} * @see java.lang.System#getProperty(java.lang.String) * @see java.lang.System#getProperty(java.lang.String, java.lang.String) */ @@ -1321,8 +1319,6 @@ public static Long getLong(String nm) { * @param nm property name. * @param val default value. * @return the {@code Long} value of the property. - * @throws SecurityException for the same reasons as - * {@link System#getProperty(String) System.getProperty} * @see java.lang.System#getProperty(java.lang.String) * @see java.lang.System#getProperty(java.lang.String, java.lang.String) */ @@ -1370,8 +1366,6 @@ public static Long getLong(String nm, long val) { * @param nm property name. * @param val default value. * @return the {@code Long} value of the property. - * @throws SecurityException for the same reasons as - * {@link System#getProperty(String) System.getProperty} * @see System#getProperty(java.lang.String) * @see System#getProperty(java.lang.String, java.lang.String) */ diff --git a/src/java.base/share/classes/java/lang/Module.java b/src/java.base/share/classes/java/lang/Module.java index 4d4593b3197..a90fbc99260 100644 --- a/src/java.base/share/classes/java/lang/Module.java +++ b/src/java.base/share/classes/java/lang/Module.java @@ -195,15 +195,7 @@ public String getName() { /** * Returns the {@code ClassLoader} for this module. * - *

If there is a security manager then its {@code checkPermission} - * method if first called with a {@code RuntimePermission("getClassLoader")} - * permission to check that the caller is allowed to get access to the - * class loader.

- * * @return The class loader for this module - * - * @throws SecurityException - * If denied by the security manager */ public ClassLoader getClassLoader() { @SuppressWarnings("removal") @@ -1689,9 +1681,8 @@ protected Class loadClass(String cn, boolean resolve) * with the name "{@code META-INF/MANIFEST.MF}" is never encapsulated * because "{@code META-INF}" is not a legal package name.

* - *

This method returns {@code null} if the resource is not in this - * module, the resource is encapsulated and cannot be located by the caller, - * or access to the resource is denied by the security manager.

+ *

This method returns {@code null} if the resource is not in this module + * or the resource is encapsulated and cannot be located by the caller.

* * @param name * The resource name diff --git a/src/java.base/share/classes/java/lang/ModuleLayer.java b/src/java.base/share/classes/java/lang/ModuleLayer.java index 80d392470cb..4ee2b02414d 100644 --- a/src/java.base/share/classes/java/lang/ModuleLayer.java +++ b/src/java.base/share/classes/java/lang/ModuleLayer.java @@ -359,10 +359,6 @@ public Controller enableNativeAccess(Module target) { * @throws LayerInstantiationException * If the layer cannot be created for any of the reasons specified * by the static {@code defineModulesWithOneLoader} method - * @throws SecurityException - * If {@code RuntimePermission("createClassLoader")} or - * {@code RuntimePermission("getClassLoader")} is denied by - * the security manager * * @see #findLoader */ @@ -401,10 +397,6 @@ public ModuleLayer defineModulesWithOneLoader(Configuration cf, * @throws LayerInstantiationException * If the layer cannot be created for any of the reasons specified * by the static {@code defineModulesWithManyLoaders} method - * @throws SecurityException - * If {@code RuntimePermission("createClassLoader")} or - * {@code RuntimePermission("getClassLoader")} is denied by - * the security manager * * @see #findLoader */ @@ -440,9 +432,6 @@ public ModuleLayer defineModulesWithManyLoaders(Configuration cf, * @throws LayerInstantiationException * If the layer cannot be created for any of the reasons specified * by the static {@code defineModules} method - * @throws SecurityException - * If {@code RuntimePermission("getClassLoader")} is denied by - * the security manager */ public ModuleLayer defineModules(Configuration cf, Function clf) { @@ -490,10 +479,6 @@ public ModuleLayer defineModules(Configuration cf, * a module named "{@code java.base}", or a module contains a package named * "{@code java}" or a package with a name starting with "{@code java.}".

* - *

If there is a security manager then the class loader created by - * this method will load classes and resources with privileges that are - * restricted by the calling context of this method.

- * * @param cf * The configuration for the layer * @param parentLayers @@ -510,10 +495,6 @@ public ModuleLayer defineModules(Configuration cf, * @throws LayerInstantiationException * If all modules cannot be defined to the same class loader for any * of the reasons listed above - * @throws SecurityException - * If {@code RuntimePermission("createClassLoader")} or - * {@code RuntimePermission("getClassLoader")} is denied by - * the security manager * * @see #findLoader */ @@ -563,10 +544,6 @@ public static Controller defineModulesWithOneLoader(Configuration cf, * methods) in the module defined to the class loader before searching * the parent class loader.

* - *

If there is a security manager then the class loaders created by - * this method will load classes and resources with privileges that are - * restricted by the calling context of this method.

- * * @param cf * The configuration for the layer * @param parentLayers @@ -586,11 +563,6 @@ public static Controller defineModulesWithOneLoader(Configuration cf, * named "{@code java}" or a package with a name starting with * "{@code java.}" * - * @throws SecurityException - * If {@code RuntimePermission("createClassLoader")} or - * {@code RuntimePermission("getClassLoader")} is denied by - * the security manager - * * @see #findLoader */ public static Controller defineModulesWithManyLoaders(Configuration cf, @@ -673,9 +645,6 @@ public static Controller defineModulesWithManyLoaders(Configuration cf, * configuration of the parent layers, including order * @throws LayerInstantiationException * If creating the layer fails for any of the reasons listed above - * @throws SecurityException - * If {@code RuntimePermission("getClassLoader")} is denied by - * the security manager */ public static Controller defineModules(Configuration cf, List parentLayers, @@ -906,11 +875,6 @@ boolean addEnableNativeAccess(String name) { * parent} layers are searched in the manner specified by {@link * #findModule(String) findModule}. * - *

If there is a security manager then its {@code checkPermission} - * method is called with a {@code RuntimePermission("getClassLoader")} - * permission to check that the caller is allowed to get access to the - * class loader.

- * * @apiNote This method does not return an {@code Optional} * because `null` must be used to represent the bootstrap class loader. * @@ -921,8 +885,6 @@ boolean addEnableNativeAccess(String name) { * * @throws IllegalArgumentException if a module of the given name is not * defined in this layer or any parent of this layer - * - * @throws SecurityException if denied by the security manager */ public ClassLoader findLoader(String name) { Optional om = findModule(name); diff --git a/src/java.base/share/classes/java/lang/Object.java b/src/java.base/share/classes/java/lang/Object.java index 7909f053042..aa9c8f31237 100644 --- a/src/java.base/share/classes/java/lang/Object.java +++ b/src/java.base/share/classes/java/lang/Object.java @@ -25,7 +25,6 @@ package java.lang; -import jdk.internal.misc.Blocker; import jdk.internal.vm.annotation.IntrinsicCandidate; /** @@ -374,21 +373,20 @@ public final void wait() throws InterruptedException { * @see #wait(long, int) */ public final void wait(long timeoutMillis) throws InterruptedException { - if (!Thread.currentThread().isVirtual()) { - wait0(timeoutMillis); - return; + if (timeoutMillis < 0) { + throw new IllegalArgumentException("timeout value is negative"); } - // virtual thread waiting - boolean attempted = Blocker.begin(); - try { + if (Thread.currentThread() instanceof VirtualThread vthread) { + try { + wait0(timeoutMillis); + } catch (InterruptedException e) { + // virtual thread's interrupt status needs to be cleared + vthread.getAndClearInterrupt(); + throw e; + } + } else { wait0(timeoutMillis); - } catch (InterruptedException e) { - // virtual thread's interrupt status needs to be cleared - Thread.currentThread().getAndClearInterrupt(); - throw e; - } finally { - Blocker.end(attempted); } } diff --git a/src/java.base/share/classes/java/lang/PinnedThreadPrinter.java b/src/java.base/share/classes/java/lang/PinnedThreadPrinter.java deleted file mode 100644 index a9b40d028f5..00000000000 --- a/src/java.base/share/classes/java/lang/PinnedThreadPrinter.java +++ /dev/null @@ -1,151 +0,0 @@ -/* - * Copyright (c) 2020, 2023, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. Oracle designates this - * particular file as subject to the "Classpath" exception as provided - * by Oracle in the LICENSE file that accompanied this code. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - */ -package java.lang; - -import java.io.PrintStream; -import java.security.AccessController; -import java.security.PrivilegedAction; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.stream.Collectors; -import static java.lang.StackWalker.Option.*; -import jdk.internal.access.JavaIOPrintStreamAccess; -import jdk.internal.access.SharedSecrets; -import jdk.internal.misc.InternalLock; -import jdk.internal.vm.Continuation; - -/** - * Helper class to print the virtual thread stack trace when pinned. - * - * The class maintains a ClassValue with the hashes of stack traces that are pinned by - * code in that Class. This is used to avoid printing the same stack trace many times. - */ -class PinnedThreadPrinter { - private static final JavaIOPrintStreamAccess JIOPSA = SharedSecrets.getJavaIOPrintStreamAccess(); - private static final StackWalker STACK_WALKER; - static { - var options = Set.of(SHOW_REFLECT_FRAMES, RETAIN_CLASS_REFERENCE); - PrivilegedAction pa = () -> - LiveStackFrame.getStackWalker(options, VirtualThread.continuationScope()); - @SuppressWarnings("removal") - var stackWalker = AccessController.doPrivileged(pa); - STACK_WALKER = stackWalker; - } - - private static final ClassValue HASHES = new ClassValue<>() { - @Override - protected Hashes computeValue(Class type) { - return new Hashes(); - } - }; - - @SuppressWarnings("serial") - private static class Hashes extends LinkedHashMap { - boolean add(int hash) { - return (putIfAbsent(hash, Boolean.TRUE) == null); - } - @Override - protected boolean removeEldestEntry(Map.Entry oldest) { - // limit number of hashes - return size() > 8; - } - } - - /** - * Returns a hash of the given stack trace. The hash is based on the class, - * method and bytecode index. - */ - private static int hash(List stack) { - int hash = 0; - for (LiveStackFrame frame : stack) { - hash = (31 * hash) + Objects.hash(frame.getDeclaringClass(), - frame.getMethodName(), - frame.getByteCodeIndex()); - } - return hash; - } - - /** - * Returns true if the frame is native, a class initializer, or holds monitors. - */ - private static boolean isInterestingFrame(LiveStackFrame f) { - return f.isNativeMethod() - || "".equals(f.getMethodName()) - || (f.getMonitors().length > 0); - } - - /** - * Prints the current thread's stack trace. - * - * @param printAll true to print all stack frames, false to only print the - * frames that are native or holding a monitor - */ - static void printStackTrace(PrintStream out, Continuation.Pinned reason, boolean printAll) { - List stack = STACK_WALKER.walk(s -> - s.map(f -> (LiveStackFrame) f) - .filter(f -> f.getDeclaringClass() != PinnedThreadPrinter.class) - .collect(Collectors.toList()) - ); - Object lockObj = JIOPSA.lock(out); - if (lockObj instanceof InternalLock lock && lock.tryLock()) { - try { - // find the closest frame that is causing the thread to be pinned - stack.stream() - .filter(f -> isInterestingFrame(f)) - .map(LiveStackFrame::getDeclaringClass) - .findFirst() - .ifPresentOrElse(klass -> { - // print the stack trace if not already seen - int hash = hash(stack); - if (HASHES.get(klass).add(hash)) { - printStackTrace(out, reason, stack, printAll); - } - }, () -> printStackTrace(out, reason, stack, true)); // not found - - } finally { - lock.unlock(); - } - } - } - - private static void printStackTrace(PrintStream out, - Continuation.Pinned reason, - List stack, - boolean printAll) { - out.format("%s reason:%s%n", Thread.currentThread(), reason); - for (LiveStackFrame frame : stack) { - var ste = frame.toStackTraceElement(); - int monitorCount = frame.getMonitors().length; - if (monitorCount > 0) { - out.format(" %s <== monitors:%d%n", ste, monitorCount); - } else if (printAll || isInterestingFrame(frame)) { - out.format(" %s%n", ste); - } - } - } -} diff --git a/src/java.base/share/classes/java/lang/Process.java b/src/java.base/share/classes/java/lang/Process.java index af6753a236b..3e4837d2e02 100644 --- a/src/java.base/share/classes/java/lang/Process.java +++ b/src/java.base/share/classes/java/lang/Process.java @@ -753,8 +753,7 @@ public boolean isReleasable() { * * {@code Process} objects returned by {@link ProcessBuilder#start()} and * {@link Runtime#exec} implement {@code toHandle} as the equivalent of - * {@link ProcessHandle#of(long) ProcessHandle.of(pid)} including the - * check for a SecurityManager and {@code RuntimePermission("manageProcess")}. + * {@link ProcessHandle#of(long) ProcessHandle.of(pid)}. * * @implSpec * This implementation throws an instance of @@ -766,8 +765,6 @@ public boolean isReleasable() { * @return Returns a ProcessHandle for the Process * @throws UnsupportedOperationException if the Process implementation * does not support this operation - * @throws SecurityException if a security manager has been installed and - * it denies RuntimePermission("manageProcess") * @since 9 */ public ProcessHandle toHandle() { @@ -811,8 +808,6 @@ public ProcessHandle.Info info() { * direct children of the process * @throws UnsupportedOperationException if the Process implementation * does not support this operation - * @throws SecurityException if a security manager has been installed and - * it denies RuntimePermission("manageProcess") * @since 9 */ public Stream children() { @@ -837,8 +832,6 @@ public Stream children() { * are descendants of the process * @throws UnsupportedOperationException if the Process implementation * does not support this operation - * @throws SecurityException if a security manager has been installed and - * it denies RuntimePermission("manageProcess") * @since 9 */ public Stream descendants() { diff --git a/src/java.base/share/classes/java/lang/ProcessBuilder.java b/src/java.base/share/classes/java/lang/ProcessBuilder.java index e7d5d9debef..9cb5848bdff 100644 --- a/src/java.base/share/classes/java/lang/ProcessBuilder.java +++ b/src/java.base/share/classes/java/lang/ProcessBuilder.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2003, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2003, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -329,32 +329,16 @@ public List command() { * *

The returned map is typically case-sensitive on all platforms. * - *

If a security manager exists, its - * {@link SecurityManager#checkPermission checkPermission} method - * is called with a - * {@link RuntimePermission}{@code ("getenv.*")} permission. - * This may result in a {@link SecurityException} being thrown. - * *

When passing information to a Java subprocess, * system properties * are generally preferred over environment variables. * * @return this process builder's environment * - * @throws SecurityException - * if a security manager exists and its - * {@link SecurityManager#checkPermission checkPermission} - * method doesn't allow access to the process environment - * * @see Runtime#exec(String[],String[],java.io.File) * @see System#getenv() */ public Map environment() { - @SuppressWarnings("removal") - SecurityManager security = System.getSecurityManager(); - if (security != null) - security.checkPermission(new RuntimePermission("getenv.*")); - if (environment == null) environment = ProcessEnvironment.environment(); @@ -1009,12 +993,6 @@ public ProcessBuilder redirectErrorStream(boolean redirectErrorStream) { * The minimal set of system dependent environment variables * may override the values provided in the environment. * - *

If there is a security manager, its - * {@link SecurityManager#checkExec checkExec} - * method is called with the first component of this object's - * {@code command} array as its argument. This may result in - * a {@link SecurityException} being thrown. - * *

Starting an operating system process is highly system-dependent. * Among the many things that can go wrong are: *