Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Track virtual memory with native memory tracking (NMT) #8630

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Track virtual memory with native memory tracking (NMT) #8630

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
3 changes: 2 additions & 1 deletion ...ratevm/src/com.oracle.svm.core.foreign/src/com/oracle/svm/core/foreign/TrampolineSet.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@
import org.graalvm.word.WordFactory;

import com.oracle.svm.core.heap.OutOfMemoryUtil;
import com.oracle.svm.core.nmt.NmtCategory;
import com.oracle.svm.core.os.VirtualMemoryProvider;
import com.oracle.svm.core.util.VMError;

Expand Down Expand Up @@ -112,7 +113,7 @@ private Pointer prepareTrampolines(PinnedObject mhsArray, PinnedObject stubsArra
VirtualMemoryProvider memoryProvider = VirtualMemoryProvider.get();
UnsignedWord pageSize = allocationSize();
/* We request a specific alignment to guarantee correctness of getAllocationBase */
Pointer page = memoryProvider.commit(WordFactory.nullPointer(), pageSize, VirtualMemoryProvider.Access.WRITE | VirtualMemoryProvider.Access.FUTURE_EXECUTE);
Pointer page = memoryProvider.commit(WordFactory.nullPointer(), pageSize, VirtualMemoryProvider.Access.WRITE | VirtualMemoryProvider.Access.FUTURE_EXECUTE, NmtCategory.Internal);
if (page.isNull()) {
throw OutOfMemoryUtil.reportOutOfMemoryError(new OutOfMemoryError("Could not allocate memory for trampolines."));
}
Expand Down
117 changes: 108 additions & 9 deletions ...c/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/PosixVirtualMemoryProvider.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,10 +54,15 @@
import com.oracle.svm.core.c.CGlobalDataFactory;
import com.oracle.svm.core.feature.AutomaticallyRegisteredFeature;
import com.oracle.svm.core.feature.InternalFeature;
import com.oracle.svm.core.nmt.NativeMemoryTracking;
import com.oracle.svm.core.nmt.NmtCategory;
import com.oracle.svm.core.nmt.NmtPreImageHeapData;
import com.oracle.svm.core.nmt.NmtPreImageHeapDataAccess;
import com.oracle.svm.core.os.VirtualMemoryProvider;
import com.oracle.svm.core.posix.headers.Unistd;
import com.oracle.svm.core.util.PointerUtils;
import com.oracle.svm.core.util.UnsignedUtils;
import com.oracle.svm.core.VMInspectionOptions;

@AutomaticallyRegisteredFeature
class PosixVirtualMemoryProviderFeature implements InternalFeature {
Expand Down Expand Up @@ -109,7 +114,18 @@ public UnsignedWord getGranularity() {

@Override
@Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
public Pointer reserve(UnsignedWord nbytes, UnsignedWord alignment, boolean executable) {
public Pointer reserve(UnsignedWord nbytes, UnsignedWord alignment, boolean executable, NmtPreImageHeapData nmtData) {
return reserve0(nbytes, alignment, executable, nmtData, NmtCategory.ImageHeap);
}

@Override
@Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
public Pointer reserve(UnsignedWord nbytes, UnsignedWord alignment, boolean executable, NmtCategory category) {
return reserve0(nbytes, alignment, executable, WordFactory.nullPointer(), category);
}

@Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
private Pointer reserve0(UnsignedWord nbytes, UnsignedWord alignment, boolean executable, NmtPreImageHeapData nmtData, NmtCategory category) {
if (nbytes.equal(0)) {
return WordFactory.nullPointer();
}
Expand All @@ -128,25 +144,54 @@ public Pointer reserve(UnsignedWord nbytes, UnsignedWord alignment, boolean exec
return nullPointer();
}
if (!customAlignment) {
if (VMInspectionOptions.hasNativeMemoryTrackingSupport()) {
if (nmtData.isNull()) {
NativeMemoryTracking.singleton().trackReserve(mappingBegin, mappingSize, category);
} else {
NmtPreImageHeapDataAccess.enqueueReserve(nmtData, mappingBegin, mappingSize, category);
}
}
return mappingBegin;
}
UnsignedWord unmappedSize = WordFactory.zero();
Pointer begin = PointerUtils.roundUp(mappingBegin, alignment);
UnsignedWord clippedBegin = begin.subtract(mappingBegin);
if (clippedBegin.aboveOrEqual(granularity)) {
munmap(mappingBegin, UnsignedUtils.roundDown(clippedBegin, granularity));
UnsignedWord unmapSize = UnsignedUtils.roundDown(clippedBegin, granularity);
munmap(mappingBegin, unmapSize);
unmappedSize = unmappedSize.add(unmapSize);
}
Pointer mappingEnd = mappingBegin.add(mappingSize);
UnsignedWord clippedEnd = mappingEnd.subtract(begin.add(nbytes));
if (clippedEnd.aboveOrEqual(granularity)) {
UnsignedWord rounded = UnsignedUtils.roundDown(clippedEnd, granularity);
munmap(mappingEnd.subtract(rounded), rounded);
unmappedSize = unmappedSize.add(rounded);
}
if (VMInspectionOptions.hasNativeMemoryTrackingSupport()) {
if (nmtData.isNull()) {
NativeMemoryTracking.singleton().trackReserve(begin, mappingSize.subtract(unmappedSize), category);
} else {
NmtPreImageHeapDataAccess.enqueueReserve(nmtData, begin, mappingSize.subtract(unmappedSize), category);
}
}
return begin;
}

@Override
@Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
public Pointer mapFile(PointerBase start, UnsignedWord nbytes, WordBase fileHandle, UnsignedWord offset, int access) {
public Pointer mapFile(PointerBase start, UnsignedWord nbytes, WordBase fileHandle, UnsignedWord offset, int access, NmtPreImageHeapData nmtData) {
return mapFile0(start, nbytes, fileHandle, offset, access, nmtData, NmtCategory.ImageHeap);
}

@Override
@Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
public Pointer mapFile(PointerBase start, UnsignedWord nbytes, WordBase fileHandle, UnsignedWord offset, int access, NmtCategory category) {
return mapFile0(start, nbytes, fileHandle, offset, access, WordFactory.nullPointer(), category);
}

@Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
private Pointer mapFile0(PointerBase start, UnsignedWord nbytes, WordBase fileHandle, UnsignedWord offset, int access, NmtPreImageHeapData nmtData, NmtCategory category) {
if ((start.isNonNull() && !isAligned(start)) || nbytes.equal(0)) {
return WordFactory.nullPointer();
}
Expand All @@ -157,12 +202,27 @@ public Pointer mapFile(PointerBase start, UnsignedWord nbytes, WordBase fileHand
}
int fd = (int) fileHandle.rawValue();
Pointer result = mmap(start, nbytes, accessAsProt(access), flags, fd, offset.rawValue());
return result.notEqual(MAP_FAILED()) ? result : WordFactory.nullPointer();
if (result.notEqual(MAP_FAILED())) {
trackCommitAndMaybeReserve(result, start, nbytes, nmtData, category);
return result;
}
return WordFactory.nullPointer();
}

@Override
@Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
public Pointer commit(PointerBase start, UnsignedWord nbytes, int access) {
public Pointer commit(PointerBase start, UnsignedWord nbytes, int access, NmtPreImageHeapData nmtData) {
return commit0(start, nbytes, access, nmtData, NmtCategory.ImageHeap);
}

@Override
@Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
public Pointer commit(PointerBase start, UnsignedWord nbytes, int access, NmtCategory category) {
return commit0(start, nbytes, access, WordFactory.nullPointer(), category);
}

@Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
private Pointer commit0(PointerBase start, UnsignedWord nbytes, int access, NmtPreImageHeapData nmtData, NmtCategory category) {
if ((start.isNonNull() && !isAligned(start)) || nbytes.equal(0)) {
return WordFactory.nullPointer();
}
Expand All @@ -177,7 +237,31 @@ public Pointer commit(PointerBase start, UnsignedWord nbytes, int access) {
}
/* The memory returned by mmap is guaranteed to be zeroed. */
final Pointer result = mmap(start, nbytes, accessAsProt(access), flags, NO_FD, NO_FD_OFFSET);
return result.notEqual(MAP_FAILED()) ? result : nullPointer();
if (result.notEqual(MAP_FAILED())) {
trackCommitAndMaybeReserve(result, start, nbytes, nmtData, category);
return result;
}
return nullPointer();
}

@Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
private static void trackCommitAndMaybeReserve(PointerBase baseAddr, PointerBase start, UnsignedWord nbytes, NmtPreImageHeapData nmtData, NmtCategory category) {
if (!VMInspectionOptions.hasNativeMemoryTrackingSupport()) {
return;
}
// Account for possible reserve before commit
if (start.isNull()) {
if (nmtData.isNull()) {
NativeMemoryTracking.singleton().trackReserve(baseAddr, nbytes, category);
} else {
NmtPreImageHeapDataAccess.enqueueReserve(nmtData, baseAddr, nbytes, category);
}
}
if (nmtData.isNull()) {
NativeMemoryTracking.singleton().trackCommit(baseAddr, nbytes, category);
} else {
NmtPreImageHeapDataAccess.enqueueCommit(nmtData, baseAddr, nbytes, category);
}
}

@Override
Expand All @@ -196,22 +280,37 @@ public int uncommit(PointerBase start, UnsignedWord nbytes) {
if (start.isNull() || !isAligned(start) || nbytes.equal(0)) {
return -1;
}

final Pointer result = mmap(start, nbytes, PROT_NONE(), MAP_FIXED() | MAP_ANON() | MAP_PRIVATE() | MAP_NORESERVE(), NO_FD, NO_FD_OFFSET);
return result.notEqual(MAP_FAILED()) ? 0 : -1;
if (result.notEqual(MAP_FAILED())) {
if (VMInspectionOptions.hasNativeMemoryTrackingSupport()) {
NativeMemoryTracking.singleton().trackUncommit(start, nbytes);
}
return 0;
}
return -1;
}

@Override
@Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
public int free(PointerBase start, UnsignedWord nbytes) {
return free(start, nbytes, WordFactory.nullPointer());
}

@Override
@Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
public int free(PointerBase start, UnsignedWord nbytes, NmtPreImageHeapData nmtData) {
if (start.isNull() || !isAligned(start) || nbytes.equal(0)) {
return -1;
}

UnsignedWord granularity = getGranularity();
Pointer mappingBegin = PointerUtils.roundDown(start, granularity);
UnsignedWord mappingSize = UnsignedUtils.roundUp(nbytes, granularity);
return munmap(mappingBegin, mappingSize);
int ret = munmap(mappingBegin, mappingSize);
if (ret == 0 && nmtData.isNull() && VMInspectionOptions.hasNativeMemoryTrackingSupport()) {
NativeMemoryTracking.singleton().trackFree(start);
}
return ret;
}

@Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
Expand Down
34 changes: 21 additions & 13 deletions ...com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/linux/LinuxImageHeapProvider.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@
import com.oracle.svm.core.config.ConfigurationValues;
import com.oracle.svm.core.headers.LibC;
import com.oracle.svm.core.heap.Heap;
import com.oracle.svm.core.nmt.NmtPreImageHeapData;
import com.oracle.svm.core.os.AbstractImageHeapProvider;
import com.oracle.svm.core.os.VirtualMemoryProvider;
import com.oracle.svm.core.os.VirtualMemoryProvider.Access;
Expand Down Expand Up @@ -98,12 +99,12 @@ public class LinuxImageHeapProvider extends AbstractImageHeapProvider {

@Override
@Uninterruptible(reason = "Called during isolate initialization.")
public int initialize(Pointer reservedAddressSpace, UnsignedWord reservedSize, WordPointer basePointer, WordPointer endPointer) {
public int initialize(Pointer reservedAddressSpace, UnsignedWord reservedSize, WordPointer basePointer, WordPointer endPointer, NmtPreImageHeapData nmtData) {
Pointer selfReservedMemory = WordFactory.nullPointer();
UnsignedWord requiredSize = getTotalRequiredAddressSpaceSize();
if (reservedAddressSpace.isNull()) {
UnsignedWord alignment = WordFactory.unsigned(Heap.getHeap().getPreferredAddressSpaceAlignment());
selfReservedMemory = VirtualMemoryProvider.get().reserve(requiredSize, alignment, false);
selfReservedMemory = VirtualMemoryProvider.get().reserve(requiredSize, alignment, false, nmtData);
if (selfReservedMemory.isNull()) {
return CEntryPointErrors.RESERVE_ADDRESS_SPACE_FAILED;
}
Expand All @@ -127,13 +128,13 @@ public int initialize(Pointer reservedAddressSpace, UnsignedWord reservedSize, W

int error = DynamicMethodAddressResolutionHeapSupport.get().initialize();
if (error != CEntryPointErrors.NO_ERROR) {
freeImageHeap(selfReservedHeapBase);
freeImageHeap(selfReservedHeapBase, nmtData);
return error;
}

error = DynamicMethodAddressResolutionHeapSupport.get().install(heapBase);
if (error != CEntryPointErrors.NO_ERROR) {
freeImageHeap(selfReservedHeapBase);
freeImageHeap(selfReservedHeapBase, nmtData);
return error;
}
} else {
Expand All @@ -149,16 +150,17 @@ public int initialize(Pointer reservedAddressSpace, UnsignedWord reservedSize, W
CACHED_IMAGE_FD.get(), CACHED_IMAGE_HEAP_OFFSET_IN_FILE.get(), MAGIC.get(),
IMAGE_HEAP_BEGIN.get(), IMAGE_HEAP_END.get(),
IMAGE_HEAP_RELOCATABLE_BEGIN.get(), IMAGE_HEAP_A_RELOCATABLE_POINTER.get(), IMAGE_HEAP_RELOCATABLE_END.get(),
IMAGE_HEAP_WRITABLE_BEGIN.get(), IMAGE_HEAP_WRITABLE_END.get());
IMAGE_HEAP_WRITABLE_BEGIN.get(), IMAGE_HEAP_WRITABLE_END.get(), nmtData);
if (result != CEntryPointErrors.NO_ERROR) {
freeImageHeap(selfReservedHeapBase);
freeImageHeap(selfReservedHeapBase, nmtData);
}
return result;
}

@Uninterruptible(reason = "Called during isolate initialization.")
private static int initializeImageHeap(Pointer imageHeap, UnsignedWord reservedSize, WordPointer endPointer, WordPointer cachedFd, WordPointer cachedOffsetInFile,
Pointer magicAddress, Word heapBeginSym, Word heapEndSym, Word heapRelocsSym, Pointer heapAnyRelocPointer, Word heapRelocsEndSym, Word heapWritableSym, Word heapWritableEndSym) {
Pointer magicAddress, Word heapBeginSym, Word heapEndSym, Word heapRelocsSym, Pointer heapAnyRelocPointer, Word heapRelocsEndSym, Word heapWritableSym, Word heapWritableEndSym,
NmtPreImageHeapData nmtData) {
assert heapBeginSym.belowOrEqual(heapWritableSym) && heapWritableSym.belowOrEqual(heapWritableEndSym) && heapWritableEndSym.belowOrEqual(heapEndSym);
assert heapBeginSym.belowOrEqual(heapRelocsSym) && heapRelocsSym.belowOrEqual(heapRelocsEndSym) && heapRelocsEndSym.belowOrEqual(heapEndSym);
assert heapAnyRelocPointer.isNull() || (heapRelocsSym.belowOrEqual(heapAnyRelocPointer) && heapAnyRelocPointer.belowThan(heapRelocsEndSym));
Expand Down Expand Up @@ -195,12 +197,12 @@ private static int initializeImageHeap(Pointer imageHeap, UnsignedWord reservedS
* heap must be in pristine condition for that).
*/
if (fd.equal(CANNOT_OPEN_FD)) {
return initializeImageHeapByCopying(imageHeap, imageHeapSize, pageSize, heapBeginSym, heapWritableSym, heapWritableEndSym);
return initializeImageHeapByCopying(imageHeap, imageHeapSize, pageSize, heapBeginSym, heapWritableSym, heapWritableEndSym, nmtData);
}

// Create memory mappings from the image file.
UnsignedWord fileOffset = cachedOffsetInFile.read();
Pointer mappedImageHeap = VirtualMemoryProvider.get().mapFile(imageHeap, imageHeapSize, fd, fileOffset, Access.READ);
Pointer mappedImageHeap = VirtualMemoryProvider.get().mapFile(imageHeap, imageHeapSize, fd, fileOffset, Access.READ, nmtData);
if (mappedImageHeap.isNull() || mappedImageHeap != imageHeap) {
return CEntryPointErrors.MAP_HEAP_FAILED;
}
Expand All @@ -221,7 +223,7 @@ private static int initializeImageHeap(Pointer imageHeap, UnsignedWord reservedS
* be part of a chunk with writable objects, in which case the chunk header must
* also be writable, and all the chunk's pages will be unprotected below.
*/
Pointer committedRelocsBegin = VirtualMemoryProvider.get().commit(relocsBoundary, relocsAlignedSize, Access.READ | Access.WRITE);
Pointer committedRelocsBegin = VirtualMemoryProvider.get().commit(relocsBoundary, relocsAlignedSize, Access.READ | Access.WRITE, nmtData);
if (committedRelocsBegin.isNull() || committedRelocsBegin != relocsBoundary) {
return CEntryPointErrors.PROTECT_HEAP_FAILED;
}
Expand Down Expand Up @@ -249,8 +251,9 @@ private static int initializeImageHeap(Pointer imageHeap, UnsignedWord reservedS
}

@Uninterruptible(reason = "Called during isolate initialization.")
private static int initializeImageHeapByCopying(Pointer imageHeap, UnsignedWord imageHeapSize, UnsignedWord pageSize, Word heapBeginSym, Word heapWritableSym, Word heapWritableEndSym) {
Pointer committedBegin = VirtualMemoryProvider.get().commit(imageHeap, imageHeapSize, Access.READ | Access.WRITE);
private static int initializeImageHeapByCopying(Pointer imageHeap, UnsignedWord imageHeapSize, UnsignedWord pageSize, Word heapBeginSym, Word heapWritableSym, Word heapWritableEndSym,
NmtPreImageHeapData nmtData) {
Pointer committedBegin = VirtualMemoryProvider.get().commit(imageHeap, imageHeapSize, Access.READ | Access.WRITE, nmtData);
if (committedBegin.isNull()) {
return CEntryPointErrors.MAP_HEAP_FAILED;
}
Expand Down Expand Up @@ -343,6 +346,11 @@ private static boolean checkImageFileMagic(int mapfd, int imagefd, CCharPointer
@Override
@Uninterruptible(reason = "Called during isolate tear-down.")
public int freeImageHeap(PointerBase heapBase) {
return freeImageHeap(heapBase, WordFactory.nullPointer());
}

@Uninterruptible(reason = "Called during isolate tear-down.")
private int freeImageHeap(PointerBase heapBase, NmtPreImageHeapData nmtData) {
if (heapBase.isNull()) { // no memory allocated
return CEntryPointErrors.NO_ERROR;
}
Expand All @@ -352,7 +360,7 @@ public int freeImageHeap(PointerBase heapBase) {
if (DynamicMethodAddressResolutionHeapSupport.isEnabled()) {
addressSpaceStart = addressSpaceStart.subtract(getPreHeapAlignedSizeForDynamicMethodAddressResolver());
}
if (VirtualMemoryProvider.get().free(addressSpaceStart, getTotalRequiredAddressSpaceSize()) != 0) {
if (VirtualMemoryProvider.get().free(addressSpaceStart, getTotalRequiredAddressSpaceSize(), nmtData) != 0) {
return CEntryPointErrors.FREE_IMAGE_HEAP_FAILED;
}
return CEntryPointErrors.NO_ERROR;
Expand Down