"Fossies" - the Fresh Open Source Software Archive  

Source code changes of the file "test-framework/src/java/org/apache/lucene/util/RamUsageTester.java" between
lucene-7.6.0-src.tgz and lucene-7.7.0-src.tgz

About: Lucene is a Java full-text search engine (not a complete application, but rather a code library and API; java source code).

RamUsageTester.java  (lucene-7.6.0-src.tgz):RamUsageTester.java  (lucene-7.7.0-src.tgz)
skipping to change at line 24 skipping to change at line 24
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.lucene.util; package org.apache.lucene.util;
import java.io.ByteArrayOutputStream; import java.io.ByteArrayOutputStream;
import java.io.File; import java.io.File;
import java.lang.reflect.Array; import java.lang.reflect.Array;
import java.lang.reflect.Field; import java.lang.reflect.Field;
import java.lang.reflect.Modifier; import java.lang.reflect.Modifier;
import java.nio.ByteBuffer;
import java.nio.file.Path; import java.nio.file.Path;
import java.security.AccessController; import java.security.AccessController;
import java.security.PrivilegedAction; import java.security.PrivilegedAction;
import java.util.AbstractList; import java.util.AbstractList;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collection; import java.util.Collection;
import java.util.Collections; import java.util.Collections;
import java.util.HashMap; import java.util.HashMap;
import java.util.IdentityHashMap; import java.util.IdentityHashMap;
import java.util.List; import java.util.List;
import java.util.Locale;
import java.util.Map; import java.util.Map;
import java.util.Set; import java.util.Set;
import java.util.function.Predicate;
import java.util.function.ToLongFunction; import java.util.function.ToLongFunction;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import java.util.stream.Stream; import java.util.stream.Stream;
import java.util.stream.StreamSupport; import java.util.stream.StreamSupport;
/** Crawls object graph to collect RAM usage for testing */ /** Crawls object graph to collect RAM usage for testing */
public final class RamUsageTester { public final class RamUsageTester {
/** An accumulator of object references. This class allows for customizing RAM usage estimation. */ /** An accumulator of object references. This class allows for customizing RAM usage estimation. */
public static class Accumulator { public static class Accumulator {
skipping to change at line 114 skipping to change at line 117
long totalSize = 0; long totalSize = 0;
while (!stack.isEmpty()) { while (!stack.isEmpty()) {
final Object ob = stack.remove(stack.size() - 1); final Object ob = stack.remove(stack.size() - 1);
if (ob == null || seen.contains(ob)) { if (ob == null || seen.contains(ob)) {
continue; continue;
} }
seen.add(ob); seen.add(ob);
final long obSize;
final Class<?> obClazz = ob.getClass(); final Class<?> obClazz = ob.getClass();
assert obClazz != null : "jvm bug detected (Object.getClass() == null). pl ease report this to your vendor"; assert obClazz != null : "jvm bug detected (Object.getClass() == null). pl ease report this to your vendor";
if (obClazz.isArray()) { if (obClazz.isArray()) {
/* obSize = handleArray(accumulator, stack, ob, obClazz);
* Consider an array, possibly of primitive types. Push any of its refer
ences to
* the processing stack and accumulate this array's shallow size.
*/
final long shallowSize = RamUsageEstimator.shallowSizeOf(ob);
final int len = Array.getLength(ob);
final List<Object> values;
Class<?> componentClazz = obClazz.getComponentType();
if (componentClazz.isPrimitive()) {
values = Collections.emptyList();
} else {
values = new AbstractList<Object>() {
@Override
public Object get(int index) {
return Array.get(ob, index);
}
@Override
public int size() {
return len;
}
};
}
totalSize += accumulator.accumulateArray(ob, shallowSize, values, stack)
;
} else { } else {
/* obSize = handleOther(accumulator, classCache, stack, ob, obClazz);
* Consider an object. Push any references it has to the processing stac
k
* and accumulate this object's shallow size.
*/
try {
ClassCache cachedInfo = classCache.get(obClazz);
if (cachedInfo == null) {
classCache.put(obClazz, cachedInfo = createCacheEntry(obClazz));
}
boolean needsReflection = true;
if (Constants.JRE_IS_MINIMUM_JAVA9 && obClazz.getName().startsWith("ja
va.")) {
// Java 9: Best guess for some known types, as we cannot precisely l
ook into runtime classes:
final ToLongFunction<Object> func = SIMPLE_TYPES.get(obClazz);
if (func != null) { // some simple type like String where the size i
s easy to get from public properties
totalSize += accumulator.accumulateObject(ob, cachedInfo.alignedSh
allowInstanceSize + func.applyAsLong(ob),
Collections.emptyMap(), stack);
needsReflection = false;
} else if (ob instanceof Iterable) {
final List<Object> values = StreamSupport.stream(((Iterable<?>) ob
).spliterator(), false)
.collect(Collectors.toList());
totalSize += accumulator.accumulateArray(ob, cachedInfo.alignedSha
llowInstanceSize + RamUsageEstimator.NUM_BYTES_ARRAY_HEADER, values, stack);
needsReflection = false;
} else if (ob instanceof Map) {
final List<Object> values = ((Map<?,?>) ob).entrySet().stream()
.flatMap(e -> Stream.of(e.getKey(), e.getValue()))
.collect(Collectors.toList());
totalSize += accumulator.accumulateArray(ob, cachedInfo.alignedSha
llowInstanceSize + RamUsageEstimator.NUM_BYTES_ARRAY_HEADER, values, stack);
totalSize += RamUsageEstimator.NUM_BYTES_ARRAY_HEADER;
needsReflection = false;
}
}
if (needsReflection) {
final Map<Field, Object> fieldValues = new HashMap<>();
for (Field f : cachedInfo.referenceFields) {
fieldValues.put(f, f.get(ob));
}
totalSize += accumulator.accumulateObject(ob, cachedInfo.alignedShal
lowInstanceSize, fieldValues, stack);
}
} catch (IllegalAccessException e) {
// this should never happen as we enabled setAccessible().
throw new RuntimeException("Reflective field access failed?", e);
}
} }
totalSize += obSize;
// Dump size of each object for comparisons across JVMs and flags.
// System.out.println(" += " + obClazz + " | " + obSize);
} }
// Help the GC (?). // Help the GC (?).
seen.clear(); seen.clear();
stack.clear(); stack.clear();
classCache.clear(); classCache.clear();
return totalSize; return totalSize;
} }
private static long handleOther(Accumulator accumulator, IdentityHashMap<Class
<?>, ClassCache> classCache, ArrayList<Object> stack, Object ob, Class<?> obClaz
z) {
/*
* Consider an object. Push any references it has to the processing stack
* and accumulate this object's shallow size.
*/
try {
if (Constants.JRE_IS_MINIMUM_JAVA9) {
long alignedShallowInstanceSize = RamUsageEstimator.shallowSizeOf(ob);
Predicate<Class<?>> isJavaModule = (clazz) -> {
return clazz.getName().startsWith("java.");
};
// Java 9: Best guess for some known types, as we cannot precisely look
into runtime classes:
final ToLongFunction<Object> func = SIMPLE_TYPES.get(obClazz);
if (func != null) { // some simple type like String where the size is ea
sy to get from public properties
return accumulator.accumulateObject(ob, alignedShallowInstanceSize + f
unc.applyAsLong(ob),
Collections.emptyMap(), stack);
} else if (ob instanceof Enum) {
return alignedShallowInstanceSize;
} else if (ob instanceof ByteBuffer) {
// Approximate ByteBuffers with their underlying storage (ignores fiel
d overhead).
return byteArraySize(((ByteBuffer) ob).capacity());
} else if (isJavaModule.test(obClazz) && ob instanceof Map) {
final List<Object> values = ((Map<?,?>) ob).entrySet().stream()
.flatMap(e -> Stream.of(e.getKey(), e.getValue()))
.collect(Collectors.toList());
return accumulator.accumulateArray(ob, alignedShallowInstanceSize + Ra
mUsageEstimator.NUM_BYTES_ARRAY_HEADER, values, stack)
+ RamUsageEstimator.NUM_BYTES_ARRAY_HEADER;
} else if (isJavaModule.test(obClazz) && ob instanceof Iterable) {
final List<Object> values = StreamSupport.stream(((Iterable<?>) ob).sp
literator(), false)
.collect(Collectors.toList());
return accumulator.accumulateArray(ob, alignedShallowInstanceSize + Ra
mUsageEstimator.NUM_BYTES_ARRAY_HEADER, values, stack)
+ RamUsageEstimator.NUM_BYTES_ARRAY_HEADER;
} else {
// Fallback to reflective access.
}
}
ClassCache cachedInfo = classCache.get(obClazz);
if (cachedInfo == null) {
classCache.put(obClazz, cachedInfo = createCacheEntry(obClazz));
}
final Map<Field, Object> fieldValues = new HashMap<>();
for (Field f : cachedInfo.referenceFields) {
fieldValues.put(f, f.get(ob));
}
return accumulator.accumulateObject(ob, cachedInfo.alignedShallowInstanceS
ize, fieldValues, stack);
} catch (IllegalAccessException e) {
// this should never happen as we enabled setAccessible().
throw new RuntimeException("Reflective field access failed?", e);
}
}
private static long handleArray(Accumulator accumulator, ArrayList<Object> sta
ck, Object ob, Class<?> obClazz) {
/*
* Consider an array, possibly of primitive types. Push any of its reference
s to
* the processing stack and accumulate this array's shallow size.
*/
final long shallowSize = RamUsageEstimator.shallowSizeOf(ob);
final int len = Array.getLength(ob);
final List<Object> values;
Class<?> componentClazz = obClazz.getComponentType();
if (componentClazz.isPrimitive()) {
values = Collections.emptyList();
} else {
values = new AbstractList<Object>() {
@Override
public Object get(int index) {
return Array.get(ob, index);
}
@Override
public int size() {
return len;
}
};
}
return accumulator.accumulateArray(ob, shallowSize, values, stack);
}
/** /**
* This map contains a function to calculate sizes of some "simple types" like String just from their public properties. * This map contains a function to calculate sizes of some "simple types" like String just from their public properties.
* This is needed for Java 9, which does not allow to look into runtime class fields. * This is needed for Java 9, which does not allow to look into runtime class fields.
*/ */
@SuppressWarnings("serial") @SuppressWarnings("serial")
private static final Map<Class<?>, ToLongFunction<Object>> SIMPLE_TYPES = Coll ections.unmodifiableMap(new IdentityHashMap<Class<?>, ToLongFunction<Object>>() { private static final Map<Class<?>, ToLongFunction<Object>> SIMPLE_TYPES = Coll ections.unmodifiableMap(new IdentityHashMap<Class<?>, ToLongFunction<Object>>() {
{ init(); } { init(); }
@SuppressForbidden(reason = "We measure some forbidden classes") @SuppressForbidden(reason = "We measure some forbidden classes")
private void init() { private void init() {
skipping to change at line 227 skipping to change at line 252
} }
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
private <T> void a(Class<T> clazz, ToLongFunction<T> func) { private <T> void a(Class<T> clazz, ToLongFunction<T> func) {
put(clazz, (ToLongFunction<Object>) func); put(clazz, (ToLongFunction<Object>) func);
} }
private long charArraySize(int len) { private long charArraySize(int len) {
return RamUsageEstimator.alignObjectSize((long)RamUsageEstimator.NUM_BYTES _ARRAY_HEADER + (long)Character.BYTES * len); return RamUsageEstimator.alignObjectSize((long)RamUsageEstimator.NUM_BYTES _ARRAY_HEADER + (long)Character.BYTES * len);
} }
private long byteArraySize(int len) {
return RamUsageEstimator.alignObjectSize((long)RamUsageEstimator.NUM_BYTES
_ARRAY_HEADER + len);
}
}); });
/** /**
* Cached information about a given class. * Cached information about a given class.
*/ */
private static final class ClassCache { private static final class ClassCache {
public final long alignedShallowInstanceSize; public final long alignedShallowInstanceSize;
public final Field[] referenceFields; public final Field[] referenceFields;
public ClassCache(long alignedShallowInstanceSize, Field[] referenceFields) { public ClassCache(long alignedShallowInstanceSize, Field[] referenceFields) {
skipping to change at line 271 skipping to change at line 292
final Field[] fields = c.getDeclaredFields(); final Field[] fields = c.getDeclaredFields();
for (final Field f : fields) { for (final Field f : fields) {
if (!Modifier.isStatic(f.getModifiers())) { if (!Modifier.isStatic(f.getModifiers())) {
shallowInstanceSize = RamUsageEstimator.adjustForField(shallowInstan ceSize, f); shallowInstanceSize = RamUsageEstimator.adjustForField(shallowInstan ceSize, f);
if (!f.getType().isPrimitive()) { if (!f.getType().isPrimitive()) {
try { try {
f.setAccessible(true); f.setAccessible(true);
referenceFields.add(f); referenceFields.add(f);
} catch (RuntimeException re) { } catch (RuntimeException re) {
if ("java.lang.reflect.InaccessibleObjectException".equals(re.ge throw new RuntimeException(String.format(Locale.ROOT,
tClass().getName())) { "Can't access field '%s' of class '%s' for RAM estimation.",
// LUCENE-7595: this is Java 9, which prevents access to field f.getName(),
s in foreign modules clazz.getName()), re);
} else {
throw re;
}
} }
} }
} }
} }
} }
cachedInfo = new ClassCache( cachedInfo = new ClassCache(
RamUsageEstimator.alignObjectSize(shallowInstanceSize), RamUsageEstimator.alignObjectSize(shallowInstanceSize),
referenceFields.toArray(new Field[referenceFields.size()])); referenceFields.toArray(new Field[referenceFields.size()]));
return cachedInfo; return cachedInfo;
}); });
} }
private static long byteArraySize(int len) {
return RamUsageEstimator.alignObjectSize((long) RamUsageEstimator.NUM_BYTES_
ARRAY_HEADER + len);
}
} }
 End of changes. 11 change blocks. 
92 lines changed or deleted 114 lines changed or added

Home  |  About  |  Features  |  All  |  Newest  |  Dox  |  Diffs  |  RSS Feeds  |  Screenshots  |  Comments  |  Imprint  |  Privacy  |  HTTP(S)