Skip to content

Commit e79882f

Browse files
zelenijzelenij
committed
AFFINITY-22: Change the API to support more the 64 CPUs; better JNI
testing; use C++ for JNI instead of C
1 parent a0e038d commit e79882f

22 files changed

Lines changed: 428 additions & 129 deletions

‎affinity-test/pom.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@
2828

2929
<modelVersion>4.0.0</modelVersion>
3030
<artifactId>affinity-test</artifactId>
31-
<version>2.3-SNAPSHOT</version>
31+
<version>3.0-SNAPSHOT</version>
3232
<packaging>bundle</packaging>
3333

3434
<name>OpenHFT/Java-Thread-Affinity/affinity-test</name>

‎affinity/pom.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@
2828

2929
<modelVersion>4.0.0</modelVersion>
3030
<artifactId>affinity</artifactId>
31-
<version>2.3-SNAPSHOT</version>
31+
<version>3.0-SNAPSHOT</version>
3232
<packaging>bundle</packaging>
3333

3434
<name>OpenHFT/Java-Thread-Affinity/affinity</name>

‎affinity/src/main/c/Makefile‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ JAVA_CLASSES = software.chronicle.enterprise.internals.impl.NativeAffinity
1414

1515
JNI_STUBS := $(subst .,_,$(JAVA_CLASSES))
1616
JNI_HEADERS := $(patsubst %,$(WORKING_DIR)/%.h,$(JNI_STUBS))
17-
JNI_SOURCES := $(patsubst %,%.c,$(JNI_STUBS))
17+
JNI_SOURCES := $(patsubst %,%.cpp,$(JNI_STUBS))
1818
JNI_JAVA_SOURCES := $(patsubst %,$(TARGET_DIR)/%.class,$(subst .,/,$(JAVA_CLASSES)))
1919

2020
JAVA_BUILD_DIR := $(TARGET_DIR)
@@ -23,6 +23,7 @@ JAVA_HOME ?= /usr/java/default
2323
JAVA_LIB := $(JAVA_HOME)/jre/lib
2424
JVM_SHARED_LIBS := -L$(JAVA_LIB)/amd64/server -L$(JAVA_LIB)/i386/server -L$(JAVA_LIB)/amd64/jrockit/ -L$(JAVA_LIB)/i386/jrockit/
2525

26+
CXX=g++
2627
INCLUDES := -I $(JAVA_HOME)/include -I $(JAVA_HOME)/include/linux -I $(WORKING_DIR)
2728

2829
# classpath for javah
@@ -37,7 +38,7 @@ endif
3738
all: $(TARGET)
3839

3940
$(TARGET): $(JNI_HEADERS) $(JNI_SOURCES)
40-
gcc -O3 -Wall -shared -fPIC $(JVM_SHARED_LIBS) -ljvm -lrt $(INCLUDES) $(JNI_SOURCES) -o $(TARGET)
41+
$(CXX) -O3 -Wall -shared -fPIC $(JVM_SHARED_LIBS) -ljvm -lrt $(INCLUDES) $(JNI_SOURCES) -o $(TARGET)
4142

4243
$(JNI_HEADERS): $(JNI_JAVA_SOURCES)
4344
mkdir -p $(TARGET_DIR)/jni

affinity/src/main/c/software_chronicle_enterprise_internals_impl_NativeAffinity.c renamed to affinity/src/main/c/software_chronicle_enterprise_internals_impl_NativeAffinity.cpp

Lines changed: 33 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
/*
1+
/* vim: syntax=cpp
22
* Copyright 2011 Peter Lawrey
33
*
44
* Licensed under the Apache License, Version 2.0 (the "License");
@@ -14,12 +14,16 @@
1414
* limitations under the License.
1515
*/
1616

17+
#ifndef _GNU_SOURCE
1718
#define _GNU_SOURCE
19+
#endif
20+
1821
#include <jni.h>
1922
#include <sched.h>
2023
#include <sys/syscall.h>
2124
#include <sys/types.h>
2225
#include <unistd.h>
26+
#include <string.h>
2327

2428
#include "software_chronicle_enterprise_internals_impl_NativeAffinity.h"
2529

@@ -28,16 +32,27 @@
2832
* Method: getAffinity0
2933
* Signature: ()J
3034
*/
31-
JNIEXPORT jlong JNICALL Java_software_chronicle_enterprise_internals_impl_NativeAffinity_getAffinity0
32-
(JNIEnv *env, jclass c) {
35+
JNIEXPORT jbyteArray JNICALL Java_software_chronicle_enterprise_internals_impl_NativeAffinity_getAffinity0
36+
(JNIEnv *env, jclass c)
37+
{
38+
// The default size of the structure supports 1024 CPUs, should be enough
39+
// for now In the future we can use dynamic sets, which can support more
40+
// CPUs, given OS can handle them as well
3341
cpu_set_t mask;
34-
int ret = sched_getaffinity(0, sizeof(mask), &mask);
35-
if (ret < 0) return ~0LL;
36-
long long mask2 = 0, i;
37-
for(i=0;i<sizeof(mask2)*8;i++)
38-
if (CPU_ISSET(i, &mask))
39-
mask2 |= 1L << i;
40-
return (jlong) mask2;
42+
const size_t size = sizeof(mask);
43+
44+
int res = sched_getaffinity(0, size, &mask);
45+
if (res < 0)
46+
{
47+
return NULL;
48+
}
49+
50+
jbyteArray ret = env->NewByteArray(size);
51+
jbyte* bytes = env->GetByteArrayElements(ret, 0);
52+
memcpy(bytes, &mask, size);
53+
env->SetByteArrayRegion(ret, 0, size, bytes);
54+
55+
return ret;
4156
}
4257

4358
/*
@@ -46,14 +61,16 @@ JNIEXPORT jlong JNICALL Java_software_chronicle_enterprise_internals_impl_Native
4661
* Signature: (J)V
4762
*/
4863
JNIEXPORT void JNICALL Java_software_chronicle_enterprise_internals_impl_NativeAffinity_setAffinity0
49-
(JNIEnv *env, jclass c, jlong affinity) {
50-
int i;
64+
(JNIEnv *env, jclass c, jbyteArray affinity)
65+
{
5166
cpu_set_t mask;
67+
const size_t size = sizeof(mask);
5268
CPU_ZERO(&mask);
53-
for(i=0;i<sizeof(affinity)*8;i++)
54-
if ((affinity >> i) & 1)
55-
CPU_SET(i, &mask);
56-
sched_setaffinity(0, sizeof(mask), &mask);
69+
70+
jbyte* bytes = env->GetByteArrayElements(affinity, 0);
71+
memcpy(&mask, bytes, size);
72+
73+
sched_setaffinity(0, size, &mask);
5774
}
5875

5976
/*

‎affinity/src/main/java/net/openhft/affinity/AffinityLock.java‎

Lines changed: 26 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
import java.io.Closeable;
2727
import java.io.File;
2828
import java.io.IOException;
29+
import java.util.BitSet;
2930

3031
/**
3132
* This utility class support locking a thread to a single core, or reserving a whole core for a thread.
@@ -41,8 +42,8 @@ public class AffinityLock implements Closeable {
4142
// TODO It seems like on virtualized platforms .availableProcessors() value can change at
4243
// TODO runtime. We should think about how to adopt to such change
4344
public static final int PROCESSORS = Runtime.getRuntime().availableProcessors();
44-
public static final long BASE_AFFINITY = AffinitySupport.getAffinity();
45-
public static final long RESERVED_AFFINITY = getReservedAffinity0();
45+
public static final BitSet BASE_AFFINITY = AffinitySupport.getAffinity();
46+
public static final BitSet RESERVED_AFFINITY = getReservedAffinity0();
4647
private static final LockInventory LOCK_INVENTORY = new LockInventory(new NoCpuLayout(PROCESSORS));
4748

4849
static {
@@ -102,17 +103,29 @@ public static CpuLayout cpuLayout() {
102103
return LOCK_INVENTORY.getCpuLayout();
103104
}
104105

105-
private static long getReservedAffinity0() {
106+
private static BitSet getReservedAffinity0()
107+
{
106108
String reservedAffinity = System.getProperty(AFFINITY_RESERVED);
107-
if (reservedAffinity == null || reservedAffinity.trim().isEmpty()) {
108-
long reserverable = ((1 << PROCESSORS) - 1) ^ BASE_AFFINITY;
109-
if (reserverable == 0 && PROCESSORS > 1) {
109+
if (reservedAffinity == null || reservedAffinity.trim().isEmpty())
110+
{
111+
BitSet reserverable = new BitSet(PROCESSORS);
112+
reserverable.set(0, PROCESSORS - 1, true);
113+
reserverable.and(BASE_AFFINITY);
114+
if (reserverable.isEmpty() && PROCESSORS > 1)
115+
{
110116
LoggerFactory.getLogger(AffinityLock.class).info("No isolated CPUs found, so assuming CPUs 1 to {} available.", (PROCESSORS - 1));
111-
return ((1 << PROCESSORS) - 2);
117+
reserverable = new BitSet(PROCESSORS);
118+
// make the first CPU unavailable
119+
reserverable.set(1, PROCESSORS - 1, true);
120+
reserverable.set(0, false);
121+
return reserverable;
112122
}
113123
return reserverable;
114124
}
115-
return Long.parseLong(reservedAffinity, 16);
125+
126+
long[] longs = new long[1];
127+
longs[0] = Long.parseLong(reservedAffinity, 16);
128+
return BitSet.valueOf(longs);
116129
}
117130

118131
/**
@@ -212,7 +225,11 @@ public void bind(boolean wholeCore) {
212225
LOGGER.info("Assigning cpu {} to {}", cpuId, assignedThread);
213226
}
214227
if (cpuId >= 0)
215-
AffinitySupport.setAffinity(1L << cpuId);
228+
{
229+
BitSet affinity = new BitSet();
230+
affinity.set(cpuId, true);
231+
AffinitySupport.setAffinity(affinity);
232+
}
216233
}
217234

218235
final boolean canReserve() {

‎affinity/src/main/java/net/openhft/affinity/AffinitySupport.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
import java.io.PrintWriter;
2525
import java.io.StringWriter;
2626
import java.lang.reflect.Field;
27+
import java.util.BitSet;
2728

2829
/**
2930
* Library to wrap low level JNI or JNA calls. Can be called without needing to know the actual implementation used.
@@ -144,11 +145,11 @@ private static void logThrowable(Throwable t, String description) {
144145
LOGGER.warn(sw.toString());
145146
}
146147

147-
public static long getAffinity() {
148+
public static BitSet getAffinity() {
148149
return AFFINITY_IMPL.getAffinity();
149150
}
150151

151-
public static void setAffinity(final long affinity) {
152+
public static void setAffinity(final BitSet affinity) {
152153
AFFINITY_IMPL.setAffinity(affinity);
153154
}
154155

‎affinity/src/main/java/net/openhft/affinity/IAffinity.java‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,8 @@
1616

1717
package net.openhft.affinity;
1818

19+
import java.util.BitSet;
20+
1921
/**
2022
* Implementation interface
2123
*
@@ -24,14 +26,14 @@
2426
*/
2527
public interface IAffinity {
2628
/**
27-
* @return returns affinity mask for current thread, or -1 if unknown
29+
* @return returns affinity mask for current thread, or null if unknown
2830
*/
29-
long getAffinity();
31+
BitSet getAffinity();
3032

3133
/**
3234
* @param affinity sets affinity mask of current thread to specified value
3335
*/
34-
void setAffinity(final long affinity);
36+
void setAffinity(final BitSet affinity);
3537

3638
/**
3739
* @return the current cpu id, or -1 if unknown.

‎affinity/src/main/java/net/openhft/affinity/LockInventory.java‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -53,9 +53,10 @@ public final synchronized void set(CpuLayout cpuLayout) {
5353
return;
5454
}
5555
reset(cpuLayout);
56-
for (int i = 0; i < cpuLayout.cpus(); i++) {
57-
boolean base = ((AffinityLock.BASE_AFFINITY >> i) & 1) != 0;
58-
boolean reservable = ((AffinityLock.RESERVED_AFFINITY >> i) & 1) != 0;
56+
for (int i = 0; i < cpuLayout.cpus(); i++)
57+
{
58+
final boolean base = AffinityLock.BASE_AFFINITY.get(i);;
59+
final boolean reservable = AffinityLock.RESERVED_AFFINITY.get(i);
5960

6061
LOGGER.trace("cpu " + i + " base={} reservable= {}", i, base, reservable);
6162
AffinityLock lock = logicalCoreLocks[i] = newLock(i, base, reservable);

‎affinity/src/main/java/net/openhft/affinity/impl/LinuxHelper.java‎

Lines changed: 12 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
import org.jetbrains.annotations.NotNull;
2222

2323
import java.util.Arrays;
24+
import java.util.BitSet;
2425
import java.util.List;
2526

2627
public class LinuxHelper {
@@ -222,25 +223,27 @@ int sched_getaffinity(final int pid,
222223
return cpuset;
223224
}
224225

225-
public static void sched_setaffinity(final long affinity) {
226+
public static void sched_setaffinity(final BitSet affinity) {
226227
final CLibrary lib = CLibrary.INSTANCE;
227228
final cpu_set_t cpuset = new cpu_set_t();
228229
final int size = version.isSameOrNewer(VERSION_2_6) ? cpu_set_t.SIZE_OF_CPU_SET_T : NativeLong.SIZE;
229-
if(Platform.is64Bit()) {
230-
cpuset.__bits[0].setValue(affinity);
231-
232-
} else {
233-
cpuset.__bits[0].setValue(affinity & 0xFFFFFFFFL);
234-
cpuset.__bits[1].setValue((affinity >>> 32) & 0xFFFFFFFFL);
230+
final long[] bits = affinity.toLongArray();
231+
for (int i = 0; i < bits.length; i++) {
232+
if (Platform.is64Bit()) {
233+
cpuset.__bits[i].setValue(bits[i]);
234+
} else {
235+
cpuset.__bits[i*2].setValue(bits[i] & 0xFFFFFFFFL);
236+
cpuset.__bits[i*2+1].setValue((bits[i] >>> 32) & 0xFFFFFFFFL);
237+
}
235238
}
236239
try {
237240
if(lib.sched_setaffinity(0, size, cpuset) != 0) {
238241
throw new IllegalStateException("sched_setaffinity(0, " + size +
239-
", 0x" + Long.toHexString(affinity) + " failed; errno=" + Native.getLastError());
242+
", 0x" + Utilities.toHexString(affinity) + " failed; errno=" + Native.getLastError());
240243
}
241244
} catch (LastErrorException e) {
242245
throw new IllegalStateException("sched_setaffinity(0, " + size +
243-
", 0x" + Long.toHexString(affinity) + " failed; errno=" + e.getErrorCode(), e);
246+
", 0x" + Utilities.toHexString(affinity) + " failed; errno=" + e.getErrorCode(), e);
244247
}
245248
}
246249

‎affinity/src/main/java/net/openhft/affinity/impl/LinuxJNAAffinity.java‎

Lines changed: 63 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,21 +21,81 @@
2121
import org.slf4j.Logger;
2222
import org.slf4j.LoggerFactory;
2323

24+
import java.nio.ByteBuffer;
25+
import java.util.ArrayList;
26+
import java.util.BitSet;
27+
import java.util.Collections;
28+
import java.util.LinkedList;
29+
2430
public enum LinuxJNAAffinity implements IAffinity {
2531
INSTANCE;
2632
private static final Logger LOGGER = LoggerFactory.getLogger(LinuxJNAAffinity.class);
2733
public static final boolean LOADED;
2834

2935
// TODO: FIXME!!! CHANGE IAffinity TO SUPPORT PLATFORMS WITH 64+ CORES FIXME!!!
3036
@Override
31-
public long getAffinity() {
37+
public BitSet getAffinity() {
3238
final LinuxHelper.cpu_set_t cpuset = LinuxHelper.sched_getaffinity();
33-
return cpuset.__bits[0].longValue();
39+
40+
boolean collect = false;
41+
ArrayList<Byte> bytes = new ArrayList<Byte>();
42+
43+
ByteBuffer buff = null;
44+
if (Platform.is64Bit())
45+
{
46+
buff = ByteBuffer.allocate(Long.SIZE / 8);
47+
}
48+
else
49+
{
50+
buff = ByteBuffer.allocate(Integer.SIZE / 8);
51+
}
52+
53+
for (int i = cpuset.__bits.length - 1; i >= 0; --i)
54+
{
55+
if (!collect && cpuset.__bits[i].longValue() != 0)
56+
{
57+
collect = true;
58+
}
59+
60+
if (collect)
61+
{
62+
if (Platform.is64Bit())
63+
{
64+
buff.putLong(cpuset.__bits[i].longValue());
65+
}
66+
else
67+
{
68+
buff.putInt((int) cpuset.__bits[i].longValue());
69+
}
70+
71+
final byte[] arr = buff.array();
72+
//for (int j = arr.length - 1; j >= 0; --j)
73+
for (int j = 0; j < arr.length; j++)
74+
{
75+
bytes.add(arr[j]);
76+
}
77+
}
78+
}
79+
80+
if (!bytes.isEmpty())
81+
{
82+
byte[] data = new byte[bytes.size()];
83+
for (int i = 0; i < bytes.size(); i++)
84+
{
85+
// don't forget to reverse the order of long values
86+
data[data.length - i - 1] = bytes.get(i);
87+
}
88+
return BitSet.valueOf(data);
89+
}
90+
else
91+
{
92+
return new BitSet();
93+
}
3494
}
3595

3696
// TODO: FIXME!!! CHANGE IAffinity TO SUPPORT PLATFORMS WITH 64+ CORES FIXME!!!
3797
@Override
38-
public void setAffinity(final long affinity) {
98+
public void setAffinity(final BitSet affinity) {
3999
LinuxHelper.sched_setaffinity(affinity);
40100
}
41101

0 commit comments

Comments
 (0)