Skip to content

Commit b7bd552

Browse files
CRZbulabulaCRZbulabula
authored
[Remove DataNode] Increase the waiting time for removing DN test (#15769)
1 parent 84b56d2 commit b7bd552

3 files changed

Lines changed: 35 additions & 25 deletions

File tree

‎integration-test/src/test/java/org/apache/iotdb/confignode/it/removedatanode/IoTDBRemoveDataNodeUtils.java‎

Lines changed: 7 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,8 @@
3333

3434
import java.sql.Connection;
3535
import java.sql.SQLException;
36+
import java.util.ArrayList;
37+
import java.util.Collections;
3638
import java.util.HashSet;
3739
import java.util.List;
3840
import java.util.Set;
@@ -66,13 +68,9 @@ public static Connection getConnectionWithSQLType(SQLModel model) throws SQLExce
6668

6769
public static Set<Integer> selectRemoveDataNodes(
6870
Set<Integer> allDataNodeId, int removeDataNodeNum) {
69-
Set<Integer> removeDataNodeIds = new HashSet<>();
70-
for (int i = 0; i < removeDataNodeNum; i++) {
71-
int removeDataNodeId = allDataNodeId.iterator().next();
72-
removeDataNodeIds.add(removeDataNodeId);
73-
allDataNodeId.remove(removeDataNodeId);
74-
}
75-
return removeDataNodeIds;
71+
List<Integer> shuffledDataNodeIds = new ArrayList<>(allDataNodeId);
72+
Collections.shuffle(shuffledDataNodeIds);
73+
return new HashSet<>(shuffledDataNodeIds.subList(0, removeDataNodeNum));
7674
}
7775

7876
public static void restartDataNodes(List<DataNodeWrapper> dataNodeWrappers) {
@@ -120,7 +118,7 @@ public static void awaitUntilSuccess(
120118

121119
try {
122120
Awaitility.await()
123-
.atMost(2, TimeUnit.MINUTES)
121+
.atMost(5, TimeUnit.MINUTES)
124122
.pollDelay(2, TimeUnit.SECONDS)
125123
.until(
126124
() -> {
@@ -164,7 +162,7 @@ public static void awaitUntilSuccess(
164162
lastTimeDataNodeLocations.get().removeAll(removeDataNodeLocations);
165163
String expectedSetStr = lastTimeDataNodeLocations.get().toString();
166164
LOGGER.error(
167-
"Remove DataNodes timeout in 2 minutes, expected set: {}, actual set: {}",
165+
"Remove DataNodes timeout in 5 minutes, expected set: {}, actual set: {}",
168166
expectedSetStr,
169167
actualSetStr);
170168
if (lastException.get() == null) {

‎integration-test/src/test/java/org/apache/iotdb/confignode/it/removedatanode/IoTDBRemoveUnknownDataNodeIT.java‎

Lines changed: 17 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@
4949
import java.sql.Connection;
5050
import java.sql.ResultSet;
5151
import java.sql.Statement;
52+
import java.util.ArrayList;
5253
import java.util.HashSet;
5354
import java.util.List;
5455
import java.util.Map;
@@ -240,6 +241,9 @@ public void testRemoveDataNode(
240241
dataRegionPerDataNode * dataNodeNum / dataReplicateFactor);
241242
EnvFactory.getEnv().initClusterEnvironment(configNodeNum, dataNodeNum);
242243

244+
final Set<Integer> removeDataNodes = new HashSet<>();
245+
final List<TDataNodeLocation> removeDataNodeLocations = new ArrayList<>();
246+
243247
try (final Connection connection = makeItCloseQuietly(getConnectionWithSQLType(model));
244248
final Statement statement = makeItCloseQuietly(connection.createStatement());
245249
SyncConfigNodeIServiceClient client =
@@ -269,16 +273,14 @@ public void testRemoveDataNode(
269273
allDataNodeId.add(result.getInt(ColumnHeaderConstant.NODE_ID));
270274
}
271275

272-
// Select data nodes to remove
273-
final Set<Integer> removeDataNodes = selectRemoveDataNodes(allDataNodeId, removeDataNodeNum);
274-
276+
// Randomly select data nodes to remove
277+
removeDataNodes.addAll(selectRemoveDataNodes(allDataNodeId, removeDataNodeNum));
275278
List<DataNodeWrapper> removeDataNodeWrappers =
276279
removeDataNodes.stream()
277280
.map(dataNodeId -> EnvFactory.getEnv().dataNodeIdToWrapper(dataNodeId).get())
278281
.collect(Collectors.toList());
279-
280282
AtomicReference<SyncConfigNodeIServiceClient> clientRef = new AtomicReference<>(client);
281-
List<TDataNodeLocation> removeDataNodeLocations =
283+
removeDataNodeLocations.addAll(
282284
clientRef
283285
.get()
284286
.getDataNodeConfiguration(-1)
@@ -287,15 +289,22 @@ public void testRemoveDataNode(
287289
.stream()
288290
.map(TDataNodeConfiguration::getLocation)
289291
.filter(location -> removeDataNodes.contains(location.getDataNodeId()))
290-
.collect(Collectors.toList());
291-
292+
.collect(Collectors.toList()));
292293
// Stop DataNodes before removing them
293294
stopDataNodes(removeDataNodeWrappers);
294295
LOGGER.info("RemoveDataNodes: {} are stopped.", removeDataNodes);
296+
} catch (InconsistentDataException e) {
297+
LOGGER.error("Unexpected error:", e);
298+
}
295299

300+
// Establish a new connection after stopping data nodes
301+
try (final Connection connection = makeItCloseQuietly(EnvFactory.getEnv().getConnection());
302+
final Statement statement = makeItCloseQuietly(connection.createStatement());
303+
SyncConfigNodeIServiceClient client =
304+
(SyncConfigNodeIServiceClient) EnvFactory.getEnv().getLeaderConfigNodeConnection()) {
305+
AtomicReference<SyncConfigNodeIServiceClient> clientRef = new AtomicReference<>(client);
296306
if (SQLModel.NOT_USE_SQL.equals(model)) {
297307
TDataNodeRemoveReq removeReq = new TDataNodeRemoveReq(removeDataNodeLocations);
298-
299308
// Remove data nodes
300309
TDataNodeRemoveResp removeResp = clientRef.get().removeDataNode(removeReq);
301310
LOGGER.info("Submit Remove DataNodes result {} ", removeResp);
@@ -345,12 +354,6 @@ public void testRemoveDataNode(
345354
}
346355

347356
LOGGER.info("Remove DataNodes success");
348-
} catch (InconsistentDataException e) {
349-
LOGGER.error("Unexpected error:", e);
350-
}
351-
352-
try (final Connection connection = makeItCloseQuietly(EnvFactory.getEnv().getConnection());
353-
final Statement statement = makeItCloseQuietly(connection.createStatement())) {
354357

355358
// Check the data region distribution after removing data nodes
356359
Map<Integer, Set<Integer>> afterRegionMap = getDataRegionMap(statement);

‎iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/pipe/PipeConsensusServerImpl.java‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -384,7 +384,15 @@ public void setRemotePeerActive(Peer peer, boolean isActive, boolean isForDeleti
384384
String.format("error when set peer %s to active %s", peer, isActive), e);
385385
}
386386
} catch (ClientManagerException e) {
387-
throw new ConsensusGroupModifyPeerException(e);
387+
if (isForDeletionPurpose) {
388+
// for remove peer, if target peer is already down, we can skip this step.
389+
LOGGER.warn(
390+
"target peer may be down, error when set peer {} to active {}", peer, isActive, e);
391+
} else {
392+
// for add peer, if target peer is down, we need to throw exception to identify the failure
393+
// of this addPeerProcedure.
394+
throw new ConsensusGroupModifyPeerException(e);
395+
}
388396
}
389397
}
390398

@@ -653,7 +661,8 @@ public void waitReleaseAllRegionRelatedResource(Peer targetPeer)
653661
Thread.sleep(checkIntervalInMs);
654662
}
655663
} catch (ClientManagerException | TException e) {
656-
throw new ConsensusGroupModifyPeerException(
664+
// in case of target peer is down or can not serve, we simply skip it.
665+
LOGGER.warn(
657666
String.format(
658667
"error when waiting %s to release all region related resource. %s",
659668
targetPeer, e.getMessage()),

0 commit comments

Comments
 (0)