From bf94adf6f1c470a9fd22a474ce25cbfd2e94af06 Mon Sep 17 00:00:00 2001 From: Mukul Kumar Singh Date: Fri, 5 Jul 2019 17:04:40 +0530 Subject: [PATCH] HDDS-1758. Add replication and key deletion tests to MiniOzoneChaosCluster. Contributed by Mukul Kumar Singh. (#1049) --- .../hadoop/ozone/MiniOzoneChaosCluster.java | 57 ++++++++++++++++--- .../hadoop/ozone/MiniOzoneLoadGenerator.java | 5 ++ .../ozone/TestMiniChaosOzoneCluster.java | 2 +- .../hadoop/ozone/om/KeyManagerImpl.java | 3 +- 4 files changed, 58 insertions(+), 9 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneChaosCluster.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneChaosCluster.java index 059af5a8b5922..dc2787de6e98c 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneChaosCluster.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneChaosCluster.java @@ -22,6 +22,7 @@ import org.apache.hadoop.conf.StorageUnit; import org.apache.hadoop.hdds.HddsConfigKeys; import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.scm.ScmConfigKeys; import org.apache.hadoop.hdds.scm.server.StorageContainerManager; import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; @@ -55,7 +56,8 @@ public class MiniOzoneChaosCluster extends MiniOzoneClusterImpl { private ScheduledFuture scheduledFuture; private enum FailureMode { - NODES + NODES_RESTART, + NODES_SHUTDOWN } public MiniOzoneChaosCluster(OzoneConfiguration conf, @@ -81,21 +83,55 @@ private boolean isFastRestart() { return RandomUtils.nextBoolean(); } + // Should the selected node be stopped or started. + private boolean shouldStop() { + return RandomUtils.nextBoolean(); + } + // Get the datanode index of the datanode to fail. private int getNodeToFail() { return RandomUtils.nextInt() % numDatanodes; } - private void failNodes() { + private void restartNodes() { final int numNodesToFail = getNumberOfNodesToFail(); LOG.info("Will restart {} nodes to simulate failure", numNodesToFail); for (int i = 0; i < numNodesToFail; i++) { boolean failureMode = isFastRestart(); int failedNodeIndex = getNodeToFail(); + String failString = failureMode ? "Fast" : "Slow"; + DatanodeDetails dn = + getHddsDatanodes().get(failedNodeIndex).getDatanodeDetails(); try { - LOG.info("Restarting DataNodeIndex {}", failedNodeIndex); + LOG.info("{} Restarting DataNode: {}", failString, dn.getUuid()); restartHddsDatanode(failedNodeIndex, failureMode); - LOG.info("Completed restarting DataNodeIndex {}", failedNodeIndex); + LOG.info("{} Completed restarting Datanode: {}", failString, + dn.getUuid()); + } catch (Exception e) { + + } + } + } + + private void shutdownNodes() { + final int numNodesToFail = getNumberOfNodesToFail(); + LOG.info("Will shutdown {} nodes to simulate failure", numNodesToFail); + for (int i = 0; i < numNodesToFail; i++) { + boolean shouldStop = shouldStop(); + int failedNodeIndex = getNodeToFail(); + String stopString = shouldStop ? "Stopping" : "Starting"; + DatanodeDetails dn = + getHddsDatanodes().get(failedNodeIndex).getDatanodeDetails(); + try { + LOG.info("{} DataNode {}", stopString, dn.getUuid()); + + if (shouldStop) { + shutdownHddsDatanode(failedNodeIndex); + } else { + restartHddsDatanode(failedNodeIndex, true); + } + LOG.info("Completed {} DataNode {}", stopString, dn.getUuid()); + } catch (Exception e) { } @@ -111,8 +147,11 @@ private FailureMode getFailureMode() { private void fail() { FailureMode mode = getFailureMode(); switch (mode) { - case NODES: - failNodes(); + case NODES_RESTART: + restartNodes(); + break; + case NODES_SHUTDOWN: + shutdownNodes(); break; default: @@ -190,7 +229,9 @@ void initializeConfiguration() throws IOException { 1, StorageUnit.MB); conf.setTimeDuration(ScmConfigKeys.HDDS_SCM_WATCHER_TIMEOUT, 1000, TimeUnit.MILLISECONDS); - conf.setTimeDuration(ScmConfigKeys.OZONE_SCM_STALENODE_INTERVAL, 5, + conf.setTimeDuration(ScmConfigKeys.OZONE_SCM_STALENODE_INTERVAL, 10, + TimeUnit.SECONDS); + conf.setTimeDuration(ScmConfigKeys.OZONE_SCM_DEADNODE_INTERVAL, 20, TimeUnit.SECONDS); conf.setTimeDuration(HddsConfigKeys.HDDS_CONTAINER_REPORT_INTERVAL, 1, TimeUnit.SECONDS); @@ -204,6 +245,8 @@ void initializeConfiguration() throws IOException { conf.setTimeDuration(HddsConfigKeys.HDDS_HEARTBEAT_INTERVAL, 1, TimeUnit.SECONDS); conf.setInt(OzoneConfigKeys.OZONE_CONTAINER_CACHE_SIZE, 8); + conf.setInt("hdds.scm.replication.thread.interval", 10 * 1000); + conf.setInt("hdds.scm.replication.event.timeout", 20 * 1000); } @Override diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneLoadGenerator.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneLoadGenerator.java index 3623747984605..67edb15238013 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneLoadGenerator.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneLoadGenerator.java @@ -130,6 +130,11 @@ private void load(long runTimeMillis) { break; } + try { + bucket.deleteKey(keyName); + } catch (Exception e) { + LOG.error("LOADGEN: Unable to delete key:{}", keyName, e); + } } // This will terminate other threads too. isWriteThreadRunning.set(false); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestMiniChaosOzoneCluster.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestMiniChaosOzoneCluster.java index 8bc3a52c86e61..0cb7f81afe3a6 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestMiniChaosOzoneCluster.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestMiniChaosOzoneCluster.java @@ -62,7 +62,7 @@ public class TestMiniChaosOzoneCluster implements Runnable { @Option(names = {"-i", "--failureInterval"}, description = "time between failure events in seconds") - private static int failureInterval = 5; // 5 second period between failures. + private static int failureInterval = 300; // 5 second period between failures. private static MiniOzoneChaosCluster cluster; private static MiniOzoneLoadGenerator loadGenerator; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java index 75d1af5bad4b9..99e3e660f642e 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java @@ -711,7 +711,8 @@ public OmKeyInfo lookupKey(OmKeyArgs args) throws IOException { k.setPipeline(cp.getPipeline()); } } catch (IOException e) { - LOG.debug("Unable to update pipeline for container"); + LOG.error("Unable to update pipeline for container:{}", + k.getContainerID()); } } });