YARN-6403. Invalid local resource request can raise NPE and make NM e…

…xit. Contributed by Tao Yang
cc13ny · Apr 5, 2017 · e8071aa · e8071aa
1 parent 34ab8e7
commit e8071aa
Show file tree

Hide file tree

Showing 5 changed files with 108 additions and 0 deletions.
diff --git a/...rc/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerLaunchContextPBImpl.java b/...rc/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerLaunchContextPBImpl.java
@@ -208,11 +208,24 @@ public void setLocalResources(
       final Map<String, LocalResource> localResources) {
     if (localResources == null)
       return;
+    checkLocalResources(localResources);
     initLocalResources();
     this.localResources.clear();
     this.localResources.putAll(localResources);
   }
 
+  private void checkLocalResources(Map<String, LocalResource> localResources) {
+    for (Map.Entry<String, LocalResource> rsrcEntry : localResources
+        .entrySet()) {
+      if (rsrcEntry.getValue() == null
+          || rsrcEntry.getValue().getResource() == null) {
+        throw new NullPointerException(
+            "Null resource URL for local resource " + rsrcEntry.getKey() + " : "
+                + rsrcEntry.getValue());
+      }
+    }
+  }
+
   private void addLocalResourcesToProto() {
     maybeInitBuilder();
     builder.clearLocalResources();

diff --git a/...java/org/apache/hadoop/yarn/api/records/impl/pb/TestApplicationClientProtocolRecords.java b/...java/org/apache/hadoop/yarn/api/records/impl/pb/TestApplicationClientProtocolRecords.java
@@ -30,6 +30,10 @@
 import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
 import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
 import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.LocalResourceType;
+import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
+import org.apache.hadoop.yarn.factories.RecordFactory;
+import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
 import org.junit.Assert;
 import org.junit.Test;
 
@@ -66,4 +70,29 @@ public void testCLCPBImplNullEnv() throws IOException {
         clcProto.getEnvironment().get("testCLCPBImplNullEnv"));
 
   }
+
+  /*
+   * This test validates the scenario in which the client sets a null value for
+   * local resource URL.
+   */
+  @Test
+  public void testCLCPBImplNullResourceURL() throws IOException {
+    RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
+    try {
+      LocalResource rsrc_alpha = recordFactory.newRecordInstance(LocalResource.class);
+      rsrc_alpha.setResource(null);
+      rsrc_alpha.setSize(-1);
+      rsrc_alpha.setVisibility(LocalResourceVisibility.APPLICATION);
+      rsrc_alpha.setType(LocalResourceType.FILE);
+      rsrc_alpha.setTimestamp(System.currentTimeMillis());
+      Map<String, LocalResource> localResources =
+          new HashMap<String, LocalResource>();
+      localResources.put("null_url_resource", rsrc_alpha);
+      ContainerLaunchContext containerLaunchContext = recordFactory.newRecordInstance(ContainerLaunchContext.class);
+      containerLaunchContext.setLocalResources(localResources);
+      Assert.fail("Setting an invalid local resource should be an error!");
+    } catch (NullPointerException e) {
+      Assert.assertTrue(e.getMessage().contains("Null resource URL for local resource"));
+    }
+  }
 }
diff --git a/...java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java b/...java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
@@ -64,6 +64,7 @@
 import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
 import org.apache.hadoop.yarn.api.records.ContainerState;
 import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.LocalResource;
 import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
 import org.apache.hadoop.yarn.api.records.LogAggregationContext;
 import org.apache.hadoop.yarn.api.records.NodeId;
@@ -996,6 +997,15 @@ protected void startContainerInternal(
 
     ContainerLaunchContext launchContext = request.getContainerLaunchContext();
 
+    // Sanity check for local resources
+    for (Map.Entry<String, LocalResource> rsrc : launchContext
+        .getLocalResources().entrySet()) {
+      if (rsrc.getValue() == null || rsrc.getValue().getResource() == null) {
+        throw new YarnException(
+            "Null resource URL for local resource " + rsrc.getKey() + " : " + rsrc.getValue());
+      }
+    }
+
     Credentials credentials =
         YarnServerSecurityUtils.parseCredentials(launchContext);
 

diff --git a/.../src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestContainerManagerWithLCE.java b/.../src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestContainerManagerWithLCE.java
@@ -364,6 +364,17 @@ public void testContainerRestart() throws IOException, InterruptedException,
     super.testContainerRestart();
   }
 
+  @Override
+  public void testStartContainerFailureWithInvalidLocalResource() throws Exception {
+    // Don't run the test if the binary is not available.
+    if (!shouldRunTest()) {
+      LOG.info("LCE binary path is not passed. Not running the test");
+      return;
+    }
+    LOG.info("Running testStartContainerFailureWithInvalidLocalResource");
+    super.testStartContainerFailureWithInvalidLocalResource();
+  }
+
   private boolean shouldRunTest() {
     return System
         .getProperty(YarnConfiguration.NM_LINUX_CONTAINER_EXECUTOR_PATH) != null;

diff --git a/...java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java b/...java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java
@@ -1859,4 +1859,49 @@ private void testContainerLaunchAndSignal(SignalContainerCommand command)
       Assert.assertEquals(signal, signalContext.getSignal());
     }
   }
+
+  @Test
+  public void testStartContainerFailureWithInvalidLocalResource()
+      throws Exception {
+    containerManager.start();
+    LocalResource rsrc_alpha =
+        recordFactory.newRecordInstance(LocalResource.class);
+    rsrc_alpha.setResource(null);
+    rsrc_alpha.setSize(-1);
+    rsrc_alpha.setVisibility(LocalResourceVisibility.APPLICATION);
+    rsrc_alpha.setType(LocalResourceType.FILE);
+    rsrc_alpha.setTimestamp(System.currentTimeMillis());
+    Map<String, LocalResource> localResources =
+        new HashMap<String, LocalResource>();
+    localResources.put("invalid_resource", rsrc_alpha);
+    ContainerLaunchContext containerLaunchContext =
+        recordFactory.newRecordInstance(ContainerLaunchContext.class);
+    ContainerLaunchContext spyContainerLaunchContext =
+        Mockito.spy(containerLaunchContext);
+    Mockito.when(spyContainerLaunchContext.getLocalResources())
+        .thenReturn(localResources);
+
+    ContainerId cId = createContainerId(0);
+    String user = "start_container_fail";
+    Token containerToken =
+        createContainerToken(cId, DUMMY_RM_IDENTIFIER, context.getNodeId(),
+            user, context.getContainerTokenSecretManager());
+    StartContainerRequest request = StartContainerRequest
+        .newInstance(spyContainerLaunchContext, containerToken);
+
+    // start containers
+    List<StartContainerRequest> startRequest =
+        new ArrayList<StartContainerRequest>();
+    startRequest.add(request);
+    StartContainersRequest requestList =
+        StartContainersRequest.newInstance(startRequest);
+
+    StartContainersResponse response =
+        containerManager.startContainers(requestList);
+    Assert.assertTrue(response.getFailedRequests().size() == 1);
+    Assert.assertTrue(response.getSuccessfullyStartedContainers().size() == 0);
+    Assert.assertTrue(response.getFailedRequests().containsKey(cId));
+    Assert.assertTrue(response.getFailedRequests().get(cId).getMessage()
+        .contains("Null resource URL for local resource"));
+  }
 }