YARN-3413. Changed Nodelabel attributes (like exclusivity) to be sett…

…able only via addToClusterNodeLabels but not changeable at runtime. (Wangda Tan via vinodkv)
dianhu · Apr 23, 2015 · f5fe35e · f5fe35e
1 parent aa4a192
commit f5fe35e
Show file tree

Hide file tree

Showing 52 changed files with 600 additions and 879 deletions.
diff --git a/...apreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java b/...apreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java
@@ -58,6 +58,7 @@
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.ContainerReport;
 import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.NodeLabel;
 import org.apache.hadoop.yarn.api.records.NodeReport;
 import org.apache.hadoop.yarn.api.records.NodeState;
 import org.apache.hadoop.yarn.api.records.QueueUserACLInfo;
@@ -461,7 +462,7 @@ public Map<String, Set<NodeId>> getLabelsToNodes(Set<String> labels)
   }
 
   @Override
-  public Set<String> getClusterNodeLabels()
+  public List<NodeLabel> getClusterNodeLabels()
       throws YarnException, IOException {
     return client.getClusterNodeLabels();
   }

diff --git a/...rc/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetClusterNodeLabelsResponse.java b/...rc/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetClusterNodeLabelsResponse.java
@@ -18,16 +18,17 @@
 
 package org.apache.hadoop.yarn.api.protocolrecords;
 
-import java.util.Set;
+import java.util.List;
 
 import org.apache.hadoop.classification.InterfaceAudience.Public;
 import org.apache.hadoop.classification.InterfaceStability.Evolving;
+import org.apache.hadoop.yarn.api.records.NodeLabel;
 import org.apache.hadoop.yarn.util.Records;
 
 @Public
 @Evolving
 public abstract class GetClusterNodeLabelsResponse {
-  public static GetClusterNodeLabelsResponse newInstance(Set<String> labels) {
+  public static GetClusterNodeLabelsResponse newInstance(List<NodeLabel> labels) {
     GetClusterNodeLabelsResponse request =
         Records.newRecord(GetClusterNodeLabelsResponse.class);
     request.setNodeLabels(labels);
@@ -36,9 +37,9 @@ public static GetClusterNodeLabelsResponse newInstance(Set<String> labels) {
 
   @Public
   @Evolving
-  public abstract void setNodeLabels(Set<String> labels);
+  public abstract void setNodeLabels(List<NodeLabel> labels);
 
   @Public
   @Evolving
-  public abstract Set<String> getNodeLabels();
+  public abstract List<NodeLabel> getNodeLabels();
 }
diff --git a/...doop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeLabel.java b/...doop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeLabel.java
@@ -18,38 +18,81 @@
 
 package org.apache.hadoop.yarn.api.records;
 
+import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.classification.InterfaceAudience.Public;
 import org.apache.hadoop.classification.InterfaceStability.Stable;
 import org.apache.hadoop.classification.InterfaceStability.Unstable;
 import org.apache.hadoop.yarn.util.Records;
 
 @Public
 @Unstable
-public abstract class NodeLabel {
-  @Public
+public abstract class NodeLabel implements Comparable<NodeLabel> {
+  /**
+   * By default, node label is exclusive or not
+   */
+  @Private
+  @Unstable
+  public static final boolean DEFAULT_NODE_LABEL_EXCLUSIVITY = true;
+
+  @Private
+  @Unstable
+  public static NodeLabel newInstance(String name) {
+    return newInstance(name, DEFAULT_NODE_LABEL_EXCLUSIVITY);
+  }
+
+  @Private
   @Unstable
-  public static NodeLabel newInstance(String nodeLabel,
-      boolean isExclusive) {
-    NodeLabel request =
-        Records.newRecord(NodeLabel.class);
-    request.setNodeLabel(nodeLabel);
-    request.setIsExclusive(isExclusive);
+  public static NodeLabel newInstance(String name, boolean isExclusive) {
+    NodeLabel request = Records.newRecord(NodeLabel.class);
+    request.setName(name);
+    request.setExclusivity(isExclusive);
     return request;
   }
-  
+
   @Public
   @Stable
-  public abstract String getNodeLabel();
-  
-  @Public
+  public abstract String getName();
+
+  @Private
   @Unstable
-  public abstract void setNodeLabel(String nodeLabel);
-  
+  public abstract void setName(String name);
+
   @Public
   @Stable
-  public abstract boolean getIsExclusive();
-  
-  @Public
+  public abstract boolean isExclusive();
+
+  @Private
   @Unstable
-  public abstract void setIsExclusive(boolean isExclusive);
+  public abstract void setExclusivity(boolean isExclusive);
+
+  @Override
+  public int compareTo(NodeLabel other) {
+    return getName().compareTo(other.getName());
+  }
+
+  @Override
+  public boolean equals(Object obj) {
+    if (obj instanceof NodeLabel) {
+      NodeLabel nl = (NodeLabel) obj;
+      return nl.getName().equals(getName())
+          && nl.isExclusive() == isExclusive();
+    }
+    return false;
+  }
+
+  @Override
+  public String toString() {
+    StringBuilder sb = new StringBuilder();
+    sb.append("<");
+    sb.append(getName());
+    sb.append(":exclusivity=");
+    sb.append(isExclusive());
+    sb.append(">");
+    return sb.toString();
+  }
+
+  @Override
+  public int hashCode() {
+    return (getName().hashCode() << 16) + (isExclusive() ? 1 : 0);
+  }
 }
diff --git a/...rc/main/java/org/apache/hadoop/yarn/server/api/ResourceManagerAdministrationProtocol.java b/...rc/main/java/org/apache/hadoop/yarn/server/api/ResourceManagerAdministrationProtocol.java
@@ -50,8 +50,6 @@
 import org.apache.hadoop.yarn.server.api.protocolrecords.RemoveFromClusterNodeLabelsResponse;
 import org.apache.hadoop.yarn.server.api.protocolrecords.ReplaceLabelsOnNodeRequest;
 import org.apache.hadoop.yarn.server.api.protocolrecords.ReplaceLabelsOnNodeResponse;
-import org.apache.hadoop.yarn.server.api.protocolrecords.UpdateNodeLabelsRequest;
-import org.apache.hadoop.yarn.server.api.protocolrecords.UpdateNodeLabelsResponse;
 import org.apache.hadoop.yarn.server.api.protocolrecords.UpdateNodeResourceRequest;
 import org.apache.hadoop.yarn.server.api.protocolrecords.UpdateNodeResourceResponse;
 
@@ -139,12 +137,6 @@ public RemoveFromClusterNodeLabelsResponse removeFromClusterNodeLabels(
   public ReplaceLabelsOnNodeResponse replaceLabelsOnNode(
       ReplaceLabelsOnNodeRequest request) throws YarnException, IOException;
 
-  @Public
-  @Evolving
-  @Idempotent
-  public UpdateNodeLabelsResponse updateNodeLabels(
-      UpdateNodeLabelsRequest request) throws YarnException, IOException;
-
   @Public
   @Evolving
   @Idempotent

diff --git a/...java/org/apache/hadoop/yarn/server/api/protocolrecords/AddToClusterNodeLabelsRequest.java b/...java/org/apache/hadoop/yarn/server/api/protocolrecords/AddToClusterNodeLabelsRequest.java
@@ -18,27 +18,31 @@
 
 package org.apache.hadoop.yarn.server.api.protocolrecords;
 
-import java.util.Set;
+import java.util.List;
 
 import org.apache.hadoop.classification.InterfaceAudience.Public;
-import org.apache.hadoop.classification.InterfaceStability.Evolving;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.api.records.NodeLabel;
 import org.apache.hadoop.yarn.util.Records;
 
 @Public
-@Evolving
+@Unstable
 public abstract class AddToClusterNodeLabelsRequest {
-  public static AddToClusterNodeLabelsRequest newInstance(Set<String> labels) {
+  @Public
+  @Unstable
+  public static AddToClusterNodeLabelsRequest newInstance(
+      List<NodeLabel> NodeLabels) {
     AddToClusterNodeLabelsRequest request =
-        Records.newRecord(AddToClusterNodeLabelsRequest.class);
-    request.setNodeLabels(labels);
+      Records.newRecord(AddToClusterNodeLabelsRequest.class);
+    request.setNodeLabels(NodeLabels);
     return request;
   }
 
   @Public
-  @Evolving
-  public abstract void setNodeLabels(Set<String> labels);
+  @Unstable
+  public abstract void setNodeLabels(List<NodeLabel> NodeLabels);
 
   @Public
-  @Evolving
-  public abstract Set<String> getNodeLabels();
+  @Unstable
+  public abstract List<NodeLabel> getNodeLabels();
 }
diff --git a/.../main/java/org/apache/hadoop/yarn/server/api/protocolrecords/UpdateNodeLabelsRequest.java b/.../main/java/org/apache/hadoop/yarn/server/api/protocolrecords/UpdateNodeLabelsRequest.java
diff --git a/...main/java/org/apache/hadoop/yarn/server/api/protocolrecords/UpdateNodeLabelsResponse.java b/...main/java/org/apache/hadoop/yarn/server/api/protocolrecords/UpdateNodeLabelsResponse.java
diff --git a/...-yarn/hadoop-yarn-api/src/main/proto/server/resourcemanager_administration_protocol.proto b/...-yarn/hadoop-yarn-api/src/main/proto/server/resourcemanager_administration_protocol.proto
@@ -42,6 +42,5 @@ service ResourceManagerAdministrationProtocolService {
   rpc addToClusterNodeLabels(AddToClusterNodeLabelsRequestProto) returns (AddToClusterNodeLabelsResponseProto);
   rpc removeFromClusterNodeLabels(RemoveFromClusterNodeLabelsRequestProto) returns (RemoveFromClusterNodeLabelsResponseProto);
   rpc replaceLabelsOnNodes(ReplaceLabelsOnNodeRequestProto) returns (ReplaceLabelsOnNodeResponseProto);
-  rpc updateNodeLabels(UpdateNodeLabelsRequestProto) returns (UpdateNodeLabelsResponseProto);
   rpc checkForDecommissioningNodes(CheckForDecommissioningNodesRequestProto) returns (CheckForDecommissioningNodesResponseProto);
 }
diff --git a/...rn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto b/...rn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto
@@ -77,7 +77,7 @@ message UpdateNodeResourceResponseProto {
 }
 
 message AddToClusterNodeLabelsRequestProto {
-  repeated string nodeLabels = 1;
+  repeated NodeLabelProto nodeLabels = 1;
 }
 
 message AddToClusterNodeLabelsResponseProto {
@@ -98,11 +98,6 @@ message ReplaceLabelsOnNodeResponseProto {
 
 }
 
-message UpdateNodeLabelsRequestProto {
-  repeated NodeLabelProto nodeLabels = 1;
-}
-
-
 message UpdateNodeLabelsResponseProto {
 }
 

diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
@@ -262,7 +262,7 @@ message LabelsToNodeIdsProto {
 }
 
 message NodeLabelProto {
-  optional string nodeLabel = 1;
+  optional string name = 1;
   optional bool isExclusive = 2 [default = true]; 
 }
 

diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
@@ -213,7 +213,7 @@ message GetClusterNodeLabelsRequestProto {
 }
 
 message GetClusterNodeLabelsResponseProto {
-  repeated string nodeLabels = 1;
+  repeated NodeLabelProto nodeLabels = 1;
 }
 
 //////////////////////////////////////////////////////

diff --git a/.../apache/hadoop/yarn/applications/distributedshell/TestDistributedShellWithNodeLabels.java b/.../apache/hadoop/yarn/applications/distributedshell/TestDistributedShellWithNodeLabels.java
@@ -54,7 +54,7 @@ private void initializeNodeLabels() throws IOException {
     RMNodeLabelsManager labelsMgr = rmContext.getNodeLabelManager();
     Set<String> labels = new HashSet<String>();
     labels.add("x");
-    labelsMgr.addToCluserNodeLabels(labels);
+    labelsMgr.addToCluserNodeLabelsWithDefaultExclusivity(labels);
 
     // Setup queue access to node labels
     distShellTest.conf.set("yarn.scheduler.capacity.root.accessible-node-labels", "x");

diff --git a/...p-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java b/...p-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java
@@ -46,6 +46,7 @@
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.ContainerReport;
 import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.NodeLabel;
 import org.apache.hadoop.yarn.api.records.NodeReport;
 import org.apache.hadoop.yarn.api.records.NodeState;
 import org.apache.hadoop.yarn.api.records.QueueInfo;
@@ -663,6 +664,6 @@ public abstract Map<String, Set<NodeId>> getLabelsToNodes(Set<String> labels)
    */
   @Public
   @Unstable
-  public abstract Set<String> getClusterNodeLabels()
+  public abstract List<NodeLabel> getClusterNodeLabels()
       throws YarnException, IOException;
 }
diff --git a/...doop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java b/...doop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
@@ -86,6 +86,7 @@
 import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
 import org.apache.hadoop.yarn.api.records.ContainerReport;
 import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.NodeLabel;
 import org.apache.hadoop.yarn.api.records.NodeReport;
 import org.apache.hadoop.yarn.api.records.NodeState;
 import org.apache.hadoop.yarn.api.records.QueueInfo;
@@ -815,7 +816,7 @@ public Map<String, Set<NodeId>> getLabelsToNodes(Set<String> labels)
   }
 
   @Override
-  public Set<String> getClusterNodeLabels() throws YarnException, IOException {
+  public List<NodeLabel> getClusterNodeLabels() throws YarnException, IOException {
     return rmClient.getClusterNodeLabels(
         GetClusterNodeLabelsRequest.newInstance()).getNodeLabels();
   }