HIVE-8707 - Fix ordering differences due to Java 8 HashMap function (…

…Mohit Sabharwal via Brock) git-svn-id: https://svn.apache.org/repos/asf/hive/trunk@1641162 13f79535-47bb-0310-9956-ffa450edef68
gautamboddeda · Nov 23, 2014 · 0ef01bf · 0ef01bf
1 parent f17f122
commit 0ef01bf
Show file tree

Hide file tree

Showing 309 changed files with 9,686 additions and 9,676 deletions.
diff --git a/hbase-handler/src/test/results/positive/hbase_queries.q.out b/hbase-handler/src/test/results/positive/hbase_queries.q.out
@@ -268,36 +268,36 @@ STAGE PLANS:
     Map Reduce
       Map Operator Tree:
           TableScan
-            alias: hbase_table_2
+            alias: hbase_table_1
             Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
             Filter Operator
-              predicate: (key < 120) (type: boolean)
+              predicate: (100 < key) (type: boolean)
               Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
               Select Operator
-                expressions: key (type: int), value (type: string)
-                outputColumnNames: _col0, _col1
+                expressions: key (type: int)
+                outputColumnNames: _col0
                 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
                 Reduce Output Operator
                   key expressions: _col0 (type: int)
                   sort order: +
                   Map-reduce partition columns: _col0 (type: int)
                   Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
-                  value expressions: _col1 (type: string)
           TableScan
-            alias: hbase_table_1
+            alias: hbase_table_2
             Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
             Filter Operator
-              predicate: (100 < key) (type: boolean)
+              predicate: (key < 120) (type: boolean)
               Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
               Select Operator
-                expressions: key (type: int)
-                outputColumnNames: _col0
+                expressions: key (type: int), value (type: string)
+                outputColumnNames: _col0, _col1
                 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
                 Reduce Output Operator
                   key expressions: _col0 (type: int)
                   sort order: +
                   Map-reduce partition columns: _col0 (type: int)
                   Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+                  value expressions: _col1 (type: string)
       Reduce Operator Tree:
         Join Operator
           condition map:

diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
@@ -1023,7 +1023,8 @@ private <T> List<T> convertList(List<T> dnList) {
 
   /** Makes shallow copy of a map to avoid DataNucleus mucking with our objects. */
   private <K, V> Map<K, V> convertMap(Map<K, V> dnMap) {
-    return (dnMap == null) ? null : Maps.newHashMap(dnMap);
+    // Must be deterministic order map - see HIVE-8707
+    return (dnMap == null) ? null : Maps.newLinkedHashMap(dnMap);
   }
 
   private Table convertToTable(MTable mtbl) throws MetaException {

diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/QB.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/QB.java
@@ -20,6 +20,7 @@
 
 import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -103,17 +104,19 @@ public QB() {
   }
 
   public QB(String outer_id, String alias, boolean isSubQ) {
-    aliasToTabs = new HashMap<String, String>();
-    aliasToSubq = new HashMap<String, QBExpr>();
-    aliasToProps = new HashMap<String, Map<String, String>>();
+    // Must be deterministic order maps - see HIVE-8707
+    aliasToTabs = new LinkedHashMap<String, String>();
+    aliasToSubq = new LinkedHashMap<String, QBExpr>();
+    aliasToProps = new LinkedHashMap<String, Map<String, String>>();
     aliases = new ArrayList<String>();
     if (alias != null) {
       alias = alias.toLowerCase();
     }
     qbp = new QBParseInfo(alias, isSubQ);
     qbm = new QBMetaData();
-    ptfNodeToSpec = new HashMap<ASTNode, PTFInvocationSpec>();
-    destToWindowingSpec = new HashMap<String, WindowingSpec>();
+    // Must be deterministic order maps - see HIVE-8707
+    ptfNodeToSpec = new LinkedHashMap<ASTNode, PTFInvocationSpec>();
+    destToWindowingSpec = new LinkedHashMap<String, WindowingSpec>();
     id = getAppendedAliasFromId(outer_id, alias);
   }
 
@@ -320,7 +323,8 @@ public PTFInvocationSpec getPTFInvocationSpec(ASTNode node) {
   }
 
   public void addPTFNodeToSpec(ASTNode node, PTFInvocationSpec spec) {
-    ptfNodeToSpec = ptfNodeToSpec == null ? new HashMap<ASTNode, PTFInvocationSpec>() : ptfNodeToSpec;
+    // Must be deterministic order map - see HIVE-8707
+    ptfNodeToSpec = ptfNodeToSpec == null ? new LinkedHashMap<ASTNode, PTFInvocationSpec>() : ptfNodeToSpec;
     ptfNodeToSpec.put(node, spec);
   }
 

diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/QBMetaData.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/QBMetaData.java
@@ -19,6 +19,7 @@
 package org.apache.hadoop.hive.ql.parse;
 
 import java.util.HashMap;
+import java.util.LinkedHashMap;
 import java.util.Map;
 
 import org.apache.commons.logging.Log;
@@ -53,13 +54,15 @@ public class QBMetaData {
   private static final Log LOG = LogFactory.getLog(QBMetaData.class.getName());
 
   public QBMetaData() {
-    aliasToTable = new HashMap<String, Table>();
+    // Must be deterministic order map - see HIVE-8707
+    aliasToTable = new LinkedHashMap<String, Table>();
     nameToDestTable = new HashMap<String, Table>();
     nameToDestPartition = new HashMap<String, Partition>();
     nameToDestFile = new HashMap<String, String>();
     nameToDestType = new HashMap<String, Integer>();
-    aliasToPartSpec = new HashMap<String, Map<String, String>>();
-    aliasToDPCtx  = new HashMap<String, DynamicPartitionCtx>();
+    // Must be deterministic order maps - see HIVE-8707
+    aliasToPartSpec = new LinkedHashMap<String, Map<String, String>>();
+    aliasToDPCtx  = new LinkedHashMap<String, DynamicPartitionCtx>();
   }
 
   // All getXXX needs toLowerCase() because they are directly called from

diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
@@ -400,8 +400,9 @@ public SemanticAnalyzer(HiveConf conf) throws SemanticException {
     opToPartList = new HashMap<TableScanOperator, PrunedPartitionList>();
     opToSamplePruner = new HashMap<TableScanOperator, sampleDesc>();
     nameToSplitSample = new HashMap<String, SplitSample>();
-    topOps = new HashMap<String, Operator<? extends OperatorDesc>>();
-    topSelOps = new HashMap<String, Operator<? extends OperatorDesc>>();
+    // Must be deterministic order maps - see HIVE-8707
+    topOps = new LinkedHashMap<String, Operator<? extends OperatorDesc>>();
+    topSelOps = new LinkedHashMap<String, Operator<? extends OperatorDesc>>();
     loadTableWork = new ArrayList<LoadTableDesc>();
     loadFileWork = new ArrayList<LoadFileDesc>();
     opParseCtx = new LinkedHashMap<Operator<? extends OperatorDesc>, OpParseContext>();
@@ -9641,7 +9642,8 @@ public Operator genPlan(QB qb, boolean skipAmbiguityCheck)
       throws SemanticException {
 
     // First generate all the opInfos for the elements in the from clause
-    Map<String, Operator> aliasToOpInfo = new HashMap<String, Operator>();
+    // Must be deterministic order map - see HIVE-8707
+    Map<String, Operator> aliasToOpInfo = new LinkedHashMap<String, Operator>();
 
     // Recurse over the subqueries to fill the subquery part of the plan
     for (String alias : qb.getSubqAliases()) {

diff --git a/ql/src/test/results/clientnegative/bucket_mapjoin_mismatch1.q.out b/ql/src/test/results/clientnegative/bucket_mapjoin_mismatch1.q.out
@@ -103,28 +103,28 @@ STAGE PLANS:
     Map Reduce
       Map Operator Tree:
           TableScan
-            alias: b
-            Statistics: Num rows: 29 Data size: 3062 Basic stats: COMPLETE Column stats: NONE
+            alias: a
+            Statistics: Num rows: 40 Data size: 4200 Basic stats: COMPLETE Column stats: NONE
             Filter Operator
               predicate: key is not null (type: boolean)
-              Statistics: Num rows: 15 Data size: 1583 Basic stats: COMPLETE Column stats: NONE
+              Statistics: Num rows: 20 Data size: 2100 Basic stats: COMPLETE Column stats: NONE
               Reduce Output Operator
                 key expressions: key (type: int)
                 sort order: +
                 Map-reduce partition columns: key (type: int)
-                Statistics: Num rows: 15 Data size: 1583 Basic stats: COMPLETE Column stats: NONE
+                Statistics: Num rows: 20 Data size: 2100 Basic stats: COMPLETE Column stats: NONE
                 value expressions: value (type: string)
           TableScan
-            alias: a
-            Statistics: Num rows: 40 Data size: 4200 Basic stats: COMPLETE Column stats: NONE
+            alias: b
+            Statistics: Num rows: 29 Data size: 3062 Basic stats: COMPLETE Column stats: NONE
             Filter Operator
               predicate: key is not null (type: boolean)
-              Statistics: Num rows: 20 Data size: 2100 Basic stats: COMPLETE Column stats: NONE
+              Statistics: Num rows: 15 Data size: 1583 Basic stats: COMPLETE Column stats: NONE
               Reduce Output Operator
                 key expressions: key (type: int)
                 sort order: +
                 Map-reduce partition columns: key (type: int)
-                Statistics: Num rows: 20 Data size: 2100 Basic stats: COMPLETE Column stats: NONE
+                Statistics: Num rows: 15 Data size: 1583 Basic stats: COMPLETE Column stats: NONE
                 value expressions: value (type: string)
       Reduce Operator Tree:
         Join Operator

diff --git a/ql/src/test/results/clientnegative/join_nonexistent_part.q.out b/ql/src/test/results/clientnegative/join_nonexistent_part.q.out
@@ -1 +1 @@
-Authorization failed:No privilege 'Select' found for inputs { database:default, table:src, columnName:key}. Use SHOW GRANT to get more details.
+Authorization failed:No privilege 'Select' found for inputs { database:default, table:srcpart, columnName:key}. Use SHOW GRANT to get more details.
diff --git a/ql/src/test/results/clientpositive/allcolref_in_udf.q.out b/ql/src/test/results/clientpositive/allcolref_in_udf.q.out
@@ -81,18 +81,6 @@ STAGE PLANS:
   Stage: Stage-1
     Map Reduce
       Map Operator Tree:
-          TableScan
-            alias: b
-            Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
-            Filter Operator
-              predicate: UDFToDouble(key) is not null (type: boolean)
-              Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
-              Reduce Output Operator
-                key expressions: UDFToDouble(key) (type: double)
-                sort order: +
-                Map-reduce partition columns: UDFToDouble(key) (type: double)
-                Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
-                value expressions: key (type: string), value (type: string)
           TableScan
             alias: a
             Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
@@ -105,6 +93,18 @@ STAGE PLANS:
                 Map-reduce partition columns: (key + 1) (type: double)
                 Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE
                 value expressions: key (type: string), value (type: string)
+          TableScan
+            alias: b
+            Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+            Filter Operator
+              predicate: UDFToDouble(key) is not null (type: boolean)
+              Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+              Reduce Output Operator
+                key expressions: UDFToDouble(key) (type: double)
+                sort order: +
+                Map-reduce partition columns: UDFToDouble(key) (type: double)
+                Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+                value expressions: key (type: string), value (type: string)
       Reduce Operator Tree:
         Join Operator
           condition map: