Skip to content

Commit

Permalink
HIVE-8994 - Merge from trunk Nov 28 2014 (post merge fix)
Browse files Browse the repository at this point in the history 
git-svn-id: https://svn.apache.org/repos/asf/hive/branches/spark@1642395 13f79535-47bb-0310-9956-ffa450edef68
  • Loading branch information
Brock Noland committed Nov 29, 2014
1 parent ee1770f commit 35b7dfc
Show file tree
Hide file tree
Showing 292 changed files with 7,979 additions and 9,661 deletions.
3 changes: 0 additions & 3 deletions ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkPlanGenerator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -156,9 +156,6 @@ private Class getInputFormat(JobConf jobConf, MapWork mWork) throws HiveExceptio
}
String inpFormat = HiveConf.getVar(jobConf,
HiveConf.ConfVars.HIVEINPUTFORMAT);
if ((inpFormat == null) || (StringUtils.isBlank(inpFormat))) {
inpFormat = ShimLoader.getHadoopShims().getInputFormatClassName();
}

if (mWork.isUseBucketizedHiveInputFormat()) {
inpFormat = BucketizedHiveInputFormat.class.getName();
Expand Down
10 changes: 5 additions & 5 deletions ql/src/java/org/apache/hadoop/hive/ql/exec/spark/session/SparkSessionManagerImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.shims.ShimLoader;
import org.apache.hadoop.hive.shims.Utils;
import org.apache.hadoop.security.UserGroupInformation;

import java.util.Collections;
Expand Down Expand Up @@ -133,12 +134,11 @@ public SparkSession getSession(SparkSession existingSession, HiveConf conf,
*/
private boolean canReuseSession(SparkSession existingSession, HiveConf conf) throws HiveException {
try {
UserGroupInformation newUgi = ShimLoader.getHadoopShims().getUGIForConf(conf);
String newUserName = ShimLoader.getHadoopShims().getShortUserName(newUgi);
UserGroupInformation newUgi = Utils.getUGIForConf(conf);
String newUserName = newUgi.getShortUserName();

UserGroupInformation ugiInSession =
ShimLoader.getHadoopShims().getUGIForConf(existingSession.getConf());
String userNameInSession = ShimLoader.getHadoopShims().getShortUserName(ugiInSession);
UserGroupInformation ugiInSession = Utils.getUGIForConf(existingSession.getConf());
String userNameInSession = ugiInSession.getShortUserName();

return newUserName.equals(userNameInSession);
} catch(Exception ex) {
Expand Down
4 changes: 3 additions & 1 deletion ql/src/test/results/clientpositive/join_vc.q.out
View file
Original file line number Diff line number Diff line change
Expand Up @@ -268,6 +268,7 @@ POSTHOOK: Input: default@src
2632
2652
2770
2770
2792
2792
2802
Expand All @@ -280,7 +281,6 @@ POSTHOOK: Input: default@src
3060
3128
3128
1238
3138
3138
3138
Expand All @@ -289,6 +289,7 @@ POSTHOOK: Input: default@src
3170
3192
328
328
3298
3298
3368
Expand Down Expand Up @@ -349,6 +350,7 @@ POSTHOOK: Input: default@src
5572
5572
5606
5606
5616
5626
5626
Expand Down
48 changes: 20 additions & 28 deletions ql/src/test/results/clientpositive/multi_insert_mixed.q.out
View file
Original file line number Diff line number Diff line change
Expand Up @@ -112,17 +112,13 @@ STAGE PLANS:
keys: KEY._col0 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: string), _col1 (type: bigint)
outputColumnNames: _col0, _col1
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe

Stage: Stage-4
Map Reduce
Expand All @@ -131,16 +127,16 @@ STAGE PLANS:
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: bigint)
Reduce Operator Tree:
Select Operator
expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: bigint)
outputColumnNames: _col0, _col1
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
Expand Down Expand Up @@ -176,17 +172,13 @@ STAGE PLANS:
keys: KEY._col0 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: string), _col1 (type: bigint)
outputColumnNames: _col0, _col1
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe

Stage: Stage-7
Map Reduce
Expand All @@ -195,16 +187,16 @@ STAGE PLANS:
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: bigint)
Reduce Operator Tree:
Select Operator
expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: bigint)
outputColumnNames: _col0, _col1
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
Expand Down
102 changes: 49 additions & 53 deletions ql/src/test/results/clientpositive/parallel_join0.q.out
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,16 +65,12 @@ STAGE PLANS:
1 {VALUE._col0} {VALUE._col1}
outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string)
outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
File Output Operator
compressed: false
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe

Stage: Stage-2
Map Reduce
Expand Down Expand Up @@ -120,7 +116,7 @@ SELECT src1.key as k1, src1.value as v1,
(SELECT * FROM src WHERE src.key < 10) src2
SORT BY k1, v1, k2, v2
POSTHOOK: type: QUERY
{"STAGE PLANS":{"Stage-2":{"Map Reduce":{"Reduce Operator Tree:":{"Select Operator":{"expressions:":"KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey2 (type: string), KEY.reducesinkkey3 (type: string)","outputColumnNames:":["_col0","_col1","_col2","_col3"],"children":{"File Output Operator":{"Statistics:":"Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE","compressed:":"false","table:":{"serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe","input format:":"org.apache.hadoop.mapred.TextInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat"}}},"Statistics:":"Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE"}},"Map Operator Tree:":[{"TableScan":{"children":{"Reduce Output Operator":{"sort order:":"++++","Statistics:":"Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE","key expressions:":"_col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string)"}}}}]}},"Stage-1":{"Map Reduce":{"Reduce Operator Tree:":{"Join Operator":{"outputColumnNames:":["_col0","_col1","_col2","_col3"],"children":{"Select Operator":{"expressions:":"_col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string)","outputColumnNames:":["_col0","_col1","_col2","_col3"],"children":{"File Output Operator":{"compressed:":"false","table:":{"serde:":"org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe","input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat"}}},"Statistics:":"Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE"}},"Statistics:":"Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE","condition map:":[{"":"Inner Join 0 to 1"}],"condition expressions:":{"1":"{VALUE._col0} {VALUE._col1}","0":"{VALUE._col0} {VALUE._col1}"}}},"Map Operator Tree:":[{"TableScan":{"alias:":"src","children":{"Filter Operator":{"predicate:":"(key < 10) (type: boolean)","children":{"Select Operator":{"expressions:":"key (type: string), value (type: string)","outputColumnNames:":["_col0","_col1"],"children":{"Reduce Output Operator":{"sort order:":"","value expressions:":"_col0 (type: string), _col1 (type: string)","Statistics:":"Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE"}},"Statistics:":"Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE"}},"Statistics:":"Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE"}},"Statistics:":"Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE"}},{"TableScan":{"alias:":"src","children":{"Filter Operator":{"predicate:":"(key < 10) (type: boolean)","children":{"Select Operator":{"expressions:":"key (type: string), value (type: string)","outputColumnNames:":["_col0","_col1"],"children":{"Reduce Output Operator":{"sort order:":"","value expressions:":"_col0 (type: string), _col1 (type: string)","Statistics:":"Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE"}},"Statistics:":"Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE"}},"Statistics:":"Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE"}},"Statistics:":"Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE"}}]}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{}}}}},"STAGE DEPENDENCIES":{"Stage-2":{"DEPENDENT STAGES":"Stage-1"},"Stage-1":{"ROOT STAGE":"TRUE"},"Stage-0":{"DEPENDENT STAGES":"Stage-2"}}}
{"STAGE PLANS":{"Stage-2":{"Map Reduce":{"Reduce Operator Tree:":{"Select Operator":{"expressions:":"KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey2 (type: string), KEY.reducesinkkey3 (type: string)","outputColumnNames:":["_col0","_col1","_col2","_col3"],"children":{"File Output Operator":{"Statistics:":"Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE","compressed:":"false","table:":{"serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe","input format:":"org.apache.hadoop.mapred.TextInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat"}}},"Statistics:":"Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE"}},"Map Operator Tree:":[{"TableScan":{"children":{"Reduce Output Operator":{"sort order:":"++++","Statistics:":"Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE","key expressions:":"_col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string)"}}}}]}},"Stage-1":{"Map Reduce":{"Reduce Operator Tree:":{"Join Operator":{"outputColumnNames:":["_col0","_col1","_col2","_col3"],"children":{"File Output Operator":{"compressed:":"false","table:":{"serde:":"org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe","input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat"}}},"Statistics:":"Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE","condition map:":[{"":"Inner Join 0 to 1"}],"condition expressions:":{"1":"{VALUE._col0} {VALUE._col1}","0":"{VALUE._col0} {VALUE._col1}"}}},"Map Operator Tree:":[{"TableScan":{"alias:":"src","children":{"Filter Operator":{"predicate:":"(key < 10) (type: boolean)","children":{"Select Operator":{"expressions:":"key (type: string), value (type: string)","outputColumnNames:":["_col0","_col1"],"children":{"Reduce Output Operator":{"sort order:":"","value expressions:":"_col0 (type: string), _col1 (type: string)","Statistics:":"Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE"}},"Statistics:":"Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE"}},"Statistics:":"Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE"}},"Statistics:":"Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE"}},{"TableScan":{"alias:":"src","children":{"Filter Operator":{"predicate:":"(key < 10) (type: boolean)","children":{"Select Operator":{"expressions:":"key (type: string), value (type: string)","outputColumnNames:":["_col0","_col1"],"children":{"Reduce Output Operator":{"sort order:":"","value expressions:":"_col0 (type: string), _col1 (type: string)","Statistics:":"Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE"}},"Statistics:":"Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE"}},"Statistics:":"Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE"}},"Statistics:":"Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE"}}]}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{}}}}},"STAGE DEPENDENCIES":{"Stage-2":{"DEPENDENT STAGES":"Stage-1"},"Stage-1":{"ROOT STAGE":"TRUE"},"Stage-0":{"DEPENDENT STAGES":"Stage-2"}}}
Warning: Shuffle Join JOIN[8][tables = [src1, src2]] in Stage 'Stage-1:MAPRED' is a cross product
PREHOOK: query: SELECT src1.key as k1, src1.value as v1,
src2.key as k2, src2.value as v2 FROM
Expand All @@ -142,101 +138,101 @@ POSTHOOK: Input: default@src
#### A masked pattern was here ####
0 val_0 0 val_0
0 val_0 0 val_0
0 val_0 2 val_2
0 val_0 4 val_4
0 val_0 5 val_5
0 val_0 5 val_5
0 val_0 8 val_8
0 val_0 9 val_9
2 val_2 0 val_0
2 val_2 2 val_2
2 val_2 5 val_5
4 val_4 0 val_0
4 val_4 2 val_2
4 val_4 4 val_4
4 val_4 5 val_5
4 val_4 5 val_5
4 val_4 9 val_9
5 val_5 0 val_0
5 val_5 0 val_0
5 val_5 0 val_0
5 val_5 2 val_2
5 val_5 2 val_2
5 val_5 4 val_4
5 val_5 5 val_5
5 val_5 8 val_8
5 val_5 8 val_8
5 val_5 9 val_9
5 val_5 9 val_9
8 val_8 0 val_0
8 val_8 4 val_4
8 val_8 5 val_5
8 val_8 8 val_8
8 val_8 5 val_5
8 val_8 5 val_5
9 val_9 0 val_0
9 val_9 0 val_0
9 val_9 0 val_0
9 val_9 2 val_2
9 val_9 5 val_5
9 val_9 5 val_5
0 val_0 0 val_0
0 val_0 0 val_0
0 val_0 4 val_4
0 val_0 4 val_4
0 val_0 0 val_0
0 val_0 2 val_2
0 val_0 5 val_5
0 val_0 5 val_5
0 val_0 8 val_8
0 val_0 9 val_9
2 val_2 0 val_0
2 val_2 5 val_5
4 val_4 5 val_5
5 val_5 0 val_0
5 val_5 0 val_0
5 val_5 4 val_4
5 val_5 2 val_2
5 val_5 2 val_2
5 val_5 4 val_4
5 val_5 5 val_5
5 val_5 5 val_5
5 val_5 5 val_5
5 val_5 8 val_8
5 val_5 9 val_9
8 val_8 0 val_0
8 val_8 2 val_2
8 val_8 5 val_5
9 val_9 2 val_2
9 val_9 9 val_9
0 val_0 0 val_0
0 val_0 0 val_0
0 val_0 0 val_0
0 val_0 2 val_2
0 val_0 4 val_4
0 val_0 4 val_4
0 val_0 5 val_5
0 val_0 5 val_5
0 val_0 5 val_5
0 val_0 5 val_5
0 val_0 5 val_5
0 val_0 8 val_8
0 val_0 9 val_9
0 val_0 9 val_9
2 val_2 0 val_0
2 val_2 0 val_0
2 val_2 4 val_4
2 val_2 5 val_5
2 val_2 5 val_5
2 val_2 8 val_8
2 val_2 9 val_9
4 val_4 0 val_0
4 val_4 0 val_0
4 val_4 5 val_5
4 val_4 8 val_8
5 val_5 0 val_0
5 val_5 0 val_0
5 val_5 0 val_0
5 val_5 2 val_2
5 val_5 4 val_4
5 val_5 5 val_5
5 val_5 5 val_5
5 val_5 5 val_5
5 val_5 8 val_8
8 val_8 0 val_0
8 val_8 5 val_5
9 val_9 0 val_0
9 val_9 0 val_0
8 val_8 4 val_4
9 val_9 4 val_4
9 val_9 5 val_5
9 val_9 8 val_8
9 val_9 9 val_9
0 val_0 0 val_0
0 val_0 0 val_0
0 val_0 0 val_0
0 val_0 2 val_2
0 val_0 2 val_2
0 val_0 5 val_5
0 val_0 8 val_8
0 val_0 9 val_9
2 val_2 2 val_2
2 val_2 9 val_9
0 val_0 5 val_5
0 val_0 5 val_5
2 val_2 5 val_5
4 val_4 0 val_0
4 val_4 2 val_2
4 val_4 5 val_5
5 val_5 0 val_0
5 val_5 0 val_0
5 val_5 0 val_0
5 val_5 2 val_2
5 val_5 5 val_5
5 val_5 5 val_5
5 val_5 5 val_5
5 val_5 8 val_8
5 val_5 9 val_9
8 val_8 2 val_2
8 val_8 8 val_8
8 val_8 9 val_9
9 val_9 0 val_0
9 val_9 4 val_4
6 changes: 3 additions & 3 deletions ql/src/test/results/clientpositive/parallel_join1.q.out
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ STAGE PLANS:
Map Reduce
Map Operator Tree:
TableScan
alias: src2
alias: src1
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: key is not null (type: boolean)
Expand All @@ -38,9 +38,8 @@ STAGE PLANS:
sort order: +
Map-reduce partition columns: key (type: string)
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
value expressions: value (type: string)
TableScan
alias: src1
alias: src2
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: key is not null (type: boolean)
Expand All @@ -50,6 +49,7 @@ STAGE PLANS:
sort order: +
Map-reduce partition columns: key (type: string)
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
value expressions: value (type: string)
Reduce Operator Tree:
Join Operator
condition map:
Expand Down
Loading

0 comments on commit 35b7dfc

Please sign in to comment.