From fc4a31457a8c5546d6b7d0840aaed6339975b0d0 Mon Sep 17 00:00:00 2001 From: Xuefu Zhang Date: Thu, 27 Nov 2014 12:39:38 +0000 Subject: [PATCH] HIVE-8978: Fix test determinism issue for qfile: smb_mapjoin_1.q etc (Jimmy via Xuefu, reviewed by Brock) git-svn-id: https://svn.apache.org/repos/asf/hive/trunk@1642127 13f79535-47bb-0310-9956-ffa450edef68 --- .../queries/clientpositive/join_reorder.q | 2 + .../queries/clientpositive/outer_join_ppr.q | 2 + .../queries/clientpositive/smb_mapjoin_1.q | 4 +- .../queries/clientpositive/smb_mapjoin_2.q | 4 +- .../queries/clientpositive/smb_mapjoin_4.q | 4 +- .../queries/clientpositive/smb_mapjoin_5.q | 4 +- .../queries/clientpositive/smb_mapjoin_8.q | 1 + .../test/queries/clientpositive/uniquejoin.q | 2 + .../clientpositive/vector_decimal_aggregate.q | 2 + .../queries/clientpositive/vectorization_13.q | 4 + .../results/clientpositive/join_reorder.q.out | 28 ++--- .../clientpositive/outer_join_ppr.q.out | Bin 34695 -> 34741 bytes .../clientpositive/smb_mapjoin_1.q.out | 16 +-- .../clientpositive/smb_mapjoin_2.q.out | 24 +++-- .../clientpositive/smb_mapjoin_4.q.out | 48 +++++---- .../clientpositive/smb_mapjoin_5.q.out | 48 +++++---- .../clientpositive/smb_mapjoin_8.q.out | 24 +++-- .../tez/vector_decimal_aggregate.q.out | 20 ++-- .../clientpositive/tez/vectorization_13.q.out | 78 ++++++++++---- .../results/clientpositive/uniquejoin.q.out | 26 +++-- .../vector_decimal_aggregate.q.out | 52 ++++----- .../clientpositive/vectorization_13.q.out | 100 +++++++++++++----- 22 files changed, 318 insertions(+), 175 deletions(-) diff --git a/ql/src/test/queries/clientpositive/join_reorder.q b/ql/src/test/queries/clientpositive/join_reorder.q index b209c50b6619..8846113d6e08 100644 --- a/ql/src/test/queries/clientpositive/join_reorder.q +++ b/ql/src/test/queries/clientpositive/join_reorder.q @@ -10,6 +10,8 @@ LOAD DATA LOCAL INPATH '../../data/files/T1.txt' INTO TABLE T1; LOAD DATA LOCAL INPATH '../../data/files/T2.txt' INTO TABLE T2; LOAD DATA LOCAL INPATH '../../data/files/T3.txt' INTO TABLE T3; +-- SORT_QUERY_RESULTS + EXPLAIN FROM T1 a JOIN src c ON c.key+1=a.key SELECT a.key, a.val, c.key; diff --git a/ql/src/test/queries/clientpositive/outer_join_ppr.q b/ql/src/test/queries/clientpositive/outer_join_ppr.q index 38007cab5f67..c32e02305930 100644 --- a/ql/src/test/queries/clientpositive/outer_join_ppr.q +++ b/ql/src/test/queries/clientpositive/outer_join_ppr.q @@ -1,5 +1,7 @@ set hive.optimize.ppd=true; +-- SORT_QUERY_RESULTS + EXPLAIN EXTENDED FROM src a diff --git a/ql/src/test/queries/clientpositive/smb_mapjoin_1.q b/ql/src/test/queries/clientpositive/smb_mapjoin_1.q index 9dee4110f599..4f913e728dc4 100644 --- a/ql/src/test/queries/clientpositive/smb_mapjoin_1.q +++ b/ql/src/test/queries/clientpositive/smb_mapjoin_1.q @@ -13,7 +13,9 @@ load data local inpath '../../data/files/smbbucket_3.rc' overwrite into table sm set hive.optimize.bucketmapjoin = true; set hive.optimize.bucketmapjoin.sortedmerge = true; set hive.input.format = org.apache.hadoop.hive.ql.io.BucketizedHiveInputFormat; - + +-- SORT_QUERY_RESULTS + explain select /*+mapjoin(a)*/ * from smb_bucket_1 a join smb_bucket_2 b on a.key = b.key; select /*+mapjoin(a)*/ * from smb_bucket_1 a join smb_bucket_2 b on a.key = b.key; diff --git a/ql/src/test/queries/clientpositive/smb_mapjoin_2.q b/ql/src/test/queries/clientpositive/smb_mapjoin_2.q index e2b24333ad41..5e729d9fd167 100644 --- a/ql/src/test/queries/clientpositive/smb_mapjoin_2.q +++ b/ql/src/test/queries/clientpositive/smb_mapjoin_2.q @@ -13,7 +13,9 @@ load data local inpath '../../data/files/smbbucket_3.rc' overwrite into table sm set hive.optimize.bucketmapjoin = true; set hive.optimize.bucketmapjoin.sortedmerge = true; set hive.input.format = org.apache.hadoop.hive.ql.io.BucketizedHiveInputFormat; - + +-- SORT_QUERY_RESULTS + explain select /*+mapjoin(a)*/ * from smb_bucket_1 a join smb_bucket_3 b on a.key = b.key; select /*+mapjoin(a)*/ * from smb_bucket_1 a join smb_bucket_3 b on a.key = b.key; diff --git a/ql/src/test/queries/clientpositive/smb_mapjoin_4.q b/ql/src/test/queries/clientpositive/smb_mapjoin_4.q index 2b3f67ea4eea..e9d28babf98b 100644 --- a/ql/src/test/queries/clientpositive/smb_mapjoin_4.q +++ b/ql/src/test/queries/clientpositive/smb_mapjoin_4.q @@ -13,7 +13,9 @@ load data local inpath '../../data/files/smbbucket_3.rc' overwrite into table sm set hive.optimize.bucketmapjoin = true; set hive.optimize.bucketmapjoin.sortedmerge = true; set hive.input.format = org.apache.hadoop.hive.ql.io.BucketizedHiveInputFormat; - + +-- SORT_QUERY_RESULTS + explain select /*+mapjoin(a,b)*/ * from smb_bucket_1 a join smb_bucket_2 b on a.key = b.key join smb_bucket_3 c on b.key=c.key; select /*+mapjoin(a,b)*/ * from smb_bucket_1 a join smb_bucket_2 b on a.key = b.key join smb_bucket_3 c on b.key=c.key; diff --git a/ql/src/test/queries/clientpositive/smb_mapjoin_5.q b/ql/src/test/queries/clientpositive/smb_mapjoin_5.q index 406604e621ad..5b6eeb858878 100644 --- a/ql/src/test/queries/clientpositive/smb_mapjoin_5.q +++ b/ql/src/test/queries/clientpositive/smb_mapjoin_5.q @@ -13,7 +13,9 @@ load data local inpath '../../data/files/smbbucket_3.rc' overwrite into table sm set hive.optimize.bucketmapjoin = true; set hive.optimize.bucketmapjoin.sortedmerge = true; set hive.input.format = org.apache.hadoop.hive.ql.io.BucketizedHiveInputFormat; - + +-- SORT_QUERY_RESULTS + explain select /*+mapjoin(a,c)*/ * from smb_bucket_1 a join smb_bucket_2 b on a.key = b.key join smb_bucket_3 c on b.key=c.key; select /*+mapjoin(a,c)*/ * from smb_bucket_1 a join smb_bucket_2 b on a.key = b.key join smb_bucket_3 c on b.key=c.key; diff --git a/ql/src/test/queries/clientpositive/smb_mapjoin_8.q b/ql/src/test/queries/clientpositive/smb_mapjoin_8.q index f296057d43e3..4b4e16757208 100644 --- a/ql/src/test/queries/clientpositive/smb_mapjoin_8.q +++ b/ql/src/test/queries/clientpositive/smb_mapjoin_8.q @@ -3,6 +3,7 @@ set hive.enforce.bucketing = true; set hive.enforce.sorting = true; set hive.exec.reducers.max = 1; +-- SORT_QUERY_RESULTS create table smb_bucket_input (key int, value string) stored as rcfile; load data local inpath '../../data/files/smb_bucket_input.rc' into table smb_bucket_input; diff --git a/ql/src/test/queries/clientpositive/uniquejoin.q b/ql/src/test/queries/clientpositive/uniquejoin.q index 3bc8ef931bcd..5b9e94d2b64f 100644 --- a/ql/src/test/queries/clientpositive/uniquejoin.q +++ b/ql/src/test/queries/clientpositive/uniquejoin.q @@ -6,6 +6,8 @@ LOAD DATA LOCAL INPATH '../../data/files/T1.txt' INTO TABLE T1; LOAD DATA LOCAL INPATH '../../data/files/T2.txt' INTO TABLE T2; LOAD DATA LOCAL INPATH '../../data/files/T3.txt' INTO TABLE T3; +-- SORT_QUERY_RESULTS + FROM UNIQUEJOIN PRESERVE T1 a (a.key), PRESERVE T2 b (b.key), PRESERVE T3 c (c.key) SELECT a.key, b.key, c.key; diff --git a/ql/src/test/queries/clientpositive/vector_decimal_aggregate.q b/ql/src/test/queries/clientpositive/vector_decimal_aggregate.q index bab972fa210b..ee83f5bcf437 100644 --- a/ql/src/test/queries/clientpositive/vector_decimal_aggregate.q +++ b/ql/src/test/queries/clientpositive/vector_decimal_aggregate.q @@ -6,6 +6,8 @@ CREATE TABLE decimal_vgby STORED AS ORC AS SET hive.vectorized.execution.enabled=true; +-- SORT_QUERY_RESULTS + -- First only do simple aggregations that output primitives only EXPLAIN SELECT cint, COUNT(cdecimal1), MAX(cdecimal1), MIN(cdecimal1), SUM(cdecimal1), diff --git a/ql/src/test/queries/clientpositive/vectorization_13.q b/ql/src/test/queries/clientpositive/vectorization_13.q index 613f431a200c..2103015388ed 100644 --- a/ql/src/test/queries/clientpositive/vectorization_13.q +++ b/ql/src/test/queries/clientpositive/vectorization_13.q @@ -31,6 +31,7 @@ WHERE (((cfloat < 3569) AND ((ctimestamp2 != 12) AND (ctinyint < 9763215.5639)))) GROUP BY cboolean1, ctinyint, ctimestamp1, cfloat, cstring1 +ORDER BY cboolean1, ctinyint, ctimestamp1, cfloat, cstring1 LIMIT 40; SELECT cboolean1, @@ -62,6 +63,7 @@ WHERE (((cfloat < 3569) AND ((ctimestamp2 != 12) AND (ctinyint < 9763215.5639)))) GROUP BY cboolean1, ctinyint, ctimestamp1, cfloat, cstring1 +ORDER BY cboolean1, ctinyint, ctimestamp1, cfloat, cstring1 LIMIT 40; -- double compare timestamp @@ -95,6 +97,7 @@ WHERE (((cfloat < 3569) AND ((ctimestamp2 != -1.3359999999999999) AND (ctinyint < 9763215.5639)))) GROUP BY cboolean1, ctinyint, ctimestamp1, cfloat, cstring1 +ORDER BY cboolean1, ctinyint, ctimestamp1, cfloat, cstring1 LIMIT 40; SELECT cboolean1, @@ -126,5 +129,6 @@ WHERE (((cfloat < 3569) AND ((ctimestamp2 != -1.3359999999999999) AND (ctinyint < 9763215.5639)))) GROUP BY cboolean1, ctinyint, ctimestamp1, cfloat, cstring1 +ORDER BY cboolean1, ctinyint, ctimestamp1, cfloat, cstring1 LIMIT 40; diff --git a/ql/src/test/results/clientpositive/join_reorder.q.out b/ql/src/test/results/clientpositive/join_reorder.q.out index 6c6843ac1a39..713ae740b851 100644 --- a/ql/src/test/results/clientpositive/join_reorder.q.out +++ b/ql/src/test/results/clientpositive/join_reorder.q.out @@ -46,10 +46,14 @@ POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/T3.txt' INTO TABLE T3 POSTHOOK: type: LOAD #### A masked pattern was here #### POSTHOOK: Output: default@t3 -PREHOOK: query: EXPLAIN FROM T1 a JOIN src c ON c.key+1=a.key +PREHOOK: query: -- SORT_QUERY_RESULTS + +EXPLAIN FROM T1 a JOIN src c ON c.key+1=a.key SELECT a.key, a.val, c.key PREHOOK: type: QUERY -POSTHOOK: query: EXPLAIN FROM T1 a JOIN src c ON c.key+1=a.key +POSTHOOK: query: -- SORT_QUERY_RESULTS + +EXPLAIN FROM T1 a JOIN src c ON c.key+1=a.key SELECT a.key, a.val, c.key POSTHOOK: type: QUERY STAGE DEPENDENCIES: @@ -421,9 +425,9 @@ POSTHOOK: Input: default@t2 POSTHOOK: Input: default@t3 #### A masked pattern was here #### 2 2 12 12 +7 NULL 17 17 NULL NULL NULL 14 NULL NULL NULL 16 -7 NULL 17 17 PREHOOK: query: FROM T1 a LEFT OUTER JOIN T2 b ON (b.key=a.key) RIGHT OUTER JOIN T3 c ON (c.val = a.val) @@ -443,9 +447,9 @@ POSTHOOK: Input: default@t2 POSTHOOK: Input: default@t3 #### A masked pattern was here #### 2 2 12 12 +7 NULL 17 17 NULL NULL NULL 14 NULL NULL NULL 16 -7 NULL 17 17 PREHOOK: query: EXPLAIN FROM UNIQUEJOIN PRESERVE T1 a (a.key, a.val), PRESERVE T2 b (b.key, b.val), @@ -616,15 +620,15 @@ POSTHOOK: Input: default@t3 #### A masked pattern was here #### 1 NULL NULL 2 NULL 2 -NULL 2 NULL 3 3 NULL -NULL 4 4 -NULL 5 NULL -NULL NULL 6 7 NULL 7 8 8 NULL 8 8 NULL 8 NULL NULL +NULL 2 NULL +NULL 4 4 +NULL 5 NULL +NULL NULL 6 PREHOOK: query: FROM UNIQUEJOIN PRESERVE T1 a (a.key, a.val), PRESERVE T2 b (b.key, b.val), @@ -647,12 +651,12 @@ POSTHOOK: Input: default@t3 #### A masked pattern was here #### 1 NULL NULL 2 NULL 2 -NULL 2 NULL 3 3 NULL -NULL 4 4 -NULL 5 NULL -NULL NULL 6 7 NULL 7 8 8 NULL 8 8 NULL 8 NULL NULL +NULL 2 NULL +NULL 4 4 +NULL 5 NULL +NULL NULL 6 diff --git a/ql/src/test/results/clientpositive/outer_join_ppr.q.out b/ql/src/test/results/clientpositive/outer_join_ppr.q.out index efc8b0190ec6e017f5118c9bb964cd8924269a9c..93cfde24eecb2f37ca9aaf75c266ba84f5460030 100644 GIT binary patch delta 81 zcmZqgXWH7&q#h9D>f!J2ZKY6Hnp#w8rJ$>;5bPfm5+4}q8Wb5H 1 PREHOOK: type: QUERY -POSTHOOK: query: -- First only do simple aggregations that output primitives only +POSTHOOK: query: -- SORT_QUERY_RESULTS + +-- First only do simple aggregations that output primitives only EXPLAIN SELECT cint, COUNT(cdecimal1), MAX(cdecimal1), MIN(cdecimal1), SUM(cdecimal1), COUNT(cdecimal2), MAX(cdecimal2), MIN(cdecimal2), SUM(cdecimal2) @@ -113,14 +117,14 @@ POSTHOOK: query: SELECT cint, POSTHOOK: type: QUERY POSTHOOK: Input: default@decimal_vgby #### A masked pattern was here #### -NULL 3072 9318.4351351351 -4298.1513513514 5018444.1081079808 3072 11160.715384615385 -5147.907692307693 6010604.3076923073536 -3728 6 5831542.269248378 -3367.6517567568 5817556.0411483778 6 6984454.211097692 -4033.445769230769 6967702.8672438458471 -563 2 -515.621072973 -3367.6517567568 -3883.2728297298 2 -617.5607769230769 -4033.445769230769 -4651.0065461538459 -762 2 5831542.269248378 1531.2194054054 5833073.4886537834 2 6984454.211097692 1833.9456923076925 6986288.1567899996925 -6981 3 5831542.269248378 -515.621072973 5830511.027102432 3 6984454.211097692 -617.5607769230769 6983219.0895438458462 253665376 1024 9767.0054054054 -9779.5486486487 -347484.0818378374 1024 11697.969230769231 -11712.99230769231 -416182.64030769233089 528534767 1024 5831542.269248378 -9777.1594594595 11646372.8607481068 1024 6984454.211097692 -11710.130769230771 13948892.79980307629003 626923679 1024 9723.4027027027 -9778.9513513514 10541.0525297287 1024 11645.746153846154 -11712.276923076923 12625.04759999997746 +6981 3 5831542.269248378 -515.621072973 5830511.027102432 3 6984454.211097692 -617.5607769230769 6983219.0895438458462 +762 2 5831542.269248378 1531.2194054054 5833073.4886537834 2 6984454.211097692 1833.9456923076925 6986288.1567899996925 +NULL 3072 9318.4351351351 -4298.1513513514 5018444.1081079808 3072 11160.715384615385 -5147.907692307693 6010604.3076923073536 PREHOOK: query: -- Now add the others... EXPLAIN SELECT cint, COUNT(cdecimal1), MAX(cdecimal1), MIN(cdecimal1), SUM(cdecimal1), AVG(cdecimal1), STDDEV_POP(cdecimal1), STDDEV_SAMP(cdecimal1), @@ -217,11 +221,11 @@ POSTHOOK: query: SELECT cint, POSTHOOK: type: QUERY POSTHOOK: Input: default@decimal_vgby #### A masked pattern was here #### -NULL 3072 9318.4351351351 -4298.1513513514 5018444.1081079808 1633.60810810806667 5695.483082135364 5696.4103077145055 3072 11160.715384615385 -5147.907692307693 6010604.3076923073536 1956.576923076922966667 6821.495748565159 6822.606289190924 -3728 6 5831542.269248378 -3367.6517567568 5817556.0411483778 969592.67352472963333 2174330.2092403853 2381859.406131774 6 6984454.211097692 -4033.445769230769 6967702.8672438458471 1161283.811207307641183333 2604201.2704476737 2852759.5602156054 -563 2 -515.621072973 -3367.6517567568 -3883.2728297298 -1941.6364148649 1426.0153418918999 2016.6902366556308 2 -617.5607769230769 -4033.445769230769 -4651.0065461538459 -2325.50327307692295 1707.9424961538462 2415.395441814127 -762 2 5831542.269248378 1531.2194054054 5833073.4886537834 2916536.7443268917 2915005.5249214866 4122440.3477364695 2 6984454.211097692 1833.9456923076925 6986288.1567899996925 3493144.07839499984625 3491310.1327026924 4937458.140118758 -6981 3 5831542.269248378 -515.621072973 5830511.027102432 1943503.67570081066667 2749258.455012492 3367140.1929065133 3 6984454.211097692 -617.5607769230769 6983219.0895438458462 2327739.696514615282066667 3292794.4113115156 4032833.0678006653 253665376 1024 9767.0054054054 -9779.5486486487 -347484.0818378374 -339.33992366976309 5708.9563478862 5711.745967572779 1024 11697.969230769231 -11712.99230769231 -416182.64030769233089 -406.428359675480791885 6837.632716002934 6840.973851172274 528534767 1024 5831542.269248378 -9777.1594594595 11646372.8607481068 11373.41099682432305 257528.92988206653 257654.7686043977 1024 6984454.211097692 -11710.130769230771 13948892.79980307629003 13621.965624807691689482 308443.1074570801 308593.82484083984 626923679 1024 9723.4027027027 -9778.9513513514 10541.0525297287 10.29399661106318 5742.09145323734 5744.897264034267 1024 11645.746153846154 -11712.276923076923 12625.04759999997746 12.329148046874977988 6877.318722794877 6880.679250101603 +6981 3 5831542.269248378 -515.621072973 5830511.027102432 1943503.67570081066667 2749258.455012492 3367140.1929065133 3 6984454.211097692 -617.5607769230769 6983219.0895438458462 2327739.696514615282066667 3292794.4113115156 4032833.0678006653 +762 2 5831542.269248378 1531.2194054054 5833073.4886537834 2916536.7443268917 2915005.5249214866 4122440.3477364695 2 6984454.211097692 1833.9456923076925 6986288.1567899996925 3493144.07839499984625 3491310.1327026924 4937458.140118758 +NULL 3072 9318.4351351351 -4298.1513513514 5018444.1081079808 1633.60810810806667 5695.483082135364 5696.4103077145055 3072 11160.715384615385 -5147.907692307693 6010604.3076923073536 1956.576923076922966667 6821.495748565159 6822.606289190924 diff --git a/ql/src/test/results/clientpositive/tez/vectorization_13.q.out b/ql/src/test/results/clientpositive/tez/vectorization_13.q.out index 3410c338c882..6f062dbfe04d 100644 --- a/ql/src/test/results/clientpositive/tez/vectorization_13.q.out +++ b/ql/src/test/results/clientpositive/tez/vectorization_13.q.out @@ -28,6 +28,7 @@ WHERE (((cfloat < 3569) AND ((ctimestamp2 != 12) AND (ctinyint < 9763215.5639)))) GROUP BY cboolean1, ctinyint, ctimestamp1, cfloat, cstring1 +ORDER BY cboolean1, ctinyint, ctimestamp1, cfloat, cstring1 LIMIT 40 PREHOOK: type: QUERY POSTHOOK: query: EXPLAIN @@ -60,6 +61,7 @@ WHERE (((cfloat < 3569) AND ((ctimestamp2 != 12) AND (ctinyint < 9763215.5639)))) GROUP BY cboolean1, ctinyint, ctimestamp1, cfloat, cstring1 +ORDER BY cboolean1, ctinyint, ctimestamp1, cfloat, cstring1 LIMIT 40 POSTHOOK: type: QUERY STAGE DEPENDENCIES: @@ -71,6 +73,7 @@ STAGE PLANS: Tez Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -107,19 +110,31 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10 Statistics: Num rows: 1365 Data size: 41904 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: boolean), _col1 (type: tinyint), _col2 (type: timestamp), _col3 (type: float), _col4 (type: string), (- _col1) (type: tinyint), _col5 (type: tinyint), ((- _col1) + _col5) (type: tinyint), _col6 (type: double), (_col6 * ((- _col1) + _col5)) (type: double), (- _col6) (type: double), (79.553 * _col3) (type: double), _col7 (type: double), (- _col6) (type: double), _col8 (type: double), (((- _col1) + _col5) - 10.175) (type: double), (- (- _col6)) (type: double), (-26.28 / (- (- _col6))) (type: double), _col9 (type: float), ((_col6 * ((- _col1) + _col5)) / _col1) (type: double), _col10 (type: tinyint) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20 + expressions: _col0 (type: boolean), _col1 (type: tinyint), (- _col6) (type: double), (79.553 * _col3) (type: double), _col7 (type: double), _col8 (type: double), (((- _col1) + _col5) - 10.175) (type: double), (- (- _col6)) (type: double), (-26.28 / (- (- _col6))) (type: double), _col9 (type: float), ((_col6 * ((- _col1) + _col5)) / _col1) (type: double), _col2 (type: timestamp), _col10 (type: tinyint), _col3 (type: float), _col4 (type: string), (- _col1) (type: tinyint), _col5 (type: tinyint), ((- _col1) + _col5) (type: tinyint), _col6 (type: double), (_col6 * ((- _col1) + _col5)) (type: double) + outputColumnNames: _col0, _col1, _col10, _col11, _col12, _col14, _col15, _col16, _col17, _col18, _col19, _col2, _col20, _col3, _col4, _col5, _col6, _col7, _col8, _col9 Statistics: Num rows: 1365 Data size: 41904 Basic stats: COMPLETE Column stats: NONE - Limit - Number of rows: 40 + Reduce Output Operator + key expressions: _col0 (type: boolean), _col1 (type: tinyint), _col2 (type: timestamp), _col3 (type: float), _col4 (type: string) + sort order: +++++ + Statistics: Num rows: 1365 Data size: 41904 Basic stats: COMPLETE Column stats: NONE + value expressions: _col5 (type: tinyint), _col6 (type: tinyint), _col7 (type: tinyint), _col8 (type: double), _col9 (type: double), _col10 (type: double), _col11 (type: double), _col12 (type: double), _col14 (type: double), _col15 (type: double), _col16 (type: double), _col17 (type: double), _col18 (type: float), _col19 (type: double), _col20 (type: tinyint) + Reducer 3 + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: boolean), KEY.reducesinkkey1 (type: tinyint), KEY.reducesinkkey2 (type: timestamp), KEY.reducesinkkey3 (type: float), KEY.reducesinkkey4 (type: string), VALUE._col0 (type: tinyint), VALUE._col1 (type: tinyint), VALUE._col2 (type: tinyint), VALUE._col3 (type: double), VALUE._col4 (type: double), VALUE._col5 (type: double), VALUE._col6 (type: double), VALUE._col7 (type: double), VALUE._col5 (type: double), VALUE._col8 (type: double), VALUE._col9 (type: double), VALUE._col10 (type: double), VALUE._col11 (type: double), VALUE._col12 (type: float), VALUE._col13 (type: double), VALUE._col14 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20 + Statistics: Num rows: 1365 Data size: 41904 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 40 + Statistics: Num rows: 40 Data size: 1200 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false Statistics: Num rows: 40 Data size: 1200 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 40 Data size: 1200 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized Stage: Stage-0 Fetch Operator @@ -156,6 +171,7 @@ WHERE (((cfloat < 3569) AND ((ctimestamp2 != 12) AND (ctinyint < 9763215.5639)))) GROUP BY cboolean1, ctinyint, ctimestamp1, cfloat, cstring1 +ORDER BY cboolean1, ctinyint, ctimestamp1, cfloat, cstring1 LIMIT 40 PREHOOK: type: QUERY PREHOOK: Input: default@alltypesorc @@ -189,6 +205,7 @@ WHERE (((cfloat < 3569) AND ((ctimestamp2 != 12) AND (ctinyint < 9763215.5639)))) GROUP BY cboolean1, ctinyint, ctimestamp1, cfloat, cstring1 +ORDER BY cboolean1, ctinyint, ctimestamp1, cfloat, cstring1 LIMIT 40 POSTHOOK: type: QUERY POSTHOOK: Input: default@alltypesorc @@ -264,6 +281,7 @@ WHERE (((cfloat < 3569) AND ((ctimestamp2 != -1.3359999999999999) AND (ctinyint < 9763215.5639)))) GROUP BY cboolean1, ctinyint, ctimestamp1, cfloat, cstring1 +ORDER BY cboolean1, ctinyint, ctimestamp1, cfloat, cstring1 LIMIT 40 PREHOOK: type: QUERY POSTHOOK: query: -- double compare timestamp @@ -297,6 +315,7 @@ WHERE (((cfloat < 3569) AND ((ctimestamp2 != -1.3359999999999999) AND (ctinyint < 9763215.5639)))) GROUP BY cboolean1, ctinyint, ctimestamp1, cfloat, cstring1 +ORDER BY cboolean1, ctinyint, ctimestamp1, cfloat, cstring1 LIMIT 40 POSTHOOK: type: QUERY STAGE DEPENDENCIES: @@ -308,6 +327,7 @@ STAGE PLANS: Tez Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -344,19 +364,31 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10 Statistics: Num rows: 1365 Data size: 41904 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: boolean), _col1 (type: tinyint), _col2 (type: timestamp), _col3 (type: float), _col4 (type: string), (- _col1) (type: tinyint), _col5 (type: tinyint), ((- _col1) + _col5) (type: tinyint), _col6 (type: double), (_col6 * ((- _col1) + _col5)) (type: double), (- _col6) (type: double), (79.553 * _col3) (type: double), _col7 (type: double), (- _col6) (type: double), _col8 (type: double), (((- _col1) + _col5) - 10.175) (type: double), (- (- _col6)) (type: double), (-26.28 / (- (- _col6))) (type: double), _col9 (type: float), ((_col6 * ((- _col1) + _col5)) / _col1) (type: double), _col10 (type: tinyint) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20 + expressions: _col0 (type: boolean), _col1 (type: tinyint), (- _col6) (type: double), (79.553 * _col3) (type: double), _col7 (type: double), _col8 (type: double), (((- _col1) + _col5) - 10.175) (type: double), (- (- _col6)) (type: double), (-26.28 / (- (- _col6))) (type: double), _col9 (type: float), ((_col6 * ((- _col1) + _col5)) / _col1) (type: double), _col2 (type: timestamp), _col10 (type: tinyint), _col3 (type: float), _col4 (type: string), (- _col1) (type: tinyint), _col5 (type: tinyint), ((- _col1) + _col5) (type: tinyint), _col6 (type: double), (_col6 * ((- _col1) + _col5)) (type: double) + outputColumnNames: _col0, _col1, _col10, _col11, _col12, _col14, _col15, _col16, _col17, _col18, _col19, _col2, _col20, _col3, _col4, _col5, _col6, _col7, _col8, _col9 Statistics: Num rows: 1365 Data size: 41904 Basic stats: COMPLETE Column stats: NONE - Limit - Number of rows: 40 + Reduce Output Operator + key expressions: _col0 (type: boolean), _col1 (type: tinyint), _col2 (type: timestamp), _col3 (type: float), _col4 (type: string) + sort order: +++++ + Statistics: Num rows: 1365 Data size: 41904 Basic stats: COMPLETE Column stats: NONE + value expressions: _col5 (type: tinyint), _col6 (type: tinyint), _col7 (type: tinyint), _col8 (type: double), _col9 (type: double), _col10 (type: double), _col11 (type: double), _col12 (type: double), _col14 (type: double), _col15 (type: double), _col16 (type: double), _col17 (type: double), _col18 (type: float), _col19 (type: double), _col20 (type: tinyint) + Reducer 3 + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: boolean), KEY.reducesinkkey1 (type: tinyint), KEY.reducesinkkey2 (type: timestamp), KEY.reducesinkkey3 (type: float), KEY.reducesinkkey4 (type: string), VALUE._col0 (type: tinyint), VALUE._col1 (type: tinyint), VALUE._col2 (type: tinyint), VALUE._col3 (type: double), VALUE._col4 (type: double), VALUE._col5 (type: double), VALUE._col6 (type: double), VALUE._col7 (type: double), VALUE._col5 (type: double), VALUE._col8 (type: double), VALUE._col9 (type: double), VALUE._col10 (type: double), VALUE._col11 (type: double), VALUE._col12 (type: float), VALUE._col13 (type: double), VALUE._col14 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20 + Statistics: Num rows: 1365 Data size: 41904 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 40 + Statistics: Num rows: 40 Data size: 1200 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false Statistics: Num rows: 40 Data size: 1200 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 40 Data size: 1200 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized Stage: Stage-0 Fetch Operator @@ -393,6 +425,7 @@ WHERE (((cfloat < 3569) AND ((ctimestamp2 != -1.3359999999999999) AND (ctinyint < 9763215.5639)))) GROUP BY cboolean1, ctinyint, ctimestamp1, cfloat, cstring1 +ORDER BY cboolean1, ctinyint, ctimestamp1, cfloat, cstring1 LIMIT 40 PREHOOK: type: QUERY PREHOOK: Input: default@alltypesorc @@ -426,6 +459,7 @@ WHERE (((cfloat < 3569) AND ((ctimestamp2 != -1.3359999999999999) AND (ctinyint < 9763215.5639)))) GROUP BY cboolean1, ctinyint, ctimestamp1, cfloat, cstring1 +ORDER BY cboolean1, ctinyint, ctimestamp1, cfloat, cstring1 LIMIT 40 POSTHOOK: type: QUERY POSTHOOK: Input: default@alltypesorc diff --git a/ql/src/test/results/clientpositive/uniquejoin.q.out b/ql/src/test/results/clientpositive/uniquejoin.q.out index b6d355339575..b71d5b11dce7 100644 --- a/ql/src/test/results/clientpositive/uniquejoin.q.out +++ b/ql/src/test/results/clientpositive/uniquejoin.q.out @@ -46,14 +46,18 @@ POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/T3.txt' INTO TABLE T3 POSTHOOK: type: LOAD #### A masked pattern was here #### POSTHOOK: Output: default@t3 -PREHOOK: query: FROM UNIQUEJOIN PRESERVE T1 a (a.key), PRESERVE T2 b (b.key), PRESERVE T3 c (c.key) +PREHOOK: query: -- SORT_QUERY_RESULTS + +FROM UNIQUEJOIN PRESERVE T1 a (a.key), PRESERVE T2 b (b.key), PRESERVE T3 c (c.key) SELECT a.key, b.key, c.key PREHOOK: type: QUERY PREHOOK: Input: default@t1 PREHOOK: Input: default@t2 PREHOOK: Input: default@t3 #### A masked pattern was here #### -POSTHOOK: query: FROM UNIQUEJOIN PRESERVE T1 a (a.key), PRESERVE T2 b (b.key), PRESERVE T3 c (c.key) +POSTHOOK: query: -- SORT_QUERY_RESULTS + +FROM UNIQUEJOIN PRESERVE T1 a (a.key), PRESERVE T2 b (b.key), PRESERVE T3 c (c.key) SELECT a.key, b.key, c.key POSTHOOK: type: QUERY POSTHOOK: Input: default@t1 @@ -63,14 +67,14 @@ POSTHOOK: Input: default@t3 1 NULL NULL 2 2 2 3 3 NULL -NULL 4 4 -NULL 5 NULL -NULL NULL 6 7 NULL 7 8 8 NULL 8 8 NULL 8 8 NULL 8 8 NULL +NULL 4 4 +NULL 5 NULL +NULL NULL 6 PREHOOK: query: FROM UNIQUEJOIN T1 a (a.key), T2 b (b.key), T3 c (c.key) SELECT a.key, b.key, c.key PREHOOK: type: QUERY @@ -119,15 +123,15 @@ POSTHOOK: Input: default@t3 #### A masked pattern was here #### 1 11 NULL NULL NULL NULL 2 12 NULL NULL 2 12 -NULL NULL 2 22 NULL NULL 3 13 3 13 NULL NULL -NULL NULL 4 14 4 14 -NULL NULL 5 15 NULL NULL -NULL NULL NULL NULL 6 16 7 17 NULL NULL 7 17 8 18 8 18 NULL NULL 8 18 8 18 NULL NULL 8 28 NULL NULL NULL NULL +NULL NULL 2 22 NULL NULL +NULL NULL 4 14 4 14 +NULL NULL 5 15 NULL NULL +NULL NULL NULL NULL 6 16 PREHOOK: query: FROM UNIQUEJOIN PRESERVE T1 a (a.key), T2 b (b.key), PRESERVE T3 c (c.key) SELECT a.key, b.key, c.key PREHOOK: type: QUERY @@ -145,13 +149,13 @@ POSTHOOK: Input: default@t3 1 NULL NULL 2 2 2 3 3 NULL -NULL 4 4 -NULL NULL 6 7 NULL 7 8 8 NULL 8 8 NULL 8 8 NULL 8 8 NULL +NULL 4 4 +NULL NULL 6 PREHOOK: query: FROM UNIQUEJOIN PRESERVE T1 a (a.key), T2 b(b.key) SELECT a.key, b.key PREHOOK: type: QUERY diff --git a/ql/src/test/results/clientpositive/vector_decimal_aggregate.q.out b/ql/src/test/results/clientpositive/vector_decimal_aggregate.q.out index 586f29b2b026..787d5dd52ad8 100644 --- a/ql/src/test/results/clientpositive/vector_decimal_aggregate.q.out +++ b/ql/src/test/results/clientpositive/vector_decimal_aggregate.q.out @@ -16,7 +16,9 @@ POSTHOOK: type: CREATETABLE_AS_SELECT POSTHOOK: Input: default@alltypesorc POSTHOOK: Output: database:default POSTHOOK: Output: default@decimal_vgby -PREHOOK: query: -- First only do simple aggregations that output primitives only +PREHOOK: query: -- SORT_QUERY_RESULTS + +-- First only do simple aggregations that output primitives only EXPLAIN SELECT cint, COUNT(cdecimal1), MAX(cdecimal1), MIN(cdecimal1), SUM(cdecimal1), COUNT(cdecimal2), MAX(cdecimal2), MIN(cdecimal2), SUM(cdecimal2) @@ -24,7 +26,9 @@ EXPLAIN SELECT cint, GROUP BY cint HAVING COUNT(*) > 1 PREHOOK: type: QUERY -POSTHOOK: query: -- First only do simple aggregations that output primitives only +POSTHOOK: query: -- SORT_QUERY_RESULTS + +-- First only do simple aggregations that output primitives only EXPLAIN SELECT cint, COUNT(cdecimal1), MAX(cdecimal1), MIN(cdecimal1), SUM(cdecimal1), COUNT(cdecimal2), MAX(cdecimal2), MIN(cdecimal2), SUM(cdecimal2) @@ -42,22 +46,22 @@ STAGE PLANS: Map Operator Tree: TableScan alias: decimal_vgby - Statistics: Num rows: 12288 Data size: 2165060 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 561 Data size: 128041 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: cint (type: int), cdecimal1 (type: decimal(20,10)), cdecimal2 (type: decimal(23,14)) outputColumnNames: cint, cdecimal1, cdecimal2 - Statistics: Num rows: 12288 Data size: 2165060 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 561 Data size: 128041 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(cdecimal1), max(cdecimal1), min(cdecimal1), sum(cdecimal1), count(cdecimal2), max(cdecimal2), min(cdecimal2), sum(cdecimal2), count() keys: cint (type: int) mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9 - Statistics: Num rows: 12288 Data size: 2165060 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 561 Data size: 128041 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 12288 Data size: 2165060 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 561 Data size: 128041 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint), _col2 (type: decimal(20,10)), _col3 (type: decimal(20,10)), _col4 (type: decimal(30,10)), _col5 (type: bigint), _col6 (type: decimal(23,14)), _col7 (type: decimal(23,14)), _col8 (type: decimal(33,14)), _col9 (type: bigint) Execution mode: vectorized Reduce Operator Tree: @@ -66,17 +70,17 @@ STAGE PLANS: keys: KEY._col0 (type: int) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9 - Statistics: Num rows: 6144 Data size: 1082530 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 280 Data size: 63906 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (_col9 > 1) (type: boolean) - Statistics: Num rows: 2048 Data size: 360843 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 93 Data size: 21225 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: decimal(20,10)), _col3 (type: decimal(20,10)), _col4 (type: decimal(30,10)), _col5 (type: bigint), _col6 (type: decimal(23,14)), _col7 (type: decimal(23,14)), _col8 (type: decimal(33,14)) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 - Statistics: Num rows: 2048 Data size: 360843 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 93 Data size: 21225 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 2048 Data size: 360843 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 93 Data size: 21225 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -106,14 +110,14 @@ POSTHOOK: query: SELECT cint, POSTHOOK: type: QUERY POSTHOOK: Input: default@decimal_vgby #### A masked pattern was here #### -NULL 3072 9318.4351351351 -4298.1513513514 5018444.1081079808 3072 11160.715384615385 -5147.907692307693 6010604.3076923073536 -3728 6 5831542.269248378 -3367.6517567568 5817556.0411483778 6 6984454.211097692 -4033.445769230769 6967702.8672438458471 -563 2 -515.621072973 -3367.6517567568 -3883.2728297298 2 -617.5607769230769 -4033.445769230769 -4651.0065461538459 -762 2 5831542.269248378 1531.2194054054 5833073.4886537834 2 6984454.211097692 1833.9456923076925 6986288.1567899996925 -6981 3 5831542.269248378 -515.621072973 5830511.027102432 3 6984454.211097692 -617.5607769230769 6983219.0895438458462 253665376 1024 9767.0054054054 -9779.5486486487 -347484.0818378374 1024 11697.969230769231 -11712.99230769231 -416182.64030769233089 528534767 1024 5831542.269248378 -9777.1594594595 11646372.8607481068 1024 6984454.211097692 -11710.130769230771 13948892.79980307629003 626923679 1024 9723.4027027027 -9778.9513513514 10541.0525297287 1024 11645.746153846154 -11712.276923076923 12625.04759999997746 +6981 3 5831542.269248378 -515.621072973 5830511.027102432 3 6984454.211097692 -617.5607769230769 6983219.0895438458462 +762 2 5831542.269248378 1531.2194054054 5833073.4886537834 2 6984454.211097692 1833.9456923076925 6986288.1567899996925 +NULL 3072 9318.4351351351 -4298.1513513514 5018444.1081079808 3072 11160.715384615385 -5147.907692307693 6010604.3076923073536 PREHOOK: query: -- Now add the others... EXPLAIN SELECT cint, COUNT(cdecimal1), MAX(cdecimal1), MIN(cdecimal1), SUM(cdecimal1), AVG(cdecimal1), STDDEV_POP(cdecimal1), STDDEV_SAMP(cdecimal1), @@ -140,22 +144,22 @@ STAGE PLANS: Map Operator Tree: TableScan alias: decimal_vgby - Statistics: Num rows: 12288 Data size: 2165060 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 561 Data size: 128041 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: cint (type: int), cdecimal1 (type: decimal(20,10)), cdecimal2 (type: decimal(23,14)) outputColumnNames: cint, cdecimal1, cdecimal2 - Statistics: Num rows: 12288 Data size: 2165060 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 561 Data size: 128041 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(cdecimal1), max(cdecimal1), min(cdecimal1), sum(cdecimal1), avg(cdecimal1), stddev_pop(cdecimal1), stddev_samp(cdecimal1), count(cdecimal2), max(cdecimal2), min(cdecimal2), sum(cdecimal2), avg(cdecimal2), stddev_pop(cdecimal2), stddev_samp(cdecimal2), count() keys: cint (type: int) mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15 - Statistics: Num rows: 12288 Data size: 2165060 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 561 Data size: 128041 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 12288 Data size: 2165060 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 561 Data size: 128041 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint), _col2 (type: decimal(20,10)), _col3 (type: decimal(20,10)), _col4 (type: decimal(30,10)), _col5 (type: struct), _col6 (type: struct), _col7 (type: struct), _col8 (type: bigint), _col9 (type: decimal(23,14)), _col10 (type: decimal(23,14)), _col11 (type: decimal(33,14)), _col12 (type: struct), _col13 (type: struct), _col14 (type: struct), _col15 (type: bigint) Execution mode: vectorized Reduce Operator Tree: @@ -164,17 +168,17 @@ STAGE PLANS: keys: KEY._col0 (type: int) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15 - Statistics: Num rows: 6144 Data size: 1082530 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 280 Data size: 63906 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (_col15 > 1) (type: boolean) - Statistics: Num rows: 2048 Data size: 360843 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 93 Data size: 21225 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: decimal(20,10)), _col3 (type: decimal(20,10)), _col4 (type: decimal(30,10)), _col5 (type: decimal(24,14)), _col6 (type: double), _col7 (type: double), _col8 (type: bigint), _col9 (type: decimal(23,14)), _col10 (type: decimal(23,14)), _col11 (type: decimal(33,14)), _col12 (type: decimal(27,18)), _col13 (type: double), _col14 (type: double) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14 - Statistics: Num rows: 2048 Data size: 360843 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 93 Data size: 21225 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 2048 Data size: 360843 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 93 Data size: 21225 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -204,11 +208,11 @@ POSTHOOK: query: SELECT cint, POSTHOOK: type: QUERY POSTHOOK: Input: default@decimal_vgby #### A masked pattern was here #### -NULL 3072 9318.4351351351 -4298.1513513514 5018444.1081079808 1633.60810810806667 5695.483082135364 5696.4103077145055 3072 11160.715384615385 -5147.907692307693 6010604.3076923073536 1956.576923076922966667 6821.495748565159 6822.606289190924 -3728 6 5831542.269248378 -3367.6517567568 5817556.0411483778 969592.67352472963333 2174330.2092403853 2381859.406131774 6 6984454.211097692 -4033.445769230769 6967702.8672438458471 1161283.811207307641183333 2604201.2704476737 2852759.5602156054 -563 2 -515.621072973 -3367.6517567568 -3883.2728297298 -1941.6364148649 1426.0153418918999 2016.6902366556308 2 -617.5607769230769 -4033.445769230769 -4651.0065461538459 -2325.50327307692295 1707.9424961538462 2415.395441814127 -762 2 5831542.269248378 1531.2194054054 5833073.4886537834 2916536.7443268917 2915005.5249214866 4122440.3477364695 2 6984454.211097692 1833.9456923076925 6986288.1567899996925 3493144.07839499984625 3491310.1327026924 4937458.140118758 -6981 3 5831542.269248378 -515.621072973 5830511.027102432 1943503.67570081066667 2749258.455012492 3367140.1929065133 3 6984454.211097692 -617.5607769230769 6983219.0895438458462 2327739.696514615282066667 3292794.4113115156 4032833.0678006653 253665376 1024 9767.0054054054 -9779.5486486487 -347484.0818378374 -339.33992366976309 5708.9563478862 5711.745967572779 1024 11697.969230769231 -11712.99230769231 -416182.64030769233089 -406.428359675480791885 6837.632716002934 6840.973851172274 528534767 1024 5831542.269248378 -9777.1594594595 11646372.8607481068 11373.41099682432305 257528.92988206653 257654.7686043977 1024 6984454.211097692 -11710.130769230771 13948892.79980307629003 13621.965624807691689482 308443.1074570801 308593.82484083984 626923679 1024 9723.4027027027 -9778.9513513514 10541.0525297287 10.29399661106318 5742.09145323734 5744.897264034267 1024 11645.746153846154 -11712.276923076923 12625.04759999997746 12.329148046874977988 6877.318722794877 6880.679250101603 +6981 3 5831542.269248378 -515.621072973 5830511.027102432 1943503.67570081066667 2749258.455012492 3367140.1929065133 3 6984454.211097692 -617.5607769230769 6983219.0895438458462 2327739.696514615282066667 3292794.4113115156 4032833.0678006653 +762 2 5831542.269248378 1531.2194054054 5833073.4886537834 2916536.7443268917 2915005.5249214866 4122440.3477364695 2 6984454.211097692 1833.9456923076925 6986288.1567899996925 3493144.07839499984625 3491310.1327026924 4937458.140118758 +NULL 3072 9318.4351351351 -4298.1513513514 5018444.1081079808 1633.60810810806667 5695.483082135364 5696.4103077145055 3072 11160.715384615385 -5147.907692307693 6010604.3076923073536 1956.576923076922966667 6821.495748565159 6822.606289190924 diff --git a/ql/src/test/results/clientpositive/vectorization_13.q.out b/ql/src/test/results/clientpositive/vectorization_13.q.out index 1cefe30b3f55..061cb0da39ff 100644 --- a/ql/src/test/results/clientpositive/vectorization_13.q.out +++ b/ql/src/test/results/clientpositive/vectorization_13.q.out @@ -28,6 +28,7 @@ WHERE (((cfloat < 3569) AND ((ctimestamp2 != 12) AND (ctinyint < 9763215.5639)))) GROUP BY cboolean1, ctinyint, ctimestamp1, cfloat, cstring1 +ORDER BY cboolean1, ctinyint, ctimestamp1, cfloat, cstring1 LIMIT 40 PREHOOK: type: QUERY POSTHOOK: query: EXPLAIN @@ -60,11 +61,13 @@ WHERE (((cfloat < 3569) AND ((ctimestamp2 != 12) AND (ctinyint < 9763215.5639)))) GROUP BY cboolean1, ctinyint, ctimestamp1, cfloat, cstring1 +ORDER BY cboolean1, ctinyint, ctimestamp1, cfloat, cstring1 LIMIT 40 POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 STAGE PLANS: Stage: Stage-1 @@ -101,19 +104,40 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10 Statistics: Num rows: 1365 Data size: 41904 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: boolean), _col1 (type: tinyint), _col2 (type: timestamp), _col3 (type: float), _col4 (type: string), (- _col1) (type: tinyint), _col5 (type: tinyint), ((- _col1) + _col5) (type: tinyint), _col6 (type: double), (_col6 * ((- _col1) + _col5)) (type: double), (- _col6) (type: double), (79.553 * _col3) (type: double), _col7 (type: double), (- _col6) (type: double), _col8 (type: double), (((- _col1) + _col5) - 10.175) (type: double), (- (- _col6)) (type: double), (-26.28 / (- (- _col6))) (type: double), _col9 (type: float), ((_col6 * ((- _col1) + _col5)) / _col1) (type: double), _col10 (type: tinyint) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20 + expressions: _col0 (type: boolean), _col1 (type: tinyint), (- _col6) (type: double), (79.553 * _col3) (type: double), _col7 (type: double), _col8 (type: double), (((- _col1) + _col5) - 10.175) (type: double), (- (- _col6)) (type: double), (-26.28 / (- (- _col6))) (type: double), _col9 (type: float), ((_col6 * ((- _col1) + _col5)) / _col1) (type: double), _col2 (type: timestamp), _col10 (type: tinyint), _col3 (type: float), _col4 (type: string), (- _col1) (type: tinyint), _col5 (type: tinyint), ((- _col1) + _col5) (type: tinyint), _col6 (type: double), (_col6 * ((- _col1) + _col5)) (type: double) + outputColumnNames: _col0, _col1, _col10, _col11, _col12, _col14, _col15, _col16, _col17, _col18, _col19, _col2, _col20, _col3, _col4, _col5, _col6, _col7, _col8, _col9 Statistics: Num rows: 1365 Data size: 41904 Basic stats: COMPLETE Column stats: NONE - Limit - Number of rows: 40 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + key expressions: _col0 (type: boolean), _col1 (type: tinyint), _col2 (type: timestamp), _col3 (type: float), _col4 (type: string) + sort order: +++++ + Statistics: Num rows: 1365 Data size: 41904 Basic stats: COMPLETE Column stats: NONE + value expressions: _col5 (type: tinyint), _col6 (type: tinyint), _col7 (type: tinyint), _col8 (type: double), _col9 (type: double), _col10 (type: double), _col11 (type: double), _col12 (type: double), _col14 (type: double), _col15 (type: double), _col16 (type: double), _col17 (type: double), _col18 (type: float), _col19 (type: double), _col20 (type: tinyint) + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: boolean), KEY.reducesinkkey1 (type: tinyint), KEY.reducesinkkey2 (type: timestamp), KEY.reducesinkkey3 (type: float), KEY.reducesinkkey4 (type: string), VALUE._col0 (type: tinyint), VALUE._col1 (type: tinyint), VALUE._col2 (type: tinyint), VALUE._col3 (type: double), VALUE._col4 (type: double), VALUE._col5 (type: double), VALUE._col6 (type: double), VALUE._col7 (type: double), VALUE._col5 (type: double), VALUE._col8 (type: double), VALUE._col9 (type: double), VALUE._col10 (type: double), VALUE._col11 (type: double), VALUE._col12 (type: float), VALUE._col13 (type: double), VALUE._col14 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20 + Statistics: Num rows: 1365 Data size: 41904 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 40 + Statistics: Num rows: 40 Data size: 1200 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false Statistics: Num rows: 40 Data size: 1200 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 40 Data size: 1200 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -150,6 +174,7 @@ WHERE (((cfloat < 3569) AND ((ctimestamp2 != 12) AND (ctinyint < 9763215.5639)))) GROUP BY cboolean1, ctinyint, ctimestamp1, cfloat, cstring1 +ORDER BY cboolean1, ctinyint, ctimestamp1, cfloat, cstring1 LIMIT 40 PREHOOK: type: QUERY PREHOOK: Input: default@alltypesorc @@ -183,6 +208,7 @@ WHERE (((cfloat < 3569) AND ((ctimestamp2 != 12) AND (ctinyint < 9763215.5639)))) GROUP BY cboolean1, ctinyint, ctimestamp1, cfloat, cstring1 +ORDER BY cboolean1, ctinyint, ctimestamp1, cfloat, cstring1 LIMIT 40 POSTHOOK: type: QUERY POSTHOOK: Input: default@alltypesorc @@ -258,6 +284,7 @@ WHERE (((cfloat < 3569) AND ((ctimestamp2 != -1.3359999999999999) AND (ctinyint < 9763215.5639)))) GROUP BY cboolean1, ctinyint, ctimestamp1, cfloat, cstring1 +ORDER BY cboolean1, ctinyint, ctimestamp1, cfloat, cstring1 LIMIT 40 PREHOOK: type: QUERY POSTHOOK: query: -- double compare timestamp @@ -291,11 +318,13 @@ WHERE (((cfloat < 3569) AND ((ctimestamp2 != -1.3359999999999999) AND (ctinyint < 9763215.5639)))) GROUP BY cboolean1, ctinyint, ctimestamp1, cfloat, cstring1 +ORDER BY cboolean1, ctinyint, ctimestamp1, cfloat, cstring1 LIMIT 40 POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 STAGE PLANS: Stage: Stage-1 @@ -332,19 +361,40 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10 Statistics: Num rows: 1365 Data size: 41904 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: boolean), _col1 (type: tinyint), _col2 (type: timestamp), _col3 (type: float), _col4 (type: string), (- _col1) (type: tinyint), _col5 (type: tinyint), ((- _col1) + _col5) (type: tinyint), _col6 (type: double), (_col6 * ((- _col1) + _col5)) (type: double), (- _col6) (type: double), (79.553 * _col3) (type: double), _col7 (type: double), (- _col6) (type: double), _col8 (type: double), (((- _col1) + _col5) - 10.175) (type: double), (- (- _col6)) (type: double), (-26.28 / (- (- _col6))) (type: double), _col9 (type: float), ((_col6 * ((- _col1) + _col5)) / _col1) (type: double), _col10 (type: tinyint) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20 + expressions: _col0 (type: boolean), _col1 (type: tinyint), (- _col6) (type: double), (79.553 * _col3) (type: double), _col7 (type: double), _col8 (type: double), (((- _col1) + _col5) - 10.175) (type: double), (- (- _col6)) (type: double), (-26.28 / (- (- _col6))) (type: double), _col9 (type: float), ((_col6 * ((- _col1) + _col5)) / _col1) (type: double), _col2 (type: timestamp), _col10 (type: tinyint), _col3 (type: float), _col4 (type: string), (- _col1) (type: tinyint), _col5 (type: tinyint), ((- _col1) + _col5) (type: tinyint), _col6 (type: double), (_col6 * ((- _col1) + _col5)) (type: double) + outputColumnNames: _col0, _col1, _col10, _col11, _col12, _col14, _col15, _col16, _col17, _col18, _col19, _col2, _col20, _col3, _col4, _col5, _col6, _col7, _col8, _col9 Statistics: Num rows: 1365 Data size: 41904 Basic stats: COMPLETE Column stats: NONE - Limit - Number of rows: 40 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + key expressions: _col0 (type: boolean), _col1 (type: tinyint), _col2 (type: timestamp), _col3 (type: float), _col4 (type: string) + sort order: +++++ + Statistics: Num rows: 1365 Data size: 41904 Basic stats: COMPLETE Column stats: NONE + value expressions: _col5 (type: tinyint), _col6 (type: tinyint), _col7 (type: tinyint), _col8 (type: double), _col9 (type: double), _col10 (type: double), _col11 (type: double), _col12 (type: double), _col14 (type: double), _col15 (type: double), _col16 (type: double), _col17 (type: double), _col18 (type: float), _col19 (type: double), _col20 (type: tinyint) + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: boolean), KEY.reducesinkkey1 (type: tinyint), KEY.reducesinkkey2 (type: timestamp), KEY.reducesinkkey3 (type: float), KEY.reducesinkkey4 (type: string), VALUE._col0 (type: tinyint), VALUE._col1 (type: tinyint), VALUE._col2 (type: tinyint), VALUE._col3 (type: double), VALUE._col4 (type: double), VALUE._col5 (type: double), VALUE._col6 (type: double), VALUE._col7 (type: double), VALUE._col5 (type: double), VALUE._col8 (type: double), VALUE._col9 (type: double), VALUE._col10 (type: double), VALUE._col11 (type: double), VALUE._col12 (type: float), VALUE._col13 (type: double), VALUE._col14 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20 + Statistics: Num rows: 1365 Data size: 41904 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 40 + Statistics: Num rows: 40 Data size: 1200 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false Statistics: Num rows: 40 Data size: 1200 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 40 Data size: 1200 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -381,6 +431,7 @@ WHERE (((cfloat < 3569) AND ((ctimestamp2 != -1.3359999999999999) AND (ctinyint < 9763215.5639)))) GROUP BY cboolean1, ctinyint, ctimestamp1, cfloat, cstring1 +ORDER BY cboolean1, ctinyint, ctimestamp1, cfloat, cstring1 LIMIT 40 PREHOOK: type: QUERY PREHOOK: Input: default@alltypesorc @@ -414,6 +465,7 @@ WHERE (((cfloat < 3569) AND ((ctimestamp2 != -1.3359999999999999) AND (ctinyint < 9763215.5639)))) GROUP BY cboolean1, ctinyint, ctimestamp1, cfloat, cstring1 +ORDER BY cboolean1, ctinyint, ctimestamp1, cfloat, cstring1 LIMIT 40 POSTHOOK: type: QUERY POSTHOOK: Input: default@alltypesorc