From f275ee40b6c9c31c4e5b6405f5ae17f2a6f9dd68 Mon Sep 17 00:00:00 2001 From: Stamatis Zampetakis Date: Fri, 17 Jul 2026 17:28:42 +0300 Subject: [PATCH] HIVE-29745: Remove Druid integration Remove the Druid storage handler and all associated infrastructure from the codebase since its broken and no longer used. Changes include: - Delete the druid-handler module (storage handler, SerDe, I/O formats, Kafka ingestion support, Kerberos security utilities) - Delete the itests/qtest-druid module (MiniDruidCluster, test drivers) - Remove all Druid-related query test files (.q and .q.out) - Remove hive.druid.* configuration variables from HiveConf - Remove Druid constants and calcite-druid dependency - Remove Calcite optimizer rules: HiveDruidRules, DruidSqlOperatorConverter, SortedDynPartitionTimeGranularityOptimizer - Remove DruidQuery handling from CalcitePlanner, HiveVolcanoPlanner, HiveFilterProjectTransposeRule, HiveRelFieldTrimmer, ASTBuilder, ASTConverter, and HiveMaterializedViewsRegistry - Remove DRUID/DRUID_LOCAL/DRUID_KAFKA cluster types from test infra - Relocate Kafka test utilities (SingleNodeKafkaCluster, Wikipedia) from itests/qtest-druid to itests/util under org.apache.hadoop.hive.kafka - Update itests/util dependencies to use kafka artifacts directly instead of pulling them transitively via hive-it-druid --- .../apache/hadoop/hive/conf/Constants.java | 17 - .../org/apache/hadoop/hive/conf/HiveConf.java | 70 - .../druid_table_alltypesorc/load.hive.sql | 40 - data/scripts/q_test_cleanup_druid.sql | 2 - data/scripts/q_test_druid_init.sql | 0 druid-handler/README.md | 21 - druid-handler/pom.xml | 434 --- .../hadoop/hive/druid/DruidKafkaUtils.java | 232 -- .../hive/druid/DruidStorageHandler.java | 1020 ------ .../hive/druid/DruidStorageHandlerInfo.java | 67 - .../hive/druid/DruidStorageHandlerUtils.java | 1113 ------- .../hive/druid/conf/DruidConstants.java | 81 - .../hadoop/hive/druid/conf/package-info.java | 23 - .../hive/druid/io/DruidOutputFormat.java | 178 -- .../druid/io/DruidQueryBasedInputFormat.java | 262 -- .../hive/druid/io/DruidRecordWriter.java | 300 -- .../hive/druid/io/DruidVectorizedWrapper.java | 139 - .../hadoop/hive/druid/io/HiveDruidSplit.java | 80 - .../hadoop/hive/druid/io/package-info.java | 23 - .../hive/druid/json/AvroBytesDecoder.java | 31 - .../hadoop/hive/druid/json/AvroParseSpec.java | 83 - .../druid/json/AvroStreamInputRowParser.java | 76 - .../json/InlineSchemaAvroBytesDecoder.java | 44 - .../json/KafkaIndexTaskTuningConfig.java | 128 - .../druid/json/KafkaSupervisorIOConfig.java | 238 -- .../druid/json/KafkaSupervisorReport.java | 202 -- .../hive/druid/json/KafkaSupervisorSpec.java | 117 - .../json/KafkaSupervisorTuningConfig.java | 161 - .../SeekableStreamIndexTaskTuningConfig.java | 308 -- .../SeekableStreamSupervisorTuningConfig.java | 59 - .../hive/druid/json/TaskReportData.java | 102 - .../hadoop/hive/druid/json/package-info.java | 23 - .../hadoop/hive/druid/package-info.java | 23 - .../druid/security/DruidKerberosUtil.java | 119 - .../druid/security/KerberosHttpClient.java | 132 - .../druid/security/ResponseCookieHandler.java | 75 - .../RetryIfUnauthorizedResponseHandler.java | 101 - .../druid/security/RetryResponseHolder.java | 45 - .../hive/druid/security/package-info.java | 23 - .../serde/DruidGroupByQueryRecordReader.java | 93 - .../druid/serde/DruidQueryRecordReader.java | 303 -- .../serde/DruidScanQueryRecordReader.java | 88 - .../hadoop/hive/druid/serde/DruidSerDe.java | 497 --- .../hive/druid/serde/DruidSerDeUtils.java | 64 - .../DruidTimeseriesQueryRecordReader.java | 89 - .../serde/DruidTopNQueryRecordReader.java | 96 - .../hive/druid/serde/DruidWritable.java | 131 - .../hadoop/hive/druid/serde/package-info.java | 23 - .../hive/druid/DerbyConnectorTestUtility.java | 129 - ...dQueryBasedInputFormatToAddFaultyHost.java | 53 - .../hadoop/hive/druid/QTestDruidSerDe.java | 104 - .../hadoop/hive/druid/QTestDruidSerDe2.java | 106 - .../hive/druid/QTestDruidStorageHandler.java | 34 - .../hive/druid/QTestDruidStorageHandler2.java | 34 - ...estDruidStorageHandlerToAddFaultyHost.java | 34 - .../hive/druid/TestDruidStorageHandler.java | 774 ----- .../TestHiveDruidQueryBasedInputFormat.java | 177 -- .../hive/druid/io/TestHiveDruidSplit.java | 49 - .../hive/druid/serde/TestDruidSerDe.java | 884 ------ .../hive/ql/io/TestDruidRecordWriter.java | 245 -- itests/hive-minikdc/pom.xml | 12 - itests/hive-unit-hadoop2/pom.xml | 6 - itests/hive-unit/pom.xml | 6 - .../apache/hive/jdbc/TestRestrictedList.java | 2 - itests/pom.xml | 17 - itests/qtest-accumulo/pom.xml | 4 - itests/qtest-druid/pom.xml | 319 -- .../java/org/apache/hive/druid/DruidNode.java | 44 - .../apache/hive/druid/ForkingDruidNode.java | 157 - .../apache/hive/druid/MiniDruidCluster.java | 283 -- itests/qtest-iceberg/pom.xml | 34 - itests/qtest-kudu/pom.xml | 4 - itests/qtest/pom.xml | 34 - .../hive/cli/MiniDruidLlapLocalCliDriver.java | 63 - .../hive/cli/TestMiniDruidCliDriver.java | 64 - .../hive/cli/TestMiniDruidKafkaCliDriver.java | 64 - .../resources/testconfiguration.properties | 28 - itests/util/pom.xml | 33 +- .../hadoop/hive/cli/control/CliConfigs.java | 63 - .../hive/kafka/SingleNodeKafkaCluster.java | 6 +- .../apache/hadoop}/hive/kafka/Wikipedia.java | 90 +- .../hadoop/hive/ql/QTestMiniClusters.java | 64 +- ...LStdHiveAuthorizationValidatorForTest.java | 4 - .../cli/service/AsyncTaskCopyAuxJars.java | 2 +- packaging/pom.xml | 5 - packaging/src/main/assembly/src.xml | 1 - pom.xml | 9 - ql/pom.xml | 10 - .../table/create/CreateTableOperation.java | 1 - .../HiveMaterializedViewsRegistry.java | 105 +- .../hadoop/hive/ql/optimizer/Optimizer.java | 4 - ...dDynPartitionTimeGranularityOptimizer.java | 479 --- .../calcite/cost/HiveVolcanoPlanner.java | 23 - .../calcite/rules/HiveDruidRules.java | 304 -- .../rules/HiveFilterProjectTransposeRule.java | 8 - .../calcite/rules/HiveRelFieldTrimmer.java | 71 +- .../views/HiveMaterializedViewUtils.java | 20 +- .../calcite/translator/ASTBuilder.java | 35 +- .../calcite/translator/ASTConverter.java | 14 - .../translator/PlanModifierForASTConv.java | 5 - .../hadoop/hive/ql/parse/CalcitePlanner.java | 190 +- .../ql/parse/DruidSqlOperatorConverter.java | 392 --- .../TestSQLStdHiveAccessControllerHS2.java | 2 +- .../queries/clientnegative/druid_address.q | 5 - .../queries/clientnegative/druid_buckets.q | 6 - .../test/queries/clientnegative/druid_case.q | 6 - .../queries/clientnegative/druid_datasource.q | 3 - .../clientnegative/druid_datasource2.q | 4 - .../queries/clientnegative/druid_location.q | 6 - .../queries/clientnegative/druid_partitions.q | 6 - .../druid_materialized_view_rewrite_ssb.q | 318 -- .../queries/clientpositive/druid_timeseries.q | 111 - .../clientpositive/druid_timestamptz.q | 94 - .../clientpositive/druid_timestamptz2.q | 64 - .../test/queries/clientpositive/druid_topn.q | 75 - .../clientpositive/druidkafkamini_avro.q | 100 - .../clientpositive/druidkafkamini_basic.q | 131 - .../clientpositive/druidkafkamini_csv.q | 38 - .../clientpositive/druidkafkamini_delimited.q | 39 - .../druidmini_dynamic_partition.q | 190 -- .../clientpositive/druidmini_expressions.q | 237 -- .../clientpositive/druidmini_extractTime.q | 187 -- .../clientpositive/druidmini_floorTime.q | 148 - .../queries/clientpositive/druidmini_joins.q | 62 - .../clientpositive/druidmini_masking.q | 26 - .../queries/clientpositive/druidmini_mv.q | 115 - .../druidmini_semijoin_reduction_all_types.q | 130 - .../queries/clientpositive/druidmini_test1.q | 122 - .../clientpositive/druidmini_test_alter.q | 56 - .../clientpositive/druidmini_test_insert.q | 157 - .../clientpositive/druidmini_test_ts.q | 109 - .../clientnegative/druid_address.q.out | 7 - .../clientnegative/druid_buckets.q.out | 8 - .../results/clientnegative/druid_case.q.out | 7 - .../clientnegative/druid_datasource.q.out | 7 - .../clientnegative/druid_datasource2.q.out | 7 - .../clientnegative/druid_location.q.out | 9 - .../clientnegative/druid_partitions.q.out | 8 - .../druid_materialized_view_rewrite_ssb.q.out | 813 ----- .../druid/druid_timeseries.q.out | 640 ---- .../druid/druid_timestamptz2.q.out | 209 -- .../clientpositive/druid/druid_topn.q.out | 374 --- .../druid/druidkafkamini_avro.q.out | 263 -- .../druid/druidkafkamini_basic.q.out | 767 ----- .../druid/druidkafkamini_csv.q.out | 138 - .../druid/druidkafkamini_delimited.q.out | 140 - .../druid/druidmini_dynamic_partition.q.out | 682 ---- .../druid/druidmini_expressions.q.out | 2737 ----------------- .../druid/druidmini_extractTime.q.out | 886 ------ .../druid/druidmini_floorTime.q.out | 692 ----- .../clientpositive/druid/druidmini_mv.q.out | 741 ----- ...uidmini_semijoin_reduction_all_types.q.out | 2033 ------------ .../druid/druidmini_test1.q.out | 960 ------ .../druid/druidmini_test_alter.q.out | 210 -- .../druid/druidmini_test_insert.q.out | 875 ------ .../druid/druidmini_test_ts.q.out | 556 ---- 156 files changed, 136 insertions(+), 29458 deletions(-) delete mode 100644 data/files/datasets/druid_table_alltypesorc/load.hive.sql delete mode 100644 data/scripts/q_test_cleanup_druid.sql delete mode 100644 data/scripts/q_test_druid_init.sql delete mode 100644 druid-handler/README.md delete mode 100644 druid-handler/pom.xml delete mode 100644 druid-handler/src/java/org/apache/hadoop/hive/druid/DruidKafkaUtils.java delete mode 100644 druid-handler/src/java/org/apache/hadoop/hive/druid/DruidStorageHandler.java delete mode 100644 druid-handler/src/java/org/apache/hadoop/hive/druid/DruidStorageHandlerInfo.java delete mode 100644 druid-handler/src/java/org/apache/hadoop/hive/druid/DruidStorageHandlerUtils.java delete mode 100644 druid-handler/src/java/org/apache/hadoop/hive/druid/conf/DruidConstants.java delete mode 100644 druid-handler/src/java/org/apache/hadoop/hive/druid/conf/package-info.java delete mode 100644 druid-handler/src/java/org/apache/hadoop/hive/druid/io/DruidOutputFormat.java delete mode 100644 druid-handler/src/java/org/apache/hadoop/hive/druid/io/DruidQueryBasedInputFormat.java delete mode 100644 druid-handler/src/java/org/apache/hadoop/hive/druid/io/DruidRecordWriter.java delete mode 100644 druid-handler/src/java/org/apache/hadoop/hive/druid/io/DruidVectorizedWrapper.java delete mode 100644 druid-handler/src/java/org/apache/hadoop/hive/druid/io/HiveDruidSplit.java delete mode 100644 druid-handler/src/java/org/apache/hadoop/hive/druid/io/package-info.java delete mode 100644 druid-handler/src/java/org/apache/hadoop/hive/druid/json/AvroBytesDecoder.java delete mode 100644 druid-handler/src/java/org/apache/hadoop/hive/druid/json/AvroParseSpec.java delete mode 100644 druid-handler/src/java/org/apache/hadoop/hive/druid/json/AvroStreamInputRowParser.java delete mode 100644 druid-handler/src/java/org/apache/hadoop/hive/druid/json/InlineSchemaAvroBytesDecoder.java delete mode 100644 druid-handler/src/java/org/apache/hadoop/hive/druid/json/KafkaIndexTaskTuningConfig.java delete mode 100644 druid-handler/src/java/org/apache/hadoop/hive/druid/json/KafkaSupervisorIOConfig.java delete mode 100644 druid-handler/src/java/org/apache/hadoop/hive/druid/json/KafkaSupervisorReport.java delete mode 100644 druid-handler/src/java/org/apache/hadoop/hive/druid/json/KafkaSupervisorSpec.java delete mode 100644 druid-handler/src/java/org/apache/hadoop/hive/druid/json/KafkaSupervisorTuningConfig.java delete mode 100644 druid-handler/src/java/org/apache/hadoop/hive/druid/json/SeekableStreamIndexTaskTuningConfig.java delete mode 100644 druid-handler/src/java/org/apache/hadoop/hive/druid/json/SeekableStreamSupervisorTuningConfig.java delete mode 100644 druid-handler/src/java/org/apache/hadoop/hive/druid/json/TaskReportData.java delete mode 100644 druid-handler/src/java/org/apache/hadoop/hive/druid/json/package-info.java delete mode 100644 druid-handler/src/java/org/apache/hadoop/hive/druid/package-info.java delete mode 100644 druid-handler/src/java/org/apache/hadoop/hive/druid/security/DruidKerberosUtil.java delete mode 100644 druid-handler/src/java/org/apache/hadoop/hive/druid/security/KerberosHttpClient.java delete mode 100644 druid-handler/src/java/org/apache/hadoop/hive/druid/security/ResponseCookieHandler.java delete mode 100644 druid-handler/src/java/org/apache/hadoop/hive/druid/security/RetryIfUnauthorizedResponseHandler.java delete mode 100644 druid-handler/src/java/org/apache/hadoop/hive/druid/security/RetryResponseHolder.java delete mode 100644 druid-handler/src/java/org/apache/hadoop/hive/druid/security/package-info.java delete mode 100644 druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidGroupByQueryRecordReader.java delete mode 100644 druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidQueryRecordReader.java delete mode 100644 druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidScanQueryRecordReader.java delete mode 100644 druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidSerDe.java delete mode 100644 druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidSerDeUtils.java delete mode 100644 druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidTimeseriesQueryRecordReader.java delete mode 100644 druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidTopNQueryRecordReader.java delete mode 100644 druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidWritable.java delete mode 100644 druid-handler/src/java/org/apache/hadoop/hive/druid/serde/package-info.java delete mode 100644 druid-handler/src/test/org/apache/hadoop/hive/druid/DerbyConnectorTestUtility.java delete mode 100644 druid-handler/src/test/org/apache/hadoop/hive/druid/QTestDruidQueryBasedInputFormatToAddFaultyHost.java delete mode 100644 druid-handler/src/test/org/apache/hadoop/hive/druid/QTestDruidSerDe.java delete mode 100644 druid-handler/src/test/org/apache/hadoop/hive/druid/QTestDruidSerDe2.java delete mode 100644 druid-handler/src/test/org/apache/hadoop/hive/druid/QTestDruidStorageHandler.java delete mode 100644 druid-handler/src/test/org/apache/hadoop/hive/druid/QTestDruidStorageHandler2.java delete mode 100644 druid-handler/src/test/org/apache/hadoop/hive/druid/QTestDruidStorageHandlerToAddFaultyHost.java delete mode 100644 druid-handler/src/test/org/apache/hadoop/hive/druid/TestDruidStorageHandler.java delete mode 100644 druid-handler/src/test/org/apache/hadoop/hive/druid/TestHiveDruidQueryBasedInputFormat.java delete mode 100644 druid-handler/src/test/org/apache/hadoop/hive/druid/io/TestHiveDruidSplit.java delete mode 100644 druid-handler/src/test/org/apache/hadoop/hive/druid/serde/TestDruidSerDe.java delete mode 100644 druid-handler/src/test/org/apache/hadoop/hive/ql/io/TestDruidRecordWriter.java delete mode 100644 itests/qtest-druid/pom.xml delete mode 100644 itests/qtest-druid/src/main/java/org/apache/hive/druid/DruidNode.java delete mode 100644 itests/qtest-druid/src/main/java/org/apache/hive/druid/ForkingDruidNode.java delete mode 100644 itests/qtest-druid/src/main/java/org/apache/hive/druid/MiniDruidCluster.java delete mode 100644 itests/qtest/src/test/java/org/apache/hadoop/hive/cli/MiniDruidLlapLocalCliDriver.java delete mode 100644 itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestMiniDruidCliDriver.java delete mode 100644 itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestMiniDruidKafkaCliDriver.java rename itests/{qtest-druid/src/main/java/org/apache => util/src/main/java/org/apache/hadoop}/hive/kafka/SingleNodeKafkaCluster.java (97%) rename itests/{qtest-druid/src/main/java/org/apache => util/src/main/java/org/apache/hadoop}/hive/kafka/Wikipedia.java (92%) delete mode 100644 ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionTimeGranularityOptimizer.java delete mode 100644 ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveDruidRules.java delete mode 100644 ql/src/java/org/apache/hadoop/hive/ql/parse/DruidSqlOperatorConverter.java delete mode 100644 ql/src/test/queries/clientnegative/druid_address.q delete mode 100644 ql/src/test/queries/clientnegative/druid_buckets.q delete mode 100644 ql/src/test/queries/clientnegative/druid_case.q delete mode 100644 ql/src/test/queries/clientnegative/druid_datasource.q delete mode 100644 ql/src/test/queries/clientnegative/druid_datasource2.q delete mode 100644 ql/src/test/queries/clientnegative/druid_location.q delete mode 100644 ql/src/test/queries/clientnegative/druid_partitions.q delete mode 100644 ql/src/test/queries/clientpositive/druid_materialized_view_rewrite_ssb.q delete mode 100644 ql/src/test/queries/clientpositive/druid_timeseries.q delete mode 100644 ql/src/test/queries/clientpositive/druid_timestamptz.q delete mode 100644 ql/src/test/queries/clientpositive/druid_timestamptz2.q delete mode 100644 ql/src/test/queries/clientpositive/druid_topn.q delete mode 100644 ql/src/test/queries/clientpositive/druidkafkamini_avro.q delete mode 100644 ql/src/test/queries/clientpositive/druidkafkamini_basic.q delete mode 100644 ql/src/test/queries/clientpositive/druidkafkamini_csv.q delete mode 100644 ql/src/test/queries/clientpositive/druidkafkamini_delimited.q delete mode 100644 ql/src/test/queries/clientpositive/druidmini_dynamic_partition.q delete mode 100644 ql/src/test/queries/clientpositive/druidmini_expressions.q delete mode 100644 ql/src/test/queries/clientpositive/druidmini_extractTime.q delete mode 100644 ql/src/test/queries/clientpositive/druidmini_floorTime.q delete mode 100644 ql/src/test/queries/clientpositive/druidmini_joins.q delete mode 100644 ql/src/test/queries/clientpositive/druidmini_masking.q delete mode 100644 ql/src/test/queries/clientpositive/druidmini_mv.q delete mode 100644 ql/src/test/queries/clientpositive/druidmini_semijoin_reduction_all_types.q delete mode 100644 ql/src/test/queries/clientpositive/druidmini_test1.q delete mode 100644 ql/src/test/queries/clientpositive/druidmini_test_alter.q delete mode 100644 ql/src/test/queries/clientpositive/druidmini_test_insert.q delete mode 100644 ql/src/test/queries/clientpositive/druidmini_test_ts.q delete mode 100644 ql/src/test/results/clientnegative/druid_address.q.out delete mode 100644 ql/src/test/results/clientnegative/druid_buckets.q.out delete mode 100644 ql/src/test/results/clientnegative/druid_case.q.out delete mode 100644 ql/src/test/results/clientnegative/druid_datasource.q.out delete mode 100644 ql/src/test/results/clientnegative/druid_datasource2.q.out delete mode 100644 ql/src/test/results/clientnegative/druid_location.q.out delete mode 100644 ql/src/test/results/clientnegative/druid_partitions.q.out delete mode 100644 ql/src/test/results/clientpositive/druid/druid_materialized_view_rewrite_ssb.q.out delete mode 100644 ql/src/test/results/clientpositive/druid/druid_timeseries.q.out delete mode 100644 ql/src/test/results/clientpositive/druid/druid_timestamptz2.q.out delete mode 100644 ql/src/test/results/clientpositive/druid/druid_topn.q.out delete mode 100644 ql/src/test/results/clientpositive/druid/druidkafkamini_avro.q.out delete mode 100644 ql/src/test/results/clientpositive/druid/druidkafkamini_basic.q.out delete mode 100644 ql/src/test/results/clientpositive/druid/druidkafkamini_csv.q.out delete mode 100644 ql/src/test/results/clientpositive/druid/druidkafkamini_delimited.q.out delete mode 100644 ql/src/test/results/clientpositive/druid/druidmini_dynamic_partition.q.out delete mode 100644 ql/src/test/results/clientpositive/druid/druidmini_expressions.q.out delete mode 100644 ql/src/test/results/clientpositive/druid/druidmini_extractTime.q.out delete mode 100644 ql/src/test/results/clientpositive/druid/druidmini_floorTime.q.out delete mode 100644 ql/src/test/results/clientpositive/druid/druidmini_mv.q.out delete mode 100644 ql/src/test/results/clientpositive/druid/druidmini_semijoin_reduction_all_types.q.out delete mode 100644 ql/src/test/results/clientpositive/druid/druidmini_test1.q.out delete mode 100644 ql/src/test/results/clientpositive/druid/druidmini_test_alter.q.out delete mode 100644 ql/src/test/results/clientpositive/druid/druidmini_test_insert.q.out delete mode 100644 ql/src/test/results/clientpositive/druid/druidmini_test_ts.q.out diff --git a/common/src/java/org/apache/hadoop/hive/conf/Constants.java b/common/src/java/org/apache/hadoop/hive/conf/Constants.java index 4b0e75fafb51..7aa6945c6886 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/Constants.java +++ b/common/src/java/org/apache/hadoop/hive/conf/Constants.java @@ -30,23 +30,6 @@ public class Constants { public static final String LLAP_NUM_BUCKETS = "llap.num.buckets"; public static final String LLAP_BUCKET_ID = "llap.bucket.id"; - /* Constants for Druid storage handler */ - public static final String DRUID_HIVE_STORAGE_HANDLER_ID = - "org.apache.hadoop.hive.druid.DruidStorageHandler"; - public static final String DRUID_HIVE_OUTPUT_FORMAT = - "org.apache.hadoop.hive.druid.io.DruidOutputFormat"; - public static final String DRUID_DATA_SOURCE = "druid.datasource"; - public static final String DRUID_SEGMENT_GRANULARITY = "druid.segment.granularity"; - - public static final String DRUID_TARGET_SHARDS_PER_GRANULARITY = - "druid.segment.targetShardsPerGranularity"; - public static final String DRUID_TIMESTAMP_GRANULARITY_COL_NAME = "__time_granularity"; - public static final String DRUID_SHARD_KEY_COL_NAME = "__druid_extra_partition_key"; - public static final String DRUID_QUERY_JSON = "druid.query.json"; - public static final String DRUID_QUERY_FIELD_NAMES = "druid.fieldNames"; - public static final String DRUID_QUERY_FIELD_TYPES = "druid.fieldTypes"; - public static final String DRUID_QUERY_TYPE = "druid.query.type"; - public static final String JDBC_HIVE_STORAGE_HANDLER_ID = "org.apache.hive.storage.jdbc.JdbcStorageHandler"; public static final String JDBC_CONFIG_PREFIX = "hive.sql"; diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 65ad8a82bbef..54516007d32a 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -3382,72 +3382,6 @@ public static enum ConfVars { OPTIMIZE_ACID_META_COLUMNS("hive.optimize.acid.meta.columns", true, "If true, don't decode Acid metadata columns from storage unless" + " they are needed."), - - // For Druid storage handler - HIVE_DRUID_INDEXING_GRANULARITY("hive.druid.indexer.segments.granularity", "DAY", - new PatternSet("YEAR", "MONTH", "WEEK", "DAY", "HOUR", "MINUTE", "SECOND"), - "Granularity for the segments created by the Druid storage handler" - ), - HIVE_DRUID_MAX_PARTITION_SIZE("hive.druid.indexer.partition.size.max", 5000000, - "Maximum number of records per segment partition" - ), - HIVE_DRUID_MAX_ROW_IN_MEMORY("hive.druid.indexer.memory.rownum.max", 75000, - "Maximum number of records in memory while storing data in Druid" - ), - HIVE_DRUID_BROKER_DEFAULT_ADDRESS("hive.druid.broker.address.default", "localhost:8082", - "Address of the Druid broker. If we are querying Druid from Hive, this address needs to be\n" - + - "declared" - ), - HIVE_DRUID_COORDINATOR_DEFAULT_ADDRESS("hive.druid.coordinator.address.default", "localhost:8081", - "Address of the Druid coordinator. It is used to check the load status of newly created segments" - ), - HIVE_DRUID_OVERLORD_DEFAULT_ADDRESS("hive.druid.overlord.address.default", "localhost:8090", - "Address of the Druid overlord. It is used to submit indexing tasks to druid." - ), - HIVE_DRUID_SELECT_THRESHOLD("hive.druid.select.threshold", 10000, - "Takes only effect when hive.druid.select.distribute is set to false. \n" + - "When we can split a Select query, this is the maximum number of rows that we try to retrieve\n" + - "per query. In order to do that, we obtain the estimated size for the complete result. If the\n" + - "number of records of the query results is larger than this threshold, we split the query in\n" + - "total number of rows/threshold parts across the time dimension. Note that we assume the\n" + - "records to be split uniformly across the time dimension."), - HIVE_DRUID_NUM_HTTP_CONNECTION("hive.druid.http.numConnection", 20, "Number of connections used by\n" + - "the HTTP client."), - HIVE_DRUID_HTTP_READ_TIMEOUT("hive.druid.http.read.timeout", "PT1M", "Read timeout period for the HTTP\n" + - "client in ISO8601 format (for example P2W, P3M, PT1H30M, PT0.750S), default is period of 1 minute."), - HIVE_DRUID_SLEEP_TIME("hive.druid.sleep.time", "PT10S", - "Sleep time between retries in ISO8601 format (for example P2W, P3M, PT1H30M, PT0.750S), default is period of 10 seconds." - ), - HIVE_DRUID_BASE_PERSIST_DIRECTORY("hive.druid.basePersistDirectory", "", - "Local temporary directory used to persist intermediate indexing state, will default to JVM system property java.io.tmpdir." - ), - HIVE_DRUID_ROLLUP("hive.druid.rollup", true, "Whether to rollup druid rows or not."), - DRUID_SEGMENT_DIRECTORY("hive.druid.storage.storageDirectory", "/druid/segments" - , "druid deep storage location."), - DRUID_METADATA_BASE("hive.druid.metadata.base", "druid", "Default prefix for metadata tables"), - DRUID_METADATA_DB_TYPE("hive.druid.metadata.db.type", "mysql", - new PatternSet("mysql", "postgresql", "derby"), "Type of the metadata database." - ), - DRUID_METADATA_DB_USERNAME("hive.druid.metadata.username", "", - "Username to connect to Type of the metadata DB." - ), - DRUID_METADATA_DB_PASSWORD("hive.druid.metadata.password", "", - "Password to connect to Type of the metadata DB." - ), - DRUID_METADATA_DB_URI("hive.druid.metadata.uri", "", - "URI to connect to the database (for example jdbc:mysql://hostname:port/DBName)." - ), - DRUID_WORKING_DIR("hive.druid.working.directory", "/tmp/workingDirectory", - "Default hdfs working directory used to store some intermediate metadata" - ), - HIVE_DRUID_MAX_TRIES("hive.druid.maxTries", 5, "Maximum number of retries before giving up"), - HIVE_DRUID_PASSIVE_WAIT_TIME("hive.druid.passiveWaitTimeMs", 30000L, - "Wait time in ms default to 30 seconds." - ), - HIVE_DRUID_BITMAP_FACTORY_TYPE("hive.druid.bitmap.type", "roaring", new PatternSet("roaring", "concise"), "Coding algorithm use to encode the bitmaps"), - HIVE_DRUID_KERBEROS_ENABLE("hive.druid.kerberos.enable", true, - "Enable/Disable Kerberos authentication explicitly while connecting to a druid cluster."), // For HBase storage handler HIVE_HBASE_WAL_ENABLED("hive.hbase.wal.enabled", true, "Whether writes to HBase should be forced to the write-ahead log. \n" + @@ -5570,8 +5504,6 @@ public static enum ConfVars { "hive.privilege.synchronizer," + "hive.privilege.synchronizer.interval," + "hive.query.max.length," + - "hive.druid.broker.address.default," + - "hive.druid.coordinator.address.default," + "hikaricp.," + "hadoop.bin.path," + "yarn.bin.path," + @@ -5591,7 +5523,6 @@ public static enum ConfVars { HIVE_CONF_HIDDEN_LIST("hive.conf.hidden.list", METASTORE_PWD.varname + "," + HIVE_SERVER2_SSL_KEYSTORE_PASSWORD.varname + "," + HIVE_SERVER2_WEBUI_SSL_KEYSTORE_PASSWORD.varname - + "," + DRUID_METADATA_DB_PASSWORD.varname // Adding the S3 credentials from Hadoop config to be hidden + ",fs.s3.awsAccessKeyId" + ",fs.s3.awsSecretAccessKey" @@ -6787,7 +6718,6 @@ private static String[] convertVarsToRegex(String[] paramList) { "hive\\.auto\\..*", "hive\\.cbo\\..*", "hive\\.convert\\..*", - "hive\\.druid\\..*", "hive\\.exec\\.dynamic\\.partition.*", "hive\\.exec\\.max\\.dynamic\\.partitions.*", "hive\\.exec\\.compress\\..*", diff --git a/data/files/datasets/druid_table_alltypesorc/load.hive.sql b/data/files/datasets/druid_table_alltypesorc/load.hive.sql deleted file mode 100644 index c322e3fd3f43..000000000000 --- a/data/files/datasets/druid_table_alltypesorc/load.hive.sql +++ /dev/null @@ -1,40 +0,0 @@ -CREATE TABLE alltypesorc1( - ctinyint TINYINT, - csmallint SMALLINT, - cint INT, - cbigint BIGINT, - cfloat FLOAT, - cdouble DOUBLE, - cstring1 STRING, - cstring2 STRING, - ctimestamp1 TIMESTAMP, - ctimestamp2 TIMESTAMP, - cboolean1 BOOLEAN, - cboolean2 BOOLEAN, - cintstring STRING, - cfloatstring STRING, - cdoublestring STRING) - STORED AS ORC; - -LOAD DATA LOCAL INPATH "${hiveconf:test.data.dir}/alltypesorc" -OVERWRITE INTO TABLE alltypesorc1; - -CREATE EXTERNAL TABLE druid_table_alltypesorc -STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' -TBLPROPERTIES ("druid.segment.granularity" = "HOUR", "druid.query.granularity" = "MINUTE") -AS -SELECT cast (`ctimestamp1` as timestamp with local time zone) as `__time`, - cstring1, - cstring2, - cdouble, - cfloat, - ctinyint, - csmallint, - cint, - cbigint, - cboolean1, - cboolean2, - cast(cint as string) as cintstring, - cast(cfloat as string) as cfloatstring, - cast(cdouble as string) as cdoublestring - FROM alltypesorc1 where ctimestamp1 IS NOT NULL; diff --git a/data/scripts/q_test_cleanup_druid.sql b/data/scripts/q_test_cleanup_druid.sql deleted file mode 100644 index b0d342570472..000000000000 --- a/data/scripts/q_test_cleanup_druid.sql +++ /dev/null @@ -1,2 +0,0 @@ -DROP TABLE IF EXISTS alltypesorc; -DROP TABLE IF EXISTS druid_table; diff --git a/data/scripts/q_test_druid_init.sql b/data/scripts/q_test_druid_init.sql deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/druid-handler/README.md b/druid-handler/README.md deleted file mode 100644 index 285f7ec162a8..000000000000 --- a/druid-handler/README.md +++ /dev/null @@ -1,21 +0,0 @@ - -# Druid Storage Handler - -[Link for documentation](https://hive.apache.org/docs/latest/user/druid-integration) diff --git a/druid-handler/pom.xml b/druid-handler/pom.xml deleted file mode 100644 index dcc5ebd8a481..000000000000 --- a/druid-handler/pom.xml +++ /dev/null @@ -1,434 +0,0 @@ - - - - 4.0.0 - - org.apache.hive - hive - 4.3.0-SNAPSHOT - ../pom.xml - - hive-druid-handler - jar - Hive Druid Handler - - .. - 16.0.1 - - - - - - com.fasterxml.jackson.dataformat - jackson-dataformat-smile - - - com.fasterxml.jackson.core - jackson-databind - - - javax.validation - validation-api - ${validation-api.version} - - - org.apache.hive - hive-exec - ${project.version} - - - io.netty - * - - - com.google.guava - guava - - - com.fasterxml.jackson.core - jackson-core - - - com.fasterxml.jackson.core - jackson-annotations - - - com.fasterxml.jackson.core - jackson-databind - - - - - - com.google.guava - guava - ${druid.guava.version} - - - joda-time - joda-time - - - io.netty - netty - ${netty3.version} - - - org.apache.druid - druid-server - ${druid.version} - - - io.tesla.aether - tesla-aether - - - com.fasterxml.jackson.core - jackson-core - - - com.fasterxml.jackson.core - jackson-annotations - - - com.fasterxml.jackson.core - jackson-databind - - - org.apache.druid - druid-aws-common - - - io.netty - * - - - org.glassfish - javax.el - - - true - - - org.apache.druid - druid-processing - ${druid.version} - - - com.ibm.icu - icu4j - - - org.codehaus.plexus - plexus-utils - - - org.ow2.asm - asm-commons - - - org.ow2.asm - asm - - - org.checkerframework - checker-qual - - - - com.google.code.findbugs - annotations - - - true - - - org.apache.druid.extensions - druid-hdfs-storage - ${druid.version} - - - com.fasterxml.jackson.core - jackson-core - - - com.fasterxml.jackson.core - jackson-annotations - - - com.fasterxml.jackson.core - jackson-databind - - - org.apache.hadoop - hadoop-aws - - - com.amazonaws - aws-java-sdk-s3 - - - true - - - org.apache.druid.extensions - mysql-metadata-storage - ${druid.version} - - - mysql - mysql-connector-java - - - true - - - org.apache.druid.extensions - postgresql-metadata-storage - ${druid.version} - true - - - org.apache.druid.extensions - druid-bloom-filter - ${druid.version} - true - - - org.apache.hadoop - hadoop-common - provided - true - - - org.slf4j - slf4j-log4j12 - - - org.slf4j - slf4j-reload4j - - - ch.qos.reload4j - reload4j - - - com.google.guava - guava - - - commons-beanutils - commons-beanutils - - - commons-logging - commons-logging - - - com.fasterxml.jackson.core - jackson-databind - - - - - org.apache.hadoop - provided - hadoop-mapreduce-client-core - true - - - com.google.inject - guice - - - com.google.inject.extensions - guice-servlet - - - io.netty - * - - - com.google.guava - guava - - - com.fasterxml.jackson.core - jackson-core - - - com.fasterxml.jackson.core - jackson-annotations - - - com.fasterxml.jackson.core - jackson-databind - - - - - - org.mockito - mockito-core - ${mockito-core.version} - test - - - junit - junit - test - - - org.apache.druid - druid-indexing-hadoop - ${druid.version} - test - - - com.fasterxml.jackson.core - jackson-core - - - com.fasterxml.jackson.core - jackson-annotations - - - com.fasterxml.jackson.core - jackson-databind - - - - - org.apache.logging.log4j - log4j-api - ${log4j2.version} - test - - - - - dev-fast-build - - - skipShade - !true - - - - - - org.apache.maven.plugins - maven-shade-plugin - - - - - package - - shade - - - true - false - - - org.apache.druid - org.apache.hive.druid.org.apache.druid - - - io.netty - org.apache.hive.druid.io.netty - - - org.apache.calcite - org.apache.hive.druid.org.apache.calcite - - - org.jboss.netty - org.apache.hive.druid.org.jboss.netty - - - com.fasterxml.jackson - org.apache.hive.druid.com.fasterxml.jackson - - - com.google.common - org.apache.hive.druid.com.google.common - - - org.roaringbitmap - org.apache.hive.druid.org.roaringbitmap - - - - - org.apache.druid:* - org.apache.druid.extensions:* - io.netty:* - org.apache.calcite:* - com.fasterxml.jackson.core:* - com.fasterxml.jackson.datatype:* - com.fasterxml.jackson.dataformat:* - com.google.guava:* - it.unimi.dsi:* - org.jdbi:* - net.jpountz.lz4:* - org.apache.commons:* - org.roaringbitmap:* - org.apache.derby:* - org.asynchttpclient:* - org.antlr:* - - - - - *:* - - META-INF/*.SF - META-INF/*.DSA - META-INF/*.RSA - META-INF/versions/** - static/ - - - - - - - - - - - - - ${basedir}/src/java - ${basedir}/src/test - - - org.apache.maven.plugins - maven-jar-plugin - - - - test-jar - - - - - - - diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/DruidKafkaUtils.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/DruidKafkaUtils.java deleted file mode 100644 index fb6ce308fbc7..000000000000 --- a/druid-handler/src/java/org/apache/hadoop/hive/druid/DruidKafkaUtils.java +++ /dev/null @@ -1,232 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.druid; - -import com.fasterxml.jackson.core.type.TypeReference; -import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableMap; -import org.apache.druid.data.input.impl.CSVParseSpec; -import org.apache.druid.data.input.impl.DelimitedParseSpec; -import org.apache.druid.data.input.impl.DimensionsSpec; -import org.apache.druid.data.input.impl.InputRowParser; -import org.apache.druid.data.input.impl.JSONParseSpec; -import org.apache.druid.data.input.impl.StringInputRowParser; -import org.apache.druid.data.input.impl.TimestampSpec; -import org.apache.druid.java.util.http.client.Request; -import org.apache.druid.java.util.http.client.response.StringFullResponseHandler; -import org.apache.druid.java.util.http.client.response.StringFullResponseHolder; -import org.apache.druid.segment.IndexSpec; -import org.apache.druid.segment.indexing.DataSchema; -import org.apache.druid.segment.writeout.TmpFileSegmentWriteOutMediumFactory; -import org.apache.hadoop.hive.druid.conf.DruidConstants; -import org.apache.hadoop.hive.druid.json.AvroParseSpec; -import org.apache.hadoop.hive.druid.json.AvroStreamInputRowParser; -import org.apache.hadoop.hive.druid.json.InlineSchemaAvroBytesDecoder; -import org.apache.hadoop.hive.druid.json.KafkaSupervisorIOConfig; -import org.apache.hadoop.hive.druid.json.KafkaSupervisorSpec; -import org.apache.hadoop.hive.druid.json.KafkaSupervisorTuningConfig; -import org.apache.hadoop.hive.metastore.api.Table; -import org.apache.hadoop.hive.ql.session.SessionState; -import org.jboss.netty.handler.codec.http.HttpMethod; -import org.jboss.netty.handler.codec.http.HttpResponseStatus; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.net.URL; -import java.nio.charset.Charset; -import java.util.HashMap; -import java.util.Map; - -import static org.apache.hadoop.hive.druid.DruidStorageHandlerUtils.JSON_MAPPER; - -/** - * Class containing some Utility methods for Kafka Ingestion. - */ -final class DruidKafkaUtils { - private static final Logger LOG = LoggerFactory.getLogger(DruidKafkaUtils.class); - private static final SessionState.LogHelper CONSOLE = new SessionState.LogHelper(LOG); - - private DruidKafkaUtils() { - } - - static KafkaSupervisorSpec createKafkaSupervisorSpec( - Table table, - String kafkaTopic, - String kafkaServers, - DataSchema dataSchema, - IndexSpec indexSpec - ) { - return new KafkaSupervisorSpec(dataSchema, - new KafkaSupervisorTuningConfig(DruidStorageHandlerUtils.getIntegerProperty(table, - DruidConstants.DRUID_KAFKA_INGESTION_PROPERTY_PREFIX + "maxRowsInMemory"), - DruidStorageHandlerUtils.getLongProperty(table, - DruidConstants.DRUID_KAFKA_INGESTION_PROPERTY_PREFIX + "maxBytesInMemory"), - - DruidStorageHandlerUtils.getIntegerProperty(table, - DruidConstants.DRUID_KAFKA_INGESTION_PROPERTY_PREFIX + "maxRowsPerSegment"), DruidStorageHandlerUtils - .getLongProperty(table, DruidConstants.DRUID_KAFKA_INGESTION_PROPERTY_PREFIX + "maxTotalRows"), - DruidStorageHandlerUtils.getPeriodProperty(table, - DruidConstants.DRUID_KAFKA_INGESTION_PROPERTY_PREFIX + "intermediatePersistPeriod"), null, - // basePersistDirectory - use druid default, no need to be configured by user - DruidStorageHandlerUtils - .getIntegerProperty(table, DruidConstants.DRUID_KAFKA_INGESTION_PROPERTY_PREFIX + "maxPendingPersists"), - indexSpec, null, null, - // buildV9Directly - use druid default, no need to be configured by user - DruidStorageHandlerUtils.getBooleanProperty(table, - DruidConstants.DRUID_KAFKA_INGESTION_PROPERTY_PREFIX + "reportParseExceptions"), - DruidStorageHandlerUtils.getLongProperty(table, - DruidConstants.DRUID_KAFKA_INGESTION_PROPERTY_PREFIX + "handoffConditionTimeout"), - DruidStorageHandlerUtils.getBooleanProperty(table, - DruidConstants.DRUID_KAFKA_INGESTION_PROPERTY_PREFIX + "resetOffsetAutomatically"), - TmpFileSegmentWriteOutMediumFactory.instance(), DruidStorageHandlerUtils - .getIntegerProperty(table, DruidConstants.DRUID_KAFKA_INGESTION_PROPERTY_PREFIX + "workerThreads"), - DruidStorageHandlerUtils.getIntegerProperty(table, - DruidConstants.DRUID_KAFKA_INGESTION_PROPERTY_PREFIX + "chatThreads"), - DruidStorageHandlerUtils.getLongProperty(table, - DruidConstants.DRUID_KAFKA_INGESTION_PROPERTY_PREFIX + "chatRetries"), - DruidStorageHandlerUtils.getPeriodProperty(table, - DruidConstants.DRUID_KAFKA_INGESTION_PROPERTY_PREFIX + "httpTimeout"), - DruidStorageHandlerUtils.getPeriodProperty(table, - DruidConstants.DRUID_KAFKA_INGESTION_PROPERTY_PREFIX + "shutdownTimeout"), - DruidStorageHandlerUtils.getPeriodProperty(table, - DruidConstants.DRUID_KAFKA_INGESTION_PROPERTY_PREFIX + "offsetFetchPeriod"), - DruidStorageHandlerUtils.getPeriodProperty(table, - DruidConstants.DRUID_KAFKA_INGESTION_PROPERTY_PREFIX + "intermediateHandoffPeriod"), - DruidStorageHandlerUtils.getBooleanProperty(table, - DruidConstants.DRUID_KAFKA_INGESTION_PROPERTY_PREFIX + "logParseExceptions"), - DruidStorageHandlerUtils.getIntegerProperty(table, - DruidConstants.DRUID_KAFKA_INGESTION_PROPERTY_PREFIX + "maxParseExceptions"), - DruidStorageHandlerUtils.getIntegerProperty(table, - DruidConstants.DRUID_KAFKA_INGESTION_PROPERTY_PREFIX + "maxSavedParseExceptions")), - new KafkaSupervisorIOConfig(kafkaTopic, - // Mandatory Property - DruidStorageHandlerUtils.getIntegerProperty(table, - DruidConstants.DRUID_KAFKA_INGESTION_PROPERTY_PREFIX + "replicas"), - DruidStorageHandlerUtils.getIntegerProperty(table, - DruidConstants.DRUID_KAFKA_INGESTION_PROPERTY_PREFIX + "taskCount"), - DruidStorageHandlerUtils.getPeriodProperty(table, - DruidConstants.DRUID_KAFKA_INGESTION_PROPERTY_PREFIX + "taskDuration"), - getKafkaConsumerProperties(table, kafkaServers), - // Mandatory Property - DruidStorageHandlerUtils.getPeriodProperty(table, - DruidConstants.DRUID_KAFKA_INGESTION_PROPERTY_PREFIX + "startDelay"), - DruidStorageHandlerUtils.getPeriodProperty(table, - DruidConstants.DRUID_KAFKA_INGESTION_PROPERTY_PREFIX + "period"), - DruidStorageHandlerUtils.getBooleanProperty(table, - DruidConstants.DRUID_KAFKA_INGESTION_PROPERTY_PREFIX + "useEarliestOffset"), - DruidStorageHandlerUtils.getPeriodProperty(table, - DruidConstants.DRUID_KAFKA_INGESTION_PROPERTY_PREFIX + "completionTimeout"), - DruidStorageHandlerUtils.getPeriodProperty(table, - DruidConstants.DRUID_KAFKA_INGESTION_PROPERTY_PREFIX + "lateMessageRejectionPeriod"), - DruidStorageHandlerUtils.getPeriodProperty(table, - DruidConstants.DRUID_KAFKA_INGESTION_PROPERTY_PREFIX + "earlyMessageRejectionPeriod"), - DruidStorageHandlerUtils.getBooleanProperty(table, - DruidConstants.DRUID_KAFKA_INGESTION_PROPERTY_PREFIX + "skipOffsetGaps")), - new HashMap<>()); - } - - private static Map getKafkaConsumerProperties(Table table, String kafkaServers) { - ImmutableMap.Builder builder = ImmutableMap.builder(); - builder.put(KafkaSupervisorIOConfig.BOOTSTRAP_SERVERS_KEY, kafkaServers); - for (Map.Entry entry : table.getParameters().entrySet()) { - if (entry.getKey().startsWith(DruidConstants.DRUID_KAFKA_CONSUMER_PROPERTY_PREFIX)) { - String propertyName = entry.getKey().substring(DruidConstants.DRUID_KAFKA_CONSUMER_PROPERTY_PREFIX.length()); - builder.put(propertyName, entry.getValue()); - } - } - return builder.build(); - } - - static void updateKafkaIngestionSpec(String overlordAddress, KafkaSupervisorSpec spec) { - try { - String task = JSON_MAPPER.writeValueAsString(spec); - CONSOLE.printInfo("submitting kafka Spec {}", task); - LOG.info("submitting kafka Supervisor Spec {}", task); - StringFullResponseHolder response = DruidStorageHandlerUtils - .getResponseFromCurrentLeader(DruidStorageHandler.getHttpClient(), new Request(HttpMethod.POST, - new URL(String.format("http://%s/druid/indexer/v1/supervisor", overlordAddress))) - .setContent("application/json", JSON_MAPPER.writeValueAsBytes(spec)), - new StringFullResponseHandler(Charset.forName("UTF-8"))); - if (response.getStatus().equals(HttpResponseStatus.OK)) { - String - msg = - String.format("Kafka Supervisor for [%s] Submitted Successfully to druid.", - spec.getDataSchema().getDataSource()); - LOG.info(msg); - CONSOLE.printInfo(msg); - } else { - throw new IOException(String.format("Unable to update Kafka Ingestion for Druid status [%d] full response [%s]", - response.getStatus().getCode(), - response.getContent())); - } - } catch (Exception e) { - throw new RuntimeException(e); - } - } - - static boolean isKafkaStreamingTable(Table table) { - // For kafka Streaming tables it is mandatory to set a kafka topic. - return DruidStorageHandlerUtils.getTableProperty(table, DruidConstants.KAFKA_TOPIC) != null; - } - - static InputRowParser getInputRowParser(Table table, TimestampSpec timestampSpec, DimensionsSpec dimensionsSpec) { - String parseSpecFormat = DruidStorageHandlerUtils.getTableProperty(table, DruidConstants.DRUID_PARSE_SPEC_FORMAT); - - // Default case JSON - if ((parseSpecFormat == null) || "json".equalsIgnoreCase(parseSpecFormat)) { - return new StringInputRowParser(new JSONParseSpec(timestampSpec, dimensionsSpec, null, null), "UTF-8"); - } else if ("csv".equalsIgnoreCase(parseSpecFormat)) { - return new StringInputRowParser(new CSVParseSpec(timestampSpec, - dimensionsSpec, - DruidStorageHandlerUtils.getTableProperty(table, DruidConstants.DRUID_PARSE_SPEC_LIST_DELIMITER), - DruidStorageHandlerUtils.getListProperty(table, DruidConstants.DRUID_PARSE_SPEC_COLUMNS), - DruidStorageHandlerUtils.getBooleanProperty(table, DruidConstants.DRUID_PARSE_SPEC_HAS_HEADER_ROWS, false), - DruidStorageHandlerUtils.getIntegerProperty(table, DruidConstants.DRUID_PARSE_SPEC_SKIP_HEADER_ROWS, 0)), - "UTF-8"); - } else if ("delimited".equalsIgnoreCase(parseSpecFormat)) { - return new StringInputRowParser(new DelimitedParseSpec(timestampSpec, - dimensionsSpec, - DruidStorageHandlerUtils.getTableProperty(table, DruidConstants.DRUID_PARSE_SPEC_DELIMITER), - DruidStorageHandlerUtils.getTableProperty(table, DruidConstants.DRUID_PARSE_SPEC_LIST_DELIMITER), - DruidStorageHandlerUtils.getListProperty(table, DruidConstants.DRUID_PARSE_SPEC_COLUMNS), - DruidStorageHandlerUtils.getBooleanProperty(table, DruidConstants.DRUID_PARSE_SPEC_HAS_HEADER_ROWS, false), - DruidStorageHandlerUtils.getIntegerProperty(table, DruidConstants.DRUID_PARSE_SPEC_SKIP_HEADER_ROWS, 0)), - "UTF-8"); - } else if ("avro".equalsIgnoreCase(parseSpecFormat)) { - try { - String avroSchemaLiteral = DruidStorageHandlerUtils.getTableProperty(table, DruidConstants.AVRO_SCHEMA_LITERAL); - Preconditions.checkNotNull(avroSchemaLiteral, "Please specify avro schema literal when using avro parser"); - Map - avroSchema = - JSON_MAPPER.readValue(avroSchemaLiteral, new TypeReference>() { - }); - return new AvroStreamInputRowParser(new AvroParseSpec(timestampSpec, dimensionsSpec, null), - new InlineSchemaAvroBytesDecoder(avroSchema)); - } catch (Exception e) { - throw new IllegalStateException("Exception while creating avro schema", e); - } - } - - throw new IllegalArgumentException("Invalid parse spec format [" - + parseSpecFormat - + "]. " - + "Supported types are : json, csv, tsv, avro"); - } -} diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/DruidStorageHandler.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/DruidStorageHandler.java deleted file mode 100644 index 656fa40c03fa..000000000000 --- a/druid-handler/src/java/org/apache/hadoop/hive/druid/DruidStorageHandler.java +++ /dev/null @@ -1,1020 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.druid; - -import com.fasterxml.jackson.core.type.TypeReference; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; -import com.google.common.base.Strings; -import com.google.common.base.Supplier; -import com.google.common.base.Suppliers; -import com.google.common.base.Throwables; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Maps; -import com.google.common.collect.Sets; -import org.apache.druid.data.input.impl.DimensionSchema; -import org.apache.druid.data.input.impl.DimensionsSpec; -import org.apache.druid.data.input.impl.InputRowParser; -import org.apache.druid.data.input.impl.TimestampSpec; -import org.apache.druid.java.util.common.Pair; -import org.apache.druid.java.util.common.RetryUtils; -import org.apache.druid.java.util.common.lifecycle.Lifecycle; -import org.apache.druid.java.util.http.client.HttpClient; -import org.apache.druid.java.util.http.client.HttpClientConfig; -import org.apache.druid.java.util.http.client.HttpClientInit; -import org.apache.druid.java.util.http.client.Request; -import org.apache.druid.java.util.http.client.response.StringFullResponseHandler; -import org.apache.druid.java.util.http.client.response.StringFullResponseHolder; -import org.apache.druid.metadata.MetadataStorageConnectorConfig; -import org.apache.druid.metadata.MetadataStorageTablesConfig; -import org.apache.druid.metadata.SQLMetadataConnector; -import org.apache.druid.metadata.storage.derby.DerbyConnector; -import org.apache.druid.metadata.storage.derby.DerbyMetadataStorage; -import org.apache.druid.metadata.storage.mysql.MySQLConnector; -import org.apache.druid.metadata.storage.mysql.MySQLConnectorConfig; -import org.apache.druid.metadata.storage.postgresql.PostgreSQLConnector; -import org.apache.druid.metadata.storage.postgresql.PostgreSQLConnectorConfig; -import org.apache.druid.metadata.storage.postgresql.PostgreSQLTablesConfig; -import org.apache.druid.query.BaseQuery; -import org.apache.druid.query.Query; -import org.apache.druid.query.aggregation.AggregatorFactory; -import org.apache.druid.segment.IndexSpec; -import org.apache.druid.segment.indexing.DataSchema; -import org.apache.druid.segment.indexing.granularity.GranularitySpec; -import org.apache.druid.segment.loading.DataSegmentPusher; -import org.apache.druid.segment.loading.SegmentLoadingException; -import org.apache.druid.storage.hdfs.HdfsDataSegmentPusher; -import org.apache.druid.storage.hdfs.HdfsDataSegmentPusherConfig; -import org.apache.druid.timeline.DataSegment; -import org.apache.commons.lang3.StringUtils; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hive.common.TableName; -import org.apache.hadoop.hive.conf.Constants; -import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.druid.conf.DruidConstants; -import org.apache.hadoop.hive.druid.io.DruidOutputFormat; -import org.apache.hadoop.hive.druid.io.DruidQueryBasedInputFormat; -import org.apache.hadoop.hive.druid.io.DruidRecordWriter; -import org.apache.hadoop.hive.druid.json.KafkaSupervisorReport; -import org.apache.hadoop.hive.druid.json.KafkaSupervisorSpec; -import org.apache.hadoop.hive.druid.security.KerberosHttpClient; -import org.apache.hadoop.hive.druid.serde.DruidSerDe; -import org.apache.hadoop.hive.metastore.DefaultHiveMetaHook; -import org.apache.hadoop.hive.metastore.HiveMetaHook; -import org.apache.hadoop.hive.metastore.api.EnvironmentContext; -import org.apache.hadoop.hive.metastore.api.FieldSchema; -import org.apache.hadoop.hive.metastore.api.LockType; -import org.apache.hadoop.hive.metastore.api.MetaException; -import org.apache.hadoop.hive.metastore.api.Table; -import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils; -import org.apache.hadoop.hive.ql.exec.Utilities; -import org.apache.hadoop.hive.ql.hooks.WriteEntity; -import org.apache.hadoop.hive.ql.metadata.HiveStorageHandler; -import org.apache.hadoop.hive.ql.metadata.StorageHandlerInfo; -import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc; -import org.apache.hadoop.hive.ql.plan.OperatorDesc; -import org.apache.hadoop.hive.ql.plan.TableDesc; -import org.apache.hadoop.hive.ql.plan.TableScanDesc; -import org.apache.hadoop.hive.ql.security.authorization.DefaultHiveAuthorizationProvider; -import org.apache.hadoop.hive.ql.security.authorization.HiveAuthorizationProvider; -import org.apache.hadoop.hive.ql.security.authorization.HiveCustomStorageHandlerUtils; -import org.apache.hadoop.hive.ql.session.SessionState; -import org.apache.hadoop.hive.serde2.AbstractSerDe; -import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; -import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; -import org.apache.hadoop.mapred.InputFormat; -import org.apache.hadoop.mapred.JobConf; -import org.apache.hadoop.mapred.OutputFormat; -import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hive.common.util.ShutdownHookManager; -import org.jboss.netty.handler.codec.http.HttpMethod; -import org.jboss.netty.handler.codec.http.HttpResponseStatus; -import org.joda.time.DateTime; -import org.joda.time.Period; -import org.skife.jdbi.v2.exceptions.CallbackFailedException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import javax.annotation.Nullable; -import java.io.IOException; -import java.net.MalformedURLException; -import java.net.URL; -import java.net.URI; -import java.net.URISyntaxException; -import java.nio.charset.Charset; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Properties; -import java.util.Set; -import java.util.concurrent.ExecutionException; -import java.util.stream.Collectors; - -import static org.apache.hadoop.hive.druid.DruidStorageHandlerUtils.JSON_MAPPER; - -/** - * DruidStorageHandler provides a HiveStorageHandler implementation for Druid. - */ -@SuppressWarnings({ "rawtypes" }) public class DruidStorageHandler extends DefaultHiveMetaHook - implements HiveStorageHandler { - - private static final Logger LOG = LoggerFactory.getLogger(DruidStorageHandler.class); - - private static final SessionState.LogHelper CONSOLE = new SessionState.LogHelper(LOG); - - public static final String SEGMENTS_DESCRIPTOR_DIR_NAME = "segmentsDescriptorDir"; - - private static final String INTERMEDIATE_SEGMENT_DIR_NAME = "intermediateSegmentDir"; - - private static final HttpClient HTTP_CLIENT; - - private static final List ALLOWED_ALTER_TYPES = - ImmutableList.of("ADDPROPS", "DROPPROPS", "ADDCOLS"); - - /** Druid prefix to form the URI for authentication */ - private static final String DRUID_PREFIX = "druid:"; - /** Druid config for determining the host name */ - private static final String DRUID_HOST_NAME = "druid.zk.service.host"; - - static { - final Lifecycle lifecycle = new Lifecycle(); - try { - lifecycle.start(); - } catch (Exception e) { - LOG.error("Issues with lifecycle start", e); - } - HTTP_CLIENT = makeHttpClient(lifecycle); - ShutdownHookManager.addShutdownHook(lifecycle::stop); - } - - private SQLMetadataConnector connector; - - private MetadataStorageTablesConfig druidMetadataStorageTablesConfig = null; - - private String uniqueId = null; - - private String rootWorkingDir = null; - - private Configuration conf; - - public DruidStorageHandler() { - } - - @VisibleForTesting public DruidStorageHandler(SQLMetadataConnector connector, - MetadataStorageTablesConfig druidMetadataStorageTablesConfig) { - this.connector = connector; - this.druidMetadataStorageTablesConfig = druidMetadataStorageTablesConfig; - } - - @Override public Class getInputFormatClass() { - return DruidQueryBasedInputFormat.class; - } - - @Override public Class getOutputFormatClass() { - return DruidOutputFormat.class; - } - - @Override public Class getSerDeClass() { - return DruidSerDe.class; - } - - @Override public HiveMetaHook getMetaHook() { - return this; - } - - @Override public HiveAuthorizationProvider getAuthorizationProvider() { - return new DefaultHiveAuthorizationProvider(); - } - - @Override public void configureInputJobProperties(TableDesc tableDesc, Map jobProperties) { - - } - - @Override public void configureInputJobCredentials(TableDesc tableDesc, Map jobSecrets) { - - } - - @Override public void preCreateTable(Table table) throws MetaException { - if (!StringUtils.isEmpty(table.getSd().getLocation())) { - throw new MetaException("LOCATION may not be specified for Druid"); - } - if (table.getPartitionKeysSize() != 0) { - throw new MetaException("PARTITIONED BY may not be specified for Druid"); - } - if (table.getSd().getBucketColsSize() != 0) { - throw new MetaException("CLUSTERED BY may not be specified for Druid"); - } - String dataSourceName = table.getParameters().get(Constants.DRUID_DATA_SOURCE); - if (dataSourceName != null) { - // Already Existing datasource in Druid. - return; - } - - // create dataSourceName based on Hive Table name - dataSourceName = TableName.getDbTable(table.getDbName(), table.getTableName()); - try { - // NOTE: This just created druid_segments table in Druid metastore. - // This is needed for the case when hive is started before any of druid services - // and druid_segments table has not been created yet. - getConnector().createSegmentTable(); - } catch (Exception e) { - LOG.error("Exception while trying to create druid segments table", e); - throw new MetaException(e.getMessage()); - } - Collection - existingDataSources = - DruidStorageHandlerUtils.getAllDataSourceNames(getConnector(), getDruidMetadataStorageTablesConfig()); - LOG.debug("pre-create data source with name {}", dataSourceName); - // Check for existence of for the datasource we are going to create in druid_segments table. - if (existingDataSources.contains(dataSourceName)) { - throw new MetaException(String.format("Data source [%s] already existing", dataSourceName)); - } - table.getParameters().put(Constants.DRUID_DATA_SOURCE, dataSourceName); - } - - @Override public void rollbackCreateTable(Table table) { - cleanWorkingDir(); - } - - @Override public void commitCreateTable(Table table) throws MetaException { - if (DruidKafkaUtils.isKafkaStreamingTable(table)) { - updateKafkaIngestion(table); - } - // For CTAS queries when user has explicitly specified the datasource. - // We will append the data to existing druid datasource. - this.commitInsertTable(table, false); - } - - @Override - public URI getURIForAuth(Table table) throws URISyntaxException{ - Map tableProperties = HiveCustomStorageHandlerUtils.getTableProperties(table); - String host_name = conf.get(DRUID_HOST_NAME) != null ? conf.get(DRUID_HOST_NAME) : - HiveConf.getVar(getConf(), HiveConf.ConfVars.HIVE_DRUID_BROKER_DEFAULT_ADDRESS); - String table_name = tableProperties.get(Constants.DRUID_DATA_SOURCE); - String column_names = tableProperties.get(Constants.DRUID_QUERY_FIELD_NAMES); - if (column_names != null) - return new URI(DRUID_PREFIX+"//"+host_name+"/"+table_name+"/"+column_names); - else - return new URI(DRUID_PREFIX+"//"+host_name+"/"+table_name); - } - - private void updateKafkaIngestion(Table table) { - final String overlordAddress = HiveConf.getVar(getConf(), HiveConf.ConfVars.HIVE_DRUID_OVERLORD_DEFAULT_ADDRESS); - - final String - dataSourceName = - Preconditions.checkNotNull(DruidStorageHandlerUtils.getTableProperty(table, Constants.DRUID_DATA_SOURCE), - "Druid datasource name is null"); - - final String - kafkaTopic = - Preconditions.checkNotNull(DruidStorageHandlerUtils.getTableProperty(table, - DruidConstants.KAFKA_TOPIC), "kafka topic is null"); - final String - kafkaServers = - Preconditions.checkNotNull(DruidStorageHandlerUtils.getTableProperty(table, - DruidConstants.KAFKA_BOOTSTRAP_SERVERS), "kafka connect string is null"); - - Properties tableProperties = new Properties(); - tableProperties.putAll(table.getParameters()); - - final GranularitySpec granularitySpec = DruidStorageHandlerUtils.getGranularitySpec(getConf(), tableProperties); - - List columns = table.getSd().getCols(); - List columnNames = new ArrayList<>(columns.size()); - List columnTypes = new ArrayList<>(columns.size()); - - for (FieldSchema schema : columns) { - columnNames.add(schema.getName()); - columnTypes.add(TypeInfoUtils.getTypeInfoFromTypeString(schema.getType())); - } - - Pair, AggregatorFactory[]> - dimensionsAndAggregates = - DruidStorageHandlerUtils.getDimensionsAndAggregates(columnNames, columnTypes); - if (!columnNames.contains(DruidConstants.DEFAULT_TIMESTAMP_COLUMN)) { - throw new IllegalStateException("Timestamp column (' " - + DruidConstants.DEFAULT_TIMESTAMP_COLUMN - + "') not specified in create table; list of columns is : " - + columnNames); - } - - DimensionsSpec dimensionsSpec = new DimensionsSpec(dimensionsAndAggregates.lhs, null, null); - String timestampFormat = DruidStorageHandlerUtils - .getTableProperty(table, DruidConstants.DRUID_TIMESTAMP_FORMAT); - String timestampColumnName = DruidStorageHandlerUtils - .getTableProperty(table, DruidConstants.DRUID_TIMESTAMP_COLUMN); - if(timestampColumnName == null) { - timestampColumnName = DruidConstants.DEFAULT_TIMESTAMP_COLUMN; - } - final TimestampSpec timestampSpec = new TimestampSpec(timestampColumnName, timestampFormat, - null - ); - final InputRowParser inputRowParser = DruidKafkaUtils - .getInputRowParser(table, timestampSpec, dimensionsSpec); - - final Map - inputParser = - JSON_MAPPER.convertValue(inputRowParser, new TypeReference>() { - }); - final DataSchema - dataSchema = - new DataSchema(dataSourceName, - inputParser, - dimensionsAndAggregates.rhs, - granularitySpec, - null, - DruidStorageHandlerUtils.JSON_MAPPER); - - IndexSpec indexSpec = DruidStorageHandlerUtils.getIndexSpec(getConf()); - - KafkaSupervisorSpec - spec = - DruidKafkaUtils.createKafkaSupervisorSpec(table, kafkaTopic, kafkaServers, dataSchema, indexSpec); - - // Fetch existing Ingestion Spec from Druid, if any - KafkaSupervisorSpec existingSpec = fetchKafkaIngestionSpec(table); - String - targetState = - DruidStorageHandlerUtils.getTableProperty(table, DruidConstants.DRUID_KAFKA_INGESTION); - if (targetState == null) { - // Case when user has not specified any ingestion state in the current command - // if there is a kafka supervisor running then keep it last known state is START otherwise STOP. - targetState = existingSpec == null ? "STOP" : "START"; - } - - if ("STOP".equalsIgnoreCase(targetState)) { - if (existingSpec != null) { - stopKafkaIngestion(overlordAddress, dataSourceName); - } - } else if ("START".equalsIgnoreCase(targetState)) { - if (existingSpec == null || !existingSpec.equals(spec)) { - DruidKafkaUtils.updateKafkaIngestionSpec(overlordAddress, spec); - } - } else if ("RESET".equalsIgnoreCase(targetState)) { - // Case when there are changes in multiple table properties. - if (existingSpec != null && !existingSpec.equals(spec)) { - DruidKafkaUtils.updateKafkaIngestionSpec(overlordAddress, spec); - } - resetKafkaIngestion(overlordAddress, dataSourceName); - } else { - throw new IllegalArgumentException(String.format( - "Invalid value for property [%s], Valid values are [START, STOP, RESET]", - DruidConstants.DRUID_KAFKA_INGESTION)); - } - // We do not want to keep state in two separate places so remove from hive table properties. - table.getParameters().remove(DruidConstants.DRUID_KAFKA_INGESTION); - } - - private void resetKafkaIngestion(String overlordAddress, String dataSourceName) { - try { - StringFullResponseHolder - response = - RetryUtils.retry(() -> DruidStorageHandlerUtils.getResponseFromCurrentLeader(getHttpClient(), - new Request(HttpMethod.POST, new URL( - String.format("http://%s/druid/indexer/v1/supervisor/%s/reset", overlordAddress, dataSourceName))), - new StringFullResponseHandler(Charset.forName("UTF-8"))), input -> input instanceof IOException, - getMaxRetryCount()); - if (response.getStatus().equals(HttpResponseStatus.OK)) { - CONSOLE.printInfo("Druid Kafka Ingestion Reset successful."); - } else { - throw new IOException(String.format("Unable to reset Kafka Ingestion Druid status [%d] full response [%s]", - response.getStatus().getCode(), - response.getContent())); - } - } catch (Exception e) { - throw new RuntimeException(e); - } - } - - private void stopKafkaIngestion(String overlordAddress, String dataSourceName) { - try { - StringFullResponseHolder - response = - RetryUtils.retry(() -> DruidStorageHandlerUtils.getResponseFromCurrentLeader(getHttpClient(), - new Request(HttpMethod.POST, new URL( - String.format("http://%s/druid/indexer/v1/supervisor/%s/shutdown", overlordAddress, dataSourceName))), - new StringFullResponseHandler(Charset.forName("UTF-8"))), input -> input instanceof IOException, - getMaxRetryCount()); - if (response.getStatus().equals(HttpResponseStatus.OK)) { - CONSOLE.printInfo("Druid Kafka Ingestion shutdown successful."); - } else { - throw new IOException(String.format("Unable to stop Kafka Ingestion Druid status [%d] full response [%s]", - response.getStatus().getCode(), - response.getContent())); - } - } catch (Exception e) { - throw new RuntimeException(e); - } - - } - - private KafkaSupervisorSpec fetchKafkaIngestionSpec(Table table) { - // Stop Kafka Ingestion first - final String - overlordAddress = - Preconditions.checkNotNull(HiveConf.getVar(getConf(), HiveConf.ConfVars.HIVE_DRUID_OVERLORD_DEFAULT_ADDRESS), - "Druid Overlord Address is null"); - String - dataSourceName = - Preconditions.checkNotNull(DruidStorageHandlerUtils.getTableProperty(table, Constants.DRUID_DATA_SOURCE), - "Druid Datasource name is null"); - try { - StringFullResponseHolder - response = - RetryUtils.retry(() -> DruidStorageHandlerUtils.getResponseFromCurrentLeader(getHttpClient(), - new Request(HttpMethod.GET, - new URL(String.format("http://%s/druid/indexer/v1/supervisor/%s", overlordAddress, dataSourceName))), - new StringFullResponseHandler(Charset.forName("UTF-8"))), input -> input instanceof IOException, - getMaxRetryCount()); - if (response.getStatus().equals(HttpResponseStatus.OK)) { - return JSON_MAPPER.readValue(response.getContent(), KafkaSupervisorSpec.class); - // Druid Returns 400 Bad Request when not found. - } else if (response.getStatus().equals(HttpResponseStatus.NOT_FOUND) || response.getStatus() - .equals(HttpResponseStatus.BAD_REQUEST)) { - LOG.debug("No Kafka Supervisor found for datasource[%s]", dataSourceName); - return null; - } else { - throw new IOException(String.format( - "Unable to fetch Kafka Ingestion Spec from Druid status [%d] full response [%s]", - response.getStatus().getCode(), - response.getContent())); - } - } catch (Exception e) { - throw new RuntimeException("Exception while fetching kafka ingestion spec from druid", e); - } - } - - /** - * Fetches kafka supervisor status report from druid overlord. This method will return null if can not fetch report - * - * @param table object. - * @return kafka supervisor report or null when druid overlord is unreachable. - */ - @Nullable private KafkaSupervisorReport fetchKafkaSupervisorReport(Table table) { - final String - overlordAddress = - Preconditions.checkNotNull(HiveConf.getVar(getConf(), HiveConf.ConfVars.HIVE_DRUID_OVERLORD_DEFAULT_ADDRESS), - "Druid Overlord Address is null"); - final String - dataSourceName = - Preconditions.checkNotNull(DruidStorageHandlerUtils.getTableProperty(table, Constants.DRUID_DATA_SOURCE), - "Druid Datasource name is null"); - try { - StringFullResponseHolder - response = - RetryUtils.retry(() -> DruidStorageHandlerUtils.getResponseFromCurrentLeader(getHttpClient(), - new Request(HttpMethod.GET, new URL( - String.format("http://%s/druid/indexer/v1/supervisor/%s/status", overlordAddress, dataSourceName))), - new StringFullResponseHandler(Charset.forName("UTF-8"))), input -> input instanceof IOException, - getMaxRetryCount()); - if (response.getStatus().equals(HttpResponseStatus.OK)) { - return DruidStorageHandlerUtils.JSON_MAPPER.readValue(response.getContent(), KafkaSupervisorReport.class); - // Druid Returns 400 Bad Request when not found. - } else if (response.getStatus().equals(HttpResponseStatus.NOT_FOUND) || response.getStatus() - .equals(HttpResponseStatus.BAD_REQUEST)) { - LOG.info("No Kafka Supervisor found for datasource[%s]", dataSourceName); - return null; - } else { - LOG.error("Unable to fetch Kafka Supervisor status [%d] full response [%s]", - response.getStatus().getCode(), - response.getContent()); - return null; - } - } catch (Exception e) { - LOG.error("Exception while fetching kafka ingestion spec from druid", e); - return null; - } - } - - /** - * Creates metadata moves then commit the Segment's metadata to Druid metadata store in one TxN. - * - * @param table Hive table - * @param overwrite true if it is an insert overwrite table. - */ - private List loadAndCommitDruidSegments(Table table, boolean overwrite, List segmentsToLoad) - throws IOException, CallbackFailedException { - final String dataSourceName = table.getParameters().get(Constants.DRUID_DATA_SOURCE); - final String - segmentDirectory = - table.getParameters().get(DruidConstants.DRUID_SEGMENT_DIRECTORY) != null ? - table.getParameters().get(DruidConstants.DRUID_SEGMENT_DIRECTORY) : - HiveConf.getVar(getConf(), HiveConf.ConfVars.DRUID_SEGMENT_DIRECTORY); - - final HdfsDataSegmentPusherConfig hdfsSegmentPusherConfig = new HdfsDataSegmentPusherConfig(); - List publishedDataSegmentList; - - LOG.info(String.format("Moving [%s] Druid segments from staging directory [%s] to Deep storage [%s]", - segmentsToLoad.size(), - getStagingWorkingDir().toString(), - segmentDirectory)); - hdfsSegmentPusherConfig.setStorageDirectory(segmentDirectory); - DataSegmentPusher dataSegmentPusher = new HdfsDataSegmentPusher(hdfsSegmentPusherConfig, getConf(), JSON_MAPPER); - publishedDataSegmentList = - DruidStorageHandlerUtils.publishSegmentsAndCommit(getConnector(), - getDruidMetadataStorageTablesConfig(), - dataSourceName, - segmentsToLoad, - overwrite, - getConf(), - dataSegmentPusher); - return publishedDataSegmentList; - } - - /** - * This function checks the load status of Druid segments by polling druid coordinator. - * @param segments List of druid segments to check for - */ - private void checkLoadStatus(List segments) { - final String - coordinatorAddress = - HiveConf.getVar(getConf(), HiveConf.ConfVars.HIVE_DRUID_COORDINATOR_DEFAULT_ADDRESS); - int maxTries = getMaxRetryCount(); - - LOG.debug("checking load status from coordinator {}", coordinatorAddress); - - String coordinatorResponse; - try { - coordinatorResponse = - RetryUtils.retry(() -> DruidStorageHandlerUtils.getResponseFromCurrentLeader(getHttpClient(), - new Request(HttpMethod.GET, new URL(String.format("http://%s/status", coordinatorAddress))), - new StringFullResponseHandler(Charset.forName("UTF-8"))).getContent(), - input -> input instanceof IOException, maxTries); - } catch (Exception e) { - CONSOLE.printInfo("Will skip waiting for data loading, coordinator unavailable"); - return; - } - if (Strings.isNullOrEmpty(coordinatorResponse)) { - CONSOLE.printInfo("Will skip waiting for data loading empty response from coordinator"); - } - CONSOLE.printInfo(String.format("Waiting for the loading of [%s] segments", segments.size())); - long passiveWaitTimeMs = HiveConf.getLongVar(getConf(), HiveConf.ConfVars.HIVE_DRUID_PASSIVE_WAIT_TIME); - Set urlsOfUnloadedSegments = segments - .stream() - .map(dataSegment -> { - try { - //Need to make sure that we are using segment identifier - return new URL(String.format("http://%s/druid/coordinator/v1/datasources/%s/segments/%s", - coordinatorAddress, - dataSegment.getDataSource(), - dataSegment.getId().toString())); - } catch (MalformedURLException e) { - Throwables.propagate(e); - } - return null; - }) - .collect(Collectors.toSet()); - - int numRetries = 0; - while (numRetries++ < maxTries && !urlsOfUnloadedSegments.isEmpty()) { - urlsOfUnloadedSegments = ImmutableSet.copyOf(Sets.filter(urlsOfUnloadedSegments, input -> { - try { - String result = DruidStorageHandlerUtils - .getResponseFromCurrentLeader(getHttpClient(), new Request(HttpMethod.GET, input), - new StringFullResponseHandler(Charset.forName("UTF-8"))).getContent(); - - LOG.debug("Checking segment [{}] response is [{}]", input, result); - return Strings.isNullOrEmpty(result); - } catch (InterruptedException | ExecutionException e) { - LOG.error(String.format("Error while checking URL [%s]", input), e); - return true; - } - })); - - try { - if (!urlsOfUnloadedSegments.isEmpty()) { - Thread.sleep(passiveWaitTimeMs); - } - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - } - if (!urlsOfUnloadedSegments.isEmpty()) { - // We are not Throwing an exception since it might be a transient issue that is blocking loading - CONSOLE.printError(String.format("Wait time exhausted and we have [%s] out of [%s] segments not loaded yet", - urlsOfUnloadedSegments.size(), - segments.size())); - } - } - - @VisibleForTesting void deleteSegment(DataSegment segment) throws SegmentLoadingException { - - final Path path = DruidStorageHandlerUtils.getPath(segment); - LOG.info("removing segment {}, located at path {}", segment.getId().toString(), path); - - try { - if (path.getName().endsWith(".zip")) { - - final FileSystem fs = path.getFileSystem(getConf()); - - if (!fs.exists(path)) { - LOG.warn("Segment Path {} does not exist. It appears to have been deleted already.", path); - return; - } - - // path format -- > .../dataSource/interval/version/partitionNum/xxx.zip - Path partitionNumDir = path.getParent(); - if (!fs.delete(partitionNumDir, true)) { - throw new SegmentLoadingException("Unable to kill segment, failed to delete dir [%s]", - partitionNumDir.toString()); - } - - //try to delete other directories if possible - Path versionDir = partitionNumDir.getParent(); - if (safeNonRecursiveDelete(fs, versionDir)) { - Path intervalDir = versionDir.getParent(); - if (safeNonRecursiveDelete(fs, intervalDir)) { - Path dataSourceDir = intervalDir.getParent(); - safeNonRecursiveDelete(fs, dataSourceDir); - } - } - } else { - throw new SegmentLoadingException("Unknown file type[%s]", path); - } - } catch (IOException e) { - throw new SegmentLoadingException(e, "Unable to kill segment"); - } - } - - private static boolean safeNonRecursiveDelete(FileSystem fs, Path path) { - try { - return fs.delete(path, false); - } catch (Exception ex) { - return false; - } - } - - @Override public void preDropTable(Table table) { - // Nothing to do - } - - @Override public void rollbackDropTable(Table table) { - // Nothing to do - } - - @Override public void commitDropTable(Table table, boolean deleteData) { - if (DruidKafkaUtils.isKafkaStreamingTable(table)) { - // Stop Kafka Ingestion first - final String - overlordAddress = - Preconditions.checkNotNull(HiveConf.getVar(getConf(), HiveConf.ConfVars.HIVE_DRUID_OVERLORD_DEFAULT_ADDRESS), - "Druid Overlord Address is null"); - String - dataSourceName = - Preconditions.checkNotNull(DruidStorageHandlerUtils.getTableProperty(table, Constants.DRUID_DATA_SOURCE), - "Druid Datasource name is null"); - stopKafkaIngestion(overlordAddress, dataSourceName); - } - - String - dataSourceName = - Preconditions.checkNotNull(table.getParameters().get(Constants.DRUID_DATA_SOURCE), "DataSource name is null !"); - // Move MetaStoreUtils.isExternalTablePurge(table) calls to a common place for all StorageHandlers - // deleteData flag passed down to StorageHandler should be true only if - // MetaStoreUtils.isExternalTablePurge(table) returns true. - if (deleteData && MetaStoreUtils.isExternalTablePurge(table)) { - LOG.info("Dropping with purge all the data for data source {}", dataSourceName); - List - dataSegmentList = - DruidStorageHandlerUtils.getDataSegmentList(getConnector(), - getDruidMetadataStorageTablesConfig(), - dataSourceName); - if (dataSegmentList.isEmpty()) { - LOG.info("Nothing to delete for data source {}", dataSourceName); - return; - } - for (DataSegment dataSegment : dataSegmentList) { - try { - deleteSegment(dataSegment); - } catch (SegmentLoadingException e) { - LOG.error(String.format("Error while deleting segment [%s]", dataSegment.getId().toString()), e); - } - } - } - if (DruidStorageHandlerUtils.disableDataSource(getConnector(), - getDruidMetadataStorageTablesConfig(), - dataSourceName)) { - LOG.info("Successfully dropped druid data source {}", dataSourceName); - } - } - - @Override public void commitInsertTable(Table table, boolean overwrite) throws MetaException { - LOG.debug("commit insert into table {} overwrite {}", table.getTableName(), overwrite); - try { - // Check if there segments to load - final Path segmentDescriptorDir = getSegmentDescriptorDir(); - final List segmentsToLoad = fetchSegmentsMetadata(segmentDescriptorDir); - final String dataSourceName = table.getParameters().get(Constants.DRUID_DATA_SOURCE); - //No segments to load still need to honer overwrite - if (segmentsToLoad.isEmpty() && overwrite) { - //disable datasource - //Case it is an insert overwrite we have to disable the existing Druid DataSource - DruidStorageHandlerUtils.disableDataSource(getConnector(), - getDruidMetadataStorageTablesConfig(), - dataSourceName); - } else if (!segmentsToLoad.isEmpty()) { - // at this point we have Druid segments from reducers but we need to atomically - // rename and commit to metadata - // Moving Druid segments and committing to druid metadata as one transaction. - checkLoadStatus(loadAndCommitDruidSegments(table, overwrite, segmentsToLoad)); - } - } catch (IOException e) { - throw new MetaException(e.getMessage()); - } catch (CallbackFailedException c) { - LOG.error("Error while committing transaction to druid metadata storage", c); - throw new MetaException(c.getCause().getMessage()); - } finally { - cleanWorkingDir(); - } - } - - private List fetchSegmentsMetadata(Path segmentDescriptorDir) throws IOException { - if (!segmentDescriptorDir.getFileSystem(getConf()).exists(segmentDescriptorDir)) { - LOG.info("Directory {} does not exist, ignore this if it is create statement or inserts of 0 rows," - + " no Druid segments to move, cleaning working directory {}", - segmentDescriptorDir.toString(), - getStagingWorkingDir().toString()); - return Collections.emptyList(); - } - return DruidStorageHandlerUtils.getCreatedSegments(segmentDescriptorDir, getConf()); - } - - @Override public void preInsertTable(Table table, boolean overwrite) { - - } - - @Override public void rollbackInsertTable(Table table, boolean overwrite) { - // do nothing - } - - @Override public void configureOutputJobProperties(TableDesc tableDesc, Map jobProperties) { - jobProperties.put(Constants.DRUID_DATA_SOURCE, tableDesc.getTableName()); - jobProperties.put(DruidConstants.DRUID_SEGMENT_VERSION, new DateTime().toString()); - jobProperties.put(DruidConstants.DRUID_JOB_WORKING_DIRECTORY, getStagingWorkingDir().toString()); - // DruidOutputFormat will write segments in an intermediate directory - jobProperties.put(DruidConstants.DRUID_SEGMENT_INTERMEDIATE_DIRECTORY, - getIntermediateSegmentDir().toString()); - } - - @Override public void configureTableJobProperties(TableDesc tableDesc, Map jobProperties) { - - } - - @Override public void configureJobConf(TableDesc tableDesc, JobConf jobConf) { - final boolean kerberosEnabled = HiveConf.getBoolVar(getConf(), HiveConf.ConfVars.HIVE_DRUID_KERBEROS_ENABLE); - if (kerberosEnabled && UserGroupInformation.isSecurityEnabled()) { - // AM can not do Kerberos Auth so will do the input split generation in the HS2 - LOG.debug("Setting {} to {} to enable split generation on HS2", - HiveConf.ConfVars.HIVE_AM_SPLIT_GENERATION.toString(), - Boolean.FALSE.toString()); - jobConf.set(HiveConf.ConfVars.HIVE_AM_SPLIT_GENERATION.toString(), Boolean.FALSE.toString()); - } - try { - Utilities.addDependencyJars(jobConf, DruidRecordWriter.class); - } catch (IOException e) { - Throwables.propagate(e); - } - } - - @Override public void setConf(Configuration conf) { - this.conf = conf; - } - - @Override public Configuration getConf() { - return conf; - } - - @Override public LockType getLockType(WriteEntity writeEntity) { - if (writeEntity.getWriteType().equals(WriteEntity.WriteType.INSERT)) { - return LockType.SHARED_READ; - } - return LockType.SHARED_WRITE; - } - - @Override public String toString() { - return Constants.DRUID_HIVE_STORAGE_HANDLER_ID; - } - - private String getUniqueId() { - if (uniqueId == null) { - uniqueId = - Preconditions.checkNotNull(Strings.emptyToNull(HiveConf.getVar(getConf(), HiveConf.ConfVars.HIVE_QUERY_ID)), - "Hive query id is null"); - } - return uniqueId; - } - - private Path getStagingWorkingDir() { - return new Path(getRootWorkingDir(), makeStagingName()); - } - - private MetadataStorageTablesConfig getDruidMetadataStorageTablesConfig() { - if (druidMetadataStorageTablesConfig != null) { - return druidMetadataStorageTablesConfig; - } - final String base = HiveConf.getVar(getConf(), HiveConf.ConfVars.DRUID_METADATA_BASE); - druidMetadataStorageTablesConfig = MetadataStorageTablesConfig.fromBase(base); - return druidMetadataStorageTablesConfig; - } - - private SQLMetadataConnector getConnector() { - return Suppliers.memoize(this::buildConnector).get(); - } - - private static String getPassword(Configuration conf, HiveConf.ConfVars var) { - try { - final char[] password = conf.getPassword(var.varname); - return password == null ? null : String.valueOf(password); - } catch (IOException e) { - // Reading password from configuration for backwards compatibility. - LOG.warn("Unable to retrieve password from credential providers. Trying to read it from config..", e); - return conf.get(var.varname); - } - } - - private SQLMetadataConnector buildConnector() { - - if (connector != null) { - return connector; - } - - final String dbType = HiveConf.getVar(getConf(), HiveConf.ConfVars.DRUID_METADATA_DB_TYPE); - final String username = HiveConf.getVar(getConf(), HiveConf.ConfVars.DRUID_METADATA_DB_USERNAME); - final String password = getPassword(getConf(), HiveConf.ConfVars.DRUID_METADATA_DB_PASSWORD); - final String uri = HiveConf.getVar(getConf(), HiveConf.ConfVars.DRUID_METADATA_DB_URI); - LOG.debug("Supplying SQL Connector with DB type {}, URI {}, User {}", dbType, uri, username); - @SuppressWarnings("Guava") final Supplier - storageConnectorConfigSupplier = - Suppliers.ofInstance(new MetadataStorageConnectorConfig() { - @Override public String getConnectURI() { - return uri; - } - - @Override public String getUser() { - return Strings.emptyToNull(username); - } - - @Override public String getPassword() { - return Strings.emptyToNull(password); - } - }); - switch (dbType) { - case "mysql": - connector = - new MySQLConnector(storageConnectorConfigSupplier, - Suppliers.ofInstance(getDruidMetadataStorageTablesConfig()), - new MySQLConnectorConfig()); - break; - case "postgresql": - connector = - new PostgreSQLConnector(storageConnectorConfigSupplier, - Suppliers.ofInstance(getDruidMetadataStorageTablesConfig()), new PostgreSQLConnectorConfig(), - new PostgreSQLTablesConfig()); - - break; - case "derby": - connector = - new DerbyConnector(new DerbyMetadataStorage(storageConnectorConfigSupplier.get()), - storageConnectorConfigSupplier, - Suppliers.ofInstance(getDruidMetadataStorageTablesConfig())); - break; - default: - throw new IllegalStateException(String.format("Unknown metadata storage type [%s]", dbType)); - } - return connector; - } - - @VisibleForTesting String makeStagingName() { - return ".staging-".concat(getUniqueId().replace(":", "")); - } - - private Path getSegmentDescriptorDir() { - return new Path(getStagingWorkingDir(), SEGMENTS_DESCRIPTOR_DIR_NAME); - } - - private Path getIntermediateSegmentDir() { - return new Path(getStagingWorkingDir(), INTERMEDIATE_SEGMENT_DIR_NAME); - } - - private void cleanWorkingDir() { - final FileSystem fileSystem; - try { - fileSystem = getStagingWorkingDir().getFileSystem(getConf()); - fileSystem.delete(getStagingWorkingDir(), true); - } catch (IOException e) { - LOG.error("Got Exception while cleaning working directory", e); - } - } - - private String getRootWorkingDir() { - if (Strings.isNullOrEmpty(rootWorkingDir)) { - rootWorkingDir = HiveConf.getVar(getConf(), HiveConf.ConfVars.DRUID_WORKING_DIR); - } - return rootWorkingDir; - } - - private static HttpClient makeHttpClient(Lifecycle lifecycle) { - final int - numConnection = - HiveConf.getIntVar(SessionState.getSessionConf(), HiveConf.ConfVars.HIVE_DRUID_NUM_HTTP_CONNECTION); - final Period - readTimeout = - new Period(HiveConf.getVar(SessionState.getSessionConf(), HiveConf.ConfVars.HIVE_DRUID_HTTP_READ_TIMEOUT)); - LOG.info("Creating Druid HTTP client with {} max parallel connections and {}ms read timeout", - numConnection, - readTimeout.toStandardDuration().getMillis()); - - final HttpClient - httpClient = - HttpClientInit.createClient(HttpClientConfig.builder() - .withNumConnections(numConnection) - .withReadTimeout(new Period(readTimeout).toStandardDuration()) - .build(), lifecycle); - final boolean kerberosEnabled = - HiveConf.getBoolVar(SessionState.getSessionConf(), HiveConf.ConfVars.HIVE_DRUID_KERBEROS_ENABLE); - if (kerberosEnabled && UserGroupInformation.isSecurityEnabled()) { - LOG.info("building Kerberos Http Client"); - return new KerberosHttpClient(httpClient); - } - return httpClient; - } - - public static HttpClient getHttpClient() { - return HTTP_CLIENT; - } - - @Override public void preAlterTable(Table table, EnvironmentContext context) throws MetaException { - String alterOpType = context == null ? null : context.getProperties().get(ALTER_TABLE_OPERATION_TYPE); - // alterOpType is null in case of stats update - if (alterOpType != null && !ALLOWED_ALTER_TYPES.contains(alterOpType)) { - throw new MetaException("ALTER TABLE can not be used for " + alterOpType + " to a non-native table "); - } - if (DruidKafkaUtils.isKafkaStreamingTable(table)) { - updateKafkaIngestion(table); - } - } - - private int getMaxRetryCount() { - return HiveConf.getIntVar(getConf(), HiveConf.ConfVars.HIVE_DRUID_MAX_TRIES); - } - - @Override public StorageHandlerInfo getStorageHandlerInfo(Table table) throws MetaException { - if (DruidKafkaUtils.isKafkaStreamingTable(table)) { - KafkaSupervisorReport kafkaSupervisorReport = fetchKafkaSupervisorReport(table); - if (kafkaSupervisorReport == null) { - return DruidStorageHandlerInfo.UNREACHABLE; - } - return new DruidStorageHandlerInfo(kafkaSupervisorReport); - } else { - // Currently we do not expose any runtime info for non-streaming tables. - // In future extend this add more information regarding table status. - // e.g. Total size of segments in druid, load status of table on historical nodes etc. - return null; - } - } - - @Override public Map getOperatorDescProperties(OperatorDesc operatorDesc, - Map initialProps) { - if (operatorDesc instanceof TableScanDesc) { - TableScanDesc tableScanDesc = (TableScanDesc) operatorDesc; - ExprNodeGenericFuncDesc filterExpr = tableScanDesc.getFilterExpr(); - String druidQuery = initialProps.get(Constants.DRUID_QUERY_JSON); - - if (filterExpr != null && druidQuery != null) { - try { - Query query = DruidStorageHandlerUtils.JSON_MAPPER.readValue(druidQuery, BaseQuery.class); - Query queryWithDynamicFilters = DruidStorageHandlerUtils.addDynamicFilters(query, filterExpr, conf, false); - Map props = Maps.newHashMap(initialProps); - props.put(Constants.DRUID_QUERY_JSON, - DruidStorageHandlerUtils.JSON_MAPPER.writeValueAsString(queryWithDynamicFilters)); - return props; - } catch (IOException e) { - LOG.error("Exception while deserializing druid query. Explain plan may not have final druid query", e); - } - } - } - // Case when we do not have any additional info to add. - return initialProps; - } -} diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/DruidStorageHandlerInfo.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/DruidStorageHandlerInfo.java deleted file mode 100644 index e4ab53a6c849..000000000000 --- a/druid-handler/src/java/org/apache/hadoop/hive/druid/DruidStorageHandlerInfo.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.druid; - -import org.apache.hadoop.hive.druid.json.KafkaSupervisorReport; -import org.apache.hadoop.hive.ql.metadata.StorageHandlerInfo; - -/** - * DruidStorageHandlerInfo provides a runtime information for DruidStorageHandler. - */ -public class DruidStorageHandlerInfo implements StorageHandlerInfo { - - static final StorageHandlerInfo - UNREACHABLE = - (StorageHandlerInfo) () -> "Druid Overlord is Unreachable, Runtime Status : unknown"; - - private final KafkaSupervisorReport kafkaSupervisorReport; - - DruidStorageHandlerInfo(KafkaSupervisorReport kafkaSupervisorReport) { - this.kafkaSupervisorReport = kafkaSupervisorReport; - } - - @Override public String formatAsText() { - StringBuilder sb = new StringBuilder(); - sb.append("Druid Storage Handler Runtime Status for ") - .append(kafkaSupervisorReport.getId()) - .append("\n") - .append("kafkaPartitions=") - .append(kafkaSupervisorReport.getPayload().getPartitions()) - .append("\n") - .append("activeTasks=") - .append(kafkaSupervisorReport.getPayload().getActiveTasks()) - .append("\n") - .append("publishingTasks=") - .append(kafkaSupervisorReport.getPayload().getPublishingTasks()); - - if (kafkaSupervisorReport.getPayload().getLatestOffsets() != null) { - sb.append("\n").append("latestOffsets=").append(kafkaSupervisorReport.getPayload().getLatestOffsets()); - } - if (kafkaSupervisorReport.getPayload().getMinimumLag() != null) { - sb.append("\n").append("minimumLag=").append(kafkaSupervisorReport.getPayload().getMinimumLag()); - } - if (kafkaSupervisorReport.getPayload().getAggregateLag() != null) { - sb.append("\n").append("aggregateLag=").append(kafkaSupervisorReport.getPayload().getAggregateLag()); - } - if (kafkaSupervisorReport.getPayload().getOffsetsLastUpdated() != null) { - sb.append("\n").append("lastUpdateTime=").append(kafkaSupervisorReport.getPayload().getOffsetsLastUpdated()); - } - return sb.toString(); - } -} diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/DruidStorageHandlerUtils.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/DruidStorageHandlerUtils.java deleted file mode 100644 index 7e49b0b7c96e..000000000000 --- a/druid-handler/src/java/org/apache/hadoop/hive/druid/DruidStorageHandlerUtils.java +++ /dev/null @@ -1,1113 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.druid; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.InjectableValues; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.jsontype.NamedType; -import com.fasterxml.jackson.dataformat.smile.SmileFactory; -import com.google.common.base.Throwables; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Interner; -import com.google.common.collect.Interners; -import com.google.common.collect.Iterables; -import com.google.common.collect.Lists; -import com.google.common.collect.Ordering; -import org.apache.calcite.adapter.druid.DruidQuery; -import org.apache.calcite.sql.validate.SqlValidatorUtil; -import org.apache.druid.common.config.NullHandling; -import org.apache.druid.data.input.impl.DimensionSchema; -import org.apache.druid.data.input.impl.StringDimensionSchema; -import org.apache.druid.guice.BloomFilterSerializersModule; -import org.apache.druid.jackson.DefaultObjectMapper; -import org.apache.druid.java.util.common.JodaUtils; -import org.apache.druid.java.util.common.MapUtils; -import org.apache.druid.java.util.common.Pair; -import org.apache.druid.java.util.common.granularity.Granularity; -import org.apache.druid.java.util.emitter.EmittingLogger; -import org.apache.druid.java.util.emitter.core.NoopEmitter; -import org.apache.druid.java.util.emitter.service.ServiceEmitter; -import org.apache.druid.java.util.http.client.HttpClient; -import org.apache.druid.java.util.http.client.Request; -import org.apache.druid.java.util.http.client.response.InputStreamResponseHandler; -import org.apache.druid.java.util.http.client.response.StringFullResponseHandler; -import org.apache.druid.java.util.http.client.response.StringFullResponseHolder; -import org.apache.druid.math.expr.ExprMacroTable; -import org.apache.druid.metadata.MetadataStorageTablesConfig; -import org.apache.druid.metadata.SQLMetadataConnector; -import org.apache.druid.metadata.storage.mysql.MySQLConnector; -import org.apache.druid.query.DruidProcessingConfig; -import org.apache.druid.query.Druids; -import org.apache.druid.query.aggregation.AggregatorFactory; -import org.apache.druid.query.aggregation.DoubleSumAggregatorFactory; -import org.apache.druid.query.aggregation.FloatSumAggregatorFactory; -import org.apache.druid.query.aggregation.LongSumAggregatorFactory; -import org.apache.druid.query.expression.LikeExprMacro; -import org.apache.druid.query.expression.RegexpExtractExprMacro; -import org.apache.druid.query.expression.TimestampCeilExprMacro; -import org.apache.druid.query.expression.TimestampExtractExprMacro; -import org.apache.druid.query.expression.TimestampFloorExprMacro; -import org.apache.druid.query.expression.TimestampFormatExprMacro; -import org.apache.druid.query.expression.TimestampParseExprMacro; -import org.apache.druid.query.expression.TimestampShiftExprMacro; -import org.apache.druid.query.expression.TrimExprMacro; -import org.apache.druid.query.filter.AndDimFilter; -import org.apache.druid.query.filter.BloomDimFilter; -import org.apache.druid.query.filter.BloomKFilter; -import org.apache.druid.query.filter.BloomKFilterHolder; -import org.apache.druid.query.filter.BoundDimFilter; -import org.apache.druid.query.filter.DimFilter; -import org.apache.druid.query.filter.OrDimFilter; -import org.apache.druid.query.groupby.GroupByQuery; -import org.apache.druid.query.ordering.StringComparator; -import org.apache.druid.query.ordering.StringComparators; -import org.apache.druid.query.scan.ScanQuery; -import org.apache.druid.query.spec.MultipleIntervalSegmentSpec; -import org.apache.druid.query.timeseries.TimeseriesQuery; -import org.apache.druid.query.topn.TopNQuery; -import org.apache.druid.query.topn.TopNQueryBuilder; -import org.apache.druid.segment.IndexIO; -import org.apache.druid.segment.IndexMergerV9; -import org.apache.druid.segment.IndexSpec; -import org.apache.druid.segment.VirtualColumn; -import org.apache.druid.segment.VirtualColumns; -import org.apache.druid.segment.column.ValueType; -import org.apache.druid.segment.data.BitmapSerdeFactory; -import org.apache.druid.segment.data.ConciseBitmapSerdeFactory; -import org.apache.druid.segment.data.RoaringBitmapSerdeFactory; -import org.apache.druid.segment.indexing.granularity.GranularitySpec; -import org.apache.druid.segment.indexing.granularity.UniformGranularitySpec; -import org.apache.druid.segment.loading.DataSegmentPusher; -import org.apache.druid.segment.realtime.appenderator.SegmentIdWithShardSpec; -import org.apache.druid.segment.virtual.ExpressionVirtualColumn; -import org.apache.druid.segment.writeout.TmpFileSegmentWriteOutMediumFactory; -import org.apache.druid.storage.hdfs.HdfsDataSegmentPusher; -import org.apache.druid.storage.hdfs.HdfsDataSegmentPusherConfig; -import org.apache.druid.timeline.DataSegment; -import org.apache.druid.timeline.TimelineObjectHolder; -import org.apache.druid.timeline.VersionedIntervalTimeline; -import org.apache.druid.timeline.partition.LinearShardSpec; -import org.apache.druid.timeline.partition.NoneShardSpec; -import org.apache.druid.timeline.partition.NumberedShardSpec; -import org.apache.druid.timeline.partition.PartitionChunk; -import org.apache.druid.timeline.partition.ShardSpec; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hive.conf.Constants; -import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.druid.conf.DruidConstants; -import org.apache.hadoop.hive.druid.json.AvroParseSpec; -import org.apache.hadoop.hive.druid.json.AvroStreamInputRowParser; -import org.apache.hadoop.hive.metastore.api.Table; -import org.apache.hadoop.hive.ql.exec.ExprNodeDynamicValueEvaluator; -import org.apache.hadoop.hive.ql.exec.ExprNodeEvaluator; -import org.apache.hadoop.hive.ql.exec.ExprNodeEvaluatorFactory; -import org.apache.hadoop.hive.ql.exec.FunctionRegistry; -import org.apache.hadoop.hive.ql.exec.UDF; -import org.apache.hadoop.hive.ql.metadata.HiveException; -import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc; -import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; -import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc; -import org.apache.hadoop.hive.ql.udf.UDFToDouble; -import org.apache.hadoop.hive.ql.udf.UDFToFloat; -import org.apache.hadoop.hive.ql.udf.UDFToLong; -import org.apache.hadoop.hive.ql.udf.generic.GenericUDF; -import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBetween; -import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBridge; -import org.apache.hadoop.hive.ql.udf.generic.GenericUDFInBloomFilter; -import org.apache.hadoop.hive.ql.udf.generic.GenericUDFToString; -import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; -import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; -import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; -import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; -import org.apache.hadoop.io.BytesWritable; -import org.apache.hadoop.io.retry.RetryPolicies; -import org.apache.hadoop.io.retry.RetryProxy; -import org.jboss.netty.handler.codec.http.HttpHeaders; -import org.jboss.netty.handler.codec.http.HttpMethod; -import org.jboss.netty.handler.codec.http.HttpResponseStatus; -import org.joda.time.DateTime; -import org.joda.time.Interval; -import org.joda.time.Period; -import org.joda.time.chrono.ISOChronology; -import org.skife.jdbi.v2.Folder3; -import org.skife.jdbi.v2.Handle; -import org.skife.jdbi.v2.PreparedBatch; -import org.skife.jdbi.v2.Query; -import org.skife.jdbi.v2.ResultIterator; -import org.skife.jdbi.v2.exceptions.CallbackFailedException; -import org.skife.jdbi.v2.tweak.HandleCallback; -import org.skife.jdbi.v2.util.ByteArrayMapper; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import javax.annotation.Nullable; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.net.InetAddress; -import java.net.MalformedURLException; -import java.net.URL; -import java.net.UnknownHostException; -import java.nio.ByteBuffer; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Properties; -import java.util.Set; -import java.util.TimeZone; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; - -/** - * Utils class for Druid storage handler. - */ -public final class DruidStorageHandlerUtils { - private DruidStorageHandlerUtils() { - } - - private static final Logger LOG = LoggerFactory.getLogger(DruidStorageHandlerUtils.class); - - private static final int NUM_RETRIES = 8; - private static final int SECONDS_BETWEEN_RETRIES = 2; - private static final int DEFAULT_FS_BUFFER_SIZE = 1 << 18; // 256KB - private static final int DEFAULT_STREAMING_RESULT_SIZE = 100; - private static final String SMILE_CONTENT_TYPE = "application/x-jackson-smile"; - - static final String INDEX_ZIP = "index.zip"; - private static final String DESCRIPTOR_JSON = "descriptor.json"; - private static final Interval - DEFAULT_INTERVAL = - new Interval(new DateTime("1900-01-01", ISOChronology.getInstanceUTC()), - new DateTime("3000-01-01", ISOChronology.getInstanceUTC())).withChronology(ISOChronology.getInstanceUTC()); - - /** - * Mapper to use to serialize/deserialize Druid objects (JSON). - */ - public static final ObjectMapper JSON_MAPPER = new DefaultObjectMapper(); - - /** - * Mapper to use to serialize/deserialize Druid objects (SMILE). - */ - public static final ObjectMapper SMILE_MAPPER = new DefaultObjectMapper(new SmileFactory()); - private static final int DEFAULT_MAX_TRIES = 10; - - static { - // This is needed to initliaze NullHandling for druid without guice. - NullHandling.initializeForTests(); - // This is needed for serde of PagingSpec as it uses JacksonInject for injecting SelectQueryConfig - InjectableValues.Std injectableValues = new InjectableValues.Std() - // Expressions macro table used when we deserialize the query from calcite plan - .addValue(ExprMacroTable.class, new ExprMacroTable(ImmutableList - .of(new LikeExprMacro(), new RegexpExtractExprMacro(), new TimestampCeilExprMacro(), - new TimestampExtractExprMacro(), new TimestampFormatExprMacro(), new TimestampParseExprMacro(), - new TimestampShiftExprMacro(), new TimestampFloorExprMacro(), new TrimExprMacro.BothTrimExprMacro(), - new TrimExprMacro.LeftTrimExprMacro(), new TrimExprMacro.RightTrimExprMacro()))) - .addValue(ObjectMapper.class, JSON_MAPPER) - .addValue(DataSegment.PruneSpecsHolder.class, DataSegment.PruneSpecsHolder.DEFAULT); - - JSON_MAPPER.setInjectableValues(injectableValues); - SMILE_MAPPER.setInjectableValues(injectableValues); - // Register the shard sub type to be used by the mapper - JSON_MAPPER.registerSubtypes(new NamedType(LinearShardSpec.class, "linear")); - JSON_MAPPER.registerSubtypes(new NamedType(NumberedShardSpec.class, "numbered")); - JSON_MAPPER.registerSubtypes(new NamedType(AvroParseSpec.class, "avro")); - SMILE_MAPPER.registerSubtypes(new NamedType(AvroParseSpec.class, "avro")); - JSON_MAPPER.registerSubtypes(new NamedType(AvroStreamInputRowParser.class, "avro_stream")); - SMILE_MAPPER.registerSubtypes(new NamedType(AvroStreamInputRowParser.class, "avro_stream")); - // Register Bloom Filter Serializers - BloomFilterSerializersModule bloomFilterSerializersModule = new BloomFilterSerializersModule(); - JSON_MAPPER.registerModule(bloomFilterSerializersModule); - SMILE_MAPPER.registerModule(bloomFilterSerializersModule); - - // set the timezone of the object mapper - // THIS IS NOT WORKING workaround is to set it as part of java opts -Duser.timezone="UTC" - JSON_MAPPER.setTimeZone(TimeZone.getTimeZone("UTC")); - try { - // No operation emitter will be used by some internal druid classes. - EmittingLogger.registerEmitter(new ServiceEmitter("druid-hive-indexer", - InetAddress.getLocalHost().getHostName(), - new NoopEmitter())); - } catch (UnknownHostException e) { - throw Throwables.propagate(e); - } - } - - /** - * Used by druid to perform IO on indexes. - */ - public static final IndexIO INDEX_IO = new IndexIO(JSON_MAPPER, new DruidProcessingConfig() { - @Override public String getFormatString() { - return "%s-%s"; - } - }); - - /** - * Used by druid to merge indexes. - */ - public static final IndexMergerV9 - INDEX_MERGER_V9 = - new IndexMergerV9(JSON_MAPPER, DruidStorageHandlerUtils.INDEX_IO, TmpFileSegmentWriteOutMediumFactory.instance()); - - /** - * Generic Interner implementation used to read segments object from metadata storage. - */ - private static final Interner DATA_SEGMENT_INTERNER = Interners.newWeakInterner(); - - /** - * Method that creates a request for Druid query using SMILE format. - * - * @param address of the host target. - * @param query druid query. - * @return Request object to be submitted. - */ - public static Request createSmileRequest(String address, org.apache.druid.query.Query query) { - try { - return new Request(HttpMethod.POST, new URL(String.format("%s/druid/v2/", "http://" + address))).setContent( - SMILE_MAPPER.writeValueAsBytes(query)).setHeader(HttpHeaders.Names.CONTENT_TYPE, SMILE_CONTENT_TYPE); - } catch (MalformedURLException e) { - LOG.error("URL Malformed address {}", address); - throw new RuntimeException(e); - } catch (JsonProcessingException e) { - LOG.error("can not Serialize the Query [{}]", query.toString()); - throw new RuntimeException(e); - } - } - - /** - * Method that submits a request to an Http address and retrieves the result. - * The caller is responsible for closing the stream once it finishes consuming it. - * - * @param client Http Client will be used to submit request. - * @param request Http request to be submitted. - * @return response object. - * @throws IOException in case of request IO error. - */ - public static InputStream submitRequest(HttpClient client, Request request) throws IOException { - try { - return client.go(request, new InputStreamResponseHandler()).get(); - } catch (ExecutionException | InterruptedException e) { - throw new IOException(e.getCause()); - } - - } - - static StringFullResponseHolder getResponseFromCurrentLeader(HttpClient client, Request request, - StringFullResponseHandler fullResponseHandler) throws ExecutionException, InterruptedException { - StringFullResponseHolder responseHolder = client.go(request, fullResponseHandler).get(); - if (HttpResponseStatus.TEMPORARY_REDIRECT.equals(responseHolder.getStatus())) { - String redirectUrlStr = responseHolder.getResponse().headers().get("Location"); - LOG.debug("Request[%s] received redirect response to location [%s].", request.getUrl(), redirectUrlStr); - final URL redirectUrl; - try { - redirectUrl = new URL(redirectUrlStr); - } catch (MalformedURLException ex) { - throw new ExecutionException(String - .format("Malformed redirect location is found in response from url[%s], new location[%s].", - request.getUrl(), - redirectUrlStr), ex); - } - responseHolder = client.go(withUrl(request, redirectUrl), fullResponseHandler).get(); - } - return responseHolder; - } - - private static Request withUrl(Request old, URL url) { - Request req = new Request(old.getMethod(), url); - req.addHeaderValues(old.getHeaders()); - if (old.hasContent()) { - req.setContent(old.getContent()); - } - return req; - } - - /** - * @param taskDir path to the directory containing the segments descriptor info - * the descriptor path will be - * ../workingPath/task_id/{@link DruidStorageHandler#SEGMENTS_DESCRIPTOR_DIR_NAME}/*.json - * @param conf hadoop conf to get the file system - * @return List of DataSegments - * @throws IOException can be for the case we did not produce data. - */ - public static List getCreatedSegments(Path taskDir, Configuration conf) throws IOException { - ImmutableList.Builder publishedSegmentsBuilder = ImmutableList.builder(); - FileSystem fs = taskDir.getFileSystem(conf); - FileStatus[] fss; - fss = fs.listStatus(taskDir); - for (FileStatus fileStatus : fss) { - final DataSegment segment = JSON_MAPPER.readValue((InputStream) fs.open(fileStatus.getPath()), DataSegment.class); - publishedSegmentsBuilder.add(segment); - } - return publishedSegmentsBuilder.build(); - } - - /** - * Writes to filesystem serialized form of segment descriptor if an existing file exists it will try to replace it. - * - * @param outputFS filesystem. - * @param segment DataSegment object. - * @param descriptorPath path. - * @throws IOException in case any IO issues occur. - */ - public static void writeSegmentDescriptor(final FileSystem outputFS, - final DataSegment segment, - final Path descriptorPath) throws IOException { - final DataPusher descriptorPusher = (DataPusher) RetryProxy.create(DataPusher.class, () -> { - if (outputFS.exists(descriptorPath)) { - if (!outputFS.delete(descriptorPath, false)) { - throw new IOException(String.format("Failed to delete descriptor at [%s]", descriptorPath)); - } - } - try (final OutputStream descriptorOut = outputFS.create(descriptorPath, true, DEFAULT_FS_BUFFER_SIZE)) { - JSON_MAPPER.writeValue(descriptorOut, segment); - descriptorOut.flush(); - } - }, RetryPolicies.exponentialBackoffRetry(NUM_RETRIES, SECONDS_BETWEEN_RETRIES, TimeUnit.SECONDS)); - descriptorPusher.push(); - } - - /** - * @param connector SQL metadata connector to the metadata storage - * @param metadataStorageTablesConfig Table config - * @return all the active data sources in the metadata storage - */ - static Collection getAllDataSourceNames(SQLMetadataConnector connector, - final MetadataStorageTablesConfig metadataStorageTablesConfig) { - return connector.getDBI() - .withHandle((HandleCallback>) handle -> handle.createQuery(String.format( - "SELECT DISTINCT(datasource) FROM %s WHERE used = true", - metadataStorageTablesConfig.getSegmentsTable())) - .fold(Lists.newArrayList(), - (druidDataSources, stringObjectMap, foldController, statementContext) -> { - druidDataSources.add(MapUtils.getString(stringObjectMap, "datasource")); - return druidDataSources; - })); - } - - /** - * @param connector SQL connector to metadata - * @param metadataStorageTablesConfig Tables configuration - * @param dataSource Name of data source - * @return true if the data source was successfully disabled false otherwise - */ - static boolean disableDataSource(SQLMetadataConnector connector, - final MetadataStorageTablesConfig metadataStorageTablesConfig, - final String dataSource) { - try { - if (!getAllDataSourceNames(connector, metadataStorageTablesConfig).contains(dataSource)) { - LOG.warn("Cannot delete data source {}, does not exist", dataSource); - return false; - } - - connector.getDBI().withHandle((HandleCallback) handle -> { - disableDataSourceWithHandle(handle, metadataStorageTablesConfig, dataSource); - return null; - }); - - } catch (Exception e) { - LOG.error(String.format("Error removing dataSource %s", dataSource), e); - return false; - } - return true; - } - - /** - * First computes the segments timeline to accommodate new segments for insert into case. - * Then moves segments to druid deep storage with updated metadata/version. - * ALL IS DONE IN ONE TRANSACTION - * - * @param connector DBI connector to commit - * @param metadataStorageTablesConfig Druid metadata tables definitions - * @param dataSource Druid datasource name - * @param segments List of segments to move and commit to metadata - * @param overwrite if it is an insert overwrite - * @param conf Configuration - * @param dataSegmentPusher segment pusher - * @return List of successfully published Druid segments. - * This list has the updated versions and metadata about segments after move and timeline sorting - * @throws CallbackFailedException in case the connector can not add the segment to the DB. - */ - @SuppressWarnings("unchecked") static List publishSegmentsAndCommit(final SQLMetadataConnector connector, - final MetadataStorageTablesConfig metadataStorageTablesConfig, - final String dataSource, - final List segments, - boolean overwrite, - Configuration conf, - DataSegmentPusher dataSegmentPusher) throws CallbackFailedException { - return connector.getDBI().inTransaction((handle, transactionStatus) -> { - // We create the timeline for the existing and new segments - VersionedIntervalTimeline timeline; - if (overwrite) { - // If we are overwriting, we disable existing sources - disableDataSourceWithHandle(handle, metadataStorageTablesConfig, dataSource); - - // When overwriting, we just start with empty timeline, - // as we are overwriting segments with new versions - timeline = new VersionedIntervalTimeline<>(Ordering.natural()); - } else { - // Append Mode - if (segments.isEmpty()) { - // If there are no new segments, we can just bail out - return Collections.EMPTY_LIST; - } - // Otherwise, build a timeline of existing segments in metadata storage - Interval - indexedInterval = - JodaUtils.umbrellaInterval(segments.stream().map(DataSegment::getInterval).collect(Collectors.toList())); - LOG.info("Building timeline for umbrella Interval [{}]", indexedInterval); - timeline = getTimelineForIntervalWithHandle(handle, dataSource, indexedInterval, metadataStorageTablesConfig); - } - - final List finalSegmentsToPublish = Lists.newArrayList(); - for (DataSegment segment : segments) { - List> existingChunks = timeline.lookup(segment.getInterval()); - if (existingChunks.size() > 1) { - // Not possible to expand since we have more than one chunk with a single segment. - // This is the case when user wants to append a segment with coarser granularity. - // case metadata storage has segments with granularity HOUR and segments to append have DAY granularity. - // Druid shard specs does not support multiple partitions for same interval with different granularity. - throw new IllegalStateException(String.format( - "Cannot allocate new segment for dataSource[%s], interval[%s], already have [%,d] chunks. " - + "Not possible to append new segment.", - dataSource, - segment.getInterval(), - existingChunks.size())); - } - // Find out the segment with latest version and maximum partition number - SegmentIdWithShardSpec max = null; - final ShardSpec newShardSpec; - final String newVersion; - if (!existingChunks.isEmpty()) { - // Some existing chunk, Find max - TimelineObjectHolder existingHolder = Iterables.getOnlyElement(existingChunks); - for (PartitionChunk existing : existingHolder.getObject()) { - if (max == null || max.getShardSpec().getPartitionNum() < existing.getObject() - .getShardSpec() - .getPartitionNum()) { - max = SegmentIdWithShardSpec.fromDataSegment(existing.getObject()); - } - } - } - - if (max == null) { - // No existing shard present in the database, use the current version. - newShardSpec = segment.getShardSpec(); - newVersion = segment.getVersion(); - } else { - // use version of existing max segment to generate new shard spec - newShardSpec = getNextPartitionShardSpec(max.getShardSpec()); - newVersion = max.getVersion(); - } - DataSegment - publishedSegment = - publishSegmentWithShardSpec(segment, - newShardSpec, - newVersion, - getPath(segment).getFileSystem(conf), - dataSegmentPusher); - finalSegmentsToPublish.add(publishedSegment); - timeline.add(publishedSegment.getInterval(), - publishedSegment.getVersion(), - publishedSegment.getShardSpec().createChunk(publishedSegment)); - - } - - // Publish new segments to metadata storage - final PreparedBatch - batch = - handle.prepareBatch(String.format( - "INSERT INTO %1$s (id, dataSource, created_date, start, \"end\", partitioned, version, used, payload) " - + "VALUES (:id, :dataSource, :created_date, :start, :end, :partitioned, :version, :used, :payload)", - metadataStorageTablesConfig.getSegmentsTable()) - - ); - - for (final DataSegment segment : finalSegmentsToPublish) { - - batch.add(new ImmutableMap.Builder().put("id", segment.getId().toString()) - .put("dataSource", segment.getDataSource()) - .put("created_date", new DateTime().toString()) - .put("start", segment.getInterval().getStart().toString()) - .put("end", segment.getInterval().getEnd().toString()) - .put("partitioned", !(segment.getShardSpec() instanceof NoneShardSpec)) - .put("version", segment.getVersion()) - .put("used", true) - .put("payload", JSON_MAPPER.writeValueAsBytes(segment)) - .build()); - - LOG.info("Published {}", segment.getId().toString()); - } - batch.execute(); - - return finalSegmentsToPublish; - }); - } - - private static void disableDataSourceWithHandle(Handle handle, - MetadataStorageTablesConfig metadataStorageTablesConfig, - String dataSource) { - handle.createStatement(String.format("UPDATE %s SET used=false WHERE dataSource = :dataSource", - metadataStorageTablesConfig.getSegmentsTable())).bind("dataSource", dataSource).execute(); - } - - /** - * @param connector SQL connector to metadata - * @param metadataStorageTablesConfig Tables configuration - * @param dataSource Name of data source - * @return List of all data segments part of the given data source - */ - static List getDataSegmentList(final SQLMetadataConnector connector, - final MetadataStorageTablesConfig metadataStorageTablesConfig, - final String dataSource) { - return connector.retryTransaction((handle, status) -> handle.createQuery(String.format( - "SELECT payload FROM %s WHERE dataSource = :dataSource", - metadataStorageTablesConfig.getSegmentsTable())) - .setFetchSize(getStreamingFetchSize(connector)) - .bind("dataSource", dataSource) - .map(ByteArrayMapper.FIRST) - .fold(new ArrayList<>(), (Folder3, byte[]>) (accumulator, payload, control, ctx) -> { - try { - final DataSegment segment = DATA_SEGMENT_INTERNER.intern(JSON_MAPPER.readValue(payload, DataSegment.class)); - - accumulator.add(segment); - return accumulator; - } catch (Exception e) { - throw new SQLException(e.toString()); - } - }), 3, DEFAULT_MAX_TRIES); - } - - /** - * @param connector SQL DBI connector. - * @return streaming fetch size. - */ - private static int getStreamingFetchSize(SQLMetadataConnector connector) { - if (connector instanceof MySQLConnector) { - return Integer.MIN_VALUE; - } - return DEFAULT_STREAMING_RESULT_SIZE; - } - - /** - * @param pushedSegment the pushed data segment object - * @param segmentsDescriptorDir actual directory path for descriptors. - * @return a sanitize file name - */ - public static Path makeSegmentDescriptorOutputPath(DataSegment pushedSegment, Path segmentsDescriptorDir) { - return new Path(segmentsDescriptorDir, String.format("%s.json", pushedSegment.getId().toString().replace(":", ""))); - } - - public static String createScanAllQuery(String dataSourceName, List columns) throws JsonProcessingException { - final Druids.ScanQueryBuilder scanQueryBuilder = Druids.newScanQueryBuilder(); - final List intervals = Collections.singletonList(DEFAULT_INTERVAL); - ScanQuery - scanQuery = - scanQueryBuilder.dataSource(dataSourceName).resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST) - .intervals(new MultipleIntervalSegmentSpec(intervals)) - .columns(columns) - .build(); - return JSON_MAPPER.writeValueAsString(scanQuery); - } - - @Nullable static Boolean getBooleanProperty(Table table, String propertyName) { - String val = getTableProperty(table, propertyName); - if (val == null) { - return null; - } - return Boolean.parseBoolean(val); - } - - static boolean getBooleanProperty(Table table, String propertyName, boolean defaultVal) { - Boolean val = getBooleanProperty(table, propertyName); - return val == null ? defaultVal : val; - } - - @Nullable static Integer getIntegerProperty(Table table, String propertyName) { - String val = getTableProperty(table, propertyName); - if (val == null) { - return null; - } - try { - return Integer.parseInt(val); - } catch (NumberFormatException e) { - throw new NumberFormatException(String.format("Exception while parsing property[%s] with Value [%s] as Integer", - propertyName, - val)); - } - } - - static int getIntegerProperty(Table table, String propertyName, int defaultVal) { - Integer val = getIntegerProperty(table, propertyName); - return val == null ? defaultVal : val; - } - - @Nullable static Long getLongProperty(Table table, String propertyName) { - String val = getTableProperty(table, propertyName); - if (val == null) { - return null; - } - try { - return Long.parseLong(val); - } catch (NumberFormatException e) { - throw new NumberFormatException(String.format("Exception while parsing property[%s] with Value [%s] as Long", - propertyName, - val)); - } - } - - @Nullable static Period getPeriodProperty(Table table, String propertyName) { - String val = getTableProperty(table, propertyName); - if (val == null) { - return null; - } - try { - return Period.parse(val); - } catch (IllegalArgumentException e) { - throw new IllegalArgumentException(String.format("Exception while parsing property[%s] with Value [%s] as Period", - propertyName, - val)); - } - } - - @Nullable public static List getListProperty(Table table, String propertyName) { - List rv = new ArrayList<>(); - String values = getTableProperty(table, propertyName); - if (values == null) { - return null; - } - String[] vals = values.trim().split(","); - for (String val : vals) { - if (org.apache.commons.lang3.StringUtils.isNotBlank(val)) { - rv.add(val); - } - } - return rv; - } - - static String getTableProperty(Table table, String propertyName) { - return table.getParameters().get(propertyName); - } - - /** - * Simple interface for retry operations. - */ - public interface DataPusher { - void push() throws IOException; - } - - private static VersionedIntervalTimeline getTimelineForIntervalWithHandle(final Handle handle, - final String dataSource, - final Interval interval, - final MetadataStorageTablesConfig dbTables) throws IOException { - Query> - sql = - handle.createQuery(String.format( - "SELECT payload FROM %s WHERE used = true AND dataSource = ? AND start <= ? AND \"end\" >= ?", - dbTables.getSegmentsTable())) - .bind(0, dataSource) - .bind(1, interval.getEnd().toString()) - .bind(2, interval.getStart().toString()); - - final VersionedIntervalTimeline timeline = new VersionedIntervalTimeline<>(Ordering.natural()); - try (ResultIterator dbSegments = sql.map(ByteArrayMapper.FIRST).iterator()) { - while (dbSegments.hasNext()) { - final byte[] payload = dbSegments.next(); - DataSegment segment = JSON_MAPPER.readValue(payload, DataSegment.class); - timeline.add(segment.getInterval(), segment.getVersion(), segment.getShardSpec().createChunk(segment)); - } - } - return timeline; - } - - public static DataSegmentPusher createSegmentPusherForDirectory(String segmentDirectory, Configuration configuration) - throws IOException { - final HdfsDataSegmentPusherConfig hdfsDataSegmentPusherConfig = new HdfsDataSegmentPusherConfig(); - hdfsDataSegmentPusherConfig.setStorageDirectory(segmentDirectory); - return new HdfsDataSegmentPusher(hdfsDataSegmentPusherConfig, configuration, JSON_MAPPER); - } - - private static DataSegment publishSegmentWithShardSpec(DataSegment segment, - ShardSpec shardSpec, - String version, - FileSystem fs, - DataSegmentPusher dataSegmentPusher) throws IOException { - boolean retry = true; - DataSegment.Builder dataSegmentBuilder = new DataSegment.Builder(segment).version(version); - Path finalPath = null; - while (retry) { - retry = false; - dataSegmentBuilder.shardSpec(shardSpec); - final Path intermediatePath = getPath(segment); - - finalPath = - new Path(dataSegmentPusher.getPathForHadoop(), - dataSegmentPusher.makeIndexPathName(dataSegmentBuilder.build(), DruidStorageHandlerUtils.INDEX_ZIP)); - // Create parent if it does not exist, recreation is not an error - fs.mkdirs(finalPath.getParent()); - - if (!fs.rename(intermediatePath, finalPath)) { - if (fs.exists(finalPath)) { - // Someone else is also trying to append - shardSpec = getNextPartitionShardSpec(shardSpec); - retry = true; - } else { - throw new IOException(String.format( - "Failed to rename intermediate segment[%s] to final segment[%s] is not present.", - intermediatePath, - finalPath)); - } - } - } - DataSegment dataSegment = dataSegmentBuilder.loadSpec(dataSegmentPusher.makeLoadSpec(finalPath.toUri())).build(); - - writeSegmentDescriptor(fs, dataSegment, new Path(finalPath.getParent(), DruidStorageHandlerUtils.DESCRIPTOR_JSON)); - - return dataSegment; - } - - private static ShardSpec getNextPartitionShardSpec(ShardSpec shardSpec) { - if (shardSpec instanceof LinearShardSpec) { - return new LinearShardSpec(shardSpec.getPartitionNum() + 1); - } else if (shardSpec instanceof NumberedShardSpec) { - return new NumberedShardSpec(shardSpec.getPartitionNum(), ((NumberedShardSpec) shardSpec).getPartitions()); - } else { - // Druid only support appending more partitions to Linear and Numbered ShardSpecs. - throw new IllegalStateException(String.format("Cannot expand shard spec [%s]", shardSpec)); - } - } - - static Path getPath(DataSegment dataSegment) { - return new Path(String.valueOf(Objects.requireNonNull(dataSegment.getLoadSpec()).get("path"))); - } - - public static GranularitySpec getGranularitySpec(Configuration configuration, Properties tableProperties) { - final String - segmentGranularity = - tableProperties.getProperty(Constants.DRUID_SEGMENT_GRANULARITY) != null ? - tableProperties.getProperty(Constants.DRUID_SEGMENT_GRANULARITY) : - HiveConf.getVar(configuration, HiveConf.ConfVars.HIVE_DRUID_INDEXING_GRANULARITY); - final boolean - rollup = - tableProperties.getProperty(DruidConstants.DRUID_ROLLUP) != null ? - Boolean.parseBoolean(tableProperties.getProperty(Constants.DRUID_SEGMENT_GRANULARITY)) : - HiveConf.getBoolVar(configuration, HiveConf.ConfVars.HIVE_DRUID_ROLLUP); - return new UniformGranularitySpec(Granularity.fromString(segmentGranularity), - Granularity.fromString(tableProperties.getProperty(DruidConstants.DRUID_QUERY_GRANULARITY) == null ? - "NONE" : - tableProperties.getProperty(DruidConstants.DRUID_QUERY_GRANULARITY)), - rollup, - null); - } - - public static IndexSpec getIndexSpec(Configuration jc) { - final BitmapSerdeFactory bitmapSerdeFactory; - if ("concise".equals(HiveConf.getVar(jc, HiveConf.ConfVars.HIVE_DRUID_BITMAP_FACTORY_TYPE))) { - bitmapSerdeFactory = new ConciseBitmapSerdeFactory(); - } else { - bitmapSerdeFactory = new RoaringBitmapSerdeFactory(true); - } - return new IndexSpec(bitmapSerdeFactory, - IndexSpec.DEFAULT_DIMENSION_COMPRESSION, - IndexSpec.DEFAULT_METRIC_COMPRESSION, - IndexSpec.DEFAULT_LONG_ENCODING); - } - - public static Pair, AggregatorFactory[]> getDimensionsAndAggregates(List columnNames, - List columnTypes) { - // Default, all columns that are not metrics or timestamp, are treated as dimensions - final List dimensions = new ArrayList<>(); - ImmutableList.Builder aggregatorFactoryBuilder = ImmutableList.builder(); - for (int i = 0; i < columnTypes.size(); i++) { - final PrimitiveObjectInspector.PrimitiveCategory - primitiveCategory = - ((PrimitiveTypeInfo) columnTypes.get(i)).getPrimitiveCategory(); - AggregatorFactory af; - switch (primitiveCategory) { - case BYTE: - case SHORT: - case INT: - case LONG: - af = new LongSumAggregatorFactory(columnNames.get(i), columnNames.get(i)); - break; - case FLOAT: - af = new FloatSumAggregatorFactory(columnNames.get(i), columnNames.get(i)); - break; - case DOUBLE: - af = new DoubleSumAggregatorFactory(columnNames.get(i), columnNames.get(i)); - break; - case DECIMAL: - throw new UnsupportedOperationException(String.format("Druid does not support decimal column type cast column " - + "[%s] to double", columnNames.get(i))); - case TIMESTAMP: - // Granularity column - String tColumnName = columnNames.get(i); - if (!tColumnName.equals(Constants.DRUID_TIMESTAMP_GRANULARITY_COL_NAME) - && !tColumnName.equals(DruidConstants.DEFAULT_TIMESTAMP_COLUMN)) { - throw new IllegalArgumentException("Dimension " - + tColumnName - + " does not have STRING type: " - + primitiveCategory); - } - continue; - case TIMESTAMPLOCALTZ: - // Druid timestamp column - String tLocalTZColumnName = columnNames.get(i); - if (!tLocalTZColumnName.equals(DruidConstants.DEFAULT_TIMESTAMP_COLUMN)) { - throw new IllegalArgumentException("Dimension " - + tLocalTZColumnName - + " does not have STRING type: " - + primitiveCategory); - } - continue; - default: - // Dimension - String dColumnName = columnNames.get(i); - if (PrimitiveObjectInspectorUtils.getPrimitiveGrouping(primitiveCategory) - != PrimitiveObjectInspectorUtils.PrimitiveGrouping.STRING_GROUP - && primitiveCategory != PrimitiveObjectInspector.PrimitiveCategory.BOOLEAN) { - throw new IllegalArgumentException("Dimension " - + dColumnName - + " does not have STRING type: " - + primitiveCategory); - } - dimensions.add(new StringDimensionSchema(dColumnName)); - continue; - } - aggregatorFactoryBuilder.add(af); - } - ImmutableList aggregatorFactories = aggregatorFactoryBuilder.build(); - return Pair.of(dimensions, aggregatorFactories.toArray(new AggregatorFactory[0])); - } - - // Druid only supports String,Long,Float,Double selectors - private static Set druidSupportedTypeInfos = - ImmutableSet.of(TypeInfoFactory.stringTypeInfo, TypeInfoFactory.charTypeInfo, - TypeInfoFactory.varcharTypeInfo, TypeInfoFactory.byteTypeInfo, TypeInfoFactory.intTypeInfo, - TypeInfoFactory.longTypeInfo, TypeInfoFactory.shortTypeInfo, TypeInfoFactory.doubleTypeInfo); - - private static Set stringTypeInfos = - ImmutableSet.of(TypeInfoFactory.stringTypeInfo, TypeInfoFactory.charTypeInfo, - TypeInfoFactory.varcharTypeInfo); - - public static org.apache.druid.query.Query addDynamicFilters(org.apache.druid.query.Query query, - ExprNodeGenericFuncDesc filterExpr, Configuration conf, boolean resolveDynamicValues) { - List virtualColumns = Arrays.asList(getVirtualColumns(query).getVirtualColumns()); - org.apache.druid.query.Query rv = query; - DimFilter joinReductionFilter = toDruidFilter(filterExpr, conf, virtualColumns, resolveDynamicValues); - if (joinReductionFilter != null) { - String type = query.getType(); - DimFilter filter = new AndDimFilter(joinReductionFilter, query.getFilter()); - switch (type) { - case org.apache.druid.query.Query.TIMESERIES: - rv = Druids.TimeseriesQueryBuilder.copy((TimeseriesQuery) query).filters(filter) - .virtualColumns(VirtualColumns.create(virtualColumns)).build(); - break; - case org.apache.druid.query.Query.TOPN: - rv = new TopNQueryBuilder((TopNQuery) query).filters(filter) - .virtualColumns(VirtualColumns.create(virtualColumns)).build(); - break; - case org.apache.druid.query.Query.GROUP_BY: - rv = new GroupByQuery.Builder((GroupByQuery) query).setDimFilter(filter) - .setVirtualColumns(VirtualColumns.create(virtualColumns)).build(); - break; - case org.apache.druid.query.Query.SCAN: - rv = Druids.ScanQueryBuilder.copy((ScanQuery) query).filters(filter) - .virtualColumns(VirtualColumns.create(virtualColumns)).build(); - break; - default: - throw new UnsupportedOperationException("Unsupported Query type " + type); - } - } - return rv; - } - - @Nullable private static DimFilter toDruidFilter(ExprNodeDesc filterExpr, Configuration configuration, - List virtualColumns, boolean resolveDynamicValues) { - if (filterExpr == null) { - return null; - } - Class genericUDFClass = getGenericUDFClassFromExprDesc(filterExpr); - if (FunctionRegistry.isOpAnd(filterExpr)) { - Iterator iterator = filterExpr.getChildren().iterator(); - List delegates = Lists.newArrayList(); - while (iterator.hasNext()) { - DimFilter filter = toDruidFilter(iterator.next(), configuration, virtualColumns, resolveDynamicValues); - if (filter != null) { - delegates.add(filter); - } - } - if (!delegates.isEmpty()) { - return new AndDimFilter(delegates); - } - } - if (FunctionRegistry.isOpOr(filterExpr)) { - Iterator iterator = filterExpr.getChildren().iterator(); - List delegates = Lists.newArrayList(); - while (iterator.hasNext()) { - DimFilter filter = toDruidFilter(iterator.next(), configuration, virtualColumns, resolveDynamicValues); - if (filter != null) { - delegates.add(filter); - } - } - if (!delegates.isEmpty()) { - return new OrDimFilter(delegates); - } - } else if (GenericUDFBetween.class == genericUDFClass) { - List child = filterExpr.getChildren(); - String col = extractColName(child.get(1), virtualColumns); - if (col != null) { - try { - StringComparator comparator = stringTypeInfos - .contains(child.get(1).getTypeInfo()) ? StringComparators.LEXICOGRAPHIC : StringComparators.NUMERIC; - String lower = evaluate(child.get(2), configuration, resolveDynamicValues); - String upper = evaluate(child.get(3), configuration, resolveDynamicValues); - return new BoundDimFilter(col, lower, upper, false, false, null, null, comparator); - - } catch (HiveException e) { - throw new RuntimeException(e); - } - } - } else if (GenericUDFInBloomFilter.class == genericUDFClass) { - List child = filterExpr.getChildren(); - String col = extractColName(child.get(0), virtualColumns); - if (col != null) { - try { - BloomKFilter bloomFilter = evaluateBloomFilter(child.get(1), configuration, resolveDynamicValues); - return new BloomDimFilter(col, BloomKFilterHolder.fromBloomKFilter(bloomFilter), null); - } catch (HiveException | IOException e) { - throw new RuntimeException(e); - } - } - } - return null; - } - - private static String evaluate(ExprNodeDesc desc, Configuration configuration, boolean resolveDynamicValue) - throws HiveException { - ExprNodeEvaluator exprNodeEvaluator = ExprNodeEvaluatorFactory.get(desc, configuration); - if (exprNodeEvaluator instanceof ExprNodeDynamicValueEvaluator && !resolveDynamicValue) { - return desc.getExprStringForExplain(); - } else { - return exprNodeEvaluator.evaluate(null).toString(); - } - } - - private static BloomKFilter evaluateBloomFilter(ExprNodeDesc desc, Configuration configuration, - boolean resolveDynamicValue) throws HiveException, IOException { - if (!resolveDynamicValue) { - // return a dummy bloom filter for explain - return new BloomKFilter(1); - } else { - BytesWritable bw = (BytesWritable) ExprNodeEvaluatorFactory.get(desc, configuration).evaluate(null); - return BloomKFilter.deserialize(ByteBuffer.wrap(bw.getBytes())); - } - } - - @Nullable public static String extractColName(ExprNodeDesc expr, List virtualColumns) { - if (!druidSupportedTypeInfos.contains(expr.getTypeInfo())) { - // This column type is currently not supported in druid.(e.g boolean) - // We cannot pass the bloom filter to druid since bloom filter tests for exact object bytes. - return null; - } - if (expr instanceof ExprNodeColumnDesc) { - return ((ExprNodeColumnDesc) expr).getColumn(); - } - - ExprNodeGenericFuncDesc funcDesc = null; - if (expr instanceof ExprNodeGenericFuncDesc) { - funcDesc = (ExprNodeGenericFuncDesc) expr; - } - if (null == funcDesc) { - return null; - } - GenericUDF udf = funcDesc.getGenericUDF(); - // bail out if its not a simple cast expression. - if (funcDesc.getChildren().size() == 1 && funcDesc.getChildren().get(0) instanceof ExprNodeColumnDesc) { - return null; - } - String columnName = ((ExprNodeColumnDesc) (funcDesc.getChildren().get(0))).getColumn(); - ValueType targetType = null; - if (udf instanceof GenericUDFBridge) { - Class udfClass = ((GenericUDFBridge) udf).getUdfClass(); - if (udfClass.equals(UDFToDouble.class)) { - targetType = ValueType.DOUBLE; - } else if (udfClass.equals(UDFToFloat.class)) { - targetType = ValueType.FLOAT; - } else if (udfClass.equals(UDFToLong.class)) { - targetType = ValueType.LONG; - } - } else if (udf instanceof GenericUDFToString) { - targetType = ValueType.STRING; - } - - if (targetType == null) { - return null; - } - String virtualColumnExpr = DruidQuery.format("CAST(%s, '%s')", columnName, targetType.toString()); - for (VirtualColumn column : virtualColumns) { - if (column instanceof ExpressionVirtualColumn && ((ExpressionVirtualColumn) column).getExpression() - .equals(virtualColumnExpr)) { - // Found an existing virtual column with same expression, no need to add another virtual column - return column.getOutputName(); - } - } - Set usedColumnNames = virtualColumns.stream().map(col -> col.getOutputName()).collect(Collectors.toSet()); - final String name = SqlValidatorUtil.uniquify("vc", usedColumnNames, SqlValidatorUtil.EXPR_SUGGESTER); - ExpressionVirtualColumn expressionVirtualColumn = - new ExpressionVirtualColumn(name, virtualColumnExpr, targetType, ExprMacroTable.nil()); - virtualColumns.add(expressionVirtualColumn); - return name; - } - - public static VirtualColumns getVirtualColumns(org.apache.druid.query.Query query) { - String type = query.getType(); - switch (type) { - case org.apache.druid.query.Query.TIMESERIES: - return ((TimeseriesQuery) query).getVirtualColumns(); - case org.apache.druid.query.Query.TOPN: - return ((TopNQuery) query).getVirtualColumns(); - case org.apache.druid.query.Query.GROUP_BY: - return ((GroupByQuery) query).getVirtualColumns(); - case org.apache.druid.query.Query.SCAN: - return ((ScanQuery) query).getVirtualColumns(); - default: - throw new UnsupportedOperationException("Unsupported Query type " + query); - } - } - - @Nullable private static Class getGenericUDFClassFromExprDesc(ExprNodeDesc desc) { - if (!(desc instanceof ExprNodeGenericFuncDesc)) { - return null; - } - ExprNodeGenericFuncDesc genericFuncDesc = (ExprNodeGenericFuncDesc) desc; - return genericFuncDesc.getGenericUDF().getClass(); - } -} diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/conf/DruidConstants.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/conf/DruidConstants.java deleted file mode 100644 index 070580406629..000000000000 --- a/druid-handler/src/java/org/apache/hadoop/hive/druid/conf/DruidConstants.java +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.druid.conf; - -/** - * Utility class for Druid Constants. - */ -public final class DruidConstants { - private DruidConstants() { - } - - public static final String DRUID_QUERY_FETCH = "druid.query.fetch"; - - public static final String DRUID_ROLLUP = "druid.rollup"; - - public static final String DRUID_QUERY_GRANULARITY = "druid.query.granularity"; - - public static final String DRUID_SEGMENT_DIRECTORY = "druid.storage.storageDirectory"; - - public static final String DRUID_SEGMENT_INTERMEDIATE_DIRECTORY = "druid.storage.storageDirectory.intermediate"; - - public static final String DRUID_SEGMENT_VERSION = "druid.segment.version"; - - public static final String DRUID_JOB_WORKING_DIRECTORY = "druid.job.workingDirectory"; - - public static final String KAFKA_TOPIC = "kafka.topic"; - - public static final String KAFKA_BOOTSTRAP_SERVERS = "kafka.bootstrap.servers"; - - public static final String DRUID_KAFKA_INGESTION_PROPERTY_PREFIX = "druid.kafka.ingestion."; - - public static final String DRUID_KAFKA_CONSUMER_PROPERTY_PREFIX = DRUID_KAFKA_INGESTION_PROPERTY_PREFIX + "consumer."; - - /* Kafka Ingestion state - valid values - START/STOP/RESET */ - public static final String DRUID_KAFKA_INGESTION = "druid.kafka.ingestion"; - - //Druid storage timestamp column name - public static final String DEFAULT_TIMESTAMP_COLUMN = "__time"; - - public static final String DRUID_TIMESTAMP_FORMAT = "druid.timestamp.format"; - - // Used when the field name in ingested data via streaming ingestion does not match - // druid default timestamp column i.e `__time` - public static final String DRUID_TIMESTAMP_COLUMN = "druid.timestamp.column"; - - //Druid Json timestamp column name for GroupBy results - public static final String EVENT_TIMESTAMP_COLUMN = "timestamp"; - - // Druid ParseSpec Type - JSON/CSV/TSV/AVRO - public static final String DRUID_PARSE_SPEC_FORMAT = "druid.parseSpec.format"; - - public static final String AVRO_SCHEMA_LITERAL = "avro.schema.literal"; - - // value delimiter for druid columns - public static final String DRUID_PARSE_SPEC_DELIMITER = "druid.parseSpec.delimiter"; - - // list delimiter for multi-valued columns - public static final String DRUID_PARSE_SPEC_LIST_DELIMITER = "druid.parseSpec.listDelimiter"; - - // order of columns for delimiter and csv parse specs. - public static final String DRUID_PARSE_SPEC_COLUMNS = "druid.parseSpec.columns"; - - public static final String DRUID_PARSE_SPEC_SKIP_HEADER_ROWS = "druid.parseSpec.skipHeaderRows"; - - public static final String DRUID_PARSE_SPEC_HAS_HEADER_ROWS = "druid.parseSpec.hasHeaderRows"; -} diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/conf/package-info.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/conf/package-info.java deleted file mode 100644 index 1049acb2a94c..000000000000 --- a/druid-handler/src/java/org/apache/hadoop/hive/druid/conf/package-info.java +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * Package info file. - */ - -package org.apache.hadoop.hive.druid.conf; diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/io/DruidOutputFormat.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/io/DruidOutputFormat.java deleted file mode 100644 index d90db9cbda92..000000000000 --- a/druid-handler/src/java/org/apache/hadoop/hive/druid/io/DruidOutputFormat.java +++ /dev/null @@ -1,178 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.druid.io; - -import com.fasterxml.jackson.core.type.TypeReference; -import org.apache.druid.data.input.impl.DimensionSchema; -import org.apache.druid.data.input.impl.DimensionsSpec; -import org.apache.druid.data.input.impl.InputRowParser; -import org.apache.druid.data.input.impl.MapInputRowParser; -import org.apache.druid.data.input.impl.TimeAndDimsParseSpec; -import org.apache.druid.data.input.impl.TimestampSpec; -import org.apache.druid.java.util.common.Pair; -import org.apache.druid.query.aggregation.AggregatorFactory; -import org.apache.druid.segment.IndexSpec; -import org.apache.druid.segment.indexing.DataSchema; -import org.apache.druid.segment.indexing.RealtimeTuningConfig; -import org.apache.druid.segment.indexing.granularity.GranularitySpec; -import org.apache.druid.segment.realtime.plumber.CustomVersioningPolicy; - -import org.apache.commons.lang3.StringUtils; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hive.conf.Constants; -import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.druid.DruidStorageHandlerUtils; -import org.apache.hadoop.hive.druid.conf.DruidConstants; -import org.apache.hadoop.hive.druid.serde.DruidWritable; -import org.apache.hadoop.hive.ql.exec.FileSinkOperator; -import org.apache.hadoop.hive.ql.io.HiveOutputFormat; -import org.apache.hadoop.hive.serde.serdeConstants; -import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; -import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; -import org.apache.hadoop.io.NullWritable; -import org.apache.hadoop.io.Writable; -import org.apache.hadoop.mapred.JobConf; -import org.apache.hadoop.mapred.RecordWriter; -import org.apache.hadoop.util.Progressable; - -import static org.apache.hadoop.hive.druid.DruidStorageHandler.SEGMENTS_DESCRIPTOR_DIR_NAME; - -import com.google.common.base.Preconditions; -import com.google.common.base.Strings; -import com.google.common.collect.Lists; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Properties; - -/** - * Druid Output format class used to write data as Native Druid Segment. - */ -public class DruidOutputFormat implements HiveOutputFormat { - - private static final Logger LOG = LoggerFactory.getLogger(DruidOutputFormat.class); - - @Override - public FileSinkOperator.RecordWriter getHiveRecordWriter( - JobConf jc, - Path finalOutPath, - Class valueClass, - boolean isCompressed, - Properties tableProperties, - Progressable progress - ) throws IOException { - - - final int targetNumShardsPerGranularity = Integer.parseUnsignedInt( - tableProperties.getProperty(Constants.DRUID_TARGET_SHARDS_PER_GRANULARITY, "0")); - final int maxPartitionSize = targetNumShardsPerGranularity > 0 ? -1 : HiveConf - .getIntVar(jc, HiveConf.ConfVars.HIVE_DRUID_MAX_PARTITION_SIZE); - // If datasource is in the table properties, it is an INSERT/INSERT OVERWRITE as the datasource - // name was already persisted. Otherwise, it is a CT/CTAS and we need to get the name from the - // job properties that are set by configureOutputJobProperties in the DruidStorageHandler - final String dataSource = tableProperties.getProperty(Constants.DRUID_DATA_SOURCE) == null - ? jc.get(Constants.DRUID_DATA_SOURCE) - : tableProperties.getProperty(Constants.DRUID_DATA_SOURCE); - final String segmentDirectory = jc.get(DruidConstants.DRUID_SEGMENT_INTERMEDIATE_DIRECTORY); - - final GranularitySpec granularitySpec = DruidStorageHandlerUtils.getGranularitySpec(jc, tableProperties); - - final String columnNameProperty = tableProperties.getProperty(serdeConstants.LIST_COLUMNS); - final String columnTypeProperty = tableProperties.getProperty(serdeConstants.LIST_COLUMN_TYPES); - - if (StringUtils.isEmpty(columnNameProperty) || StringUtils.isEmpty(columnTypeProperty)) { - throw new IllegalStateException( - String.format("List of columns names [%s] or columns type [%s] is/are not present", - columnNameProperty, columnTypeProperty - )); - } - ArrayList columnNames = Lists.newArrayList(columnNameProperty.split(",")); - if (!columnNames.contains(DruidConstants.DEFAULT_TIMESTAMP_COLUMN)) { - throw new IllegalStateException("Timestamp column (' " + DruidConstants.DEFAULT_TIMESTAMP_COLUMN + - "') not specified in create table; list of columns is : " + - tableProperties.getProperty(serdeConstants.LIST_COLUMNS)); - } - ArrayList columnTypes = TypeInfoUtils.getTypeInfosFromTypeString(columnTypeProperty); - - Pair, AggregatorFactory[]> dimensionsAndAggregates = DruidStorageHandlerUtils - .getDimensionsAndAggregates(columnNames, columnTypes); - final InputRowParser inputRowParser = new MapInputRowParser(new TimeAndDimsParseSpec( - new TimestampSpec(DruidConstants.DEFAULT_TIMESTAMP_COLUMN, "auto", null), - new DimensionsSpec(dimensionsAndAggregates.lhs, Lists - .newArrayList(Constants.DRUID_TIMESTAMP_GRANULARITY_COL_NAME, - Constants.DRUID_SHARD_KEY_COL_NAME - ), null - ) - )); - - Map - inputParser = - DruidStorageHandlerUtils.JSON_MAPPER.convertValue(inputRowParser, new TypeReference>() { - }); - - final DataSchema dataSchema = new DataSchema( - Preconditions.checkNotNull(dataSource, "Data source name is null"), - inputParser, - dimensionsAndAggregates.rhs, - granularitySpec, - null, - DruidStorageHandlerUtils.JSON_MAPPER - ); - - final String workingPath = jc.get(DruidConstants.DRUID_JOB_WORKING_DIRECTORY); - final String version = jc.get(DruidConstants.DRUID_SEGMENT_VERSION); - String basePersistDirectory = HiveConf - .getVar(jc, HiveConf.ConfVars.HIVE_DRUID_BASE_PERSIST_DIRECTORY); - if (Strings.isNullOrEmpty(basePersistDirectory)) { - basePersistDirectory = System.getProperty("java.io.tmpdir"); - } - Integer maxRowInMemory = HiveConf.getIntVar(jc, HiveConf.ConfVars.HIVE_DRUID_MAX_ROW_IN_MEMORY); - - IndexSpec indexSpec = DruidStorageHandlerUtils.getIndexSpec(jc); - RealtimeTuningConfig realtimeTuningConfig = new RealtimeTuningConfig(maxRowInMemory, null, null, null, - new File(basePersistDirectory, dataSource), new CustomVersioningPolicy(version), null, null, null, indexSpec, - null, true, 0, 0, true, null, 0L, null, null); - - LOG.debug(String.format("running with Data schema [%s] ", dataSchema)); - return new DruidRecordWriter(dataSchema, realtimeTuningConfig, - DruidStorageHandlerUtils.createSegmentPusherForDirectory(segmentDirectory, jc), - maxPartitionSize, new Path(workingPath, SEGMENTS_DESCRIPTOR_DIR_NAME), - finalOutPath.getFileSystem(jc) - ); - } - - @Override - public RecordWriter getRecordWriter( - FileSystem ignored, JobConf job, String name, Progressable progress - ) throws IOException { - throw new UnsupportedOperationException("please implement me !"); - } - - @Override - public void checkOutputSpecs(FileSystem ignored, JobConf job) throws IOException { - // NOOP - } -} diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/io/DruidQueryBasedInputFormat.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/io/DruidQueryBasedInputFormat.java deleted file mode 100644 index 2a2be067125f..000000000000 --- a/druid-handler/src/java/org/apache/hadoop/hive/druid/io/DruidQueryBasedInputFormat.java +++ /dev/null @@ -1,262 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.druid.io; - -import com.fasterxml.jackson.core.type.TypeReference; -import com.google.common.collect.Lists; -import org.apache.commons.lang3.StringEscapeUtils; -import org.apache.commons.lang3.StringUtils; -import org.apache.druid.java.util.http.client.Request; -import org.apache.druid.query.BaseQuery; -import org.apache.druid.query.LocatedSegmentDescriptor; -import org.apache.druid.query.Query; -import org.apache.druid.query.SegmentDescriptor; -import org.apache.druid.query.scan.ScanQuery; -import org.apache.druid.query.spec.MultipleSpecificSegmentSpec; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hive.conf.Constants; -import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.druid.DruidStorageHandler; -import org.apache.hadoop.hive.druid.DruidStorageHandlerUtils; -import org.apache.hadoop.hive.druid.serde.DruidGroupByQueryRecordReader; -import org.apache.hadoop.hive.druid.serde.DruidQueryRecordReader; -import org.apache.hadoop.hive.druid.serde.DruidScanQueryRecordReader; -import org.apache.hadoop.hive.druid.serde.DruidTimeseriesQueryRecordReader; -import org.apache.hadoop.hive.druid.serde.DruidTopNQueryRecordReader; -import org.apache.hadoop.hive.druid.serde.DruidWritable; -import org.apache.hadoop.hive.ql.exec.Utilities; -import org.apache.hadoop.hive.ql.exec.vector.VectorizedInputFormatInterface; -import org.apache.hadoop.hive.ql.exec.vector.VectorizedSupport; -import org.apache.hadoop.hive.shims.ShimLoader; -import org.apache.hadoop.io.NullWritable; -import org.apache.hadoop.mapred.JobConf; -import org.apache.hadoop.mapred.Reporter; -import org.apache.hadoop.mapreduce.InputFormat; -import org.apache.hadoop.mapreduce.InputSplit; -import org.apache.hadoop.mapreduce.Job; -import org.apache.hadoop.mapreduce.JobContext; -import org.apache.hadoop.mapreduce.RecordReader; -import org.apache.hadoop.mapreduce.TaskAttemptContext; -import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; -import org.jboss.netty.handler.codec.http.HttpMethod; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.io.InputStream; -import java.net.URL; -import java.net.URLEncoder; -import java.util.Arrays; -import java.util.List; - -/** - * Druid query based input format. - * - * Given a query and the Druid broker address, it will send it, and retrieve - * and parse the results. - */ -public class DruidQueryBasedInputFormat extends InputFormat - implements org.apache.hadoop.mapred.InputFormat, VectorizedInputFormatInterface { - - private static final Logger LOG = LoggerFactory.getLogger(DruidQueryBasedInputFormat.class); - - public static DruidQueryRecordReader getDruidQueryReader(String druidQueryType) { - switch (druidQueryType) { - case Query.TIMESERIES: - return new DruidTimeseriesQueryRecordReader(); - case Query.TOPN: - return new DruidTopNQueryRecordReader(); - case Query.GROUP_BY: - return new DruidGroupByQueryRecordReader(); - case Query.SCAN: - return new DruidScanQueryRecordReader(); - default: - throw new IllegalStateException("Druid query type " + druidQueryType + " not recognized"); - } - } - - @Override public org.apache.hadoop.mapred.InputSplit[] getSplits(JobConf job, int numSplits) throws IOException { - return getInputSplits(job); - } - - @Override public List getSplits(JobContext context) throws IOException, InterruptedException { - return Arrays.asList(getInputSplits(context.getConfiguration())); - } - - protected HiveDruidSplit[] getInputSplits(Configuration conf) throws IOException { - String address = HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_DRUID_BROKER_DEFAULT_ADDRESS); - String queryId = HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_QUERY_ID); - if (StringUtils.isEmpty(address)) { - throw new IOException("Druid broker address not specified in configuration"); - } - String druidQuery = StringEscapeUtils.unescapeJava(conf.get(Constants.DRUID_QUERY_JSON)); - - String druidQueryType; - if (StringUtils.isEmpty(druidQuery)) { - // Empty, maybe because CBO did not run; we fall back to - // full Select query - LOG.warn("Druid query is empty; creating Select query"); - String dataSource = conf.get(Constants.DRUID_DATA_SOURCE); - if (dataSource == null || dataSource.isEmpty()) { - throw new IOException("Druid data source cannot be empty or null"); - } - druidQuery = DruidStorageHandlerUtils.createScanAllQuery(dataSource, Utilities.getColumnNames(conf)); - druidQueryType = Query.SCAN; - conf.set(Constants.DRUID_QUERY_TYPE, druidQueryType); - } else { - druidQueryType = conf.get(Constants.DRUID_QUERY_TYPE); - if (druidQueryType == null) { - throw new IOException("Druid query type not recognized"); - } - } - - // Add Hive Query ID to Druid Query - if (queryId != null) { - druidQuery = withQueryId(druidQuery, queryId); - } - - // hive depends on FileSplits - Job job = Job.getInstance(conf); - JobContext jobContext = ShimLoader.getHadoopShims().newJobContext(job); - Path[] paths = FileInputFormat.getInputPaths(jobContext); - - // We need to deserialize and serialize query so intervals are written in the JSON - // Druid query with user timezone, as this is default Hive time semantics. - // Then, create splits with the Druid queries. - switch (druidQueryType) { - case Query.TIMESERIES: - case Query.TOPN: - case Query.GROUP_BY: - return new HiveDruidSplit[] {new HiveDruidSplit(druidQuery, paths[0], new String[] {address})}; - case Query.SCAN: - ScanQuery scanQuery = DruidStorageHandlerUtils.JSON_MAPPER.readValue(druidQuery, ScanQuery.class); - return distributeScanQuery(address, scanQuery, paths[0]); - default: - throw new IOException("Druid query type not recognized"); - } - } - - /* New method that distributes the Scan query by creating splits containing - * information about different Druid nodes that have the data for the given - * query. */ - private static HiveDruidSplit[] distributeScanQuery(String address, ScanQuery query, Path dummyPath) - throws IOException { - // If it has a limit, we use it and we do not distribute the query - final boolean isFetch = query.getScanRowsLimit() < Long.MAX_VALUE; - if (isFetch) { - return new HiveDruidSplit[] {new HiveDruidSplit(DruidStorageHandlerUtils.JSON_MAPPER.writeValueAsString(query), - dummyPath, - new String[] {address}) - }; - } - - final List segmentDescriptors = fetchLocatedSegmentDescriptors(address, query); - - // Create one input split for each segment - final int numSplits = segmentDescriptors.size(); - final HiveDruidSplit[] splits = new HiveDruidSplit[segmentDescriptors.size()]; - for (int i = 0; i < numSplits; i++) { - final LocatedSegmentDescriptor locatedSD = segmentDescriptors.get(i); - final String[] hosts = new String[locatedSD.getLocations().size() + 1]; - for (int j = 0; j < locatedSD.getLocations().size(); j++) { - hosts[j] = locatedSD.getLocations().get(j).getHost(); - } - // Default to broker if all other hosts fail. - hosts[locatedSD.getLocations().size()] = address; - - // Create partial Select query - final SegmentDescriptor - newSD = - new SegmentDescriptor(locatedSD.getInterval(), locatedSD.getVersion(), locatedSD.getPartitionNumber()); - final Query partialQuery = query.withQuerySegmentSpec(new MultipleSpecificSegmentSpec(Lists.newArrayList(newSD))); - splits[i] = - new HiveDruidSplit(DruidStorageHandlerUtils.JSON_MAPPER.writeValueAsString(partialQuery), dummyPath, hosts); - } - return splits; - } - - private static List fetchLocatedSegmentDescriptors(String address, BaseQuery query) - throws IOException { - final String intervals = StringUtils.join(query.getIntervals(), ","); // Comma-separated intervals without brackets - final String - request = - String.format("http://%s/druid/v2/datasources/%s/candidates?intervals=%s", - address, - query.getDataSource().getNames().get(0), - URLEncoder.encode(intervals, "UTF-8")); - LOG.debug("sending request {} to query for segments", request); - final InputStream response; - try { - response = - DruidStorageHandlerUtils.submitRequest(DruidStorageHandler.getHttpClient(), - new Request(HttpMethod.GET, new URL(request))); - } catch (Exception e) { - throw new IOException(org.apache.hadoop.util.StringUtils.stringifyException(e)); - } - - // Retrieve results - final List segmentDescriptors; - try { - segmentDescriptors = - DruidStorageHandlerUtils.JSON_MAPPER.readValue(response, new TypeReference>() { - }); - } catch (Exception e) { - response.close(); - throw new IOException(org.apache.hadoop.util.StringUtils.stringifyException(e)); - } - return segmentDescriptors; - } - - private static String withQueryId(String druidQuery, String queryId) throws IOException { - Query queryWithId = DruidStorageHandlerUtils.JSON_MAPPER.readValue(druidQuery, BaseQuery.class).withId(queryId); - return DruidStorageHandlerUtils.JSON_MAPPER.writeValueAsString(queryWithId); - } - - @Override public org.apache.hadoop.mapred.RecordReader getRecordReader( - org.apache.hadoop.mapred.InputSplit split, - JobConf job, - Reporter reporter) throws IOException { - // We need to provide a different record reader for every type of Druid query. - // The reason is that Druid results format is different for each type. - final DruidQueryRecordReader reader; - // By default, we use druid scan query as fallback. - final String druidQueryType = job.get(Constants.DRUID_QUERY_TYPE, Query.SCAN); - reader = getDruidQueryReader(druidQueryType); - reader.initialize((HiveDruidSplit) split, job); - if (Utilities.getIsVectorized(job)) { - //noinspection unchecked - return (org.apache.hadoop.mapred.RecordReader) new DruidVectorizedWrapper(reader, job); - } - return reader; - } - - @Override public RecordReader createRecordReader(InputSplit split, - TaskAttemptContext context) throws IOException, InterruptedException { - // By default, we use druid scan query as fallback. - final String druidQueryType = context.getConfiguration().get(Constants.DRUID_QUERY_TYPE, Query.SCAN); - // We need to provide a different record reader for every type of Druid query. - // The reason is that Druid results format is different for each type. - //noinspection unchecked - return getDruidQueryReader(druidQueryType); - } - - @Override public VectorizedSupport.Support[] getSupportedFeatures() { - return new VectorizedSupport.Support[0]; - } -} diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/io/DruidRecordWriter.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/io/DruidRecordWriter.java deleted file mode 100644 index dc16c4e3f791..000000000000 --- a/druid-handler/src/java/org/apache/hadoop/hive/druid/io/DruidRecordWriter.java +++ /dev/null @@ -1,300 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.druid.io; - -import com.google.common.base.Joiner; -import com.google.common.base.Preconditions; -import com.google.common.base.Suppliers; -import com.google.common.base.Throwables; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.Lists; -import org.apache.druid.data.input.Committer; -import org.apache.druid.data.input.InputRow; -import org.apache.druid.data.input.MapBasedInputRow; -import org.apache.druid.java.util.common.DateTimes; -import org.apache.druid.java.util.common.granularity.Granularity; -import org.apache.druid.segment.indexing.DataSchema; -import org.apache.druid.segment.indexing.RealtimeTuningConfig; -import org.apache.druid.segment.loading.DataSegmentPusher; -import org.apache.druid.segment.realtime.FireDepartmentMetrics; -import org.apache.druid.segment.realtime.appenderator.Appenderator; -import org.apache.druid.segment.realtime.appenderator.Appenderators; -import org.apache.druid.segment.realtime.appenderator.SegmentIdWithShardSpec; -import org.apache.druid.segment.realtime.appenderator.SegmentNotWritableException; -import org.apache.druid.segment.realtime.appenderator.SegmentsAndMetadata; -import org.apache.druid.segment.realtime.plumber.Committers; -import org.apache.druid.timeline.DataSegment; -import org.apache.druid.timeline.partition.LinearShardSpec; -import org.apache.commons.io.FileUtils; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hive.conf.Constants; -import org.apache.hadoop.hive.druid.DruidStorageHandlerUtils; -import org.apache.hadoop.hive.druid.conf.DruidConstants; -import org.apache.hadoop.hive.druid.serde.DruidWritable; -import org.apache.hadoop.hive.ql.exec.FileSinkOperator; -import org.apache.hadoop.io.NullWritable; -import org.apache.hadoop.io.Writable; -import org.apache.hadoop.mapred.RecordWriter; -import org.apache.hadoop.mapred.Reporter; -import org.joda.time.DateTime; -import org.joda.time.Interval; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; -import java.io.IOException; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import java.util.UUID; -import java.util.concurrent.ExecutionException; -import java.util.function.Supplier; -import java.util.stream.Collectors; - -/** - * Druid Record Writer, implementing a file sink operator. - */ -public class DruidRecordWriter implements RecordWriter, FileSinkOperator.RecordWriter { - protected static final Logger LOG = LoggerFactory.getLogger(DruidRecordWriter.class); - - private final DataSchema dataSchema; - - private final Appenderator appenderator; - - private final RealtimeTuningConfig tuningConfig; - - private final Path segmentsDescriptorDir; - - private SegmentIdWithShardSpec currentOpenSegment = null; - - private final int maxPartitionSize; - - private final FileSystem fileSystem; - - private final Supplier committerSupplier; - - private final Granularity segmentGranularity; - - public DruidRecordWriter(DataSchema dataSchema, - RealtimeTuningConfig realtimeTuningConfig, - DataSegmentPusher dataSegmentPusher, - int maxPartitionSize, - final Path segmentsDescriptorsDir, - final FileSystem fileSystem) { - File basePersistDir = new File(realtimeTuningConfig.getBasePersistDirectory(), UUID.randomUUID().toString()); - this.tuningConfig = - Preconditions.checkNotNull(realtimeTuningConfig.withBasePersistDirectory(basePersistDir), - "realtimeTuningConfig is null"); - this.dataSchema = Preconditions.checkNotNull(dataSchema, "data schema is null"); - - appenderator = Appenderators - .createOffline("hive-offline-appenderator", this.dataSchema, tuningConfig, false, new FireDepartmentMetrics(), - dataSegmentPusher, DruidStorageHandlerUtils.JSON_MAPPER, DruidStorageHandlerUtils.INDEX_IO, - DruidStorageHandlerUtils.INDEX_MERGER_V9); - this.maxPartitionSize = maxPartitionSize; - appenderator.startJob(); - this.segmentsDescriptorDir = Preconditions.checkNotNull(segmentsDescriptorsDir, "segmentsDescriptorsDir is null"); - this.fileSystem = Preconditions.checkNotNull(fileSystem, "file system is null"); - this.segmentGranularity = this.dataSchema.getGranularitySpec().getSegmentGranularity(); - committerSupplier = Suppliers.ofInstance(Committers.nil())::get; - } - - /** - * This function computes the segment identifier and push the current open segment - * The push will occur if max size is reached or the event belongs to the next interval. - * Note that this function assumes that timestamps are pseudo sorted. - * This function will close and move to the next segment granularity as soon as - * an event from the next interval appears. The sorting is done by the previous stage. - * - * @return segmentIdentifier with of the truncatedTime and maybe push the current open segment. - */ - private SegmentIdWithShardSpec getSegmentIdentifierAndMaybePush(long truncatedTime) { - - DateTime truncatedDateTime = segmentGranularity.bucketStart(DateTimes.utc(truncatedTime)); - final Interval interval = new Interval(truncatedDateTime, segmentGranularity.increment(truncatedDateTime)); - - SegmentIdWithShardSpec retVal; - if (currentOpenSegment == null) { - currentOpenSegment = - new SegmentIdWithShardSpec(dataSchema.getDataSource(), - interval, - tuningConfig.getVersioningPolicy().getVersion(interval), - new LinearShardSpec(0)); - return currentOpenSegment; - } else if (currentOpenSegment.getInterval().equals(interval)) { - retVal = currentOpenSegment; - int rowCount = appenderator.getRowCount(retVal); - if (rowCount < maxPartitionSize) { - return retVal; - } else { - retVal = - new SegmentIdWithShardSpec(dataSchema.getDataSource(), - interval, - tuningConfig.getVersioningPolicy().getVersion(interval), - new LinearShardSpec(currentOpenSegment.getShardSpec().getPartitionNum() + 1)); - pushSegments(Lists.newArrayList(currentOpenSegment)); - LOG.info("Creating new partition for segment {}, partition num {}", - retVal.toString(), - retVal.getShardSpec().getPartitionNum()); - currentOpenSegment = retVal; - return retVal; - } - } else { - retVal = - new SegmentIdWithShardSpec(dataSchema.getDataSource(), - interval, - tuningConfig.getVersioningPolicy().getVersion(interval), - new LinearShardSpec(0)); - pushSegments(Lists.newArrayList(currentOpenSegment)); - LOG.info("Creating segment {}", retVal.toString()); - currentOpenSegment = retVal; - return retVal; - } - } - - private void pushSegments(List segmentsToPush) { - try { - SegmentsAndMetadata segmentsAndMetadata = appenderator.push(segmentsToPush, committerSupplier.get(), false).get(); - final Set pushedSegmentIdentifierHashSet = new HashSet<>(); - - for (DataSegment pushedSegment : segmentsAndMetadata.getSegments()) { - pushedSegmentIdentifierHashSet.add(SegmentIdWithShardSpec.fromDataSegment(pushedSegment).toString()); - final Path - segmentDescriptorOutputPath = - DruidStorageHandlerUtils.makeSegmentDescriptorOutputPath(pushedSegment, segmentsDescriptorDir); - DruidStorageHandlerUtils.writeSegmentDescriptor(fileSystem, pushedSegment, segmentDescriptorOutputPath); - LOG.info(String.format("Pushed the segment [%s] and persisted the descriptor located at [%s]", - pushedSegment, - segmentDescriptorOutputPath)); - } - - final Set - toPushSegmentsHashSet = - segmentsToPush.stream() - .map(SegmentIdWithShardSpec::toString) - .collect(Collectors.toCollection(HashSet::new)); - - if (!pushedSegmentIdentifierHashSet.equals(toPushSegmentsHashSet)) { - throw new IllegalStateException(String.format("was asked to publish [%s] but was able to publish only [%s]", - Joiner.on(", ").join(toPushSegmentsHashSet), - Joiner.on(", ").join(pushedSegmentIdentifierHashSet))); - } - for (SegmentIdWithShardSpec dataSegmentId : segmentsToPush) { - LOG.info("Dropping segment {}", dataSegmentId.toString()); - appenderator.drop(dataSegmentId).get(); - } - - LOG.info(String.format("Published [%,d] segments.", segmentsToPush.size())); - } catch (InterruptedException e) { - LOG.error(String.format("got interrupted, failed to push [%,d] segments.", segmentsToPush.size()), e); - Thread.currentThread().interrupt(); - } catch (IOException | ExecutionException e) { - LOG.error(String.format("Failed to push [%,d] segments.", segmentsToPush.size()), e); - Throwables.propagate(e); - } - } - - @Override public void write(Writable w) throws IOException { - DruidWritable record = (DruidWritable) w; - final long timestamp = (long) record.getValue().get(DruidConstants.DEFAULT_TIMESTAMP_COLUMN); - final int - partitionNumber = - Math.toIntExact((long) record.getValue().getOrDefault(Constants.DRUID_SHARD_KEY_COL_NAME, -1L)); - final InputRow - inputRow = - new MapBasedInputRow(timestamp, - dataSchema.getParser().getParseSpec().getDimensionsSpec().getDimensionNames(), - record.getValue()); - - try { - - if (partitionNumber != -1 && maxPartitionSize == -1) { - /* - Case data is sorted by time and an extra hashing dimension see DRUID_SHARD_KEY_COL_NAME - Thus use DRUID_SHARD_KEY_COL_NAME as segment partition in addition to time dimension - Data with the same DRUID_SHARD_KEY_COL_NAME and Time interval will end in the same segment - */ - DateTime truncatedDateTime = segmentGranularity.bucketStart(DateTimes.utc(timestamp)); - final Interval interval = new Interval(truncatedDateTime, segmentGranularity.increment(truncatedDateTime)); - - if (currentOpenSegment != null) { - if (currentOpenSegment.getShardSpec().getPartitionNum() != partitionNumber - || !currentOpenSegment.getInterval().equals(interval)) { - pushSegments(ImmutableList.of(currentOpenSegment)); - currentOpenSegment = - new SegmentIdWithShardSpec(dataSchema.getDataSource(), - interval, - tuningConfig.getVersioningPolicy().getVersion(interval), - new LinearShardSpec(partitionNumber)); - } - } else { - currentOpenSegment = - new SegmentIdWithShardSpec(dataSchema.getDataSource(), - interval, - tuningConfig.getVersioningPolicy().getVersion(interval), - new LinearShardSpec(partitionNumber)); - - } - appenderator.add(currentOpenSegment, inputRow, committerSupplier::get); - - } else if (partitionNumber == -1 && maxPartitionSize != -1) { - /*Case we are partitioning the segments based on time and max row per segment maxPartitionSize*/ - appenderator.add(getSegmentIdentifierAndMaybePush(timestamp), inputRow, committerSupplier::get); - } else { - throw new IllegalArgumentException(String.format( - "partitionNumber and maxPartitionSize should be mutually exclusive " - + "got partitionNum [%s] and maxPartitionSize [%s]", - partitionNumber, - maxPartitionSize)); - } - - } catch (SegmentNotWritableException e) { - throw new IOException(e); - } - } - - @Override public void close(boolean abort) throws IOException { - try { - if (!abort) { - final List segmentsToPush = Lists.newArrayList(); - segmentsToPush.addAll(appenderator.getSegments()); - pushSegments(segmentsToPush); - } - appenderator.clear(); - } catch (InterruptedException e) { - Throwables.propagate(e); - } finally { - try { - FileUtils.deleteDirectory(tuningConfig.getBasePersistDirectory()); - } catch (Exception e) { - LOG.error("error cleaning of base persist directory", e); - } - appenderator.close(); - } - } - - @Override public void write(NullWritable key, DruidWritable value) throws IOException { - this.write(value); - } - - @Override public void close(Reporter reporter) throws IOException { - this.close(false); - } - -} diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/io/DruidVectorizedWrapper.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/io/DruidVectorizedWrapper.java deleted file mode 100644 index 2345e13ab075..000000000000 --- a/druid-handler/src/java/org/apache/hadoop/hive/druid/io/DruidVectorizedWrapper.java +++ /dev/null @@ -1,139 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.druid.io; - -import com.google.common.base.Preconditions; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.druid.serde.DruidQueryRecordReader; -import org.apache.hadoop.hive.druid.serde.DruidSerDe; -import org.apache.hadoop.hive.druid.serde.DruidWritable; -import org.apache.hadoop.hive.ql.exec.Utilities; -import org.apache.hadoop.hive.ql.exec.vector.VectorAssignRow; -import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; -import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatchCtx; -import org.apache.hadoop.hive.ql.metadata.HiveException; -import org.apache.hadoop.hive.ql.plan.MapWork; -import org.apache.hadoop.hive.serde2.SerDeException; -import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; -import org.apache.hadoop.io.NullWritable; -import org.apache.hadoop.mapred.RecordReader; - -import java.io.IOException; -import java.util.Iterator; -import java.util.Properties; - -/** - * A Wrapper class that consumes row-by-row from base Druid Record Reader and provides a Vectorized one. - * @param type of the Druid query. - */ -public class DruidVectorizedWrapper> implements RecordReader { - private final VectorAssignRow vectorAssignRow = new VectorAssignRow(); - private final DruidQueryRecordReader baseReader; - private final VectorizedRowBatchCtx rbCtx; - private final DruidSerDe serDe; - private final Object[] rowBoat; - - /** - * Actual projected columns needed by the query, this can be empty in case of query like: select count(*) from src. - */ - private final int[] projectedColumns; - - private final DruidWritable druidWritable; - - public DruidVectorizedWrapper(DruidQueryRecordReader reader, Configuration jobConf) { - this.rbCtx = Utilities.getVectorizedRowBatchCtx(jobConf); - if (rbCtx.getDataColumnNums() != null) { - projectedColumns = rbCtx.getDataColumnNums(); - } else { - // case all the columns are selected - projectedColumns = new int[rbCtx.getRowColumnTypeInfos().length]; - for (int i = 0; i < projectedColumns.length; i++) { - projectedColumns[i] = i; - } - } - this.serDe = createAndInitializeSerde(jobConf); - this.baseReader = Preconditions.checkNotNull(reader); - - // row parser and row assigner initializing - try { - vectorAssignRow.init((StructObjectInspector) serDe.getObjectInspector()); - } catch (HiveException e) { - throw new RuntimeException(e); - } - - druidWritable = baseReader.createValue(); - rowBoat = new Object[rbCtx.getDataColumnCount()]; - } - - @Override public boolean next(NullWritable nullWritable, VectorizedRowBatch vectorizedRowBatch) throws IOException { - vectorizedRowBatch.reset(); - int rowsCount = 0; - while (rowsCount < vectorizedRowBatch.getMaxSize() && baseReader.next(nullWritable, druidWritable)) { - if (projectedColumns.length > 0) { - try { - serDe.deserializeAsPrimitive(druidWritable, rowBoat); - } catch (SerDeException e) { - throw new IOException(e); - } - for (int i : projectedColumns) { - vectorAssignRow.assignRowColumn(vectorizedRowBatch, rowsCount, i, rowBoat[i]); - } - } - rowsCount++; - } - vectorizedRowBatch.size = rowsCount; - return rowsCount > 0; - } - - @Override public NullWritable createKey() { - return NullWritable.get(); - } - - @Override public VectorizedRowBatch createValue() { - return rbCtx.createVectorizedRowBatch(); - } - - @Override public long getPos() throws IOException { - return baseReader.getPos(); - } - - @Override public void close() throws IOException { - baseReader.close(); - } - - @Override public float getProgress() throws IOException { - return baseReader.getProgress(); - } - - private static DruidSerDe createAndInitializeSerde(Configuration jobConf) { - DruidSerDe serDe = new DruidSerDe(); - MapWork mapWork = Preconditions.checkNotNull(Utilities.getMapWork(jobConf), "Map work is null"); - Iterator partitionIterator = mapWork.getPartitionDescs(); - if (!partitionIterator.hasNext()) { - throw new RuntimeException("Can not find table property at the map work"); - } - Properties properties = partitionIterator.next().getTableDesc().getProperties(); - try { - serDe.initialize(jobConf, properties, null); - } catch (SerDeException e) { - throw new RuntimeException("Can not initialized the serde", e); - } - return serDe; - } -} diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/io/HiveDruidSplit.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/io/HiveDruidSplit.java deleted file mode 100644 index 89d74e38bfc5..000000000000 --- a/druid-handler/src/java/org/apache/hadoop/hive/druid/io/HiveDruidSplit.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.druid.io; - -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; -import java.util.Arrays; - -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.mapred.FileSplit; - -/** - * Druid split. Its purpose is to trigger query execution in Druid. - */ -public class HiveDruidSplit extends FileSplit implements org.apache.hadoop.mapred.InputSplit { - - private String druidQuery; - - private String[] hosts; - - // required for deserialization - public HiveDruidSplit() { - super(null, 0, 0, (String[]) null); - } - - public HiveDruidSplit(String druidQuery, Path dummyPath, String[] hosts) { - super(dummyPath, 0, 0, hosts); - this.druidQuery = druidQuery; - this.hosts = hosts; - } - - @Override public void write(DataOutput out) throws IOException { - super.write(out); - out.writeUTF(druidQuery); - out.writeInt(hosts.length); - for (String host : hosts) { - out.writeUTF(host); - } - } - - @Override public void readFields(DataInput in) throws IOException { - super.readFields(in); - druidQuery = in.readUTF(); - int length = in.readInt(); - String[] listHosts = new String[length]; - for (int i = 0; i < length; i++) { - listHosts[i] = in.readUTF(); - } - hosts = listHosts; - } - - public String getDruidQuery() { - return druidQuery; - } - - @Override public String[] getLocations() throws IOException { - return hosts; - } - - @Override public String toString() { - return "HiveDruidSplit{" + druidQuery + ", " + (hosts == null ? "empty hosts" : Arrays.toString(hosts)) + "}"; - } - -} diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/io/package-info.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/io/package-info.java deleted file mode 100644 index e5aac26f1bc8..000000000000 --- a/druid-handler/src/java/org/apache/hadoop/hive/druid/io/package-info.java +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * Package info file. - */ - -package org.apache.hadoop.hive.druid.io; diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/json/AvroBytesDecoder.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/json/AvroBytesDecoder.java deleted file mode 100644 index 199987a9c105..000000000000 --- a/druid-handler/src/java/org/apache/hadoop/hive/druid/json/AvroBytesDecoder.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.druid.json; - -import com.fasterxml.jackson.annotation.JsonSubTypes; -import com.fasterxml.jackson.annotation.JsonTypeInfo; - - -/** - * This class is copied from druid source code - * in order to avoid adding additional dependencies on druid-indexing-service. - */ -@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type") @JsonSubTypes(value = { - @JsonSubTypes.Type(name = "schema_inline", value = InlineSchemaAvroBytesDecoder.class) }) -public interface AvroBytesDecoder { -} diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/json/AvroParseSpec.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/json/AvroParseSpec.java deleted file mode 100644 index 48d6cf2d6c8a..000000000000 --- a/druid-handler/src/java/org/apache/hadoop/hive/druid/json/AvroParseSpec.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.druid.json; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.fasterxml.jackson.annotation.JsonProperty; -import org.apache.druid.data.input.impl.DimensionsSpec; -import org.apache.druid.data.input.impl.ParseSpec; -import org.apache.druid.data.input.impl.TimestampSpec; -import org.apache.druid.java.util.common.parsers.JSONPathSpec; -import org.apache.druid.java.util.common.parsers.Parser; - -import java.util.Objects; - -/** - * This class is copied from druid source code - * in order to avoid adding additional dependencies on druid-indexing-service. - */ -public class AvroParseSpec extends ParseSpec { - - @JsonIgnore private final JSONPathSpec flattenSpec; - - @JsonCreator public AvroParseSpec(@JsonProperty("timestampSpec") TimestampSpec timestampSpec, - @JsonProperty("dimensionsSpec") DimensionsSpec dimensionsSpec, - @JsonProperty("flattenSpec") JSONPathSpec flattenSpec) { - super(timestampSpec != null ? timestampSpec : new TimestampSpec(null, null, null), - dimensionsSpec != null ? dimensionsSpec : new DimensionsSpec(null, null, null)); - - this.flattenSpec = flattenSpec != null ? flattenSpec : JSONPathSpec.DEFAULT; - } - - @JsonProperty public JSONPathSpec getFlattenSpec() { - return flattenSpec; - } - - @Override public Parser makeParser() { - // makeParser is only used by StringInputRowParser, which cannot parse avro anyway. - throw new UnsupportedOperationException("makeParser not supported"); - } - - @Override public ParseSpec withTimestampSpec(TimestampSpec spec) { - return new AvroParseSpec(spec, getDimensionsSpec(), flattenSpec); - } - - @Override public ParseSpec withDimensionsSpec(DimensionsSpec spec) { - return new AvroParseSpec(getTimestampSpec(), spec, flattenSpec); - } - - @Override public boolean equals(final Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - if (!super.equals(o)) { - return false; - } - final AvroParseSpec that = (AvroParseSpec) o; - return Objects.equals(flattenSpec, that.flattenSpec); - } - - @Override public int hashCode() { - return Objects.hash(super.hashCode(), flattenSpec); - } -} diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/json/AvroStreamInputRowParser.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/json/AvroStreamInputRowParser.java deleted file mode 100644 index b689b63031a6..000000000000 --- a/druid-handler/src/java/org/apache/hadoop/hive/druid/json/AvroStreamInputRowParser.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.druid.json; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.common.base.Preconditions; -import org.apache.druid.data.input.ByteBufferInputRowParser; -import org.apache.druid.data.input.InputRow; -import org.apache.druid.data.input.impl.ParseSpec; - -import javax.validation.constraints.NotNull; -import java.nio.ByteBuffer; -import java.util.List; -import java.util.Objects; - -/** - * This class is copied from druid source code - * in order to avoid adding additional dependencies on druid-indexing-service. - */ -public class AvroStreamInputRowParser implements ByteBufferInputRowParser { - private final ParseSpec parseSpec; - private final AvroBytesDecoder avroBytesDecoder; - - @JsonCreator public AvroStreamInputRowParser(@JsonProperty("parseSpec") ParseSpec parseSpec, - @JsonProperty("avroBytesDecoder") AvroBytesDecoder avroBytesDecoder) { - this.parseSpec = Preconditions.checkNotNull(parseSpec, "parseSpec"); - this.avroBytesDecoder = Preconditions.checkNotNull(avroBytesDecoder, "avroBytesDecoder"); - } - - @NotNull @Override public List parseBatch(ByteBuffer input) { - throw new UnsupportedOperationException("This class is only used for JSON serde"); - } - - @JsonProperty @Override public ParseSpec getParseSpec() { - return parseSpec; - } - - @JsonProperty public AvroBytesDecoder getAvroBytesDecoder() { - return avroBytesDecoder; - } - - @Override public ByteBufferInputRowParser withParseSpec(ParseSpec parseSpec) { - return new AvroStreamInputRowParser(parseSpec, avroBytesDecoder); - } - - @Override public boolean equals(final Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - final AvroStreamInputRowParser that = (AvroStreamInputRowParser) o; - return Objects.equals(parseSpec, that.parseSpec) && Objects.equals(avroBytesDecoder, that.avroBytesDecoder); - } - - @Override public int hashCode() { - return Objects.hash(parseSpec, avroBytesDecoder); - } -} diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/json/InlineSchemaAvroBytesDecoder.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/json/InlineSchemaAvroBytesDecoder.java deleted file mode 100644 index 1b6a08e74668..000000000000 --- a/druid-handler/src/java/org/apache/hadoop/hive/druid/json/InlineSchemaAvroBytesDecoder.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.druid.json; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.common.base.Preconditions; - -import java.util.Map; - -/** - * This class is copied from druid source code - * in order to avoid adding additional dependencies on druid-indexing-service. - */ -public class InlineSchemaAvroBytesDecoder implements AvroBytesDecoder { - private final Map schema; - - @JsonCreator public InlineSchemaAvroBytesDecoder(@JsonProperty("schema") Map schema) { - Preconditions.checkArgument(schema != null, "schema must be provided"); - - this.schema = schema; - } - - @JsonProperty public Map getSchema() { - return schema; - } - -} diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/json/KafkaIndexTaskTuningConfig.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/json/KafkaIndexTaskTuningConfig.java deleted file mode 100644 index 92800cef5fd9..000000000000 --- a/druid-handler/src/java/org/apache/hadoop/hive/druid/json/KafkaIndexTaskTuningConfig.java +++ /dev/null @@ -1,128 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.druid.json; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import org.apache.druid.segment.IndexSpec; -import org.apache.druid.segment.writeout.SegmentWriteOutMediumFactory; -import org.joda.time.Period; - -import javax.annotation.Nullable; -import java.io.File; - -/** - * This class is copied from druid source code - * in order to avoid adding additional dependencies on druid-indexing-service. - */ -public class KafkaIndexTaskTuningConfig extends SeekableStreamIndexTaskTuningConfig { - @JsonCreator - public KafkaIndexTaskTuningConfig( - @JsonProperty("maxRowsInMemory") @Nullable Integer maxRowsInMemory, - @JsonProperty("maxBytesInMemory") @Nullable Long maxBytesInMemory, - @JsonProperty("maxRowsPerSegment") @Nullable Integer maxRowsPerSegment, - @JsonProperty("maxTotalRows") @Nullable Long maxTotalRows, - @JsonProperty("intermediatePersistPeriod") @Nullable Period intermediatePersistPeriod, - @JsonProperty("basePersistDirectory") @Nullable File basePersistDirectory, - @JsonProperty("maxPendingPersists") @Nullable Integer maxPendingPersists, - @JsonProperty("indexSpec") @Nullable IndexSpec indexSpec, - @JsonProperty("indexSpecForIntermediatePersists") @Nullable IndexSpec indexSpecForIntermediatePersists, - // This parameter is left for compatibility when reading existing configs, to be removed in Druid 0.12. - @JsonProperty("buildV9Directly") @Nullable Boolean buildV9Directly, - @Deprecated @JsonProperty("reportParseExceptions") @Nullable Boolean reportParseExceptions, - @JsonProperty("handoffConditionTimeout") @Nullable Long handoffConditionTimeout, - @JsonProperty("resetOffsetAutomatically") @Nullable Boolean resetOffsetAutomatically, - @JsonProperty("segmentWriteOutMediumFactory") @Nullable SegmentWriteOutMediumFactory segmentWriteOutMediumFactory, - @JsonProperty("intermediateHandoffPeriod") @Nullable Period intermediateHandoffPeriod, - @JsonProperty("logParseExceptions") @Nullable Boolean logParseExceptions, - @JsonProperty("maxParseExceptions") @Nullable Integer maxParseExceptions, - @JsonProperty("maxSavedParseExceptions") @Nullable Integer maxSavedParseExceptions - ) { - super( - maxRowsInMemory, - maxBytesInMemory, - maxRowsPerSegment, - maxTotalRows, - intermediatePersistPeriod, - basePersistDirectory, - maxPendingPersists, - indexSpec, - indexSpecForIntermediatePersists, - true, - reportParseExceptions, - handoffConditionTimeout, - resetOffsetAutomatically, - false, - segmentWriteOutMediumFactory, - intermediateHandoffPeriod, - logParseExceptions, - maxParseExceptions, - maxSavedParseExceptions - ); - } - - @Override - public KafkaIndexTaskTuningConfig withBasePersistDirectory(File dir) { - return new KafkaIndexTaskTuningConfig( - getMaxRowsInMemory(), - getMaxBytesInMemory(), - getMaxRowsPerSegment(), - getMaxTotalRows(), - getIntermediatePersistPeriod(), - dir, - getMaxPendingPersists(), - getIndexSpec(), - getIndexSpecForIntermediatePersists(), - true, - isReportParseExceptions(), - getHandoffConditionTimeout(), - isResetOffsetAutomatically(), - getSegmentWriteOutMediumFactory(), - getIntermediateHandoffPeriod(), - isLogParseExceptions(), - getMaxParseExceptions(), - getMaxSavedParseExceptions() - ); - } - - - @Override - public String toString() { - return "KafkaIndexTaskTuningConfig{" + - "maxRowsInMemory=" + getMaxRowsInMemory() + - ", maxRowsPerSegment=" + getMaxRowsPerSegment() + - ", maxTotalRows=" + getMaxTotalRows() + - ", maxBytesInMemory=" + getMaxBytesInMemory() + - ", intermediatePersistPeriod=" + getIntermediatePersistPeriod() + - ", basePersistDirectory=" + getBasePersistDirectory() + - ", maxPendingPersists=" + getMaxPendingPersists() + - ", indexSpec=" + getIndexSpec() + - ", indexSpecForIntermediatePersists=" + getIndexSpecForIntermediatePersists() + - ", reportParseExceptions=" + isReportParseExceptions() + - ", handoffConditionTimeout=" + getHandoffConditionTimeout() + - ", resetOffsetAutomatically=" + isResetOffsetAutomatically() + - ", segmentWriteOutMediumFactory=" + getSegmentWriteOutMediumFactory() + - ", intermediateHandoffPeriod=" + getIntermediateHandoffPeriod() + - ", logParseExceptions=" + isLogParseExceptions() + - ", maxParseExceptions=" + getMaxParseExceptions() + - ", maxSavedParseExceptions=" + getMaxSavedParseExceptions() + - '}'; - } - -} diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/json/KafkaSupervisorIOConfig.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/json/KafkaSupervisorIOConfig.java deleted file mode 100644 index 9f5fc48e6e6d..000000000000 --- a/druid-handler/src/java/org/apache/hadoop/hive/druid/json/KafkaSupervisorIOConfig.java +++ /dev/null @@ -1,238 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.druid.json; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.common.base.Optional; -import com.google.common.base.Preconditions; -import org.apache.druid.java.util.common.StringUtils; -import org.joda.time.Duration; -import org.joda.time.Period; - -import java.util.Map; - -/** - * This class is copied from druid source code - * in order to avoid adding additional dependencies on druid-indexing-service. - */ -public class KafkaSupervisorIOConfig { - public static final String BOOTSTRAP_SERVERS_KEY = "bootstrap.servers"; - - private final String topic; - private final Integer replicas; - private final Integer taskCount; - private final Duration taskDuration; - private final Map consumerProperties; - private final Duration startDelay; - private final Duration period; - private final boolean useEarliestOffset; - private final Duration completionTimeout; - @SuppressWarnings({ "OptionalUsedAsFieldOrParameterType", "Guava" }) private final Optional - lateMessageRejectionPeriod; - @SuppressWarnings({ "OptionalUsedAsFieldOrParameterType", "Guava" }) private final Optional - earlyMessageRejectionPeriod; - private final boolean skipOffsetGaps; - - @JsonCreator public KafkaSupervisorIOConfig(@JsonProperty("topic") String topic, - @JsonProperty("replicas") Integer replicas, - @JsonProperty("taskCount") Integer taskCount, - @JsonProperty("taskDuration") Period taskDuration, - @JsonProperty("consumerProperties") Map consumerProperties, - @JsonProperty("startDelay") Period startDelay, - @JsonProperty("period") Period period, - @JsonProperty("useEarliestOffset") Boolean useEarliestOffset, - @JsonProperty("completionTimeout") Period completionTimeout, - @JsonProperty("lateMessageRejectionPeriod") Period lateMessageRejectionPeriod, - @JsonProperty("earlyMessageRejectionPeriod") Period earlyMessageRejectionPeriod, - @JsonProperty("skipOffsetGaps") Boolean skipOffsetGaps) { - this.topic = Preconditions.checkNotNull(topic, "topic"); - this.consumerProperties = Preconditions.checkNotNull(consumerProperties, "consumerProperties"); - Preconditions.checkNotNull(consumerProperties.get(BOOTSTRAP_SERVERS_KEY), - StringUtils.format("consumerProperties must contain entry for [%s]", BOOTSTRAP_SERVERS_KEY)); - - this.replicas = replicas != null ? replicas : 1; - this.taskCount = taskCount != null ? taskCount : 1; - this.taskDuration = defaultDuration(taskDuration, "PT1H"); - this.startDelay = defaultDuration(startDelay, "PT5S"); - this.period = defaultDuration(period, "PT30S"); - this.useEarliestOffset = useEarliestOffset != null ? useEarliestOffset : false; - this.completionTimeout = defaultDuration(completionTimeout, "PT30M"); - //noinspection Guava - this.lateMessageRejectionPeriod = - lateMessageRejectionPeriod == null ? - Optional.absent() : - Optional.of(lateMessageRejectionPeriod.toStandardDuration()); - //noinspection Guava - this.earlyMessageRejectionPeriod = - earlyMessageRejectionPeriod == null ? - Optional.absent() : - Optional.of(earlyMessageRejectionPeriod.toStandardDuration()); - this.skipOffsetGaps = skipOffsetGaps != null ? skipOffsetGaps : false; - } - - @JsonProperty public String getTopic() { - return topic; - } - - @JsonProperty public Integer getReplicas() { - return replicas; - } - - @JsonProperty public Integer getTaskCount() { - return taskCount; - } - - @JsonProperty public Duration getTaskDuration() { - return taskDuration; - } - - @JsonProperty public Map getConsumerProperties() { - return consumerProperties; - } - - @JsonProperty public Duration getStartDelay() { - return startDelay; - } - - @JsonProperty public Duration getPeriod() { - return period; - } - - @JsonProperty public boolean isUseEarliestOffset() { - return useEarliestOffset; - } - - @JsonProperty public Duration getCompletionTimeout() { - return completionTimeout; - } - - @SuppressWarnings("Guava") @JsonProperty public Optional getEarlyMessageRejectionPeriod() { - return earlyMessageRejectionPeriod; - } - - @SuppressWarnings("Guava") @JsonProperty public Optional getLateMessageRejectionPeriod() { - return lateMessageRejectionPeriod; - } - - @JsonProperty public boolean isSkipOffsetGaps() { - return skipOffsetGaps; - } - - @Override public String toString() { - return "KafkaSupervisorIOConfig{" - + "topic='" - + topic - + '\'' - + ", replicas=" - + replicas - + ", taskCount=" - + taskCount - + ", taskDuration=" - + taskDuration - + ", consumerProperties=" - + consumerProperties - + ", startDelay=" - + startDelay - + ", period=" - + period - + ", useEarliestOffset=" - + useEarliestOffset - + ", completionTimeout=" - + completionTimeout - + ", lateMessageRejectionPeriod=" - + lateMessageRejectionPeriod - + ", skipOffsetGaps=" - + skipOffsetGaps - + '}'; - } - - private static Duration defaultDuration(final Period period, final String theDefault) { - return (period == null ? new Period(theDefault) : period).toStandardDuration(); - } - - @Override public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - - KafkaSupervisorIOConfig that = (KafkaSupervisorIOConfig) o; - - if (useEarliestOffset != that.useEarliestOffset) { - return false; - } - if (skipOffsetGaps != that.skipOffsetGaps) { - return false; - } - if (topic != null ? !topic.equals(that.topic) : that.topic != null) { - return false; - } - if (replicas != null ? !replicas.equals(that.replicas) : that.replicas != null) { - return false; - } - if (taskCount != null ? !taskCount.equals(that.taskCount) : that.taskCount != null) { - return false; - } - if (taskDuration != null ? !taskDuration.equals(that.taskDuration) : that.taskDuration != null) { - return false; - } - if (consumerProperties != null ? - !consumerProperties.equals(that.consumerProperties) : - that.consumerProperties != null) { - return false; - } - if (startDelay != null ? !startDelay.equals(that.startDelay) : that.startDelay != null) { - return false; - } - if (period != null ? !period.equals(that.period) : that.period != null) { - return false; - } - if (completionTimeout != null ? - !completionTimeout.equals(that.completionTimeout) : - that.completionTimeout != null) { - return false; - } - if (lateMessageRejectionPeriod.isPresent() ? - !lateMessageRejectionPeriod.equals(that.lateMessageRejectionPeriod) : - that.lateMessageRejectionPeriod.isPresent()) { - return false; - } - return earlyMessageRejectionPeriod.isPresent() ? - earlyMessageRejectionPeriod.equals(that.earlyMessageRejectionPeriod) : - !that.earlyMessageRejectionPeriod.isPresent(); - } - - @Override public int hashCode() { - int result = topic != null ? topic.hashCode() : 0; - result = 31 * result + (replicas != null ? replicas.hashCode() : 0); - result = 31 * result + (taskCount != null ? taskCount.hashCode() : 0); - result = 31 * result + (taskDuration != null ? taskDuration.hashCode() : 0); - result = 31 * result + (consumerProperties != null ? consumerProperties.hashCode() : 0); - result = 31 * result + (startDelay != null ? startDelay.hashCode() : 0); - result = 31 * result + (period != null ? period.hashCode() : 0); - result = 31 * result + (useEarliestOffset ? 1 : 0); - result = 31 * result + (completionTimeout != null ? completionTimeout.hashCode() : 0); - result = 31 * result + (lateMessageRejectionPeriod.isPresent() ? lateMessageRejectionPeriod.hashCode() : 0); - result = 31 * result + (earlyMessageRejectionPeriod.isPresent() ? earlyMessageRejectionPeriod.hashCode() : 0); - result = 31 * result + (skipOffsetGaps ? 1 : 0); - return result; - } -} diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/json/KafkaSupervisorReport.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/json/KafkaSupervisorReport.java deleted file mode 100644 index bca1c7896848..000000000000 --- a/druid-handler/src/java/org/apache/hadoop/hive/druid/json/KafkaSupervisorReport.java +++ /dev/null @@ -1,202 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.druid.json; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.common.collect.Lists; -import org.apache.druid.indexing.overlord.supervisor.SupervisorReport; -import org.joda.time.DateTime; - -import javax.annotation.Nullable; -import java.util.List; -import java.util.Map; -import java.util.Objects; - -/** - * This class is copied from druid source code - * in order to avoid adding additional dependencies on druid-indexing-service. - */ -public class KafkaSupervisorReport extends SupervisorReport { - /** - * Report Payload class. - */ - public static class KafkaSupervisorReportPayload { - private final String dataSource; - private final String topic; - private final Integer partitions; - private final Integer replicas; - private final Long durationSeconds; - private final List activeTasks; - private final List publishingTasks; - private final Map latestOffsets; - private final Map minimumLag; - private final Long aggregateLag; - private final DateTime offsetsLastUpdated; - - @JsonCreator public KafkaSupervisorReportPayload(@JsonProperty("dataSource") String dataSource, - @JsonProperty("topic") String topic, - @JsonProperty("partitions") Integer partitions, - @JsonProperty("replicas") Integer replicas, - @JsonProperty("durationSeconds") Long durationSeconds, - @Nullable @JsonProperty("latestOffsets") Map latestOffsets, - @Nullable @JsonProperty("minimumLag") Map minimumLag, - @Nullable @JsonProperty("aggregateLag") Long aggregateLag, - @Nullable @JsonProperty("offsetsLastUpdated") DateTime offsetsLastUpdated) { - this.dataSource = dataSource; - this.topic = topic; - this.partitions = partitions; - this.replicas = replicas; - this.durationSeconds = durationSeconds; - this.activeTasks = Lists.newArrayList(); - this.publishingTasks = Lists.newArrayList(); - this.latestOffsets = latestOffsets; - this.minimumLag = minimumLag; - this.aggregateLag = aggregateLag; - this.offsetsLastUpdated = offsetsLastUpdated; - } - - @JsonProperty public String getDataSource() { - return dataSource; - } - - @JsonProperty public String getTopic() { - return topic; - } - - @JsonProperty public Integer getPartitions() { - return partitions; - } - - @JsonProperty public Integer getReplicas() { - return replicas; - } - - @JsonProperty public Long getDurationSeconds() { - return durationSeconds; - } - - @JsonProperty public List getActiveTasks() { - return activeTasks; - } - - @JsonProperty public List getPublishingTasks() { - return publishingTasks; - } - - @JsonProperty @JsonInclude(JsonInclude.Include.NON_NULL) public Map getLatestOffsets() { - return latestOffsets; - } - - @JsonProperty @JsonInclude(JsonInclude.Include.NON_NULL) public Map getMinimumLag() { - return minimumLag; - } - - @JsonProperty @JsonInclude(JsonInclude.Include.NON_NULL) public Long getAggregateLag() { - return aggregateLag; - } - - @JsonProperty public DateTime getOffsetsLastUpdated() { - return offsetsLastUpdated; - } - - @Override public String toString() { - return "{" - + "dataSource='" - + dataSource - + '\'' - + ", topic='" - + topic - + '\'' - + ", partitions=" - + partitions - + ", replicas=" - + replicas - + ", durationSeconds=" - + durationSeconds - + ", active=" - + activeTasks - + ", publishing=" - + publishingTasks - + (latestOffsets != null ? ", latestOffsets=" + latestOffsets : "") - + (minimumLag != null ? ", minimumLag=" + minimumLag : "") - + (aggregateLag != null ? ", aggregateLag=" + aggregateLag : "") - + (offsetsLastUpdated != null ? ", offsetsLastUpdated=" + offsetsLastUpdated : "") - + '}'; - } - } - - private final KafkaSupervisorReportPayload payload; - - @JsonCreator public KafkaSupervisorReport(@JsonProperty("id") String id, - @JsonProperty("generationTime") DateTime generationTime, - @JsonProperty("payload") KafkaSupervisorReportPayload payload) { - super(id, generationTime, payload); - this.payload = payload; - } - - public KafkaSupervisorReport(String dataSource, - DateTime generationTime, - String topic, - Integer partitions, - Integer replicas, - Long durationSeconds, - @Nullable Map latestOffsets, - @Nullable Map minimumLag, - @Nullable Long aggregateLag, - @Nullable DateTime offsetsLastUpdated) { - this(dataSource, - generationTime, - new KafkaSupervisorReportPayload(dataSource, - topic, - partitions, - replicas, - durationSeconds, - latestOffsets, - minimumLag, - aggregateLag, - offsetsLastUpdated)); - } - - @Override public KafkaSupervisorReportPayload getPayload() { - return payload; - } - - @Override public String toString() { - return "{" + "id='" + getId() + '\'' + ", generationTime=" + getGenerationTime() + ", payload=" + payload + '}'; - } - - @Override public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - if (!super.equals(o)) { - return false; - } - KafkaSupervisorReport that = (KafkaSupervisorReport) o; - return Objects.equals(payload, that.payload); - } - - @Override public int hashCode() { - return Objects.hash(super.hashCode(), payload); - } -} diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/json/KafkaSupervisorSpec.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/json/KafkaSupervisorSpec.java deleted file mode 100644 index 18177b6640cd..000000000000 --- a/druid-handler/src/java/org/apache/hadoop/hive/druid/json/KafkaSupervisorSpec.java +++ /dev/null @@ -1,117 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.druid.json; - -import org.apache.druid.segment.indexing.DataSchema; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonSubTypes; -import com.fasterxml.jackson.annotation.JsonTypeInfo; -import com.google.common.base.Preconditions; - -import java.util.Map; - -/** - * This class is copied from druid source code - * in order to avoid adding additional dependencies on druid-indexing-service. - */ -@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type") @JsonSubTypes({ - @JsonSubTypes.Type(name = "kafka", value = KafkaSupervisorSpec.class) }) public class KafkaSupervisorSpec { - private final DataSchema dataSchema; - private final KafkaSupervisorTuningConfig tuningConfig; - private final KafkaSupervisorIOConfig ioConfig; - private final Map context; - - @JsonCreator public KafkaSupervisorSpec(@JsonProperty("dataSchema") DataSchema dataSchema, - @JsonProperty("tuningConfig") KafkaSupervisorTuningConfig tuningConfig, - @JsonProperty("ioConfig") KafkaSupervisorIOConfig ioConfig, - @JsonProperty("context") Map context) { - this.dataSchema = Preconditions.checkNotNull(dataSchema, "dataSchema"); - this.tuningConfig = - tuningConfig != null ? - tuningConfig : - new KafkaSupervisorTuningConfig(null, - null, - null, - null, - null, - null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, - null, null); - this.ioConfig = Preconditions.checkNotNull(ioConfig, "ioConfig"); - this.context = context; - } - - @JsonProperty public DataSchema getDataSchema() { - return dataSchema; - } - - @JsonProperty public KafkaSupervisorTuningConfig getTuningConfig() { - return tuningConfig; - } - - @JsonProperty public KafkaSupervisorIOConfig getIoConfig() { - return ioConfig; - } - - @JsonProperty public Map getContext() { - return context; - } - - @Override public String toString() { - return "KafkaSupervisorSpec{" - + "dataSchema=" - + dataSchema - + ", tuningConfig=" - + tuningConfig - + ", ioConfig=" - + ioConfig - + '}'; - } - - @Override public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - - KafkaSupervisorSpec that = (KafkaSupervisorSpec) o; - - if (dataSchema != null ? !dataSchema.equals(that.dataSchema) : that.dataSchema != null) { - return false; - } - if (tuningConfig != null ? !tuningConfig.equals(that.tuningConfig) : that.tuningConfig != null) { - return false; - } - if (ioConfig != null ? !ioConfig.equals(that.ioConfig) : that.ioConfig != null) { - return false; - } - return context != null ? context.equals(that.context) : that.context == null; - } - - @Override public int hashCode() { - int result = dataSchema != null ? dataSchema.hashCode() : 0; - result = 31 * result + (tuningConfig != null ? tuningConfig.hashCode() : 0); - result = 31 * result + (ioConfig != null ? ioConfig.hashCode() : 0); - result = 31 * result + (context != null ? context.hashCode() : 0); - return result; - } -} diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/json/KafkaSupervisorTuningConfig.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/json/KafkaSupervisorTuningConfig.java deleted file mode 100644 index 4e171612b39b..000000000000 --- a/druid-handler/src/java/org/apache/hadoop/hive/druid/json/KafkaSupervisorTuningConfig.java +++ /dev/null @@ -1,161 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.druid.json; - -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonSubTypes; -import com.fasterxml.jackson.annotation.JsonTypeInfo; -import org.apache.druid.segment.IndexSpec; -import org.apache.druid.segment.indexing.TuningConfigs; -import org.apache.druid.segment.writeout.SegmentWriteOutMediumFactory; -import org.joda.time.Duration; -import org.joda.time.Period; - -import javax.annotation.Nullable; -import java.io.File; - -/** - * This class is copied from druid source code - * in order to avoid adding additional dependencies on druid-indexing-service. - */ -@SuppressWarnings("unused") -@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type") -@JsonSubTypes({ - @JsonSubTypes.Type(name = "kafka", value = KafkaSupervisorTuningConfig.class)}) -public class KafkaSupervisorTuningConfig - extends KafkaIndexTaskTuningConfig implements SeekableStreamSupervisorTuningConfig { - private static final String DEFAULT_OFFSET_FETCH_PERIOD = "PT30S"; - - private final Integer workerThreads; - private final Integer chatThreads; - private final Long chatRetries; - private final Duration httpTimeout; - private final Duration shutdownTimeout; - private final Duration offsetFetchPeriod; - - public static KafkaSupervisorTuningConfig defaultConfig() { - return new KafkaSupervisorTuningConfig(null, null, null, null, null, null, null, null, null, null, null, null, null, - null, null, null, null, null, null, null, null, null, null, null); - } - - public KafkaSupervisorTuningConfig( - @JsonProperty("maxRowsInMemory") Integer maxRowsInMemory, - @JsonProperty("maxBytesInMemory") Long maxBytesInMemory, - @JsonProperty("maxRowsPerSegment") Integer maxRowsPerSegment, @JsonProperty("maxTotalRows") Long maxTotalRows, - @JsonProperty("intermediatePersistPeriod") Period intermediatePersistPeriod, - @JsonProperty("basePersistDirectory") File basePersistDirectory, - @JsonProperty("maxPendingPersists") Integer maxPendingPersists, @JsonProperty("indexSpec") IndexSpec indexSpec, - @JsonProperty("indexSpecForIntermediatePersists") @Nullable IndexSpec indexSpecForIntermediatePersists, - // This parameter is left for compatibility when reading existing configs, to be removed in Druid 0.12. - @JsonProperty("buildV9Directly") Boolean buildV9Directly, - @JsonProperty("reportParseExceptions") Boolean reportParseExceptions, - @JsonProperty("handoffConditionTimeout") Long handoffConditionTimeout, - @JsonProperty("resetOffsetAutomatically") Boolean resetOffsetAutomatically, - @JsonProperty("segmentWriteOutMediumFactory") @Nullable SegmentWriteOutMediumFactory segmentWriteOutMediumFactory, - @JsonProperty("workerThreads") Integer workerThreads, @JsonProperty("chatThreads") Integer chatThreads, - @JsonProperty("chatRetries") Long chatRetries, @JsonProperty("httpTimeout") Period httpTimeout, - @JsonProperty("shutdownTimeout") Period shutdownTimeout, - @JsonProperty("offsetFetchPeriod") Period offsetFetchPeriod, - @JsonProperty("intermediateHandoffPeriod") Period intermediateHandoffPeriod, - @JsonProperty("logParseExceptions") @Nullable Boolean logParseExceptions, - @JsonProperty("maxParseExceptions") @Nullable Integer maxParseExceptions, - @JsonProperty("maxSavedParseExceptions") @Nullable Integer maxSavedParseExceptions) { - super(maxRowsInMemory, maxBytesInMemory, maxRowsPerSegment, maxTotalRows, intermediatePersistPeriod, - basePersistDirectory, maxPendingPersists, indexSpec, indexSpecForIntermediatePersists, true, - reportParseExceptions, handoffConditionTimeout, resetOffsetAutomatically, segmentWriteOutMediumFactory, - intermediateHandoffPeriod, logParseExceptions, maxParseExceptions, maxSavedParseExceptions); - this.workerThreads = workerThreads; - this.chatThreads = chatThreads; - this.chatRetries = (chatRetries != null ? chatRetries : DEFAULT_CHAT_RETRIES); - this.httpTimeout = SeekableStreamSupervisorTuningConfig.defaultDuration(httpTimeout, DEFAULT_HTTP_TIMEOUT); - this.shutdownTimeout = - SeekableStreamSupervisorTuningConfig.defaultDuration(shutdownTimeout, DEFAULT_SHUTDOWN_TIMEOUT); - this.offsetFetchPeriod = - SeekableStreamSupervisorTuningConfig.defaultDuration(offsetFetchPeriod, DEFAULT_OFFSET_FETCH_PERIOD); - } - - @Override - @JsonProperty - public Integer getWorkerThreads() { - return workerThreads; - } - - @Override - @JsonProperty - public Integer getChatThreads() { - return chatThreads; - } - - @Override - @JsonProperty - public Long getChatRetries() { - return chatRetries; - } - - @Override - @JsonProperty - public Duration getHttpTimeout() { - return httpTimeout; - } - - @Override - @JsonProperty - public Duration getShutdownTimeout() { - return shutdownTimeout; - } - - @Override - public Duration getRepartitionTransitionDuration() { - // Stopping tasks early for Kafka ingestion on partition set change is not supported yet, - // just return a default for now. - return SeekableStreamSupervisorTuningConfig - .defaultDuration(null, SeekableStreamSupervisorTuningConfig.DEFAULT_REPARTITION_TRANSITION_DURATION); - } - - @JsonProperty - public Duration getOffsetFetchPeriod() { - return offsetFetchPeriod; - } - - @Override - public String toString() { - return "KafkaSupervisorTuningConfig{" + "maxRowsInMemory=" + getMaxRowsInMemory() + ", maxRowsPerSegment=" - + getMaxRowsPerSegment() + ", maxTotalRows=" + getMaxTotalRows() + ", maxBytesInMemory=" + TuningConfigs - .getMaxBytesInMemoryOrDefault(getMaxBytesInMemory()) + ", intermediatePersistPeriod=" - + getIntermediatePersistPeriod() + ", basePersistDirectory=" + getBasePersistDirectory() - + ", maxPendingPersists=" + getMaxPendingPersists() + ", indexSpec=" + getIndexSpec() - + ", reportParseExceptions=" + isReportParseExceptions() + ", handoffConditionTimeout=" - + getHandoffConditionTimeout() + ", resetOffsetAutomatically=" + isResetOffsetAutomatically() - + ", segmentWriteOutMediumFactory=" + getSegmentWriteOutMediumFactory() + ", workerThreads=" + workerThreads - + ", chatThreads=" + chatThreads + ", chatRetries=" + chatRetries + ", httpTimeout=" + httpTimeout - + ", shutdownTimeout=" + shutdownTimeout + ", offsetFetchPeriod=" + offsetFetchPeriod - + ", intermediateHandoffPeriod=" + getIntermediateHandoffPeriod() + ", logParseExceptions=" - + isLogParseExceptions() + ", maxParseExceptions=" + getMaxParseExceptions() + ", maxSavedParseExceptions=" - + getMaxSavedParseExceptions() + '}'; - } - - @Override - public KafkaIndexTaskTuningConfig convertToTaskTuningConfig() { - return new KafkaIndexTaskTuningConfig(getMaxRowsInMemory(), getMaxBytesInMemory(), getMaxRowsPerSegment(), - getMaxTotalRows(), getIntermediatePersistPeriod(), getBasePersistDirectory(), getMaxPendingPersists(), - getIndexSpec(), getIndexSpecForIntermediatePersists(), true, isReportParseExceptions(), - getHandoffConditionTimeout(), isResetOffsetAutomatically(), getSegmentWriteOutMediumFactory(), - getIntermediateHandoffPeriod(), isLogParseExceptions(), getMaxParseExceptions(), getMaxSavedParseExceptions()); - } -} diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/json/SeekableStreamIndexTaskTuningConfig.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/json/SeekableStreamIndexTaskTuningConfig.java deleted file mode 100644 index 289f0e8d4308..000000000000 --- a/druid-handler/src/java/org/apache/hadoop/hive/druid/json/SeekableStreamIndexTaskTuningConfig.java +++ /dev/null @@ -1,308 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.druid.json; - -import com.fasterxml.jackson.annotation.JsonProperty; -import org.apache.druid.indexer.partitions.DynamicPartitionsSpec; -import org.apache.druid.segment.IndexSpec; -import org.apache.druid.segment.indexing.RealtimeTuningConfig; -import org.apache.druid.segment.indexing.TuningConfig; -import org.apache.druid.segment.realtime.appenderator.AppenderatorConfig; -import org.apache.druid.segment.writeout.SegmentWriteOutMediumFactory; -import org.joda.time.Period; - -import javax.annotation.Nullable; -import java.io.File; -import java.util.Objects; - -/** - * This class is copied from druid source code - * in order to avoid adding additional dependencies on druid-indexing-service. - */ -public abstract class SeekableStreamIndexTaskTuningConfig implements TuningConfig, AppenderatorConfig { - private static final boolean DEFAULT_RESET_OFFSET_AUTOMATICALLY = false; - private static final boolean DEFAULT_SKIP_SEQUENCE_NUMBER_AVAILABILITY_CHECK = false; - - private final int maxRowsInMemory; - private final long maxBytesInMemory; - private final DynamicPartitionsSpec partitionsSpec; - private final Period intermediatePersistPeriod; - private final File basePersistDirectory; - @Deprecated - private final int maxPendingPersists; - private final IndexSpec indexSpec; - private final IndexSpec indexSpecForIntermediatePersists; - private final boolean reportParseExceptions; - private final long handoffConditionTimeout; - private final boolean resetOffsetAutomatically; - @Nullable - private final SegmentWriteOutMediumFactory segmentWriteOutMediumFactory; - private final Period intermediateHandoffPeriod; - private final boolean skipSequenceNumberAvailabilityCheck; - - private final boolean logParseExceptions; - private final int maxParseExceptions; - private final int maxSavedParseExceptions; - - public SeekableStreamIndexTaskTuningConfig( - @Nullable Integer maxRowsInMemory, - @Nullable Long maxBytesInMemory, - @Nullable Integer maxRowsPerSegment, - @Nullable Long maxTotalRows, - @Nullable Period intermediatePersistPeriod, - @Nullable File basePersistDirectory, - @Nullable Integer maxPendingPersists, - @Nullable IndexSpec indexSpec, - @Nullable IndexSpec indexSpecForIntermediatePersists, - // This parameter is left for compatibility when reading existing configs, to be removed in Druid 0.12. - @Deprecated @JsonProperty("buildV9Directly") @Nullable Boolean buildV9Directly, - @Deprecated @Nullable Boolean reportParseExceptions, - @Nullable Long handoffConditionTimeout, - @Nullable Boolean resetOffsetAutomatically, - Boolean skipSequenceNumberAvailabilityCheck, - @Nullable SegmentWriteOutMediumFactory segmentWriteOutMediumFactory, - @Nullable Period intermediateHandoffPeriod, - @Nullable Boolean logParseExceptions, - @Nullable Integer maxParseExceptions, - @Nullable Integer maxSavedParseExceptions - ) { - // Cannot be a static because default basePersistDirectory is unique per-instance - final RealtimeTuningConfig defaults = RealtimeTuningConfig.makeDefaultTuningConfig(basePersistDirectory); - - this.maxRowsInMemory = maxRowsInMemory == null ? defaults.getMaxRowsInMemory() : maxRowsInMemory; - this.partitionsSpec = new DynamicPartitionsSpec(maxRowsPerSegment, maxTotalRows); - // initializing this to 0, it will be lazily initialized to a value - // @see server.src.main.java.org.apache.druid.segment.indexing.TuningConfigs#getMaxBytesInMemoryOrDefault(long) - this.maxBytesInMemory = maxBytesInMemory == null ? 0 : maxBytesInMemory; - this.intermediatePersistPeriod = intermediatePersistPeriod == null - ? defaults.getIntermediatePersistPeriod() - : intermediatePersistPeriod; - this.basePersistDirectory = defaults.getBasePersistDirectory(); - this.maxPendingPersists = maxPendingPersists == null ? 0 : maxPendingPersists; - this.indexSpec = indexSpec == null ? defaults.getIndexSpec() : indexSpec; - this.indexSpecForIntermediatePersists = indexSpecForIntermediatePersists == null ? - this.indexSpec : indexSpecForIntermediatePersists; - this.reportParseExceptions = reportParseExceptions == null - ? defaults.isReportParseExceptions() - : reportParseExceptions; - this.handoffConditionTimeout = handoffConditionTimeout == null - ? defaults.getHandoffConditionTimeout() - : handoffConditionTimeout; - this.resetOffsetAutomatically = resetOffsetAutomatically == null - ? DEFAULT_RESET_OFFSET_AUTOMATICALLY - : resetOffsetAutomatically; - this.segmentWriteOutMediumFactory = segmentWriteOutMediumFactory; - this.intermediateHandoffPeriod = intermediateHandoffPeriod == null - ? new Period().withDays(Integer.MAX_VALUE) - : intermediateHandoffPeriod; - this.skipSequenceNumberAvailabilityCheck = skipSequenceNumberAvailabilityCheck == null - ? DEFAULT_SKIP_SEQUENCE_NUMBER_AVAILABILITY_CHECK - : skipSequenceNumberAvailabilityCheck; - - if (this.reportParseExceptions) { - this.maxParseExceptions = 0; - this.maxSavedParseExceptions = maxSavedParseExceptions == null ? 0 : Math.min(1, maxSavedParseExceptions); - } else { - this.maxParseExceptions = maxParseExceptions == null - ? TuningConfig.DEFAULT_MAX_PARSE_EXCEPTIONS - : maxParseExceptions; - this.maxSavedParseExceptions = maxSavedParseExceptions == null - ? TuningConfig.DEFAULT_MAX_SAVED_PARSE_EXCEPTIONS - : maxSavedParseExceptions; - } - this.logParseExceptions = logParseExceptions == null - ? TuningConfig.DEFAULT_LOG_PARSE_EXCEPTIONS - : logParseExceptions; - } - - @Override - @JsonProperty - public int getMaxRowsInMemory() { - return maxRowsInMemory; - } - - @Override - @JsonProperty - public long getMaxBytesInMemory() { - return maxBytesInMemory; - } - - @Override - @JsonProperty - public Integer getMaxRowsPerSegment() { - return partitionsSpec.getMaxRowsPerSegment(); - } - - @JsonProperty - @Override - @Nullable - public Long getMaxTotalRows() { - return partitionsSpec.getMaxTotalRows(); - } - - @Override - public DynamicPartitionsSpec getPartitionsSpec() { - return partitionsSpec; - } - - @Override - @JsonProperty - public Period getIntermediatePersistPeriod() { - return intermediatePersistPeriod; - } - - @Override - @JsonProperty - public File getBasePersistDirectory() { - return basePersistDirectory; - } - - @Override - @JsonProperty - @Deprecated - public int getMaxPendingPersists() { - return maxPendingPersists; - } - - @Override - @JsonProperty - public IndexSpec getIndexSpec() { - return indexSpec; - } - - @JsonProperty - @Override - public IndexSpec getIndexSpecForIntermediatePersists() { - return indexSpecForIntermediatePersists; - } - - /** - * Always returns true, doesn't affect the version being built. - */ - @Deprecated - @JsonProperty - public boolean getBuildV9Directly() { - return true; - } - - @Override - @JsonProperty - public boolean isReportParseExceptions() { - return reportParseExceptions; - } - - @JsonProperty - public long getHandoffConditionTimeout() { - return handoffConditionTimeout; - } - - @JsonProperty - public boolean isResetOffsetAutomatically() { - return resetOffsetAutomatically; - } - - @Override - @JsonProperty - @Nullable - public SegmentWriteOutMediumFactory getSegmentWriteOutMediumFactory() { - return segmentWriteOutMediumFactory; - } - - @JsonProperty - public Period getIntermediateHandoffPeriod() { - return intermediateHandoffPeriod; - } - - @JsonProperty - public boolean isLogParseExceptions() { - return logParseExceptions; - } - - @JsonProperty - public int getMaxParseExceptions() { - return maxParseExceptions; - } - - @JsonProperty - public int getMaxSavedParseExceptions() { - return maxSavedParseExceptions; - } - - @JsonProperty - public boolean isSkipSequenceNumberAvailabilityCheck() { - return skipSequenceNumberAvailabilityCheck; - } - - @Override - public abstract SeekableStreamIndexTaskTuningConfig withBasePersistDirectory(File dir); - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - SeekableStreamIndexTaskTuningConfig that = (SeekableStreamIndexTaskTuningConfig) o; - return maxRowsInMemory == that.maxRowsInMemory && - maxBytesInMemory == that.maxBytesInMemory && - maxPendingPersists == that.maxPendingPersists && - reportParseExceptions == that.reportParseExceptions && - handoffConditionTimeout == that.handoffConditionTimeout && - resetOffsetAutomatically == that.resetOffsetAutomatically && - skipSequenceNumberAvailabilityCheck == that.skipSequenceNumberAvailabilityCheck && - logParseExceptions == that.logParseExceptions && - maxParseExceptions == that.maxParseExceptions && - maxSavedParseExceptions == that.maxSavedParseExceptions && - Objects.equals(partitionsSpec, that.partitionsSpec) && - Objects.equals(intermediatePersistPeriod, that.intermediatePersistPeriod) && - Objects.equals(basePersistDirectory, that.basePersistDirectory) && - Objects.equals(indexSpec, that.indexSpec) && - Objects.equals(indexSpecForIntermediatePersists, that.indexSpecForIntermediatePersists) && - Objects.equals(segmentWriteOutMediumFactory, that.segmentWriteOutMediumFactory) && - Objects.equals(intermediateHandoffPeriod, that.intermediateHandoffPeriod); - } - - @Override - public int hashCode() { - return Objects.hash( - maxRowsInMemory, - maxBytesInMemory, - partitionsSpec, - intermediatePersistPeriod, - basePersistDirectory, - maxPendingPersists, - indexSpec, - indexSpecForIntermediatePersists, - reportParseExceptions, - handoffConditionTimeout, - resetOffsetAutomatically, - segmentWriteOutMediumFactory, - intermediateHandoffPeriod, - skipSequenceNumberAvailabilityCheck, - logParseExceptions, - maxParseExceptions, - maxSavedParseExceptions - ); - } - - @Override - public abstract String toString(); -} diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/json/SeekableStreamSupervisorTuningConfig.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/json/SeekableStreamSupervisorTuningConfig.java deleted file mode 100644 index ec71518f46c8..000000000000 --- a/druid-handler/src/java/org/apache/hadoop/hive/druid/json/SeekableStreamSupervisorTuningConfig.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.druid.json; - -import com.fasterxml.jackson.annotation.JsonProperty; -import org.joda.time.Duration; -import org.joda.time.Period; - -/** - * This class is copied from druid source code - * in order to avoid adding additional dependencies on druid-indexing-service. - */ -public interface SeekableStreamSupervisorTuningConfig { - - int DEFAULT_CHAT_RETRIES = 8; - String DEFAULT_HTTP_TIMEOUT = "PT10S"; - String DEFAULT_SHUTDOWN_TIMEOUT = "PT80S"; - String DEFAULT_REPARTITION_TRANSITION_DURATION = "PT2M"; - - static Duration defaultDuration(final Period period, final String theDefault) { - return (period == null ? new Period(theDefault) : period).toStandardDuration(); - } - - @JsonProperty - Integer getWorkerThreads(); - - @JsonProperty - Integer getChatThreads(); - - @JsonProperty - Long getChatRetries(); - - @JsonProperty - Duration getHttpTimeout(); - - @JsonProperty - Duration getShutdownTimeout(); - - @JsonProperty - Duration getRepartitionTransitionDuration(); - - SeekableStreamIndexTaskTuningConfig convertToTaskTuningConfig(); -} diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/json/TaskReportData.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/json/TaskReportData.java deleted file mode 100644 index f35db6d01cba..000000000000 --- a/druid-handler/src/java/org/apache/hadoop/hive/druid/json/TaskReportData.java +++ /dev/null @@ -1,102 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.druid.json; - -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; - -import org.joda.time.DateTime; - -import java.util.Map; - -import javax.annotation.Nullable; - -/** - * This class is copied from druid source code - * in order to avoid adding additional dependencies on druid-indexing-service. - */ -@SuppressWarnings("unused") public class TaskReportData { - /** - * Task type used by serializer but does not have any functionality as far i can tell. - */ - @SuppressWarnings("unused") public enum TaskType { - ACTIVE, PUBLISHING, UNKNOWN - } - - private final String id; - private final Map startingOffsets; - private final DateTime startTime; - private final Long remainingSeconds; - private final TaskType type; - private final Map currentOffsets; - private final Map lag; - - public TaskReportData(String id, - @Nullable Map startingOffsets, - @Nullable Map currentOffsets, - DateTime startTime, - Long remainingSeconds, - TaskType type, - @Nullable Map lag) { - this.id = id; - this.startingOffsets = startingOffsets; - this.currentOffsets = currentOffsets; - this.startTime = startTime; - this.remainingSeconds = remainingSeconds; - this.type = type; - this.lag = lag; - } - - @JsonProperty public String getId() { - return id; - } - - @JsonProperty @JsonInclude(JsonInclude.Include.NON_NULL) public Map getStartingOffsets() { - return startingOffsets; - } - - @JsonProperty @JsonInclude(JsonInclude.Include.NON_NULL) public Map getCurrentOffsets() { - return currentOffsets; - } - - @JsonProperty public DateTime getStartTime() { - return startTime; - } - - @JsonProperty public Long getRemainingSeconds() { - return remainingSeconds; - } - - @JsonProperty public TaskType getType() { - return type; - } - - @JsonProperty @JsonInclude(JsonInclude.Include.NON_NULL) public Map getLag() { - return lag; - } - - @Override public String toString() { - return "{" + "id='" + id + '\'' + (startingOffsets != null ? ", startingOffsets=" + startingOffsets : "") + ( - currentOffsets != null ? - ", currentOffsets=" + currentOffsets : - "") + ", startTime=" + startTime + ", remainingSeconds=" + remainingSeconds + (lag != null ? - ", lag=" + lag : - "") + '}'; - } -} diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/json/package-info.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/json/package-info.java deleted file mode 100644 index dc53fa7de707..000000000000 --- a/druid-handler/src/java/org/apache/hadoop/hive/druid/json/package-info.java +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * package info file. - */ - -package org.apache.hadoop.hive.druid.json; diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/package-info.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/package-info.java deleted file mode 100644 index 19acf64bb93e..000000000000 --- a/druid-handler/src/java/org/apache/hadoop/hive/druid/package-info.java +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * Package info file. - */ - -package org.apache.hadoop.hive.druid; diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/security/DruidKerberosUtil.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/security/DruidKerberosUtil.java deleted file mode 100644 index 12603c10ec7e..000000000000 --- a/druid-handler/src/java/org/apache/hadoop/hive/druid/security/DruidKerberosUtil.java +++ /dev/null @@ -1,119 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.druid.security; - -import org.apache.hadoop.security.authentication.client.AuthenticatedURL; -import org.apache.hadoop.security.authentication.client.AuthenticationException; -import org.apache.hadoop.security.authentication.util.KerberosUtil; -import org.ietf.jgss.GSSContext; -import org.ietf.jgss.GSSException; -import org.ietf.jgss.GSSManager; -import org.ietf.jgss.GSSName; -import org.ietf.jgss.Oid; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.net.CookieStore; -import java.net.HttpCookie; -import java.net.URI; -import java.util.Base64; -import java.util.List; -import java.util.concurrent.locks.ReentrantLock; - -/** - * Utils class for Druid Kerberos stuff. - */ -public final class DruidKerberosUtil { - protected static final Logger LOG = LoggerFactory.getLogger(DruidKerberosUtil.class); - private static final ReentrantLock KERBEROS_LOCK = new ReentrantLock(true); - - private DruidKerberosUtil() { - } - - /** - * This method always needs to be called within a doAs block so that the client's TGT credentials - * can be read from the Subject. - * - * @return Kerberos Challenge String. - * - * @throws AuthenticationException on authentication errors. - */ - - static String kerberosChallenge(String server) throws AuthenticationException { - KERBEROS_LOCK.lock(); - try { - // This Oid for Kerberos GSS-API mechanism. - Oid mechOid = KerberosUtil.getOidInstance("GSS_KRB5_MECH_OID"); - GSSManager manager = GSSManager.getInstance(); - // GSS name for server - GSSName serverName = manager.createName("HTTP@" + server, GSSName.NT_HOSTBASED_SERVICE); - // Create a GSSContext for authentication with the service. - // We're passing client credentials as null since we want them to be read from the Subject. - GSSContext - gssContext = - manager.createContext(serverName.canonicalize(mechOid), mechOid, null, GSSContext.DEFAULT_LIFETIME); - gssContext.requestMutualAuth(true); - gssContext.requestCredDeleg(true); - // Establish context - byte[] inToken = new byte[0]; - byte[] outToken = gssContext.initSecContext(inToken, 0, inToken.length); - gssContext.dispose(); - // Base64 encoded and stringified token for server - LOG.debug("Got valid challenge for host {}", serverName); - return Base64.getEncoder().encodeToString(outToken); - } catch (GSSException | IllegalAccessException | NoSuchFieldException | ClassNotFoundException e) { - throw new AuthenticationException(e); - } finally { - KERBEROS_LOCK.unlock(); - } - } - - static HttpCookie getAuthCookie(CookieStore cookieStore, URI uri) { - if (cookieStore == null) { - return null; - } - boolean isSSL = uri.getScheme().equals("https"); - List cookies = cookieStore.getCookies(); - - for (HttpCookie c : cookies) { - // If this is a secured cookie and the current connection is non-secured, - // then, skip this cookie. We need to skip this cookie because, the cookie - // replay will not be transmitted to the server. - if (c.getSecure() && !isSSL) { - continue; - } - if (c.getName().equals(AuthenticatedURL.AUTH_COOKIE)) { - return c; - } - } - return null; - } - - static void removeAuthCookie(CookieStore cookieStore, URI uri) { - HttpCookie authCookie = getAuthCookie(cookieStore, uri); - if (authCookie != null) { - cookieStore.remove(uri, authCookie); - } - } - - static boolean needToSendCredentials(CookieStore cookieStore, URI uri) { - return getAuthCookie(cookieStore, uri) == null; - } - -} diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/security/KerberosHttpClient.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/security/KerberosHttpClient.java deleted file mode 100644 index fdbbfcc7d7f6..000000000000 --- a/druid-handler/src/java/org/apache/hadoop/hive/druid/security/KerberosHttpClient.java +++ /dev/null @@ -1,132 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.druid.security; - -import com.google.common.base.Throwables; -import com.google.common.util.concurrent.ListenableFuture; -import com.google.common.util.concurrent.SettableFuture; -import org.apache.druid.java.util.http.client.AbstractHttpClient; -import org.apache.druid.java.util.http.client.HttpClient; -import org.apache.druid.java.util.http.client.Request; -import org.apache.druid.java.util.http.client.response.HttpResponseHandler; -import org.apache.hadoop.security.UserGroupInformation; -import org.jboss.netty.handler.codec.http.HttpHeaders; -import org.joda.time.Duration; -import org.slf4j.LoggerFactory; - -import java.net.CookieManager; -import java.net.URI; -import java.security.PrivilegedExceptionAction; -import java.util.Collections; -import java.util.List; -import java.util.Map; - -/** - * This is a slightly modified version of kerberos module borrowed from druid project - * Couple of reasons behind the copy/modification instead of mvn dependency. - * 1/ Need to remove the authentication step since it not required - * 2/ To avoid some un-needed transitive dependencies that can clash on the classpath like jetty-XX. - */ -public class KerberosHttpClient extends AbstractHttpClient { - protected static final org.slf4j.Logger LOG = LoggerFactory.getLogger(KerberosHttpClient.class); - private final HttpClient delegate; - private final CookieManager cookieManager; - - public KerberosHttpClient(HttpClient delegate) { - this.delegate = delegate; - this.cookieManager = new CookieManager(); - } - - @Override public ListenableFuture go(Request request, - HttpResponseHandler httpResponseHandler, - Duration duration) { - final SettableFuture retVal = SettableFuture.create(); - innerGo(request, httpResponseHandler, duration, retVal); - return retVal; - } - - private void innerGo(final Request request, - final HttpResponseHandler httpResponseHandler, - final Duration duration, - final SettableFuture future) { - try { - final String host = request.getUrl().getHost(); - final URI uri = request.getUrl().toURI(); - - /* Cookies Manager is used to cache cookie returned by service. - The goal us to avoid doing KDC requests for every request.*/ - - Map> cookieMap = cookieManager.get(uri, Collections.emptyMap()); - for (Map.Entry> entry : cookieMap.entrySet()) { - request.addHeaderValues(entry.getKey(), entry.getValue()); - } - final boolean shouldRetryOnUnauthorizedResponse; - - if (DruidKerberosUtil.needToSendCredentials(cookieManager.getCookieStore(), uri)) { - // No Cookies for requested URI, authenticate user and add authentication header - LOG.debug("No Auth Cookie found for URI{}. Existing Cookies{} Authenticating... ", - uri, - cookieManager.getCookieStore().getCookies()); - // Assuming that a valid UGI with kerberos cred is created by HS2 or LLAP - UserGroupInformation currentUser = UserGroupInformation.getCurrentUser(); - currentUser.checkTGTAndReloginFromKeytab(); - LOG.debug("The user credential is {}", currentUser); - String - challenge = - currentUser.doAs((PrivilegedExceptionAction) () -> DruidKerberosUtil.kerberosChallenge(host)); - request.setHeader(HttpHeaders.Names.AUTHORIZATION, "Negotiate " + challenge); - /* no reason to retry if the challenge ticket is not valid. */ - shouldRetryOnUnauthorizedResponse = false; - } else { - /* In this branch we had already a cookie that did expire - therefore we need to resend a valid Kerberos challenge*/ - LOG.debug("Found Auth Cookie found for URI {} cookie {}", - uri, - DruidKerberosUtil.getAuthCookie(cookieManager.getCookieStore(), uri).toString()); - shouldRetryOnUnauthorizedResponse = true; - } - - @SuppressWarnings("unchecked") ListenableFuture> - internalFuture = - delegate.go(request, - new RetryIfUnauthorizedResponseHandler(new ResponseCookieHandler(request.getUrl() - .toURI(), cookieManager, httpResponseHandler)), - duration); - - RetryResponseHolder responseHolder = internalFuture.get(); - - if (shouldRetryOnUnauthorizedResponse && responseHolder.shouldRetry()) { - LOG.debug("Preparing for Retry boolean {} and result {}, object{} ", true, - responseHolder.shouldRetry(), - responseHolder.getObj()); - // remove Auth cookie - DruidKerberosUtil.removeAuthCookie(cookieManager.getCookieStore(), uri); - // clear existing cookie - request.setHeader("Cookie", ""); - innerGo(request.copy(), httpResponseHandler, duration, future); - - } else { - future.set(responseHolder.getObj()); - } - } catch (Throwable e) { - throw Throwables.propagate(e); - } - } - -} diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/security/ResponseCookieHandler.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/security/ResponseCookieHandler.java deleted file mode 100644 index 223000e62a04..000000000000 --- a/druid-handler/src/java/org/apache/hadoop/hive/druid/security/ResponseCookieHandler.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.druid.security; - -import com.google.common.collect.Maps; -import org.apache.druid.java.util.http.client.response.ClientResponse; -import org.apache.druid.java.util.http.client.response.HttpResponseHandler; -import org.jboss.netty.handler.codec.http.HttpChunk; -import org.jboss.netty.handler.codec.http.HttpHeaders; -import org.jboss.netty.handler.codec.http.HttpResponse; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.net.CookieManager; -import java.net.URI; - -/** - * Class to handle Cookies used to cache the Kerberos Credentials. - * @param intermediate response type. - * @param final response type. - */ -public class ResponseCookieHandler implements HttpResponseHandler { - private static final Logger LOG = LoggerFactory.getLogger(ResponseCookieHandler.class); - - private final URI uri; - private final CookieManager manager; - private final HttpResponseHandler delegate; - - ResponseCookieHandler(URI uri, CookieManager manager, HttpResponseHandler delegate) { - this.uri = uri; - this.manager = manager; - this.delegate = delegate; - } - - @Override public ClientResponse handleResponse(HttpResponse httpResponse, TrafficCop trafficCop) { - try { - final HttpHeaders headers = httpResponse.headers(); - manager.put(uri, Maps.asMap(headers.names(), headers::getAll)); - return delegate.handleResponse(httpResponse, trafficCop); - } catch (IOException e) { - LOG.error("Error while processing Cookies from header", e); - throw new RuntimeException(e); - } - } - - @Override public ClientResponse handleChunk(ClientResponse clientResponse, - HttpChunk httpChunk, long chunkNum) { - return delegate.handleChunk(clientResponse, httpChunk, chunkNum); - } - - @Override public ClientResponse done(ClientResponse clientResponse) { - return delegate.done(clientResponse); - } - - @Override public void exceptionCaught(ClientResponse clientResponse, Throwable throwable) { - delegate.exceptionCaught(clientResponse, throwable); - } -} diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/security/RetryIfUnauthorizedResponseHandler.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/security/RetryIfUnauthorizedResponseHandler.java deleted file mode 100644 index d6702f5a1fe0..000000000000 --- a/druid-handler/src/java/org/apache/hadoop/hive/druid/security/RetryIfUnauthorizedResponseHandler.java +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.druid.security; - -import org.apache.druid.java.util.http.client.response.ClientResponse; -import org.apache.druid.java.util.http.client.response.HttpResponseHandler; -import org.jboss.netty.handler.codec.http.HttpChunk; -import org.jboss.netty.handler.codec.http.HttpResponse; -import org.jboss.netty.handler.codec.http.HttpResponseStatus; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Class handling retry on Unauthorized responses. - * - * @param Intermediate response type. - * @param final result type. - */ -public class RetryIfUnauthorizedResponseHandler - implements HttpResponseHandler, RetryResponseHolder> { - protected static final Logger LOG = LoggerFactory.getLogger(RetryIfUnauthorizedResponseHandler.class); - - private final HttpResponseHandler httpResponseHandler; - - public RetryIfUnauthorizedResponseHandler(HttpResponseHandler httpResponseHandler) { - this.httpResponseHandler = httpResponseHandler; - } - - @Override public ClientResponse> handleResponse(HttpResponse httpResponse, - TrafficCop trafficCop) { - LOG.debug("UnauthorizedResponseHandler - Got response status {}", httpResponse.getStatus()); - if (httpResponse.getStatus().equals(HttpResponseStatus.UNAUTHORIZED)) { - // Drain the buffer - //noinspection ResultOfMethodCallIgnored - httpResponse.getContent().toString(); - return ClientResponse.unfinished(RetryResponseHolder.retry()); - } else { - return wrap(httpResponseHandler.handleResponse(httpResponse, trafficCop)); - } - } - - @Override public ClientResponse> handleChunk( - ClientResponse> clientResponse, - HttpChunk httpChunk, long chunkNum) { - if (clientResponse.getObj().shouldRetry()) { - // Drain the buffer - //noinspection ResultOfMethodCallIgnored - httpChunk.getContent().toString(); - return clientResponse; - } else { - return wrap(httpResponseHandler.handleChunk(unwrap(clientResponse), httpChunk, chunkNum)); - } - } - - @Override public ClientResponse> done( - ClientResponse> clientResponse) { - if (clientResponse.getObj().shouldRetry()) { - return ClientResponse.finished(RetryResponseHolder.retry()); - } else { - return wrap(httpResponseHandler.done(unwrap(clientResponse))); - } - } - - @Override public void exceptionCaught(ClientResponse> clientResponse, - Throwable throwable) { - httpResponseHandler.exceptionCaught(unwrap(clientResponse), throwable); - } - - private ClientResponse> wrap(ClientResponse response) { - if (response.isFinished()) { - return ClientResponse.finished(new RetryResponseHolder<>(false, response.getObj())); - } else { - return ClientResponse.unfinished(new RetryResponseHolder<>(false, response.getObj())); - } - } - - private ClientResponse unwrap(ClientResponse> response) { - if (response.isFinished()) { - return ClientResponse.finished(response.getObj().getObj()); - } else { - return ClientResponse.unfinished(response.getObj().getObj()); - } - } - -} diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/security/RetryResponseHolder.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/security/RetryResponseHolder.java deleted file mode 100644 index 067107e1d3c8..000000000000 --- a/druid-handler/src/java/org/apache/hadoop/hive/druid/security/RetryResponseHolder.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.druid.security; - -/** - * Retry Response Handler class. - * @param type of the response object. - */ -public class RetryResponseHolder { - private final boolean shouldRetry; - private final T obj; - - public RetryResponseHolder(boolean shouldRetry, T obj) { - this.shouldRetry = shouldRetry; - this.obj = obj; - } - - public static RetryResponseHolder retry() { - return new RetryResponseHolder<>(true, null); - } - - public boolean shouldRetry() { - return shouldRetry; - } - - public T getObj() { - return obj; - } -} diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/security/package-info.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/security/package-info.java deleted file mode 100644 index 5d8201e73ace..000000000000 --- a/druid-handler/src/java/org/apache/hadoop/hive/druid/security/package-info.java +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * Package info file. - */ - -package org.apache.hadoop.hive.druid.security; diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidGroupByQueryRecordReader.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidGroupByQueryRecordReader.java deleted file mode 100644 index 4def3864c72a..000000000000 --- a/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidGroupByQueryRecordReader.java +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.druid.serde; - -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.JavaType; -import org.apache.druid.data.input.MapBasedRow; -import org.apache.druid.data.input.Row; -import org.apache.hadoop.hive.druid.DruidStorageHandlerUtils; -import org.apache.hadoop.hive.druid.conf.DruidConstants; -import org.apache.hadoop.io.NullWritable; - -import java.io.IOException; -import java.util.Map; - -/** - * Record reader for results for Druid GroupByQuery. - */ -public class DruidGroupByQueryRecordReader extends DruidQueryRecordReader { - private static final TypeReference TYPE_REFERENCE = new TypeReference() { - }; - - private MapBasedRow currentRow; - private Map currentEvent; - - @Override protected JavaType getResultTypeDef() { - return DruidStorageHandlerUtils.JSON_MAPPER.getTypeFactory().constructType(TYPE_REFERENCE); - } - - @Override public boolean nextKeyValue() { - // Results - - if (getQueryResultsIterator().hasNext()) { - final Row row = getQueryResultsIterator().next(); - // currently druid supports only MapBasedRow as Jackson SerDe so it should safe to cast without check - currentRow = (MapBasedRow) row; - currentEvent = currentRow.getEvent(); - return true; - } - return false; - } - - @Override public NullWritable getCurrentKey() throws IOException, InterruptedException { - return NullWritable.get(); - } - - @Override public DruidWritable getCurrentValue() throws IOException, InterruptedException { - // Create new value - DruidWritable value = new DruidWritable(false); - // 1) The timestamp column - value.getValue().put(DruidConstants.EVENT_TIMESTAMP_COLUMN, - currentRow.getTimestamp() == null ? null : currentRow.getTimestamp().getMillis() - ); - // 2) The dimension columns - value.getValue().putAll(currentEvent); - return value; - } - - @Override public boolean next(NullWritable key, DruidWritable value) { - if (nextKeyValue()) { - // Update value - value.getValue().clear(); - // 1) The timestamp column - value.getValue().put(DruidConstants.EVENT_TIMESTAMP_COLUMN, - currentRow.getTimestamp() == null ? null : currentRow.getTimestamp().getMillis() - ); - // 2) The dimension columns - value.getValue().putAll(currentEvent); - return true; - } - return false; - } - - @Override public float getProgress() throws IOException { - return getQueryResultsIterator().hasNext() ? 0 : 1; - } - -} diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidQueryRecordReader.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidQueryRecordReader.java deleted file mode 100644 index 19379e1724da..000000000000 --- a/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidQueryRecordReader.java +++ /dev/null @@ -1,303 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.druid.serde; - -import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.core.JsonToken; -import com.fasterxml.jackson.core.ObjectCodec; -import com.fasterxml.jackson.databind.JavaType; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.base.Preconditions; -import com.google.common.base.Throwables; -import org.apache.druid.java.util.common.IAE; -import org.apache.druid.java.util.common.RE; -import org.apache.druid.java.util.common.guava.CloseQuietly; -import org.apache.druid.java.util.http.client.HttpClient; -import org.apache.druid.java.util.http.client.Request; -import org.apache.druid.java.util.http.client.response.InputStreamResponseHandler; -import org.apache.druid.query.Query; -import org.apache.druid.query.QueryInterruptedException; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.druid.DruidStorageHandler; -import org.apache.hadoop.hive.druid.DruidStorageHandlerUtils; -import org.apache.hadoop.hive.druid.io.HiveDruidSplit; -import org.apache.hadoop.hive.ql.exec.SerializationUtilities; -import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc; -import org.apache.hadoop.hive.ql.plan.TableScanDesc; -import org.apache.hadoop.io.NullWritable; -import org.apache.hadoop.mapreduce.InputSplit; -import org.apache.hadoop.mapreduce.RecordReader; -import org.apache.hadoop.mapreduce.TaskAttemptContext; -import org.apache.parquet.Strings; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.Closeable; -import java.io.IOException; -import java.io.InputStream; -import java.util.Iterator; -import java.util.Objects; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.Future; - -/** - * Base record reader for given a Druid query. This class contains the logic to - * send the query to the broker and retrieve the results. The transformation to - * emit records needs to be done by the classes that extend the reader. - *

- * The key for each record will be a NullWritable, while the value will be a - * DruidWritable containing the timestamp as well as all values resulting from - * the query. - */ -public abstract class DruidQueryRecordReader> extends RecordReader - implements org.apache.hadoop.mapred.RecordReader { - - private static final Logger LOG = LoggerFactory.getLogger(DruidQueryRecordReader.class); - private final Object initLock = new Object(); - // Smile mapper is used to read query results that are serialized as binary instead of json - private ObjectMapper smileMapper; - private Configuration conf; - private String[] locations; - private HttpClient httpClient; - /** - * Query that Druid executes. - */ - protected Query query; - - /** - * Query results as a streaming iterator. - */ - private volatile JsonParserIterator queryResultsIterator = null; - - public JsonParserIterator getQueryResultsIterator() { - if (this.queryResultsIterator == null) { - synchronized (initLock) { - if (this.queryResultsIterator == null) { - this.queryResultsIterator = createQueryResultsIterator(); - } - } - } - return this.queryResultsIterator; - } - - public JsonParserIterator createQueryResultsIterator() { - JsonParserIterator iterator = null; - String filterExprSerialized = conf.get(TableScanDesc.FILTER_EXPR_CONF_STR); - if (filterExprSerialized != null) { - ExprNodeGenericFuncDesc filterExpr = SerializationUtilities.deserializeExpression(filterExprSerialized); - query = DruidStorageHandlerUtils.addDynamicFilters(query, filterExpr, conf, true); - } - - // Result type definition used to read the rows, this is query dependent. - JavaType resultsType = getResultTypeDef(); - boolean initialized = false; - int currentLocationIndex = 0; - Exception ex = null; - while (!initialized && currentLocationIndex < locations.length) { - String address = locations[currentLocationIndex++]; - if (Strings.isNullOrEmpty(address)) { - throw new RE("can not fetch results from empty or null host value"); - } - // Execute query - LOG.debug("Retrieving data from druid location[{}] using query:[{}] ", address, query); - try { - Request request = DruidStorageHandlerUtils.createSmileRequest(address, query); - Future inputStreamFuture = httpClient.go(request, new InputStreamResponseHandler()); - //noinspection unchecked - iterator = - new JsonParserIterator(smileMapper, resultsType, inputStreamFuture, request.getUrl().toString(), query); - iterator.init(); - initialized = true; - } catch (Exception e) { - if (iterator != null) { - // We got exception while querying results from this host. - CloseQuietly.close(iterator); - } - LOG.error("Failure getting results for query[{}] from host[{}] because of [{}]", - query, address, e.getMessage()); - if (ex == null) { - ex = e; - } else { - ex.addSuppressed(e); - } - } - } - - if (!initialized) { - throw new RE(ex, "Failure getting results for query[%s] from locations[%s] because of [%s]", query, locations, - Objects.requireNonNull(ex).getMessage()); - } - return iterator; - } - - @Override public void initialize(InputSplit split, TaskAttemptContext context) throws IOException { - initialize(split, context.getConfiguration()); - } - - public void initialize(InputSplit split, ObjectMapper mapper, ObjectMapper smileMapper, HttpClient httpClient, - Configuration conf) throws IOException { - this.conf = conf; - HiveDruidSplit hiveDruidSplit = (HiveDruidSplit) split; - Preconditions.checkNotNull(hiveDruidSplit, "input split is null ???"); - Preconditions.checkNotNull(httpClient, "need Http Client can not be null"); - // Smile mapper is used to read query results that are serialized as binary instead of json - this.smileMapper = Preconditions.checkNotNull(smileMapper, "Smile Mapper can not be null"); - // Create query - this.query = mapper.readValue(Preconditions.checkNotNull(hiveDruidSplit.getDruidQuery()), Query.class); - Preconditions.checkNotNull(query); - - this.locations = hiveDruidSplit.getLocations(); - this.httpClient = httpClient; - } - - public void initialize(InputSplit split, Configuration conf) throws IOException { - initialize(split, - DruidStorageHandlerUtils.JSON_MAPPER, - DruidStorageHandlerUtils.SMILE_MAPPER, DruidStorageHandler.getHttpClient(), conf); - } - - protected abstract JavaType getResultTypeDef(); - - @Override public NullWritable createKey() { - return NullWritable.get(); - } - - @Override public DruidWritable createValue() { - return new DruidWritable(false); - } - - @Override public abstract boolean next(NullWritable key, DruidWritable value) throws IOException; - - @Override public long getPos() { - // HiveContextAwareRecordReader uses this position to track the block position and check - // whether to skip header and footer. return -1 to since we need not skip any header and - // footer rows for druid. - return -1; - } - - @Override public abstract boolean nextKeyValue() throws IOException; - - @Override public abstract NullWritable getCurrentKey() throws IOException, InterruptedException; - - @Override public abstract DruidWritable getCurrentValue() throws IOException, InterruptedException; - - @Override public abstract float getProgress() throws IOException; - - @Override public void close() { - if (queryResultsIterator != null) { - CloseQuietly.close(queryResultsIterator); - } - } - - /** - * This is a helper wrapper class used to create an iterator of druid rows out of InputStream. - * The type of the rows is defined by - * org.apache.hadoop.hive.druid.serde.DruidQueryRecordReader.JsonParserIterator#typeRef - * - * @param druid Row type returned as result - */ - @SuppressWarnings("TypeParameterHidesVisibleType") protected class JsonParserIterator> - implements Iterator, Closeable { - private JsonParser jp; - private ObjectCodec objectCodec; - private final ObjectMapper mapper; - private final JavaType typeRef; - private final Future future; - private final Query query; - private final String url; - - /** - * @param mapper mapper used to deserialize the stream of data (we use smile factory) - * @param typeRef Type definition of the results objects - * @param future Future holding the input stream (the input stream is not owned but it will be closed - * when org.apache.hadoop.hive.druid.serde.DruidQueryRecordReader.JsonParserIterator#close() is - * called - * or reach the end of the steam) - * @param url URL used to fetch the data, used mostly as message with exception stack to identify the faulty - * stream, - * thus this can be empty string. - * @param query Query used to fetch the data, used mostly as message with exception stack, thus can be empty - * string. - */ - JsonParserIterator(ObjectMapper mapper, JavaType typeRef, Future future, String url, Query query) { - this.typeRef = typeRef; - this.future = future; - this.url = url; - this.query = query; - this.mapper = mapper; - jp = null; - } - - @Override public boolean hasNext() { - if (jp.isClosed()) { - return false; - } - if (jp.getCurrentToken() == JsonToken.END_ARRAY) { - CloseQuietly.close(jp); - return false; - } - - return true; - } - - @Override public R next() { - try { - final R retVal = objectCodec.readValue(jp, typeRef); - jp.nextToken(); - return retVal; - } catch (IOException e) { - throw Throwables.propagate(e); - } - } - - @Override public void remove() { - throw new UnsupportedOperationException(); - } - - private void init() { - if (jp == null) { - try { - InputStream is = future.get(); - if (is == null) { - throw new IOException(String.format("query[%s] url[%s] timed out", query, url)); - } else { - jp = mapper.getFactory().createParser(is).configure(JsonParser.Feature.AUTO_CLOSE_SOURCE, true); - } - final JsonToken nextToken = jp.nextToken(); - if (nextToken == JsonToken.START_OBJECT) { - QueryInterruptedException cause = jp.getCodec().readValue(jp, QueryInterruptedException.class); - throw new QueryInterruptedException(cause); - } else if (nextToken != JsonToken.START_ARRAY) { - throw new IAE("Next token wasn't a START_ARRAY, was[%s] from url [%s]", jp.getCurrentToken(), url); - } else { - jp.nextToken(); - objectCodec = jp.getCodec(); - } - - } catch (IOException | InterruptedException | ExecutionException e) { - throw new RE(e, "Failure getting results for query[%s] url[%s] because of [%s]", query, url, e.getMessage()); - } - } - } - - @Override public void close() throws IOException { - CloseQuietly.close(jp); - } - } - -} diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidScanQueryRecordReader.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidScanQueryRecordReader.java deleted file mode 100644 index 9afb76a12f4a..000000000000 --- a/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidScanQueryRecordReader.java +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.druid.serde; - -import org.apache.druid.query.scan.ScanResultValue; - -import org.apache.hadoop.hive.druid.DruidStorageHandlerUtils; -import org.apache.hadoop.io.NullWritable; - -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.JavaType; - -import java.io.IOException; -import java.util.Collections; -import java.util.Iterator; -import java.util.List; - -/** - * Record reader for results for Druid ScanQuery. - */ -public class DruidScanQueryRecordReader extends DruidQueryRecordReader { - - private static final TypeReference TYPE_REFERENCE = new TypeReference() { - }; - - private Iterator> compactedValues = Collections.emptyIterator(); - - @Override protected JavaType getResultTypeDef() { - return DruidStorageHandlerUtils.JSON_MAPPER.getTypeFactory().constructType(TYPE_REFERENCE); - } - - @Override public boolean nextKeyValue() throws IOException { - if (compactedValues.hasNext()) { - return true; - } - if (getQueryResultsIterator().hasNext()) { - ScanResultValue current = getQueryResultsIterator().next(); - //noinspection unchecked - compactedValues = ((List>) current.getEvents()).iterator(); - return nextKeyValue(); - } - return false; - } - - @Override public NullWritable getCurrentKey() throws IOException, InterruptedException { - return NullWritable.get(); - } - - @Override public DruidWritable createValue() { - return new DruidWritable(true); - } - - @Override public DruidWritable getCurrentValue() throws IOException, InterruptedException { - // Create new value - DruidWritable value = new DruidWritable(true); - value.setCompactedValue(compactedValues.next()); - return value; - } - - @Override public boolean next(NullWritable key, DruidWritable value) throws IOException { - if (nextKeyValue()) { - // Update value - value.setCompactedValue(compactedValues.next()); - return true; - } - return false; - } - - @Override public float getProgress() { - return getQueryResultsIterator().hasNext() || compactedValues.hasNext() ? 0 : 1; - } - -} diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidSerDe.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidSerDe.java deleted file mode 100644 index 25a35ea47d51..000000000000 --- a/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidSerDe.java +++ /dev/null @@ -1,497 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.druid.serde; - -import com.fasterxml.jackson.core.type.TypeReference; -import com.google.common.base.Preconditions; -import com.google.common.collect.Lists; -import org.apache.druid.query.Druids; -import org.apache.druid.query.Druids.SegmentMetadataQueryBuilder; -import org.apache.druid.query.metadata.metadata.ColumnAnalysis; -import org.apache.druid.query.metadata.metadata.SegmentAnalysis; -import org.apache.druid.query.metadata.metadata.SegmentMetadataQuery; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.common.type.Date; -import org.apache.hadoop.hive.common.type.HiveChar; -import org.apache.hadoop.hive.common.type.HiveVarchar; -import org.apache.hadoop.hive.common.type.Timestamp; -import org.apache.hadoop.hive.common.type.TimestampTZ; -import org.apache.hadoop.hive.conf.Constants; -import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.druid.DruidStorageHandler; -import org.apache.hadoop.hive.druid.DruidStorageHandlerUtils; -import org.apache.hadoop.hive.druid.conf.DruidConstants; -import org.apache.hadoop.hive.ql.exec.Utilities; -import org.apache.hadoop.hive.serde.serdeConstants; -import org.apache.hadoop.hive.serde2.AbstractSerDe; -import org.apache.hadoop.hive.serde2.SerDeException; -import org.apache.hadoop.hive.serde2.SerDeSpec; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; -import org.apache.hadoop.hive.serde2.objectinspector.StructField; -import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.BooleanObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.ByteObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.DoubleObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.FloatObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveCharObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveVarcharObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.IntObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.LongObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.ShortObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.TimestampLocalTZObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.TimestampObjectInspector; -import org.apache.hadoop.hive.serde2.typeinfo.CharTypeInfo; -import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; -import org.apache.hadoop.hive.serde2.typeinfo.TimestampLocalTZTypeInfo; -import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; -import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; -import org.apache.hadoop.hive.serde2.typeinfo.VarcharTypeInfo; -import org.apache.hadoop.io.Writable; -import org.apache.hadoop.util.StringUtils; - -import java.io.IOException; -import java.io.InputStream; -import java.time.Instant; -import java.time.ZonedDateTime; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Properties; -import java.util.stream.Collectors; - -import static org.joda.time.format.ISODateTimeFormat.dateOptionalTimeParser; - -/** - * DruidSerDe that is used to deserialize objects from a Druid data source. - */ -@SerDeSpec(schemaProps = { Constants.DRUID_DATA_SOURCE }) -public class DruidSerDe extends AbstractSerDe { - - private String[] columns; - private PrimitiveTypeInfo[] types; - private ObjectInspector inspector; - private TimestampLocalTZTypeInfo tsTZTypeInfo; - - @Override - public void initialize(Configuration configuration, Properties tableProperties, Properties partitionProperties) - throws SerDeException { - super.initialize(configuration, tableProperties, partitionProperties); - - tsTZTypeInfo = new TimestampLocalTZTypeInfo(configuration.get(HiveConf.ConfVars.HIVE_LOCAL_TIME_ZONE.varname)); - // Druid query - final String druidQuery = properties.getProperty(Constants.DRUID_QUERY_JSON, null); - if (druidQuery != null && !druidQuery.isEmpty()) { - initFromDruidQueryPlan(properties, druidQuery); - } else { - // No query. Either it is a CTAS, or we need to create a Druid meta data Query - if (!org.apache.commons.lang3.StringUtils.isEmpty(properties.getProperty(serdeConstants.LIST_COLUMNS)) - && !org.apache.commons.lang3.StringUtils.isEmpty(properties.getProperty(serdeConstants.LIST_COLUMN_TYPES))) { - // CASE CTAS statement - initFromProperties(properties); - } else { - // Segment Metadata query that retrieves all columns present in - // the data source (dimensions and metrics). - initFromMetaDataQuery(configuration, properties); - } - } - if (log.isDebugEnabled()) { - log.debug("DruidSerDe initialized with\n" - + "\t columns: " - + Arrays.toString(columns) - + "\n\t types: " - + Arrays.toString(types)); - } - } - - private void initFromMetaDataQuery(final Configuration configuration, final Properties properties) - throws SerDeException { - final List columnNames = new ArrayList<>(); - final List columnTypes = new ArrayList<>(); - final List inspectors = new ArrayList<>(); - - String dataSource = properties.getProperty(Constants.DRUID_DATA_SOURCE); - if (dataSource == null) { - throw new SerDeException("Druid data source not specified; use " - + Constants.DRUID_DATA_SOURCE - + " in table properties"); - } - SegmentMetadataQueryBuilder builder = new Druids.SegmentMetadataQueryBuilder(); - builder.dataSource(dataSource); - builder.merge(true); - builder.analysisTypes(); - SegmentMetadataQuery query = builder.build(); - - // Execute query in Druid - String address = HiveConf.getVar(configuration, HiveConf.ConfVars.HIVE_DRUID_BROKER_DEFAULT_ADDRESS); - if (org.apache.commons.lang3.StringUtils.isEmpty(address)) { - throw new SerDeException("Druid broker address not specified in configuration"); - } - // Infer schema - SegmentAnalysis schemaInfo; - try { - schemaInfo = submitMetadataRequest(address, query); - } catch (IOException e) { - throw new SerDeException(e); - } - for (Entry columnInfo : schemaInfo.getColumns().entrySet()) { - if (columnInfo.getKey().equals(DruidConstants.DEFAULT_TIMESTAMP_COLUMN)) { - // Special handling for timestamp column - columnNames.add(columnInfo.getKey()); // field name - PrimitiveTypeInfo type = tsTZTypeInfo; // field type - columnTypes.add(type); - inspectors.add(PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(type)); - continue; - } - columnNames.add(columnInfo.getKey()); // field name - PrimitiveTypeInfo type = DruidSerDeUtils.convertDruidToHiveType(columnInfo.getValue().getType()); // field type - columnTypes.add(type instanceof TimestampLocalTZTypeInfo ? tsTZTypeInfo : type); - inspectors.add(PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(type)); - } - columns = columnNames.toArray(new String[0]); - types = columnTypes.toArray(new PrimitiveTypeInfo[0]); - inspector = ObjectInspectorFactory.getStandardStructObjectInspector(columnNames, inspectors); - } - - private void initFromProperties(final Properties properties) throws SerDeException { - - final List columnNames = new ArrayList<>(Utilities.getColumnNames(properties)); - if (!columnNames.contains(DruidConstants.DEFAULT_TIMESTAMP_COLUMN)) { - throw new SerDeException("Timestamp column (' " - + DruidConstants.DEFAULT_TIMESTAMP_COLUMN - + "') not specified in create table; list of columns is : " - + properties.getProperty(serdeConstants.LIST_COLUMNS)); - } - final List - columnTypes = - Utilities.getColumnTypes(properties) - .stream() - .map(TypeInfoFactory::getPrimitiveTypeInfo) - .collect(Collectors.toList()) - .stream() - .map(e -> e instanceof TimestampLocalTZTypeInfo ? tsTZTypeInfo : e) - .collect(Collectors.toList()); - final List - inspectors = - columnTypes.stream() - .map(PrimitiveObjectInspectorFactory::getPrimitiveJavaObjectInspector) - .collect(Collectors.toList()); - columns = columnNames.toArray(new String[0]); - types = columnTypes.toArray(new PrimitiveTypeInfo[0]); - inspector = ObjectInspectorFactory.getStandardStructObjectInspector(columnNames, inspectors); - } - - private void initFromDruidQueryPlan(Properties properties, String druidQuery) { - Preconditions.checkNotNull(druidQuery, "Why Druid query is null"); - final List inspectors = new ArrayList<>(); - final List columnNames; - final List columnTypes; - final String - fieldNamesProperty = - Preconditions.checkNotNull(properties.getProperty(Constants.DRUID_QUERY_FIELD_NAMES, null)); - final String - fieldTypesProperty = - Preconditions.checkNotNull(properties.getProperty(Constants.DRUID_QUERY_FIELD_TYPES, null)); - if (fieldNamesProperty.isEmpty()) { - // this might seem counter intuitive but some queries like query - // SELECT YEAR(Calcs.date0) AS yr_date0_ok FROM druid_tableau.calcs Calcs WHERE (YEAR(Calcs.date0) IS NULL) - // LIMIT 1 - // is planed in a way where we only push a filter down and keep the project of null as hive project. Thus empty - // columns - columnNames = Collections.emptyList(); - columnTypes = Collections.emptyList(); - } else { - columnNames = Arrays.stream(fieldNamesProperty.trim().split(",")).collect(Collectors.toList()); - columnTypes = - TypeInfoUtils.getTypeInfosFromTypeString(fieldTypesProperty) - .stream() - .map(e -> TypeInfoFactory.getPrimitiveTypeInfo(e.getTypeName())) - .map(primitiveTypeInfo -> { - if (primitiveTypeInfo instanceof TimestampLocalTZTypeInfo) { - return tsTZTypeInfo; - } - return primitiveTypeInfo; - }) - .collect(Collectors.toList()); - } - columns = new String[columnNames.size()]; - types = new PrimitiveTypeInfo[columnNames.size()]; - for (int i = 0; i < columnTypes.size(); ++i) { - columns[i] = columnNames.get(i); - types[i] = columnTypes.get(i); - inspectors.add(PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(types[i])); - } - inspector = ObjectInspectorFactory.getStandardStructObjectInspector(columnNames, inspectors); - } - - /* Submits the request and returns */ - protected SegmentAnalysis submitMetadataRequest(String address, SegmentMetadataQuery query) - throws SerDeException, IOException { - InputStream response; - try { - response = - DruidStorageHandlerUtils.submitRequest(DruidStorageHandler.getHttpClient(), - DruidStorageHandlerUtils.createSmileRequest(address, query)); - } catch (Exception e) { - throw new SerDeException(StringUtils.stringifyException(e)); - } - - // Retrieve results - List resultsList; - try { - // This will throw an exception in case of the response from druid is not an array - // this case occurs if for instance druid query execution returns an exception instead of array of results. - resultsList = - DruidStorageHandlerUtils.SMILE_MAPPER.readValue(response, new TypeReference>() { - }); - } catch (Exception e) { - response.close(); - throw new SerDeException(StringUtils.stringifyException(e)); - } - if (resultsList == null || resultsList.isEmpty()) { - throw new SerDeException("Connected to Druid but could not retrieve datasource information"); - } - if (resultsList.size() != 1) { - throw new SerDeException("Information about segments should have been merged"); - } - - return resultsList.get(0); - } - - @Override public Class getSerializedClass() { - return DruidWritable.class; - } - - @Override public Writable serialize(Object o, ObjectInspector objectInspector) throws SerDeException { - if (objectInspector.getCategory() != ObjectInspector.Category.STRUCT) { - throw new SerDeException(getClass().toString() - + " can only serialize struct types, but we got: " - + objectInspector.getTypeName()); - } - - // Prepare the field ObjectInspectors - StructObjectInspector soi = (StructObjectInspector) objectInspector; - List fields = soi.getAllStructFieldRefs(); - List values = soi.getStructFieldsDataAsList(o); - // We deserialize the result - final Map value = new HashMap<>(); - for (int i = 0; i < columns.length; i++) { - if (values.get(i) == null) { - // null, we just add it - value.put(columns[i], null); - continue; - } - final Object res; - switch (types[i].getPrimitiveCategory()) { - case TIMESTAMP: - res = - ((TimestampObjectInspector) fields.get(i).getFieldObjectInspector()).getPrimitiveJavaObject(values.get(i)) - .toEpochMilli(); - break; - case TIMESTAMPLOCALTZ: - res = - ((TimestampLocalTZObjectInspector) fields.get(i) - .getFieldObjectInspector()).getPrimitiveJavaObject(values.get(i)) - .getZonedDateTime() - .toInstant() - .toEpochMilli(); - break; - case BYTE: - res = ((ByteObjectInspector) fields.get(i).getFieldObjectInspector()).get(values.get(i)); - break; - case SHORT: - res = ((ShortObjectInspector) fields.get(i).getFieldObjectInspector()).get(values.get(i)); - break; - case INT: - res = ((IntObjectInspector) fields.get(i).getFieldObjectInspector()).get(values.get(i)); - break; - case LONG: - res = ((LongObjectInspector) fields.get(i).getFieldObjectInspector()).get(values.get(i)); - break; - case FLOAT: - res = ((FloatObjectInspector) fields.get(i).getFieldObjectInspector()).get(values.get(i)); - break; - case DOUBLE: - res = ((DoubleObjectInspector) fields.get(i).getFieldObjectInspector()).get(values.get(i)); - break; - case CHAR: - res = - ((HiveCharObjectInspector) fields.get(i).getFieldObjectInspector()).getPrimitiveJavaObject(values.get(i)) - .getValue(); - break; - case VARCHAR: - res = - ((HiveVarcharObjectInspector) fields.get(i).getFieldObjectInspector()).getPrimitiveJavaObject(values.get(i)) - .getValue(); - break; - case STRING: - res = ((StringObjectInspector) fields.get(i).getFieldObjectInspector()).getPrimitiveJavaObject(values.get(i)); - break; - case BOOLEAN: - res = ((BooleanObjectInspector) fields.get(i).getFieldObjectInspector()).get(values.get(i)) ? 1L : 0L; - break; - default: - throw new SerDeException("Unsupported type: " + types[i].getPrimitiveCategory()); - } - value.put(columns[i], res); - } - //Extract the partitions keys segments granularity and partition key if any - // First Segment Granularity has to be here. - final int granularityFieldIndex = columns.length; - assert values.size() > granularityFieldIndex; - Preconditions.checkArgument(fields.get(granularityFieldIndex) - .getFieldName() - .equals(Constants.DRUID_TIMESTAMP_GRANULARITY_COL_NAME)); - - Timestamp - timestamp = - ((TimestampObjectInspector) fields.get(granularityFieldIndex).getFieldObjectInspector()).getPrimitiveJavaObject( - values.get(granularityFieldIndex)); - Preconditions.checkNotNull(timestamp, "Timestamp column cannot have null value"); - value.put(Constants.DRUID_TIMESTAMP_GRANULARITY_COL_NAME, timestamp.toEpochMilli()); - - if (values.size() == columns.length + 2) { - // Then partition number if any. - final int partitionNumPos = granularityFieldIndex + 1; - Preconditions.checkArgument(fields.get(partitionNumPos).getFieldName().equals(Constants.DRUID_SHARD_KEY_COL_NAME), - String.format("expecting to encounter %s but was %s", - Constants.DRUID_SHARD_KEY_COL_NAME, - fields.get(partitionNumPos).getFieldName())); - value.put(Constants.DRUID_SHARD_KEY_COL_NAME, - ((LongObjectInspector) fields.get(partitionNumPos) - .getFieldObjectInspector()).get(values.get(partitionNumPos))); - } - - return new DruidWritable(value); - } - - /** - * @param writable Druid Writable to be deserialized. - * @return List of Hive Writables. - * @throws SerDeException if there is Serde issues. - */ - @Override public Object deserialize(Writable writable) throws SerDeException { - final DruidWritable input = (DruidWritable) writable; - final List output = Lists.newArrayListWithExpectedSize(columns.length); - for (int i = 0; i < columns.length; i++) { - final Object value = input.isCompacted() ? input.getCompactedValue().get(i) : input.getValue().get(columns[i]); - if (value == null) { - output.add(null); - } else { - output.add(convertAsPrimitive(value, types[i])); - } - } - return output; - } - - /** - * Function to convert Druid Primitive values to Hive Primitives. Main usage of this is to pipe data to VectorRow. - * This has the exact same logic as {@link DruidSerDe#deserialize(Writable)}, any modification here should be done - * there as well. - * Reason to have 2 function is that no vectorized path expects writables. - * - * @param writable Druid Writable. - * @param rowBoat Rowboat used to carry columns values. - * @throws SerDeException in case of deserialization errors. - */ - public void deserializeAsPrimitive(Writable writable, final Object[] rowBoat) throws SerDeException { - final DruidWritable input = (DruidWritable) writable; - for (int i = 0; i < columns.length; i++) { - final Object value = input.isCompacted() ? input.getCompactedValue().get(i) : input.getValue().get(columns[i]); - rowBoat[i] = null; - if (value != null) { - rowBoat[i] = convertAsPrimitive(value, types[i]); - } - } - } - - private static Object convertAsPrimitive(Object value, PrimitiveTypeInfo typeInfo) throws SerDeException { - switch (typeInfo.getPrimitiveCategory()) { - case TIMESTAMP: - return Timestamp.ofEpochMilli(deserializeToMillis(value)); - case TIMESTAMPLOCALTZ: - return new TimestampTZ(ZonedDateTime.ofInstant(Instant.ofEpochMilli(deserializeToMillis(value)), - ((TimestampLocalTZTypeInfo) typeInfo).timeZone())); - case DATE: - return (Date.ofEpochMilli(deserializeToMillis(value))); - case BYTE: - return ((Number) value).byteValue(); - case SHORT: - return ((Number) value).shortValue(); - case INT: - if (value instanceof Number) { - return ((Number) value).intValue(); - } else { - // This is a corner case where we have an extract of time unit like day/month pushed as Extraction Fn - //@TODO The best way to fix this is to add explicit output Druid types to Calcite Extraction Functions impls - return (Integer.valueOf((String) value)); - } - case LONG: - return ((Number) value).longValue(); - case FLOAT: - return ((Number) value).floatValue(); - case DOUBLE: - return ((Number) value).doubleValue(); - case CHAR: - return new HiveChar(value.toString(), ((CharTypeInfo) typeInfo).getLength()); - case VARCHAR: - return new HiveVarchar(value.toString(), ((VarcharTypeInfo) typeInfo).getLength()); - case STRING: - return value.toString(); - case BOOLEAN: - if (value instanceof Number) { - return (((Number) value).intValue() != 0); - } else { - return (Boolean.valueOf(value.toString())); - } - default: - throw new SerDeException("Unknown type: " + typeInfo.getPrimitiveCategory()); - } - } - - private static long deserializeToMillis(Object value) { - long numberOfMillis; - if (value instanceof Number) { - numberOfMillis = ((Number) value).longValue(); - } else { - // it is an extraction fn need to be parsed - try { - numberOfMillis = dateOptionalTimeParser().parseDateTime((String) value).getMillis(); - } catch (IllegalArgumentException e) { - // we may not be able to parse the date if it already comes in Hive format, - // we retry and otherwise fail - numberOfMillis = Timestamp.valueOf((String) value).toEpochMilli(); - } - } - return numberOfMillis; - } - - @Override public ObjectInspector getObjectInspector() { - return inspector; - } - - @Override public boolean shouldStoreFieldsInMetastore(Map tableParams) { - return true; - } -} diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidSerDeUtils.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidSerDeUtils.java deleted file mode 100644 index 663d4c2484aa..000000000000 --- a/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidSerDeUtils.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.druid.serde; - -import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; -import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Utils class for Druid SerDe. - */ -final class DruidSerDeUtils { - private DruidSerDeUtils() { - } - - private static final Logger LOG = LoggerFactory.getLogger(DruidSerDeUtils.class); - - private static final String FLOAT_TYPE = "FLOAT"; - private static final String DOUBLE_TYPE = "DOUBLE"; - private static final String LONG_TYPE = "LONG"; - private static final String STRING_TYPE = "STRING"; - - /* This method converts from the String representation of Druid type - * to the corresponding Hive type */ - static PrimitiveTypeInfo convertDruidToHiveType(String typeName) { - typeName = typeName.toUpperCase(); - switch (typeName) { - case FLOAT_TYPE: - return TypeInfoFactory.floatTypeInfo; - case DOUBLE_TYPE: - return TypeInfoFactory.doubleTypeInfo; - case LONG_TYPE: - return TypeInfoFactory.longTypeInfo; - case STRING_TYPE: - return TypeInfoFactory.stringTypeInfo; - default: - // This is a guard for special Druid types e.g. hyperUnique - // (http://druid.io/docs/0.9.1.1/querying/aggregations.html#hyperunique-aggregator). - // Currently, we do not support doing anything special with them in Hive. - // However, those columns are there, and they can be actually read as normal - // dimensions e.g. with a select query. Thus, we print the warning and just read them - // as String. - LOG.warn("Transformation to STRING for unknown type " + typeName); - return TypeInfoFactory.stringTypeInfo; - } - } - -} diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidTimeseriesQueryRecordReader.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidTimeseriesQueryRecordReader.java deleted file mode 100644 index 95af15d2fa76..000000000000 --- a/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidTimeseriesQueryRecordReader.java +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.druid.serde; - -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.JavaType; -import org.apache.druid.query.Result; -import org.apache.druid.query.timeseries.TimeseriesResultValue; -import org.apache.hadoop.hive.druid.DruidStorageHandlerUtils; -import org.apache.hadoop.hive.druid.conf.DruidConstants; -import org.apache.hadoop.io.NullWritable; - -import java.io.IOException; - -/** - * Record reader for results for Druid TimeseriesQuery. - */ -public class DruidTimeseriesQueryRecordReader - extends DruidQueryRecordReader> { - - private static final TypeReference TYPE_REFERENCE = new TypeReference>() { - }; - private Result current; - - @Override - protected JavaType getResultTypeDef() { - return DruidStorageHandlerUtils.JSON_MAPPER.getTypeFactory().constructType(TYPE_REFERENCE); - } - - @Override - public boolean nextKeyValue() { - if (getQueryResultsIterator().hasNext()) { - current = getQueryResultsIterator().next(); - return true; - } - return false; - } - - @Override - public NullWritable getCurrentKey() throws IOException, InterruptedException { - return NullWritable.get(); - } - - @Override - public DruidWritable getCurrentValue() throws IOException, InterruptedException { - // Create new value - DruidWritable value = new DruidWritable(false); - value.getValue().put(DruidConstants.EVENT_TIMESTAMP_COLUMN, - current.getTimestamp() == null ? null : current.getTimestamp().getMillis() - ); - value.getValue().putAll(current.getValue().getBaseObject()); - return value; - } - - @Override - public boolean next(NullWritable key, DruidWritable value) { - if (nextKeyValue()) { - // Update value - value.getValue().clear(); - value.getValue().put(DruidConstants.EVENT_TIMESTAMP_COLUMN, - current.getTimestamp() == null ? null : current.getTimestamp().getMillis() - ); - value.getValue().putAll(current.getValue().getBaseObject()); - return true; - } - return false; - } - - @Override - public float getProgress() throws IOException { - return getQueryResultsIterator().hasNext() ? 0 : 1; - } - -} diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidTopNQueryRecordReader.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidTopNQueryRecordReader.java deleted file mode 100644 index 57c8b0ef146a..000000000000 --- a/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidTopNQueryRecordReader.java +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.druid.serde; - -import java.io.IOException; -import java.util.Collections; -import java.util.Iterator; - -import com.fasterxml.jackson.databind.JavaType; -import org.apache.hadoop.hive.druid.DruidStorageHandlerUtils; -import org.apache.hadoop.io.NullWritable; - -import com.fasterxml.jackson.core.type.TypeReference; - -import org.apache.druid.query.Result; -import org.apache.druid.query.topn.DimensionAndMetricValueExtractor; -import org.apache.druid.query.topn.TopNResultValue; - -/** - * Record reader for results for Druid TopNQuery. - */ -public class DruidTopNQueryRecordReader extends DruidQueryRecordReader> { - - private static final TypeReference> - TYPE_REFERENCE = - new TypeReference>() { - }; - - private Result current; - - private Iterator values = Collections.emptyIterator(); - - @Override protected JavaType getResultTypeDef() { - return DruidStorageHandlerUtils.JSON_MAPPER.getTypeFactory().constructType(TYPE_REFERENCE); - } - - @Override public boolean nextKeyValue() { - if (values.hasNext()) { - return true; - } - if (getQueryResultsIterator().hasNext()) { - current = getQueryResultsIterator().next(); - values = current.getValue().getValue().iterator(); - return nextKeyValue(); - } - return false; - } - - @Override public NullWritable getCurrentKey() throws IOException, InterruptedException { - return NullWritable.get(); - } - - @Override public DruidWritable getCurrentValue() throws IOException, InterruptedException { - // Create new value - DruidWritable value = new DruidWritable(false); - value.getValue().put("timestamp", current.getTimestamp().getMillis()); - if (values.hasNext()) { - value.getValue().putAll(values.next().getBaseObject()); - return value; - } - return value; - } - - @Override public boolean next(NullWritable key, DruidWritable value) { - if (nextKeyValue()) { - // Update value - value.getValue().clear(); - value.getValue().put("timestamp", current.getTimestamp().getMillis()); - if (values.hasNext()) { - value.getValue().putAll(values.next().getBaseObject()); - } - return true; - } - return false; - } - - @Override public float getProgress() { - return getQueryResultsIterator().hasNext() || values.hasNext() ? 0 : 1; - } - -} diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidWritable.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidWritable.java deleted file mode 100644 index b5850d46ac17..000000000000 --- a/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidWritable.java +++ /dev/null @@ -1,131 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.druid.serde; - -import com.google.common.collect.Lists; -import org.apache.hadoop.io.Writable; - -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; - -/** - * Writable for Druid results. - */ -public class DruidWritable implements Writable { - - /** - * value map stores column name to value mapping. - * This is only used when the result is not compacted. - */ - private final Map value; - - /** - * list of values in a compacted form, Serializer/Deserializer needs to agree on the order of values. - * This is only used when the result is compacted. - */ - private transient List compactedValue; - - private final boolean compacted; - - public DruidWritable(boolean compacted) { - this.compacted = compacted; - if (compacted) { - compactedValue = Lists.newArrayList(); - value = null; - } else { - value = new HashMap<>(); - compactedValue = null; - } - } - - public DruidWritable(Map value) { - this.value = value; - this.compactedValue = null; - compacted = false; - } - - public DruidWritable(List value) { - this.compacted = true; - this.compactedValue = value; - this.value = null; - } - - public Map getValue() { - if (compacted) { - throw new UnsupportedOperationException( - "compacted DruidWritable does not support getValue(), use getCompactedValue()"); - } - return value; - } - - List getCompactedValue() { - if (!compacted) { - throw new UnsupportedOperationException( - "non compacted DruidWritable does not support getCompactedValue(), use getValue()"); - } - return compactedValue; - } - - void setCompactedValue(List compactedValue) { - this.compactedValue = compactedValue; - } - - boolean isCompacted() { - return compacted; - } - - @Override public void write(DataOutput out) throws IOException { - throw new UnsupportedOperationException(); - } - - @Override public void readFields(DataInput in) throws IOException { - throw new UnsupportedOperationException(); - } - - @Override public int hashCode() { - return Objects.hash(value, compactedValue, compacted); - } - - @Override public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - DruidWritable that = (DruidWritable) o; - return compacted == that.compacted && Objects.equals(value, that.value) && Objects.equals(compactedValue, - that.compactedValue); - } - - @Override public String toString() { - return "DruidWritable{" - + "value=" - + value - + ", compactedValue=" - + compactedValue - + ", compacted=" - + compacted - + '}'; - } -} diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/package-info.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/package-info.java deleted file mode 100644 index ba6a103af1fb..000000000000 --- a/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/package-info.java +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * Package info file. - */ - -package org.apache.hadoop.hive.druid.serde; diff --git a/druid-handler/src/test/org/apache/hadoop/hive/druid/DerbyConnectorTestUtility.java b/druid-handler/src/test/org/apache/hadoop/hive/druid/DerbyConnectorTestUtility.java deleted file mode 100644 index 9119aa5e6b57..000000000000 --- a/druid-handler/src/test/org/apache/hadoop/hive/druid/DerbyConnectorTestUtility.java +++ /dev/null @@ -1,129 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.druid; - -import com.google.common.base.Suppliers; -import org.apache.druid.metadata.MetadataStorageConnectorConfig; -import org.apache.druid.metadata.MetadataStorageTablesConfig; -import org.apache.druid.metadata.storage.derby.DerbyConnector; -import org.apache.druid.metadata.storage.derby.DerbyMetadataStorage; -import org.junit.Assert; -import org.junit.rules.ExternalResource; -import org.skife.jdbi.v2.DBI; -import org.skife.jdbi.v2.exceptions.UnableToObtainConnectionException; - -import java.sql.SQLException; -import java.util.UUID; -import java.util.function.Supplier; - -/** - * Derby test class. - */ -public class DerbyConnectorTestUtility extends DerbyConnector { - private final String jdbcUri; - - public DerbyConnectorTestUtility( - Supplier config, - Supplier dbTables - ) { - this(config, dbTables, "jdbc:derby:memory:druidTest" + dbSafeUUID()); - } - - protected DerbyConnectorTestUtility( - Supplier config, - Supplier dbTables, - String jdbcUri - ) { - super(new DerbyMetadataStorage(config.get()), config::get, dbTables::get, new DBI(jdbcUri + ";create=true")); - this.jdbcUri = jdbcUri; - } - - public void tearDown() { - try { - new DBI(jdbcUri + ";drop=true").open().close(); - } catch (UnableToObtainConnectionException e) { - SQLException cause = (SQLException) e.getCause(); - // error code "08006" indicates proper shutdown - Assert.assertEquals(String.format("Derby not shutdown: [%s]", cause.toString()), "08006", - cause.getSQLState() - ); - } - } - - public static String dbSafeUUID() { - return UUID.randomUUID().toString().replace("-", ""); - } - - public String getJdbcUri() { - return jdbcUri; - } - - /** - * Derby connector rule. - */ - public static class DerbyConnectorRule extends ExternalResource { - private DerbyConnectorTestUtility connector; - - private final Supplier dbTables; - - private final MetadataStorageConnectorConfig connectorConfig; - - public DerbyConnectorRule() { - this("druidTest" + dbSafeUUID()); - } - - private DerbyConnectorRule( - final String defaultBase - ) { - this(Suppliers.ofInstance(MetadataStorageTablesConfig.fromBase(defaultBase))::get); - } - - public DerbyConnectorRule( - Supplier dbTables - ) { - this.dbTables = dbTables; - this.connectorConfig = new MetadataStorageConnectorConfig() { - @Override - public String getConnectURI() { - return connector.getJdbcUri(); - } - }; - } - - @Override - protected void before() throws Throwable { - connector = new DerbyConnectorTestUtility(Suppliers.ofInstance(connectorConfig)::get, dbTables); - connector.getDBI().open().close(); // create db - } - - @Override - protected void after() { - connector.tearDown(); - } - - public DerbyConnectorTestUtility getConnector() { - return connector; - } - - public Supplier metadataTablesConfigSupplier() { - return dbTables; - } - } - -} diff --git a/druid-handler/src/test/org/apache/hadoop/hive/druid/QTestDruidQueryBasedInputFormatToAddFaultyHost.java b/druid-handler/src/test/org/apache/hadoop/hive/druid/QTestDruidQueryBasedInputFormatToAddFaultyHost.java deleted file mode 100644 index b47982f2a1f0..000000000000 --- a/druid-handler/src/test/org/apache/hadoop/hive/druid/QTestDruidQueryBasedInputFormatToAddFaultyHost.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.druid; - -import com.google.common.collect.Lists; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.druid.io.DruidQueryBasedInputFormat; -import org.apache.hadoop.hive.druid.io.HiveDruidSplit; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -/** - * This input format adds a faulty host as first input split location. - * Tests should be able to query results successfully by trying next split locations. - */ -public class QTestDruidQueryBasedInputFormatToAddFaultyHost extends DruidQueryBasedInputFormat { - - @Override - protected HiveDruidSplit[] getInputSplits(Configuration conf) throws IOException { - HiveDruidSplit[] inputSplits = super.getInputSplits(conf); - List list = new ArrayList<>(); - for(HiveDruidSplit split : inputSplits) { - String[] locations = split.getLocations(); - List locationsWithFaultyHost = Lists.newArrayListWithCapacity(locations.length + 1); - // A non-queryable host location. - locationsWithFaultyHost.add("localhost:8081"); - locationsWithFaultyHost.addAll(Arrays.asList(locations)); - HiveDruidSplit hiveDruidSplit = new HiveDruidSplit(split.getDruidQuery(), split.getPath(), - locationsWithFaultyHost.toArray(new String[0]) - ); - list.add(hiveDruidSplit); - } - return list.toArray(new HiveDruidSplit[0]); - } -} diff --git a/druid-handler/src/test/org/apache/hadoop/hive/druid/QTestDruidSerDe.java b/druid-handler/src/test/org/apache/hadoop/hive/druid/QTestDruidSerDe.java deleted file mode 100644 index 1764944ac04b..000000000000 --- a/druid-handler/src/test/org/apache/hadoop/hive/druid/QTestDruidSerDe.java +++ /dev/null @@ -1,104 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.druid; - -import java.util.List; - -import org.apache.hadoop.hive.druid.serde.DruidSerDe; -import org.apache.hadoop.hive.serde2.SerDeException; -import org.apache.hadoop.util.StringUtils; - -import com.fasterxml.jackson.core.type.TypeReference; - -import org.apache.druid.query.metadata.metadata.SegmentAnalysis; -import org.apache.druid.query.metadata.metadata.SegmentMetadataQuery; - -/** - * Druid SerDe to be used in tests. - */ -public class QTestDruidSerDe extends DruidSerDe { - - // Request : - // "{\"queryType\":\"segmentMetadata\",\"dataSource\":{\"type\":\"table\",\"name\":\"wikipedia\"}," - // + "\"intervals\":{\"type\":\"intervals\"," - // + "\"intervals\":[\"-146136543-09-08T00:30:34.096-07:52:58/146140482-04-24T08:36:27.903-07:00\"]}," - // + "\"toInclude\":{\"type\":\"all\"},\"merge\":true,\"context\":null,\"analysisTypes\":[]," - // + "\"usingDefaultInterval\":true,\"lenientAggregatorMerge\":false,\"descending\":false}"; - private static final String - RESPONSE = - "[ {\r\n " - + " \"id\" : \"merged\",\r\n " - + " \"intervals\" : [ \"2010-01-01T00:00:00.000Z/2015-12-31T00:00:00.000Z\" ],\r\n " - + " \"columns\" : {\r\n " - + " \"__time\" : { \"type\" : \"LONG\", \"hasMultipleValues\" : false, \"size\" : 407240380, " - + "\"cardinality\" : null, \"errorMessage\" : null },\r\n " - + " \"robot\" : { \"type\" : \"STRING\", \"hasMultipleValues\" : false, \"size\" : 100000, \"cardinality\"" - + " : 1944, \"errorMessage\" : null },\r\n " - + " \"namespace\" : { \"type\" : \"STRING\", \"hasMultipleValues\" : true, \"size\" : 100000, " - + "\"cardinality\" : 1504, \"errorMessage\" : null },\r\n " - + " \"anonymous\" : { \"type\" : \"STRING\", \"hasMultipleValues\" : false, \"size\" : 100000, " - + "\"cardinality\" : 1944, \"errorMessage\" : null },\r\n " - + " \"unpatrolled\" : { \"type\" : \"STRING\", \"hasMultipleValues\" : false, \"size\" : 100000, " - + "\"cardinality\" : 1944, \"errorMessage\" : null },\r\n " - + " \"page\" : { \"type\" : \"STRING\", \"hasMultipleValues\" : false, \"size\" : 100000, \"cardinality\" " - + ": 1944, \"errorMessage\" : null },\r\n " - + " \"language\" : { \"type\" : \"STRING\", \"hasMultipleValues\" : false, \"size\" : 100000, " - + "\"cardinality\" : 1944, \"errorMessage\" : null },\r\n " - + " \"newpage\" : { \"type\" : \"STRING\", \"hasMultipleValues\" : false, \"size\" : 100000, " - + "\"cardinality\" : 1944, \"errorMessage\" : null },\r\n " - + " \"user\" : { \"type\" : \"STRING\", \"hasMultipleValues\" : false, \"size\" : 100000, \"cardinality\" " - + ": 1944, \"errorMessage\" : null },\r\n " - + " \"count\" : { \"type\" : \"FLOAT\", \"hasMultipleValues\" : false, \"size\" : 100000, \"cardinality\" " - + ": null, \"errorMessage\" : null },\r\n " - + " \"added\" : { \"type\" : \"FLOAT\", \"hasMultipleValues\" : false, \"size\" : 100000, \"cardinality\" " - + ": null, \"errorMessage\" : null },\r\n " - + " \"delta\" : { \"type\" : \"FLOAT\", \"hasMultipleValues\" : false, \"size\" : 100000, \"cardinality\" " - + ": null, \"errorMessage\" : null },\r\n " - + " \"variation\" : { \"type\" : \"FLOAT\", \"hasMultipleValues\" : false, \"size\" : 100000, " - + "\"cardinality\" : null, \"errorMessage\" : null },\r\n " - + " \"deleted\" : { \"type\" : \"FLOAT\", \"hasMultipleValues\" : false, \"size\" : 100000, " - + "\"cardinality\" : null, \"errorMessage\" : null }\r\n " - + " },\r\n " - + " \"aggregators\" : {\r\n " - + " \"count\" : { \"type\" : \"longSum\", \"name\" : \"count\", \"fieldName\" : \"count\" },\r\n " - + " \"added\" : { \"type\" : \"doubleSum\", \"name\" : \"added\", \"fieldName\" : \"added\" },\r\n " - + " \"delta\" : { \"type\" : \"doubleSum\", \"name\" : \"delta\", \"fieldName\" : \"delta\" },\r\n " - + " \"variation\" : { \"type\" : \"doubleSum\", \"name\" : \"variation\", \"fieldName\" : \"variation\" }," - + "\r\n " - + " \"deleted\" : { \"type\" : \"doubleSum\", \"name\" : \"deleted\", \"fieldName\" : \"deleted\" }\r\n " - + " },\r\n " - + " \"queryGranularity\" : {\r\n \"type\": \"none\"\r\n },\r\n " - + " \"size\" : 300000,\r\n " - + " \"numRows\" : 5000000\r\n} ]"; - - /* Submits the request and returns */ - @Override protected SegmentAnalysis submitMetadataRequest(String address, SegmentMetadataQuery query) - throws SerDeException { - // Retrieve results - List resultsList; - try { - resultsList = - DruidStorageHandlerUtils.JSON_MAPPER.readValue(RESPONSE, new TypeReference>() { - }); - } catch (Exception e) { - throw new SerDeException(StringUtils.stringifyException(e)); - } - return resultsList.get(0); - } - -} diff --git a/druid-handler/src/test/org/apache/hadoop/hive/druid/QTestDruidSerDe2.java b/druid-handler/src/test/org/apache/hadoop/hive/druid/QTestDruidSerDe2.java deleted file mode 100644 index 55aa1c84b7aa..000000000000 --- a/druid-handler/src/test/org/apache/hadoop/hive/druid/QTestDruidSerDe2.java +++ /dev/null @@ -1,106 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.druid; - -import java.util.List; - -import org.apache.hadoop.hive.druid.serde.DruidSerDe; -import org.apache.hadoop.hive.serde2.SerDeException; -import org.apache.hadoop.util.StringUtils; - -import com.fasterxml.jackson.core.type.TypeReference; - -import org.apache.druid.query.metadata.metadata.SegmentAnalysis; -import org.apache.druid.query.metadata.metadata.SegmentMetadataQuery; - -/** - * Druid SerDe to be used in tests. - */ -public class QTestDruidSerDe2 extends DruidSerDe { - - // Request : - // "{\"queryType\":\"segmentMetadata\",\"dataSource\":{\"type\":\"table\",\"name\":\"wikipedia\"}," - // + "\"intervals\":{\"type\":\"intervals\"," - // + "\"intervals\":[\"-146136543-09-08T00:30:34.096-07:52:58/146140482-04-24T08:36:27.903-07:00\"]}," - // + "\"toInclude\":{\"type\":\"all\"},\"merge\":true,\"context\":null,\"analysisTypes\":[]," - // + "\"usingDefaultInterval\":true,\"lenientAggregatorMerge\":false,\"descending\":false}"; - private static final String - RESPONSE = - "[ {\r\n " - + " \"id\" : \"merged\",\r\n " - + " \"intervals\" : [ \"2010-01-01T00:00:00.000Z/2015-12-31T00:00:00.000Z\" ],\r\n " - + " \"columns\" : {\r\n " - + " \"__time\" : { \"type\" : \"LONG\", \"hasMultipleValues\" : false, \"size\" : 407240380, " - + "\"cardinality\" : null, \"errorMessage\" : null },\r\n " - + " \"robot\" : { \"type\" : \"STRING\", \"hasMultipleValues\" : false, \"size\" : 100000, \"cardinality\"" - + " : 1944, \"errorMessage\" : null },\r\n " - + " \"namespace\" : { \"type\" : \"STRING\", \"hasMultipleValues\" : true, \"size\" : 100000, " - + "\"cardinality\" : 1504, \"errorMessage\" : null },\r\n " - + " \"anonymous\" : { \"type\" : \"STRING\", \"hasMultipleValues\" : false, \"size\" : 100000, " - + "\"cardinality\" : 1944, \"errorMessage\" : null },\r\n " - // Next column has a similar name as previous, but different casing. - // This is allowed in Druid, but it should fail in Hive. - + " \"Anonymous\" : { \"type\" : \"STRING\", \"hasMultipleValues\" : false, \"size\" : 100000, " - + "\"cardinality\" : 1944, \"errorMessage\" : null },\r\n " - + " \"page\" : { \"type\" : \"STRING\", \"hasMultipleValues\" : false, \"size\" : 100000, \"cardinality\" " - + ": 1944, \"errorMessage\" : null },\r\n " - + " \"language\" : { \"type\" : \"STRING\", \"hasMultipleValues\" : false, \"size\" : 100000, " - + "\"cardinality\" : 1944, \"errorMessage\" : null },\r\n " - + " \"newpage\" : { \"type\" : \"STRING\", \"hasMultipleValues\" : false, \"size\" : 100000, " - + "\"cardinality\" : 1944, \"errorMessage\" : null },\r\n " - + " \"user\" : { \"type\" : \"STRING\", \"hasMultipleValues\" : false, \"size\" : 100000, \"cardinality\" " - + ": 1944, \"errorMessage\" : null },\r\n " - + " \"count\" : { \"type\" : \"FLOAT\", \"hasMultipleValues\" : false, \"size\" : 100000, \"cardinality\" " - + ": null, \"errorMessage\" : null },\r\n " - + " \"added\" : { \"type\" : \"FLOAT\", \"hasMultipleValues\" : false, \"size\" : 100000, \"cardinality\" " - + ": null, \"errorMessage\" : null },\r\n " - + " \"delta\" : { \"type\" : \"FLOAT\", \"hasMultipleValues\" : false, \"size\" : 100000, \"cardinality\" " - + ": null, \"errorMessage\" : null },\r\n " - + " \"variation\" : { \"type\" : \"FLOAT\", \"hasMultipleValues\" : false, \"size\" : 100000, " - + "\"cardinality\" : null, \"errorMessage\" : null },\r\n " - + " \"deleted\" : { \"type\" : \"FLOAT\", \"hasMultipleValues\" : false, \"size\" : 100000, " - + "\"cardinality\" : null, \"errorMessage\" : null }\r\n " - + " },\r\n " - + " \"aggregators\" : {\r\n " - + " \"count\" : { \"type\" : \"longSum\", \"name\" : \"count\", \"fieldName\" : \"count\" },\r\n " - + " \"added\" : { \"type\" : \"doubleSum\", \"name\" : \"added\", \"fieldName\" : \"added\" },\r\n " - + " \"delta\" : { \"type\" : \"doubleSum\", \"name\" : \"delta\", \"fieldName\" : \"delta\" },\r\n " - + " \"variation\" : { \"type\" : \"doubleSum\", \"name\" : \"variation\", \"fieldName\" : \"variation\" }," - + "\r\n " - + " \"deleted\" : { \"type\" : \"doubleSum\", \"name\" : \"deleted\", \"fieldName\" : \"deleted\" }\r\n " - + " },\r\n " - + " \"queryGranularity\" : {\r\n \"type\": \"none\"\r\n },\r\n " - + " \"size\" : 300000,\r\n " - + " \"numRows\" : 5000000\r\n} ]"; - - /* Submits the request and returns */ - @Override protected SegmentAnalysis submitMetadataRequest(String address, SegmentMetadataQuery query) - throws SerDeException { - // Retrieve results - List resultsList; - try { - resultsList = - DruidStorageHandlerUtils.JSON_MAPPER.readValue(RESPONSE, new TypeReference>() { - }); - } catch (Exception e) { - throw new SerDeException(StringUtils.stringifyException(e)); - } - return resultsList.get(0); - } - -} diff --git a/druid-handler/src/test/org/apache/hadoop/hive/druid/QTestDruidStorageHandler.java b/druid-handler/src/test/org/apache/hadoop/hive/druid/QTestDruidStorageHandler.java deleted file mode 100644 index e17948c3fe6f..000000000000 --- a/druid-handler/src/test/org/apache/hadoop/hive/druid/QTestDruidStorageHandler.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.druid; - -import org.apache.hadoop.hive.serde2.AbstractSerDe; - -/** - * Storage handler for Druid to be used in tests. It cannot connect to - * Druid, and thus it cannot execute queries. - */ -@SuppressWarnings("deprecation") -public class QTestDruidStorageHandler extends DruidStorageHandler { - - @Override - public Class getSerDeClass() { - return QTestDruidSerDe.class; - } - -} diff --git a/druid-handler/src/test/org/apache/hadoop/hive/druid/QTestDruidStorageHandler2.java b/druid-handler/src/test/org/apache/hadoop/hive/druid/QTestDruidStorageHandler2.java deleted file mode 100644 index 9265a5d0a558..000000000000 --- a/druid-handler/src/test/org/apache/hadoop/hive/druid/QTestDruidStorageHandler2.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.druid; - -import org.apache.hadoop.hive.serde2.AbstractSerDe; - -/** - * Storage handler for Druid to be used in tests. It cannot connect to - * Druid, and thus it cannot execute queries. - */ -@SuppressWarnings("deprecation") -public class QTestDruidStorageHandler2 extends DruidStorageHandler { - - @Override - public Class getSerDeClass() { - return QTestDruidSerDe2.class; - } - -} diff --git a/druid-handler/src/test/org/apache/hadoop/hive/druid/QTestDruidStorageHandlerToAddFaultyHost.java b/druid-handler/src/test/org/apache/hadoop/hive/druid/QTestDruidStorageHandlerToAddFaultyHost.java deleted file mode 100644 index e0dec01e0614..000000000000 --- a/druid-handler/src/test/org/apache/hadoop/hive/druid/QTestDruidStorageHandlerToAddFaultyHost.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.druid; - -import org.apache.hadoop.mapred.InputFormat; - -/** - * Storage handler for Druid to be used in tests. - * It uses an input format that adds a faulty host as first input split location. - * Tests should be able to query results successfully by trying next split locations. - */ -@SuppressWarnings("deprecation") -public class QTestDruidStorageHandlerToAddFaultyHost extends DruidStorageHandler { - - @Override - public Class getInputFormatClass() { - return QTestDruidQueryBasedInputFormatToAddFaultyHost.class; - } -} diff --git a/druid-handler/src/test/org/apache/hadoop/hive/druid/TestDruidStorageHandler.java b/druid-handler/src/test/org/apache/hadoop/hive/druid/TestDruidStorageHandler.java deleted file mode 100644 index 5040831b9a94..000000000000 --- a/druid-handler/src/test/org/apache/hadoop/hive/druid/TestDruidStorageHandler.java +++ /dev/null @@ -1,774 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.druid; - -import com.google.common.base.Throwables; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Iterables; -import com.google.common.collect.Lists; -import org.apache.druid.indexer.JobHelper; -import org.apache.druid.indexer.SQLMetadataStorageUpdaterJobHandler; -import org.apache.druid.metadata.MetadataStorageTablesConfig; -import org.apache.druid.segment.loading.DataSegmentPusher; -import org.apache.druid.segment.loading.SegmentLoadingException; -import org.apache.druid.storage.hdfs.HdfsDataSegmentPusher; -import org.apache.druid.storage.hdfs.HdfsDataSegmentPusherConfig; -import org.apache.druid.timeline.DataSegment; -import org.apache.druid.timeline.partition.LinearShardSpec; -import org.apache.druid.timeline.partition.NoneShardSpec; -import org.apache.druid.timeline.partition.ShardSpec; -import org.apache.commons.io.FileUtils; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.LocalFileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.metastore.api.MetaException; -import org.apache.hadoop.hive.metastore.api.StorageDescriptor; -import org.apache.hadoop.hive.metastore.api.Table; -import org.joda.time.DateTimeZone; -import org.joda.time.Interval; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; -import org.mockito.Mockito; -import org.skife.jdbi.v2.Handle; - -import java.io.File; -import java.io.IOException; -import java.io.OutputStream; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.UUID; - -/** - * Test class for Druid Storage handler. - */ -public class TestDruidStorageHandler { - - @Rule public final DerbyConnectorTestUtility.DerbyConnectorRule - derbyConnectorRule = - new DerbyConnectorTestUtility.DerbyConnectorRule(); - - @Rule public final TemporaryFolder temporaryFolder = new TemporaryFolder(); - - private static final String DB_NAME = "default"; - private static final String TABLE_NAME = "testName"; - private static final String DATA_SOURCE_NAME = "default.testName"; - - private String segmentsTable; - - private String tableWorkingPath; - - private Configuration config; - - private DruidStorageHandler druidStorageHandler; - - private DataSegment createSegment(String location) throws IOException { - return createSegment(location, new Interval(100, 170, DateTimeZone.UTC), "v1", new LinearShardSpec(0)); - } - - private DataSegment createSegment(String location, Interval interval, String version, ShardSpec shardSpec) - throws IOException { - FileUtils.writeStringToFile(new File(location), "dummySegmentData"); - return DataSegment.builder().dataSource(DATA_SOURCE_NAME).version(version).interval(interval).shardSpec(shardSpec) - .loadSpec(ImmutableMap.of("path", location)).size(1000L) - .build(); - } - - @Before public void before() throws Throwable { - tableWorkingPath = temporaryFolder.newFolder().getAbsolutePath(); - segmentsTable = derbyConnectorRule.metadataTablesConfigSupplier().get().getSegmentsTable(); - Map params = new HashMap<>(); - params.put("external.table.purge", "TRUE"); - Mockito.when(tableMock.getParameters()).thenReturn(params); - Mockito.when(tableMock.getPartitionKeysSize()).thenReturn(0); - StorageDescriptor storageDes = Mockito.mock(StorageDescriptor.class); - Mockito.when(storageDes.getBucketColsSize()).thenReturn(0); - Mockito.when(tableMock.getSd()).thenReturn(storageDes); - Mockito.when(tableMock.getDbName()).thenReturn(DB_NAME); - Mockito.when(tableMock.getTableName()).thenReturn(TABLE_NAME); - config = new Configuration(); - config.set(String.valueOf(HiveConf.ConfVars.HIVE_QUERY_ID), "hive-" + UUID.randomUUID().toString()); - config.set(String.valueOf(HiveConf.ConfVars.DRUID_WORKING_DIR), tableWorkingPath); - config.set(String.valueOf(HiveConf.ConfVars.DRUID_SEGMENT_DIRECTORY), - new Path(tableWorkingPath, "finalSegmentDir").toString()); - config.set("hive.druid.maxTries", "0"); - druidStorageHandler = - new DruidStorageHandler(derbyConnectorRule.getConnector(), - derbyConnectorRule.metadataTablesConfigSupplier().get()); - druidStorageHandler.setConf(config); - } - - @After public void tearDown() { - temporaryFolder.delete(); - } - - private final Table tableMock = Mockito.mock(Table.class); - - @Test public void testPreCreateTableWillCreateSegmentsTable() throws MetaException { - - try (Handle handle = derbyConnectorRule.getConnector().getDBI().open()) { - Assert.assertFalse(derbyConnectorRule.getConnector().tableExists(handle, segmentsTable)); - druidStorageHandler.preCreateTable(tableMock); - Assert.assertTrue(derbyConnectorRule.getConnector().tableExists(handle, segmentsTable)); - } - - } - - @Test(expected = MetaException.class) public void testPreCreateTableWhenDataSourceExists() - throws MetaException, IOException { - derbyConnectorRule.getConnector().createSegmentTable(); - SQLMetadataStorageUpdaterJobHandler - sqlMetadataStorageUpdaterJobHandler = - new SQLMetadataStorageUpdaterJobHandler(derbyConnectorRule.getConnector()); - Path taskDirPath = new Path(tableWorkingPath, druidStorageHandler.makeStagingName()); - DataSegment dataSegment = createSegment(new Path(taskDirPath, "intermediatePath").toString()); - - sqlMetadataStorageUpdaterJobHandler.publishSegments(segmentsTable, - Collections.singletonList(dataSegment), - DruidStorageHandlerUtils.JSON_MAPPER); - - druidStorageHandler.preCreateTable(tableMock); - } - - @Test public void testCommitCreateTablePlusCommitDropTableWithoutPurge() throws MetaException, IOException { - druidStorageHandler.preCreateTable(tableMock); - LocalFileSystem localFileSystem = FileSystem.getLocal(config); - Path taskDirPath = new Path(tableWorkingPath, druidStorageHandler.makeStagingName()); - DataSegment dataSegment = createSegment(new Path(taskDirPath, "index.zip").toString()); - - Path - descriptorPath = - DruidStorageHandlerUtils.makeSegmentDescriptorOutputPath(dataSegment, - new Path(taskDirPath, DruidStorageHandler.SEGMENTS_DESCRIPTOR_DIR_NAME)); - DruidStorageHandlerUtils.writeSegmentDescriptor(localFileSystem, dataSegment, descriptorPath); - druidStorageHandler.commitCreateTable(tableMock); - Assert.assertArrayEquals(Lists.newArrayList(DATA_SOURCE_NAME).toArray(), - Lists.newArrayList(DruidStorageHandlerUtils.getAllDataSourceNames(derbyConnectorRule.getConnector(), - derbyConnectorRule.metadataTablesConfigSupplier().get())).toArray()); - druidStorageHandler.commitDropTable(tableMock, false); - Assert.assertArrayEquals(Lists.newArrayList().toArray(), - Lists.newArrayList(DruidStorageHandlerUtils.getAllDataSourceNames(derbyConnectorRule.getConnector(), - derbyConnectorRule.metadataTablesConfigSupplier().get())).toArray()); - } - - @Test public void testCommitCreateTablePlusCommitDropTableWithPurge() throws MetaException, IOException { - druidStorageHandler.preCreateTable(tableMock); - LocalFileSystem localFileSystem = FileSystem.getLocal(config); - Path taskDirPath = new Path(tableWorkingPath, druidStorageHandler.makeStagingName()); - DataSegment dataSegment = createSegment(new Path(taskDirPath, "index.zip").toString()); - - Path - descriptorPath = - DruidStorageHandlerUtils.makeSegmentDescriptorOutputPath(dataSegment, - new Path(taskDirPath, DruidStorageHandler.SEGMENTS_DESCRIPTOR_DIR_NAME)); - DruidStorageHandlerUtils.writeSegmentDescriptor(localFileSystem, dataSegment, descriptorPath); - druidStorageHandler.commitCreateTable(tableMock); - Assert.assertArrayEquals(Lists.newArrayList(DATA_SOURCE_NAME).toArray(), - Lists.newArrayList(DruidStorageHandlerUtils.getAllDataSourceNames(derbyConnectorRule.getConnector(), - derbyConnectorRule.metadataTablesConfigSupplier().get())).toArray()); - druidStorageHandler.commitDropTable(tableMock, true); - Assert.assertArrayEquals(Lists.newArrayList().toArray(), - Lists.newArrayList(DruidStorageHandlerUtils.getAllDataSourceNames(derbyConnectorRule.getConnector(), - derbyConnectorRule.metadataTablesConfigSupplier().get())).toArray()); - } - - @Test public void testCommitCreateEmptyTablePlusCommitDropTableWithoutPurge() throws MetaException, IOException { - druidStorageHandler.preCreateTable(tableMock); - druidStorageHandler.commitCreateTable(tableMock); - Assert.assertArrayEquals(Lists.newArrayList().toArray(), - Lists.newArrayList(DruidStorageHandlerUtils.getAllDataSourceNames(derbyConnectorRule.getConnector(), - derbyConnectorRule.metadataTablesConfigSupplier().get())).toArray()); - druidStorageHandler.commitDropTable(tableMock, false); - Assert.assertArrayEquals(Lists.newArrayList().toArray(), - Lists.newArrayList(DruidStorageHandlerUtils.getAllDataSourceNames(derbyConnectorRule.getConnector(), - derbyConnectorRule.metadataTablesConfigSupplier().get())).toArray()); - } - - @Test public void testCommitCreateEmptyTablePlusCommitDropTableWithPurge() throws MetaException, IOException { - druidStorageHandler.preCreateTable(tableMock); - druidStorageHandler.commitCreateTable(tableMock); - Assert.assertArrayEquals(Lists.newArrayList().toArray(), - Lists.newArrayList(DruidStorageHandlerUtils.getAllDataSourceNames(derbyConnectorRule.getConnector(), - derbyConnectorRule.metadataTablesConfigSupplier().get())).toArray()); - druidStorageHandler.commitDropTable(tableMock, true); - Assert.assertArrayEquals(Lists.newArrayList().toArray(), - Lists.newArrayList(DruidStorageHandlerUtils.getAllDataSourceNames(derbyConnectorRule.getConnector(), - derbyConnectorRule.metadataTablesConfigSupplier().get())).toArray()); - } - - @Test public void testCommitInsertTable() throws MetaException, IOException { - druidStorageHandler.preCreateTable(tableMock); - LocalFileSystem localFileSystem = FileSystem.getLocal(config); - Path taskDirPath = new Path(tableWorkingPath, druidStorageHandler.makeStagingName()); - DataSegment dataSegment = createSegment(new Path(taskDirPath, "index.zip").toString()); - Path - descriptorPath = - DruidStorageHandlerUtils.makeSegmentDescriptorOutputPath(dataSegment, - new Path(taskDirPath, DruidStorageHandler.SEGMENTS_DESCRIPTOR_DIR_NAME)); - DruidStorageHandlerUtils.writeSegmentDescriptor(localFileSystem, dataSegment, descriptorPath); - druidStorageHandler.commitCreateTable(tableMock); - Assert.assertArrayEquals(Lists.newArrayList(DATA_SOURCE_NAME).toArray(), - Lists.newArrayList(DruidStorageHandlerUtils.getAllDataSourceNames(derbyConnectorRule.getConnector(), - derbyConnectorRule.metadataTablesConfigSupplier().get())).toArray()); - } - - @Test public void testCommitEmptyInsertTable() throws MetaException, IOException { - druidStorageHandler.preCreateTable(tableMock); - druidStorageHandler.commitCreateTable(tableMock); - Assert.assertArrayEquals(Lists.newArrayList().toArray(), - Lists.newArrayList(DruidStorageHandlerUtils.getAllDataSourceNames(derbyConnectorRule.getConnector(), - derbyConnectorRule.metadataTablesConfigSupplier().get())).toArray()); - } - - @Test public void testDeleteSegment() throws IOException, SegmentLoadingException { - String segmentRootPath = temporaryFolder.newFolder().getAbsolutePath(); - LocalFileSystem localFileSystem = FileSystem.getLocal(config); - Path taskDirPath = new Path(tableWorkingPath, druidStorageHandler.makeStagingName()); - DataSegment dataSegment = createSegment(new Path(taskDirPath, "index.zip").toString()); - HdfsDataSegmentPusherConfig hdfsDSPConfig = new HdfsDataSegmentPusherConfig(); - hdfsDSPConfig.setStorageDirectory(segmentRootPath); - HdfsDataSegmentPusher - hdfsDataSegmentPusher = - new HdfsDataSegmentPusher(hdfsDSPConfig, config, DruidStorageHandlerUtils.JSON_MAPPER); - Path - segmentOutputPath = - JobHelper.makeFileNamePath(new Path(segmentRootPath), - localFileSystem, - dataSegment, - JobHelper.INDEX_ZIP, - hdfsDataSegmentPusher); - Path indexPath = new Path(segmentOutputPath, "index.zip"); - DataSegment - dataSegmentWithLoadspect = - DataSegment.builder(dataSegment).loadSpec(ImmutableMap.of("path", indexPath)).build(); - OutputStream outputStream = localFileSystem.create(indexPath, true); - outputStream.close(); - Assert.assertTrue("index file is not created ??", localFileSystem.exists(indexPath)); - Assert.assertTrue(localFileSystem.exists(segmentOutputPath)); - - druidStorageHandler.deleteSegment(dataSegmentWithLoadspect); - // path format -- > .../dataSource/interval/version/partitionNum/xxx.zip - Assert.assertFalse("Index file still there ??", localFileSystem.exists(indexPath)); - // path format of segmentOutputPath -- > .../dataSource/interval/version/partitionNum/ - Assert.assertFalse("PartitionNum directory still there ??", localFileSystem.exists(segmentOutputPath)); - Assert.assertFalse("Version directory still there ??", localFileSystem.exists(segmentOutputPath.getParent())); - Assert.assertFalse("Interval directory still there ??", - localFileSystem.exists(segmentOutputPath.getParent().getParent())); - Assert.assertFalse("Data source directory still there ??", - localFileSystem.exists(segmentOutputPath.getParent().getParent().getParent())); - } - - @Test public void testCommitInsertOverwriteTable() throws MetaException, IOException { - DerbyConnectorTestUtility connector = derbyConnectorRule.getConnector(); - MetadataStorageTablesConfig metadataStorageTablesConfig = derbyConnectorRule.metadataTablesConfigSupplier().get(); - druidStorageHandler.preCreateTable(tableMock); - LocalFileSystem localFileSystem = FileSystem.getLocal(config); - Path taskDirPath = new Path(tableWorkingPath, druidStorageHandler.makeStagingName()); - HdfsDataSegmentPusherConfig pusherConfig = new HdfsDataSegmentPusherConfig(); - pusherConfig.setStorageDirectory(config.get(String.valueOf(HiveConf.ConfVars.DRUID_SEGMENT_DIRECTORY))); - DataSegmentPusher - dataSegmentPusher = - new HdfsDataSegmentPusher(pusherConfig, config, DruidStorageHandlerUtils.JSON_MAPPER); - - // This create and publish the segment to be overwritten - List - existingSegments = - Collections.singletonList(createSegment(new Path(taskDirPath, DruidStorageHandlerUtils.INDEX_ZIP).toString(), - new Interval(100, 150, DateTimeZone.UTC), - "v0", - new LinearShardSpec(0))); - DruidStorageHandlerUtils.publishSegmentsAndCommit(connector, - metadataStorageTablesConfig, - DATA_SOURCE_NAME, - existingSegments, - true, - config, - dataSegmentPusher); - - // This creates and publish new segment - DataSegment - dataSegment = - createSegment(new Path(taskDirPath, DruidStorageHandlerUtils.INDEX_ZIP).toString(), - new Interval(180, 250, DateTimeZone.UTC), - "v1", - new LinearShardSpec(0)); - - Path - descriptorPath = - DruidStorageHandlerUtils.makeSegmentDescriptorOutputPath(dataSegment, - new Path(taskDirPath, DruidStorageHandler.SEGMENTS_DESCRIPTOR_DIR_NAME)); - DruidStorageHandlerUtils.writeSegmentDescriptor(localFileSystem, dataSegment, descriptorPath); - - druidStorageHandler.commitInsertTable(tableMock, true); - Assert.assertArrayEquals(Lists.newArrayList(DATA_SOURCE_NAME).toArray(), - Lists.newArrayList(DruidStorageHandlerUtils.getAllDataSourceNames(connector, metadataStorageTablesConfig)) - .toArray()); - - final List dataSegmentList = getUsedSegmentsList(connector, metadataStorageTablesConfig); - Assert.assertEquals(1, dataSegmentList.size()); - DataSegment persistedSegment = Iterables.getOnlyElement(dataSegmentList); - Assert.assertEquals(dataSegment, persistedSegment); - Assert.assertEquals(dataSegment.getVersion(), persistedSegment.getVersion()); - Path - expectedFinalHadoopPath = - new Path(dataSegmentPusher.getPathForHadoop(), - dataSegmentPusher.makeIndexPathName(persistedSegment, DruidStorageHandlerUtils.INDEX_ZIP)); - Assert.assertEquals(ImmutableMap.of("type", "hdfs", "path", expectedFinalHadoopPath.toString()), - persistedSegment.getLoadSpec()); - Assert.assertEquals("dummySegmentData", FileUtils.readFileToString(new File(expectedFinalHadoopPath.toUri()))); - } - - @Test public void testCommitMultiInsertOverwriteTable() throws MetaException, IOException { - DerbyConnectorTestUtility connector = derbyConnectorRule.getConnector(); - MetadataStorageTablesConfig metadataStorageTablesConfig = derbyConnectorRule.metadataTablesConfigSupplier().get(); - LocalFileSystem localFileSystem = FileSystem.getLocal(config); - druidStorageHandler.preCreateTable(tableMock); - Path taskDirPath = new Path(tableWorkingPath, druidStorageHandler.makeStagingName()); - HdfsDataSegmentPusherConfig pusherConfig = new HdfsDataSegmentPusherConfig(); - pusherConfig.setStorageDirectory(config.get(String.valueOf(HiveConf.ConfVars.DRUID_SEGMENT_DIRECTORY))); - DataSegmentPusher - dataSegmentPusher = - new HdfsDataSegmentPusher(pusherConfig, config, DruidStorageHandlerUtils.JSON_MAPPER); - - // This create and publish the segment to be overwritten - List - existingSegments = - Collections.singletonList(createSegment(new Path(taskDirPath, DruidStorageHandlerUtils.INDEX_ZIP).toString(), - new Interval(100, 150, DateTimeZone.UTC), - "v0", - new LinearShardSpec(0))); - DruidStorageHandlerUtils.publishSegmentsAndCommit(connector, - metadataStorageTablesConfig, - DATA_SOURCE_NAME, - existingSegments, - true, - config, - dataSegmentPusher); - // Check that there is one datasource with the published segment - Assert.assertArrayEquals(Lists.newArrayList(DATA_SOURCE_NAME).toArray(), - Lists.newArrayList(DruidStorageHandlerUtils.getAllDataSourceNames(connector, metadataStorageTablesConfig)) - .toArray()); - - // Sequence is the following: - // 1) INSERT with no segments -> Original segment still present in the datasource - // 2) INSERT OVERWRITE with no segments -> Datasource is empty - // 3) INSERT OVERWRITE with no segments -> Datasource is empty - // 4) INSERT with no segments -> Datasource is empty - // 5) INSERT with one segment -> Datasource has one segment - // 6) INSERT OVERWRITE with one segment -> Datasource has one segment - // 7) INSERT with one segment -> Datasource has two segments - // 8) INSERT OVERWRITE with no segments -> Datasource is empty - - // We start: - // #1 - druidStorageHandler.commitInsertTable(tableMock, false); - Assert.assertArrayEquals(Lists.newArrayList(DATA_SOURCE_NAME).toArray(), - Lists.newArrayList(DruidStorageHandlerUtils.getAllDataSourceNames(connector, metadataStorageTablesConfig)) - .toArray()); - Assert.assertEquals(1, getUsedSegmentsList(connector, metadataStorageTablesConfig).size()); - - // #2 - druidStorageHandler.commitInsertTable(tableMock, true); - Assert.assertEquals(0, getUsedSegmentsList(connector, metadataStorageTablesConfig).size()); - - // #3 - druidStorageHandler.commitInsertTable(tableMock, true); - Assert.assertEquals(0, getUsedSegmentsList(connector, metadataStorageTablesConfig).size()); - - // #4 - druidStorageHandler.commitInsertTable(tableMock, true); - Assert.assertEquals(0, getUsedSegmentsList(connector, metadataStorageTablesConfig).size()); - - // #5 - DataSegment - dataSegment1 = - createSegment(new Path(taskDirPath, DruidStorageHandlerUtils.INDEX_ZIP).toString(), - new Interval(180, 250, DateTimeZone.UTC), - "v1", - new LinearShardSpec(0)); - Path - descriptorPath1 = - DruidStorageHandlerUtils.makeSegmentDescriptorOutputPath(dataSegment1, - new Path(taskDirPath, DruidStorageHandler.SEGMENTS_DESCRIPTOR_DIR_NAME)); - DruidStorageHandlerUtils.writeSegmentDescriptor(localFileSystem, dataSegment1, descriptorPath1); - druidStorageHandler.commitInsertTable(tableMock, false); - Assert.assertArrayEquals(Lists.newArrayList(DATA_SOURCE_NAME).toArray(), - Lists.newArrayList(DruidStorageHandlerUtils.getAllDataSourceNames(connector, metadataStorageTablesConfig)) - .toArray()); - Assert.assertEquals(1, getUsedSegmentsList(connector, metadataStorageTablesConfig).size()); - - // #6 - DataSegment - dataSegment2 = - createSegment(new Path(taskDirPath, DruidStorageHandlerUtils.INDEX_ZIP).toString(), - new Interval(200, 250, DateTimeZone.UTC), - "v1", - new LinearShardSpec(0)); - Path - descriptorPath2 = - DruidStorageHandlerUtils.makeSegmentDescriptorOutputPath(dataSegment2, - new Path(taskDirPath, DruidStorageHandler.SEGMENTS_DESCRIPTOR_DIR_NAME)); - DruidStorageHandlerUtils.writeSegmentDescriptor(localFileSystem, dataSegment2, descriptorPath2); - druidStorageHandler.commitInsertTable(tableMock, true); - Assert.assertArrayEquals(Lists.newArrayList(DATA_SOURCE_NAME).toArray(), - Lists.newArrayList(DruidStorageHandlerUtils.getAllDataSourceNames(connector, metadataStorageTablesConfig)) - .toArray()); - Assert.assertEquals(1, getUsedSegmentsList(connector, metadataStorageTablesConfig).size()); - - // #7 - DataSegment - dataSegment3 = - createSegment(new Path(taskDirPath, DruidStorageHandlerUtils.INDEX_ZIP).toString(), - new Interval(100, 200, DateTimeZone.UTC), - "v1", - new LinearShardSpec(0)); - Path - descriptorPath3 = - DruidStorageHandlerUtils.makeSegmentDescriptorOutputPath(dataSegment3, - new Path(taskDirPath, DruidStorageHandler.SEGMENTS_DESCRIPTOR_DIR_NAME)); - DruidStorageHandlerUtils.writeSegmentDescriptor(localFileSystem, dataSegment3, descriptorPath3); - druidStorageHandler.commitInsertTable(tableMock, false); - Assert.assertArrayEquals(Lists.newArrayList(DATA_SOURCE_NAME).toArray(), - Lists.newArrayList(DruidStorageHandlerUtils.getAllDataSourceNames(connector, metadataStorageTablesConfig)) - .toArray()); - Assert.assertEquals(2, getUsedSegmentsList(connector, metadataStorageTablesConfig).size()); - - // #8 - druidStorageHandler.commitInsertTable(tableMock, true); - Assert.assertEquals(0, getUsedSegmentsList(connector, metadataStorageTablesConfig).size()); - } - - private List getUsedSegmentsList(DerbyConnectorTestUtility connector, - final MetadataStorageTablesConfig metadataStorageTablesConfig) { - return connector.getDBI() - .withHandle(handle -> handle.createQuery(String.format( - "SELECT payload FROM %s WHERE used=true ORDER BY created_date ASC", - metadataStorageTablesConfig.getSegmentsTable())) - .map((i, resultSet, statementContext) -> { - try { - return DruidStorageHandlerUtils.JSON_MAPPER.readValue(resultSet.getBytes("payload"), DataSegment.class); - } catch (IOException e) { - throw Throwables.propagate(e); - } - }).list()); - } - - @Test public void testCommitInsertIntoTable() throws MetaException, IOException { - DerbyConnectorTestUtility connector = derbyConnectorRule.getConnector(); - MetadataStorageTablesConfig metadataStorageTablesConfig = derbyConnectorRule.metadataTablesConfigSupplier().get(); - druidStorageHandler.preCreateTable(tableMock); - LocalFileSystem localFileSystem = FileSystem.getLocal(config); - Path taskDirPath = new Path(tableWorkingPath, druidStorageHandler.makeStagingName()); - List - existingSegments = - Collections.singletonList(createSegment(new Path(taskDirPath, DruidStorageHandlerUtils.INDEX_ZIP).toString(), - new Interval(100, 150, DateTimeZone.UTC), - "v0", - new LinearShardSpec(1))); - HdfsDataSegmentPusherConfig pusherConfig = new HdfsDataSegmentPusherConfig(); - pusherConfig.setStorageDirectory(config.get(String.valueOf(HiveConf.ConfVars.DRUID_SEGMENT_DIRECTORY))); - DataSegmentPusher - dataSegmentPusher = - new HdfsDataSegmentPusher(pusherConfig, config, DruidStorageHandlerUtils.JSON_MAPPER); - - DruidStorageHandlerUtils.publishSegmentsAndCommit(connector, - metadataStorageTablesConfig, - DATA_SOURCE_NAME, - existingSegments, - true, - config, - dataSegmentPusher); - DataSegment - dataSegment = - createSegment(new Path(taskDirPath, DruidStorageHandlerUtils.INDEX_ZIP).toString(), - new Interval(100, 150, DateTimeZone.UTC), - "v1", - new LinearShardSpec(0)); - Path - descriptorPath = - DruidStorageHandlerUtils.makeSegmentDescriptorOutputPath(dataSegment, - new Path(taskDirPath, DruidStorageHandler.SEGMENTS_DESCRIPTOR_DIR_NAME)); - DruidStorageHandlerUtils.writeSegmentDescriptor(localFileSystem, dataSegment, descriptorPath); - druidStorageHandler.commitInsertTable(tableMock, false); - Assert.assertArrayEquals(Lists.newArrayList(DATA_SOURCE_NAME).toArray(), - Lists.newArrayList(DruidStorageHandlerUtils.getAllDataSourceNames(connector, metadataStorageTablesConfig)) - .toArray()); - - final List dataSegmentList = getUsedSegmentsList(connector, metadataStorageTablesConfig); - Assert.assertEquals(2, dataSegmentList.size()); - - DataSegment persistedSegment = dataSegmentList.get(1); - // Insert into appends to old version - Assert.assertEquals("v0", persistedSegment.getVersion()); - Assert.assertTrue(persistedSegment.getShardSpec() instanceof LinearShardSpec); - Assert.assertEquals(2, persistedSegment.getShardSpec().getPartitionNum()); - - Path - expectedFinalHadoopPath = - new Path(dataSegmentPusher.getPathForHadoop(), - dataSegmentPusher.makeIndexPathName(persistedSegment, DruidStorageHandlerUtils.INDEX_ZIP)); - - Assert.assertEquals(ImmutableMap.of("type", "hdfs", "path", expectedFinalHadoopPath.toString()), - persistedSegment.getLoadSpec()); - Assert.assertEquals("dummySegmentData", FileUtils.readFileToString(new File(expectedFinalHadoopPath.toUri()))); - } - - @Test public void testInsertIntoAppendOneMorePartition() throws MetaException, IOException { - DerbyConnectorTestUtility connector = derbyConnectorRule.getConnector(); - MetadataStorageTablesConfig metadataStorageTablesConfig = derbyConnectorRule.metadataTablesConfigSupplier().get(); - druidStorageHandler.preCreateTable(tableMock); - LocalFileSystem localFileSystem = FileSystem.getLocal(config); - Path taskDirPath = new Path(tableWorkingPath, druidStorageHandler.makeStagingName()); - HdfsDataSegmentPusherConfig pusherConfig = new HdfsDataSegmentPusherConfig(); - pusherConfig.setStorageDirectory(config.get(String.valueOf(HiveConf.ConfVars.DRUID_SEGMENT_DIRECTORY))); - DataSegmentPusher - dataSegmentPusher = - new HdfsDataSegmentPusher(pusherConfig, config, DruidStorageHandlerUtils.JSON_MAPPER); - - List - existingSegments = - Collections.singletonList(createSegment(new Path(taskDirPath, DruidStorageHandlerUtils.INDEX_ZIP).toString(), - new Interval(100, 150, DateTimeZone.UTC), - "v0", - new LinearShardSpec(0))); - DruidStorageHandlerUtils.publishSegmentsAndCommit(connector, - metadataStorageTablesConfig, - DATA_SOURCE_NAME, - existingSegments, - true, - config, - dataSegmentPusher); - - DataSegment - dataSegment = - createSegment(new Path(taskDirPath, DruidStorageHandlerUtils.INDEX_ZIP).toString(), - new Interval(100, 150, DateTimeZone.UTC), - "v0", - new LinearShardSpec(0)); - Path - descriptorPath = - DruidStorageHandlerUtils.makeSegmentDescriptorOutputPath(dataSegment, - new Path(taskDirPath, DruidStorageHandler.SEGMENTS_DESCRIPTOR_DIR_NAME)); - DruidStorageHandlerUtils.writeSegmentDescriptor(localFileSystem, dataSegment, descriptorPath); - druidStorageHandler.commitInsertTable(tableMock, false); - Assert.assertArrayEquals(Lists.newArrayList(DATA_SOURCE_NAME).toArray(), Lists.newArrayList( - DruidStorageHandlerUtils.getAllDataSourceNames(connector, - metadataStorageTablesConfig - )).toArray()); - - final List dataSegmentList = getUsedSegmentsList(connector, metadataStorageTablesConfig); - Assert.assertEquals(2, dataSegmentList.size()); - - DataSegment persistedSegment = dataSegmentList.get(1); - Assert.assertEquals("v0", persistedSegment.getVersion()); - Assert.assertTrue(persistedSegment.getShardSpec() instanceof LinearShardSpec); - Assert.assertEquals(1, persistedSegment.getShardSpec().getPartitionNum()); - - Path - expectedFinalHadoopPath = - new Path(dataSegmentPusher.getPathForHadoop(), - dataSegmentPusher.makeIndexPathName(persistedSegment, DruidStorageHandlerUtils.INDEX_ZIP)); - - Assert.assertEquals(ImmutableMap.of("type", "hdfs", "path", expectedFinalHadoopPath.toString()), - persistedSegment.getLoadSpec()); - Assert.assertEquals("dummySegmentData", FileUtils.readFileToString(new File(expectedFinalHadoopPath.toUri()))); - } - - @Test public void testCommitInsertIntoWhenDestinationSegmentFileExist() throws MetaException, IOException { - DerbyConnectorTestUtility connector = derbyConnectorRule.getConnector(); - MetadataStorageTablesConfig metadataStorageTablesConfig = derbyConnectorRule.metadataTablesConfigSupplier().get(); - druidStorageHandler.preCreateTable(tableMock); - LocalFileSystem localFileSystem = FileSystem.getLocal(config); - Path taskDirPath = new Path(tableWorkingPath, druidStorageHandler.makeStagingName()); - List - existingSegments = - Collections.singletonList(createSegment(new Path(taskDirPath, "index_old.zip").toString(), - new Interval(100, 150, DateTimeZone.UTC), - "v0", - new LinearShardSpec(1))); - HdfsDataSegmentPusherConfig pusherConfig = new HdfsDataSegmentPusherConfig(); - pusherConfig.setStorageDirectory(config.get(String.valueOf(HiveConf.ConfVars.DRUID_SEGMENT_DIRECTORY))); - DataSegmentPusher - dataSegmentPusher = - new HdfsDataSegmentPusher(pusherConfig, config, DruidStorageHandlerUtils.JSON_MAPPER); - DruidStorageHandlerUtils.publishSegmentsAndCommit(connector, - metadataStorageTablesConfig, - DATA_SOURCE_NAME, - existingSegments, - true, - config, - dataSegmentPusher); - DataSegment - dataSegment = - createSegment(new Path(taskDirPath, "index.zip").toString(), - new Interval(100, 150, DateTimeZone.UTC), - "v1", - new LinearShardSpec(0)); - Path - descriptorPath = - DruidStorageHandlerUtils.makeSegmentDescriptorOutputPath(dataSegment, - new Path(taskDirPath, DruidStorageHandler.SEGMENTS_DESCRIPTOR_DIR_NAME)); - DruidStorageHandlerUtils.writeSegmentDescriptor(localFileSystem, dataSegment, descriptorPath); - - // Create segment file at the destination location with LinearShardSpec(2) - DataSegment - segment = - createSegment(new Path(taskDirPath, "index_conflict.zip").toString(), - new Interval(100, 150, DateTimeZone.UTC), - "v1", - new LinearShardSpec(1)); - Path - segmentPath = - new Path(dataSegmentPusher.getPathForHadoop(), - dataSegmentPusher.makeIndexPathName(segment, DruidStorageHandlerUtils.INDEX_ZIP)); - FileUtils.writeStringToFile(new File(segmentPath.toUri()), "dummy"); - - druidStorageHandler.commitInsertTable(tableMock, false); - Assert.assertArrayEquals(Lists.newArrayList(DATA_SOURCE_NAME).toArray(), - Lists.newArrayList(DruidStorageHandlerUtils.getAllDataSourceNames(connector, metadataStorageTablesConfig)) - .toArray()); - - final List dataSegmentList = getUsedSegmentsList(connector, metadataStorageTablesConfig); - Assert.assertEquals(2, dataSegmentList.size()); - - DataSegment persistedSegment = dataSegmentList.get(1); - // Insert into appends to old version - Assert.assertEquals("v0", persistedSegment.getVersion()); - Assert.assertTrue(persistedSegment.getShardSpec() instanceof LinearShardSpec); - // insert into should skip and increment partition number to 3 - Assert.assertEquals(2, persistedSegment.getShardSpec().getPartitionNum()); - Path - expectedFinalHadoopPath = - new Path(dataSegmentPusher.getPathForHadoop(), - dataSegmentPusher.makeIndexPathName(persistedSegment, DruidStorageHandlerUtils.INDEX_ZIP)); - - Assert.assertEquals(ImmutableMap.of("type", "hdfs", "path", expectedFinalHadoopPath.toString()), - persistedSegment.getLoadSpec()); - Assert.assertEquals("dummySegmentData", FileUtils.readFileToString(new File(expectedFinalHadoopPath.toUri()))); - } - - @Test(expected = MetaException.class) public void testCommitInsertIntoWithConflictingIntervalSegment() - throws MetaException, IOException { - DerbyConnectorTestUtility connector = derbyConnectorRule.getConnector(); - MetadataStorageTablesConfig metadataStorageTablesConfig = derbyConnectorRule.metadataTablesConfigSupplier().get(); - druidStorageHandler.preCreateTable(tableMock); - LocalFileSystem localFileSystem = FileSystem.getLocal(config); - Path taskDirPath = new Path(tableWorkingPath, druidStorageHandler.makeStagingName()); - List - existingSegments = - Arrays.asList(createSegment(new Path(taskDirPath, "index_old_1.zip").toString(), - new Interval(100, 150, DateTimeZone.UTC), - "v0", - new LinearShardSpec(0)), - createSegment(new Path(taskDirPath, "index_old_2.zip").toString(), - new Interval(150, 200, DateTimeZone.UTC), - "v0", - new LinearShardSpec(0)), - createSegment(new Path(taskDirPath, "index_old_3.zip").toString(), - new Interval(200, 300, DateTimeZone.UTC), - "v0", - new LinearShardSpec(0))); - HdfsDataSegmentPusherConfig pusherConfig = new HdfsDataSegmentPusherConfig(); - pusherConfig.setStorageDirectory(taskDirPath.toString()); - DataSegmentPusher - dataSegmentPusher = - new HdfsDataSegmentPusher(pusherConfig, config, DruidStorageHandlerUtils.JSON_MAPPER); - DruidStorageHandlerUtils.publishSegmentsAndCommit(connector, - metadataStorageTablesConfig, - DATA_SOURCE_NAME, - existingSegments, - true, - config, - dataSegmentPusher); - - // Try appending segment with conflicting interval - DataSegment - conflictingSegment = - createSegment(new Path(taskDirPath, DruidStorageHandlerUtils.INDEX_ZIP).toString(), - new Interval(100, 300, DateTimeZone.UTC), - "v1", - new LinearShardSpec(0)); - Path - descriptorPath = - DruidStorageHandlerUtils.makeSegmentDescriptorOutputPath(conflictingSegment, - new Path(taskDirPath, DruidStorageHandler.SEGMENTS_DESCRIPTOR_DIR_NAME)); - DruidStorageHandlerUtils.writeSegmentDescriptor(localFileSystem, conflictingSegment, descriptorPath); - druidStorageHandler.commitInsertTable(tableMock, false); - } - - @Test(expected = MetaException.class) public void testCommitInsertIntoWithNonExtendableSegment() - throws MetaException, IOException { - DerbyConnectorTestUtility connector = derbyConnectorRule.getConnector(); - MetadataStorageTablesConfig metadataStorageTablesConfig = derbyConnectorRule.metadataTablesConfigSupplier().get(); - druidStorageHandler.preCreateTable(tableMock); - LocalFileSystem localFileSystem = FileSystem.getLocal(config); - Path taskDirPath = new Path(tableWorkingPath, druidStorageHandler.makeStagingName()); - List - existingSegments = - Arrays.asList(createSegment(new Path(taskDirPath, "index_old_1.zip").toString(), - new Interval(100, 150, DateTimeZone.UTC), - "v0", - NoneShardSpec.instance()), - createSegment(new Path(taskDirPath, "index_old_2.zip").toString(), - new Interval(200, 250, DateTimeZone.UTC), - "v0", - new LinearShardSpec(0)), - createSegment(new Path(taskDirPath, "index_old_3.zip").toString(), - new Interval(250, 300, DateTimeZone.UTC), - "v0", - new LinearShardSpec(0))); - HdfsDataSegmentPusherConfig pusherConfig = new HdfsDataSegmentPusherConfig(); - pusherConfig.setStorageDirectory(taskDirPath.toString()); - DataSegmentPusher - dataSegmentPusher = - new HdfsDataSegmentPusher(pusherConfig, config, DruidStorageHandlerUtils.JSON_MAPPER); - DruidStorageHandlerUtils.publishSegmentsAndCommit(connector, - metadataStorageTablesConfig, - DATA_SOURCE_NAME, - existingSegments, - true, - config, - dataSegmentPusher); - - // Try appending to non extendable shard spec - DataSegment - conflictingSegment = - createSegment(new Path(taskDirPath, DruidStorageHandlerUtils.INDEX_ZIP).toString(), - new Interval(100, 150, DateTimeZone.UTC), - "v1", - new LinearShardSpec(0)); - Path - descriptorPath = - DruidStorageHandlerUtils.makeSegmentDescriptorOutputPath(conflictingSegment, - new Path(taskDirPath, DruidStorageHandler.SEGMENTS_DESCRIPTOR_DIR_NAME)); - DruidStorageHandlerUtils.writeSegmentDescriptor(localFileSystem, conflictingSegment, descriptorPath); - - druidStorageHandler.commitInsertTable(tableMock, false); - - } - -} diff --git a/druid-handler/src/test/org/apache/hadoop/hive/druid/TestHiveDruidQueryBasedInputFormat.java b/druid-handler/src/test/org/apache/hadoop/hive/druid/TestHiveDruidQueryBasedInputFormat.java deleted file mode 100644 index 2bcbb14f9fbf..000000000000 --- a/druid-handler/src/test/org/apache/hadoop/hive/druid/TestHiveDruidQueryBasedInputFormat.java +++ /dev/null @@ -1,177 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.druid; - -import java.lang.reflect.Method; -import java.util.Arrays; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.conf.Constants; -import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.druid.io.DruidQueryBasedInputFormat; -import org.apache.hadoop.hive.druid.io.HiveDruidSplit; -import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; - -import org.apache.druid.query.Query; -import static org.junit.Assert.assertEquals; -import org.junit.Test; - -/** - * Test Class. - */ -@SuppressWarnings("SameParameterValue") public class TestHiveDruidQueryBasedInputFormat { - - private static final String - TIMESERIES_QUERY = - "{ \"queryType\": \"timeseries\", " - + " \"dataSource\": \"sample_datasource\", " - + " \"granularity\": \"DAY\", " - + " \"descending\": \"true\", " - + " \"intervals\": [ \"2012-01-01T00:00:00.000-08:00/2012-01-03T00:00:00.000-08:00\" ]}"; - private static final String - TIMESERIES_QUERY_SPLIT = - "[HiveDruidSplit{{\"queryType\":\"timeseries\"," - + "\"dataSource\":{\"type\":\"table\",\"name\":\"sample_datasource\"}," - + "\"intervals\":{\"type\":\"LegacySegmentSpec\",\"intervals\":" - + "[\"2012-01-01T08:00:00.000Z/2012-01-03T08:00:00.000Z\"]}," - + "\"descending\":true," - + "\"virtualColumns\":[]," - + "\"filter\":null," - + "\"granularity\":\"DAY\"," - + "\"aggregations\":[]," - + "\"postAggregations\":[]," - + "\"limit\":2147483647," - + "\"context\":{\"queryId\":\"\"}}, [localhost:8082]}]"; - - private static final String - TOPN_QUERY = - "{ \"queryType\": \"topN\", " - + " \"dataSource\": \"sample_data\", " - + " \"dimension\": \"sample_dim\", " - + " \"threshold\": 5, " - + " \"metric\": \"count\", " - + " \"aggregations\": [ " - + " { " - + " \"type\": \"longSum\", " - + " \"name\": \"count\", " - + " \"fieldName\": \"count\" " - + " }, " - + " { " - + " \"type\": \"doubleSum\", " - + " \"name\": \"some_metric\", " - + " \"fieldName\": \"some_metric\" " - + " } " - + " ], " - + " \"granularity\": \"all\", " - + " \"intervals\": [ " - + " \"2013-08-31T00:00:00.000-07:00/2013-09-03T00:00:00.000-07:00\" " - + " ]}"; - private static final String - TOPN_QUERY_SPLIT = - "[HiveDruidSplit{{\"queryType\":\"topN\"," - + "\"dataSource\":{\"type\":\"table\",\"name\":\"sample_data\"}," - + "\"virtualColumns\":[]," - + "\"dimension\":{\"type\":\"LegacyDimensionSpec\",\"dimension\":\"sample_dim\"," - + "\"outputName\":\"sample_dim\",\"outputType\":\"STRING\"}," - + "\"metric\":{\"type\":\"LegacyTopNMetricSpec\",\"metric\":\"count\"}," - + "\"threshold\":5," - + "\"intervals\":{\"type\":\"LegacySegmentSpec\",\"intervals\":[\"2013-08-31T07:00:00" - + ".000Z/2013-09-03T07:00:00.000Z\"]}," - + "\"filter\":null," - + "\"granularity\":{\"type\":\"all\"}," - + "\"aggregations\":[{\"type\":\"longSum\",\"name\":\"count\",\"fieldName\":\"count\",\"expression\":null}," - + "{\"type\":\"doubleSum\",\"name\":\"some_metric\",\"fieldName\":\"some_metric\",\"expression\":null}]," - + "\"postAggregations\":[]," - + "\"context\":{\"queryId\":\"\"}," - + "\"descending\":false}, [localhost:8082]}]"; - - private static final String - GROUP_BY_QUERY = - "{ \"queryType\": \"groupBy\", " - + " \"dataSource\": \"sample_datasource\", " - + " \"granularity\": \"day\", " - + " \"dimensions\": [\"country\", \"device\"], " - + " \"limitSpec\": {" - + " \"type\": \"default\"," - + " \"limit\": 5000," - + " \"columns\": [\"country\", \"data_transfer\"] }, " - + " \"aggregations\": [ " - + " { \"type\": \"longSum\", \"name\": \"total_usage\", \"fieldName\": \"user_count\" }, " - + " { \"type\": \"doubleSum\", \"name\": \"data_transfer\", \"fieldName\": \"data_transfer\" } " - + " ], " - + " \"intervals\": [ \"2012-01-01T00:00:00.000-08:00/2012-01-03T00:00:00.000-08:00\" ]" - + " }"; - private static final String - GROUP_BY_QUERY_SPLIT = - "[HiveDruidSplit{{\"queryType\":\"groupBy\"," - + "\"dataSource\":{\"type\":\"table\",\"name\":\"sample_datasource\"}," - + "\"intervals\":{\"type\":\"LegacySegmentSpec\",\"intervals\":[\"2012-01-01T08:00:00" - + ".000Z/2012-01-03T08:00:00.000Z\"]}," - + "\"virtualColumns\":[]," - + "\"filter\":null," - + "\"granularity\":\"DAY\"," - + "\"dimensions\":[{\"type\":\"LegacyDimensionSpec\",\"dimension\":\"country\",\"outputName\":\"country\"," - + "\"outputType\":\"STRING\"}," - + "{\"type\":\"LegacyDimensionSpec\",\"dimension\":\"device\",\"outputName\":\"device\"," - + "\"outputType\":\"STRING\"}]," - + "\"aggregations\":[{\"type\":\"longSum\",\"name\":\"total_usage\",\"fieldName\":\"user_count\"," - + "\"expression\":null}," - + "{\"type\":\"doubleSum\",\"name\":\"data_transfer\",\"fieldName\":\"data_transfer\",\"expression\":null}]," - + "\"postAggregations\":[]," - + "\"having\":null," - + "\"limitSpec\":{\"type\":\"default\",\"columns\":[{\"dimension\":\"country\",\"direction\":\"ascending\"," - + "\"dimensionOrder\":{\"type\":\"lexicographic\"}}," - + "{\"dimension\":\"data_transfer\",\"direction\":\"ascending\"," - + "\"dimensionOrder\":{\"type\":\"lexicographic\"}}],\"limit\":5000}," - + "\"context\":{\"queryId\":\"\"}," - + "\"descending\":false}, [localhost:8082]}]"; - - @Test - public void testTimeZone() throws Exception { - DruidQueryBasedInputFormat input = new DruidQueryBasedInputFormat(); - - Method method1 = DruidQueryBasedInputFormat.class.getDeclaredMethod("getInputSplits", Configuration.class); - method1.setAccessible(true); - - // Create, initialize, and test - Configuration conf = createPropertiesQuery("sample_datasource", Query.TIMESERIES, TIMESERIES_QUERY); - HiveDruidSplit[] resultSplits = (HiveDruidSplit[]) method1.invoke(input, conf); - assertEquals(TIMESERIES_QUERY_SPLIT, Arrays.toString(resultSplits)); - - conf = createPropertiesQuery("sample_datasource", Query.TOPN, TOPN_QUERY); - resultSplits = (HiveDruidSplit[]) method1.invoke(input, conf); - assertEquals(TOPN_QUERY_SPLIT, Arrays.toString(resultSplits)); - - conf = createPropertiesQuery("sample_datasource", Query.GROUP_BY, GROUP_BY_QUERY); - resultSplits = (HiveDruidSplit[]) method1.invoke(input, conf); - assertEquals(GROUP_BY_QUERY_SPLIT, Arrays.toString(resultSplits)); - - } - - private static Configuration createPropertiesQuery(String dataSource, String queryType, String jsonQuery) { - Configuration conf = new Configuration(); - // Set the configuration parameters - conf.set(FileInputFormat.INPUT_DIR, "/my/dir"); - conf.set(HiveConf.ConfVars.HIVE_DRUID_BROKER_DEFAULT_ADDRESS.varname, "localhost:8082"); - conf.set(Constants.DRUID_DATA_SOURCE, dataSource); - conf.set(Constants.DRUID_QUERY_JSON, jsonQuery); - conf.set(Constants.DRUID_QUERY_TYPE, queryType); - return conf; - } - -} diff --git a/druid-handler/src/test/org/apache/hadoop/hive/druid/io/TestHiveDruidSplit.java b/druid-handler/src/test/org/apache/hadoop/hive/druid/io/TestHiveDruidSplit.java deleted file mode 100644 index be3ff6fee112..000000000000 --- a/druid-handler/src/test/org/apache/hadoop/hive/druid/io/TestHiveDruidSplit.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.druid.io; - -import org.apache.hadoop.fs.Path; -import org.junit.Assert; -import org.junit.Test; - -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.DataInputStream; -import java.io.DataOutput; -import java.io.DataOutputStream; -import java.io.IOException; - -/** - * Test Class for input splits. - */ -public class TestHiveDruidSplit { - @Test public void testSerDeser() throws IOException { - HiveDruidSplit - hiveDruidSplit = - new HiveDruidSplit("query string", new Path("test-path"), new String[] {"host:8080", "host2:8090"}); - ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); - DataOutput dataOutput = new DataOutputStream(byteArrayOutputStream); - hiveDruidSplit.write(dataOutput); - ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(byteArrayOutputStream.toByteArray()); - HiveDruidSplit actualHiveDruidSplit = new HiveDruidSplit(); - actualHiveDruidSplit.readFields(new DataInputStream(byteArrayInputStream)); - Assert.assertEquals(actualHiveDruidSplit.getDruidQuery(), "query string"); - Assert.assertArrayEquals(actualHiveDruidSplit.getLocations(), new String[] {"host:8080", "host2:8090"}); - } -} diff --git a/druid-handler/src/test/org/apache/hadoop/hive/druid/serde/TestDruidSerDe.java b/druid-handler/src/test/org/apache/hadoop/hive/druid/serde/TestDruidSerDe.java deleted file mode 100644 index 5157e4656c4d..000000000000 --- a/druid-handler/src/test/org/apache/hadoop/hive/druid/serde/TestDruidSerDe.java +++ /dev/null @@ -1,884 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.druid.serde; - -import static org.junit.Assert.assertEquals; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.lang.reflect.InvocationTargetException; -import java.time.Instant; -import java.time.ZoneOffset; -import java.util.ArrayList; -import java.util.List; -import java.util.Map.Entry; -import java.util.Properties; -import java.util.stream.Collectors; - -import org.apache.druid.java.util.http.client.HttpClient; -import org.apache.druid.java.util.http.client.response.HttpResponseHandler; -import org.apache.druid.query.scan.ScanResultValue; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hive.common.type.HiveChar; -import org.apache.hadoop.hive.common.type.HiveVarchar; -import org.apache.hadoop.hive.common.type.Timestamp; -import org.apache.hadoop.hive.common.type.TimestampTZ; -import org.apache.hadoop.hive.conf.Constants; -import org.apache.hadoop.hive.druid.DruidStorageHandlerUtils; -import org.apache.hadoop.hive.druid.QTestDruidSerDe; -import org.apache.hadoop.hive.druid.io.DruidQueryBasedInputFormat; -import org.apache.hadoop.hive.druid.io.HiveDruidSplit; -import org.apache.hadoop.hive.ql.exec.Utilities; -import org.apache.hadoop.hive.serde.serdeConstants; -import org.apache.hadoop.hive.serde2.SerDeException; -import org.apache.hadoop.hive.serde2.io.ByteWritable; -import org.apache.hadoop.hive.serde2.io.DoubleWritable; -import org.apache.hadoop.hive.serde2.io.HiveCharWritable; -import org.apache.hadoop.hive.serde2.io.HiveVarcharWritable; -import org.apache.hadoop.hive.serde2.io.ShortWritable; -import org.apache.hadoop.hive.serde2.io.TimestampLocalTZWritable; -import org.apache.hadoop.hive.serde2.io.TimestampWritableV2; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; -import org.apache.hadoop.hive.serde2.objectinspector.StructField; -import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; -import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; -import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; -import org.apache.hadoop.io.FloatWritable; -import org.apache.hadoop.io.IntWritable; -import org.apache.hadoop.io.LongWritable; -import org.apache.hadoop.io.NullWritable; -import org.apache.hadoop.io.Text; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; - -import com.fasterxml.jackson.core.type.TypeReference; -import com.google.common.collect.ImmutableMap; -import com.google.common.util.concurrent.SettableFuture; - -import org.apache.druid.data.input.Row; -import org.apache.druid.query.Query; -import org.apache.druid.query.Result; -import org.apache.druid.query.timeseries.TimeseriesResultValue; -import org.apache.druid.query.topn.TopNResultValue; -import org.junit.rules.ExpectedException; - -/** - * Basic tests for Druid SerDe. The examples are taken from Druid 0.9.1.1 - * documentation. - */ -@SuppressWarnings({"SameParameterValue", "SpellCheckingInspection"}) -public class TestDruidSerDe { - // Timeseries query - private static final String - TIMESERIES_QUERY = - "{ \"queryType\": \"timeseries\", " - + " \"dataSource\": \"sample_datasource\", " - + " \"granularity\": \"day\", " - + " \"descending\": \"true\", " - + " \"filter\": { " - + " \"type\": \"and\", " - + " \"fields\": [ " - + " { \"type\": \"selector\", \"dimension\": \"sample_dimension1\", \"value\": \"sample_value1\" }, " - + " { \"type\": \"or\", " - + " \"fields\": [ " - + " { \"type\": \"selector\", \"dimension\": \"sample_dimension2\", \"value\": \"sample_value2\" }, " - + " { \"type\": \"selector\", \"dimension\": \"sample_dimension3\", \"value\": \"sample_value3\" } " - + " ] " - + " } " - + " ] " - + " }, " - + " \"aggregations\": [ " - + " { \"type\": \"longSum\", \"name\": \"sample_name1\", \"fieldName\": \"sample_fieldName1\" }, " - + " { \"type\": \"doubleSum\", \"name\": \"sample_name2\", \"fieldName\": \"sample_fieldName2\" } " - + " ], " - + " \"postAggregations\": [ " - + " { \"type\": \"arithmetic\", " - + " \"name\": \"sample_divide\", " - + " \"fn\": \"/\", " - + " \"fields\": [ " - + " { \"type\": \"fieldAccess\", \"name\": \"postAgg__sample_name1\", \"fieldName\": \"sample_name1\" " - + "}, " - + " { \"type\": \"fieldAccess\", \"name\": \"postAgg__sample_name2\", \"fieldName\": \"sample_name2\" } " - + " ] " - + " } " - + " ], " - + " \"intervals\": [ \"2012-01-01T00:00:00.000/2012-01-03T00:00:00.000\" ]}"; - - // Timeseries query results - private static final String - TIMESERIES_QUERY_RESULTS = - "[ " - + "{ " - + " \"timestamp\": \"2012-01-01T00:00:00.000Z\", " - + " \"result\": { \"sample_name1\": 0, \"sample_name2\": 1.0, \"sample_divide\": 2.2222 } " - + "}, " - + "{ " - + " \"timestamp\": \"2012-01-02T00:00:00.000Z\", " - + " \"result\": { \"sample_name1\": 2, \"sample_name2\": 3.32, \"sample_divide\": 4 } " - + "}]"; - - private byte[] tsQueryResults; - private byte[] topNQueryResults; - private byte[] groupByQueryResults; - private byte[] groupByTimeExtractQueryResults; - private byte[] selectQueryResults; - private byte[] groupByMonthExtractQueryResults; - private byte[] scanQueryResults; - - // Timeseries query results as records - private static final Object[][] TIMESERIES_QUERY_RESULTS_RECORDS = new Object[][]{ - new Object[]{ - new TimestampTZ(Instant.ofEpochMilli(1325376000000L).atZone(ZoneOffset.UTC)), - 0L, - 1.0F, - 2.2222F}, - new Object[]{ - new TimestampTZ(Instant.ofEpochMilli(1325462400000L).atZone(ZoneOffset.UTC)), - 2L, - 3.32F, - 4F}}; - - // Timeseries query results as records (types defined by metastore) - private static final String TIMESERIES_COLUMN_NAMES = "timestamp,sample_name1,sample_name2,sample_divide"; - private static final String TIMESERIES_COLUMN_TYPES = "timestamp with local time zone,bigint,float,float"; - // TopN query - private static final String - TOPN_QUERY = - "{ \"queryType\": \"topN\", " - + " \"dataSource\": \"sample_data\", " - + " \"dimension\": \"sample_dim\", " - + " \"threshold\": 5, " - + " \"metric\": \"count\", " - + " \"granularity\": \"all\", " - + " \"filter\": { " - + " \"type\": \"and\", " - + " \"fields\": [ " - + " { " - + " \"type\": \"selector\", " - + " \"dimension\": \"dim1\", " - + " \"value\": \"some_value\" " - + " }, " - + " { " - + " \"type\": \"selector\", " - + " \"dimension\": \"dim2\", " - + " \"value\": \"some_other_val\" " - + " } " - + " ] " - + " }, " - + " \"aggregations\": [ " - + " { " - + " \"type\": \"longSum\", " - + " \"name\": \"count\", " - + " \"fieldName\": \"count\" " - + " }, " - + " { " - + " \"type\": \"doubleSum\", " - + " \"name\": \"some_metric\", " - + " \"fieldName\": \"some_metric\" " - + " } " - + " ], " - + " \"postAggregations\": [ " - + " { " - + " \"type\": \"arithmetic\", " - + " \"name\": \"sample_divide\", " - + " \"fn\": \"/\", " - + " \"fields\": [ " - + " { " - + " \"type\": \"fieldAccess\", " - + " \"name\": \"some_metric\", " - + " \"fieldName\": \"some_metric\" " - + " }, " - + " { " - + " \"type\": \"fieldAccess\", " - + " \"name\": \"count\", " - + " \"fieldName\": \"count\" " - + " } " - + " ] " - + " } " - + " ], " - + " \"intervals\": [ " - + " \"2013-08-31T00:00:00.000/2013-09-03T00:00:00.000\" " - + " ]}"; - - // TopN query results - private static final String - TOPN_QUERY_RESULTS = - "[ " - + " { " - + " \"timestamp\": \"2013-08-31T00:00:00.000Z\", " - + " \"result\": [ " - + " { " - + " \"sample_dim\": \"dim1_val\", " - + " \"count\": 111, " - + " \"some_metric\": 10669, " - + " \"sample_divide\": 96.11711711711712 " - + " }, " - + " { " - + " \"sample_dim\": \"another_dim1_val\", " - + " \"count\": 88, " - + " \"some_metric\": 28344, " - + " \"sample_divide\": 322.09090909090907 " - + " }, " - + " { " - + " \"sample_dim\": \"dim1_val3\", " - + " \"count\": 70, " - + " \"some_metric\": 871, " - + " \"sample_divide\": 12.442857142857143 " - + " }, " - + " { " - + " \"sample_dim\": \"dim1_val4\", " - + " \"count\": 62, " - + " \"some_metric\": 815, " - + " \"sample_divide\": 13.14516129032258 " - + " }, " - + " { " - + " \"sample_dim\": \"dim1_val5\", " - + " \"count\": 60, " - + " \"some_metric\": 2787, " - + " \"sample_divide\": 46.45 " - + " } " - + " ] " - + " }]"; - - // TopN query results as records - private static final Object[][] TOPN_QUERY_RESULTS_RECORDS = new Object[][]{ - new Object[]{ - new TimestampTZ(Instant.ofEpochMilli(1377907200000L).atZone(ZoneOffset.UTC)), - "dim1_val", 111L, 10669F, 96.11711711711712F}, - new Object[]{ - (new TimestampTZ(Instant.ofEpochMilli(1377907200000L).atZone(ZoneOffset.UTC))), - "another_dim1_val", 88L, 28344F, 322.09090909090907F}, - new Object[]{ - (new TimestampTZ(Instant.ofEpochMilli(1377907200000L).atZone(ZoneOffset.UTC))), - "dim1_val3", 70L, 871F, 12.442857142857143F}, - new Object[]{ - new TimestampTZ(Instant.ofEpochMilli(1377907200000L).atZone(ZoneOffset.UTC)), - "dim1_val4", 62L, 815F, 13.14516129032258F}, - new Object[]{ - (new TimestampTZ(Instant.ofEpochMilli(1377907200000L).atZone(ZoneOffset.UTC))), - "dim1_val5", 60L, 2787F, 46.45F}}; - - // TopN query results as records (types defined by metastore) - private static final String TOPN_COLUMN_NAMES = "timestamp,sample_dim,count,some_metric,sample_divide"; - private static final String TOPN_COLUMN_TYPES = "timestamp with local time zone,string,bigint,float,float"; - - // GroupBy query - private static final String - GROUP_BY_QUERY = - "{ " - + " \"queryType\": \"groupBy\", " - + " \"dataSource\": \"sample_datasource\", " - + " \"granularity\": \"day\", " - + " \"dimensions\": [\"country\", \"device\"], " - + " \"limitSpec\": {" - + " \"type\": \"default\"," - + " \"limit\": 5000," - + " \"columns\": [\"country\", \"data_transfer\"] }, " - + " \"filter\": { " - + " \"type\": \"and\", " - + " \"fields\": [ " - + " { \"type\": \"selector\", \"dimension\": \"carrier\", \"value\": \"AT&T\" }, " - + " { \"type\": \"or\", " - + " \"fields\": [ " - + " { \"type\": \"selector\", \"dimension\": \"make\", \"value\": \"Apple\" }, " - + " { \"type\": \"selector\", \"dimension\": \"make\", \"value\": \"Samsung\" } " - + " ] " - + " } " - + " ] " - + " }, " - + " \"aggregations\": [ " - + " { \"type\": \"longSum\", \"name\": \"total_usage\", \"fieldName\": \"user_count\" }, " - + " { \"type\": \"doubleSum\", \"name\": \"data_transfer\", \"fieldName\": \"data_transfer\" } " - + " ], " - + " \"postAggregations\": [ " - + " { \"type\": \"arithmetic\", " - + " \"name\": \"avg_usage\", " - + " \"fn\": \"/\", " - + " \"fields\": [ " - + " { \"type\": \"fieldAccess\", \"fieldName\": \"data_transfer\" }, " - + " { \"type\": \"fieldAccess\", \"fieldName\": \"total_usage\" } " - + " ] " - + " } " - + " ], " - + " \"intervals\": [ \"2012-01-01T00:00:00.000/2012-01-03T00:00:00.000\" ], " - + " \"having\": { " - + " \"type\": \"greaterThan\", " - + " \"aggregation\": \"total_usage\", " - + " \"value\": 100 " - + " }}"; - - // GroupBy query results - private static final String - GROUP_BY_QUERY_RESULTS = - "[ " - + " { " - + " \"version\" : \"v1\", " - + " \"timestamp\" : \"2012-01-01T00:00:00.000Z\", " - + " \"event\" : { " - + " \"country\" : \"India\", " - + " \"device\" : \"phone\", " - + " \"total_usage\" : 88, " - + " \"data_transfer\" : 29.91233453, " - + " \"avg_usage\" : 60.32 " - + " } " - + " }, " - + " { " - + " \"version\" : \"v1\", " - + " \"timestamp\" : \"2012-01-01T00:00:12.000Z\", " - + " \"event\" : { " - + " \"country\" : \"Spain\", " - + " \"device\" : \"pc\", " - + " \"total_usage\" : 16, " - + " \"data_transfer\" : 172.93494959, " - + " \"avg_usage\" : 6.333333 " - + " } " - + " }]"; - - private static final String - GB_TIME_EXTRACTIONS = - "{\"queryType\":\"groupBy\",\"dataSource\":\"sample_datasource\"," - + "\"granularity\":\"all\",\"dimensions\":" - + "[{\"type\":\"extraction\",\"dimension\":\"__time\",\"outputName\":\"extract\",\"extractionFn\":" - + "{\"type\":\"timeFormat\",\"format\":\"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'\",\"timeZone\":\"UTC\"}}]," - + "\"limitSpec\":{\"type\":\"default\"}," - + "\"aggregations\":[{\"type\":\"count\",\"name\":\"$f1\"}]," - + "\"intervals\":[\"1900-01-01T00:00:00.000/3000-01-01T00:00:00.000\"]}"; - - private static final String - GB_TIME_EXTRACTIONS_RESULTS = - "[ " - + " { " - + " \"version\" : \"v1\", " - + " \"timestamp\" : \"2012-01-01T00:00:00.000Z\", " - + " \"event\" : { " - + " \"extract\" : \"2012-01-01T00:00:00.000Z\", " - + " \"$f1\" : 200" - + " } " - + " }, " - + " { " - + " \"version\" : \"v1\", " - + " \"timestamp\" : \"2012-01-01T00:00:12.000Z\", " - + " \"event\" : { " - + " \"extract\" : \"2012-01-01T00:00:12.000Z\", " - + " \"$f1\" : 400" - + " } " - + " }]"; - - private static final String - GB_MONTH_EXTRACTIONS_RESULTS = - "[ " - + " { " - + " \"version\" : \"v1\", " - + " \"timestamp\" : \"2012-01-01T00:00:00.000Z\", " - + " \"event\" : { " - + " \"extract_month\" : \"01\", " - + " \"$f1\" : 200" - + " } " - + " }, " - + " { " - + " \"version\" : \"v1\", " - + " \"timestamp\" : \"2012-01-01T00:00:12.000Z\", " - + " \"event\" : { " - + " \"extract_month\" : \"01\", " - + " \"$f1\" : 400" - + " } " - + " }]"; - - private static final String - GB_MONTH_EXTRACTIONS = - "{\"queryType\":\"groupBy\",\"dataSource\":\"sample_datasource\"," - + "\"granularity\":\"all\"," - + "\"dimensions\":[{\"type\":\"extraction\",\"dimension\":\"__time\",\"outputName\":\"extract_month\"," - + "\"extractionFn\":{\"type\":\"timeFormat\",\"format\":\"M\",\"timeZone\":\"UTC\",\"locale\":\"en-US\"}}]," - + "\"limitSpec\":{\"type\":\"default\"},\"aggregations\":[{\"type\":\"count\",\"name\":\"$f1\"}]," - + "\"intervals\":[\"1900-01-01T00:00:00.000/3000-01-01T00:00:00.000\"]}"; - - // GroupBy query results as records - private static final Object[][] GROUP_BY_QUERY_EXTRACTION_RESULTS_RECORDS = new Object[][]{ - new Object[]{ - (new TimestampTZ(Instant.ofEpochMilli(1325376000000L).atZone(ZoneOffset.UTC))), - (new TimestampTZ(Instant.ofEpochMilli(1325376000000L).atZone(ZoneOffset.UTC))), 200L}, - new Object[]{ - (new TimestampTZ(Instant.ofEpochMilli(1325376012000L).atZone(ZoneOffset.UTC))), - (new TimestampTZ(Instant.ofEpochMilli(1325376012000L).atZone(ZoneOffset.UTC))), 400L}}; - - private static final Object[][] GROUP_BY_QUERY_RESULTS_RECORDS = new Object[][]{ - new Object[]{ - (new TimestampTZ(Instant.ofEpochMilli(1325376000000L).atZone(ZoneOffset.UTC))), "India", - "phone", 88L, 29.91233453, 60.32F}, - new Object[]{ - (new TimestampTZ(Instant.ofEpochMilli(1325376012000L).atZone(ZoneOffset.UTC))), - "Spain", "pc", 16L, 172.93494959, 6.333333F}}; - - private static final Object[][] GB_MONTH_EXTRACTION_RESULTS_RECORDS = new Object[][]{ - new Object[]{ - (new TimestampTZ(Instant.ofEpochMilli(1325376000000L).atZone(ZoneOffset.UTC))), 1, 200L}, - new Object[]{ - (new TimestampTZ(Instant.ofEpochMilli(1325376012000L).atZone(ZoneOffset.UTC))), 1, 400L}}; - - // GroupBy query results as records (types defined by metastore) - private static final String GROUP_BY_COLUMN_NAMES = "timestamp,country,device,total_usage,data_transfer,avg_usage"; - private static final String - GROUP_BY_COLUMN_TYPES = - "timestamp with local time zone,string,string,bigint,double,float"; - - private static final String GB_TIME_EXTRACTIONS_COLUMN_NAMES = "timestamp,extract,$f1"; - private static final String - GB_TIME_EXTRACTIONS_COLUMN_TYPES = - "timestamp with local time zone,timestamp with local time zone,bigint"; - - private static final String GB_MONTH_EXTRACTIONS_COLUMN_NAMES = "timestamp,extract_month,$f1"; - private static final String GB_MONTH_EXTRACTIONS_COLUMN_TYPES = "timestamp with local time zone,int,bigint"; - - private static final String SCAN_COLUMN_NAMES = - "__time,robot,namespace,anonymous,unpatrolled,page,language,newpage,user,count,added,delta,variation,deleted"; - private static final String SCAN_COLUMN_TYPES = - "timestamp with local time zone,boolean,string,string,string,string,string,string,string,double,double,float," - + "float,float"; - private static final Object[][] SCAN_QUERY_RESULTS_RECORDS = new Object[][]{ - new Object[]{(new TimestampTZ(Instant.ofEpochMilli(1356998400000L).atZone(ZoneOffset.UTC))), Boolean.TRUE, - "article", "0", "0", "11._korpus_(NOVJ)", "sl", "0", "EmausBot", 1.0d, 39.0d, 39.0F, 39.0F, 0.0F}, - new Object[]{ - (new TimestampTZ(Instant.ofEpochMilli(1356998400000L).atZone(ZoneOffset.UTC))), Boolean.FALSE, - "article", - "0", - "0", - "112_U.S._580", - "en", - "1", - "MZMcBride", 1.0d, 70.0d, 70.0F, 70.0F, 0.0F}, - new Object[]{ - (new TimestampTZ(Instant.ofEpochMilli(1356998412000L).atZone(ZoneOffset.UTC))), Boolean.FALSE, - "article", - "0", - "0", - "113_U.S._243", - "en", - "1", - "MZMcBride", 1.0d, 77.0d, 77.0F, 77.0F, 0.0F}, - new Object[]{ - (new TimestampTZ(Instant.ofEpochMilli(1356998412000L).atZone(ZoneOffset.UTC))), Boolean.FALSE, - "article", - "0", - "0", - "113_U.S._73", - "en", - "1", - "MZMcBride", 1.0d, 70.0d, 70.0F, 70.0F, 0.0F}, - new Object[]{ - (new TimestampTZ(Instant.ofEpochMilli(1356998412000L).atZone(ZoneOffset.UTC))), Boolean.FALSE, - "article", - "0", - "0", - "113_U.S._756", - "en", - "1", - "MZMcBride", 1.0d, 68.0d, 68.0F, 68.0F, 0.0F}}; - - // Scan query - private static final String - SCAN_QUERY = - "{ \"queryType\": \"scan\", " - + " \"dataSource\": \"wikipedia\", \"descending\": \"false\", " - + " \"columns\":[\"robot\",\"namespace\",\"anonymous\",\"unpatrolled\",\"page\",\"language\"," - + "\"newpage\",\"user\",\"count\",\"added\",\"delta\",\"variation\",\"deleted\"], " - + " \"granularity\": \"all\", " - + " \"intervals\": [ \"2013-01-01/2013-01-02\" ]," - + " \"resultFormat\": \"compactedList\"," - + " \"limit\": 5" - + "}"; - - private static final String - SCAN_QUERY_RESULTS = - "[{" - + "\"segmentId\":\"wikipedia_2012-12-29T00:00:00.000Z_2013-01-10T08:00:00.000Z_2013-01-10T08:13:47.830Z_v9\"," - + "\"columns\":[\"__time\",\"robot\",\"namespace\",\"anonymous\",\"unpatrolled\",\"page\",\"language\"," - + "\"newpage\",\"user\",\"count\",\"added\",\"delta\",\"variation\",\"deleted\"]," - + "\"events\":[" - + "[\"2013-01-01T00:00:00.000Z\", 1,\"article\",\"0\",\"0\",\"11._korpus_(NOVJ)\",\"sl\",\"0\"," - + "\"EmausBot\",1.0,39.0,39.0,39.0,0.0]," - + "[\"2013-01-01T00:00:00.000Z\", 0,\"article\",\"0\",\"0\",\"112_U.S._580\",\"en\",\"1\",\"MZMcBride\",1" - + ".0,70.0,70.0,70.0,0.0]," - + "[\"2013-01-01T00:00:12.000Z\", 0,\"article\",\"0\",\"0\",\"113_U.S._243\",\"en\",\"1\",\"MZMcBride\",1" - + ".0,77.0,77.0,77.0,0.0]," - + "[\"2013-01-01T00:00:12.000Z\", 0,\"article\",\"0\",\"0\",\"113_U.S._73\",\"en\",\"1\",\"MZMcBride\",1.0," - + "70.0,70.0,70.0,0.0]," - + "[\"2013-01-01T00:00:12.000Z\", 0,\"article\",\"0\",\"0\",\"113_U.S._756\",\"en\",\"1\",\"MZMcBride\",1" - + ".0,68.0,68.0,68.0,0.0]" - + "]}]"; - - @Before - public void setup() throws IOException { - tsQueryResults = - DruidStorageHandlerUtils.SMILE_MAPPER.writeValueAsBytes(DruidStorageHandlerUtils.JSON_MAPPER.readValue( - TIMESERIES_QUERY_RESULTS, - new TypeReference>>() { - })); - - topNQueryResults = - DruidStorageHandlerUtils.SMILE_MAPPER.writeValueAsBytes(DruidStorageHandlerUtils.JSON_MAPPER.readValue( - TOPN_QUERY_RESULTS, - new TypeReference>>() { - })); - groupByQueryResults = - DruidStorageHandlerUtils.SMILE_MAPPER.writeValueAsBytes(DruidStorageHandlerUtils.JSON_MAPPER.readValue( - GROUP_BY_QUERY_RESULTS, - new TypeReference>() { - })); - groupByTimeExtractQueryResults = - DruidStorageHandlerUtils.SMILE_MAPPER.writeValueAsBytes(DruidStorageHandlerUtils.JSON_MAPPER.readValue( - GB_TIME_EXTRACTIONS_RESULTS, - new TypeReference>() { - })); - groupByMonthExtractQueryResults = - DruidStorageHandlerUtils.SMILE_MAPPER.writeValueAsBytes(DruidStorageHandlerUtils.JSON_MAPPER.readValue( - GB_MONTH_EXTRACTIONS_RESULTS, - new TypeReference>() { - })); - scanQueryResults = - DruidStorageHandlerUtils.SMILE_MAPPER.writeValueAsBytes(DruidStorageHandlerUtils.JSON_MAPPER.readValue( - SCAN_QUERY_RESULTS, - new TypeReference>() { - })); - } - - @Test - public void testDruidDeserializer() - throws SerDeException, NoSuchFieldException, SecurityException, IllegalArgumentException, IllegalAccessException, - IOException, InterruptedException, NoSuchMethodException, InvocationTargetException { - // Create, initialize, and test the SerDe - QTestDruidSerDe serDe = new QTestDruidSerDe(); - Configuration conf = new Configuration(); - Properties tbl; - // Timeseries query - tbl = - createPropertiesQuery("sample_datasource", - Query.TIMESERIES, - TIMESERIES_QUERY, - TIMESERIES_COLUMN_NAMES, - TIMESERIES_COLUMN_TYPES); - serDe.initialize(conf, tbl, null); - deserializeQueryResults(serDe, - Query.TIMESERIES, - TIMESERIES_QUERY, - tsQueryResults, - TIMESERIES_QUERY_RESULTS_RECORDS); - - // TopN query - tbl = createPropertiesQuery("sample_data", Query.TOPN, TOPN_QUERY, TOPN_COLUMN_NAMES, TOPN_COLUMN_TYPES); - serDe.initialize(conf, tbl, null); - deserializeQueryResults(serDe, Query.TOPN, TOPN_QUERY, topNQueryResults, TOPN_QUERY_RESULTS_RECORDS); - - // GroupBy query - tbl = - createPropertiesQuery("sample_datasource", - Query.GROUP_BY, - GROUP_BY_QUERY, - GROUP_BY_COLUMN_NAMES, - GROUP_BY_COLUMN_TYPES); - serDe.initialize(conf, tbl, null); - deserializeQueryResults(serDe, Query.GROUP_BY, GROUP_BY_QUERY, groupByQueryResults, GROUP_BY_QUERY_RESULTS_RECORDS); - - tbl = - createPropertiesQuery("sample_datasource", - Query.GROUP_BY, - GB_TIME_EXTRACTIONS, - GB_TIME_EXTRACTIONS_COLUMN_NAMES, - GB_TIME_EXTRACTIONS_COLUMN_TYPES); - serDe.initialize(conf, tbl, null); - deserializeQueryResults(serDe, - Query.GROUP_BY, - GB_TIME_EXTRACTIONS, - groupByTimeExtractQueryResults, - GROUP_BY_QUERY_EXTRACTION_RESULTS_RECORDS); - - tbl = - createPropertiesQuery("sample_datasource", - Query.GROUP_BY, - GB_MONTH_EXTRACTIONS, - GB_MONTH_EXTRACTIONS_COLUMN_NAMES, - GB_MONTH_EXTRACTIONS_COLUMN_TYPES); - serDe.initialize(conf, tbl, null); - deserializeQueryResults(serDe, - Query.GROUP_BY, - GB_MONTH_EXTRACTIONS, - groupByMonthExtractQueryResults, - GB_MONTH_EXTRACTION_RESULTS_RECORDS); - - // Scan query -- results should be same as select query - tbl = createPropertiesQuery("wikipedia", Query.SCAN, SCAN_QUERY, SCAN_COLUMN_NAMES, SCAN_COLUMN_TYPES); - serDe.initialize(conf, tbl, null); - deserializeQueryResults(serDe, Query.SCAN, SCAN_QUERY, scanQueryResults, SCAN_QUERY_RESULTS_RECORDS); - } - - private static Properties createPropertiesQuery(String dataSource, - String queryType, - String jsonQuery, - String columnNames, - String columnTypes) { - Properties tbl = new Properties(); - - // Set the configuration parameters - tbl.setProperty(Constants.DRUID_DATA_SOURCE, dataSource); - tbl.setProperty(Constants.DRUID_QUERY_JSON, jsonQuery); - tbl.setProperty(Constants.DRUID_QUERY_TYPE, queryType); - tbl.setProperty(Constants.DRUID_QUERY_FIELD_NAMES, columnNames); - tbl.setProperty(Constants.DRUID_QUERY_FIELD_TYPES, columnTypes); - return tbl; - } - - @SuppressWarnings("unchecked") - private void deserializeQueryResults(DruidSerDe serDe, - String queryType, - String jsonQuery, - byte[] resultString, - Object[][] records) - throws SerDeException, IOException, NoSuchFieldException, SecurityException, IllegalArgumentException, - IllegalAccessException, InterruptedException, NoSuchMethodException, InvocationTargetException { - - // Initialize - HttpClient httpClient = mock(HttpClient.class); - SettableFuture futureResult = SettableFuture.create(); - futureResult.set(new ByteArrayInputStream(resultString)); - when(httpClient.go(any(), any(HttpResponseHandler.class))).thenReturn(futureResult); - DruidQueryRecordReader reader = DruidQueryBasedInputFormat.getDruidQueryReader(queryType); - - final HiveDruidSplit split = new HiveDruidSplit(jsonQuery, new Path("empty"), new String[]{"testing_host"}); - Configuration conf = new Configuration(); - reader.initialize(split, DruidStorageHandlerUtils.JSON_MAPPER, DruidStorageHandlerUtils.SMILE_MAPPER, httpClient, - conf); - StructObjectInspector oi = (StructObjectInspector) serDe.getObjectInspector(); - List fieldRefs = oi.getAllStructFieldRefs(); - - // Check mapred - DruidWritable writable = reader.createValue(); - int pos = 0; - while (reader.next(NullWritable.get(), writable)) { - List row = (List) serDe.deserialize(writable); - Object[] expectedFieldsData = records[pos]; - assertEquals(expectedFieldsData.length, fieldRefs.size()); - for (int i = 0; i < fieldRefs.size(); i++) { - assertEquals("Field " + i + " type", expectedFieldsData[i].getClass(), row.get(i).getClass()); - Object fieldData = oi.getStructFieldData(row, fieldRefs.get(i)); - assertEquals("Field " + i, expectedFieldsData[i], fieldData); - } - pos++; - } - assertEquals(pos, records.length); - - // Check mapreduce path - futureResult = SettableFuture.create(); - futureResult.set(new ByteArrayInputStream(resultString)); - when(httpClient.go(any(), any(HttpResponseHandler.class))).thenReturn(futureResult); - reader = DruidQueryBasedInputFormat.getDruidQueryReader(queryType); - reader.initialize(split, DruidStorageHandlerUtils.JSON_MAPPER, DruidStorageHandlerUtils.SMILE_MAPPER, httpClient, - conf); - - pos = 0; - while (reader.nextKeyValue()) { - List row = (List) serDe.deserialize(reader.getCurrentValue()); - Object[] expectedFieldsData = records[pos]; - assertEquals(expectedFieldsData.length, fieldRefs.size()); - for (int i = 0; i < fieldRefs.size(); i++) { - assertEquals("Field " + i + " type", expectedFieldsData[i].getClass(), row.get(i).getClass()); - Object fieldData = oi.getStructFieldData(row, fieldRefs.get(i)); - assertEquals("Field " + i, expectedFieldsData[i], fieldData); - } - pos++; - } - assertEquals(pos, records.length); - } - - private static final String COLUMN_NAMES = "__time,c0,c1,c2,c3,c4,c5,c6,c7,c8"; - private static final String - COLUMN_TYPES = - "timestamp with local time zone,string,char(6),varchar(8),double,float,bigint,int,smallint,tinyint"; - private static final Object[] ROW_OBJECT = new Object[]{ - new TimestampLocalTZWritable(new TimestampTZ(Instant.ofEpochMilli(1377907200000L).atZone(ZoneOffset.UTC))), - new Text("dim1_val"), - new HiveCharWritable(new HiveChar("dim2_v", 6)), - new HiveVarcharWritable(new HiveVarchar("dim3_val", 8)), - new DoubleWritable(10669.3D), - new FloatWritable(10669.45F), - new LongWritable(1113939), - new IntWritable(1112123), - new ShortWritable((short) 12), - new ByteWritable((byte) 0), - new TimestampWritableV2(Timestamp.ofEpochSecond(1377907200L)) - // granularity - }; - private static final DruidWritable - DRUID_WRITABLE = - new DruidWritable(ImmutableMap.builder().put("__time", 1377907200000L) - .put("c0", "dim1_val") - .put("c1", "dim2_v") - .put("c2", "dim3_val") - .put("c3", 10669.3D) - .put("c4", 10669.45F) - .put("c5", 1113939L) - .put("c6", 1112123) - .put("c7", (short) 12) - .put("c8", (byte) 0) - .put("__time_granularity", 1377907200000L) - .build()); - - @Test - public void testDruidObjectSerializer() - throws SerDeException, NoSuchFieldException, SecurityException, IllegalArgumentException, IllegalAccessException, - IOException, InterruptedException, NoSuchMethodException, InvocationTargetException { - // Create, initialize, and test the SerDe - DruidSerDe serDe = new DruidSerDe(); - Configuration conf = new Configuration(); - Properties tbl; - // Mixed source (all types) - tbl = createPropertiesSource(COLUMN_NAMES, COLUMN_TYPES); - serDe.initialize(conf, tbl, null); - serializeObject(tbl, serDe, ROW_OBJECT, DRUID_WRITABLE); - } - - @Rule - public ExpectedException expectedEx = ExpectedException.none(); - - @Test - public void testDruidObjectSerializerwithNullTimestamp() throws Exception { - // Create, initialize, and test the SerDe - DruidSerDe serDe = new DruidSerDe(); - Configuration conf = new Configuration(); - Properties tbl; - // Mixed source (all types) - tbl = createPropertiesSource(COLUMN_NAMES, COLUMN_TYPES); - serDe.initialize(conf, tbl, null); - Object[] row = new Object[]{ - null, - new Text("dim1_val"), - new HiveCharWritable(new HiveChar("dim2_v", 6)), - new HiveVarcharWritable(new HiveVarchar("dim3_val", 8)), - new DoubleWritable(10669.3D), - new FloatWritable(10669.45F), - new LongWritable(1113939), - new IntWritable(1112123), - new ShortWritable((short) 12), - new ByteWritable((byte) 0), - null - // granularity - }; - expectedEx.expect(NullPointerException.class); - expectedEx.expectMessage("Timestamp column cannot have null value"); - // should fail as timestamp is null - serializeObject(tbl, serDe, row, DRUID_WRITABLE); - } - - private static Properties createPropertiesSource(String columnNames, String columnTypes) { - Properties tbl = new Properties(); - - // Set the configuration parameters - tbl.setProperty(serdeConstants.LIST_COLUMNS, columnNames); - tbl.setProperty(serdeConstants.LIST_COLUMN_TYPES, columnTypes); - return tbl; - } - - private static void serializeObject(Properties properties, - DruidSerDe serDe, - Object[] rowObject, - DruidWritable druidWritable) throws SerDeException { - // Build OI with timestamp granularity column - final List columnNames = new ArrayList<>(Utilities.getColumnNames(properties)); - columnNames.add(Constants.DRUID_TIMESTAMP_GRANULARITY_COL_NAME); - final List - columnTypes = - Utilities.getColumnTypes(properties) - .stream() - .map(TypeInfoFactory::getPrimitiveTypeInfo) - .collect(Collectors.toList()); - columnTypes.add(TypeInfoFactory.getPrimitiveTypeInfo("timestamp")); - List - inspectors = - columnTypes.stream() - .map(PrimitiveObjectInspectorFactory::getPrimitiveWritableObjectInspector) - .collect(Collectors.toList()); - ObjectInspector inspector = ObjectInspectorFactory.getStandardStructObjectInspector(columnNames, inspectors); - // Serialize - DruidWritable writable = (DruidWritable) serDe.serialize(rowObject, inspector); - // Check result - assertEquals(druidWritable.getValue().size(), writable.getValue().size()); - for (Entry e : druidWritable.getValue().entrySet()) { - assertEquals(e.getValue(), writable.getValue().get(e.getKey())); - } - } - - private static final Object[] ROW_OBJECT_2 = new Object[]{ - new TimestampTZ(Instant.ofEpochMilli(1377907200000L).atZone(ZoneOffset.UTC)), "dim1_val", - new HiveChar("dim2_v", 6), - new HiveVarchar("dim3_val", 8), - 10669.3D, - 10669.45F, - 1113939L, - 1112123, - ((short) 12), - ((byte) 0)}; - private static final DruidWritable - DRUID_WRITABLE_2 = - new DruidWritable(ImmutableMap.builder().put("__time", 1377907200000L) - .put("c0", "dim1_val") - .put("c1", "dim2_v") - .put("c2", "dim3_val") - .put("c3", 10669.3D) - .put("c4", 10669.45F) - .put("c5", 1113939L) - .put("c6", 1112123) - .put("c7", (short) 12) - .put("c8", (byte) 0) - .build()); - - @Test - public void testDruidObjectDeserializer() - throws SerDeException, NoSuchFieldException, SecurityException, IllegalArgumentException, IllegalAccessException, - IOException, InterruptedException, NoSuchMethodException, InvocationTargetException { - // Create, initialize, and test the SerDe - DruidSerDe serDe = new DruidSerDe(); - Configuration conf = new Configuration(); - Properties tbl; - // Mixed source (all types) - tbl = createPropertiesSource(COLUMN_NAMES, COLUMN_TYPES); - serDe.initialize(conf, tbl, null); - deserializeObject(serDe, ROW_OBJECT_2, DRUID_WRITABLE_2); - } - - @SuppressWarnings("unchecked") - private static void deserializeObject(DruidSerDe serDe, - Object[] rowObject, - DruidWritable druidWritable) throws SerDeException { - // Deserialize - List object = (List) serDe.deserialize(druidWritable); - // Check result - assertEquals(rowObject.length, object.size()); - for (int i = 0; i < rowObject.length; i++) { - assertEquals(rowObject[i].getClass(), object.get(i).getClass()); - assertEquals(rowObject[i], object.get(i)); - } - } - -} diff --git a/druid-handler/src/test/org/apache/hadoop/hive/ql/io/TestDruidRecordWriter.java b/druid-handler/src/test/org/apache/hadoop/hive/ql/io/TestDruidRecordWriter.java deleted file mode 100644 index 3fb7bdf6ca19..000000000000 --- a/druid-handler/src/test/org/apache/hadoop/hive/ql/io/TestDruidRecordWriter.java +++ /dev/null @@ -1,245 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.io; - -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Lists; -import org.apache.druid.data.input.Firehose; -import org.apache.druid.data.input.InputRow; -import org.apache.druid.data.input.impl.DimensionsSpec; -import org.apache.druid.data.input.impl.InputRowParser; -import org.apache.druid.data.input.impl.MapInputRowParser; -import org.apache.druid.data.input.impl.StringDimensionSchema; -import org.apache.druid.data.input.impl.TimeAndDimsParseSpec; -import org.apache.druid.data.input.impl.TimestampSpec; -import org.apache.druid.java.util.common.granularity.Granularities; -import org.apache.druid.query.aggregation.AggregatorFactory; -import org.apache.druid.query.aggregation.LongSumAggregatorFactory; -import org.apache.druid.query.aggregation.hyperloglog.HyperUniquesAggregatorFactory; -import org.apache.druid.segment.IndexSpec; -import org.apache.druid.segment.QueryableIndex; -import org.apache.druid.segment.QueryableIndexStorageAdapter; -import org.apache.druid.segment.data.RoaringBitmapSerdeFactory; -import org.apache.druid.segment.indexing.DataSchema; -import org.apache.druid.segment.indexing.RealtimeTuningConfig; -import org.apache.druid.segment.indexing.granularity.UniformGranularitySpec; -import org.apache.druid.segment.loading.DataSegmentPusher; -import org.apache.druid.segment.loading.LocalDataSegmentPuller; -import org.apache.druid.segment.loading.LocalDataSegmentPusher; -import org.apache.druid.segment.loading.LocalDataSegmentPusherConfig; -import org.apache.druid.segment.loading.SegmentLoadingException; -import org.apache.druid.segment.realtime.firehose.IngestSegmentFirehose; -import org.apache.druid.segment.realtime.firehose.WindowedStorageAdapter; -import org.apache.druid.timeline.DataSegment; -import org.apache.calcite.adapter.druid.DruidTable; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.LocalFileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hive.conf.Constants; -import org.apache.hadoop.hive.druid.DruidStorageHandler; -import org.apache.hadoop.hive.druid.DruidStorageHandlerUtils; -import org.apache.hadoop.hive.druid.conf.DruidConstants; -import org.apache.hadoop.hive.druid.io.DruidRecordWriter; -import org.apache.hadoop.hive.druid.serde.DruidWritable; -import org.joda.time.DateTime; -import org.joda.time.Interval; -import org.junit.Assert; -import org.junit.Ignore; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; - -import java.io.File; -import java.io.IOException; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -/** - * Test Class for Druid Record Writer. - */ -@SuppressWarnings("ConstantConditions") public class TestDruidRecordWriter { - private final ObjectMapper objectMapper = DruidStorageHandlerUtils.JSON_MAPPER; - - private static final Interval INTERVAL_FULL = new Interval("2014-10-22T00:00:00Z/P1D"); - - @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); - - final List> - expectedRows = - ImmutableList.of(ImmutableMap.of(DruidConstants.DEFAULT_TIMESTAMP_COLUMN, - DateTime.parse("2014-10-22T00:00:00.000Z").getMillis(), - "host", - ImmutableList.of("a.example.com"), - "visited_sum", - 190L, - "unique_hosts", - 1.0d), - ImmutableMap.of(DruidConstants.DEFAULT_TIMESTAMP_COLUMN, - DateTime.parse("2014-10-22T01:00:00.000Z").getMillis(), - "host", - ImmutableList.of("b.example.com"), - "visited_sum", - 175L, - "unique_hosts", - 1.0d), - ImmutableMap.of(DruidConstants.DEFAULT_TIMESTAMP_COLUMN, - DateTime.parse("2014-10-22T02:00:00.000Z").getMillis(), - "host", - ImmutableList.of("c.example.com"), - "visited_sum", - 270L, - "unique_hosts", - 1.0d)); - - @Test public void testTimeStampColumnName() { - Assert.assertEquals("Time column name need to match to ensure serdeser compatibility", - DruidConstants.DEFAULT_TIMESTAMP_COLUMN, - DruidTable.DEFAULT_TIMESTAMP_COLUMN); - } - - //Test is failing due to Guava dependency, Druid 0.13.0 should have less dependency on Guava - @Ignore @Test public void testWrite() throws IOException, SegmentLoadingException { - - final String dataSourceName = "testDataSource"; - final File segmentOutputDir = temporaryFolder.newFolder(); - final File workingDir = temporaryFolder.newFolder(); - Configuration config = new Configuration(); - - final InputRowParser - inputRowParser = - new MapInputRowParser(new TimeAndDimsParseSpec(new TimestampSpec(DruidConstants.DEFAULT_TIMESTAMP_COLUMN, - "auto", - null), new DimensionsSpec(ImmutableList.of(new StringDimensionSchema("host")), null, null))); - final Map - parserMap = - objectMapper.convertValue(inputRowParser, new TypeReference>() { - }); - - DataSchema - dataSchema = - new DataSchema(dataSourceName, - parserMap, - new AggregatorFactory[] {new LongSumAggregatorFactory("visited_sum", "visited_sum"), - new HyperUniquesAggregatorFactory("unique_hosts", "unique_hosts") }, - new UniformGranularitySpec(Granularities.DAY, Granularities.NONE, ImmutableList.of(INTERVAL_FULL)), - null, - objectMapper); - - IndexSpec indexSpec = new IndexSpec(new RoaringBitmapSerdeFactory(true), null, null, null); - RealtimeTuningConfig - tuningConfig = - new RealtimeTuningConfig(null, - null, null, null, temporaryFolder.newFolder(), null, null, null, null, indexSpec, null, null, 0, 0, null, - null, 0L, null, null); - LocalFileSystem localFileSystem = FileSystem.getLocal(config); - DataSegmentPusher dataSegmentPusher = new LocalDataSegmentPusher(new LocalDataSegmentPusherConfig() { - @Override public File getStorageDirectory() { - return segmentOutputDir; - } - }); - - Path - segmentDescriptorPath = - new Path(workingDir.getAbsolutePath(), DruidStorageHandler.SEGMENTS_DESCRIPTOR_DIR_NAME); - DruidRecordWriter - druidRecordWriter = - new DruidRecordWriter(dataSchema, tuningConfig, dataSegmentPusher, 20, segmentDescriptorPath, localFileSystem); - - List - druidWritables = - expectedRows.stream() - .map(input -> new DruidWritable(ImmutableMap.builder().putAll(input) - .put(Constants.DRUID_TIMESTAMP_GRANULARITY_COL_NAME, - Granularities - .DAY - .bucketStart(new DateTime((long) input.get(DruidConstants.DEFAULT_TIMESTAMP_COLUMN))) - .getMillis()) - .build())) - .collect(Collectors.toList()); - for (DruidWritable druidWritable : druidWritables) { - druidRecordWriter.write(druidWritable); - } - druidRecordWriter.close(false); - List dataSegmentList = DruidStorageHandlerUtils.getCreatedSegments(segmentDescriptorPath, config); - Assert.assertEquals(1, dataSegmentList.size()); - File tmpUnzippedSegmentDir = temporaryFolder.newFolder(); - new LocalDataSegmentPuller().getSegmentFiles(dataSegmentList.get(0), tmpUnzippedSegmentDir); - final QueryableIndex queryableIndex = DruidStorageHandlerUtils.INDEX_IO.loadIndex(tmpUnzippedSegmentDir); - - QueryableIndexStorageAdapter adapter = new QueryableIndexStorageAdapter(queryableIndex); - - Firehose - firehose = - new IngestSegmentFirehose(ImmutableList.of(new WindowedStorageAdapter(adapter, adapter.getInterval())), - null, - ImmutableList.of("host"), - ImmutableList.of("visited_sum", "unique_hosts"), - null); - - List rows = Lists.newArrayList(); - while (firehose.hasMore()) { - rows.add(firehose.nextRow()); - } - - verifyRows(expectedRows, rows); - - } - - private void verifyRows(List> expectedRows, List actualRows) { - System.out.println("actualRows = " + actualRows); - Assert.assertEquals(expectedRows.size(), actualRows.size()); - - for (int i = 0; i < expectedRows.size(); i++) { - Map expected = expectedRows.get(i); - InputRow actual = actualRows.get(i); - - Assert.assertEquals(ImmutableList.of("host"), actual.getDimensions()); - - Assert.assertEquals(expected.get(DruidConstants.DEFAULT_TIMESTAMP_COLUMN), - actual.getTimestamp().getMillis()); - Assert.assertEquals(expected.get("host"), actual.getDimension("host")); - Assert.assertEquals(expected.get("visited_sum"), actual.getMetric("visited_sum")); - Assert.assertEquals((Double) expected.get("unique_hosts"), - (Double) HyperUniquesAggregatorFactory.estimateCardinality(actual.getRaw("unique_hosts"), false), - 0.001); - } - } - - @Test public void testSerDesr() throws IOException { - String - segment = - "{\"dataSource\":\"datasource2015\",\"interval\":\"2015-06-01T00:00:00.000-04:00/" - + "2015-06-02T00:00:00.000-04:00\"" - + ",\"version\":\"2016-11-04T19:24:01.732-04:00\",\"loadSpec\":{\"type\":\"hdfs\"," - + "\"path\":\"hdfs://cn105-10.l42scl.hortonworks.com:8020/apps/hive/warehouse/druid.db/" - + ".hive-staging_hive_2016-11-04_19-23-50_168_1550339856804207572-1/_task_tmp.-ext-10002/_tmp.000000_0/" - + "datasource2015/20150601T000000.000-0400_20150602T000000.000-0400/2016-11-04T19_24_01.732-04_00/0/" - + "index.zip\"},\"dimensions\":\"dimension1\",\"metrics\":\"bigint\",\"shardSpec\":{\"type\":\"linear\"," - + "\"partitionNum\":0},\"binaryVersion\":9,\"size\":1765,\"identifier\":\"datasource2015_2015-06-01" - + "T00:00:00.000-04:00_2015-06-02T00:00:00.000-04:00_2016-11-04T19:24:01.732-04:00\"}"; - DataSegment dataSegment = objectMapper.readerFor(DataSegment.class).readValue(segment); - Assert.assertEquals("datasource2015", dataSegment.getDataSource()); - } - -} diff --git a/itests/hive-minikdc/pom.xml b/itests/hive-minikdc/pom.xml index 6fa425024258..6befc2c64603 100644 --- a/itests/hive-minikdc/pom.xml +++ b/itests/hive-minikdc/pom.xml @@ -76,24 +76,12 @@ org.apache.hive hive-it-unit test - - - org.apache.hive - hive-it-druid - - org.apache.hive hive-it-unit test tests - - - org.apache.hive - hive-it-druid - - org.apache.hive diff --git a/itests/hive-unit-hadoop2/pom.xml b/itests/hive-unit-hadoop2/pom.xml index 03ac61c004c5..cc6d840ac3dc 100644 --- a/itests/hive-unit-hadoop2/pom.xml +++ b/itests/hive-unit-hadoop2/pom.xml @@ -84,12 +84,6 @@ org.apache.hive hive-it-util - - - org.apache.hive - hive-it-druid - - test diff --git a/itests/hive-unit/pom.xml b/itests/hive-unit/pom.xml index e50d8ffa5733..978c1788a4a9 100644 --- a/itests/hive-unit/pom.xml +++ b/itests/hive-unit/pom.xml @@ -86,12 +86,6 @@ org.apache.hive hive-it-util - - - org.apache.hive - hive-it-druid - - org.apache.hadoop diff --git a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestRestrictedList.java b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestRestrictedList.java index 6ae1a193c601..e3d29063db64 100644 --- a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestRestrictedList.java +++ b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestRestrictedList.java @@ -91,8 +91,6 @@ public static void startServices() throws Exception { addToExpectedRestrictedMap("hive.server2.service.users"); addToExpectedRestrictedMap("hive.server2.graceful.stop.timeout"); addToExpectedRestrictedMap("hive.query.max.length"); - addToExpectedRestrictedMap("hive.druid.broker.address.default"); - addToExpectedRestrictedMap("hive.druid.coordinator.address.default"); addToExpectedRestrictedMap("hikaricp.test"); addToExpectedRestrictedMap("hadoop.bin.path"); addToExpectedRestrictedMap("yarn.bin.path"); diff --git a/itests/pom.xml b/itests/pom.xml index 566bd2fde3dd..a59b5a872266 100644 --- a/itests/pom.xml +++ b/itests/pom.xml @@ -43,7 +43,6 @@ hive-jmh hive-unit-hadoop2 hive-minikdc - qtest-druid qtest-kudu qtest-iceberg test-docker @@ -227,22 +226,6 @@ hive-it-util ${project.version} - - org.apache.hive - hive-druid-handler - ${project.version} - - - org.apache.hive - hive-druid-handler - ${project.version} - tests - - - org.apache.hive - hive-it-druid - ${project.version} - org.apache.hive hive-hbase-handler diff --git a/itests/qtest-accumulo/pom.xml b/itests/qtest-accumulo/pom.xml index f54316c8ed12..87cd503cf6c8 100644 --- a/itests/qtest-accumulo/pom.xml +++ b/itests/qtest-accumulo/pom.xml @@ -92,10 +92,6 @@ org.apache.calcite calcite-core - - org.apache.hive - hive-it-druid - diff --git a/itests/qtest-druid/pom.xml b/itests/qtest-druid/pom.xml deleted file mode 100644 index 5227f9b05e28..000000000000 --- a/itests/qtest-druid/pom.xml +++ /dev/null @@ -1,319 +0,0 @@ - - - - 4.0.0 - - hive-it - org.apache.hive - 4.3.0-SNAPSHOT - ../pom.xml - - hive-it-druid - jar - Hive Integration - QFile Druid Tests - - - ../.. - 4.0.0 - 1.19.3 - 9.4.57.v20241219 - 10.11.1.1 - 16.0.1 - 4.1.0 - 4.1.0 - 1.7.30 - - - - org.apache.druid - druid-server - ${druid.version} - - - jersey-server - com.sun.jersey - - - jersey-servlet - com.sun.jersey - - - jersey-core - com.sun.jersey - - - org.jboss.netty - * - - - - - org.apache.druid - druid-services - ${druid.version} - - - jersey-server - com.sun.jersey - - - jersey-servlet - com.sun.jersey - - - jersey-core - com.sun.jersey - - - org.jboss.netty - * - - - - - com.google.guava - guava - ${druid.guava.version} - - - org.apache.druid.extensions - druid-hdfs-storage - ${druid.version} - - - org.apache.hadoop - hadoop-hdfs-client - - - com.sun.jersey - jersey-servlet - - - com.sun.jersey - jersey-client - - - - - org.apache.druid.extensions - druid-bloom-filter - ${druid.version} - - - org.apache.druid.extensions - druid-kafka-indexing-service - ${druid.version} - - - org.apache.druid.extensions - druid-avro-extensions - ${druid.version} - - - org.mortbay.jetty - servlet-api - - - - - org.apache.logging.log4j - log4j-api - ${log4j2.version} - - - org.apache.logging.log4j - log4j-core - ${log4j2.version} - - - org.apache.curator - curator-framework - ${druid.curator.version} - - - org.apache.curator - curator-client - ${druid.curator.version} - - - org.apache.curator - curator-x-discovery - ${druid.curator.version} - - - org.apache.curator - curator-recipes - ${druid.curator.version} - - - com.sun.jersey - jersey-bundle - ${druid.jersey.version} - - - org.eclipse.jetty - jetty-server - ${druid.jetty.version} - - - org.eclipse.jetty - jetty-servlet - ${druid.jetty.version} - - - org.eclipse.jetty - jetty-servlets - ${druid.jetty.version} - - - org.eclipse.jetty - jetty-proxy - ${druid.jetty.version} - - - org.eclipse.jetty - jetty-util - ${druid.jetty.version} - - - org.eclipse.jetty - jetty-security - ${druid.jetty.version} - - - com.fasterxml.jackson.core - jackson-core - - - com.fasterxml.jackson.core - jackson-databind - - - org.apache.hadoop - hadoop-annotations - compile - ${hadoop.version} - - - com.google.inject - guice - ${druid.guice.version} - - - org.apache.kafka - kafka_2.12 - ${kafka.version} - - - org.apache.kafka - kafka-clients - ${kafka.version} - - - org.apache.kafka - kafka-server - ${kafka.version} - - - org.slf4j - slf4j-api - - - - - - - org.apache.maven.plugins - maven-shade-plugin - - - - package - - shade - - - false - false - - - org.apache.druid.cli.Main - - - - - - junit:* - jmock:* - *:xml-apis - org.apache.maven:lib:tests - javax.ws.rs:jsr311-api - *:javax.el-api - *:jsp-api* - - - - - org.apache.kafka - org.apache.kafkatests - - - org.jboss.netty - org.apache.hive.druid.org.jboss.netty - - - - - *:* - - META-INF/*.SF - META-INF/*.DSA - META-INF/*.RSA - META-INF/versions/** - - - - - - - - - org.apache.maven.plugins - maven-antrun-plugin - - - setup-metastore-scripts - process-test-resources - - run - - - - - - - - - - - - - - - diff --git a/itests/qtest-druid/src/main/java/org/apache/hive/druid/DruidNode.java b/itests/qtest-druid/src/main/java/org/apache/hive/druid/DruidNode.java deleted file mode 100644 index bc37d8c1d6b9..000000000000 --- a/itests/qtest-druid/src/main/java/org/apache/hive/druid/DruidNode.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hive.druid; - -import java.io.Closeable; -import java.io.IOException; - -public abstract class DruidNode implements Closeable{ - - private final String nodeType; - - public DruidNode(String nodeId) {this.nodeType = nodeId;} - - final public String getNodeType() { - return nodeType; - } - - /** - * starts the druid node - */ - public abstract void start() throws IOException; - - /** - * @return true if the process is working - */ - public abstract boolean isAlive(); - -} diff --git a/itests/qtest-druid/src/main/java/org/apache/hive/druid/ForkingDruidNode.java b/itests/qtest-druid/src/main/java/org/apache/hive/druid/ForkingDruidNode.java deleted file mode 100644 index 9703faa4b6e3..000000000000 --- a/itests/qtest-druid/src/main/java/org/apache/hive/druid/ForkingDruidNode.java +++ /dev/null @@ -1,157 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hive.druid; - -import com.google.common.base.Preconditions; -import com.google.common.base.Strings; -import com.google.common.base.Throwables; -import com.google.common.collect.Lists; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; -import java.io.IOException; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.TimeUnit; - -public class ForkingDruidNode extends DruidNode { - private final static String DEFAULT_JAVA_CMD = "java"; - - private final static Logger log = LoggerFactory.getLogger(ForkingDruidNode.class); - - private final String classpath; - - private final Map properties; - - private final List jvmArgs; - - private final File logLocation; - - private final File logFile; - - private final String javaCmd; - - private final ProcessBuilder processBuilder = new ProcessBuilder(); - - private Process druidProcess = null; - - private Boolean started = false; - - private final List allowedPrefixes = Lists.newArrayList( - "com.metamx", - "druid", - "org.apache.druid", - "java.io.tmpdir", - "hadoop" - ); - - public ForkingDruidNode(String nodeType, - String extraClasspath, - Map properties, - List jvmArgs, - File logLocation, - String javaCmd - ) { - super(nodeType); - - final List command = Lists.newArrayList(); - this.classpath = Strings.isNullOrEmpty(extraClasspath) - ? System.getProperty("java.class.path") - : extraClasspath; - this.properties = properties == null ? new HashMap<>() : properties; - this.jvmArgs = Preconditions.checkNotNull(jvmArgs); - this.logLocation = logLocation == null ? new File("/tmp/druid") : logLocation; - if (!this.logLocation.exists()) { - this.logLocation.mkdirs(); - } - - this.javaCmd = javaCmd == null ? DEFAULT_JAVA_CMD : javaCmd; - - logFile = new File(this.logLocation, getNodeType() + ".log"); - // set the log stream - processBuilder.redirectErrorStream(true); - processBuilder.redirectOutput(ProcessBuilder.Redirect.appendTo(logFile)); - command.add(this.javaCmd); - command.addAll(this.jvmArgs); - command.add("-server"); - command.add("-cp"); - command.add(classpath); - - // inject properties from the main App that matches allowedPrefix - for (String propName : System.getProperties().stringPropertyNames()) { - for (String allowedPrefix : allowedPrefixes) { - if (propName.startsWith(allowedPrefix)) { - command.add( - String.format( - "-D%s=%s", - propName, - System.getProperty(propName) - ) - ); - } - } - } - this.properties - .forEach((key, value) -> command.add(String.format("-D%s=%s", key, value))); - command.addAll(Lists.newArrayList("org.apache.druid.cli.Main", "server", getNodeType())); - processBuilder.command(command); - log.info("Creating forking druid node with " + String.join(" ", processBuilder.command())); - } - - @Override - public void start() throws IOException { - synchronized (started) { - if (started == false) { - druidProcess = processBuilder.start(); - started = true; - } - log.info("Started " + getNodeType()); - } - } - - @Override - public boolean isAlive() { - synchronized (started) { - return started && druidProcess != null && druidProcess.isAlive(); - } - } - - @Override - public void close() throws IOException { - synchronized (started) { - if (druidProcess != null && druidProcess.isAlive()) { - druidProcess.destroy(); - } - try { - log.info("Waiting for " + getNodeType()); - if (druidProcess.waitFor(5000, TimeUnit.MILLISECONDS)) { - log.info(String.format("Shutdown completed for node [%s]", getNodeType())); - } else { - log.info(String.format("Waiting to shutdown node [%s] exhausted shutting down forcibly", getNodeType())); - druidProcess.destroyForcibly(); - } - } catch (InterruptedException e) { - Thread.interrupted(); - Throwables.propagate(e); - } - } - } -} diff --git a/itests/qtest-druid/src/main/java/org/apache/hive/druid/MiniDruidCluster.java b/itests/qtest-druid/src/main/java/org/apache/hive/druid/MiniDruidCluster.java deleted file mode 100644 index 0fb63ce5c30e..000000000000 --- a/itests/qtest-druid/src/main/java/org/apache/hive/druid/MiniDruidCluster.java +++ /dev/null @@ -1,283 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hive.druid; - -import com.google.common.base.Throwables; -import com.google.common.collect.ImmutableMap; -import org.apache.commons.io.FileUtils; -import org.apache.hadoop.service.AbstractService; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; -import java.io.IOException; -import java.net.DatagramSocket; -import java.net.ServerSocket; -import java.util.Arrays; -import java.util.List; -import java.util.Locale; -import java.util.Map; - -/** - * This class has the hooks to start and stop the external Druid Nodes - */ -public class MiniDruidCluster extends AbstractService { - private static final Logger log = LoggerFactory.getLogger(MiniDruidCluster.class); - - private static final String - COMMON_DRUID_JVM_PROPERTIES = - "-Duser.timezone=UTC -Dfile.encoding=UTF-8 -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager " - + "-Ddruid.emitter=logging -Ddruid.emitter.logging.logLevel=info"; - - private static final List - HISTORICAL_JVM_CONF = - Arrays.asList("-server", "-XX:MaxDirectMemorySize=10g", "-Xmx512m", "-Xmx512m", COMMON_DRUID_JVM_PROPERTIES); - - private static final List - COORDINATOR_JVM_CONF = - Arrays.asList("-server", "-XX:MaxDirectMemorySize=2g", "-Xmx512m", "-Xms512m", COMMON_DRUID_JVM_PROPERTIES); - - private static final Map - COMMON_DRUID_CONF = - ImmutableMap.of("druid.metadata.storage.type", - "derby", - "druid.storage.type", - "hdfs", - "druid.processing.buffer.sizeBytes", - "10485760", - "druid.processing.numThreads", - "2", - "druid.worker.capacity", - "4"); - - private static final Map - COMMON_DRUID_HISTORICAL = - ImmutableMap.of("druid.server.maxSize", "130000000000"); - - private static final Map - COMMON_COORDINATOR_INDEXER = - ImmutableMap.builder() - .put("druid.indexer.logs.type", "file") - .put("druid.coordinator.asOverlord.enabled", "true") - .put("druid.coordinator.asOverlord.overlordService", "druid/overlord") - .put("druid.coordinator.period", "PT2S") - .put("druid.manager.segments.pollDuration", "PT2S") - .put("druid.indexer.runner.javaOpts", "-Xmx512m") - .build(); - private static final int MIN_PORT_NUMBER = 60000; - private static final int MAX_PORT_NUMBER = 65535; - - private final DruidNode historical; - - private final DruidNode broker; - - // Coordinator is running as Overlord as well. - private final DruidNode coordinator; - - private final List druidNodes; - - private final File dataDirectory; - - private final File logDirectory; - private final String derbyURI; - private final int coordinatorPort; - private final int historicalPort; - private final int brokerPort; - - - public MiniDruidCluster(String name, String logDir, String tmpDir, Integer zookeeperPort, String classpath) { - super(name); - this.dataDirectory = new File(tmpDir, "druid-data"); - this.logDirectory = new File(logDir); - - boolean isKafka = name.contains("kafka"); - coordinatorPort = isKafka ? 8081 : 9081; - brokerPort = coordinatorPort + 1; - historicalPort = brokerPort + 1; - int derbyPort = historicalPort + 1; - - ensureCleanDirectory(dataDirectory); - - derbyURI = - String.format("jdbc:derby://localhost:%s/%s/druid_derby/metadata.db;create=true", - derbyPort, - dataDirectory.getAbsolutePath()); - String - segmentsCache = - String.format("[{\"path\":\"%s/druid/segment-cache\",\"maxSize\":130000000000}]", - dataDirectory.getAbsolutePath()); - String indexingLogDir = new File(logDirectory, "indexer-log").getAbsolutePath(); - - ImmutableMap.Builder coordinatorMapBuilder = new ImmutableMap.Builder(); - ImmutableMap.Builder historicalMapBuilder = new ImmutableMap.Builder(); - ImmutableMap.Builder brokerMapBuilder = new ImmutableMap.Builder(); - - Map - coordinatorProperties = - coordinatorMapBuilder.putAll(COMMON_DRUID_CONF) - .putAll(COMMON_COORDINATOR_INDEXER) - .put("druid.metadata.storage.connector.connectURI", derbyURI) - .put("druid.metadata.storage.connector.port", String.valueOf(derbyPort)) - .put("druid.indexer.logs.directory", indexingLogDir) - .put("druid.zk.service.host", "localhost:" + zookeeperPort) - .put("druid.coordinator.startDelay", "PT1S") - .put("druid.indexer.runner", "local") - .put("druid.storage.storageDirectory", getDeepStorageDir()) - .put("druid.port", String.valueOf(coordinatorPort)) - .build(); - Map - historicalProperties = - historicalMapBuilder.putAll(COMMON_DRUID_CONF) - .putAll(COMMON_DRUID_HISTORICAL) - .put("druid.zk.service.host", "localhost:" + zookeeperPort) - .put("druid.segmentCache.locations", segmentsCache) - .put("druid.storage.storageDirectory", getDeepStorageDir()) - .put("druid.port", String.valueOf(historicalPort)) - .build(); - Map - brokerProperties = - brokerMapBuilder.putAll(COMMON_DRUID_CONF) - .put("druid.zk.service.host", "localhost:" + zookeeperPort) - .put("druid.port", String.valueOf(brokerPort)) - .build(); - coordinator = - new ForkingDruidNode("coordinator", classpath, coordinatorProperties, COORDINATOR_JVM_CONF, logDirectory, null); - historical = - new ForkingDruidNode("historical", classpath, historicalProperties, HISTORICAL_JVM_CONF, logDirectory, null); - broker = new ForkingDruidNode("broker", classpath, brokerProperties, HISTORICAL_JVM_CONF, logDirectory, null); - druidNodes = Arrays.asList(coordinator, historical, broker); - - } - - private int findPort(int start, int end) { - int port = start; - while (!available(port)) { - port++; - if (port == end) { - throw new RuntimeException("can not find free port for range " + start + ":" + end); - } - } - return port; - } - - /** - * Checks to see if a specific port is available. - * - * @param port the port to check for availability - */ - public static boolean available(int port) { - if (port < MIN_PORT_NUMBER || port > MAX_PORT_NUMBER) { - throw new IllegalArgumentException("Invalid start port: " + port); - } - - ServerSocket ss = null; - DatagramSocket ds = null; - try { - ss = new ServerSocket(port); - ss.setReuseAddress(true); - ds = new DatagramSocket(port); - ds.setReuseAddress(true); - return true; - } catch (IOException e) { - } finally { - if (ds != null) { - ds.close(); - } - - if (ss != null) { - try { - ss.close(); - } catch (IOException e) { - /* should not be thrown */ - } - } - } - - return false; - } - - public static void ensureCleanDirectory(File dir) { - try { - if (dir.exists()) { - // need to clean data directory to ensure that there is no interference from old runs - // Cleaning is happening here to allow debugging in case of tests fail - // we don;t have to clean logs since it is an append mode - log.info("Cleaning the druid directory [{}]", dir.getAbsolutePath()); - FileUtils.deleteDirectory(dir); - } else { - log.info("Creating the druid directory [{}]", dir.getAbsolutePath()); - dir.mkdirs(); - } - } catch (IOException e) { - log.error("Failed to clean druid directory"); - Throwables.propagate(e); - } - } - - @Override protected void serviceStart() throws Exception { - druidNodes.stream().forEach(node -> { - try { - node.start(); - } catch (IOException e) { - log.error("Failed to start node " + node.getNodeType() + " Consequently will destroy the cluster"); - druidNodes.stream().filter(node1 -> node1.isAlive()).forEach(nodeToStop -> { - try { - log.info("Stopping Node " + nodeToStop.getNodeType()); - nodeToStop.close(); - } catch (IOException e1) { - log.error("Error while stopping " + nodeToStop.getNodeType(), e1); - } - }); - Throwables.propagate(e); - } - }); - } - - @Override protected void serviceStop() throws Exception { - druidNodes.stream().forEach(node -> { - try { - node.close(); - } catch (IOException e) { - // nothing that we can really do about it - log.error(String.format("Failed to stop druid node [%s]", node.getNodeType()), e); - } - }); - } - - public String getMetadataURI() { - return derbyURI; - } - - public String getDeepStorageDir() { - return dataDirectory.getAbsolutePath() + File.separator + "deep-storage"; - } - - public String getCoordinatorURI() { - return String.format(Locale.ROOT,"localhost:%s", coordinatorPort); - } - - public String getBrokerURI() { - return String.format(Locale.ROOT,"localhost:%s", brokerPort); - } - - public String getOverlordURI() { - // Overlord and coordinator both run in same JVM. - return getCoordinatorURI(); - } -} diff --git a/itests/qtest-iceberg/pom.xml b/itests/qtest-iceberg/pom.xml index 035c79a14453..a03f310f13c9 100644 --- a/itests/qtest-iceberg/pom.xml +++ b/itests/qtest-iceberg/pom.xml @@ -97,17 +97,6 @@ hadoop-yarn-registry true - - org.apache.hive - hive-druid-handler - test - - - org.apache.hive - hive-druid-handler - tests - test - org.apache.hive hive-jdbc-handler @@ -439,29 +428,6 @@ - - org.apache.hive - hive-it-druid - test - - - com.sun.jersey - jersey-bundle - - - org.slf4j - slf4j-log4j12 - - - commons-logging - commons-logging - - - io.netty - * - - - org.slf4j slf4j-api diff --git a/itests/qtest-kudu/pom.xml b/itests/qtest-kudu/pom.xml index 94a6fd2b7172..e538abdc789c 100644 --- a/itests/qtest-kudu/pom.xml +++ b/itests/qtest-kudu/pom.xml @@ -82,10 +82,6 @@ org.apache.calcite calcite-core - - org.apache.hive - hive-it-druid - diff --git a/itests/qtest/pom.xml b/itests/qtest/pom.xml index fdf75bd890bc..e80293f774cc 100644 --- a/itests/qtest/pom.xml +++ b/itests/qtest/pom.xml @@ -95,17 +95,6 @@ hadoop-yarn-registry true - - org.apache.hive - hive-druid-handler - test - - - org.apache.hive - hive-druid-handler - tests - test - org.apache.hive hive-jdbc-handler @@ -418,29 +407,6 @@ - - org.apache.hive - hive-it-druid - test - - - com.sun.jersey - jersey-bundle - - - org.slf4j - slf4j-log4j12 - - - commons-logging - commons-logging - - - io.netty - * - - - org.slf4j slf4j-api diff --git a/itests/qtest/src/test/java/org/apache/hadoop/hive/cli/MiniDruidLlapLocalCliDriver.java b/itests/qtest/src/test/java/org/apache/hadoop/hive/cli/MiniDruidLlapLocalCliDriver.java deleted file mode 100644 index ee7775f7f549..000000000000 --- a/itests/qtest/src/test/java/org/apache/hadoop/hive/cli/MiniDruidLlapLocalCliDriver.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.cli; - -import org.apache.hadoop.hive.cli.control.CliAdapter; -import org.apache.hadoop.hive.cli.control.CliConfigs; -import org.junit.ClassRule; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TestRule; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; - -import java.io.File; -import java.util.List; - -/** - * This is used only for dev debugging if needed - */ -@RunWith(Parameterized.class) -public class MiniDruidLlapLocalCliDriver { - static CliAdapter adapter = new CliConfigs.MiniDruidLlapLocalCliConfig().getCliAdapter(); - - @Parameterized.Parameters(name = "{0}") - public static List getParameters() throws Exception { - return adapter.getParameters(); - } - - @ClassRule - public static TestRule cliClassRule = adapter.buildClassRule(); - - @Rule - public TestRule cliTestRule = adapter.buildTestRule(); - - private String name; - private File qfile; - - public MiniDruidLlapLocalCliDriver(String name, File qfile) { - this.name = name; - this.qfile = qfile; - } - - @Test - public void testCliDriver() throws Exception { - adapter.runTest(name, qfile); - } - -} diff --git a/itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestMiniDruidCliDriver.java b/itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestMiniDruidCliDriver.java deleted file mode 100644 index ff3d2d3eccdb..000000000000 --- a/itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestMiniDruidCliDriver.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.cli; - -import org.apache.hadoop.hive.cli.control.CliAdapter; -import org.apache.hadoop.hive.cli.control.CliConfigs; -import org.junit.ClassRule; -import org.junit.Ignore; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TestRule; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; -import org.junit.runners.Parameterized.Parameters; - -import java.io.File; -import java.util.List; -@Ignore("Flaky due to HIVE-24816") -@RunWith(Parameterized.class) -public class TestMiniDruidCliDriver -{ - - static CliAdapter adapter = new CliConfigs.MiniDruidCliConfig().getCliAdapter(); - - @Parameters(name = "{0}") - public static List getParameters() throws Exception { - return adapter.getParameters(); - } - - @ClassRule - public static TestRule cliClassRule = adapter.buildClassRule(); - - @Rule - public TestRule cliTestRule = adapter.buildTestRule(); - - private String name; - private File qfile; - - public TestMiniDruidCliDriver(String name, File qfile) { - this.name = name; - this.qfile = qfile; - } - - @Test - public void testCliDriver() throws Exception { - adapter.runTest(name, qfile); - } - -} diff --git a/itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestMiniDruidKafkaCliDriver.java b/itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestMiniDruidKafkaCliDriver.java deleted file mode 100644 index a6aef86c5d73..000000000000 --- a/itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestMiniDruidKafkaCliDriver.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.cli; - -import org.apache.hadoop.hive.cli.control.CliAdapter; -import org.apache.hadoop.hive.cli.control.CliConfigs; -import org.junit.ClassRule; -import org.junit.Ignore; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TestRule; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; -import org.junit.runners.Parameterized.Parameters; - -import java.io.File; -import java.util.List; -@Ignore("Flaky due to HIVE-24816") -@RunWith(Parameterized.class) -public class TestMiniDruidKafkaCliDriver -{ - - static CliAdapter adapter = new CliConfigs.MiniDruidKafkaCliConfig().getCliAdapter(); - - @Parameters(name = "{0}") - public static List getParameters() throws Exception { - return adapter.getParameters(); - } - - @ClassRule - public static TestRule cliClassRule = adapter.buildClassRule(); - - @Rule - public TestRule cliTestRule = adapter.buildTestRule(); - - private String name; - private File qfile; - - public TestMiniDruidKafkaCliDriver(String name, File qfile) { - this.name = name; - this.qfile = qfile; - } - - @Test - public void testCliDriver() throws Exception { - adapter.runTest(name, qfile); - } - -} diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties index bfefc3b5f023..3927effc5ab9 100644 --- a/itests/src/test/resources/testconfiguration.properties +++ b/itests/src/test/resources/testconfiguration.properties @@ -381,37 +381,9 @@ tez.perf.disabled.query.files=\ mv_query67.q,\ mv_query68.q -druid.query.files=\ - druid_materialized_view_rewrite_ssb.q,\ - druid_timeseries.q,\ - druid_timestamptz.q,\ - druid_timestamptz2.q,\ - druid_topn.q,\ - druidmini_dynamic_partition.q,\ - druidmini_expressions.q,\ - druidmini_extractTime.q,\ - druidmini_floorTime.q,\ - druidmini_joins.q,\ - druidmini_masking.q,\ - druidmini_mv.q,\ - druidmini_semijoin_reduction_all_types.q,\ - druidmini_test1.q,\ - druidmini_test_alter.q,\ - druidmini_test_insert.q,\ - druidmini_test_ts.q - -druid.kafka.query.files=\ - druidkafkamini_avro.q,\ - druidkafkamini_basic.q,\ - druidkafkamini_csv.q,\ - druidkafkamini_delimited.q - hive.kafka.query.files=\ kafka_storage_handler.q -druid.llap.local.query.files=\ - druidmini_noop.q - # tests to be run only by TestErasureCodingHDFSCliDriver erasurecoding.only.query.files=\ erasure_commands.q,\ diff --git a/itests/util/pom.xml b/itests/util/pom.xml index 54b5ab177562..260545053858 100644 --- a/itests/util/pom.xml +++ b/itests/util/pom.xml @@ -270,26 +270,19 @@ tests - org.apache.hive - hive-it-druid - - - com.sun.jersey - jersey-bundle - - - org.slf4j - slf4j-log4j12 - - - commons-logging - commons-logging - - - io.netty - * - - + org.apache.kafka + kafka_2.12 + ${kafka.version} + + + org.apache.kafka + kafka-clients + ${kafka.version} + + + org.apache.kafka + kafka-server + ${kafka.version} diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java index 397e46ee83fd..2850947e7b48 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java @@ -155,46 +155,6 @@ public MiniLlapCliConfig() { } } - public static class MiniDruidCliConfig extends AbstractCliConfig { - public MiniDruidCliConfig() { - super(CoreCliDriver.class); - try { - setQueryDir("ql/src/test/queries/clientpositive"); - - includesFrom(testConfigProps, "druid.query.files"); - - setResultsDir("ql/src/test/results/clientpositive/druid"); - setLogDir("itests/qtest/target/tmp/log"); - - setInitScript("q_test_druid_init.sql"); - setCleanupScript("q_test_cleanup_druid.sql"); - setHiveConfDir("data/conf/llap"); - setClusterType(MiniClusterType.DRUID); - } catch (Exception e) { - throw new RuntimeException("can't construct cliconfig", e); - } - } - } - - public static class MiniDruidKafkaCliConfig extends AbstractCliConfig { - public MiniDruidKafkaCliConfig() { - super(CoreCliDriver.class); - try { - setQueryDir("ql/src/test/queries/clientpositive"); - includesFrom(testConfigProps, "druid.kafka.query.files"); - setResultsDir("ql/src/test/results/clientpositive/druid"); - setLogDir("itests/qtest/target/tmp/log"); - - setInitScript("q_test_druid_init.sql"); - setCleanupScript("q_test_cleanup_druid.sql"); - setHiveConfDir("data/conf/llap"); - setClusterType(MiniClusterType.DRUID_KAFKA); - } catch (Exception e) { - throw new RuntimeException("can't construct cliconfig", e); - } - } - } - public static class MiniKafkaCliConfig extends AbstractCliConfig { public MiniKafkaCliConfig() { super(CoreCliDriver.class); @@ -223,8 +183,6 @@ public MiniLlapLocalCliConfig() { excludesFrom(testConfigProps, "minillap.query.files"); excludesFrom(testConfigProps, "minitez.query.files"); excludesFrom(testConfigProps, "encrypted.query.files"); - excludesFrom(testConfigProps, "druid.query.files"); - excludesFrom(testConfigProps, "druid.kafka.query.files"); excludesFrom(testConfigProps, "hive.kafka.query.files"); excludesFrom(testConfigProps, "erasurecoding.only.query.files"); excludesFrom(testConfigProps, "beeline.positive.include"); @@ -617,27 +575,6 @@ private void setHiveConfDir(MiniClusterType clusterType) { } } - public static class MiniDruidLlapLocalCliConfig extends AbstractCliConfig { - public MiniDruidLlapLocalCliConfig() { - super(CoreCliDriver.class); - try { - setQueryDir("ql/src/test/queries/clientpositive"); - - includesFrom(testConfigProps, "druid.llap.local.query.files"); - - setResultsDir("ql/src/test/results/clientpositive/druid"); - setLogDir("itests/qtest/target/tmp/log"); - - setInitScript("q_test_druid_init.sql"); - setCleanupScript("q_test_cleanup_druid.sql"); - setHiveConfDir("data/conf/llap"); - setClusterType(MiniClusterType.DRUID_LOCAL); - } catch (Exception e) { - throw new RuntimeException("can't construct cliconfig", e); - } - } - } - /** * The CliConfig implementation for Kudu. */ diff --git a/itests/qtest-druid/src/main/java/org/apache/hive/kafka/SingleNodeKafkaCluster.java b/itests/util/src/main/java/org/apache/hadoop/hive/kafka/SingleNodeKafkaCluster.java similarity index 97% rename from itests/qtest-druid/src/main/java/org/apache/hive/kafka/SingleNodeKafkaCluster.java rename to itests/util/src/main/java/org/apache/hadoop/hive/kafka/SingleNodeKafkaCluster.java index cd43a37b1848..d400d0bb61b6 100644 --- a/itests/qtest-druid/src/main/java/org/apache/hive/kafka/SingleNodeKafkaCluster.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/kafka/SingleNodeKafkaCluster.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.hive.kafka; +package org.apache.hadoop.hive.kafka; import kafka.server.KafkaConfig; import kafka.server.KafkaServer; @@ -69,11 +69,11 @@ public SingleNodeKafkaCluster(String name, String logDir, Integer zkPort, Intege // need to clean data directory to ensure that there is no interference from old runs // Cleaning is happening here to allow debugging in case of tests fail // we don;t have to clean logs since it is an append mode - log.info("Cleaning the druid directory [{}]", dir.getAbsolutePath()); + log.info("Cleaning the Kafka directory [{}]", dir.getAbsolutePath()); try { FileUtils.deleteDirectory(dir); } catch (IOException e) { - log.error("Failed to clean druid directory"); + log.error("Failed to clean Kafka directory"); throw new RuntimeException(e); } } diff --git a/itests/qtest-druid/src/main/java/org/apache/hive/kafka/Wikipedia.java b/itests/util/src/main/java/org/apache/hadoop/hive/kafka/Wikipedia.java similarity index 92% rename from itests/qtest-druid/src/main/java/org/apache/hive/kafka/Wikipedia.java rename to itests/util/src/main/java/org/apache/hadoop/hive/kafka/Wikipedia.java index 92907ca19f81..f6cee88e666c 100644 --- a/itests/qtest-druid/src/main/java/org/apache/hive/kafka/Wikipedia.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/kafka/Wikipedia.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.hive.kafka; +package org.apache.hadoop.hive.kafka; import org.apache.avro.specific.SpecificData; import org.apache.avro.message.BinaryMessageEncoder; @@ -475,8 +475,8 @@ public void setNamespace(java.lang.String value) { * Creates a new Wikipedia RecordBuilder. * @return A new Wikipedia RecordBuilder */ - public static org.apache.hive.kafka.Wikipedia.Builder newBuilder() { - return new org.apache.hive.kafka.Wikipedia.Builder(); + public static Wikipedia.Builder newBuilder() { + return new Wikipedia.Builder(); } /** @@ -484,8 +484,8 @@ public static org.apache.hive.kafka.Wikipedia.Builder newBuilder() { * @param other The existing builder to copy. * @return A new Wikipedia RecordBuilder */ - public static org.apache.hive.kafka.Wikipedia.Builder newBuilder(org.apache.hive.kafka.Wikipedia.Builder other) { - return new org.apache.hive.kafka.Wikipedia.Builder(other); + public static Wikipedia.Builder newBuilder(Wikipedia.Builder other) { + return new Wikipedia.Builder(other); } /** @@ -493,8 +493,8 @@ public static org.apache.hive.kafka.Wikipedia.Builder newBuilder(org.apache.hive * @param other The existing instance to copy. * @return A new Wikipedia RecordBuilder */ - public static org.apache.hive.kafka.Wikipedia.Builder newBuilder(org.apache.hive.kafka.Wikipedia other) { - return new org.apache.hive.kafka.Wikipedia.Builder(other); + public static Wikipedia.Builder newBuilder(Wikipedia other) { + return new Wikipedia.Builder(other); } /** @@ -531,7 +531,7 @@ private Builder() { * Creates a Builder by copying an existing Builder. * @param other The existing Builder to copy. */ - private Builder(org.apache.hive.kafka.Wikipedia.Builder other) { + private Builder(Wikipedia.Builder other) { super(other); if (isValidValue(fields()[0], other.isrobot)) { this.isrobot = data().deepCopy(fields()[0].schema(), other.isrobot); @@ -611,7 +611,7 @@ private Builder(org.apache.hive.kafka.Wikipedia.Builder other) { * Creates a Builder by copying an existing Wikipedia instance * @param other The existing instance to copy. */ - private Builder(org.apache.hive.kafka.Wikipedia other) { + private Builder(Wikipedia other) { super(SCHEMA$); if (isValidValue(fields()[0], other.isrobot)) { this.isrobot = data().deepCopy(fields()[0].schema(), other.isrobot); @@ -700,7 +700,7 @@ public java.lang.Boolean getIsrobot() { * @param value The value of 'isrobot'. * @return This builder. */ - public org.apache.hive.kafka.Wikipedia.Builder setIsrobot(boolean value) { + public Wikipedia.Builder setIsrobot(boolean value) { validate(fields()[0], value); this.isrobot = value; fieldSetFlags()[0] = true; @@ -720,7 +720,7 @@ public boolean hasIsrobot() { * Clears the value of the 'isrobot' field. * @return This builder. */ - public org.apache.hive.kafka.Wikipedia.Builder clearIsrobot() { + public Wikipedia.Builder clearIsrobot() { fieldSetFlags()[0] = false; return this; } @@ -738,7 +738,7 @@ public java.lang.String getChannel() { * @param value The value of 'channel'. * @return This builder. */ - public org.apache.hive.kafka.Wikipedia.Builder setChannel(java.lang.String value) { + public Wikipedia.Builder setChannel(java.lang.String value) { validate(fields()[1], value); this.channel = value; fieldSetFlags()[1] = true; @@ -758,7 +758,7 @@ public boolean hasChannel() { * Clears the value of the 'channel' field. * @return This builder. */ - public org.apache.hive.kafka.Wikipedia.Builder clearChannel() { + public Wikipedia.Builder clearChannel() { channel = null; fieldSetFlags()[1] = false; return this; @@ -777,7 +777,7 @@ public java.lang.String getTimestamp() { * @param value The value of 'timestamp'. * @return This builder. */ - public org.apache.hive.kafka.Wikipedia.Builder setTimestamp(java.lang.String value) { + public Wikipedia.Builder setTimestamp(java.lang.String value) { validate(fields()[2], value); this.timestamp = value; fieldSetFlags()[2] = true; @@ -797,7 +797,7 @@ public boolean hasTimestamp() { * Clears the value of the 'timestamp' field. * @return This builder. */ - public org.apache.hive.kafka.Wikipedia.Builder clearTimestamp() { + public Wikipedia.Builder clearTimestamp() { timestamp = null; fieldSetFlags()[2] = false; return this; @@ -816,7 +816,7 @@ public java.lang.String getFlags() { * @param value The value of 'flags'. * @return This builder. */ - public org.apache.hive.kafka.Wikipedia.Builder setFlags(java.lang.String value) { + public Wikipedia.Builder setFlags(java.lang.String value) { validate(fields()[3], value); this.flags = value; fieldSetFlags()[3] = true; @@ -836,7 +836,7 @@ public boolean hasFlags() { * Clears the value of the 'flags' field. * @return This builder. */ - public org.apache.hive.kafka.Wikipedia.Builder clearFlags() { + public Wikipedia.Builder clearFlags() { flags = null; fieldSetFlags()[3] = false; return this; @@ -855,7 +855,7 @@ public java.lang.Boolean getIsunpatrolled() { * @param value The value of 'isunpatrolled'. * @return This builder. */ - public org.apache.hive.kafka.Wikipedia.Builder setIsunpatrolled(boolean value) { + public Wikipedia.Builder setIsunpatrolled(boolean value) { validate(fields()[4], value); this.isunpatrolled = value; fieldSetFlags()[4] = true; @@ -875,7 +875,7 @@ public boolean hasIsunpatrolled() { * Clears the value of the 'isunpatrolled' field. * @return This builder. */ - public org.apache.hive.kafka.Wikipedia.Builder clearIsunpatrolled() { + public Wikipedia.Builder clearIsunpatrolled() { fieldSetFlags()[4] = false; return this; } @@ -893,7 +893,7 @@ public java.lang.String getPage() { * @param value The value of 'page'. * @return This builder. */ - public org.apache.hive.kafka.Wikipedia.Builder setPage(java.lang.String value) { + public Wikipedia.Builder setPage(java.lang.String value) { validate(fields()[5], value); this.page = value; fieldSetFlags()[5] = true; @@ -913,7 +913,7 @@ public boolean hasPage() { * Clears the value of the 'page' field. * @return This builder. */ - public org.apache.hive.kafka.Wikipedia.Builder clearPage() { + public Wikipedia.Builder clearPage() { page = null; fieldSetFlags()[5] = false; return this; @@ -932,7 +932,7 @@ public java.lang.String getDiffurl() { * @param value The value of 'diffurl'. * @return This builder. */ - public org.apache.hive.kafka.Wikipedia.Builder setDiffurl(java.lang.String value) { + public Wikipedia.Builder setDiffurl(java.lang.String value) { validate(fields()[6], value); this.diffurl = value; fieldSetFlags()[6] = true; @@ -952,7 +952,7 @@ public boolean hasDiffurl() { * Clears the value of the 'diffurl' field. * @return This builder. */ - public org.apache.hive.kafka.Wikipedia.Builder clearDiffurl() { + public Wikipedia.Builder clearDiffurl() { diffurl = null; fieldSetFlags()[6] = false; return this; @@ -971,7 +971,7 @@ public java.lang.Long getAdded() { * @param value The value of 'added'. * @return This builder. */ - public org.apache.hive.kafka.Wikipedia.Builder setAdded(long value) { + public Wikipedia.Builder setAdded(long value) { validate(fields()[7], value); this.added = value; fieldSetFlags()[7] = true; @@ -991,7 +991,7 @@ public boolean hasAdded() { * Clears the value of the 'added' field. * @return This builder. */ - public org.apache.hive.kafka.Wikipedia.Builder clearAdded() { + public Wikipedia.Builder clearAdded() { fieldSetFlags()[7] = false; return this; } @@ -1009,7 +1009,7 @@ public java.lang.String getComment() { * @param value The value of 'comment'. * @return This builder. */ - public org.apache.hive.kafka.Wikipedia.Builder setComment(java.lang.String value) { + public Wikipedia.Builder setComment(java.lang.String value) { validate(fields()[8], value); this.comment = value; fieldSetFlags()[8] = true; @@ -1029,7 +1029,7 @@ public boolean hasComment() { * Clears the value of the 'comment' field. * @return This builder. */ - public org.apache.hive.kafka.Wikipedia.Builder clearComment() { + public Wikipedia.Builder clearComment() { comment = null; fieldSetFlags()[8] = false; return this; @@ -1048,7 +1048,7 @@ public java.lang.Long getCommentlength() { * @param value The value of 'commentlength'. * @return This builder. */ - public org.apache.hive.kafka.Wikipedia.Builder setCommentlength(long value) { + public Wikipedia.Builder setCommentlength(long value) { validate(fields()[9], value); this.commentlength = value; fieldSetFlags()[9] = true; @@ -1068,7 +1068,7 @@ public boolean hasCommentlength() { * Clears the value of the 'commentlength' field. * @return This builder. */ - public org.apache.hive.kafka.Wikipedia.Builder clearCommentlength() { + public Wikipedia.Builder clearCommentlength() { fieldSetFlags()[9] = false; return this; } @@ -1086,7 +1086,7 @@ public java.lang.Boolean getIsnew() { * @param value The value of 'isnew'. * @return This builder. */ - public org.apache.hive.kafka.Wikipedia.Builder setIsnew(boolean value) { + public Wikipedia.Builder setIsnew(boolean value) { validate(fields()[10], value); this.isnew = value; fieldSetFlags()[10] = true; @@ -1106,7 +1106,7 @@ public boolean hasIsnew() { * Clears the value of the 'isnew' field. * @return This builder. */ - public org.apache.hive.kafka.Wikipedia.Builder clearIsnew() { + public Wikipedia.Builder clearIsnew() { fieldSetFlags()[10] = false; return this; } @@ -1124,7 +1124,7 @@ public java.lang.Boolean getIsminor() { * @param value The value of 'isminor'. * @return This builder. */ - public org.apache.hive.kafka.Wikipedia.Builder setIsminor(boolean value) { + public Wikipedia.Builder setIsminor(boolean value) { validate(fields()[11], value); this.isminor = value; fieldSetFlags()[11] = true; @@ -1144,7 +1144,7 @@ public boolean hasIsminor() { * Clears the value of the 'isminor' field. * @return This builder. */ - public org.apache.hive.kafka.Wikipedia.Builder clearIsminor() { + public Wikipedia.Builder clearIsminor() { fieldSetFlags()[11] = false; return this; } @@ -1162,7 +1162,7 @@ public java.lang.Long getDelta() { * @param value The value of 'delta'. * @return This builder. */ - public org.apache.hive.kafka.Wikipedia.Builder setDelta(long value) { + public Wikipedia.Builder setDelta(long value) { validate(fields()[12], value); this.delta = value; fieldSetFlags()[12] = true; @@ -1182,7 +1182,7 @@ public boolean hasDelta() { * Clears the value of the 'delta' field. * @return This builder. */ - public org.apache.hive.kafka.Wikipedia.Builder clearDelta() { + public Wikipedia.Builder clearDelta() { fieldSetFlags()[12] = false; return this; } @@ -1200,7 +1200,7 @@ public java.lang.Boolean getIsanonymous() { * @param value The value of 'isanonymous'. * @return This builder. */ - public org.apache.hive.kafka.Wikipedia.Builder setIsanonymous(boolean value) { + public Wikipedia.Builder setIsanonymous(boolean value) { validate(fields()[13], value); this.isanonymous = value; fieldSetFlags()[13] = true; @@ -1220,7 +1220,7 @@ public boolean hasIsanonymous() { * Clears the value of the 'isanonymous' field. * @return This builder. */ - public org.apache.hive.kafka.Wikipedia.Builder clearIsanonymous() { + public Wikipedia.Builder clearIsanonymous() { fieldSetFlags()[13] = false; return this; } @@ -1238,7 +1238,7 @@ public java.lang.String getUser() { * @param value The value of 'user'. * @return This builder. */ - public org.apache.hive.kafka.Wikipedia.Builder setUser(java.lang.String value) { + public Wikipedia.Builder setUser(java.lang.String value) { validate(fields()[14], value); this.user = value; fieldSetFlags()[14] = true; @@ -1258,7 +1258,7 @@ public boolean hasUser() { * Clears the value of the 'user' field. * @return This builder. */ - public org.apache.hive.kafka.Wikipedia.Builder clearUser() { + public Wikipedia.Builder clearUser() { user = null; fieldSetFlags()[14] = false; return this; @@ -1277,7 +1277,7 @@ public java.lang.Double getDeltabucket() { * @param value The value of 'deltabucket'. * @return This builder. */ - public org.apache.hive.kafka.Wikipedia.Builder setDeltabucket(double value) { + public Wikipedia.Builder setDeltabucket(double value) { validate(fields()[15], value); this.deltabucket = value; fieldSetFlags()[15] = true; @@ -1297,7 +1297,7 @@ public boolean hasDeltabucket() { * Clears the value of the 'deltabucket' field. * @return This builder. */ - public org.apache.hive.kafka.Wikipedia.Builder clearDeltabucket() { + public Wikipedia.Builder clearDeltabucket() { fieldSetFlags()[15] = false; return this; } @@ -1315,7 +1315,7 @@ public java.lang.Long getDeleted() { * @param value The value of 'deleted'. * @return This builder. */ - public org.apache.hive.kafka.Wikipedia.Builder setDeleted(long value) { + public Wikipedia.Builder setDeleted(long value) { validate(fields()[16], value); this.deleted = value; fieldSetFlags()[16] = true; @@ -1335,7 +1335,7 @@ public boolean hasDeleted() { * Clears the value of the 'deleted' field. * @return This builder. */ - public org.apache.hive.kafka.Wikipedia.Builder clearDeleted() { + public Wikipedia.Builder clearDeleted() { fieldSetFlags()[16] = false; return this; } @@ -1353,7 +1353,7 @@ public java.lang.String getNamespace() { * @param value The value of 'namespace'. * @return This builder. */ - public org.apache.hive.kafka.Wikipedia.Builder setNamespace(java.lang.String value) { + public Wikipedia.Builder setNamespace(java.lang.String value) { validate(fields()[17], value); this.namespace = value; fieldSetFlags()[17] = true; @@ -1373,7 +1373,7 @@ public boolean hasNamespace() { * Clears the value of the 'namespace' field. * @return This builder. */ - public org.apache.hive.kafka.Wikipedia.Builder clearNamespace() { + public Wikipedia.Builder clearNamespace() { namespace = null; fieldSetFlags()[17] = false; return this; diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestMiniClusters.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestMiniClusters.java index 452c6cb0b6d7..5e45e6612fc2 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestMiniClusters.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestMiniClusters.java @@ -57,9 +57,8 @@ import org.apache.hadoop.hive.shims.HadoopShims; import org.apache.hadoop.hive.shims.ShimLoader; import org.apache.hadoop.hive.shims.HadoopShims.HdfsErasureCodingShim; -import org.apache.hive.druid.MiniDruidCluster; -import org.apache.hive.kafka.SingleNodeKafkaCluster; -import org.apache.hive.kafka.Wikipedia; +import org.apache.hadoop.hive.kafka.SingleNodeKafkaCluster; +import org.apache.hadoop.hive.kafka.Wikipedia; import org.apache.hive.testutils.MiniZooKeeperCluster; import org.apache.zookeeper.WatchedEvent; import org.apache.zookeeper.Watcher; @@ -70,7 +69,7 @@ import com.google.common.base.Preconditions; /** - * QTestMiniClusters: decouples cluster details from QTestUtil (kafka/druid/llap/tez/mr, file + * QTestMiniClusters: decouples cluster details from QTestUtil (kafka/llap/tez/mr, file * system) */ public class QTestMiniClusters { @@ -94,7 +93,6 @@ public class QTestMiniClusters { private HadoopShims.MiniDFSShim dfs = null; private HadoopShims.HdfsEncryptionShim hes = null; private MiniLlapCluster llapCluster = null; - private MiniDruidCluster druidCluster = null; private SingleNodeKafkaCluster kafkaCluster = null; public enum CoreClusterType { @@ -112,9 +110,6 @@ public enum MiniClusterType { LLAP(CoreClusterType.TEZ, FsType.HDFS), LLAP_LOCAL(CoreClusterType.TEZ, FsType.LOCAL), NONE(CoreClusterType.MR,FsType.LOCAL), - DRUID_LOCAL(CoreClusterType.TEZ, FsType.LOCAL), - DRUID(CoreClusterType.TEZ, FsType.HDFS), - DRUID_KAFKA(CoreClusterType.TEZ, FsType.HDFS), KAFKA(CoreClusterType.TEZ, FsType.HDFS), KUDU(CoreClusterType.TEZ, FsType.LOCAL); @@ -146,12 +141,6 @@ public static MiniClusterType valueForString(String type) { return LLAP; } else if (type.equals("llap_local")) { return LLAP_LOCAL; - } else if (type.equals("druidLocal")) { - return DRUID_LOCAL; - } else if (type.equals("druid")) { - return DRUID; - } else if (type.equals("druid-kafka")) { - return DRUID_KAFKA; } else if (type.equals("kafka")) { return KAFKA; } else if (type.equals("kudu")) { @@ -241,23 +230,7 @@ public void setup(QTestArguments testArgs, HiveConf conf, String scriptsDir, String uriString = fs.getUri().toString(); - if (clusterType == MiniClusterType.DRUID_KAFKA || clusterType == MiniClusterType.DRUID_LOCAL - || clusterType == MiniClusterType.DRUID) { - final String tempDir = QTestSystemProperties.getTempDir(); - druidCluster = new MiniDruidCluster( - clusterType == MiniClusterType.DRUID ? "mini-druid" : "mini-druid-kafka", logDir, tempDir, - setup.zkPort, Utilities.jarFinderGetJar(MiniDruidCluster.class)); - final Path druidDeepStorage = fs.makeQualified(new Path(druidCluster.getDeepStorageDir())); - fs.mkdirs(druidDeepStorage); - final Path scratchDir = - fs.makeQualified(new Path(QTestSystemProperties.getTempDir(), "druidStagingDir")); - fs.mkdirs(scratchDir); - conf.set("hive.druid.working.directory", scratchDir.toUri().getPath()); - druidCluster.init(conf); - druidCluster.start(); - } - - if (clusterType == MiniClusterType.KAFKA || clusterType == MiniClusterType.DRUID_KAFKA) { + if (clusterType == MiniClusterType.KAFKA) { kafkaCluster = new SingleNodeKafkaCluster("kafka", QTestSystemProperties.getTempDir() + "/kafka-cluster", setup.zkPort, clusterType == MiniClusterType.KAFKA ? 9093 : 9092); @@ -277,22 +250,20 @@ public void setup(QTestArguments testArgs, HiveConf conf, String scriptsDir, } int numTrackers = 2; if (EnumSet - .of(MiniClusterType.LLAP, MiniClusterType.LLAP_LOCAL, MiniClusterType.DRUID_LOCAL, - MiniClusterType.DRUID_KAFKA, MiniClusterType.DRUID, MiniClusterType.KAFKA) + .of(MiniClusterType.LLAP, MiniClusterType.LLAP_LOCAL, MiniClusterType.KAFKA) .contains(clusterType)) { llapCluster = LlapItUtils.startAndGetMiniLlapCluster(conf, setup.zooKeeperCluster, confDir); } if (EnumSet - .of(MiniClusterType.LLAP_LOCAL, MiniClusterType.TEZ_LOCAL, MiniClusterType.DRUID_LOCAL) + .of(MiniClusterType.LLAP_LOCAL, MiniClusterType.TEZ_LOCAL) .contains(clusterType)) { mr = shims.getLocalMiniTezCluster(conf, - clusterType == MiniClusterType.LLAP_LOCAL || clusterType == MiniClusterType.DRUID_LOCAL); + clusterType == MiniClusterType.LLAP_LOCAL); } else { mr = shims .getMiniTezCluster(conf, numTrackers, uriString, EnumSet - .of(MiniClusterType.LLAP, MiniClusterType.LLAP_LOCAL, - MiniClusterType.DRUID_KAFKA, MiniClusterType.DRUID, MiniClusterType.KAFKA) + .of(MiniClusterType.LLAP, MiniClusterType.LLAP_LOCAL, MiniClusterType.KAFKA) .contains(clusterType)); } } else if (clusterType == MiniClusterType.MR) { @@ -320,20 +291,6 @@ public void initConf(HiveConf conf) throws IOException { conf.set(confEntry.getKey(), clusterSpecificConf.get(confEntry.getKey())); } } - if (druidCluster != null) { - final Path druidDeepStorage = fs.makeQualified(new Path(druidCluster.getDeepStorageDir())); - fs.mkdirs(druidDeepStorage); - conf.set("hive.druid.storage.storageDirectory", druidDeepStorage.toUri().getPath()); - conf.set("hive.druid.metadata.db.type", "derby"); - conf.set("hive.druid.metadata.uri", druidCluster.getMetadataURI()); - conf.set("hive.druid.coordinator.address.default", druidCluster.getCoordinatorURI()); - conf.set("hive.druid.overlord.address.default", druidCluster.getOverlordURI()); - conf.set("hive.druid.broker.address.default", druidCluster.getBrokerURI()); - final Path scratchDir = - fs.makeQualified(new Path(QTestSystemProperties.getTempDir(), "druidStagingDir")); - fs.mkdirs(scratchDir); - conf.set("hive.druid.working.directory", scratchDir.toUri().getPath()); - } if (testArgs.isWithLlapIo() && (clusterType == MiniClusterType.NONE)) { LOG.info("initializing llap IO"); @@ -371,11 +328,6 @@ public void shutDown() throws Exception { SessionState.get().getTezSession().destroy(); } - if (druidCluster != null) { - druidCluster.stop(); - druidCluster = null; - } - if (kafkaCluster != null) { kafkaCluster.stop(); kafkaCluster = null; diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/SQLStdHiveAuthorizationValidatorForTest.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/SQLStdHiveAuthorizationValidatorForTest.java index e12bebd15423..1e4593f6a0e4 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/SQLStdHiveAuthorizationValidatorForTest.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/SQLStdHiveAuthorizationValidatorForTest.java @@ -208,10 +208,6 @@ public List applyRowFilterAndColumnMasking(HiveAuthzContext } else if (privObj.getObjectName().equals("masking_acid_no_masking") || privObj.getObjectName().startsWith("masking_acid_no_masking_n")) { // testing acid usage when no masking/filtering is present needRewritePrivObjs.add(privObj); - } else if (privObj.getObjectName().equals("masking_test_druid") || privObj.getObjectName().startsWith("masking_test_druid_n")) { - // testing druid queries row filtering is present - privObj.setRowFilterExpression("key > 10"); - needRewritePrivObjs.add(privObj); } } return needRewritePrivObjs; diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/service/AsyncTaskCopyAuxJars.java b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/service/AsyncTaskCopyAuxJars.java index f4bb23474ca8..d3904bad638f 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/service/AsyncTaskCopyAuxJars.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/service/AsyncTaskCopyAuxJars.java @@ -44,7 +44,7 @@ class AsyncTaskCopyAuxJars implements Callable { private static final Logger LOG = LoggerFactory.getLogger(AsyncTaskCopyAuxJars.class.getName()); private static final String[] DEFAULT_AUX_CLASSES = - new String[] {"org.apache.hive.hcatalog.data.JsonSerDe", "org.apache.hadoop.hive.druid.DruidStorageHandler", + new String[] {"org.apache.hive.hcatalog.data.JsonSerDe", "org.apache.hive.storage.jdbc.JdbcStorageHandler", "org.apache.commons.dbcp2.BasicDataSourceFactory", "org.apache.commons.pool2.impl.GenericObjectPool", "org.apache.hadoop.hive.kafka.KafkaStorageHandler", "org.apache.hadoop.hive.kudu.KuduStorageHandler"}; diff --git a/packaging/pom.xml b/packaging/pom.xml index df4d33309e31..ed0090ff64ba 100644 --- a/packaging/pom.xml +++ b/packaging/pom.xml @@ -353,11 +353,6 @@ hive-hbase-handler ${project.version} - - org.apache.hive - hive-druid-handler - ${project.version} - org.apache.hive hive-iceberg-handler diff --git a/packaging/src/main/assembly/src.xml b/packaging/src/main/assembly/src.xml index 4dc658e1871e..e5aba2e494fe 100644 --- a/packaging/src/main/assembly/src.xml +++ b/packaging/src/main/assembly/src.xml @@ -74,7 +74,6 @@ contrib/**/* data/**/* dev-support/**/* - druid-handler/**/* iceberg-catalog/**/* iceberg-handler/**/* jdbc-handler/**/* diff --git a/pom.xml b/pom.xml index b7471070382c..8b0b099ef590 100644 --- a/pom.xml +++ b/pom.xml @@ -35,7 +35,6 @@ cli common contrib - druid-handler hbase-handler jdbc-handler hcatalog @@ -134,7 +133,6 @@ 10.17.1.0 3.1.0 0.1.2 - 0.17.1 2.2.4 1.12.0 22.0 @@ -193,8 +191,6 @@ 5.2.0 2.0.0-M5 4.1.127.Final - 3.10.5.Final - 4.5.8 2.8 1.16.0 @@ -637,11 +633,6 @@ calcite-linq4j ${calcite.version} - - org.apache.calcite - calcite-druid - ${calcite.version} - org.apache.curator curator-test diff --git a/ql/pom.xml b/ql/pom.xml index 57672a607910..d8600856e6dd 100644 --- a/ql/pom.xml +++ b/ql/pom.xml @@ -482,16 +482,6 @@ - - org.apache.calcite - calcite-druid - - - org.yaml - snakeyaml - - - com.google.guava guava diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/create/CreateTableOperation.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/create/CreateTableOperation.java index 8c005c8476f6..75d894bda794 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/create/CreateTableOperation.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/create/CreateTableOperation.java @@ -194,7 +194,6 @@ public static boolean doesTableNeedLocation(Table tbl) { // TODO: why doesn't this check class name rather than toString? String sh = tbl.getStorageHandler().toString(); retval = !"org.apache.hadoop.hive.hbase.HBaseStorageHandler".equals(sh) && - !Constants.DRUID_HIVE_STORAGE_HANDLER_ID.equals(sh) && !Constants.JDBC_HIVE_STORAGE_HANDLER_ID.equals(sh) && !"org.apache.hadoop.hive.accumulo.AccumuloStorageHandler".equals(sh); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java index ee55fde100f6..153e4d7e2b33 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java @@ -19,9 +19,7 @@ import java.util.ArrayList; import java.util.Collection; -import java.util.Collections; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Set; import java.util.concurrent.Executors; @@ -31,25 +29,14 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; -import com.google.common.collect.ImmutableMap; -import org.apache.calcite.adapter.druid.DruidQuery; -import org.apache.calcite.adapter.druid.DruidSchema; -import org.apache.calcite.adapter.druid.DruidTable; -import org.apache.calcite.interpreter.BindableConvention; import org.apache.calcite.jdbc.JavaTypeFactoryImpl; import org.apache.calcite.plan.RelOptCluster; import org.apache.calcite.plan.RelOptMaterialization; import org.apache.calcite.plan.RelOptPlanner; import org.apache.calcite.rel.RelNode; -import org.apache.calcite.rel.core.TableScan; import org.apache.calcite.rel.type.RelDataType; -import org.apache.calcite.rel.type.RelDataTypeFactory; -import org.apache.calcite.rel.type.RelDataTypeField; -import org.apache.calcite.rel.type.RelDataTypeImpl; import org.apache.calcite.rex.RexBuilder; -import org.apache.calcite.sql.type.SqlTypeName; import org.apache.hadoop.hive.common.TableName; -import org.apache.hadoop.hive.conf.Constants; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.metastore.DefaultMetaStoreFilterHookImpl; @@ -79,13 +66,10 @@ import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; -import org.joda.time.Interval; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.google.common.util.concurrent.ThreadFactoryBuilder; -import com.google.common.collect.ImmutableList; - import static java.util.stream.Collectors.toList; import static org.apache.hadoop.hive.ql.metadata.RewriteAlgorithm.ALL; import static org.apache.hadoop.hive.ql.optimizer.calcite.rules.views.HiveMaterializedViewUtils.extractTable; @@ -450,88 +434,17 @@ private static RelNode createMaterializedViewScan(HiveConf conf, Table viewTable RelNode tableRel; - // 3. Build operator - if (obtainTableType(viewTable) == TableType.DRUID) { - // Build Druid query - String address = HiveConf.getVar(conf, - HiveConf.ConfVars.HIVE_DRUID_BROKER_DEFAULT_ADDRESS); - String dataSource = viewTable.getParameters().get(Constants.DRUID_DATA_SOURCE); - Set metrics = new HashSet<>(); - List druidColTypes = new ArrayList<>(); - List druidColNames = new ArrayList<>(); - //@NOTE this code is very similar to the code at org/apache/hadoop/hive/ql/parse/CalcitePlanner.java:2362 - //@TODO it will be nice to refactor it - RelDataTypeFactory dtFactory = cluster.getRexBuilder().getTypeFactory(); - for (RelDataTypeField field : rowType.getFieldList()) { - if (DruidTable.DEFAULT_TIMESTAMP_COLUMN.equals(field.getName())) { - // Druid's time column is always not null. - druidColTypes.add(dtFactory.createTypeWithNullability(field.getType(), false)); - } else { - druidColTypes.add(field.getType()); - } - druidColNames.add(field.getName()); - if (field.getName().equals(DruidTable.DEFAULT_TIMESTAMP_COLUMN)) { - // timestamp - continue; - } - if (field.getType().getSqlTypeName() == SqlTypeName.VARCHAR) { - // dimension - continue; - } - metrics.add(field.getName()); - } - - List intervals = Collections.singletonList(DruidTable.DEFAULT_INTERVAL); - rowType = dtFactory.createStructType(druidColTypes, druidColNames); - // We can pass null for Hive object because it is only used to retrieve tables - // if constraints on a table object are existing, but constraints cannot be defined - // for materialized views. - RelOptHiveTable optTable = new RelOptHiveTable(null, cluster.getTypeFactory(), fullyQualifiedTabName, - rowType, viewTable, nonPartitionColumns, partitionColumns, new ArrayList<>(), - conf, new QueryTables(true), new HashMap<>(), new HashMap<>(), new AtomicInteger()); - DruidTable druidTable = new DruidTable(new DruidSchema(address, address, false), - dataSource, RelDataTypeImpl.proto(rowType), metrics, DruidTable.DEFAULT_TIMESTAMP_COLUMN, - intervals, null, null); - final TableScan scan = new HiveTableScan(cluster, cluster.traitSetOf(HiveRelNode.CONVENTION), - optTable, viewTable.getTableName(), null, false, false); - tableRel = DruidQuery.create(cluster, cluster.traitSetOf(BindableConvention.INSTANCE), - optTable, druidTable, ImmutableList.of(scan), ImmutableMap.of()); - } else { - // Build Hive Table Scan Rel. - // We can pass null for Hive object because it is only used to retrieve tables - // if constraints on a table object are existing, but constraints cannot be defined - // for materialized views. - RelOptHiveTable optTable = new RelOptHiveTable(null, cluster.getTypeFactory(), fullyQualifiedTabName, - rowType, viewTable, nonPartitionColumns, partitionColumns, new ArrayList<>(), - conf, new QueryTables(true), new HashMap<>(), new HashMap<>(), new AtomicInteger()); - tableRel = new HiveTableScan(cluster, cluster.traitSetOf(HiveRelNode.CONVENTION), optTable, - viewTable.getTableName(), null, false, false); - } + // Build Hive Table Scan Rel. + // We can pass null for Hive object because it is only used to retrieve tables + // if constraints on a table object are existing, but constraints cannot be defined + // for materialized views. + RelOptHiveTable optTable = new RelOptHiveTable(null, cluster.getTypeFactory(), fullyQualifiedTabName, + rowType, viewTable, nonPartitionColumns, partitionColumns, new ArrayList<>(), + conf, new QueryTables(true), new HashMap<>(), new HashMap<>(), new AtomicInteger()); + tableRel = new HiveTableScan(cluster, cluster.traitSetOf(HiveRelNode.CONVENTION), optTable, + viewTable.getTableName(), null, false, false); return tableRel; } - private static TableType obtainTableType(Table tabMetaData) { - if (tabMetaData.getStorageHandler() != null) { - final String storageHandlerStr = tabMetaData.getStorageHandler().toString(); - if (storageHandlerStr.equals(Constants.DRUID_HIVE_STORAGE_HANDLER_ID)) { - return TableType.DRUID; - } - - if (storageHandlerStr.equals(Constants.JDBC_HIVE_STORAGE_HANDLER_ID)) { - return TableType.JDBC; - } - - } - - return TableType.NATIVE; - } - - //@TODO this seems to be the same as org.apache.hadoop.hive.ql.parse.CalcitePlanner.TableType.DRUID do we really need both - private enum TableType { - DRUID, - NATIVE, - JDBC - } - } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java index be36ed026677..c5e861b8b721 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java @@ -107,10 +107,6 @@ public void initialize(HiveConf hiveConf) { transformations.add(new ConstantPropagate()); } - - - transformations.add(new SortedDynPartitionTimeGranularityOptimizer()); - if (HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVE_OPT_PPD)) { transformations.add(new PartitionPruner()); transformations.add(new PartitionConditionRemover()); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionTimeGranularityOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionTimeGranularityOptimizer.java deleted file mode 100644 index 39c8b0a49d53..000000000000 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionTimeGranularityOptimizer.java +++ /dev/null @@ -1,479 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.ql.optimizer; - -import org.apache.commons.lang3.StringUtils; -import org.apache.hadoop.hive.conf.Constants; -import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.metastore.api.FieldSchema; -import org.apache.hadoop.hive.ql.QueryProperties.QueryFeature; -import org.apache.hadoop.hive.ql.exec.ColumnInfo; -import org.apache.hadoop.hive.ql.exec.FileSinkOperator; -import org.apache.hadoop.hive.ql.exec.Operator; -import org.apache.hadoop.hive.ql.exec.OperatorFactory; -import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator; -import org.apache.hadoop.hive.ql.exec.RowSchema; -import org.apache.hadoop.hive.ql.exec.SelectOperator; -import org.apache.hadoop.hive.ql.exec.UDF; -import org.apache.hadoop.hive.ql.exec.Utilities; -import org.apache.hadoop.hive.ql.exec.Utilities.ReduceField; -import org.apache.hadoop.hive.ql.io.AcidUtils; -import org.apache.hadoop.hive.ql.lib.DefaultGraphWalker; -import org.apache.hadoop.hive.ql.lib.DefaultRuleDispatcher; -import org.apache.hadoop.hive.ql.lib.SemanticDispatcher; -import org.apache.hadoop.hive.ql.lib.SemanticGraphWalker; -import org.apache.hadoop.hive.ql.lib.Node; -import org.apache.hadoop.hive.ql.lib.SemanticNodeProcessor; -import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx; -import org.apache.hadoop.hive.ql.lib.SemanticRule; -import org.apache.hadoop.hive.ql.lib.RuleRegExp; -import org.apache.hadoop.hive.ql.metadata.Table; -import org.apache.hadoop.hive.ql.parse.ParseContext; -import org.apache.hadoop.hive.ql.parse.SemanticException; -import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc; -import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc; -import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; -import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc; -import org.apache.hadoop.hive.ql.plan.FileSinkDesc; -import org.apache.hadoop.hive.ql.plan.OperatorDesc; -import org.apache.hadoop.hive.ql.plan.PlanUtils; -import org.apache.hadoop.hive.ql.plan.ReduceSinkDesc; -import org.apache.hadoop.hive.ql.plan.SelectDesc; -import org.apache.hadoop.hive.ql.plan.TableDesc; -import org.apache.hadoop.hive.ql.udf.UDFDateFloorDay; -import org.apache.hadoop.hive.ql.udf.UDFDateFloorHour; -import org.apache.hadoop.hive.ql.udf.UDFDateFloorMinute; -import org.apache.hadoop.hive.ql.udf.UDFDateFloorMonth; -import org.apache.hadoop.hive.ql.udf.UDFDateFloorSecond; -import org.apache.hadoop.hive.ql.udf.UDFDateFloorWeek; -import org.apache.hadoop.hive.ql.udf.UDFDateFloorYear; -import org.apache.hadoop.hive.ql.udf.UDFRand; -import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBridge; -import org.apache.hadoop.hive.ql.udf.generic.GenericUDFEpochMilli; -import org.apache.hadoop.hive.ql.udf.generic.GenericUDFTimestamp; -import org.apache.hadoop.hive.ql.udf.generic.GenericUDFFloor; -import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPDivide; -import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPMod; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; -import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; -import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; -import org.apache.parquet.Strings; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.Stack; -import java.util.stream.Collectors; - -/** - * Introduces a RS before FS to partition data by configuration specified - * time granularity. - */ -public class SortedDynPartitionTimeGranularityOptimizer extends Transform { - - @Override - public ParseContext transform(ParseContext pCtx) throws SemanticException { - // create a walker which walks the tree in a DFS manner while maintaining the - // operator stack. The dispatcher generates the plan from the operator tree - Map opRules = new LinkedHashMap(); - - String FS = FileSinkOperator.getOperatorName() + "%"; - - opRules.put(new RuleRegExp("Sorted Dynamic Partition Time Granularity", FS), getSortDynPartProc(pCtx)); - - SemanticDispatcher disp = new DefaultRuleDispatcher(null, opRules, null); - SemanticGraphWalker ogw = new DefaultGraphWalker(disp); - - ArrayList topNodes = new ArrayList(); - topNodes.addAll(pCtx.getTopOps().values()); - ogw.startWalking(topNodes, null); - - return pCtx; - } - - private SemanticNodeProcessor getSortDynPartProc(ParseContext pCtx) { - return new SortedDynamicPartitionProc(pCtx); - } - - class SortedDynamicPartitionProc implements SemanticNodeProcessor { - - private final Logger LOG = LoggerFactory.getLogger(SortedDynPartitionTimeGranularityOptimizer.class); - protected ParseContext parseCtx; - private int targetShardsPerGranularity = 0; - private int granularityKeyPos = -1; - private int partitionKeyPos = -1; - - public SortedDynamicPartitionProc(ParseContext pCtx) { - this.parseCtx = pCtx; - } - - @Override - public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, - Object... nodeOutputs) throws SemanticException { - - // introduce RS and EX before FS - FileSinkOperator fsOp = (FileSinkOperator) nd; - final String sh = fsOp.getConf().getTableInfo().getOutputFileFormatClassName(); - if (parseCtx.getQueryProperties().hasFeature(QueryFeature.QUERY) || sh == null || !sh - .equals(Constants.DRUID_HIVE_OUTPUT_FORMAT)) { - // Bail out, nothing to do - return null; - } - String segmentGranularity; - final String targetShardsProperty; - final Table table = fsOp.getConf().getTable(); - if (table != null) { - // case the statement is an INSERT - segmentGranularity = table.getParameters().get(Constants.DRUID_SEGMENT_GRANULARITY); - targetShardsProperty = - table.getParameters().getOrDefault(Constants.DRUID_TARGET_SHARDS_PER_GRANULARITY, "0"); - - } else if (parseCtx.getCreateViewDesc() != null) { - // case the statement is a CREATE MATERIALIZED VIEW AS - segmentGranularity = parseCtx.getCreateViewDesc().getTblProps() - .get(Constants.DRUID_SEGMENT_GRANULARITY); - targetShardsProperty = parseCtx.getCreateViewDesc().getTblProps() - .getOrDefault(Constants.DRUID_TARGET_SHARDS_PER_GRANULARITY, "0"); - } else if (parseCtx.getCreateTable() != null) { - // case the statement is a CREATE TABLE AS - segmentGranularity = parseCtx.getCreateTable().getTblProps() - .get(Constants.DRUID_SEGMENT_GRANULARITY); - targetShardsProperty = parseCtx.getCreateTable().getTblProps() - .getOrDefault(Constants.DRUID_TARGET_SHARDS_PER_GRANULARITY, "0"); - } else { - throw new SemanticException("Druid storage handler used but not an INSERT, " - + "CMVAS or CTAS statement"); - } - segmentGranularity = Strings.isNullOrEmpty(segmentGranularity) ? HiveConf - .getVar(parseCtx.getConf(), - HiveConf.ConfVars.HIVE_DRUID_INDEXING_GRANULARITY - ) : segmentGranularity; - targetShardsPerGranularity = Integer.parseInt(targetShardsProperty); - - LOG.info("Sorted dynamic partitioning on time granularity optimization kicked in..."); - - // unlink connection between FS and its parent - final Operator fsParent = fsOp.getParentOperators().get(0); - fsParent.getChildOperators().clear(); - - if (targetShardsPerGranularity > 0) { - partitionKeyPos = fsParent.getSchema().getSignature().size() + 1; - } - granularityKeyPos = fsParent.getSchema().getSignature().size(); - // Create SelectOp with granularity column - final Operator granularitySelOp = getGranularitySelOp(fsParent, - segmentGranularity - ); - - // Create ReduceSinkOp operator - final ArrayList parentCols = - Lists.newArrayList(granularitySelOp.getSchema().getSignature()); - final ArrayList allRSCols = Lists.newArrayList(); - for (ColumnInfo ci : parentCols) { - allRSCols.add(new ExprNodeColumnDesc(ci)); - } - // Get the key positions - final List keyPositions; - final List sortOrder; - final List sortNullOrder; - //Order matters, assuming later that __time_granularity comes first then __druidPartitionKey - if (targetShardsPerGranularity > 0) { - keyPositions = Lists.newArrayList(granularityKeyPos, partitionKeyPos); - sortOrder = Lists.newArrayList(1, 1); // asc - sortNullOrder = Lists.newArrayList(0, 0); // nulls first - } else { - keyPositions = Lists.newArrayList(granularityKeyPos); - sortOrder = Lists.newArrayList(1); // asc - sortNullOrder = Lists.newArrayList(0); // nulls first - } - ReduceSinkOperator rsOp = getReduceSinkOp(keyPositions, sortOrder, - sortNullOrder, allRSCols, granularitySelOp, fsOp.getConf().getWriteType()); - rsOp.getConf().setBucketingVersion(fsOp.getConf().getBucketingVersion()); - - // Create backtrack SelectOp - final List descs = new ArrayList<>(allRSCols.size()); - final List colNames = new ArrayList<>(); - for (int i = 0; i < allRSCols.size(); i++) { - ExprNodeDesc col = allRSCols.get(i); - final String colName = col.getExprString(); - colNames.add(colName); - if (keyPositions.contains(i)) { - descs.add( - new ExprNodeColumnDesc(col.getTypeInfo(), ReduceField.KEY.toString() + "." + colName, - null, false - )); - } else { - descs.add(new ExprNodeColumnDesc(col.getTypeInfo(), - ReduceField.VALUE.toString() + "." + colName, null, false - )); - } - } - RowSchema selRS = new RowSchema(granularitySelOp.getSchema()); - SelectDesc selConf = new SelectDesc(descs, colNames); - SelectOperator backtrackSelOp = (SelectOperator) OperatorFactory.getAndMakeChild( - selConf, selRS, rsOp); - - // Link backtrack SelectOp to FileSinkOp - fsOp.getParentOperators().clear(); - fsOp.getParentOperators().add(backtrackSelOp); - backtrackSelOp.getChildOperators().add(fsOp); - - // Update file sink descriptor - fsOp.getConf().setDpSortState(FileSinkDesc.DPSortState.PARTITION_SORTED); - fsOp.getConf().setPartitionCols(rsOp.getConf().getPartitionCols()); - final ColumnInfo granularityColumnInfo = - new ColumnInfo(granularitySelOp.getSchema().getSignature().get(granularityKeyPos)); - fsOp.getSchema().getSignature().add(granularityColumnInfo); - if (targetShardsPerGranularity > 0) { - final ColumnInfo partitionKeyColumnInfo = - new ColumnInfo(granularitySelOp.getSchema().getSignature().get(partitionKeyPos)); - fsOp.getSchema().getSignature().add(partitionKeyColumnInfo); - } - - LOG.info("Inserted " + granularitySelOp.getOperatorId() + ", " + rsOp.getOperatorId() + " and " - + backtrackSelOp.getOperatorId() + " as parent of " + fsOp.getOperatorId() - + " and child of " + fsParent.getOperatorId()); - parseCtx.setReduceSinkAddedBySortedDynPartition(true); - return null; - } - - private Operator getGranularitySelOp( - Operator fsParent, - String segmentGranularity - ) throws SemanticException { - final ArrayList parentCols = - Lists.newArrayList(fsParent.getSchema().getSignature()); - final ArrayList descs = Lists.newArrayList(); - final List colNames = Lists.newArrayList(); - PrimitiveCategory timestampType = null; - int timestampPos = -1; - for (int i = 0; i < parentCols.size(); i++) { - ColumnInfo ci = parentCols.get(i); - ExprNodeColumnDesc columnDesc = new ExprNodeColumnDesc(ci); - descs.add(columnDesc); - colNames.add(columnDesc.getExprString()); - if (columnDesc.getTypeInfo().getCategory() == ObjectInspector.Category.PRIMITIVE - && (((PrimitiveTypeInfo) columnDesc.getTypeInfo()).getPrimitiveCategory() == PrimitiveCategory.TIMESTAMP || - ((PrimitiveTypeInfo) columnDesc.getTypeInfo()).getPrimitiveCategory() == PrimitiveCategory.TIMESTAMPLOCALTZ)) { - if (timestampPos != -1) { - throw new SemanticException("Multiple columns with timestamp/timestamp with local time-zone type on query result; " - + "could not resolve which one is the right column"); - } - timestampType = ((PrimitiveTypeInfo) columnDesc.getTypeInfo()).getPrimitiveCategory(); - timestampPos = i; - } - } - if (timestampPos == -1) { - throw new SemanticException("No column with timestamp with local time-zone type on query result; " - + "one column should be of timestamp with local time-zone type"); - } - final RowSchema selRS = new RowSchema(fsParent.getSchema()); - // Granularity (partition) column - final String udfName; - Class udfClass; - switch (segmentGranularity) { - case "YEAR": - udfName = "floor_year"; - udfClass = UDFDateFloorYear.class; - break; - case "MONTH": - udfName = "floor_month"; - udfClass = UDFDateFloorMonth.class; - break; - case "WEEK": - udfName = "floor_week"; - udfClass = UDFDateFloorWeek.class; - break; - case "DAY": - udfName = "floor_day"; - udfClass = UDFDateFloorDay.class; - break; - case "HOUR": - udfName = "floor_hour"; - udfClass = UDFDateFloorHour.class; - break; - case "MINUTE": - udfName = "floor_minute"; - udfClass = UDFDateFloorMinute.class; - break; - case "SECOND": - udfName = "floor_second"; - udfClass = UDFDateFloorSecond.class; - break; - default: - throw new SemanticException(String.format(Locale.ENGLISH, - "Unknown Druid Granularity [%s], Accepted values are [YEAR, MONTH, WEEK, DAY, HOUR, MINUTE, SECOND]", - segmentGranularity - )); - } - - - // Timestamp column type in Druid is either timestamp or timestamp with local time-zone, i.e., - // a specific instant in time. Thus, for the latest, we have this value and we need to extract the - // granularity to split the data when we are storing it in Druid. However, Druid stores - // the data in UTC. Thus, we need to apply the following logic on the data to extract - // the granularity correctly: - // 1) Read the timestamp with local time-zone value. - // 2) Extract UTC epoch (millis) from timestamp with local time-zone. - // 3) Cast the long to a timestamp. - // 4) Apply the granularity function on the timestamp value. - // That way, '2010-01-01 00:00:00 UTC' and '2009-12-31 16:00:00 PST' (same instant) - // will end up in the same Druid segment. - - // #1 - Read the column value - ExprNodeDesc expr = new ExprNodeColumnDesc(parentCols.get(timestampPos)); - if (timestampType == PrimitiveCategory.TIMESTAMPLOCALTZ) { - // #2 - UTC epoch for instant - expr = new ExprNodeGenericFuncDesc( - TypeInfoFactory.longTypeInfo, new GenericUDFEpochMilli(), Lists.newArrayList(expr)); - // #3 - Cast to timestamp - expr = new ExprNodeGenericFuncDesc( - TypeInfoFactory.timestampTypeInfo, new GenericUDFTimestamp(), Lists.newArrayList(expr)); - } - // #4 - We apply the granularity function - expr = new ExprNodeGenericFuncDesc( - TypeInfoFactory.timestampTypeInfo, - new GenericUDFBridge(udfName, false, udfClass.getName()), - Lists.newArrayList(expr)); - descs.add(expr); - colNames.add(Constants.DRUID_TIMESTAMP_GRANULARITY_COL_NAME); - // Add granularity to the row schema - final ColumnInfo ci = new ColumnInfo(Constants.DRUID_TIMESTAMP_GRANULARITY_COL_NAME, TypeInfoFactory.timestampTypeInfo, - selRS.getSignature().get(0).getTabAlias(), false, false); - selRS.getSignature().add(ci); - if (targetShardsPerGranularity > 0 ) { - // add another partitioning key based on floor(1/rand) % targetShardsPerGranularity - final ColumnInfo partitionKeyCi = - new ColumnInfo(Constants.DRUID_SHARD_KEY_COL_NAME, TypeInfoFactory.longTypeInfo, - selRS.getSignature().get(0).getTabAlias(), false, false - ); - final ExprNodeDesc targetNumShardDescNode = - new ExprNodeConstantDesc(TypeInfoFactory.intTypeInfo, targetShardsPerGranularity); - final ExprNodeGenericFuncDesc randomFn = ExprNodeGenericFuncDesc - .newInstance(new GenericUDFBridge("rand", false, UDFRand.class.getName()), - Lists.newArrayList() - ); - - final ExprNodeGenericFuncDesc random = ExprNodeGenericFuncDesc.newInstance( - new GenericUDFFloor(), Lists.newArrayList(ExprNodeGenericFuncDesc - .newInstance(new GenericUDFOPDivide(), - Lists.newArrayList(new ExprNodeConstantDesc(TypeInfoFactory.doubleTypeInfo, 1.0), randomFn) - ))); - final ExprNodeGenericFuncDesc randModMax = ExprNodeGenericFuncDesc - .newInstance(new GenericUDFOPMod(), - Lists.newArrayList(random, targetNumShardDescNode) - ); - descs.add(randModMax); - colNames.add(Constants.DRUID_SHARD_KEY_COL_NAME); - selRS.getSignature().add(partitionKeyCi); - } - // Create SelectDesc - final SelectDesc selConf = new SelectDesc(descs, colNames); - // Create Select Operator - final SelectOperator selOp = (SelectOperator) OperatorFactory.getAndMakeChild( - selConf, selRS, fsParent); - - return selOp; - } - - private ReduceSinkOperator getReduceSinkOp(List keyPositions, List sortOrder, - List sortNullOrder, ArrayList allCols, Operator parent, - AcidUtils.Operation writeType) { - // we will clone here as RS will update bucket column key with its - // corresponding with bucket number and hence their OIs - final ArrayList keyCols = keyPositions.stream() - .map(id -> allCols.get(id).clone()) - .collect(Collectors.toCollection(ArrayList::new)); - ArrayList valCols = Lists.newArrayList(); - for (int i = 0; i < allCols.size(); i++) { - if (i != granularityKeyPos && i != partitionKeyPos) { - valCols.add(allCols.get(i).clone()); - } - } - - final ArrayList partCols = - keyPositions.stream().map(id -> allCols.get(id).clone()) - .collect(Collectors.toCollection(ArrayList::new)); - - // map _col0 to KEY._col0, etc - Map colExprMap = Maps.newHashMap(); - Map nameMapping = new HashMap<>(); - final ArrayList keyColNames = Lists.newArrayList(); - final ArrayList valColNames = Lists.newArrayList(); - for (ExprNodeDesc exprNodeDesc : keyCols) { - keyColNames.add(exprNodeDesc.getExprString()); - String key = Utilities.ReduceField.KEY + "." + exprNodeDesc.getExprString(); - colExprMap.put(key, exprNodeDesc); - nameMapping.put(exprNodeDesc.getExprString(), key); - } - for (ExprNodeDesc exprNodeDesc : valCols) { - valColNames.add(exprNodeDesc.getExprString()); - String key = Utilities.ReduceField.VALUE + "." + exprNodeDesc.getExprString(); - colExprMap.put(key, exprNodeDesc); - nameMapping.put(exprNodeDesc.getExprString(), key); - } - - - // order and null order - final String orderStr = StringUtils.repeat("+", sortOrder.size()); - final String nullOrderStr = StringUtils.repeat("a", sortNullOrder.size()); - - // Create Key/Value TableDesc. When the operator plan is split into MR tasks, - // the reduce operator will initialize Extract operator with information - // from Key and Value TableDesc - final List fields = PlanUtils.getFieldSchemasFromColumnList(keyCols, - keyColNames, 0, ""); - final TableDesc keyTable = PlanUtils.getReduceKeyTableDesc(fields, orderStr, nullOrderStr); - List valFields = PlanUtils.getFieldSchemasFromColumnList(valCols, - valColNames, 0, ""); - final TableDesc valueTable = PlanUtils.getReduceValueTableDesc(valFields); - List> distinctColumnIndices = Lists.newArrayList(); - - // Number of reducers is set to default (-1) - final ReduceSinkDesc rsConf = new ReduceSinkDesc(keyCols, keyCols.size(), valCols, - keyColNames, distinctColumnIndices, valColNames, -1, partCols, -1, keyTable, - valueTable, writeType); - - final ArrayList signature = - parent.getSchema().getSignature() - .stream() - .map(e -> new ColumnInfo(e)) - .map(columnInfo -> - { - columnInfo.setInternalName(nameMapping.get(columnInfo.getInternalName())); - return columnInfo; - }) - .collect(Collectors.toCollection(ArrayList::new)); - final ReduceSinkOperator op = (ReduceSinkOperator) OperatorFactory.getAndMakeChild( - rsConf, new RowSchema(signature), parent); - op.setColumnExprMap(colExprMap); - return op; - } - - } - -} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveVolcanoPlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveVolcanoPlanner.java index 9913ffb2e478..f16c74ec10dd 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveVolcanoPlanner.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveVolcanoPlanner.java @@ -19,7 +19,6 @@ package org.apache.hadoop.hive.ql.optimizer.calcite.cost; import com.google.common.collect.Multimap; -import org.apache.calcite.adapter.druid.DruidQuery; import org.apache.calcite.plan.Convention; import org.apache.calcite.plan.ConventionTraitDef; import org.apache.calcite.plan.RelOptCost; @@ -35,7 +34,6 @@ import org.apache.hadoop.hive.ql.optimizer.calcite.HiveConfPlannerContext; import org.apache.hadoop.hive.ql.optimizer.calcite.HivePlannerContext; import org.apache.hadoop.hive.ql.optimizer.calcite.RelOptHiveTable; -import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveDruidRules; /** * Refinement of {@link org.apache.calcite.plan.volcano.VolcanoPlanner} for Hive. @@ -69,27 +67,6 @@ public static RelOptPlanner createPlanner(HivePlannerContext conf) { return planner; } - @Override - public void registerClass(RelNode node) { - if (node instanceof DruidQuery) { - // Special handling for Druid rules here as otherwise - // planner will add Druid rules with logical builder - addRule(HiveDruidRules.FILTER); - addRule(HiveDruidRules.PROJECT_FILTER_TRANSPOSE); - addRule(HiveDruidRules.AGGREGATE_FILTER_TRANSPOSE); - addRule(HiveDruidRules.AGGREGATE_PROJECT); - addRule(HiveDruidRules.PROJECT); - addRule(HiveDruidRules.AGGREGATE); - addRule(HiveDruidRules.POST_AGGREGATION_PROJECT); - addRule(HiveDruidRules.FILTER_AGGREGATE_TRANSPOSE); - addRule(HiveDruidRules.FILTER_PROJECT_TRANSPOSE); - addRule(HiveDruidRules.SORT_PROJECT_TRANSPOSE); - addRule(HiveDruidRules.SORT); - return; - } - super.registerClass(node); - } - /** * The method extends the logic of the super method to decrease * the cost of the plan if it contains materialized views diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveDruidRules.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveDruidRules.java deleted file mode 100644 index c8a3c205b40a..000000000000 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveDruidRules.java +++ /dev/null @@ -1,304 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.ql.optimizer.calcite.rules; - -import org.apache.calcite.adapter.druid.DruidQuery; -import org.apache.calcite.adapter.druid.DruidRules; -import org.apache.calcite.adapter.druid.DruidRules.DruidAggregateProjectRule; -import org.apache.calcite.adapter.druid.DruidRules.DruidAggregateRule; -import org.apache.calcite.adapter.druid.DruidRules.DruidFilterRule; -import org.apache.calcite.adapter.druid.DruidRules.DruidHavingFilterRule; -import org.apache.calcite.adapter.druid.DruidRules.DruidPostAggregationProjectRule; -import org.apache.calcite.adapter.druid.DruidRules.DruidProjectRule; -import org.apache.calcite.adapter.druid.DruidRules.DruidSortRule; -import org.apache.calcite.plan.RelOptRule; -import org.apache.calcite.plan.RelOptRuleCall; -import org.apache.calcite.rel.core.Aggregate; -import org.apache.calcite.rel.core.AggregateCall; -import org.apache.calcite.rel.rules.DateRangeRules; -import org.apache.calcite.sql.SqlKind; -import org.apache.calcite.sql.fun.SqlSumEmptyIsZeroAggFunction; -import org.apache.calcite.tools.RelBuilder; -import org.apache.calcite.tools.RelBuilderFactory; -import org.apache.calcite.util.ImmutableBitSet; -import org.apache.calcite.util.Pair; -import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelFactories; - -import com.google.common.base.Preconditions; -import com.google.common.collect.Lists; - -import java.util.ArrayList; -import java.util.HashSet; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Set; -import java.util.SortedSet; -import java.util.TreeSet; - -/** - * Druid rules with Hive builder factory. - */ -public class HiveDruidRules { - - public static final RelOptRule FILTER = - DruidFilterRule.DruidFilterRuleConfig.DEFAULT.withRelBuilderFactory(HiveRelFactories.HIVE_BUILDER).toRule(); - - public static final RelOptRule PROJECT = - DruidProjectRule.DruidProjectRuleConfig.DEFAULT.withRelBuilderFactory(HiveRelFactories.HIVE_BUILDER).toRule(); - - public static final RelOptRule AGGREGATE = - DruidAggregateRule.DruidAggregateRuleConfig.DEFAULT.withRelBuilderFactory(HiveRelFactories.HIVE_BUILDER).toRule(); - - public static final RelOptRule AGGREGATE_PROJECT = - DruidAggregateProjectRule.DruidAggregateProjectRuleConfig.DEFAULT.withRelBuilderFactory( - HiveRelFactories.HIVE_BUILDER).toRule(); - - public static final RelOptRule SORT = - DruidSortRule.DruidSortRuleConfig.DEFAULT.withRelBuilderFactory(HiveRelFactories.HIVE_BUILDER).toRule(); - - public static final RelOptRule SORT_PROJECT_TRANSPOSE = - DruidRules.SORT_PROJECT_TRANSPOSE.config.withRelBuilderFactory(HiveRelFactories.HIVE_BUILDER).toRule(); - - public static final RelOptRule PROJECT_FILTER_TRANSPOSE = - DruidRules.PROJECT_FILTER_TRANSPOSE.config.withRelBuilderFactory(HiveRelFactories.HIVE_BUILDER).toRule(); - - public static final RelOptRule FILTER_PROJECT_TRANSPOSE = - DruidRules.FILTER_PROJECT_TRANSPOSE.config.withRelBuilderFactory(HiveRelFactories.HIVE_BUILDER).toRule(); - - public static final RelOptRule AGGREGATE_FILTER_TRANSPOSE = - DruidRules.AGGREGATE_FILTER_TRANSPOSE.config.withRelBuilderFactory(HiveRelFactories.HIVE_BUILDER).toRule(); - - public static final RelOptRule FILTER_AGGREGATE_TRANSPOSE = - DruidRules.FILTER_AGGREGATE_TRANSPOSE.config.withRelBuilderFactory(HiveRelFactories.HIVE_BUILDER).toRule(); - - public static final RelOptRule POST_AGGREGATION_PROJECT = - DruidPostAggregationProjectRule.DruidPostAggregationProjectRuleConfig.DEFAULT.withRelBuilderFactory( - HiveRelFactories.HIVE_BUILDER).toRule(); - - public static final RelOptRule HAVING_FILTER_RULE = - DruidHavingFilterRule.DruidHavingFilterRuleConfig.DEFAULT.withRelBuilderFactory(HiveRelFactories.HIVE_BUILDER) - .toRule(); - - public static final AggregateExpandDistinctAggregatesDruidRule EXPAND_SINGLE_DISTINCT_AGGREGATES_DRUID_RULE = - new AggregateExpandDistinctAggregatesDruidRule(HiveRelFactories.HIVE_BUILDER); - - public static final DateRangeRules.FilterDateRangeRule FILTER_DATE_RANGE_RULE = - new DateRangeRules.FilterDateRangeRule(HiveRelFactories.HIVE_BUILDER); - - /** - * This is a simplified version of {@link org.apache.calcite.rel.rules.AggregateExpandDistinctAggregatesRule} - * The goal of this simplified version is to help pushing single count distinct as multi-phase aggregates. - * This is an okay solution before we actually support grouping sets push-down to Druid. - * We are limiting it to one Distinct count to avoid expensive cross join and running into issue - * https://issues.apache.org/jira/browse/HIVE-19601 - */ - public static class AggregateExpandDistinctAggregatesDruidRule extends RelOptRule { - - public AggregateExpandDistinctAggregatesDruidRule(RelBuilderFactory relBuilderFactory) { - super(operand(Aggregate.class, operand(DruidQuery.class, none())), relBuilderFactory, - null - ); - } - @Override public void onMatch(RelOptRuleCall call) { - Aggregate aggregate = call.rel(0); - if (!aggregate.containsDistinctCall()) { - return; - } - final long numCountDistinct = aggregate.getAggCallList() - .stream() - .filter(aggregateCall -> aggregateCall.getAggregation().getKind().equals(SqlKind.COUNT) && - aggregateCall.isDistinct()) - .count(); - if (numCountDistinct != 1) { - return; - } - - // Find all of the agg expressions. We use a LinkedHashSet to ensure determinism. - int nonDistinctAggCallCount = 0; // find all aggregate calls without distinct - int filterCount = 0; - int unsupportedNonDistinctAggCallCount = 0; - final Set, Integer>> argLists = new LinkedHashSet<>(); - for (AggregateCall aggCall : aggregate.getAggCallList()) { - if (aggCall.filterArg >= 0) { - ++filterCount; - } - if (!aggCall.isDistinct()) { - ++nonDistinctAggCallCount; - final SqlKind aggCallKind = aggCall.getAggregation().getKind(); - // We only support COUNT/SUM/MIN/MAX for the "single" count distinct optimization - switch (aggCallKind) { - case COUNT: - case SUM: - case SUM0: - case MIN: - case MAX: - break; - default: - ++unsupportedNonDistinctAggCallCount; - } - } else { - argLists.add(Pair.of(aggCall.getArgList(), aggCall.filterArg)); - } - } - // If only one distinct aggregate and one or more non-distinct aggregates, - // we can generate multi-phase aggregates - if (numCountDistinct == 1 // one distinct aggregate - && filterCount == 0 // no filter - && unsupportedNonDistinctAggCallCount == 0 // sum/min/max/count in non-distinct aggregate - && nonDistinctAggCallCount > 0) { // one or more non-distinct aggregates - final RelBuilder relBuilder = call.builder(); - convertSingletonDistinct(relBuilder, aggregate, argLists); - call.transformTo(relBuilder.build()); - return; - } - } - - /** - * Converts an aggregate with one distinct aggregate and one or more - * non-distinct aggregates to multi-phase aggregates (see reference example - * below). - * - * @param relBuilder Contains the input relational expression - * @param aggregate Original aggregate - * @param argLists Arguments and filters to the distinct aggregate function - * - */ - private RelBuilder convertSingletonDistinct(RelBuilder relBuilder, - Aggregate aggregate, Set, Integer>> argLists) { - - // In this case, we are assuming that there is a single distinct function. - // So make sure that argLists is of size one. - Preconditions.checkArgument(argLists.size() == 1); - - // For example, - // SELECT deptno, COUNT(*), SUM(bonus), MIN(DISTINCT sal) - // FROM emp - // GROUP BY deptno - // - // becomes - // - // SELECT deptno, SUM(cnt), SUM(bonus), MIN(sal) - // FROM ( - // SELECT deptno, COUNT(*) as cnt, SUM(bonus), sal - // FROM EMP - // GROUP BY deptno, sal) // Aggregate B - // GROUP BY deptno // Aggregate A - relBuilder.push(aggregate.getInput()); - - final List originalAggCalls = aggregate.getAggCallList(); - final ImmutableBitSet originalGroupSet = aggregate.getGroupSet(); - - // Add the distinct aggregate column(s) to the group-by columns, - // if not already a part of the group-by - final SortedSet bottomGroupSet = new TreeSet<>(); - bottomGroupSet.addAll(aggregate.getGroupSet().asList()); - for (AggregateCall aggCall : originalAggCalls) { - if (aggCall.isDistinct()) { - bottomGroupSet.addAll(aggCall.getArgList()); - break; // since we only have single distinct call - } - } - - // Generate the intermediate aggregate B, the one on the bottom that converts - // a distinct call to group by call. - // Bottom aggregate is the same as the original aggregate, except that - // the bottom aggregate has converted the DISTINCT aggregate to a group by clause. - final List bottomAggregateCalls = new ArrayList<>(); - for (AggregateCall aggCall : originalAggCalls) { - // Project the column corresponding to the distinct aggregate. Project - // as-is all the non-distinct aggregates - if (!aggCall.isDistinct()) { - final AggregateCall newCall = - AggregateCall.create(aggCall.getAggregation(), false, - aggCall.isApproximate(), aggCall.getArgList(), -1, - ImmutableBitSet.of(bottomGroupSet).cardinality(), - relBuilder.peek(), null, aggCall.name); - bottomAggregateCalls.add(newCall); - } - } - // Generate the aggregate B (see the reference example above) - relBuilder.push( - aggregate.copy( - aggregate.getTraitSet(), relBuilder.build(), - false, ImmutableBitSet.of(bottomGroupSet), null, bottomAggregateCalls)); - - // Add aggregate A (see the reference example above), the top aggregate - // to handle the rest of the aggregation that the bottom aggregate hasn't handled - final List topAggregateCalls = Lists.newArrayList(); - // Use the remapped arguments for the (non)distinct aggregate calls - int nonDistinctAggCallProcessedSoFar = 0; - for (AggregateCall aggCall : originalAggCalls) { - final AggregateCall newCall; - if (aggCall.isDistinct()) { - List newArgList = new ArrayList<>(); - for (int arg : aggCall.getArgList()) { - newArgList.add(bottomGroupSet.headSet(arg).size()); - } - newCall = - AggregateCall.create(aggCall.getAggregation(), - false, - aggCall.isApproximate(), - newArgList, - -1, - originalGroupSet.cardinality(), - relBuilder.peek(), - aggCall.getType(), - aggCall.name); - } else { - // If aggregate B had a COUNT aggregate call the corresponding aggregate at - // aggregate A must be SUM. For other aggregates, it remains the same. - final List newArgs = - Lists.newArrayList(bottomGroupSet.size() + nonDistinctAggCallProcessedSoFar); - if (aggCall.getAggregation().getKind() == SqlKind.COUNT) { - newCall = - AggregateCall.create(new SqlSumEmptyIsZeroAggFunction(), false, - aggCall.isApproximate(), newArgs, -1, - originalGroupSet.cardinality(), relBuilder.peek(), - aggCall.getType(), aggCall.getName()); - } else { - newCall = - AggregateCall.create(aggCall.getAggregation(), false, - aggCall.isApproximate(), newArgs, -1, - originalGroupSet.cardinality(), - relBuilder.peek(), aggCall.getType(), aggCall.name); - } - nonDistinctAggCallProcessedSoFar++; - } - - topAggregateCalls.add(newCall); - } - - // Populate the group-by keys with the remapped arguments for aggregate A - // The top groupset is basically an identity (first X fields of aggregate B's - // output), minus the distinct aggCall's input. - final Set topGroupSet = new HashSet<>(); - int groupSetToAdd = 0; - for (int bottomGroup : bottomGroupSet) { - if (originalGroupSet.get(bottomGroup)) { - topGroupSet.add(groupSetToAdd); - } - groupSetToAdd++; - } - relBuilder.push( - aggregate.copy(aggregate.getTraitSet(), - relBuilder.build(), aggregate.indicator, - ImmutableBitSet.of(topGroupSet), null, topAggregateCalls)); - return relBuilder; - } - } -} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterProjectTransposeRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterProjectTransposeRule.java index 0f334b335501..1212be6b74b0 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterProjectTransposeRule.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterProjectTransposeRule.java @@ -25,7 +25,6 @@ import java.util.Set; import java.util.function.Predicate; -import org.apache.calcite.adapter.druid.DruidQuery; import org.apache.calcite.plan.RelOptCluster; import org.apache.calcite.plan.RelOptPredicateList; import org.apache.calcite.plan.RelOptRuleCall; @@ -104,13 +103,6 @@ private static RelOptRuleOperand joinOperand() { false, ProjectMergeRule.DEFAULT_BLOAT); - public static final HiveFilterProjectTransposeRule DRUID = new HiveFilterProjectTransposeRule( - operand(Filter.class, operand(HiveProject.class, operand(DruidQuery.class, none()))), - HiveRelFactories.HIVE_BUILDER, - false, - false, - ProjectMergeRule.DEFAULT_BLOAT); - private final boolean onlyDeterministic; private final boolean pushThroughWindowing; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveRelFieldTrimmer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveRelFieldTrimmer.java index f6f011fe5823..17ab61d575cc 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveRelFieldTrimmer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveRelFieldTrimmer.java @@ -27,7 +27,6 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; -import org.apache.calcite.adapter.druid.DruidQuery; import org.apache.calcite.plan.RelOptTable; import org.apache.calcite.plan.RelOptUtil; import org.apache.calcite.rel.RelCollation; @@ -324,74 +323,6 @@ public TrimResult trimFields( return new TrimResult(newJoin, mapping); } - /** - * Variant of {@link #trimFields(RelNode, ImmutableBitSet, Set)} for - * {@link org.apache.calcite.adapter.druid.DruidQuery}. - */ - public TrimResult trimFields(DruidQuery dq, ImmutableBitSet fieldsUsed, - Set extraFields) { - final int fieldCount = dq.getRowType().getFieldCount(); - if (fieldsUsed.equals(ImmutableBitSet.range(fieldCount)) - && extraFields.isEmpty()) { - // if there is nothing to project or if we are projecting everything - // then no need to introduce another RelNode - return trimFields( - (RelNode) dq, fieldsUsed, extraFields); - } - final RelNode newTableAccessRel = project(dq, fieldsUsed, extraFields, REL_BUILDER.get()); - - // Some parts of the system can't handle rows with zero fields, so - // pretend that one field is used. - if (fieldsUsed.cardinality() == 0) { - RelNode input = newTableAccessRel; - if (input instanceof Project) { - // The table has implemented the project in the obvious way - by - // creating project with 0 fields. Strip it away, and create our own - // project with one field. - Project project = (Project) input; - if (project.getRowType().getFieldCount() == 0) { - input = project.getInput(); - } - } - return dummyProject(fieldCount, input); - } - - final Mapping mapping = createMapping(fieldsUsed, fieldCount); - return result(newTableAccessRel, mapping); - } - - private static RelNode project(DruidQuery dq, ImmutableBitSet fieldsUsed, - Set extraFields, RelBuilder relBuilder) { - final int fieldCount = dq.getRowType().getFieldCount(); - if (fieldsUsed.equals(ImmutableBitSet.range(fieldCount)) - && extraFields.isEmpty()) { - return dq; - } - final List exprList = new ArrayList<>(); - final List nameList = new ArrayList<>(); - final RexBuilder rexBuilder = dq.getCluster().getRexBuilder(); - final List fields = dq.getRowType().getFieldList(); - - // Project the subset of fields. - for (int i : fieldsUsed) { - RelDataTypeField field = fields.get(i); - exprList.add(rexBuilder.makeInputRef(dq, i)); - nameList.add(field.getName()); - } - - // Project nulls for the extra fields. (Maybe a sub-class table has - // extra fields, but we don't.) - for (RelDataTypeField extraField : extraFields) { - exprList.add( - rexBuilder.ensureType( - extraField.getType(), - rexBuilder.constantNull(), - true)); - nameList.add(extraField.getName()); - } - return relBuilder.push(dq).project(exprList, nameList).build(); - } - private boolean isRexLiteral(final RexNode rexNode) { if(rexNode instanceof RexLiteral) { return true; @@ -545,7 +476,7 @@ private ImmutableBitSet generateNewGroupset(Aggregate aggregate, ImmutableBitSet * * This is mainly done so that hive is able to push down queries with * group by 'constant key with type not supported by druid' into druid. - * + * TODO: If relevant only for Druid the review for removal */ private Aggregate rewriteGBConstantKeys(Aggregate aggregate, ImmutableBitSet fieldsUsed, ImmutableBitSet aggCallFields) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/views/HiveMaterializedViewUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/views/HiveMaterializedViewUtils.java index d3b92d585694..0fe21dfdae1a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/views/HiveMaterializedViewUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/views/HiveMaterializedViewUtils.java @@ -32,8 +32,6 @@ import java.util.function.Supplier; import java.util.stream.Collectors; -import org.apache.calcite.adapter.druid.DruidQuery; -import org.apache.calcite.interpreter.BindableConvention; import org.apache.calcite.plan.RelOptCluster; import org.apache.calcite.plan.RelOptMaterialization; import org.apache.calcite.plan.RelOptRule; @@ -82,7 +80,6 @@ import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveRelNode; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan; import org.apache.hadoop.hive.ql.optimizer.calcite.translator.TypeConverter; -import org.apache.hadoop.hive.ql.parse.DruidSqlOperatorConverter; import org.apache.hadoop.hive.ql.parse.QueryTables; import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveAccessControlException; import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveAuthzContext; @@ -502,24 +499,9 @@ public static boolean checkPrivilegeForMaterializedViews(List cachedMVTab } private static RelNode copyNodeScanNewCluster(RelOptCluster optCluster, RelNode scan) { - final RelNode newScan; - if (scan instanceof DruidQuery) { - final DruidQuery dq = (DruidQuery) scan; - // Ideally we should use HiveRelNode convention. However, since Volcano planner - // throws in that case because DruidQuery does not implement the interface, - // we set it as Bindable. Currently, we do not use convention in Hive, hence that - // should be fine. - // TODO: If we want to make use of convention (e.g., while directly generating operator - // tree instead of AST), this should be changed. - newScan = DruidQuery.create(optCluster, optCluster.traitSetOf(BindableConvention.INSTANCE), - scan.getTable(), dq.getDruidTable(), ImmutableList.of(dq.getTableScan()), - DruidSqlOperatorConverter.getDefaultMap()); - } else { - newScan = new HiveTableScan(optCluster, optCluster.traitSetOf(HiveRelNode.CONVENTION), + return new HiveTableScan(optCluster, optCluster.traitSetOf(HiveRelNode.CONVENTION), (RelOptHiveTable) scan.getTable(), ((RelOptHiveTable) scan.getTable()).getName(), null, false, false); - } - return newScan; } public static MaterializationSnapshot getSnapshotOf(DDLOperationContext context, Set tables) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTBuilder.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTBuilder.java index 12a0bcaffe08..0964b813da64 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTBuilder.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTBuilder.java @@ -20,7 +20,6 @@ import java.math.BigDecimal; import java.util.stream.Collectors; -import org.apache.calcite.adapter.druid.DruidQuery; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.core.JoinRelType; import org.apache.calcite.rex.RexDynamicParam; @@ -69,8 +68,6 @@ public static ASTNode table(final RelNode scan) { HiveTableScan hts = null; if (scan instanceof HiveJdbcConverter) { hts = ((HiveJdbcConverter) scan).getTableScan().getHiveTableScan(); - } else if (scan instanceof DruidQuery) { - hts = (HiveTableScan) ((DruidQuery) scan).getTableScan(); } else { hts = (HiveTableScan) scan; } @@ -110,33 +107,7 @@ public static ASTNode table(final RelNode scan) { } ASTBuilder propList = ASTBuilder.construct(HiveParser.TOK_TABLEPROPLIST, "TOK_TABLEPROPLIST"); - if (scan instanceof DruidQuery) { - //Passing query spec, column names and column types to be used as part of Hive Physical execution - DruidQuery dq = (DruidQuery) scan; - //Adding Query specs to be used by org.apache.hadoop.hive.druid.io.DruidQueryBasedInputFormat - propList.add(ASTBuilder.construct(HiveParser.TOK_TABLEPROPERTY, "TOK_TABLEPROPERTY") - .add(HiveParser.StringLiteral, "\"" + Constants.DRUID_QUERY_JSON + "\"") - .add(HiveParser.StringLiteral, "\"" + SemanticAnalyzer.escapeSQLString( - dq.getQueryString()) + "\"")); - // Adding column names used later by org.apache.hadoop.hive.druid.serde.DruidSerDe - propList.add(ASTBuilder.construct(HiveParser.TOK_TABLEPROPERTY, "TOK_TABLEPROPERTY") - .add(HiveParser.StringLiteral, "\"" + Constants.DRUID_QUERY_FIELD_NAMES + "\"") - .add(HiveParser.StringLiteral, - "\"" + dq.getRowType().getFieldNames().stream().map(Object::toString) - .collect(Collectors.joining(",")) + "\"" - )); - // Adding column types used later by org.apache.hadoop.hive.druid.serde.DruidSerDe - propList.add(ASTBuilder.construct(HiveParser.TOK_TABLEPROPERTY, "TOK_TABLEPROPERTY") - .add(HiveParser.StringLiteral, "\"" + Constants.DRUID_QUERY_FIELD_TYPES + "\"") - .add(HiveParser.StringLiteral, - "\"" + dq.getRowType().getFieldList().stream() - .map(e -> TypeConverter.convert(e.getType()).getTypeName()) - .collect(Collectors.joining(",")) + "\"" - )); - propList.add(ASTBuilder.construct(HiveParser.TOK_TABLEPROPERTY, "TOK_TABLEPROPERTY") - .add(HiveParser.StringLiteral, "\"" + Constants.DRUID_QUERY_TYPE + "\"") - .add(HiveParser.StringLiteral, "\"" + dq.getQueryType().getQueryName() + "\"")); - } else if (scan instanceof HiveJdbcConverter) { + if (scan instanceof HiveJdbcConverter) { HiveJdbcConverter jdbcConverter = (HiveJdbcConverter) scan; final String query = jdbcConverter.generateSql(); LOGGER.debug("Generated SQL query: " + System.lineSeparator() + query); @@ -147,14 +118,14 @@ public static ASTNode table(final RelNode scan) { propList.add(ASTBuilder.construct(HiveParser.TOK_TABLEPROPERTY, "TOK_TABLEPROPERTY") .add(HiveParser.StringLiteral, "\"" + Constants.JDBC_SPLIT_QUERY + "\"") .add(HiveParser.StringLiteral, "\"" + jdbcConverter.splittingAllowed() + "\"")); - // Adding column names used later by org.apache.hadoop.hive.druid.serde.DruidSerDe + // Adding column names used later by JDBC storage handler propList.add(ASTBuilder.construct(HiveParser.TOK_TABLEPROPERTY, "TOK_TABLEPROPERTY") .add(HiveParser.StringLiteral, "\"" + Constants.JDBC_QUERY_FIELD_NAMES + "\"") .add(HiveParser.StringLiteral, "\"" + scan.getRowType().getFieldNames().stream().map(Object::toString) .collect(Collectors.joining(",")) + "\"" )); - // Adding column types used later by org.apache.hadoop.hive.druid.serde.DruidSerDe + // Adding column types used later by JDBC storage handler propList.add(ASTBuilder.construct(HiveParser.TOK_TABLEPROPERTY, "TOK_TABLEPROPERTY") .add(HiveParser.StringLiteral, "\"" + Constants.JDBC_QUERY_FIELD_TYPES + "\"") .add(HiveParser.StringLiteral, diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTConverter.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTConverter.java index f3e47f010646..7213a4ebe935 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTConverter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTConverter.java @@ -28,7 +28,6 @@ import java.util.Set; import java.util.TreeMap; -import org.apache.calcite.adapter.druid.DruidQuery; import org.apache.calcite.plan.RelOptUtil; import org.apache.calcite.rel.RelDistribution; import org.apache.calcite.rel.RelFieldCollation; @@ -552,10 +551,6 @@ private QueryBlockInfo convertSource(RelNode r) throws CalciteSemanticException HiveJdbcConverter f = (HiveJdbcConverter) r; s = new Schema(f); ast = ASTBuilder.table(f); - } else if (r instanceof DruidQuery) { - DruidQuery f = (DruidQuery) r; - s = new Schema(f); - ast = ASTBuilder.table(f); } else if (r instanceof Join) { Join join = (Join) r; QueryBlockInfo left = convertSource(join.getLeft()); @@ -768,7 +763,6 @@ public void handle(Values values) { public void visit(RelNode node, int ordinal, RelNode parent) { if (node instanceof TableScan || - node instanceof DruidQuery || node instanceof HiveJdbcConverter) { ASTConverter.this.from = node; } else if (node instanceof Filter) { @@ -1147,14 +1141,6 @@ static class Schema extends ArrayList { } } - Schema(DruidQuery dq) { - HiveTableScan hts = (HiveTableScan) dq.getTableScan(); - String tabName = hts.getTableAlias(); - for (RelDataTypeField field : dq.getRowType().getFieldList()) { - add(new ColumnInfo(tabName, field.getName())); - } - } - Schema(HiveJdbcConverter scan) { HiveJdbcConverter jdbcHiveCoverter = scan; final JdbcHiveTableScan jdbcTableScan = jdbcHiveCoverter.getTableScan(); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/PlanModifierForASTConv.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/PlanModifierForASTConv.java index 2f625c05a8af..f6cb508f4090 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/PlanModifierForASTConv.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/PlanModifierForASTConv.java @@ -26,7 +26,6 @@ import java.util.stream.Collectors; import com.google.common.collect.Sets; -import org.apache.calcite.adapter.druid.DruidQuery; import org.apache.calcite.adapter.jdbc.JdbcConvention; import org.apache.calcite.adapter.jdbc.JdbcRel; import org.apache.calcite.adapter.jdbc.JdbcRules; @@ -264,10 +263,6 @@ public RelNode visit(HiveJdbcConverter conv) { @Override public RelNode visit(final RelNode rel) { - if (rel instanceof DruidQuery dq) { - aliases.add(((HiveTableScan) dq.getTableScan()).getTableAlias().toLowerCase()); - return dq; - } if (rel instanceof TableSpool spool) { aliases.add(spool.getTable().getQualifiedName().getLast().toLowerCase()); return spool; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java index a91a281eedf8..25489f477f55 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java @@ -35,9 +35,6 @@ import org.antlr.runtime.tree.Tree; import org.antlr.runtime.tree.TreeVisitor; import org.antlr.runtime.tree.TreeVisitorAction; -import org.apache.calcite.adapter.druid.DruidQuery; -import org.apache.calcite.adapter.druid.DruidSchema; -import org.apache.calcite.adapter.druid.DruidTable; import org.apache.calcite.adapter.java.JavaTypeFactory; import org.apache.calcite.adapter.jdbc.JdbcConvention; import org.apache.calcite.adapter.jdbc.JdbcImplementor; @@ -53,7 +50,6 @@ import org.apache.calcite.config.CalciteConnectionConfigImpl; import org.apache.calcite.config.CalciteConnectionProperty; import org.apache.calcite.config.NullCollation; -import org.apache.calcite.interpreter.BindableConvention; import org.apache.calcite.plan.RelOptCluster; import org.apache.calcite.plan.RelOptCostImpl; import org.apache.calcite.plan.RelOptMaterialization; @@ -99,7 +95,6 @@ import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeFactory; import org.apache.calcite.rel.type.RelDataTypeField; -import org.apache.calcite.rel.type.RelDataTypeImpl; import org.apache.calcite.rex.RexBuilder; import org.apache.calcite.rex.RexCall; import org.apache.calcite.rex.RexExecutor; @@ -221,7 +216,6 @@ import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveAggregateReduceRule; import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveAggregateSplitRule; import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveCardinalityPreservingJoinRule; -import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveDruidRules; import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveExceptRewriteRule; import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveExpandDistinctAggregatesRule; import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveFieldTrimmerRule; @@ -334,14 +328,12 @@ import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; -import org.joda.time.Interval; import java.lang.reflect.Field; import java.lang.reflect.InvocationTargetException; import java.math.BigDecimal; import java.util.AbstractMap.SimpleEntry; import java.util.ArrayDeque; import java.util.ArrayList; -import java.util.Arrays; import java.util.BitSet; import java.util.Collection; import java.util.Collections; @@ -441,9 +433,6 @@ public class CalcitePlanner extends SemanticAnalyzer { HiveSortLimit.class, HiveTableFunctionScan.class, HiveUnion.class, - - DruidQuery.class, - HiveJdbcConverter.class, JdbcHiveTableScan.class, JdbcAggregate.class, @@ -1883,7 +1872,7 @@ protected RelNode applyPreJoinOrderingTransforms(RelNode basePlan, RelMetadataPr // matches FIL-PROJ-TS // Also merge, remove and reduce Project if possible generatePartialProgram(program, true, HepMatchOrder.TOP_DOWN, - HiveFilterProjectTransposeRule.SCAN, HiveFilterProjectTransposeRule.DRUID, + HiveFilterProjectTransposeRule.SCAN, HiveProjectFilterPullUpConstantsRule.INSTANCE, HiveProjectMergeRule.INSTANCE, ProjectRemoveRule.Config.DEFAULT.toRule(), HiveSortMergeRule.INSTANCE); @@ -2312,22 +2301,6 @@ private RelNode applyPostJoinOrderingTransform(RelNode basePlan, RelMetadataProv HiveSearchRules.FILTER_SEARCH_EXPAND, HiveSearchRules.JOIN_SEARCH_EXPAND); - // 7. Apply Druid transformation rules - generatePartialProgram(program, false, HepMatchOrder.DEPTH_FIRST, - HiveDruidRules.FILTER_DATE_RANGE_RULE, - HiveDruidRules.FILTER, HiveDruidRules.PROJECT_FILTER_TRANSPOSE, - HiveDruidRules.AGGREGATE_FILTER_TRANSPOSE, - HiveDruidRules.AGGREGATE_PROJECT, - HiveDruidRules.PROJECT, - HiveDruidRules.EXPAND_SINGLE_DISTINCT_AGGREGATES_DRUID_RULE, - HiveDruidRules.AGGREGATE, - HiveDruidRules.POST_AGGREGATION_PROJECT, - HiveDruidRules.FILTER_AGGREGATE_TRANSPOSE, - HiveDruidRules.FILTER_PROJECT_TRANSPOSE, - HiveDruidRules.HAVING_FILTER_RULE, - HiveDruidRules.SORT_PROJECT_TRANSPOSE, - HiveDruidRules.SORT); - // 8. Apply JDBC transformation rules if (conf.getBoolVar(ConfVars.HIVE_ENABLE_JDBC_PUSHDOWN)) { List rules = Lists.newArrayList(); @@ -2366,7 +2339,7 @@ private RelNode applyPostJoinOrderingTransform(RelNode basePlan, RelMetadataProv generatePartialProgram(program, false, HepMatchOrder.DEPTH_FIRST, ProjectRemoveRule.Config.DEFAULT.toRule(), new ProjectMergeRule(false, HiveRelFactories.HIVE_BUILDER)); generatePartialProgram(program, true, HepMatchOrder.TOP_DOWN, - HiveFilterProjectTransposeRule.SCAN, HiveFilterProjectTransposeRule.DRUID, + HiveFilterProjectTransposeRule.SCAN, HiveProjectFilterPullUpConstantsRule.INSTANCE); // 9.2. Introduce exchange operators below join/multijoin operators @@ -2423,12 +2396,6 @@ public void visit(RelNode node, int ordinal, RelNode parent) { if (table.isMaterializedView()) { materializedViewsUsed.add(table); } - } else if (node instanceof DruidQuery) { - DruidQuery dq = (DruidQuery) node; - Table table = ((RelOptHiveTable) dq.getTable()).getHiveTableMD(); - if (table.isMaterializedView()) { - materializedViewsUsed.add(table); - } } super.visit(node, ordinal, parent); } @@ -3043,8 +3010,7 @@ private RelNode genTableLogicalPlan(String tableAlias, QB qb) throws SemanticExc Map tabPropsFromQuery = qb.getTabPropsForAlias(tableAlias); HiveTableScan.HiveTableScanTrait tableScanTrait = HiveTableScan.HiveTableScanTrait.from(tabPropsFromQuery); RelOptHiveTable optTable; - if (tableType == TableType.DRUID || - (tableType == TableType.JDBC && tabMetaData.getProperty(Constants.JDBC_TABLE) != null)) { + if (tableType == TableType.JDBC && tabMetaData.getProperty(Constants.JDBC_TABLE) != null) { // Create case sensitive columns list List originalColumnNames = ((StandardStructObjectInspector)rowObjectInspector).getOriginalColumnNames(); @@ -3062,109 +3028,61 @@ private RelNode genTableLogicalPlan(String tableAlias, QB qb) throws SemanticExc } fullyQualifiedTabName.add(tabMetaData.getTableName()); - if (tableType == TableType.DRUID) { - // Build Druid query - String address = HiveConf.getVar(conf, - HiveConf.ConfVars.HIVE_DRUID_BROKER_DEFAULT_ADDRESS); - String dataSource = tabMetaData.getParameters().get(Constants.DRUID_DATA_SOURCE); - Set metrics = new HashSet<>(); - RexBuilder rexBuilder = cluster.getRexBuilder(); - RelDataTypeFactory dtFactory = rexBuilder.getTypeFactory(); - List druidColTypes = new ArrayList<>(); - List druidColNames = new ArrayList<>(); - //@TODO FIX this, we actually do not need this anymore, - // in addition to that Druid allow numeric dimensions now so this check is not accurate - for (RelDataTypeField field : rowType.getFieldList()) { - if (DruidTable.DEFAULT_TIMESTAMP_COLUMN.equals(field.getName())) { - // Druid's time column is always not null. - druidColTypes.add(dtFactory.createTypeWithNullability(field.getType(), false)); - } else { - druidColTypes.add(field.getType()); - } - druidColNames.add(field.getName()); - if (field.getName().equals(DruidTable.DEFAULT_TIMESTAMP_COLUMN)) { - // timestamp - continue; - } - if (field.getType().getSqlTypeName() == SqlTypeName.VARCHAR) { - // dimension - continue; - } - metrics.add(field.getName()); - } - - List intervals = Arrays.asList(DruidTable.DEFAULT_INTERVAL); - rowType = dtFactory.createStructType(druidColTypes, druidColNames); - DruidTable druidTable = new DruidTable(new DruidSchema(address, address, false), - dataSource, RelDataTypeImpl.proto(rowType), metrics, DruidTable.DEFAULT_TIMESTAMP_COLUMN, - intervals, null, null); - optTable = new RelOptHiveTable(relOptSchema, relOptSchema.getTypeFactory(), fullyQualifiedTabName, + optTable = new RelOptHiveTable(relOptSchema, relOptSchema.getTypeFactory(), fullyQualifiedTabName, rowType, tabMetaData, nonPartitionColumns, partitionColumns, virtualCols, conf, tabNameToTabObject, partitionCache, colStatsCache, noColsMissingStats); - final TableScan scan = new HiveTableScan(cluster, cluster.traitSetOf(HiveRelNode.CONVENTION), - optTable, null == tableAlias ? tabMetaData.getTableName() : tableAlias, - getAliasId(tableAlias, qb), HiveConf.getBoolVar(conf, - HiveConf.ConfVars.HIVE_CBO_RETPATH_HIVEOP), qb.isInsideView() - || qb.getAliasInsideView().contains(tableAlias.toLowerCase()), tableScanTrait); - tableRel = DruidQuery.create(cluster, cluster.traitSetOf(BindableConvention.INSTANCE), - optTable, druidTable, ImmutableList.of(scan), DruidSqlOperatorConverter.getDefaultMap()); + final HiveTableScan hts = new HiveTableScan(cluster, + cluster.traitSetOf(HiveRelNode.CONVENTION), optTable, + null == tableAlias ? tabMetaData.getTableName() : tableAlias, + getAliasId(tableAlias, qb), + HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_CBO_RETPATH_HIVEOP), + qb.isInsideView() || qb.getAliasInsideView().contains(tableAlias.toLowerCase()), tableScanTrait); + + final String dataBaseType = tabMetaData.getProperty(Constants.JDBC_DATABASE_TYPE); + final String url = tabMetaData.getProperty(Constants.JDBC_URL); + final String driver = tabMetaData.getProperty(Constants.JDBC_DRIVER); + final String user = tabMetaData.getProperty(Constants.JDBC_USERNAME); + final String pswd; + if (tabMetaData.getProperty(Constants.JDBC_PASSWORD) != null) { + pswd = tabMetaData.getProperty(Constants.JDBC_PASSWORD); + } else if (tabMetaData.getProperty(Constants.JDBC_KEYSTORE) != null) { + String keystore = tabMetaData.getProperty(Constants.JDBC_KEYSTORE); + String key = tabMetaData.getProperty(Constants.JDBC_KEY); + pswd = Utilities.getPasswdFromKeystore(keystore, key); + } else if (tabMetaData.getProperty(Constants.JDBC_PASSWORD_URI) != null) { + pswd = Utilities.getPasswdFromUri(tabMetaData.getProperty(Constants.JDBC_PASSWORD_URI)); } else { - optTable = new RelOptHiveTable(relOptSchema, relOptSchema.getTypeFactory(), fullyQualifiedTabName, - rowType, tabMetaData, nonPartitionColumns, partitionColumns, virtualCols, conf, - tabNameToTabObject, partitionCache, colStatsCache, noColsMissingStats); - final HiveTableScan hts = new HiveTableScan(cluster, - cluster.traitSetOf(HiveRelNode.CONVENTION), optTable, - null == tableAlias ? tabMetaData.getTableName() : tableAlias, - getAliasId(tableAlias, qb), - HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_CBO_RETPATH_HIVEOP), - qb.isInsideView() || qb.getAliasInsideView().contains(tableAlias.toLowerCase()), tableScanTrait); - - final String dataBaseType = tabMetaData.getProperty(Constants.JDBC_DATABASE_TYPE); - final String url = tabMetaData.getProperty(Constants.JDBC_URL); - final String driver = tabMetaData.getProperty(Constants.JDBC_DRIVER); - final String user = tabMetaData.getProperty(Constants.JDBC_USERNAME); - final String pswd; - if (tabMetaData.getProperty(Constants.JDBC_PASSWORD) != null) { - pswd = tabMetaData.getProperty(Constants.JDBC_PASSWORD); - } else if (tabMetaData.getProperty(Constants.JDBC_KEYSTORE) != null) { - String keystore = tabMetaData.getProperty(Constants.JDBC_KEYSTORE); - String key = tabMetaData.getProperty(Constants.JDBC_KEY); - pswd = Utilities.getPasswdFromKeystore(keystore, key); - } else if (tabMetaData.getProperty(Constants.JDBC_PASSWORD_URI) != null) { - pswd = Utilities.getPasswdFromUri(tabMetaData.getProperty(Constants.JDBC_PASSWORD_URI)); - } else { - pswd = null; - LOG.warn("No password found for accessing {} table via JDBC", fullyQualifiedTabName); - } - final String catalogName = tabMetaData.getProperty(Constants.JDBC_CATALOG); - final String schemaName = tabMetaData.getProperty(Constants.JDBC_SCHEMA); - final String tableName = tabMetaData.getProperty(Constants.JDBC_TABLE); - - DataSource ds = JdbcSchema.dataSource(url, driver, user, pswd); - SqlDialect jdbcDialect = JdbcSchema.createDialect(SqlDialectFactoryImpl.INSTANCE, ds); - String dialectName = jdbcDialect.getClass().getName(); - if (LOG.isDebugEnabled()) { - LOG.debug("Dialect for table {}: {}", tableName, dialectName); - } - - List jdbcConventionKey = ImmutableNullableList.of(url, driver, user, pswd, dialectName, dataBaseType); - jdbcConventionMap.putIfAbsent(jdbcConventionKey, JdbcConvention.of(jdbcDialect, null, dataBaseType)); - JdbcConvention jc = jdbcConventionMap.get(jdbcConventionKey); + pswd = null; + LOG.warn("No password found for accessing {} table via JDBC", fullyQualifiedTabName); + } + final String catalogName = tabMetaData.getProperty(Constants.JDBC_CATALOG); + final String schemaName = tabMetaData.getProperty(Constants.JDBC_SCHEMA); + final String tableName = tabMetaData.getProperty(Constants.JDBC_TABLE); + + DataSource ds = JdbcSchema.dataSource(url, driver, user, pswd); + SqlDialect jdbcDialect = JdbcSchema.createDialect(SqlDialectFactoryImpl.INSTANCE, ds); + String dialectName = jdbcDialect.getClass().getName(); + if (LOG.isDebugEnabled()) { + LOG.debug("Dialect for table {}: {}", tableName, dialectName); + } - List schemaKey = ImmutableNullableList.of(url, driver, user, pswd, dialectName, dataBaseType, - catalogName, schemaName); - schemaMap.putIfAbsent(schemaKey, new JdbcSchema(ds, jc.dialect, jc, catalogName, schemaName)); - JdbcSchema schema = schemaMap.get(schemaKey); + List jdbcConventionKey = ImmutableNullableList.of(url, driver, user, pswd, dialectName, dataBaseType); + jdbcConventionMap.putIfAbsent(jdbcConventionKey, JdbcConvention.of(jdbcDialect, null, dataBaseType)); + JdbcConvention jc = jdbcConventionMap.get(jdbcConventionKey); - JdbcTable jt = (JdbcTable) schema.getTable(tableName); - if (jt == null) { - throw new SemanticException("Table " + tableName + " was not found in the database"); - } + List schemaKey = ImmutableNullableList.of(url, driver, user, pswd, dialectName, dataBaseType, + catalogName, schemaName); + schemaMap.putIfAbsent(schemaKey, new JdbcSchema(ds, jc.dialect, jc, catalogName, schemaName)); + JdbcSchema schema = schemaMap.get(schemaKey); - JdbcHiveTableScan jdbcTableRel = new JdbcHiveTableScan(cluster, optTable, jt, jc, hts); - tableRel = new HiveJdbcConverter(cluster, jdbcTableRel.getTraitSet().replace(HiveRelNode.CONVENTION), - jdbcTableRel, jc, url, user); + JdbcTable jt = (JdbcTable) schema.getTable(tableName); + if (jt == null) { + throw new SemanticException("Table " + tableName + " was not found in the database"); } + + JdbcHiveTableScan jdbcTableRel = new JdbcHiveTableScan(cluster, optTable, jt, jc, hts); + tableRel = new HiveJdbcConverter(cluster, jdbcTableRel.getTraitSet().replace(HiveRelNode.CONVENTION), + jdbcTableRel, jc, url, user); } else { // Build row type from field RelDataType rowType = TypeConverter.getType(cluster, rr, null); @@ -3225,11 +3143,6 @@ private boolean isNullable(String colName, NotNullConstraint notNullConstraints, private TableType obtainTableType(Table tabMetaData) { if (tabMetaData.getStorageHandler() != null) { final String storageHandlerStr = tabMetaData.getStorageHandler().toString(); - if (storageHandlerStr - .equals(Constants.DRUID_HIVE_STORAGE_HANDLER_ID)) { - return TableType.DRUID; - } - if (storageHandlerStr .equals(Constants.JDBC_HIVE_STORAGE_HANDLER_ID)) { return TableType.JDBC; @@ -5306,7 +5219,6 @@ public static void warmup() { } private enum TableType { - DRUID, NATIVE, JDBC } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/DruidSqlOperatorConverter.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/DruidSqlOperatorConverter.java deleted file mode 100644 index 36c3af5f8b3c..000000000000 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/DruidSqlOperatorConverter.java +++ /dev/null @@ -1,392 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.parse; - -import com.google.common.base.Function; -import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.Iterables; -import com.google.common.collect.Maps; -import org.apache.calcite.adapter.druid.DirectOperatorConversion; -import org.apache.calcite.adapter.druid.DruidExpressions; -import org.apache.calcite.adapter.druid.DruidQuery; -import org.apache.calcite.adapter.druid.ExtractOperatorConversion; -import org.apache.calcite.adapter.druid.FloorOperatorConversion; -import org.apache.calcite.config.CalciteConnectionConfig; -import org.apache.calcite.rel.type.RelDataType; -import org.apache.calcite.rex.RexCall; -import org.apache.calcite.rex.RexLiteral; -import org.apache.calcite.rex.RexNode; -import org.apache.calcite.sql.SqlFunction; -import org.apache.calcite.sql.SqlKind; -import org.apache.calcite.sql.SqlOperator; -import org.apache.calcite.sql.fun.SqlStdOperatorTable; -import org.apache.calcite.sql.type.SqlTypeName; -import org.apache.calcite.sql.type.SqlTypeUtil; -import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveConcat; -import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveDateAddSqlOperator; -import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveDateSubSqlOperator; -import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveExtractDate; -import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveFloorDate; -import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveFromUnixTimeSqlOperator; -import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveToDateSqlOperator; -import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTruncSqlOperator; -import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveToUnixTimestampSqlOperator; -import org.joda.time.Period; - -import javax.annotation.Nullable; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.TimeZone; - -/** - * Contains custom Druid SQL operator converter classes, contains either: - * Hive specific OperatorConversion logic that can not be part of Calcite - * Some temporary OperatorConversion that is not release by Calcite yet - */ -public class DruidSqlOperatorConverter { - - private static final String YYYY_MM_DD = "yyyy-MM-dd"; - public static final String DEFAULT_TS_FORMAT = "yyyy-MM-dd HH:mm:ss"; - - private DruidSqlOperatorConverter() { - } - - private static Map druidOperatorMap = null; - - public static final Map getDefaultMap() { - if (druidOperatorMap == null) { - druidOperatorMap = new HashMap(); - DruidQuery.DEFAULT_OPERATORS_LIST.stream().forEach(op -> druidOperatorMap.put(op.calciteOperator(), op)); - - //Override Hive specific operators - druidOperatorMap.putAll(Maps.asMap(HiveFloorDate.ALL_FUNCTIONS, - (Function) input -> new - FloorOperatorConversion() - )); - druidOperatorMap.putAll(Maps.asMap(HiveExtractDate.ALL_FUNCTIONS, - (Function) input -> new - ExtractOperatorConversion() - )); - druidOperatorMap.put(HiveConcat.INSTANCE, new DirectOperatorConversion(HiveConcat.INSTANCE, "concat")); - druidOperatorMap - .put(SqlStdOperatorTable.SUBSTRING, new DruidSqlOperatorConverter.DruidSubstringOperatorConversion()); - druidOperatorMap - .put(SqlStdOperatorTable.IS_NULL, new UnaryFunctionOperatorConversion(SqlStdOperatorTable.IS_NULL, "isnull")); - druidOperatorMap.put(SqlStdOperatorTable.IS_NOT_NULL, - new UnaryFunctionOperatorConversion(SqlStdOperatorTable.IS_NOT_NULL, "notnull") - ); - druidOperatorMap.put(HiveTruncSqlOperator.INSTANCE, new DruidDateTruncOperatorConversion()); - druidOperatorMap.put(HiveToDateSqlOperator.INSTANCE, new DruidToDateOperatorConversion()); - druidOperatorMap.put(HiveFromUnixTimeSqlOperator.INSTANCE, new DruidFormUnixTimeOperatorConversion()); - druidOperatorMap.put(HiveToUnixTimestampSqlOperator.INSTANCE, new DruidUnixTimestampOperatorConversion()); - druidOperatorMap.put(HiveDateAddSqlOperator.INSTANCE, - new DruidDateArithmeticOperatorConversion(1, HiveDateAddSqlOperator.INSTANCE) - ); - druidOperatorMap.put(HiveDateSubSqlOperator.INSTANCE, - new DruidDateArithmeticOperatorConversion(-1, HiveDateSubSqlOperator.INSTANCE) - ); - } - return druidOperatorMap; - } - - /** - * Druid operator converter from Hive Substring to Druid SubString. - * This is a temporary fix that can be removed once we move to a Calcite version including the following. - * https://issues.apache.org/jira/browse/CALCITE-2226 - */ - public static class DruidSubstringOperatorConversion - extends org.apache.calcite.adapter.druid.SubstringOperatorConversion { - @Nullable @Override public String toDruidExpression(RexNode rexNode, RelDataType rowType, DruidQuery query - ) { - final RexCall call = (RexCall) rexNode; - final String arg = DruidExpressions.toDruidExpression(call.getOperands().get(0), rowType, query); - if (arg == null) { - return null; - } - - final String indexStart; - final String length; - // SQL is 1-indexed, Druid is 0-indexed. - if (!call.getOperands().get(1).isA(SqlKind.LITERAL)) { - final String indexExp = DruidExpressions.toDruidExpression(call.getOperands().get(1), rowType, query); - if (indexExp == null) { - return null; - } - indexStart = DruidQuery.format("(%s - 1)", indexExp); - } else { - final int index = RexLiteral.intValue(call.getOperands().get(1)) - 1; - indexStart = DruidExpressions.numberLiteral(index); - } - - if (call.getOperands().size() > 2) { - //case substring from index with length - if (!call.getOperands().get(2).isA(SqlKind.LITERAL)) { - length = DruidExpressions.toDruidExpression(call.getOperands().get(2), rowType, query); - if (length == null) { - return null; - } - } else { - length = DruidExpressions.numberLiteral(RexLiteral.intValue(call.getOperands().get(2))); - } - - } else { - //case substring from index to the end - length = DruidExpressions.numberLiteral(-1); - } - return DruidQuery.format("substring(%s, %s, %s)", arg, indexStart, length); - } - } - - /** - * Operator conversion form Hive TRUNC UDF to Druid Date Time UDFs. - */ - public static class DruidDateTruncOperatorConversion - implements org.apache.calcite.adapter.druid.DruidSqlOperatorConverter { - - @Override public SqlOperator calciteOperator() { - return HiveTruncSqlOperator.INSTANCE; - } - - @Nullable @Override public String toDruidExpression(RexNode rexNode, RelDataType rowType, DruidQuery query - ) { - final RexCall call = (RexCall) rexNode; - //can handle only case trunc date type - if (call.getOperands().size() < 1) { - throw new IllegalStateException("trunc() requires at least 1 argument, got " + call.getOperands().size()); - } - if (call.getOperands().size() == 1) { - final String arg = DruidExpressions.toDruidExpression(call.getOperands().get(0), rowType, query); - if (arg == null) { - return null; - } - if (SqlTypeUtil.isDatetime(call.getOperands().get(0).getType())) { - final TimeZone tz = timezoneId(query, call.getOperands().get(0)); - return applyTimestampFormat( - DruidExpressions.applyTimestampFloor(arg, Period.days(1).toString(), "", tz), - YYYY_MM_DD, - tz); - } - return null; - } else if (call.getOperands().size() == 2) { - final String arg = DruidExpressions.toDruidExpression(call.getOperands().get(0), rowType, query); - if (arg == null) { - return null; - } - String granularity = DruidExpressions.toDruidExpression(call.getOperands().get(1), rowType, query); - if (granularity == null) { - return null; - } - final String unit; - if ("'MONTH'".equals(granularity) || "'MON'".equals(granularity) || "'MM'".equals(granularity)) { - unit = Period.months(1).toString(); - } else if ("'YEAR'".equals(granularity) || "'YYYY'".equals(granularity) || "'YY'".equals(granularity)) { - unit = Period.years(1).toString(); - } else if ("'QUARTER'".equals(granularity) || "'Q'".equals(granularity)) { - unit = Period.months(3).toString(); - } else { - unit = null; - } - if (unit == null) { - //bail out can not infer unit - return null; - } - final TimeZone tz = timezoneId(query, call.getOperands().get(0)); - return applyTimestampFormat( - DruidExpressions.applyTimestampFloor(arg, unit, "", tz), - YYYY_MM_DD, - tz); - } - return null; - } - } - - /** - * Expression operator conversion form Hive TO_DATE operator to Druid Date cast. - */ - public static class DruidToDateOperatorConversion - implements org.apache.calcite.adapter.druid.DruidSqlOperatorConverter { - - @Override public SqlOperator calciteOperator() { - return HiveToDateSqlOperator.INSTANCE; - } - - @Nullable @Override public String toDruidExpression(RexNode rexNode, RelDataType rowType, DruidQuery query - ) { - final RexCall call = (RexCall) rexNode; - if (call.getOperands().size() != 1) { - throw new IllegalStateException("to_date() requires 1 argument, got " + call.getOperands().size()); - } - final String arg = DruidExpressions.toDruidExpression(call.getOperands().get(0), rowType, query); - if (arg == null) { - return null; - } - return DruidExpressions.applyTimestampFloor( - arg, - Period.days(1).toString(), - "", - timezoneId(query, call.getOperands().get(0))); - } - } - - public static class DruidUnixTimestampOperatorConversion - implements org.apache.calcite.adapter.druid.DruidSqlOperatorConverter { - - @Override public SqlOperator calciteOperator() { - return HiveToUnixTimestampSqlOperator.INSTANCE; - } - - @Nullable @Override public String toDruidExpression(RexNode rexNode, RelDataType rowType, DruidQuery query - ) { - final RexCall call = (RexCall) rexNode; - final String arg0 = DruidExpressions.toDruidExpression(call.getOperands().get(0), rowType, query); - if (arg0 == null) { - return null; - } - if (SqlTypeUtil.isDatetime((call.getOperands().get(0).getType()))) { - // Timestamp is represented as long internally no need to any thing here - return DruidExpressions.functionCall("div", ImmutableList.of(arg0, DruidExpressions.numberLiteral(1000))); - } - // dealing with String type - final String format = call.getOperands().size() == 2 ? DruidExpressions - .toDruidExpression(call.getOperands().get(1), rowType, query) : DEFAULT_TS_FORMAT; - return DruidExpressions - .functionCall("unix_timestamp", ImmutableList.of(arg0, DruidExpressions.stringLiteral(format))); - } - } - - public static class DruidFormUnixTimeOperatorConversion - implements org.apache.calcite.adapter.druid.DruidSqlOperatorConverter { - - @Override public SqlOperator calciteOperator() { - return HiveFromUnixTimeSqlOperator.INSTANCE; - } - - @Nullable @Override public String toDruidExpression(RexNode rexNode, RelDataType rowType, DruidQuery query - ) { - final RexCall call = (RexCall) rexNode; - if (call.getOperands().size() < 1 || call.getOperands().size() > 2) { - throw new IllegalStateException("form_unixtime() requires 1 or 2 argument, got " + call.getOperands().size()); - } - final String arg = DruidExpressions.toDruidExpression(call.getOperands().get(0), rowType, query); - if (arg == null) { - return null; - } - - final String numMillis = DruidQuery.format("(%s * '1000')", arg); - final String format = - call.getOperands().size() == 1 ? DruidExpressions.stringLiteral(DEFAULT_TS_FORMAT) : DruidExpressions - .toDruidExpression(call.getOperands().get(1), rowType, query); - return DruidExpressions.functionCall("timestamp_format", - ImmutableList.of(numMillis, format, DruidExpressions.stringLiteral(TimeZone.getTimeZone("UTC").getID())) - ); - } - } - - /** - * Base class for Date Add/Sub operator conversion - */ - public static class DruidDateArithmeticOperatorConversion - implements org.apache.calcite.adapter.druid.DruidSqlOperatorConverter { - - private final int direction; - private final SqlOperator operator; - - public DruidDateArithmeticOperatorConversion(int direction, SqlOperator operator) { - this.direction = direction; - this.operator = operator; - Preconditions.checkArgument(direction == 1 || direction == -1); - } - - @Override public SqlOperator calciteOperator() { - return operator; - } - - @Nullable @Override public String toDruidExpression(RexNode rexNode, RelDataType rowType, DruidQuery query - ) { - final RexCall call = (RexCall) rexNode; - if (call.getOperands().size() != 2) { - throw new IllegalStateException("date_add/date_sub() requires 2 arguments, got " + call.getOperands().size()); - } - final String arg0 = DruidExpressions.toDruidExpression(call.getOperands().get(0), rowType, query); - final String arg1 = DruidExpressions.toDruidExpression(call.getOperands().get(1), rowType, query); - if (arg0 == null || arg1 == null) { - return null; - } - - final String steps = direction == -1 ? DruidQuery.format("-( %s )", arg1) : arg1; - return DruidExpressions.functionCall( - "timestamp_shift", - ImmutableList.of( - arg0, - DruidExpressions.stringLiteral("P1D"), - steps, - DruidExpressions.stringLiteral(timezoneId(query, call.getOperands().get(0)).getID()))); - } - } - - /** - * utility function to extract timezone id from Druid query - * @param query Druid Rel - * @return time zone - */ - private static TimeZone timezoneId(final DruidQuery query, final RexNode arg) { - return arg.getType().getSqlTypeName() == SqlTypeName.TIMESTAMP_WITH_LOCAL_TIME_ZONE - ? TimeZone.getTimeZone( - query.getTopNode().getCluster().getPlanner().getContext().unwrap(CalciteConnectionConfig.class).timeZone()) : - TimeZone.getTimeZone("UTC"); - } - - private static String applyTimestampFormat(String arg, String format, TimeZone timeZone) { - return DruidExpressions.functionCall("timestamp_format", - ImmutableList.of(arg, DruidExpressions.stringLiteral(format), DruidExpressions.stringLiteral(timeZone.getID())) - ); - } - - public static class UnaryFunctionOperatorConversion implements org.apache.calcite.adapter.druid.DruidSqlOperatorConverter { - - private final SqlOperator operator; - private final String druidOperator; - - public UnaryFunctionOperatorConversion(SqlOperator operator, String druidOperator) { - this.operator = operator; - this.druidOperator = druidOperator; - } - - @Override public SqlOperator calciteOperator() { - return operator; - } - - @Override public String toDruidExpression(RexNode rexNode, RelDataType rowType, - DruidQuery druidQuery) { - final RexCall call = (RexCall) rexNode; - - final List druidExpressions = DruidExpressions.toDruidExpressions( - druidQuery, rowType, - call.getOperands()); - - if (druidExpressions == null) { - return null; - } - - return DruidQuery.format("%s(%s)", druidOperator, Iterables.getOnlyElement(druidExpressions)); - } - } -} diff --git a/ql/src/test/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/TestSQLStdHiveAccessControllerHS2.java b/ql/src/test/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/TestSQLStdHiveAccessControllerHS2.java index 0d61eaf2573c..84b377582016 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/TestSQLStdHiveAccessControllerHS2.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/TestSQLStdHiveAccessControllerHS2.java @@ -91,7 +91,7 @@ private List getSettableParams() throws SecurityException, NoSuchFieldEx "hive.optimize.index.filter.compact.maxsize", "hive.tez.dummy", "tez.task.dummy", "hive.exec.dynamic.partition", "hive.exec.dynamic.partition.mode", "hive.exec.max.dynamic.partitions", "hive.exec.max.dynamic.partitions.pernode", - "oozie.HadoopAccessorService.created", "tez.queue.name","hive.druid.select.distribute", + "oozie.HadoopAccessorService.created", "tez.queue.name", "distcp.options.px", "hive.materializedview.rewriting"); // combine two lists diff --git a/ql/src/test/queries/clientnegative/druid_address.q b/ql/src/test/queries/clientnegative/druid_address.q deleted file mode 100644 index 35ba06a4471f..000000000000 --- a/ql/src/test/queries/clientnegative/druid_address.q +++ /dev/null @@ -1,5 +0,0 @@ -set hive.druid.broker.address.default=; - -CREATE EXTERNAL TABLE druid_table_1 -STORED BY 'org.apache.hadoop.hive.druid.QTestDruidStorageHandler' -TBLPROPERTIES ("druid.datasource" = "wikipedia"); diff --git a/ql/src/test/queries/clientnegative/druid_buckets.q b/ql/src/test/queries/clientnegative/druid_buckets.q deleted file mode 100644 index 6fc75d109667..000000000000 --- a/ql/src/test/queries/clientnegative/druid_buckets.q +++ /dev/null @@ -1,6 +0,0 @@ -set hive.druid.broker.address.default=localhost.test; - -CREATE EXTERNAL TABLE druid_table_1 -CLUSTERED BY (robot) INTO 32 BUCKETS -STORED BY 'org.apache.hadoop.hive.druid.QTestDruidStorageHandler' -TBLPROPERTIES ("druid.datasource" = "wikipedia"); diff --git a/ql/src/test/queries/clientnegative/druid_case.q b/ql/src/test/queries/clientnegative/druid_case.q deleted file mode 100644 index 4828bf846ec0..000000000000 --- a/ql/src/test/queries/clientnegative/druid_case.q +++ /dev/null @@ -1,6 +0,0 @@ -set hive.strict.checks.cartesian.product=false; -set hive.druid.broker.address.default=localhost.test; - -CREATE EXTERNAL TABLE druid_table_1 -STORED BY 'org.apache.hadoop.hive.druid.QTestDruidStorageHandler2' -TBLPROPERTIES ("druid.datasource" = "wikipedia"); diff --git a/ql/src/test/queries/clientnegative/druid_datasource.q b/ql/src/test/queries/clientnegative/druid_datasource.q deleted file mode 100644 index 87481fdc79bb..000000000000 --- a/ql/src/test/queries/clientnegative/druid_datasource.q +++ /dev/null @@ -1,3 +0,0 @@ -CREATE EXTERNAL TABLE druid_table_1 -STORED BY 'org.apache.hadoop.hive.druid.QTestDruidStorageHandler' -TBLPROPERTIES ("property" = "localhost"); diff --git a/ql/src/test/queries/clientnegative/druid_datasource2.q b/ql/src/test/queries/clientnegative/druid_datasource2.q deleted file mode 100644 index 2e7400f47bc0..000000000000 --- a/ql/src/test/queries/clientnegative/druid_datasource2.q +++ /dev/null @@ -1,4 +0,0 @@ -SET metastore.strict.managed.tables=true; -CREATE TABLE druid_table_1 -STORED BY 'org.apache.hadoop.hive.druid.QTestDruidStorageHandler' -TBLPROPERTIES ("property" = "localhost", "druid.datasource" = "mydatasource"); diff --git a/ql/src/test/queries/clientnegative/druid_location.q b/ql/src/test/queries/clientnegative/druid_location.q deleted file mode 100644 index a9705dc401ae..000000000000 --- a/ql/src/test/queries/clientnegative/druid_location.q +++ /dev/null @@ -1,6 +0,0 @@ -set hive.druid.broker.address.default=localhost.test; - -CREATE EXTERNAL TABLE druid_table_1 -STORED BY 'org.apache.hadoop.hive.druid.QTestDruidStorageHandler' -LOCATION '/testfolder/' -TBLPROPERTIES ("druid.datasource" = "wikipedia"); diff --git a/ql/src/test/queries/clientnegative/druid_partitions.q b/ql/src/test/queries/clientnegative/druid_partitions.q deleted file mode 100644 index e26a3b688a0e..000000000000 --- a/ql/src/test/queries/clientnegative/druid_partitions.q +++ /dev/null @@ -1,6 +0,0 @@ -set hive.druid.broker.address.default=localhost.test; - -CREATE EXTERNAL TABLE druid_table_1 -PARTITIONED BY (dt string) -STORED BY 'org.apache.hadoop.hive.druid.QTestDruidStorageHandler' -TBLPROPERTIES ("druid.datasource" = "wikipedia"); diff --git a/ql/src/test/queries/clientpositive/druid_materialized_view_rewrite_ssb.q b/ql/src/test/queries/clientpositive/druid_materialized_view_rewrite_ssb.q deleted file mode 100644 index b64cc49d2bf8..000000000000 --- a/ql/src/test/queries/clientpositive/druid_materialized_view_rewrite_ssb.q +++ /dev/null @@ -1,318 +0,0 @@ ---! qt:disabled:unstable; fails sometimes HIVE-23450 ---! qt:dataset:part - -set hive.support.concurrency=true; -set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; -set hive.strict.checks.cartesian.product=false; - -CREATE TABLE `customer_ext_n0`( - `c_custkey` bigint, - `c_name` string, - `c_address` string, - `c_city` string, - `c_nation` string, - `c_region` string, - `c_phone` string, - `c_mktsegment` string) -ROW FORMAT DELIMITED -FIELDS TERMINATED BY '|' -STORED AS TEXTFILE; - -LOAD DATA LOCAL INPATH '../../data/files/ssb/customer/' into table `customer_ext_n0`; - -CREATE TABLE `customer_n0`( - `c_custkey` bigint, - `c_name` string, - `c_address` string, - `c_city` string, - `c_nation` string, - `c_region` string, - `c_phone` string, - `c_mktsegment` string, - primary key (`c_custkey`) disable rely) -STORED AS ORC -TBLPROPERTIES ('transactional'='true'); - -INSERT INTO `customer_n0` -SELECT * FROM `customer_ext_n0`; - -CREATE TABLE `dates_ext_n0`( - `d_datekey` bigint, - `d_date` string, - `d_dayofweek` string, - `d_month` string, - `d_year` int, - `d_yearmonthnum` int, - `d_yearmonth` string, - `d_daynuminweek` int, - `d_daynuminmonth` int, - `d_daynuminyear` int, - `d_monthnuminyear` int, - `d_weeknuminyear` int, - `d_sellingseason` string, - `d_lastdayinweekfl` int, - `d_lastdayinmonthfl` int, - `d_holidayfl` int , - `d_weekdayfl`int) -ROW FORMAT DELIMITED -FIELDS TERMINATED BY '|' -STORED AS TEXTFILE; - -LOAD DATA LOCAL INPATH '../../data/files/ssb/date/' into table `dates_ext_n0`; - -CREATE TABLE `dates_n0`( - `d_datekey` bigint, - `__time` timestamp, - `d_date` string, - `d_dayofweek` string, - `d_month` string, - `d_year` int, - `d_yearmonthnum` int, - `d_yearmonth` string, - `d_daynuminweek` int, - `d_daynuminmonth` int, - `d_daynuminyear` int, - `d_monthnuminyear` int, - `d_weeknuminyear` int, - `d_sellingseason` string, - `d_lastdayinweekfl` int, - `d_lastdayinmonthfl` int, - `d_holidayfl` int , - `d_weekdayfl`int, - primary key (`d_datekey`) disable rely -) -STORED AS ORC -TBLPROPERTIES ('transactional'='true'); - -INSERT INTO `dates_n0` -SELECT - `d_datekey`, - cast(`d_year` || '-' || `d_monthnuminyear` || '-' || `d_daynuminmonth` as timestamp), - `d_date`, - `d_dayofweek`, - `d_month`, - `d_year`, - `d_yearmonthnum`, - `d_yearmonth`, - `d_daynuminweek`, - `d_daynuminmonth`, - `d_daynuminyear`, - `d_monthnuminyear`, - `d_weeknuminyear`, - `d_sellingseason`, - `d_lastdayinweekfl`, - `d_lastdayinmonthfl`, - `d_holidayfl`, - `d_weekdayfl` -FROM `dates_ext_n0`; - -CREATE TABLE `ssb_part_ext_n0`( - `p_partkey` bigint, - `p_name` string, - `p_mfgr` string, - `p_category` string, - `p_brand1` string, - `p_color` string, - `p_type` string, - `p_size` int, - `p_container` string) -ROW FORMAT DELIMITED -FIELDS TERMINATED BY '|' -STORED AS TEXTFILE; - -LOAD DATA LOCAL INPATH '../../data/files/ssb/part/' into table `ssb_part_ext_n0`; - -CREATE TABLE `ssb_part_n0`( - `p_partkey` bigint, - `p_name` string, - `p_mfgr` string, - `p_category` string, - `p_brand1` string, - `p_color` string, - `p_type` string, - `p_size` int, - `p_container` string, - primary key (`p_partkey`) disable rely) -STORED AS ORC -TBLPROPERTIES ('transactional'='true'); - -INSERT INTO `ssb_part_n0` -SELECT * FROM `ssb_part_ext_n0`; - -CREATE TABLE `supplier_ext_n0`( - `s_suppkey` bigint, - `s_name` string, - `s_address` string, - `s_city` string, - `s_nation` string, - `s_region` string, - `s_phone` string) -ROW FORMAT DELIMITED -FIELDS TERMINATED BY '|' -STORED AS TEXTFILE; - -LOAD DATA LOCAL INPATH '../../data/files/ssb/supplier/' into table `supplier_ext_n0`; - -CREATE TABLE `supplier_n0`( - `s_suppkey` bigint, - `s_name` string, - `s_address` string, - `s_city` string, - `s_nation` string, - `s_region` string, - `s_phone` string, - primary key (`s_suppkey`) disable rely) -STORED AS ORC -TBLPROPERTIES ('transactional'='true'); - -INSERT INTO `supplier_n0` -SELECT * FROM `supplier_ext_n0`; - -CREATE TABLE `lineorder_ext_n0`( - `lo_orderkey` bigint, - `lo_linenumber` int, - `lo_custkey` bigint not null disable rely, - `lo_partkey` bigint not null disable rely, - `lo_suppkey` bigint not null disable rely, - `lo_orderdate` bigint not null disable rely, - `lo_ordpriority` string, - `lo_shippriority` string, - `lo_quantity` double, - `lo_extendedprice` double, - `lo_ordtotalprice` double, - `lo_discount` double, - `lo_revenue` double, - `lo_supplycost` double, - `lo_tax` double, - `lo_commitdate` bigint, - `lo_shipmode` string) -ROW FORMAT DELIMITED -FIELDS TERMINATED BY '|' -STORED AS TEXTFILE; - -LOAD DATA LOCAL INPATH '../../data/files/ssb/lineorder/' into table `lineorder_ext_n0`; - -CREATE TABLE `lineorder_n0`( - `lo_orderkey` bigint, - `lo_linenumber` int, - `lo_custkey` bigint not null disable rely, - `lo_partkey` bigint not null disable rely, - `lo_suppkey` bigint not null disable rely, - `lo_orderdate` bigint not null disable rely, - `lo_ordpriority` string, - `lo_shippriority` string, - `lo_quantity` double, - `lo_extendedprice` double, - `lo_ordtotalprice` double, - `lo_discount` double, - `lo_revenue` double, - `lo_supplycost` double, - `lo_tax` double, - `lo_commitdate` bigint, - `lo_shipmode` string, - primary key (`lo_orderkey`) disable rely, - constraint fk1 foreign key (`lo_custkey`) references `customer_n0`(`c_custkey`) disable rely, - constraint fk2 foreign key (`lo_orderdate`) references `dates_n0`(`d_datekey`) disable rely, - constraint fk3 foreign key (`lo_partkey`) references `ssb_part_n0`(`p_partkey`) disable rely, - constraint fk4 foreign key (`lo_suppkey`) references `supplier_n0`(`s_suppkey`) disable rely) -STORED AS ORC -TBLPROPERTIES ('transactional'='true'); - -INSERT INTO `lineorder_n0` -SELECT * FROM `lineorder_ext_n0`; - - --- CREATE MV -CREATE MATERIALIZED VIEW `ssb_mv_druid_100` -STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' -TBLPROPERTIES ("druid.segment.granularity" = "MONTH") -AS -SELECT - `__time` as `__time` , - cast(c_city as string) c_city, - cast(c_nation as string) c_nation, - cast(c_region as string) c_region, - c_mktsegment as c_mktsegment, - cast(d_weeknuminyear as string) d_weeknuminyear, - cast(d_year as string) d_year, - cast(d_yearmonth as string) d_yearmonth, - cast(d_yearmonthnum as string) d_yearmonthnum, - cast(p_brand1 as string) p_brand1, - cast(p_category as string) p_category, - cast(p_mfgr as string) p_mfgr, - p_type, - s_name, - cast(s_city as string) s_city, - cast(s_nation as string) s_nation, - cast(s_region as string) s_region, - cast(`lo_ordpriority` as string) lo_ordpriority, - cast(`lo_shippriority` as string) lo_shippriority, - `d_sellingseason` - `lo_shipmode`, - lo_revenue, - lo_supplycost , - lo_discount , - `lo_quantity`, - `lo_extendedprice`, - `lo_ordtotalprice`, - lo_extendedprice * lo_discount discounted_price, - lo_revenue - lo_supplycost net_revenue -FROM - customer_n0, dates_n0, lineorder_n0, ssb_part_n0, supplier_n0 -where - lo_orderdate = d_datekey - and lo_partkey = p_partkey - and lo_suppkey = s_suppkey - and lo_custkey = c_custkey; - - --- QUERY OVER MV -EXPLAIN CBO -SELECT MONTH(`__time`) AS `mn___time_ok`, -CAST((MONTH(`__time`) - 1) / 3 + 1 AS BIGINT) AS `qr___time_ok`, -SUM(1) AS `sum_number_of_records_ok`, -YEAR(`__time`) AS `yr___time_ok` -FROM `ssb_mv_druid_100` -GROUP BY MONTH(`__time`), -CAST((MONTH(`__time`) - 1) / 3 + 1 AS BIGINT), -YEAR(`__time`); - -SELECT MONTH(`__time`) AS `mn___time_ok`, -CAST((MONTH(`__time`) - 1) / 3 + 1 AS BIGINT) AS `qr___time_ok`, -SUM(1) AS `sum_number_of_records_ok`, -YEAR(`__time`) AS `yr___time_ok` -FROM `ssb_mv_druid_100` -GROUP BY MONTH(`__time`), -CAST((MONTH(`__time`) - 1) / 3 + 1 AS BIGINT), -YEAR(`__time`); - - --- QUERY OVER ORIGINAL TABLES -EXPLAIN CBO -SELECT MONTH(`dates_n0`.`__time`) AS `mn___time_ok`, -CAST((MONTH(`dates_n0`.`__time`) - 1) / 3 + 1 AS BIGINT) AS `qr___time_ok`, -SUM(1) AS `sum_number_of_records_ok`, -YEAR(`dates_n0`.`__time`) AS `yr___time_ok` -FROM `lineorder_n0` `lineorder_n0` -JOIN `dates_n0` `dates_n0` ON (`lineorder_n0`.`lo_orderdate` = `dates_n0`.`d_datekey`) -JOIN `customer_n0` `customer_n0` ON (`lineorder_n0`.`lo_custkey` = `customer_n0`.`c_custkey`) -JOIN `supplier_n0` `supplier_n0` ON (`lineorder_n0`.`lo_suppkey` = `supplier_n0`.`s_suppkey`) -JOIN `ssb_part_n0` `ssb_part_n0` ON (`lineorder_n0`.`lo_partkey` = `ssb_part_n0`.`p_partkey`) -GROUP BY MONTH(`dates_n0`.`__time`), -CAST((MONTH(`dates_n0`.`__time`) - 1) / 3 + 1 AS BIGINT), -YEAR(`dates_n0`.`__time`); - -SELECT MONTH(`dates_n0`.`__time`) AS `mn___time_ok`, -CAST((MONTH(`dates_n0`.`__time`) - 1) / 3 + 1 AS BIGINT) AS `qr___time_ok`, -SUM(1) AS `sum_number_of_records_ok`, -YEAR(`dates_n0`.`__time`) AS `yr___time_ok` -FROM `lineorder_n0` `lineorder_n0` -JOIN `dates_n0` `dates_n0` ON (`lineorder_n0`.`lo_orderdate` = `dates_n0`.`d_datekey`) -JOIN `customer_n0` `customer_n0` ON (`lineorder_n0`.`lo_custkey` = `customer_n0`.`c_custkey`) -JOIN `supplier_n0` `supplier_n0` ON (`lineorder_n0`.`lo_suppkey` = `supplier_n0`.`s_suppkey`) -JOIN `ssb_part_n0` `ssb_part_n0` ON (`lineorder_n0`.`lo_partkey` = `ssb_part_n0`.`p_partkey`) -GROUP BY MONTH(`dates_n0`.`__time`), -CAST((MONTH(`dates_n0`.`__time`) - 1) / 3 + 1 AS BIGINT), -YEAR(`dates_n0`.`__time`); - -DROP MATERIALIZED VIEW `ssb_mv_druid_100`; diff --git a/ql/src/test/queries/clientpositive/druid_timeseries.q b/ql/src/test/queries/clientpositive/druid_timeseries.q deleted file mode 100644 index 6ff7d590ea38..000000000000 --- a/ql/src/test/queries/clientpositive/druid_timeseries.q +++ /dev/null @@ -1,111 +0,0 @@ -set hive.druid.broker.address.default=localhost.test; - -CREATE EXTERNAL TABLE druid_table_1_n3 -STORED BY 'org.apache.hadoop.hive.druid.QTestDruidStorageHandler' -TBLPROPERTIES ("druid.datasource" = "wikipedia"); - --- DESCRIBE FORMATTED druid_table_1_n3; - --- GRANULARITY: ALL -EXPLAIN SELECT count(`__time`) from druid_table_1_n3 where `__time` >= '2010-01-01 00:00:00 UTC' AND `__time` <= '2012-03-01 00:00:00 UTC' OR added <= 0; - -EXPLAIN SELECT count(`__time`) from druid_table_1_n3 where `__time` <= '2010-01-01 00:00:00 UTC'; - -EXPLAIN -SELECT max(added), sum(variation) -FROM druid_table_1_n3; - --- GRANULARITY: NONE -EXPLAIN -SELECT `__time`, max(added), sum(variation) -FROM druid_table_1_n3 -GROUP BY `__time`; - --- GRANULARITY: YEAR -EXPLAIN -SELECT floor_year(`__time`), max(added), sum(variation) -FROM druid_table_1_n3 -GROUP BY floor_year(`__time`); - --- @TODO FIXME https://issues.apache.org/jira/browse/CALCITE-2222 --- The current plan of this query is not optimal it can be planned as time series instead of scan --- GRANULARITY: QUARTER -EXPLAIN -SELECT floor_quarter(`__time`), max(added), sum(variation) -FROM druid_table_1_n3 -GROUP BY floor_quarter(`__time`); - --- GRANULARITY: MONTH -EXPLAIN -SELECT floor_month(`__time`), max(added), sum(variation) -FROM druid_table_1_n3 -GROUP BY floor_month(`__time`); - --- GRANULARITY: WEEK -EXPLAIN -SELECT floor_week(`__time`), max(added), sum(variation) -FROM druid_table_1_n3 -GROUP BY floor_week(`__time`); - --- GRANULARITY: DAY -EXPLAIN -SELECT floor_day(`__time`), max(added), sum(variation) -FROM druid_table_1_n3 -GROUP BY floor_day(`__time`); - --- GRANULARITY: HOUR -EXPLAIN -SELECT floor_hour(`__time`), max(added), sum(variation) -FROM druid_table_1_n3 -GROUP BY floor_hour(`__time`); - --- GRANULARITY: MINUTE -EXPLAIN -SELECT floor_minute(`__time`), max(added), sum(variation) -FROM druid_table_1_n3 -GROUP BY floor_minute(`__time`); - --- GRANULARITY: SECOND -EXPLAIN -SELECT floor_second(`__time`), max(added), sum(variation) -FROM druid_table_1_n3 -GROUP BY floor_second(`__time`); - --- WITH FILTER ON DIMENSION -EXPLAIN -SELECT floor_hour(`__time`), max(added), sum(variation) -FROM druid_table_1_n3 -WHERE robot='1' -GROUP BY floor_hour(`__time`); - --- WITH FILTER ON TIME -EXPLAIN -SELECT floor_hour(`__time`), max(added), sum(variation) -FROM druid_table_1_n3 -WHERE floor_hour(`__time`) - BETWEEN CAST('2010-01-01 00:00:00' AS TIMESTAMP WITH LOCAL TIME ZONE) - AND CAST('2014-01-01 00:00:00' AS TIMESTAMP WITH LOCAL TIME ZONE) -GROUP BY floor_hour(`__time`); - --- WITH FILTER ON TIME -EXPLAIN -SELECT subq.h, subq.m, subq.s -FROM -( - SELECT floor_hour(`__time`) as h, max(added) as m, sum(variation) as s - FROM druid_table_1_n3 - GROUP BY floor_hour(`__time`) -) subq -WHERE subq.h BETWEEN CAST('2010-01-01 00:00:00' AS TIMESTAMP WITH LOCAL TIME ZONE) - AND CAST('2014-01-01 00:00:00' AS TIMESTAMP WITH LOCAL TIME ZONE); - --- Simplification of count(__time) as count(*) since time column is not null -EXPLAIN SELECT count(`__time`) from druid_table_1_n3; - - -EXPLAIN SELECT count(`__time`) from druid_table_1_n3 where `__time` <= '2010-01-01 00:00:00 UTC'; - -EXPLAIN SELECT count(`__time`) from druid_table_1_n3 where `__time` >= '2010-01-01 00:00:00'; - - -EXPLAIN SELECT count(`__time`) from druid_table_1_n3 where `__time` <= '2010-01-01 00:00:00' OR `__time` <= '2012-03-01 00:00:00'; diff --git a/ql/src/test/queries/clientpositive/druid_timestamptz.q b/ql/src/test/queries/clientpositive/druid_timestamptz.q deleted file mode 100644 index 1c98594dd5c6..000000000000 --- a/ql/src/test/queries/clientpositive/druid_timestamptz.q +++ /dev/null @@ -1,94 +0,0 @@ ---! qt:disabled:Disabled in HIVE-20322 - -set hive.fetch.task.conversion=more; -SET hive.ctas.external.tables=true; -SET hive.external.table.purge.default = true; - -drop table tstz1_n0; - -create external table tstz1_n0(`__time` timestamp with local time zone, n string, v integer) -STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' -TBLPROPERTIES ("druid.segment.granularity" = "HOUR"); - -insert into table tstz1_n0 -values(cast('2016-01-03 12:26:34 America/Los_Angeles' as timestamp with local time zone), 'Bill', 10); - --- Create table with druid time column as timestamp -create table tstz1_n1(`__time` timestamp, n string, v integer) -STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' -TBLPROPERTIES ("druid.segment.granularity" = "HOUR"); - -insert into table tstz1_n1 -values(cast('2016-01-03 12:26:34' as timestamp), 'Bill', 10); - -EXPLAIN select `__time` from tstz1_n0; -select `__time` from tstz1_n0; - -EXPLAIN select cast(`__time` as timestamp) from tstz1_n0; -select cast(`__time` as timestamp) from tstz1_n0; - -EXPLAIN select cast(`__time` as timestamp) from tstz1_n0 where `__time` >= cast('2016-01-03 12:26:34 America/Los_Angeles' as timestamp with local time zone); -select cast(`__time` as timestamp) from tstz1_n0 where `__time` >= cast('2016-01-03 12:26:34 America/Los_Angeles' as timestamp with local time zone); - -EXPLAIN SELECT EXTRACT(HOUR FROM CAST(`__time` AS timestamp)) FROM tstz1_n0; -SELECT EXTRACT(HOUR FROM CAST(`__time` AS timestamp)) FROM tstz1_n0; - -EXPLAIN SELECT FLOOR(CAST(`__time` AS timestamp) to HOUR) FROM tstz1_n0; -SELECT FLOOR(CAST(`__time` AS timestamp) to HOUR) FROM tstz1_n0; - -EXPLAIN SELECT `__time`, max(v) FROM tstz1_n0 GROUP BY `__time`; -SELECT `__time`, max(v) FROM tstz1_n0 GROUP BY `__time`; - -EXPLAIN select `__time` from tstz1_n1; -select `__time` from tstz1_n1; - -EXPLAIN SELECT EXTRACT(HOUR FROM CAST(`__time` AS timestamp)) FROM tstz1_n1; -SELECT EXTRACT(HOUR FROM CAST(`__time` AS timestamp)) FROM tstz1_n1; - -EXPLAIN SELECT FLOOR(CAST(`__time` AS timestamp) to HOUR) FROM tstz1_n1; -SELECT FLOOR(CAST(`__time` AS timestamp) to HOUR) FROM tstz1_n1; - -EXPLAIN SELECT `__time`, max(v) FROM tstz1_n1 GROUP BY `__time`; -SELECT `__time`, max(v) FROM tstz1_n1 GROUP BY `__time`; - --- Change timezone to UTC and test again -set time zone UTC; -EXPLAIN select `__time` from tstz1_n0; -select `__time` from tstz1_n0; -EXPLAIN select cast(`__time` as timestamp) from tstz1_n0; -select cast(`__time` as timestamp) from tstz1_n0; -EXPLAIN select cast(`__time` as timestamp) from tstz1_n0 where `__time` >= cast('2016-01-03 12:26:34 America/Los_Angeles' as timestamp with local time zone); -select cast(`__time` as timestamp) from tstz1_n0 where `__time` >= cast('2016-01-03 12:26:34 America/Los_Angeles' as timestamp with local time zone); - --- THIS is failing explore why ---EXPLAIN select cast(`__time` as timestamp) from tstz1_n0 where `__time` = cast('2016-01-03 12:26:34 America/Los_Angeles' as timestamp with local time zone); ---select cast(`__time` as timestamp) from tstz1_n0 where `__time` = cast('2016-01-03 12:26:34 America/Los_Angeles' as timestamp with local time zone); - -EXPLAIN select cast(`__time` as timestamp) from tstz1_n0 where `__time` >= cast('2016-01-03 20:26:34' as timestamp); -select cast(`__time` as timestamp) from tstz1_n0 where `__time` >= cast('2016-01-03 20:26:34' as timestamp); - -EXPLAIN select cast(`__time` as timestamp) from tstz1_n0 where `__time` >= cast('2016-01-03 12:26:34 America/Los_Angeles' as timestamp with local time zone) AND `__time` <= cast('2016-01-03 12:26:34 America/Los_Angeles' as timestamp with local time zone); -select cast(`__time` as timestamp) from tstz1_n0 where `__time` >= cast('2016-01-03 12:26:34 America/Los_Angeles' as timestamp with local time zone) AND `__time` <= cast('2016-01-03 12:26:34 America/Los_Angeles' as timestamp with local time zone); - -EXPLAIN SELECT EXTRACT(HOUR FROM CAST(`__time` AS timestamp)) FROM tstz1_n0; -SELECT EXTRACT(HOUR FROM CAST(`__time` AS timestamp)) FROM tstz1_n0; - -EXPLAIN SELECT FLOOR(CAST(`__time` AS timestamp) to HOUR) FROM tstz1_n0; -SELECT FLOOR(CAST(`__time` AS timestamp) to HOUR) FROM tstz1_n0; - -EXPLAIN SELECT `__time`, max(v) FROM tstz1_n0 GROUP BY `__time`; -SELECT `__time`, max(v) FROM tstz1_n0 GROUP BY `__time`; - -EXPLAIN select `__time` from tstz1_n1; -select `__time` from tstz1_n1; - -EXPLAIN SELECT EXTRACT(HOUR FROM CAST(`__time` AS timestamp)) FROM tstz1_n1; -SELECT EXTRACT(HOUR FROM CAST(`__time` AS timestamp)) FROM tstz1_n1; - -EXPLAIN SELECT FLOOR(CAST(`__time` AS timestamp) to HOUR) FROM tstz1_n1; -SELECT FLOOR(CAST(`__time` AS timestamp) to HOUR) FROM tstz1_n1; - -EXPLAIN SELECT `__time`, max(v) FROM tstz1_n1 GROUP BY `__time`; -SELECT `__time`, max(v) FROM tstz1_n1 GROUP BY `__time`; - - diff --git a/ql/src/test/queries/clientpositive/druid_timestamptz2.q b/ql/src/test/queries/clientpositive/druid_timestamptz2.q deleted file mode 100644 index 8245b93cf1c0..000000000000 --- a/ql/src/test/queries/clientpositive/druid_timestamptz2.q +++ /dev/null @@ -1,64 +0,0 @@ ---! qt:disabled:HIVE-24816 -SET hive.ctas.external.tables=true; -SET hive.external.table.purge.default = true; -CREATE database druid_test_dst; -use druid_test_dst; - -create external table test_base_table(`timecolumn` timestamp, `interval_marker` string, `num_l` double); -insert into test_base_table values -('2015-03-08 00:00:00', 'i1-start', 4), -('2015-03-08 23:59:59', 'i1-end', 1), -('2015-03-09 00:00:00', 'i2-start', 4), -('2015-03-09 23:59:59', 'i2-end', 1), -('2015-03-10 00:00:00', 'i3-start', 2), -('2015-03-10 23:59:59', 'i3-end', 2); - -CREATE EXTERNAL TABLE druid_test_table_1 -STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' -TBLPROPERTIES ("druid.segment.granularity" = "DAY") -AS -select cast(`timecolumn` as timestamp with local time zone) as `__time`, `interval_marker`, `num_l` -FROM druid_test_dst.test_base_table; - -select * FROM druid_test_table_1; - -CREATE EXTERNAL TABLE druid_test_table_2 (`__time` timestamp with local time zone, `interval_marker` string, `num_l` double) -STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' -TBLPROPERTIES ("druid.segment.granularity" = "DAY"); - - -insert into druid_test_table_2 values -(cast('2015-03-08 00:00:00' as timestamp with local time zone), 'i1-start', 4), -(cast('2015-03-08 23:59:59' as timestamp with local time zone), 'i1-end', 1), -(cast('2015-03-09 00:00:00' as timestamp with local time zone), 'i2-start', 4), -(cast('2015-03-09 23:59:59' as timestamp with local time zone), 'i2-end', 1), -(cast('2015-03-10 00:00:00' as timestamp with local time zone), 'i3-start', 2), -(cast('2015-03-10 23:59:59' as timestamp with local time zone), 'i3-end', 2); - -select * FROM druid_test_table_2; - -SET TIME ZONE UTC; - -CREATE EXTERNAL TABLE druid_test_table_utc -STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' -TBLPROPERTIES ("druid.segment.granularity" = "DAY") -AS -select cast(`timecolumn` as timestamp with local time zone) as `__time`, `interval_marker`, `num_l` -FROM druid_test_dst.test_base_table; - -select * FROM druid_test_table_utc; - -CREATE EXTERNAL TABLE druid_test_table_utc2 (`__time` timestamp with local time zone, `interval_marker` string, `num_l` double) -STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' -TBLPROPERTIES ("druid.segment.granularity" = "DAY"); - - -insert into druid_test_table_utc2 values -(cast('2015-03-08 00:00:00' as timestamp with local time zone), 'i1-start', 4), -(cast('2015-03-08 23:59:59' as timestamp with local time zone), 'i1-end', 1), -(cast('2015-03-09 00:00:00' as timestamp with local time zone), 'i2-start', 4), -(cast('2015-03-09 23:59:59' as timestamp with local time zone), 'i2-end', 1), -(cast('2015-03-10 00:00:00' as timestamp with local time zone), 'i3-start', 2), -(cast('2015-03-10 23:59:59' as timestamp with local time zone), 'i3-end', 2); - -select * FROM druid_test_table_utc2; diff --git a/ql/src/test/queries/clientpositive/druid_topn.q b/ql/src/test/queries/clientpositive/druid_topn.q deleted file mode 100644 index 2aa6aff9544a..000000000000 --- a/ql/src/test/queries/clientpositive/druid_topn.q +++ /dev/null @@ -1,75 +0,0 @@ -set hive.druid.broker.address.default=localhost.test; - -CREATE EXTERNAL TABLE druid_table_1_n1 -STORED BY 'org.apache.hadoop.hive.druid.QTestDruidStorageHandler' -TBLPROPERTIES ("druid.datasource" = "wikipedia"); - -DESCRIBE FORMATTED druid_table_1_n1; - --- GRANULARITY: ALL -EXPLAIN -SELECT robot, max(added) as m, sum(variation) -FROM druid_table_1_n1 -GROUP BY robot -ORDER BY m DESC -LIMIT 100; - --- GRANULARITY: NONE -EXPLAIN -SELECT robot, `__time`, max(added), sum(variation) as s -FROM druid_table_1_n1 -GROUP BY robot, `__time` -ORDER BY s DESC -LIMIT 100; - --- GRANULARITY: YEAR -EXPLAIN -SELECT robot, floor_year(`__time`), max(added), sum(variation) as s -FROM druid_table_1_n1 -GROUP BY robot, floor_year(`__time`) -ORDER BY s DESC -LIMIT 10; - --- ASC: TRANSFORM INTO GROUP BY -EXPLAIN -SELECT robot, floor_month(`__time`), max(added), sum(variation) as s -FROM druid_table_1_n1 -GROUP BY robot, floor_month(`__time`) -ORDER BY s -LIMIT 10; - --- MULTIPLE ORDER: TRANSFORM INTO GROUP BY -EXPLAIN -SELECT robot, floor_month(`__time`), max(added) as m, sum(variation) as s -FROM druid_table_1_n1 -GROUP BY robot, namespace, floor_month(`__time`) -ORDER BY s DESC, m DESC -LIMIT 10; - --- MULTIPLE ORDER MIXED: TRANSFORM INTO GROUP BY -EXPLAIN -SELECT robot, floor_month(`__time`), max(added) as m, sum(variation) as s -FROM druid_table_1_n1 -GROUP BY robot, namespace, floor_month(`__time`) -ORDER BY robot ASC, m DESC -LIMIT 10; - --- WITH FILTER ON DIMENSION: TRANSFORM INTO GROUP BY -EXPLAIN -SELECT robot, floor_year(`__time`), max(added), sum(variation) as s -FROM druid_table_1_n1 -WHERE robot='1' -GROUP BY robot, floor_year(`__time`) -ORDER BY s -LIMIT 10; - --- WITH FILTER ON TIME -EXPLAIN -SELECT robot, floor_hour(`__time`), max(added) as m, sum(variation) -FROM druid_table_1_n1 -WHERE floor_hour(`__time`) - BETWEEN CAST('2010-01-01 00:00:00' AS TIMESTAMP WITH LOCAL TIME ZONE) - AND CAST('2014-01-01 00:00:00' AS TIMESTAMP WITH LOCAL TIME ZONE) -GROUP BY robot, floor_hour(`__time`) -ORDER BY m -LIMIT 100; diff --git a/ql/src/test/queries/clientpositive/druidkafkamini_avro.q b/ql/src/test/queries/clientpositive/druidkafkamini_avro.q deleted file mode 100644 index 2e72e76b216a..000000000000 --- a/ql/src/test/queries/clientpositive/druidkafkamini_avro.q +++ /dev/null @@ -1,100 +0,0 @@ ---! qt:disabled:HIVE-24816 -SET hive.vectorized.execution.enabled=false; - -CREATE EXTERNAL TABLE druid_kafka_test_avro(`__time` timestamp , `page` string, `user` string, `language` string, - `country` string,`continent` string, `namespace` string, `newPage` boolean, `unpatrolled` boolean, - `anonymous` boolean, `robot` boolean, added int, deleted int, delta bigint) - STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' - TBLPROPERTIES ( - "druid.segment.granularity" = "MONTH", - "druid.query.granularity" = "MINUTE", - "kafka.bootstrap.servers" = "localhost:9092", - "kafka.topic" = "wiki_kafka_avro_table", - "druid.kafka.ingestion.useEarliestOffset" = "true", - "druid.kafka.ingestion.maxRowsInMemory" = "5", - "druid.kafka.ingestion.startDelay" = "PT1S", - "druid.kafka.ingestion.taskDuration" = "PT30S", - "druid.kafka.ingestion.period" = "PT5S", - "druid.kafka.ingestion.consumer.retries" = "2", - "druid.kafka.ingestion.reportParseExceptions" = "true", - "druid.timestamp.column" = "timestamp", - "druid.timestamp.format" = "MM/dd/yyyy HH:mm:ss", - "druid.parseSpec.format" = "avro", - 'avro.schema.literal'='{ - "type" : "record", - "name" : "Wikipedia", - "namespace" : "org.apache.hive.kafka", - "version": "1", - "fields" : [ { - "name" : "isrobot", - "type" : "boolean" - }, { - "name" : "channel", - "type" : "string" - }, { - "name" : "timestamp", - "type" : "string" - }, { - "name" : "flags", - "type" : "string" - }, { - "name" : "isunpatrolled", - "type" : "boolean" - }, { - "name" : "page", - "type" : "string" - }, { - "name" : "diffurl", - "type" : "string" - }, { - "name" : "added", - "type" : "long" - }, { - "name" : "comment", - "type" : "string" - }, { - "name" : "commentlength", - "type" : "long" - }, { - "name" : "isnew", - "type" : "boolean" - }, { - "name" : "isminor", - "type" : "boolean" - }, { - "name" : "delta", - "type" : "long" - }, { - "name" : "isanonymous", - "type" : "boolean" - }, { - "name" : "user", - "type" : "string" - }, { - "name" : "deltabucket", - "type" : "double" - }, { - "name" : "deleted", - "type" : "long" - }, { - "name" : "namespace", - "type" : "string" - } ] - }' - ); - -ALTER TABLE druid_kafka_test_avro SET TBLPROPERTIES('druid.kafka.ingestion' = 'START'); - -!curl --noproxy * -ss http://localhost:8081/druid/indexer/v1/supervisor; - --- Sleep for some time for ingestion tasks to ingest events -!sleep 60; - -DESCRIBE druid_kafka_test_avro; -DESCRIBE EXTENDED druid_kafka_test_avro; - -Select count(*) FROM druid_kafka_test_avro; - -Select page FROM druid_kafka_test_avro; - -DROP TABLE druid_kafka_test_avro; diff --git a/ql/src/test/queries/clientpositive/druidkafkamini_basic.q b/ql/src/test/queries/clientpositive/druidkafkamini_basic.q deleted file mode 100644 index bb7554953928..000000000000 --- a/ql/src/test/queries/clientpositive/druidkafkamini_basic.q +++ /dev/null @@ -1,131 +0,0 @@ ---! qt:disabled:unstable resultset HIVE-23694 - -SET hive.vectorized.execution.enabled=true ; -CREATE EXTERNAL TABLE druid_kafka_test(`__time` timestamp, page string, `user` string, language string, added int, deleted int) - STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' - TBLPROPERTIES ( - "druid.segment.granularity" = "MONTH", - "druid.query.granularity" = "MINUTE", - "kafka.bootstrap.servers" = "localhost:9092", - "kafka.topic" = "test-topic", - "druid.kafka.ingestion.useEarliestOffset" = "true", - "druid.kafka.ingestion.maxRowsInMemory" = "5", - "druid.kafka.ingestion.startDelay" = "PT1S", - "druid.kafka.ingestion.taskDuration" = "PT60S", - "druid.kafka.ingestion.period" = "PT1S", - "druid.kafka.ingestion.consumer.retries" = "2" - ); - -ALTER TABLE druid_kafka_test SET TBLPROPERTIES('druid.kafka.ingestion' = 'START'); - -!curl --noproxy * -ss http://localhost:8081/druid/indexer/v1/supervisor; - --- Sleep for some time for ingestion tasks to ingest events -!sleep 60; - -DESCRIBE druid_kafka_test; -DESCRIBE EXTENDED druid_kafka_test; - -Select count(*) FROM druid_kafka_test; - -Select page FROM druid_kafka_test order by page; - --- Reset kafka Ingestion, this would reset the offsets and since we are using useEarliestOffset, --- We will see records duplicated after successful reset. -ALTER TABLE druid_kafka_test SET TBLPROPERTIES('druid.kafka.ingestion' = 'RESET'); - --- Sleep for some time for ingestion tasks to ingest events -!sleep 60; - -DESCRIBE druid_kafka_test; -DESCRIBE EXTENDED druid_kafka_test; - -Select count(*) FROM druid_kafka_test; - -Select page FROM druid_kafka_test order by page; - --- Join against other normal tables -CREATE TABLE languages(shortname string, fullname string); - -INSERT INTO languages values -("en", "english"), -("ru", "russian"); - -EXPLAIN EXTENDED -SELECT a.fullname, b.`user` -FROM -( -(SELECT fullname, shortname -FROM languages) a -JOIN -(SELECT language, `user` -FROM druid_kafka_test) b - ON a.shortname = b.language -); - -SELECT a.fullname, b.`user` -FROM -( -(SELECT fullname, shortname -FROM languages) a -JOIN -(SELECT language, `user` -FROM druid_kafka_test) b - ON a.shortname = b.language -) order by b.`user`; - - -EXPLAIN -SELECT language, -1 * (a + b) AS c -FROM ( - SELECT (sum(added)-sum(deleted)) / (count(*) * 3) AS a, sum(deleted) AS b, language - FROM druid_kafka_test - GROUP BY language) subq -ORDER BY c DESC; - -EXPLAIN -SELECT language, `user`, sum(added) - sum(deleted) AS a -FROM druid_kafka_test -WHERE extract (week from `__time`) IN (10,11) -GROUP BY language, `user`; - -EXPLAIN -SELECT language, sum(deleted) / count(*) AS a -FROM druid_kafka_test -GROUP BY language -ORDER BY a DESC; - -EXPLAIN -SELECT language, sum(added) / sum(deleted) AS a, - CASE WHEN sum(deleted)=0 THEN 1.0 ELSE sum(deleted) END AS b -FROM druid_kafka_test -GROUP BY language -ORDER BY a DESC; - -EXPLAIN -SELECT language, a, a - b as c -FROM ( - SELECT language, sum(added) + 100 AS a, sum(deleted) AS b - FROM druid_kafka_test - GROUP BY language) subq -ORDER BY a DESC; - -EXPLAIN -SELECT language, `user`, "A" -FROM ( - SELECT sum(added) - sum(deleted) AS a, language, `user` - FROM druid_kafka_test - GROUP BY language, `user` ) subq -ORDER BY "A" -LIMIT 5; - -EXPLAIN -SELECT language, `user`, "A" -FROM ( - SELECT language, sum(added) + sum(deleted) AS a, `user` - FROM druid_kafka_test - GROUP BY language, `user`) subq -ORDER BY `user`, language -LIMIT 5; - -DROP TABLE druid_kafka_test; diff --git a/ql/src/test/queries/clientpositive/druidkafkamini_csv.q b/ql/src/test/queries/clientpositive/druidkafkamini_csv.q deleted file mode 100644 index 43fdf37e9128..000000000000 --- a/ql/src/test/queries/clientpositive/druidkafkamini_csv.q +++ /dev/null @@ -1,38 +0,0 @@ ---! qt:disabled:HIVE-24816 -SET hive.vectorized.execution.enabled=false; - -CREATE EXTERNAL TABLE druid_kafka_test_csv(`__time` timestamp , `page` string, `user` string, `language` string, - `country` string,`continent` string, `namespace` string, `newpage` boolean, `unpatrolled` boolean, - `anonymous` boolean, `robot` boolean, added int, deleted int, delta bigint) - STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' - TBLPROPERTIES ( - "druid.segment.granularity" = "MONTH", - "druid.query.granularity" = "MINUTE", - "kafka.bootstrap.servers" = "localhost:9092", - "kafka.topic" = "wiki_kafka_csv", - "druid.kafka.ingestion.useEarliestOffset" = "true", - "druid.kafka.ingestion.maxRowsInMemory" = "5", - "druid.kafka.ingestion.startDelay" = "PT1S", - "druid.kafka.ingestion.taskDuration" = "PT30S", - "druid.kafka.ingestion.period" = "PT5S", - "druid.kafka.ingestion.consumer.retries" = "2", - "druid.kafka.ingestion.reportParseExceptions" = "true", - "druid.parseSpec.format" = "csv", - "druid.parseSpec.columns" = "__time,page,language,user,unpatrolled,newpage,robot,anonymous,namespace,continent,country,region,city,added,deleted,delta" - ); - -ALTER TABLE druid_kafka_test_csv SET TBLPROPERTIES('druid.kafka.ingestion' = 'START'); - -!curl --noproxy * -ss http://localhost:8081/druid/indexer/v1/supervisor; - --- Sleep for some time for ingestion tasks to ingest events -!sleep 60; - -DESCRIBE druid_kafka_test_csv; -DESCRIBE EXTENDED druid_kafka_test_csv; - -Select count(*) FROM druid_kafka_test_csv; - -Select page FROM druid_kafka_test_csv; - -DROP TABLE druid_kafka_test_csv; diff --git a/ql/src/test/queries/clientpositive/druidkafkamini_delimited.q b/ql/src/test/queries/clientpositive/druidkafkamini_delimited.q deleted file mode 100644 index f7c64f9f0b66..000000000000 --- a/ql/src/test/queries/clientpositive/druidkafkamini_delimited.q +++ /dev/null @@ -1,39 +0,0 @@ ---! qt:disabled:this started falling after the druid upgrade just like HIVE-23450 -SET hive.vectorized.execution.enabled=false; - -CREATE EXTERNAL TABLE druid_kafka_test_delimited(`__time` timestamp , `page` string, `user` string, `language` string, - `country` string,`continent` string, `namespace` string, `newpage` boolean, `unpatrolled` boolean, - `anonymous` boolean, `robot` boolean, added int, deleted int, delta bigint) - STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' - TBLPROPERTIES ( - "druid.segment.granularity" = "MONTH", - "druid.query.granularity" = "MINUTE", - "kafka.bootstrap.servers" = "localhost:9092", - "kafka.topic" = "wiki_kafka_csv", - "druid.kafka.ingestion.useEarliestOffset" = "true", - "druid.kafka.ingestion.maxRowsInMemory" = "5", - "druid.kafka.ingestion.startDelay" = "PT1S", - "druid.kafka.ingestion.taskDuration" = "PT30S", - "druid.kafka.ingestion.period" = "PT5S", - "druid.kafka.ingestion.consumer.retries" = "2", - "druid.kafka.ingestion.reportParseExceptions" = "true", - "druid.parseSpec.format" = "delimited", - "druid.parseSpec.columns" = "__time,page,language,user,unpatrolled,newpage,robot,anonymous,namespace,continent,country,region,city,added,deleted,delta", - "druid.parseSpec.delimiter"="," - ); - -ALTER TABLE druid_kafka_test_delimited SET TBLPROPERTIES('druid.kafka.ingestion' = 'START'); - -!curl --noproxy * -ss http://localhost:8081/druid/indexer/v1/supervisor; - --- Sleep for some time for ingestion tasks to ingest events -!sleep 60; - -DESCRIBE druid_kafka_test_delimited; -DESCRIBE EXTENDED druid_kafka_test_delimited; - -Select count(*) FROM druid_kafka_test_delimited; - -Select page FROM druid_kafka_test_delimited; - -DROP TABLE druid_kafka_test_delimited; diff --git a/ql/src/test/queries/clientpositive/druidmini_dynamic_partition.q b/ql/src/test/queries/clientpositive/druidmini_dynamic_partition.q deleted file mode 100644 index eacdcfe13190..000000000000 --- a/ql/src/test/queries/clientpositive/druidmini_dynamic_partition.q +++ /dev/null @@ -1,190 +0,0 @@ ---! qt:disabled:HIVE-23984 ---! qt:dataset:alltypesorc -SET hive.vectorized.execution.enabled=true ; -SET hive.ctas.external.tables=true; -SET hive.external.table.purge.default = true; -CREATE EXTERNAL TABLE druid_partitioned_table_0 - STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' - TBLPROPERTIES ( - "druid.segment.granularity" = "HOUR", - "druid.query.granularity" = "MINUTE", - "druid.segment.targetShardsPerGranularity" = "0" - ) - AS - SELECT cast (`ctimestamp1` as timestamp with local time zone) as `__time`, - cstring1, - cstring2, - cdouble, - cfloat, - ctinyint, - csmallint, - cint, - cbigint, - cboolean1, - cboolean2 - FROM alltypesorc where ctimestamp1 IS NOT NULL; - -EXPLAIN CREATE EXTERNAL TABLE druid_partitioned_table - STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' - TBLPROPERTIES ( - "druid.segment.granularity" = "HOUR", - "druid.query.granularity" = "MINUTE", - "druid.segment.targetShardsPerGranularity" = "6" - ) - AS - SELECT cast (`ctimestamp1` as timestamp with local time zone) as `__time`, - cstring1, - cstring2, - cdouble, - cfloat, - ctinyint, - csmallint, - cint, - cbigint, - cboolean1, - cboolean2 - FROM alltypesorc where ctimestamp1 IS NOT NULL; - - - -CREATE EXTERNAL TABLE druid_partitioned_table -STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' -TBLPROPERTIES ( -"druid.segment.granularity" = "HOUR", -"druid.query.granularity" = "MINUTE", -"druid.segment.targetShardsPerGranularity" = "6" -) -AS -SELECT cast (`ctimestamp1` as timestamp with local time zone) as `__time`, - cstring1, - cstring2, - cdouble, - cfloat, - ctinyint, - csmallint, - cint, - cbigint, - cboolean1, - cboolean2 - FROM alltypesorc where ctimestamp1 IS NOT NULL; --- @FIXME https://issues.apache.org/jira/browse/HIVE-19011 --- SELECT sum(cint), max(cbigint), sum(cbigint), max(cint) FROM druid_partitioned_table; --- SELECT sum(cint), max(cbigint), sum(cbigint), max(cint) FROM druid_partitioned_table_0; - - -SELECT sum(cint), sum(cbigint) FROM druid_partitioned_table; -SELECT sum(cint), sum(cbigint) FROM druid_partitioned_table_0; - -SELECT floor_hour(cast(`ctimestamp1` as timestamp with local time zone)) as `__time`, - cstring1, - cstring2, - cdouble, - cfloat, - ctinyint, - csmallint, - cint, - cbigint, - cboolean1, - cboolean2 - FROM alltypesorc where ctimestamp1 IS NOT NULL order by `__time`, cstring2 DESC NULLS LAST, cstring1 DESC NULLS LAST LIMIT 10 ; - - -EXPLAIN INSERT INTO TABLE druid_partitioned_table -SELECT cast (`ctimestamp2` as timestamp with local time zone) as `__time`, - cstring1, - cstring2, - cdouble, - cfloat, - ctinyint, - csmallint, - cint, - cbigint, - cboolean1, - cboolean2 - FROM alltypesorc where ctimestamp2 IS NOT NULL; - -INSERT INTO TABLE druid_partitioned_table -SELECT cast (`ctimestamp2` as timestamp with local time zone) as `__time`, - cstring1, - cstring2, - cdouble, - cfloat, - ctinyint, - csmallint, - cint, - cbigint, - cboolean1, - cboolean2 - FROM alltypesorc where ctimestamp2 IS NOT NULL; --- @FIXME https://issues.apache.org/jira/browse/HIVE-19011 --- SELECT sum(cint), max(cbigint), sum(cbigint), max(cint) FROM druid_partitioned_table; - -SELECT sum(cint), sum(cbigint) FROM druid_partitioned_table; - -EXPLAIN INSERT OVERWRITE TABLE druid_partitioned_table - SELECT cast (`ctimestamp1` as timestamp with local time zone) as `__time`, - cstring1, - cstring2, - cdouble, - cfloat, - ctinyint, - csmallint, - cint, - cbigint, - cboolean1, - cboolean2 - FROM alltypesorc where ctimestamp1 IS NOT NULL; - - -INSERT OVERWRITE TABLE druid_partitioned_table - SELECT cast (`ctimestamp1` as timestamp with local time zone) as `__time`, - cstring1, - cstring2, - cdouble, - cfloat, - ctinyint, - csmallint, - cint, - cbigint, - cboolean1, - cboolean2 - FROM alltypesorc where ctimestamp1 IS NOT NULL; - --- @FIXME https://issues.apache.org/jira/browse/HIVE-19011 ---SELECT sum(cint), max(cbigint), sum(cbigint), max(cint) FROM druid_partitioned_table ; ---SELECT sum(cint), max(cbigint), sum(cbigint), max(cint) FROM druid_partitioned_table_0; - -SELECT sum(cint), sum(cbigint) FROM druid_partitioned_table ; - -set hive.druid.indexer.partition.size.max=10; - -CREATE EXTERNAL TABLE druid_max_size_partition - STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' - TBLPROPERTIES ( - "druid.segment.granularity" = "HOUR", - "druid.query.granularity" = "MINUTE" - ) - AS - SELECT cast (`ctimestamp1` as timestamp with local time zone) as `__time`, - cstring1, - cstring2, - cdouble, - cfloat, - ctinyint, - csmallint, - cint, - cbigint, - cboolean1, - cboolean2 - FROM alltypesorc where ctimestamp1 IS NOT NULL; - -SELECT sum(cint), sum(cbigint) FROM druid_max_size_partition ; - --- @FIXME https://issues.apache.org/jira/browse/HIVE-19011 ---SELECT sum(cint), max(cbigint), sum(cbigint), max(cint) FROM druid_max_size_partition ; ---SELECT sum(cint), max(cbigint), sum(cbigint), max(cint) FROM druid_partitioned_table_0; ---SELECT sum(cint), max(cbigint), sum(cbigint), max(cint) FROM druid_partitioned_table ; - -DROP TABLE druid_partitioned_table_0; -DROP TABLE druid_partitioned_table; -DROP TABLE druid_max_size_partition; diff --git a/ql/src/test/queries/clientpositive/druidmini_expressions.q b/ql/src/test/queries/clientpositive/druidmini_expressions.q deleted file mode 100644 index 2347d625fb03..000000000000 --- a/ql/src/test/queries/clientpositive/druidmini_expressions.q +++ /dev/null @@ -1,237 +0,0 @@ ---! qt:disabled:HIVE-23984 ---! qt:dataset:druid_table_alltypesorc -SET hive.ctas.external.tables=true; - -SET hive.vectorized.execution.enabled=true; - - -- MATH AND STRING functions - -SELECT count(*) FROM druid_table_alltypesorc WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3; - -SELECT count(*) FROM druid_table_alltypesorc WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10; - -SELECT count(*) FROM druid_table_alltypesorc WHERE power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3; - -SELECT SUM(cfloat + 1), CAST(SUM(cdouble + ctinyint) AS INTEGER), SUM(ctinyint) + 1 , CAST(SUM(csmallint) + SUM(cint) AS DOUBLE), SUM(cint), SUM(cbigint) -FROM druid_table_alltypesorc WHERE ceil(cfloat) > 0 AND floor(cdouble) * 2 < 1000 OR ln(cdouble) / log10(10) > 0 AND COS(cint) > 0 OR SIN(cdouble) > 1; - -SELECT SUM(cfloat + 1), CAST(SUM(cdouble + ctinyint) AS INTEGER), SUM(ctinyint) + 1 , CAST(SUM(csmallint) + SUM(cint) AS DOUBLE), SUM(cint), SUM(cbigint) -FROM druid_table_alltypesorc WHERE ceil(cfloat) > 0 AND floor(cdouble) * 2 < 1000; - -SELECT SUM(cfloat + 1), CAST(SUM(cdouble + ctinyint) AS INTEGER), SUM(ctinyint) + 1 , CAST(SUM(csmallint) + SUM(cint) AS DOUBLE), SUM(cint), SUM(cbigint) -FROM druid_table_alltypesorc WHERE ln(cdouble) / log10(10) > 0 AND COS(cint) > 0 OR SIN(cdouble) > 1; - -SELECT SUM(cfloat + 1), CAST(SUM(cdouble + ctinyint) AS INTEGER), SUM(ctinyint) + 1 , CAST(SUM(csmallint) + SUM(cint) AS DOUBLE), SUM(cint), SUM(cbigint) -FROM druid_table_alltypesorc WHERE SIN(cdouble) > 1; - -SELECT cstring1 || '_'|| cstring2, substring(cstring2, 2, 3) as concat , upper(cstring2), lower(cstring1), SUM(cdouble) as s FROM druid_table_alltypesorc WHERE cstring1 IS NOT NULL AND cstring2 IS NOT NULL AND cstring2 like 'Y%' - GROUP BY cstring1 || '_'|| cstring2, substring(cstring2, 2, 3), upper(cstring2), lower(cstring1) ORDER BY concat DESC LIMIT 10; - -EXPLAIN SELECT count(*) FROM druid_table_alltypesorc WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3; - -EXPLAIN SELECT SUM(cfloat + 1), CAST(SUM(cdouble + ctinyint) AS INTEGER), SUM(ctinyint) + 1 , CAST(SUM(csmallint) + SUM(cint) AS DOUBLE), SUM(cint), SUM(cbigint) - FROM druid_table_alltypesorc WHERE ceil(cfloat) > 0 AND floor(cdouble) * 2 < 1000 OR ln(cdouble) / log10(10) > 0 AND COS(cint) > 0 OR SIN(cdouble) > 1; - -EXPLAIN SELECT cstring1 || '_'|| cstring2, substring(cstring2, 2, 3) as concat , upper(cstring2), lower(cstring1), SUM(cdouble) as s FROM druid_table_alltypesorc WHERE cstring1 IS NOT NULL AND cstring2 IS NOT NULL AND cstring2 like 'Y%' - GROUP BY cstring1 || '_'|| cstring2, substring(cstring2, 2, 3), upper(cstring2), lower(cstring1) ORDER BY concat DESC LIMIT 10; - -explain extended select count(*) from (select `__time` from druid_table_alltypesorc limit 1) as src ; - - - -explain -SELECT `__time` -FROM druid_table_alltypesorc -WHERE (`__time` BETWEEN '1968-01-01 00:00:00' AND '1970-01-01 00:00:00') - OR (`__time` BETWEEN '1968-02-01 00:00:00' AND '1970-04-01 00:00:00') ORDER BY `__time` ASC LIMIT 10; - -SELECT `__time` -FROM druid_table_alltypesorc -WHERE (`__time` BETWEEN '1968-01-01 00:00:00' AND '1970-01-01 00:00:00') - OR (`__time` BETWEEN '1968-02-01 00:00:00' AND '1970-04-01 00:00:00') ORDER BY `__time` ASC LIMIT 10; - - -explain -SELECT `__time` -FROM druid_table_alltypesorc -WHERE ('1968-01-01 00:00:00' <= `__time` AND `__time` <= '1970-01-01 00:00:00') - OR ('1968-02-01 00:00:00' <= `__time` AND `__time` <= '1970-04-01 00:00:00') ORDER BY `__time` ASC LIMIT 10; - -SELECT `__time` -FROM druid_table_alltypesorc -WHERE ('1968-01-01 00:00:00' <= `__time` AND `__time` <= '1970-01-01 00:00:00') - OR ('1968-02-01 00:00:00' <= `__time` AND `__time` <= '1970-04-01 00:00:00') ORDER BY `__time` ASC LIMIT 10; - --- COUNT DISTINCT TESTS --- AS PART OF https://issues.apache.org/jira/browse/HIVE-19586 - -EXPLAIN select count(DISTINCT cstring2), sum(cdouble) FROM druid_table_alltypesorc GROUP BY `__time`, `cstring1` ; - -EXPLAIN select count(distinct cdouble), sum(cdouble) FROM druid_table_alltypesorc GROUP BY `__time`, `cstring1` ; - -EXPLAIN select count(distinct cstring2), sum(2 * cdouble) FROM druid_table_alltypesorc GROUP BY `__time`, `cstring1` ; - -EXPLAIN select count(distinct cstring2 || '_'|| cstring1), sum(cdouble) FROM druid_table_alltypesorc GROUP BY `__time`, `cstring1` ; - -EXPLAIN select count(DISTINCT cstring2) FROM druid_table_alltypesorc ; -EXPLAIN select count(DISTINCT cstring2), sum(cdouble) FROM druid_table_alltypesorc ; -EXPLAIN select count(distinct cstring2 || '_'|| cstring1), sum(cdouble), min(cint) FROM druid_table_alltypesorc; - --- Force the scan query to test limit push down. -EXPLAIN select count(*) from (select `__time` from druid_table_alltypesorc limit 1025) as src; - -select count(DISTINCT cstring2), sum(cdouble) FROM druid_table_alltypesorc GROUP BY floor_year(`__time`) ; - -select count(distinct cstring2), sum(2 * cdouble) FROM druid_table_alltypesorc GROUP BY floor_year(`__time`) ; - -select count(DISTINCT cstring2) FROM druid_table_alltypesorc ; - -explain select count(DISTINCT cstring1) FROM druid_table_alltypesorc ; - -select count(DISTINCT cstring1) FROM druid_table_alltypesorc ; - -select count(DISTINCT cstring2), sum(cdouble) FROM druid_table_alltypesorc ; - -select count(distinct cstring2 || '_'|| cstring1), sum(cdouble), min(cint) FROM druid_table_alltypesorc; - --- Force a scan query to test Vectorized path -select count(*) from (select `__time` from druid_table_alltypesorc limit 1025) as src; - --- Force a scan query to test Vectorized path -select count(*) from (select `__time` from druid_table_alltypesorc limit 200000) as src; - -select count(`__time`) from (select `__time` from druid_table_alltypesorc limit 200) as src; - -select count(distinct `__time`) from druid_table_alltypesorc; - -select count(distinct `ctimestamp1`) from alltypesorc1 where ctimestamp1 IS NOT NULL; - -explain select `timets` from (select `__time` as timets from druid_table_alltypesorc order by timets limit 10) as src order by `timets`; - -explain select `timets` from (select cast(`__time` as timestamp ) as timets from druid_table_alltypesorc order by timets limit 10) as src order by `timets`; - -select `timets_with_tz` from (select `__time` as timets_with_tz from druid_table_alltypesorc order by timets_with_tz limit 10) as src order by `timets_with_tz`; - -select `timets` from (select cast(`__time` as timestamp ) as timets from druid_table_alltypesorc order by timets limit 10) as src order by `timets`; - -explain select unix_timestamp(from_unixtime(1396681200)) from druid_table_alltypesorc limit 1; -select unix_timestamp(from_unixtime(1396681200)) from druid_table_alltypesorc limit 1; - -explain select unix_timestamp(`__time`) from druid_table_alltypesorc limit 1; -select unix_timestamp(`__time`) from druid_table_alltypesorc limit 1; - -explain select FROM_UNIXTIME(UNIX_TIMESTAMP(CAST(`__time` as timestamp ),'yyyy-MM-dd HH:mm:ss' ),'yyyy-MM-dd HH:mm:ss') -from druid_table_alltypesorc -GROUP BY FROM_UNIXTIME(UNIX_TIMESTAMP(CAST(`__time` as timestamp ),'yyyy-MM-dd HH:mm:ss' ),'yyyy-MM-dd HH:mm:ss'); - -select FROM_UNIXTIME(UNIX_TIMESTAMP (CAST(`__time` as timestamp ),'yyyy-MM-dd HH:mm:ss' ),'yyyy-MM-dd HH:mm:ss') -from druid_table_alltypesorc -GROUP BY FROM_UNIXTIME(UNIX_TIMESTAMP(CAST(`__time` as timestamp ),'yyyy-MM-dd HH:mm:ss' ),'yyyy-MM-dd HH:mm:ss'); - -explain select TRUNC(cast(`__time` as timestamp), 'YY') from druid_table_alltypesorc GROUP BY TRUNC(cast(`__time` as timestamp), 'YY'); -select TRUNC(cast(`__time` as timestamp), 'YY') from druid_table_alltypesorc GROUP BY TRUNC(cast(`__time` as timestamp), 'YY'); -select TRUNC(cast(`__time` as timestamp), 'YEAR') from druid_table_alltypesorc GROUP BY TRUNC(cast(`__time` as timestamp), 'YEAR'); -select TRUNC(cast(`__time` as timestamp), 'YYYY') from druid_table_alltypesorc GROUP BY TRUNC(cast(`__time` as timestamp), 'YYYY'); - -explain select TRUNC(cast(`__time` as timestamp), 'MONTH') from druid_table_alltypesorc GROUP BY TRUNC(cast(`__time` as timestamp), 'MONTH'); -select TRUNC(cast(`__time` as timestamp), 'MONTH') from druid_table_alltypesorc GROUP BY TRUNC(cast(`__time` as timestamp), 'MONTH'); -select TRUNC(cast(`__time` as timestamp), 'MM') from druid_table_alltypesorc GROUP BY TRUNC(cast(`__time` as timestamp), 'MM'); -select TRUNC(cast(`__time` as timestamp), 'MON') from druid_table_alltypesorc GROUP BY TRUNC(cast(`__time` as timestamp), 'MON'); - -explain select TRUNC(cast(`__time` as timestamp), 'QUARTER') from druid_table_alltypesorc GROUP BY TRUNC(cast(`__time` as timestamp), 'QUARTER'); -select TRUNC(cast(`__time` as timestamp), 'QUARTER') from druid_table_alltypesorc GROUP BY TRUNC(cast(`__time` as timestamp), 'QUARTER'); -select TRUNC(cast(`__time` as timestamp), 'Q') from druid_table_alltypesorc GROUP BY TRUNC(cast(`__time` as timestamp), 'Q'); - -explain select TO_DATE(`__time`) from druid_table_alltypesorc GROUP BY TO_DATE(`__time`); -select TO_DATE(`__time`) from druid_table_alltypesorc GROUP BY TO_DATE(`__time`); - -EXPLAIN SELECT SUM((`druid_table_alias`.`cdouble` * `druid_table_alias`.`cdouble`)) AS `sum_calculation_4998925219892510720_ok`, - CAST(TRUNC(CAST(`druid_table_alias`.`__time` AS TIMESTAMP),'MM') AS DATE) AS `tmn___time_ok` -FROM `default`.`druid_table_alltypesorc` `druid_table_alias` -GROUP BY CAST(TRUNC(CAST(`druid_table_alias`.`__time` AS TIMESTAMP),'MM') AS DATE); - - -SELECT SUM((`druid_table_alias`.`cdouble` * `druid_table_alias`.`cdouble`)) AS `sum_calculation_4998925219892510720_ok`, - CAST(TRUNC(CAST(`druid_table_alias`.`__time` AS TIMESTAMP),'MM') AS DATE) AS `tmn___time_ok` -FROM `default`.`druid_table_alltypesorc` `druid_table_alias` -GROUP BY CAST(TRUNC(CAST(`druid_table_alias`.`__time` AS TIMESTAMP),'MM') AS DATE); - -explain SELECT DATE_ADD(cast(`__time` as date), CAST((cdouble / 1000) AS INT)) as date_1, DATE_SUB(cast(`__time` as date), CAST((cdouble / 1000) AS INT)) as date_2, cast(`__time` as date) as orig_date, CAST((cdouble / 1000) AS INT) as offset from druid_table_alltypesorc order by date_1, date_2 limit 3; -SELECT DATE_ADD(cast(`__time` as date), CAST((cdouble / 1000) AS INT)) as date_1, DATE_SUB(cast(`__time` as date), CAST((cdouble / 1000) AS INT)) as date_2, cast(`__time` as date) as orig_date, CAST((cdouble / 1000) AS INT) as offset from druid_table_alltypesorc order by date_1, date_2 limit 3; - - -- Boolean Values --- Expected results of this query are wrong due to https://issues.apache.org/jira/browse/CALCITE-2319 --- It should get fixed once we upgrade calcite - EXPLAIN SELECT cboolean2, count(*) from druid_table_alltypesorc GROUP BY cboolean2; - SELECT cboolean2, count(*) from druid_table_alltypesorc GROUP BY cboolean2; - - -- Expected results of this query are wrong due to https://issues.apache.org/jira/browse/CALCITE-2319 - -- It should get fixed once we upgrade calcite - SELECT ctinyint > 2, count(*) from druid_table_alltypesorc GROUP BY ctinyint > 2; - - EXPLAIN SELECT ctinyint > 2, count(*) from druid_table_alltypesorc GROUP BY ctinyint > 2; - --- group by should be rewitten and pushed into druid --- simple gby with single constant key -EXPLAIN SELECT sum(cfloat) FROM druid_table_alltypesorc WHERE cstring1 != 'en' group by 1.011; -SELECT sum(cfloat) FROM druid_table_alltypesorc WHERE cstring1 != 'en' group by 1.011; - --- gby with multiple constant keys -EXPLAIN SELECT sum(cfloat) FROM druid_table_alltypesorc WHERE cstring1 != 'en' group by 1.011, 3.40; -SELECT sum(cfloat) FROM druid_table_alltypesorc WHERE cstring1 != 'en' group by 1.011, 3.40; - --- group by with constant folded key -EXPLAIN SELECT sum(cint) FROM druid_table_alltypesorc WHERE cfloat= 0.011 group by cfloat; -SELECT sum(cint) FROM druid_table_alltypesorc WHERE cfloat= 0.011 group by cfloat; - --- group by key is referred in select -EXPLAIN SELECT cfloat, sum(cint) FROM druid_table_alltypesorc WHERE cfloat= 0.011 group by cfloat; -SELECT cfloat, sum(cint) FROM druid_table_alltypesorc WHERE cfloat= 0.011 group by cfloat; - --- Tests for testing handling of date/time funtions on druid dimensions stored as strings -CREATE TABLE druid_table_n1 -STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' -TBLPROPERTIES ("druid.segment.granularity" = "HOUR", "druid.query.granularity" = "MINUTE") -AS - SELECT cast (current_timestamp() as timestamp with local time zone) as `__time`, -cast(datetime1 as string) as datetime1, -cast(date1 as string) as date1, -cast(time1 as string) as time1 -FROM TABLE ( -VALUES -('2004-04-09 22:20:14', '2004-04-09','22:20:14'), -('2004-04-04 22:50:16', '2004-04-04', '22:50:16'), -('2004-04-12 04:40:49', '2004-04-12', '04:40:49'), -('2004-04-11 00:00:00', '2004-04-11', null), -('00:00:00 18:58:41', null, '18:58:41')) as q (datetime1, date1, time1); - -EXPLAIN SELECT TO_DATE(date1), TO_DATE(datetime1) FROM druid_table_n1; - -SELECT TO_DATE(date1), TO_DATE(datetime1) FROM druid_table_n1; - -EXPLAIN select count(*) from (select `__time` from druid_table_alltypesorc limit 1025) as src; - -select count(*) from (select `__time` from druid_table_alltypesorc limit 1025) as src; - --- No Vectorization since __time is timestamp with local time zone -explain select `timets` from (select `__time` as timets from druid_table_alltypesorc order by timets limit 10) as src order by `timets`; --- Vectorization is on now since we cast to Timestamp -explain select `timets` from (select cast(`__time` as timestamp ) as timets from druid_table_alltypesorc order by timets limit 10) as src order by `timets`; - -select `timets_with_tz` from (select `__time` as timets_with_tz from druid_table_alltypesorc order by timets_with_tz limit 10) as src order by `timets_with_tz`; - -select `timets` from (select cast(`__time` as timestamp ) as timets from druid_table_alltypesorc order by timets limit 10) as src order by `timets`; - -select count(cfloat) from (select `cfloat`, `cstring1` from druid_table_alltypesorc limit 1025) as src; - -select count(cstring1) from (select `cfloat`, `cstring1` from druid_table_alltypesorc limit 90000) as src; - -explain select count(cstring1) from (select `cfloat`, `cstring1`, `cint` from druid_table_alltypesorc limit 90000) as src; - -select max(cint * cdouble) from (select `cfloat`, `cstring1`, `cint`, `cdouble` from druid_table_alltypesorc limit 90000) as src; - -explain select max(cint * cfloat) from (select `cfloat`, `cstring1`, `cint`, `cdouble` from druid_table_alltypesorc limit 90000) as src; - -explain select count(distinct `__time`, cint) from (select * from druid_table_alltypesorc) as src; - -select count(distinct `__time`, cint) from (select * from druid_table_alltypesorc) as src; diff --git a/ql/src/test/queries/clientpositive/druidmini_extractTime.q b/ql/src/test/queries/clientpositive/druidmini_extractTime.q deleted file mode 100644 index 15560d9ca7ae..000000000000 --- a/ql/src/test/queries/clientpositive/druidmini_extractTime.q +++ /dev/null @@ -1,187 +0,0 @@ ---! qt:disabled:HIVE-24816 ---! qt:dataset:druid_table_alltypesorc -SET hive.vectorized.execution.enabled=true ; -SET hive.ctas.external.tables=true; -SET hive.external.table.purge.default = true; - - --- GROUP BY TIME EXTRACT ---SECONDS -SELECT EXTRACT(SECOND from `__time`) FROM druid_table_alltypesorc -WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 -AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 GROUP BY EXTRACT(SECOND from `__time`); - -EXPLAIN SELECT EXTRACT(SECOND from `__time`) FROM druid_table_alltypesorc -WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 -AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 GROUP BY EXTRACT(SECOND from `__time`); - - --- MINUTES -SELECT EXTRACT(MINUTE from `__time`) FROM druid_table_alltypesorc -WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 -AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 GROUP BY EXTRACT(MINUTE from `__time`); - -EXPLAIN SELECT EXTRACT(MINUTE from `__time`) FROM druid_table_alltypesorc -WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 -AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 GROUP BY EXTRACT(MINUTE from `__time`); - --- HOUR -SELECT EXTRACT(HOUR from `__time`) FROM druid_table_alltypesorc -WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 -AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 GROUP BY EXTRACT(HOUR from `__time`); - -EXPLAIN SELECT EXTRACT(HOUR from `__time`) FROM druid_table_alltypesorc -WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 -AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 GROUP BY EXTRACT(HOUR from `__time`); - --- DAY -SELECT EXTRACT(DAY from `__time`) FROM druid_table_alltypesorc -WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 -AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 GROUP BY EXTRACT(DAY from `__time`); - -EXPLAIN SELECT EXTRACT(DAY from `__time`) FROM druid_table_alltypesorc -WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 -AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 GROUP BY EXTRACT(DAY from `__time`); - ---WEEK -SELECT EXTRACT(WEEK from `__time`) FROM druid_table_alltypesorc -WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 -AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 GROUP BY EXTRACT(WEEK from `__time`); - - -EXPLAIN SELECT EXTRACT(WEEK from `__time`) FROM druid_table_alltypesorc -WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 -AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 GROUP BY EXTRACT(WEEK from `__time`); - ---MONTH -SELECT EXTRACT(MONTH from `__time`) FROM druid_table_alltypesorc -WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 -AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 GROUP BY EXTRACT(MONTH from `__time`); - -EXPLAIN SELECT EXTRACT(MONTH from `__time`) FROM druid_table_alltypesorc -WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 -AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 GROUP BY EXTRACT(MONTH from `__time`); - ---QUARTER - -SELECT EXTRACT(QUARTER from `__time`) FROM druid_table_alltypesorc -WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 -AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 GROUP BY EXTRACT(QUARTER from `__time`); - -EXPLAIN SELECT EXTRACT(QUARTER from `__time`) FROM druid_table_alltypesorc -WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 -AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 GROUP BY EXTRACT(QUARTER from `__time`); - --- YEAR -SELECT EXTRACT(YEAR from `__time`) FROM druid_table_alltypesorc -WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 -AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 GROUP BY EXTRACT(YEAR from `__time`); - - -EXPLAIN SELECT EXTRACT(YEAR from `__time`) FROM druid_table_alltypesorc -WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 -AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 GROUP BY EXTRACT(YEAR from `__time`); - --- SELECT WITHOUT GROUP BY - --- SECOND - -EXPLAIN SELECT EXTRACT(SECOND from `__time`) FROM druid_table_alltypesorc WHERE EXTRACT(SECOND from `__time`) = 0 LIMIT 1; - -SELECT EXTRACT(SECOND from `__time`) FROM druid_table_alltypesorc WHERE EXTRACT(SECOND from `__time`) = 0 LIMIT 1; - --- MINUTE - -EXPLAIN SELECT EXTRACT(MINUTE from `__time`) FROM druid_table_alltypesorc -WHERE EXTRACT(MINUTE from `__time`) >= 0 LIMIT 2; - -SELECT EXTRACT(MINUTE from `__time`) as minute FROM druid_table_alltypesorc - WHERE EXTRACT(MINUTE from `__time`) >= 0 order by minute LIMIT 2; --- HOUR - -EXPLAIN SELECT EXTRACT(HOUR from `__time`) FROM druid_table_alltypesorc -WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 -AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 LIMIT 1; - -SELECT EXTRACT(HOUR from `__time`) FROM druid_table_alltypesorc -WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 -AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 LIMIT 1; - ---DAY - -EXPLAIN SELECT EXTRACT(DAY from `__time`), EXTRACT(DAY from `__time`) DIV 7 AS WEEK, SUBSTRING(CAST(CAST(`__time` AS DATE) AS STRING), 9, 2) AS day_str -FROM druid_table_alltypesorc WHERE SUBSTRING(CAST(CAST(`__time` AS DATE) AS STRING), 9, 2) = 31 LIMIT 1; - -SELECT EXTRACT(DAY from `__time`) , EXTRACT(DAY from `__time`) DIV 7 AS WEEK, SUBSTRING(CAST(CAST(`__time` AS DATE) AS STRING), 9, 2) AS dar_str -FROM druid_table_alltypesorc WHERE SUBSTRING(CAST(CAST(`__time` AS DATE) AS STRING), 9, 2) = 31 LIMIT 1 ; - --- WEEK - -EXPLAIN SELECT EXTRACT(WEEK from `__time`) FROM druid_table_alltypesorc WHERE EXTRACT(WEEK from `__time`) >= 1 -AND EXTRACT(WEEK from `__time`) DIV 4 + 1 = 1 LIMIT 1; - -SELECT EXTRACT(WEEK from `__time`) FROM druid_table_alltypesorc WHERE EXTRACT(WEEK from `__time`) >= 1 -AND EXTRACT(WEEK from `__time`) DIV 4 + 1 = 1 LIMIT 1 ; - ---MONTH - -EXPLAIN SELECT EXTRACT(MONTH FROM `__time`) / 4 + 1, EXTRACT(MONTH FROM `__time`), SUBSTRING(CAST(CAST(`__time` AS DATE) AS STRING), 6, 2) as month_str FROM druid_table_alltypesorc -WHERE EXTRACT(MONTH FROM `__time`) / 4 + 1 = 4 AND EXTRACT(MONTH FROM `__time`) BETWEEN 11 AND 12 LIMIT 1; - -SELECT EXTRACT(MONTH FROM `__time`) / 4 + 1, EXTRACT(MONTH FROM `__time`), SUBSTRING(CAST(CAST(`__time` AS DATE) AS STRING), 6, 2) as month_str FROM druid_table_alltypesorc - WHERE EXTRACT(MONTH FROM `__time`) / 4 + 1 = 4 AND EXTRACT(MONTH FROM `__time`) BETWEEN 11 AND 12 LIMIT 1; - - ---QUARTER - -EXPLAIN SELECT EXTRACT(QUARTER from `__time`), EXTRACT(MONTH FROM `__time`) / 4 + 1 as q_number FROM druid_table_alltypesorc WHERE EXTRACT(QUARTER from `__time`) >= 4 - AND EXTRACT(MONTH FROM `__time`) / 4 + 1 = 4 LIMIT 1; - -SELECT EXTRACT(QUARTER from `__time`), EXTRACT(MONTH FROM `__time`) / 4 + 1 as q_number FROM druid_table_alltypesorc WHERE EXTRACT(QUARTER from `__time`) >= 4 - AND EXTRACT(MONTH FROM `__time`) / 4 + 1 = 4 LIMIT 1; - ---YEAR - -EXPLAIN SELECT EXTRACT(YEAR from `__time`), SUBSTRING(CAST(CAST(`__time` AS DATE) AS STRING), 1, 4) AS year_str FROM druid_table_alltypesorc WHERE EXTRACT(YEAR from `__time`) >= 1969 -AND CAST(EXTRACT(YEAR from `__time`) as STRING) = '1969' LIMIT 1; - -SELECT EXTRACT(YEAR from `__time`), SUBSTRING(CAST(CAST(`__time` AS DATE) AS STRING), 1, 4) as year_str FROM druid_table_alltypesorc WHERE EXTRACT(YEAR from `__time`) >= 1969 -AND CAST(EXTRACT(YEAR from `__time`) as STRING) = '1969' LIMIT 1; - --- Cast to Timestamp - -explain SELECT CAST(`__time` AS TIMESTAMP) AS `x_time`, SUM(cfloat) FROM druid_table_alltypesorc GROUP BY CAST(`__time` AS TIMESTAMP) ORDER BY `x_time` LIMIT 5; - -SELECT CAST(`__time` AS TIMESTAMP) AS `x_time`, SUM(cfloat) FROM druid_table_alltypesorc GROUP BY CAST(`__time` AS TIMESTAMP) ORDER BY `x_time` LIMIT 5; - --- Cast to Date - -explain SELECT CAST(`__time` AS DATE) AS `x_date`, SUM(cfloat) FROM druid_table_alltypesorc GROUP BY CAST(`__time` AS DATE) ORDER BY `x_date` LIMIT 5; - -SELECT CAST(`__time` AS DATE) AS `x_date`, SUM(cfloat) FROM druid_table_alltypesorc GROUP BY CAST(`__time` AS DATE) ORDER BY `x_date` LIMIT 5; - -SELECT CAST(`__time` AS DATE) AS `x_date` FROM druid_table_alltypesorc ORDER BY `x_date` LIMIT 5; - --- Test Extract from non datetime column - -create table test_extract_from_string_base_table(`timecolumn` timestamp, `date_c` string, `timestamp_c` string, `metric_c` double); -insert into test_extract_from_string_base_table values ('2015-03-08 00:00:00', '2015-03-10', '2015-03-08 05:30:20', 5.0); - -CREATE EXTERNAL TABLE druid_test_extract_from_string_table -STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' -TBLPROPERTIES ("druid.segment.granularity" = "DAY") -AS select -cast(`timecolumn` as timestamp with local time zone) as `__time`, `date_c`, `timestamp_c`, `metric_c` -FROM test_extract_from_string_base_table; - -explain select -year(date_c), month(date_c),day(date_c), -year(timestamp_c), month(timestamp_c),day(timestamp_c), hour(timestamp_c), minute (timestamp_c), second (timestamp_c) -from druid_test_extract_from_string_table; - -select year(date_c), month(date_c), day(date_c), -year(timestamp_c), month(timestamp_c), day(timestamp_c), hour(timestamp_c), minute (timestamp_c), second (timestamp_c) -from druid_test_extract_from_string_table; - -DROP TABLE druid_test_extract_from_string_table; -DROP TABLE test_extract_from_string_base_table; diff --git a/ql/src/test/queries/clientpositive/druidmini_floorTime.q b/ql/src/test/queries/clientpositive/druidmini_floorTime.q deleted file mode 100644 index 20289ed4a2df..000000000000 --- a/ql/src/test/queries/clientpositive/druidmini_floorTime.q +++ /dev/null @@ -1,148 +0,0 @@ ---! qt:disabled:HIVE-24816 ---! qt:dataset:druid_table_alltypesorc -SET hive.vectorized.execution.enabled=true ; -SET hive.ctas.external.tables=true; -SET hive.external.table.purge.default = true; - --- GROUP BY TIME EXTRACT ---SECONDS -SELECT floor(`__time` to SECOND) FROM druid_table_alltypesorc -WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 -AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 GROUP BY floor(`__time` to SECOND); - -EXPLAIN SELECT floor(`__time` to SECOND) FROM druid_table_alltypesorc -WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 -AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 GROUP BY floor(`__time` to SECOND); - - --- MINUTES -SELECT floor(`__time` to MINUTE) FROM druid_table_alltypesorc -WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 -AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 GROUP BY floor(`__time` to MINUTE); - -EXPLAIN SELECT floor(`__time` to MINUTE) FROM druid_table_alltypesorc -WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 -AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 GROUP BY floor(`__time` to MINUTE); - --- HOUR -SELECT floor(`__time` to HOUR) FROM druid_table_alltypesorc -WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 -AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 GROUP BY floor(`__time` to HOUR); - -EXPLAIN SELECT floor(`__time` to HOUR) FROM druid_table_alltypesorc -WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 -AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 GROUP BY floor(`__time` to HOUR); - --- DAY -SELECT EXTRACT(DAY from `__time`) FROM druid_table_alltypesorc -WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 -AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 GROUP BY EXTRACT(DAY from `__time`); - -EXPLAIN SELECT EXTRACT(DAY from `__time`) FROM druid_table_alltypesorc -WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 -AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 GROUP BY EXTRACT(DAY from `__time`); - ---WEEK -SELECT EXTRACT(WEEK from `__time`) FROM druid_table_alltypesorc -WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 -AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 GROUP BY EXTRACT(WEEK from `__time`); - - -EXPLAIN SELECT EXTRACT(WEEK from `__time`) FROM druid_table_alltypesorc -WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 -AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 GROUP BY EXTRACT(WEEK from `__time`); - ---MONTH -SELECT EXTRACT(MONTH from `__time`) FROM druid_table_alltypesorc -WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 -AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 GROUP BY EXTRACT(MONTH from `__time`); - -EXPLAIN SELECT EXTRACT(MONTH from `__time`) FROM druid_table_alltypesorc -WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 -AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 GROUP BY EXTRACT(MONTH from `__time`); - ---QUARTER - -SELECT EXTRACT(QUARTER from `__time`) FROM druid_table_alltypesorc -WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 -AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 GROUP BY EXTRACT(QUARTER from `__time`); - -EXPLAIN SELECT EXTRACT(QUARTER from `__time`) FROM druid_table_alltypesorc -WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 -AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 GROUP BY EXTRACT(QUARTER from `__time`); - --- YEAR -SELECT EXTRACT(YEAR from `__time`) FROM druid_table_alltypesorc -WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 -AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 GROUP BY EXTRACT(YEAR from `__time`); - - -EXPLAIN SELECT EXTRACT(YEAR from `__time`) FROM druid_table_alltypesorc -WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 -AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 GROUP BY EXTRACT(YEAR from `__time`); - --- SELECT WITHOUT GROUP BY - --- SECOND - -EXPLAIN SELECT EXTRACT(SECOND from `__time`) FROM druid_table_alltypesorc WHERE EXTRACT(SECOND from `__time`) = 0 LIMIT 1; - -SELECT EXTRACT(SECOND from `__time`) FROM druid_table_alltypesorc WHERE EXTRACT(SECOND from `__time`) = 0 LIMIT 1; - --- MINUTE - -EXPLAIN SELECT EXTRACT(MINUTE from `__time`) FROM druid_table_alltypesorc -WHERE EXTRACT(MINUTE from `__time`) >= 0 LIMIT 2; - -SELECT EXTRACT(MINUTE from `__time`) as minute FROM druid_table_alltypesorc - WHERE EXTRACT(MINUTE from `__time`) >= 0 order by minute LIMIT 2; --- HOUR - -EXPLAIN SELECT EXTRACT(HOUR from `__time`) FROM druid_table_alltypesorc -WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 -AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 LIMIT 1; - -SELECT EXTRACT(HOUR from `__time`) FROM druid_table_alltypesorc -WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 -AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 LIMIT 1; - ---DAY - -EXPLAIN SELECT EXTRACT(DAY from `__time`), EXTRACT(DAY from `__time`) DIV 7 AS WEEK, SUBSTRING(CAST(CAST(`__time` AS DATE) AS STRING), 9, 2) AS day_str -FROM druid_table_alltypesorc WHERE SUBSTRING(CAST(CAST(`__time` AS DATE) AS STRING), 9, 2) = 31 LIMIT 1; - -SELECT EXTRACT(DAY from `__time`) , EXTRACT(DAY from `__time`) DIV 7 AS WEEK, SUBSTRING(CAST(CAST(`__time` AS DATE) AS STRING), 9, 2) AS dar_str -FROM druid_table_alltypesorc WHERE SUBSTRING(CAST(CAST(`__time` AS DATE) AS STRING), 9, 2) = 31 LIMIT 1 ; - --- WEEK - -EXPLAIN SELECT EXTRACT(WEEK from `__time`) FROM druid_table_alltypesorc WHERE EXTRACT(WEEK from `__time`) >= 1 -AND EXTRACT(WEEK from `__time`) DIV 4 + 1 = 1 LIMIT 1; - -SELECT EXTRACT(WEEK from `__time`) FROM druid_table_alltypesorc WHERE EXTRACT(WEEK from `__time`) >= 1 -AND EXTRACT(WEEK from `__time`) DIV 4 + 1 = 1 LIMIT 1 ; - ---MONTH - -EXPLAIN SELECT EXTRACT(MONTH FROM `__time`) / 4 + 1, EXTRACT(MONTH FROM `__time`), SUBSTRING(CAST(CAST(`__time` AS DATE) AS STRING), 6, 2) as month_str FROM druid_table_alltypesorc -WHERE EXTRACT(MONTH FROM `__time`) / 4 + 1 = 4 AND EXTRACT(MONTH FROM `__time`) BETWEEN 11 AND 12 LIMIT 1; - -SELECT EXTRACT(MONTH FROM `__time`) / 4 + 1, EXTRACT(MONTH FROM `__time`), SUBSTRING(CAST(CAST(`__time` AS DATE) AS STRING), 6, 2) as month_str FROM druid_table_alltypesorc - WHERE EXTRACT(MONTH FROM `__time`) / 4 + 1 = 4 AND EXTRACT(MONTH FROM `__time`) BETWEEN 11 AND 12 LIMIT 1; - - ---QUARTER - -EXPLAIN SELECT EXTRACT(QUARTER from `__time`), EXTRACT(MONTH FROM `__time`) / 4 + 1 as q_number FROM druid_table_alltypesorc WHERE EXTRACT(QUARTER from `__time`) >= 4 - AND EXTRACT(MONTH FROM `__time`) / 4 + 1 = 4 LIMIT 1; - -SELECT EXTRACT(QUARTER from `__time`), EXTRACT(MONTH FROM `__time`) / 4 + 1 as q_number FROM druid_table_alltypesorc WHERE EXTRACT(QUARTER from `__time`) >= 4 - AND EXTRACT(MONTH FROM `__time`) / 4 + 1 = 4 LIMIT 1; - ---YEAR - -EXPLAIN SELECT EXTRACT(YEAR from `__time`), SUBSTRING(CAST(CAST(`__time` AS DATE) AS STRING), 1, 4) AS year_str FROM druid_table_alltypesorc WHERE EXTRACT(YEAR from `__time`) >= 1969 -AND CAST(EXTRACT(YEAR from `__time`) as STRING) = '1969' LIMIT 1; - -SELECT EXTRACT(YEAR from `__time`), SUBSTRING(CAST(CAST(`__time` AS DATE) AS STRING), 1, 4) as year_str FROM druid_table_alltypesorc WHERE EXTRACT(YEAR from `__time`) >= 1969 -AND CAST(EXTRACT(YEAR from `__time`) as STRING) = '1969' LIMIT 1; diff --git a/ql/src/test/queries/clientpositive/druidmini_joins.q b/ql/src/test/queries/clientpositive/druidmini_joins.q deleted file mode 100644 index 9e8231df3a9b..000000000000 --- a/ql/src/test/queries/clientpositive/druidmini_joins.q +++ /dev/null @@ -1,62 +0,0 @@ ---! qt:disabled:Disabled in HIVE-20322 - -SET hive.explain.user=false; -SET hive.ctas.external.tables=true; -SET hive.external.table.purge.default = true; ---SET hive.execution.mode=llap; - -DROP TABLE druid_table_with_nulls; - -CREATE EXTERNAL TABLE druid_table_with_nulls -STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' -TBLPROPERTIES ("druid.segment.granularity" = "HOUR") -AS -SELECT cast(current_timestamp() AS timestamp with local time zone) AS `__time`, - cast(username AS string) AS username, - cast(double1 AS double) AS double1, - cast(int1 AS int) AS int1 -FROM TABLE ( - VALUES - ('alfred', 10.30, 2), - ('bob', 3.14, null), - ('bonnie', null, 3), - ('calvin', null, null), - ('charlie', 9.8, 1), - ('charlie', 15.8, 1)) as q (username, double1, int1); - -EXPLAIN SELECT -username AS `username`, -SUM(double1) AS `sum_double1` -FROM -druid_table_with_nulls `tbl1` - JOIN ( - SELECT - username AS `username`, - SUM(double1) AS `sum_double2` - FROM druid_table_with_nulls - GROUP BY `username` - ORDER BY `sum_double2` - DESC LIMIT 10 - ) - `tbl2` - ON (`tbl1`.`username` = `tbl2`.`username`) -GROUP BY `tbl1`.`username` ORDER BY `sum_double1`; - - -SELECT -username AS `username`, -SUM(double1) AS `sum_double1` -FROM -druid_table_with_nulls `tbl1` - JOIN ( - SELECT - username AS `username`, - SUM(double1) AS `sum_double2` - FROM druid_table_with_nulls - GROUP BY `username` - ORDER BY `sum_double2` - DESC LIMIT 10 - ) - `tbl2` - ON (`tbl1`.`username` = `tbl2`.`username`) -GROUP BY `tbl1`.`username` ORDER BY `sum_double1`; \ No newline at end of file diff --git a/ql/src/test/queries/clientpositive/druidmini_masking.q b/ql/src/test/queries/clientpositive/druidmini_masking.q deleted file mode 100644 index da13c5a0b713..000000000000 --- a/ql/src/test/queries/clientpositive/druidmini_masking.q +++ /dev/null @@ -1,26 +0,0 @@ ---! qt:disabled:Disabled in HIVE-20322 - -set hive.mapred.mode=nonstrict; -set hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactoryForTest; -SET hive.ctas.external.tables=true; -SET hive.external.table.purge.default = true; - -CREATE EXTERNAL TABLE masking_test_druid -STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' -TBLPROPERTIES ("druid.segment.granularity" = "HOUR") -AS - SELECT cast(current_timestamp() AS timestamp with local time zone) AS `__time`, - cast(username AS string) AS username, - cast(double1 AS double) AS double1, - cast(key AS int) AS key - FROM TABLE ( - VALUES - ('alfred', 10.30, -2), - ('bob', 3.14, null), - ('bonnie', null, 100), - ('calvin', null, null), - ('charlie', 15.8, 20)) as q (username, double1, key); - -explain select username, key from masking_test_druid; - -select username, key from masking_test_druid; diff --git a/ql/src/test/queries/clientpositive/druidmini_mv.q b/ql/src/test/queries/clientpositive/druidmini_mv.q deleted file mode 100644 index 37708a2f4edc..000000000000 --- a/ql/src/test/queries/clientpositive/druidmini_mv.q +++ /dev/null @@ -1,115 +0,0 @@ ---! qt:disabled:HIVE-24816 --- SORT_QUERY_RESULTS - -set hive.support.concurrency=true; -set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; -set hive.strict.checks.cartesian.product=false; -set hive.materializedview.rewriting=true; - -CREATE TABLE cmv_basetable_n2 -STORED AS orc -TBLPROPERTIES ('transactional'='true') -AS -SELECT cast(current_timestamp() AS timestamp) AS t, - cast(a AS int) AS a, - cast(b AS varchar(256)) AS b, - cast(userid AS varchar(256)) AS userid, - cast(c AS double) AS c, - cast(d AS int) AS d -FROM TABLE ( - VALUES - (1, 'alfred', 'alfred', 10.30, 2), - (2, 'bob', 'bob', 3.14, 3), - (2, 'bonnie', 'bonnie', 172342.2, 3), - (3, 'calvin', 'calvin', 978.76, 3), - (3, 'charlie', 'charlie_a', 9.8, 1), - (3, 'charlie', 'charlie_b', 15.8, 1)) as q (a, b, userid, c, d); - -CREATE MATERIALIZED VIEW cmv_mat_view_n2 -STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' -TBLPROPERTIES ("druid.segment.granularity" = "HOUR") -AS -SELECT cast(t AS timestamp with local time zone) as `__time`, a, b, c, userid -FROM cmv_basetable_n2 -WHERE a = 2; - -SELECT a, b, c FROM cmv_mat_view_n2; - -SHOW TBLPROPERTIES cmv_mat_view_n2; - -CREATE MATERIALIZED VIEW IF NOT EXISTS cmv_mat_view2_n0 -STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' -TBLPROPERTIES ("druid.segment.granularity" = "HOUR") -AS -SELECT cast(t AS timestamp with local time zone) as `__time`, a, b, c, userid -FROM cmv_basetable_n2 -WHERE a = 3; - -SELECT a, c FROM cmv_mat_view2_n0; - -SHOW TBLPROPERTIES cmv_mat_view2_n0; - -EXPLAIN -SELECT a, c -FROM cmv_basetable_n2 -WHERE a = 3; - -SELECT a, c -FROM cmv_basetable_n2 -WHERE a = 3; - -EXPLAIN -SELECT * FROM ( - (SELECT a, c FROM cmv_basetable_n2 WHERE a = 3) table1 - JOIN - (SELECT a, c FROM cmv_basetable_n2 WHERE d = 3) table2 - ON table1.a = table2.a); - -SELECT * FROM ( - (SELECT a, c FROM cmv_basetable_n2 WHERE a = 3) table1 - JOIN - (SELECT a, c FROM cmv_basetable_n2 WHERE d = 3) table2 - ON table1.a = table2.a); - -INSERT INTO cmv_basetable_n2 VALUES - (cast(current_timestamp() AS timestamp), 3, 'charlie', 'charlie_c', 15.8, 1); - --- TODO: CANNOT USE THE VIEW, IT IS OUTDATED -EXPLAIN -SELECT * FROM ( - (SELECT a, c FROM cmv_basetable_n2 WHERE a = 3) table1 - JOIN - (SELECT a, c FROM cmv_basetable_n2 WHERE d = 3) table2 - ON table1.a = table2.a); - -SELECT * FROM ( - (SELECT a, c FROM cmv_basetable_n2 WHERE a = 3) table1 - JOIN - (SELECT a, c FROM cmv_basetable_n2 WHERE d = 3) table2 - ON table1.a = table2.a); - --- REBUILD -EXPLAIN -ALTER MATERIALIZED VIEW cmv_mat_view2_n0 REBUILD; - -ALTER MATERIALIZED VIEW cmv_mat_view2_n0 REBUILD; - -SHOW TBLPROPERTIES cmv_mat_view2_n0; - --- NOW IT CAN BE USED AGAIN -EXPLAIN -SELECT * FROM ( - (SELECT a, c FROM cmv_basetable_n2 WHERE a = 3) table1 - JOIN - (SELECT a, c FROM cmv_basetable_n2 WHERE d = 3) table2 - ON table1.a = table2.a); - -SELECT * FROM ( - (SELECT a, c FROM cmv_basetable_n2 WHERE a = 3) table1 - JOIN - (SELECT a, c FROM cmv_basetable_n2 WHERE d = 3) table2 - ON table1.a = table2.a); - -DROP MATERIALIZED VIEW cmv_mat_view_n2; -DROP MATERIALIZED VIEW cmv_mat_view2_n0; -DROP TABLE cmv_basetable_n2; diff --git a/ql/src/test/queries/clientpositive/druidmini_semijoin_reduction_all_types.q b/ql/src/test/queries/clientpositive/druidmini_semijoin_reduction_all_types.q deleted file mode 100644 index 7bc0d84834b2..000000000000 --- a/ql/src/test/queries/clientpositive/druidmini_semijoin_reduction_all_types.q +++ /dev/null @@ -1,130 +0,0 @@ ---! qt:disabled:HIVE-24816 ---! qt:dataset:srcpart ---! qt:dataset:druid_table_alltypesorc ---! qt:dataset:alltypesorc - -set hive.compute.query.using.stats=false; -set hive.mapred.mode=nonstrict; -set hive.explain.user=false; -set hive.optimize.ppd=true; -set hive.ppd.remove.duplicatefilters=true; -set hive.tez.dynamic.partition.pruning=true; -set hive.tez.dynamic.semijoin.reduction=true; -set hive.optimize.metadataonly=false; -set hive.optimize.index.filter=true; -set hive.stats.autogather=true; -set hive.tez.bigtable.minsize.semijoin.reduction=1; -set hive.tez.min.bloom.filter.entries=1; -set hive.stats.fetch.column.stats=true; -set hive.disable.unsafe.external.table.operations=false; -set hive.tez.dynamic.semijoin.reduction.for.mapjoin=true; - -DROP TABLE IF EXISTS alltypesorc_small; -CREATE TABLE alltypesorc_small( - ctinyint TINYINT, - csmallint SMALLINT, - cint INT, - cbigint BIGINT, - cfloat FLOAT, - cdouble DOUBLE, - cstring1 STRING, - cstring2 STRING, - ctimestamp1 TIMESTAMP, - cboolean1 BOOLEAN, - cboolean2 BOOLEAN) - STORED AS ORC; -Insert into table alltypesorc_small -Select ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, cast(`__time` as timestamp), cboolean1, cboolean2 from druid_table_alltypesorc where cstring2 like '%a%' and cstring1 like '%a%'; -Select count(*) from alltypesorc_small; -Select count(*) from druid_table_alltypesorc; - -DESCRIBE druid_table_alltypesorc; -DESCRIBE alltypesorc_small; - --- Test Joins on all column types one by one --- String -set hive.disable.unsafe.external.table.operations=false; -EXPLAIN select count(*) from alltypesorc_small join druid_table_alltypesorc on (alltypesorc_small.cstring1 = druid_table_alltypesorc.cstring1); -select count(*) from alltypesorc_small join druid_table_alltypesorc on (alltypesorc_small.cstring1 = druid_table_alltypesorc.cstring1); -set hive.disable.unsafe.external.table.operations=true; -select count(*) from alltypesorc_small join druid_table_alltypesorc on (alltypesorc_small.cstring1 = druid_table_alltypesorc.cstring1); - --- tinyint -set hive.disable.unsafe.external.table.operations=false; -EXPLAIN select count(*) from alltypesorc_small join druid_table_alltypesorc on (alltypesorc_small.ctinyint = druid_table_alltypesorc.ctinyint); -select count(*) from alltypesorc_small join druid_table_alltypesorc on (alltypesorc_small.ctinyint = druid_table_alltypesorc.ctinyint); -set hive.disable.unsafe.external.table.operations=true; -select count(*) from alltypesorc_small join druid_table_alltypesorc on (alltypesorc_small.ctinyint = druid_table_alltypesorc.ctinyint); - --- smallint -set hive.disable.unsafe.external.table.operations=false; -EXPLAIN select count(*) from alltypesorc_small join druid_table_alltypesorc on (alltypesorc_small.csmallint = druid_table_alltypesorc.csmallint); -select count(*) from alltypesorc_small join druid_table_alltypesorc on (alltypesorc_small.csmallint = druid_table_alltypesorc.csmallint); -set hive.disable.unsafe.external.table.operations=true; -select count(*) from alltypesorc_small join druid_table_alltypesorc on (alltypesorc_small.csmallint = druid_table_alltypesorc.csmallint); - --- int -set hive.disable.unsafe.external.table.operations=false; -EXPLAIN select count(*) from alltypesorc_small join druid_table_alltypesorc on (alltypesorc_small.cint = druid_table_alltypesorc.cint); -select count(*) from alltypesorc_small join druid_table_alltypesorc on (alltypesorc_small.cint = druid_table_alltypesorc.cint); -set hive.disable.unsafe.external.table.operations=true; -select count(*) from alltypesorc_small join druid_table_alltypesorc on (alltypesorc_small.cint = druid_table_alltypesorc.cint); - --- bigint -set hive.disable.unsafe.external.table.operations=false; -EXPLAIN select count(*) from alltypesorc_small join druid_table_alltypesorc on (alltypesorc_small.cbigint = druid_table_alltypesorc.cbigint); -select count(*) from alltypesorc_small join druid_table_alltypesorc on (alltypesorc_small.cbigint = druid_table_alltypesorc.cbigint); -set hive.disable.unsafe.external.table.operations=true; -select count(*) from alltypesorc_small join druid_table_alltypesorc on (alltypesorc_small.cbigint = druid_table_alltypesorc.cbigint); - --- float -set hive.disable.unsafe.external.table.operations=false; -EXPLAIN select count(*) from alltypesorc_small join druid_table_alltypesorc on (alltypesorc_small.cfloat = druid_table_alltypesorc.cfloat); -select count(*) from alltypesorc_small join druid_table_alltypesorc on (alltypesorc_small.cfloat = druid_table_alltypesorc.cfloat); -set hive.disable.unsafe.external.table.operations=true; -select count(*) from alltypesorc_small join druid_table_alltypesorc on (alltypesorc_small.cfloat = druid_table_alltypesorc.cfloat); - --- double -set hive.disable.unsafe.external.table.operations=false; -EXPLAIN select count(*) from alltypesorc_small join druid_table_alltypesorc on (alltypesorc_small.cdouble = druid_table_alltypesorc.cdouble); -set hive.disable.unsafe.external.table.operations=true; -select count(*) from alltypesorc_small join druid_table_alltypesorc on (alltypesorc_small.cdouble = druid_table_alltypesorc.cdouble); -set hive.disable.unsafe.external.table.operations=true; - --- timestamp -set hive.disable.unsafe.external.table.operations=false; -EXPLAIN select count(*) from alltypesorc_small join druid_table_alltypesorc on (alltypesorc_small.ctimestamp1 = druid_table_alltypesorc.`__time`); -select count(*) from alltypesorc_small join druid_table_alltypesorc on (alltypesorc_small.ctimestamp1 = cast(druid_table_alltypesorc.`__time` as timestamp)); -set hive.disable.unsafe.external.table.operations=true; -select count(*) from alltypesorc_small join druid_table_alltypesorc on (alltypesorc_small.ctimestamp1 = cast(druid_table_alltypesorc.`__time` as timestamp)); - --- boolean -set hive.disable.unsafe.external.table.operations=false; -EXPLAIN select count(*) from alltypesorc_small join druid_table_alltypesorc on (alltypesorc_small.cboolean1 = druid_table_alltypesorc.cboolean1); -select count(*) from alltypesorc_small join druid_table_alltypesorc on (alltypesorc_small.cboolean1 = druid_table_alltypesorc.cboolean1); -set hive.disable.unsafe.external.table.operations=true; -select count(*) from alltypesorc_small join druid_table_alltypesorc on (alltypesorc_small.cboolean1 = druid_table_alltypesorc.cboolean1); - - --- Test Casts - -set hive.disable.unsafe.external.table.operations=false; -EXPLAIN select count(*) from alltypesorc_small join druid_table_alltypesorc on (cast(alltypesorc_small.cint as string) = druid_table_alltypesorc.cintstring); -select count(*) from alltypesorc_small join druid_table_alltypesorc on (cast(alltypesorc_small.cint as string) = druid_table_alltypesorc.cintstring); -set hive.disable.unsafe.external.table.operations=true; -select count(*) from alltypesorc_small join druid_table_alltypesorc on (cast(alltypesorc_small.cint as string) = druid_table_alltypesorc.cintstring); - - -set hive.disable.unsafe.external.table.operations=false; -EXPLAIN select count(*) from alltypesorc_small join druid_table_alltypesorc on (cast(alltypesorc_small.cdouble as string) = druid_table_alltypesorc.cdoublestring); -set hive.disable.unsafe.external.table.operations=true; -select count(*) from alltypesorc_small join druid_table_alltypesorc on (cast(alltypesorc_small.cdouble as string) = druid_table_alltypesorc.cdoublestring); -set hive.disable.unsafe.external.table.operations=true; -select count(*) from alltypesorc_small join druid_table_alltypesorc on (cast(alltypesorc_small.cdouble as string) = druid_table_alltypesorc.cdoublestring); - - -set hive.disable.unsafe.external.table.operations=false; -EXPLAIN select count(*) from alltypesorc_small join druid_table_alltypesorc on (cast(alltypesorc_small.cfloat as string) = druid_table_alltypesorc.cfloatstring); -select count(*) from alltypesorc_small join druid_table_alltypesorc on (cast(alltypesorc_small.cfloat as string) = druid_table_alltypesorc.cfloatstring); -set hive.disable.unsafe.external.table.operations=true; -select count(*) from alltypesorc_small join druid_table_alltypesorc on (cast(alltypesorc_small.cfloat as string) = druid_table_alltypesorc.cfloatstring); diff --git a/ql/src/test/queries/clientpositive/druidmini_test1.q b/ql/src/test/queries/clientpositive/druidmini_test1.q deleted file mode 100644 index bafa612678ec..000000000000 --- a/ql/src/test/queries/clientpositive/druidmini_test1.q +++ /dev/null @@ -1,122 +0,0 @@ ---! qt:disabled:HIVE-24816 ---! qt:dataset:druid_table_alltypesorc -SET hive.ctas.external.tables=true; -SET hive.vectorized.execution.enabled=true; -SET hive.external.table.purge.default = true; - --- Time Series Query -explain select count(*) FROM druid_table_alltypesorc; -SELECT count(*) FROM druid_table_alltypesorc; - - -EXPLAIN SELECT floor_year(`__time`), SUM(cfloat), SUM(cdouble), SUM(ctinyint), SUM(csmallint),SUM(cint), SUM(cbigint) -FROM druid_table_alltypesorc GROUP BY floor_year(`__time`); - -SELECT floor_year(`__time`), SUM(cfloat), SUM(cdouble), SUM(ctinyint), SUM(csmallint),SUM(cint), SUM(cbigint) -FROM druid_table_alltypesorc GROUP BY floor_year(`__time`); - -EXPLAIN SELECT floor_year(`__time`), MIN(cfloat), MIN(cdouble), MIN(ctinyint), MIN(csmallint),MIN(cint), MIN(cbigint) -FROM druid_table_alltypesorc GROUP BY floor_year(`__time`); - -SELECT floor_year(`__time`), MIN(cfloat), MIN(cdouble), MIN(ctinyint), MIN(csmallint),MIN(cint), MIN(cbigint) -FROM druid_table_alltypesorc GROUP BY floor_year(`__time`); - - -EXPLAIN SELECT floor_year(`__time`), MAX(cfloat), MAX(cdouble), MAX(ctinyint), MAX(csmallint),MAX(cint), MAX(cbigint) -FROM druid_table_alltypesorc GROUP BY floor_year(`__time`); - -SELECT floor_year(`__time`), MAX(cfloat), MAX(cdouble), MAX(ctinyint), MAX(csmallint),MAX(cint), MAX(cbigint) -FROM druid_table_alltypesorc GROUP BY floor_year(`__time`); - - --- Group By - - -EXPLAIN SELECT cstring1, SUM(cdouble) as s FROM druid_table_alltypesorc GROUP BY cstring1 ORDER BY s ASC LIMIT 10; - -SELECT cstring1, SUM(cdouble) as s FROM druid_table_alltypesorc GROUP BY cstring1 ORDER BY s ASC LIMIT 10; - - -EXPLAIN SELECT cstring2, MAX(cdouble) FROM druid_table_alltypesorc GROUP BY cstring2 ORDER BY cstring2 ASC LIMIT 10; - -SELECT cstring2, MAX(cdouble) FROM druid_table_alltypesorc GROUP BY cstring2 ORDER BY cstring2 ASC LIMIT 10; - - --- TIME STUFF - -EXPLAIN -SELECT `__time` -FROM druid_table_alltypesorc ORDER BY `__time` ASC LIMIT 10; - -SELECT `__time` -FROM druid_table_alltypesorc ORDER BY `__time` ASC LIMIT 10; - -EXPLAIN -SELECT `__time` -FROM druid_table_alltypesorc -WHERE `__time` < '1970-03-01 00:00:00' ORDER BY `__time` ASC LIMIT 10; - - -SELECT `__time` -FROM druid_table_alltypesorc -WHERE `__time` < '1970-03-01 00:00:00' ORDER BY `__time` ASC LIMIT 10; - - -EXPLAIN -SELECT `__time` -FROM druid_table_alltypesorc -WHERE `__time` >= '1968-01-01 00:00:00' AND `__time` <= '1970-03-01 00:00:00' ORDER BY `__time` ASC LIMIT 10; - - -SELECT `__time` -FROM druid_table_alltypesorc -WHERE `__time` >= '1968-01-01 00:00:00' AND `__time` <= '1970-03-01 00:00:00' ORDER BY `__time` ASC LIMIT 10; - - -EXPLAIN -SELECT `__time` -FROM druid_table_alltypesorc -WHERE `__time` >= '1968-01-01 00:00:00' AND `__time` <= '1970-03-01 00:00:00' - AND `__time` < '2011-01-01 00:00:00' ORDER BY `__time` ASC LIMIT 10; - - -SELECT `__time` -FROM druid_table_alltypesorc -WHERE `__time` >= '1968-01-01 00:00:00' AND `__time` <= '1970-03-01 00:00:00' - AND `__time` < '2011-01-01 00:00:00' ORDER BY `__time` ASC LIMIT 10; - - -EXPLAIN -SELECT `__time` -FROM druid_table_alltypesorc -WHERE `__time` BETWEEN '1968-01-01 00:00:00' AND '1970-01-01 00:00:00' ORDER BY `__time` ASC LIMIT 10;; - - -SELECT `__time` -FROM druid_table_alltypesorc -WHERE `__time` BETWEEN '1968-01-01 00:00:00' AND '1970-01-01 00:00:00' ORDER BY `__time` ASC LIMIT 10;; - - -EXPLAIN -SELECT `__time` -FROM druid_table_alltypesorc -WHERE (`__time` BETWEEN '1968-01-01 00:00:00' AND '1970-01-01 00:00:00') - OR (`__time` BETWEEN '1968-02-01 00:00:00' AND '1970-04-01 00:00:00') ORDER BY `__time` ASC LIMIT 10; - - -SELECT `__time` -FROM druid_table_alltypesorc -WHERE (`__time` BETWEEN '1968-01-01 00:00:00' AND '1970-01-01 00:00:00') - OR (`__time` BETWEEN '1968-02-01 00:00:00' AND '1970-04-01 00:00:00') ORDER BY `__time` ASC LIMIT 10; - --- Running this against Druid will if Druid version does not include --- this patch https://github.com/druid-io/druid/commit/219e77aeac9b07dc20dd9ab2dd537f3f17498346 - -explain select (cstring1 is null ) AS is_null, (cint is not null ) as isnotnull FROM druid_table_alltypesorc; - -explain select substring(to_date(`__time`), 4) from druid_table_alltypesorc limit 5; -select substring(to_date(`__time`), 4) from druid_table_alltypesorc limit 5; - -explain select substring(cast(to_date(`__time`) as string), 4) from druid_table_alltypesorc limit 5; -select substring(cast(to_date(`__time`) as string), 4) from druid_table_alltypesorc limit 5; - diff --git a/ql/src/test/queries/clientpositive/druidmini_test_alter.q b/ql/src/test/queries/clientpositive/druidmini_test_alter.q deleted file mode 100644 index aae8cedcc793..000000000000 --- a/ql/src/test/queries/clientpositive/druidmini_test_alter.q +++ /dev/null @@ -1,56 +0,0 @@ ---! qt:disabled:HIVE-24816 ---! qt:dataset:alltypesorc -SET hive.ctas.external.tables=true; -SET hive.external.table.purge.default = true; -CREATE EXTERNAL TABLE druid_alltypesorc_n0 -STORED BY 'org.apache.hadoop.hive.druid.QTestDruidStorageHandlerToAddFaultyHost' -TBLPROPERTIES ("druid.segment.granularity" = "HOUR", "druid.query.granularity" = "MINUTE") -AS - SELECT cast (`ctimestamp2` as timestamp with local time zone) as `__time`, -cstring1, -cdouble, -cfloat, -ctinyint, -csmallint, -cint, -cbigint, -cboolean1 -FROM alltypesorc where ctimestamp2 IS NOT NULL; - -DESCRIBE druid_alltypesorc_n0; - -DESCRIBE extended druid_alltypesorc_n0; - -SELECT COUNT(*) FROM druid_alltypesorc_n0; - -ALTER TABLE druid_alltypesorc_n0 ADD COLUMNS (cstring2 string, cboolean2 boolean, cint2 int); - -DESCRIBE druid_alltypesorc_n0; - -DESCRIBE extended druid_alltypesorc_n0; - -SELECT COUNT(*) FROM druid_alltypesorc_n0 WHERE cstring2 IS NOT NULL; - -INSERT INTO TABLE druid_alltypesorc_n0 - SELECT cast (`ctimestamp1` as timestamp with local time zone) as `__time`, -cstring1, -cdouble, -cfloat, -ctinyint, -csmallint, -cint, -cbigint, -cboolean1, -cstring2, -cboolean2, -cint -FROM alltypesorc where ctimestamp1 IS NOT NULL; - - -SELECT COUNT(*) FROM druid_alltypesorc_n0; - -SELECT COUNT(*) FROM druid_alltypesorc_n0 WHERE cstring2 IS NULL; - -SELECT COUNT(*) FROM druid_alltypesorc_n0 WHERE cstring2 IS NOT NULL; - -DROP TABLE druid_alltypesorc_n0; diff --git a/ql/src/test/queries/clientpositive/druidmini_test_insert.q b/ql/src/test/queries/clientpositive/druidmini_test_insert.q deleted file mode 100644 index 3cef847f59a5..000000000000 --- a/ql/src/test/queries/clientpositive/druidmini_test_insert.q +++ /dev/null @@ -1,157 +0,0 @@ ---! qt:disabled:HIVE-24816 ---! qt:dataset:alltypesorc -SET hive.ctas.external.tables=true; -SET hive.external.table.purge.default = true; -CREATE EXTERNAL TABLE druid_alltypesorc -STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' -TBLPROPERTIES ("druid.segment.granularity" = "HOUR", "druid.query.granularity" = "MINUTE") -AS -SELECT cast (`ctimestamp2` as timestamp with local time zone) as `__time`, - cstring1, - cstring2, - cdouble, - cfloat, - ctinyint, - csmallint, - cint, - cbigint, - cboolean1, - cboolean2 - FROM alltypesorc where ctimestamp2 IS NOT NULL; - -SELECT COUNT(*) FROM druid_alltypesorc; - -INSERT INTO TABLE druid_alltypesorc -SELECT cast (`ctimestamp1` as timestamp with local time zone) as `__time`, - cstring1, - cstring2, - cdouble, - cfloat, - ctinyint, - csmallint, - cint, - cbigint, - cboolean1, - cboolean2 - FROM alltypesorc where ctimestamp1 IS NOT NULL; - --- ANALYZE COLUMN STATS FOR DRUID TABLE -analyze table druid_alltypesorc compute statistics for columns; - -SELECT COUNT(*) FROM druid_alltypesorc; - -INSERT OVERWRITE TABLE druid_alltypesorc -SELECT cast (`ctimestamp1` as timestamp with local time zone) as `__time`, - cstring1, - cstring2, - cdouble, - cfloat, - ctinyint, - csmallint, - cint, - cbigint, - cboolean1, - cboolean2 - FROM alltypesorc where ctimestamp1 IS NOT NULL; - -SELECT COUNT(*) FROM druid_alltypesorc; - -DROP TABLE druid_alltypesorc; - - -- Test create then insert - - create database druid_test_create_then_insert; - use druid_test_create_then_insert; - - create table test_table_n9(`timecolumn` timestamp, `userid` string, `num_l` float); - - insert into test_table_n9 values ('2015-01-08 00:00:00', 'i1-start', 4); - insert into test_table_n9 values ('2015-01-08 23:59:59', 'i1-end', 1); - - CREATE EXTERNAL TABLE druid_table_n1 (`__time` timestamp with local time zone, `userid` string, `num_l` float) - STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' - TBLPROPERTIES ("druid.segment.granularity" = "DAY"); - - - INSERT INTO TABLE druid_table_n1 - select cast(`timecolumn` as timestamp with local time zone) as `__time`, `userid`, `num_l` FROM test_table_n9; - - select count(*) FROM druid_table_n1; - - DROP TABLE test_table_n9; - DROP TABLE druid_table_n1; - DROP DATABASE druid_test_create_then_insert; - --- Day light saving time test insert into test - -create database druid_test_dst; -use druid_test_dst; - -create table test_base_table(`timecolumn` timestamp, `userid` string, `num_l` float); -insert into test_base_table values ('2015-03-08 00:00:00', 'i1-start', 4); -insert into test_base_table values ('2015-03-08 23:59:59', 'i1-end', 1); -insert into test_base_table values ('2015-03-09 00:00:00', 'i2-start', 4); -insert into test_base_table values ('2015-03-09 23:59:59', 'i2-end', 1); -insert into test_base_table values ('2015-03-10 00:00:00', 'i3-start', 2); -insert into test_base_table values ('2015-03-10 23:59:59', 'i3-end', 2); - -CREATE EXTERNAL TABLE druid_test_table_n9 -STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' -TBLPROPERTIES ("druid.segment.granularity" = "DAY") -AS -select cast(`timecolumn` as timestamp with local time zone) as `__time`, `userid`, `num_l` FROM test_base_table; - -select * FROM druid_test_table_n9; - -select * from druid_test_table_n9 where `__time` = cast('2015-03-08 00:00:00' as timestamp with local time zone); -select * from druid_test_table_n9 where `__time` = cast('2015-03-08 23:59:59' as timestamp with local time zone); - -select * from druid_test_table_n9 where `__time` = cast('2015-03-09 00:00:00' as timestamp with local time zone); -select * from druid_test_table_n9 where `__time` = cast('2015-03-09 23:59:59' as timestamp with local time zone); - -select * from druid_test_table_n9 where `__time` = cast('2015-03-10 00:00:00' as timestamp with local time zone); -select * from druid_test_table_n9 where `__time` = cast('2015-03-10 23:59:59' as timestamp with local time zone); - - -explain select * from druid_test_table_n9 where `__time` = cast('2015-03-08 00:00:00' as timestamp with local time zone); -explain select * from druid_test_table_n9 where `__time` = cast('2015-03-08 23:59:59' as timestamp with local time zone); - -explain select * from druid_test_table_n9 where `__time` = cast('2015-03-09 00:00:00' as timestamp with local time zone); -explain select * from druid_test_table_n9 where `__time` = cast('2015-03-09 23:59:59' as timestamp with local time zone); - -explain select * from druid_test_table_n9 where `__time` = cast('2015-03-10 00:00:00' as timestamp with local time zone); -explain select * from druid_test_table_n9 where `__time` = cast('2015-03-10 23:59:59' as timestamp with local time zone); - - -select * from druid_test_table_n9 where `__time` = cast('2015-03-08 00:00:00' as timestamp ); -select * from druid_test_table_n9 where `__time` = cast('2015-03-08 23:59:59' as timestamp ); - -select * from druid_test_table_n9 where `__time` = cast('2015-03-09 00:00:00' as timestamp ); -select * from druid_test_table_n9 where `__time` = cast('2015-03-09 23:59:59' as timestamp ); - -select * from druid_test_table_n9 where `__time` = cast('2015-03-10 00:00:00' as timestamp ); -select * from druid_test_table_n9 where `__time` = cast('2015-03-10 23:59:59' as timestamp ); - - -EXPLAIN select * from druid_test_table_n9 where `__time` = cast('2015-03-08 00:00:00' as timestamp ); -EXPLAIN select * from druid_test_table_n9 where `__time` = cast('2015-03-08 23:59:59' as timestamp ); - -EXPLAIN select * from druid_test_table_n9 where `__time` = cast('2015-03-09 00:00:00' as timestamp ); -EXPLAIN select * from druid_test_table_n9 where `__time` = cast('2015-03-09 23:59:59' as timestamp ); - -EXPLAIN select * from druid_test_table_n9 where `__time` = cast('2015-03-10 00:00:00' as timestamp ); -EXPLAIN select * from druid_test_table_n9 where `__time` = cast('2015-03-10 23:59:59' as timestamp ); - --- disable rollup and insert identical rows -SET hive.druid.rollup=false; -insert into druid_test_table_n9 values -('2015-03-12 00:00:00', 'i4-1', 4), -('2015-03-12 00:00:00', 'i4-1', 2), -('2015-03-12 00:00:00', 'i4-1', 1); - -select * from druid_test_table_n9 where `__time` = cast('2015-03-12 00:00:00' as timestamp ); - -DROP TABLE test_base_table; -DROP TABLE druid_test_table_n9; - -drop database druid_test_dst; diff --git a/ql/src/test/queries/clientpositive/druidmini_test_ts.q b/ql/src/test/queries/clientpositive/druidmini_test_ts.q deleted file mode 100644 index 332f11ba7e9f..000000000000 --- a/ql/src/test/queries/clientpositive/druidmini_test_ts.q +++ /dev/null @@ -1,109 +0,0 @@ ---! qt:disabled:HIVE-24816 ---! qt:dataset:druid_table_alltypesorc - --- Time Series Query -SELECT count(*) FROM druid_table_alltypesorc; - -SELECT floor_year(`__time`), SUM(cfloat), SUM(cdouble), SUM(ctinyint), SUM(csmallint),SUM(cint), SUM(cbigint) -FROM druid_table_alltypesorc GROUP BY floor_year(`__time`); - -SELECT floor_year(`__time`), MIN(cfloat), MIN(cdouble), MIN(ctinyint), MIN(csmallint),MIN(cint), MIN(cbigint) -FROM druid_table_alltypesorc GROUP BY floor_year(`__time`); - -SELECT floor_year(`__time`), MAX(cfloat), MAX(cdouble), MAX(ctinyint), MAX(csmallint),MAX(cint), MAX(cbigint) -FROM druid_table_alltypesorc GROUP BY floor_year(`__time`); - - --- Group By - -SELECT cstring1, SUM(cdouble) as s FROM druid_table_alltypesorc GROUP BY cstring1 ORDER BY s ASC LIMIT 10; - -SELECT cstring2, MAX(cdouble) FROM druid_table_alltypesorc GROUP BY cstring2 ORDER BY cstring2 ASC LIMIT 10; - - --- TIME STUFF - -SELECT `__time` -FROM druid_table_alltypesorc ORDER BY `__time` ASC LIMIT 10; - -SELECT `__time` -FROM druid_table_alltypesorc -WHERE `__time` < '1970-03-01 00:00:00' ORDER BY `__time` ASC LIMIT 10; - -SELECT `__time` -FROM druid_table_alltypesorc -WHERE `__time` >= '1968-01-01 00:00:00' AND `__time` <= '1970-03-01 00:00:00' ORDER BY `__time` ASC LIMIT 10; - -SELECT `__time` -FROM druid_table_alltypesorc -WHERE `__time` >= '1968-01-01 00:00:00' AND `__time` <= '1970-03-01 00:00:00' - AND `__time` < '2011-01-01 00:00:00' ORDER BY `__time` ASC LIMIT 10; - -SELECT `__time` -FROM druid_table_alltypesorc -WHERE `__time` BETWEEN '1968-01-01 00:00:00' AND '1970-01-01 00:00:00' ORDER BY `__time` ASC LIMIT 10;; - -SELECT `__time` -FROM druid_table_alltypesorc -WHERE (`__time` BETWEEN '1968-01-01 00:00:00' AND '1970-01-01 00:00:00') - OR (`__time` BETWEEN '1968-02-01 00:00:00' AND '1970-04-01 00:00:00') ORDER BY `__time` ASC LIMIT 10; - --- (-∞‥+∞) -EXPLAIN -SELECT `__time` -FROM druid_table_alltypesorc; - --- (-∞‥2012-03-01 00:00:00) -EXPLAIN -SELECT `__time` -FROM druid_table_alltypesorc -WHERE `__time` < '2012-03-01 00:00:00'; - --- [2010-01-01 00:00:00‥2012-03-01 00:00:00) -EXPLAIN -SELECT `__time` -FROM druid_table_alltypesorc -WHERE `__time` >= '2010-01-01 00:00:00' AND `__time` <= '2012-03-01 00:00:00'; - --- [2010-01-01 00:00:00‥2011-01-01 00:00:00) -EXPLAIN -SELECT `__time` -FROM druid_table_alltypesorc -WHERE `__time` >= '2010-01-01 00:00:00' AND `__time` <= '2012-03-01 00:00:00' - AND `__time` < '2011-01-01 00:00:00'; - --- [2010-01-01 00:00:00‥2011-01-01 00:00:00] -EXPLAIN -SELECT `__time` -FROM druid_table_alltypesorc -WHERE `__time` BETWEEN '2010-01-01 00:00:00' AND '2011-01-01 00:00:00'; - --- [2010-01-01 00:00:00‥2011-01-01 00:00:00],[2012-01-01 00:00:00‥2013-01-01 00:00:00] -EXPLAIN -SELECT `__time` -FROM druid_table_alltypesorc -WHERE (`__time` BETWEEN '2010-01-01 00:00:00' AND '2011-01-01 00:00:00') - OR (`__time` BETWEEN '2012-01-01 00:00:00' AND '2013-01-01 00:00:00'); - --- OVERLAP [2010-01-01 00:00:00‥2012-01-01 00:00:00] -EXPLAIN -SELECT `__time` -FROM druid_table_alltypesorc -WHERE (`__time` BETWEEN '2010-01-01 00:00:00' AND '2011-01-01 00:00:00') - OR (`__time` BETWEEN '2010-06-01 00:00:00' AND '2012-01-01 00:00:00'); - --- IN: MULTIPLE INTERVALS [2010-01-01 00:00:00‥2010-01-01 00:00:00),[2011-01-01 00:00:00‥2011-01-01 00:00:00) -EXPLAIN -SELECT `__time` -FROM druid_table_alltypesorc -WHERE `__time` IN ('2010-01-01 00:00:00','2011-01-01 00:00:00'); - -EXPLAIN -SELECT `__time`, cstring2 -FROM druid_table_alltypesorc -WHERE cstring2 = 'user1' AND `__time` IN ('2010-01-01 00:00:00','2011-01-01 00:00:00'); - -EXPLAIN -SELECT `__time`, cstring2 -FROM druid_table_alltypesorc -WHERE cstring2 = 'user1' OR `__time` IN ('2010-01-01 00:00:00','2011-01-01 00:00:00'); diff --git a/ql/src/test/results/clientnegative/druid_address.q.out b/ql/src/test/results/clientnegative/druid_address.q.out deleted file mode 100644 index cb9318044daf..000000000000 --- a/ql/src/test/results/clientnegative/druid_address.q.out +++ /dev/null @@ -1,7 +0,0 @@ -PREHOOK: query: CREATE EXTERNAL TABLE druid_table_1 -STORED BY 'org.apache.hadoop.hive.druid.QTestDruidStorageHandler' -TBLPROPERTIES ("druid.datasource" = "wikipedia") -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@druid_table_1 -FAILED: Execution Error, return code 40000 from org.apache.hadoop.hive.ql.ddl.DDLTask. java.lang.RuntimeException: MetaException(message:org.apache.hadoop.hive.serde2.SerDeException Druid broker address not specified in configuration) diff --git a/ql/src/test/results/clientnegative/druid_buckets.q.out b/ql/src/test/results/clientnegative/druid_buckets.q.out deleted file mode 100644 index 0f926ae90f5a..000000000000 --- a/ql/src/test/results/clientnegative/druid_buckets.q.out +++ /dev/null @@ -1,8 +0,0 @@ -PREHOOK: query: CREATE EXTERNAL TABLE druid_table_1 -CLUSTERED BY (robot) INTO 32 BUCKETS -STORED BY 'org.apache.hadoop.hive.druid.QTestDruidStorageHandler' -TBLPROPERTIES ("druid.datasource" = "wikipedia") -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@druid_table_1 -FAILED: Execution Error, return code 40000 from org.apache.hadoop.hive.ql.ddl.DDLTask. MetaException(message:CLUSTERED BY may not be specified for Druid) diff --git a/ql/src/test/results/clientnegative/druid_case.q.out b/ql/src/test/results/clientnegative/druid_case.q.out deleted file mode 100644 index 78ff4184f399..000000000000 --- a/ql/src/test/results/clientnegative/druid_case.q.out +++ /dev/null @@ -1,7 +0,0 @@ -PREHOOK: query: CREATE EXTERNAL TABLE druid_table_1 -STORED BY 'org.apache.hadoop.hive.druid.QTestDruidStorageHandler2' -TBLPROPERTIES ("druid.datasource" = "wikipedia") -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@druid_table_1 -FAILED: Execution Error, return code 40000 from org.apache.hadoop.hive.ql.ddl.DDLTask. org.apache.hadoop.hive.ql.metadata.HiveException: Duplicate column name anonymous in the table definition. diff --git a/ql/src/test/results/clientnegative/druid_datasource.q.out b/ql/src/test/results/clientnegative/druid_datasource.q.out deleted file mode 100644 index 426bb180311b..000000000000 --- a/ql/src/test/results/clientnegative/druid_datasource.q.out +++ /dev/null @@ -1,7 +0,0 @@ -PREHOOK: query: CREATE EXTERNAL TABLE druid_table_1 -STORED BY 'org.apache.hadoop.hive.druid.QTestDruidStorageHandler' -TBLPROPERTIES ("property" = "localhost") -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@druid_table_1 -FAILED: Execution Error, return code 40000 from org.apache.hadoop.hive.ql.ddl.DDLTask. java.lang.RuntimeException: MetaException(message:org.apache.hadoop.hive.serde2.SerDeException Druid data source not specified; use druid.datasource in table properties) diff --git a/ql/src/test/results/clientnegative/druid_datasource2.q.out b/ql/src/test/results/clientnegative/druid_datasource2.q.out deleted file mode 100644 index 4bbba4fa867c..000000000000 --- a/ql/src/test/results/clientnegative/druid_datasource2.q.out +++ /dev/null @@ -1,7 +0,0 @@ -PREHOOK: query: CREATE TABLE druid_table_1 -STORED BY 'org.apache.hadoop.hive.druid.QTestDruidStorageHandler' -TBLPROPERTIES ("property" = "localhost", "druid.datasource" = "mydatasource") -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@druid_table_1 -FAILED: Execution Error, return code 40000 from org.apache.hadoop.hive.ql.ddl.DDLTask. MetaException(message:Table default.druid_table_1 failed strict managed table checks due to the following reason: Table is marked as a managed table but is not transactional.) diff --git a/ql/src/test/results/clientnegative/druid_location.q.out b/ql/src/test/results/clientnegative/druid_location.q.out deleted file mode 100644 index dc47a1885908..000000000000 --- a/ql/src/test/results/clientnegative/druid_location.q.out +++ /dev/null @@ -1,9 +0,0 @@ -PREHOOK: query: CREATE EXTERNAL TABLE druid_table_1 -STORED BY 'org.apache.hadoop.hive.druid.QTestDruidStorageHandler' -#### A masked pattern was here #### -TBLPROPERTIES ("druid.datasource" = "wikipedia") -PREHOOK: type: CREATETABLE -#### A masked pattern was here #### -PREHOOK: Output: database:default -PREHOOK: Output: default@druid_table_1 -FAILED: Execution Error, return code 40000 from org.apache.hadoop.hive.ql.ddl.DDLTask. MetaException(message:LOCATION may not be specified for Druid) diff --git a/ql/src/test/results/clientnegative/druid_partitions.q.out b/ql/src/test/results/clientnegative/druid_partitions.q.out deleted file mode 100644 index f977d6cda57a..000000000000 --- a/ql/src/test/results/clientnegative/druid_partitions.q.out +++ /dev/null @@ -1,8 +0,0 @@ -PREHOOK: query: CREATE EXTERNAL TABLE druid_table_1 -PARTITIONED BY (dt string) -STORED BY 'org.apache.hadoop.hive.druid.QTestDruidStorageHandler' -TBLPROPERTIES ("druid.datasource" = "wikipedia") -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@druid_table_1 -FAILED: Execution Error, return code 40000 from org.apache.hadoop.hive.ql.ddl.DDLTask. MetaException(message:PARTITIONED BY may not be specified for Druid) diff --git a/ql/src/test/results/clientpositive/druid/druid_materialized_view_rewrite_ssb.q.out b/ql/src/test/results/clientpositive/druid/druid_materialized_view_rewrite_ssb.q.out deleted file mode 100644 index 4aee9c20ec0d..000000000000 --- a/ql/src/test/results/clientpositive/druid/druid_materialized_view_rewrite_ssb.q.out +++ /dev/null @@ -1,813 +0,0 @@ -PREHOOK: query: CREATE TABLE `customer_ext_n0`( - `c_custkey` bigint, - `c_name` string, - `c_address` string, - `c_city` string, - `c_nation` string, - `c_region` string, - `c_phone` string, - `c_mktsegment` string) -ROW FORMAT DELIMITED -FIELDS TERMINATED BY '|' -STORED AS TEXTFILE -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@customer_ext_n0 -POSTHOOK: query: CREATE TABLE `customer_ext_n0`( - `c_custkey` bigint, - `c_name` string, - `c_address` string, - `c_city` string, - `c_nation` string, - `c_region` string, - `c_phone` string, - `c_mktsegment` string) -ROW FORMAT DELIMITED -FIELDS TERMINATED BY '|' -STORED AS TEXTFILE -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@customer_ext_n0 -PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/ssb/customer/' into table `customer_ext_n0` -PREHOOK: type: LOAD -#### A masked pattern was here #### -PREHOOK: Output: default@customer_ext_n0 -POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/ssb/customer/' into table `customer_ext_n0` -POSTHOOK: type: LOAD -#### A masked pattern was here #### -POSTHOOK: Output: default@customer_ext_n0 -PREHOOK: query: CREATE TABLE `customer_n0`( - `c_custkey` bigint, - `c_name` string, - `c_address` string, - `c_city` string, - `c_nation` string, - `c_region` string, - `c_phone` string, - `c_mktsegment` string, - primary key (`c_custkey`) disable rely) -STORED AS ORC -TBLPROPERTIES ('transactional'='true') -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@customer_n0 -POSTHOOK: query: CREATE TABLE `customer_n0`( - `c_custkey` bigint, - `c_name` string, - `c_address` string, - `c_city` string, - `c_nation` string, - `c_region` string, - `c_phone` string, - `c_mktsegment` string, - primary key (`c_custkey`) disable rely) -STORED AS ORC -TBLPROPERTIES ('transactional'='true') -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@customer_n0 -PREHOOK: query: INSERT INTO `customer_n0` -SELECT * FROM `customer_ext_n0` -PREHOOK: type: QUERY -PREHOOK: Input: default@customer_ext_n0 -PREHOOK: Output: default@customer_n0 -POSTHOOK: query: INSERT INTO `customer_n0` -SELECT * FROM `customer_ext_n0` -POSTHOOK: type: QUERY -POSTHOOK: Input: default@customer_ext_n0 -POSTHOOK: Output: default@customer_n0 -POSTHOOK: Lineage: customer_n0.c_address SIMPLE [(customer_ext_n0)customer_ext_n0.FieldSchema(name:c_address, type:string, comment:null), ] -POSTHOOK: Lineage: customer_n0.c_city SIMPLE [(customer_ext_n0)customer_ext_n0.FieldSchema(name:c_city, type:string, comment:null), ] -POSTHOOK: Lineage: customer_n0.c_custkey SIMPLE [(customer_ext_n0)customer_ext_n0.FieldSchema(name:c_custkey, type:bigint, comment:null), ] -POSTHOOK: Lineage: customer_n0.c_mktsegment SIMPLE [(customer_ext_n0)customer_ext_n0.FieldSchema(name:c_mktsegment, type:string, comment:null), ] -POSTHOOK: Lineage: customer_n0.c_name SIMPLE [(customer_ext_n0)customer_ext_n0.FieldSchema(name:c_name, type:string, comment:null), ] -POSTHOOK: Lineage: customer_n0.c_nation SIMPLE [(customer_ext_n0)customer_ext_n0.FieldSchema(name:c_nation, type:string, comment:null), ] -POSTHOOK: Lineage: customer_n0.c_phone SIMPLE [(customer_ext_n0)customer_ext_n0.FieldSchema(name:c_phone, type:string, comment:null), ] -POSTHOOK: Lineage: customer_n0.c_region SIMPLE [(customer_ext_n0)customer_ext_n0.FieldSchema(name:c_region, type:string, comment:null), ] -PREHOOK: query: CREATE TABLE `dates_ext_n0`( - `d_datekey` bigint, - `d_date` string, - `d_dayofweek` string, - `d_month` string, - `d_year` int, - `d_yearmonthnum` int, - `d_yearmonth` string, - `d_daynuminweek` int, - `d_daynuminmonth` int, - `d_daynuminyear` int, - `d_monthnuminyear` int, - `d_weeknuminyear` int, - `d_sellingseason` string, - `d_lastdayinweekfl` int, - `d_lastdayinmonthfl` int, - `d_holidayfl` int , - `d_weekdayfl`int) -ROW FORMAT DELIMITED -FIELDS TERMINATED BY '|' -STORED AS TEXTFILE -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@dates_ext_n0 -POSTHOOK: query: CREATE TABLE `dates_ext_n0`( - `d_datekey` bigint, - `d_date` string, - `d_dayofweek` string, - `d_month` string, - `d_year` int, - `d_yearmonthnum` int, - `d_yearmonth` string, - `d_daynuminweek` int, - `d_daynuminmonth` int, - `d_daynuminyear` int, - `d_monthnuminyear` int, - `d_weeknuminyear` int, - `d_sellingseason` string, - `d_lastdayinweekfl` int, - `d_lastdayinmonthfl` int, - `d_holidayfl` int , - `d_weekdayfl`int) -ROW FORMAT DELIMITED -FIELDS TERMINATED BY '|' -STORED AS TEXTFILE -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@dates_ext_n0 -PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/ssb/date/' into table `dates_ext_n0` -PREHOOK: type: LOAD -#### A masked pattern was here #### -PREHOOK: Output: default@dates_ext_n0 -POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/ssb/date/' into table `dates_ext_n0` -POSTHOOK: type: LOAD -#### A masked pattern was here #### -POSTHOOK: Output: default@dates_ext_n0 -PREHOOK: query: CREATE TABLE `dates_n0`( - `d_datekey` bigint, - `__time` timestamp, - `d_date` string, - `d_dayofweek` string, - `d_month` string, - `d_year` int, - `d_yearmonthnum` int, - `d_yearmonth` string, - `d_daynuminweek` int, - `d_daynuminmonth` int, - `d_daynuminyear` int, - `d_monthnuminyear` int, - `d_weeknuminyear` int, - `d_sellingseason` string, - `d_lastdayinweekfl` int, - `d_lastdayinmonthfl` int, - `d_holidayfl` int , - `d_weekdayfl`int, - primary key (`d_datekey`) disable rely -) -STORED AS ORC -TBLPROPERTIES ('transactional'='true') -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@dates_n0 -POSTHOOK: query: CREATE TABLE `dates_n0`( - `d_datekey` bigint, - `__time` timestamp, - `d_date` string, - `d_dayofweek` string, - `d_month` string, - `d_year` int, - `d_yearmonthnum` int, - `d_yearmonth` string, - `d_daynuminweek` int, - `d_daynuminmonth` int, - `d_daynuminyear` int, - `d_monthnuminyear` int, - `d_weeknuminyear` int, - `d_sellingseason` string, - `d_lastdayinweekfl` int, - `d_lastdayinmonthfl` int, - `d_holidayfl` int , - `d_weekdayfl`int, - primary key (`d_datekey`) disable rely -) -STORED AS ORC -TBLPROPERTIES ('transactional'='true') -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@dates_n0 -PREHOOK: query: INSERT INTO `dates_n0` -SELECT - `d_datekey`, - cast(`d_year` || '-' || `d_monthnuminyear` || '-' || `d_daynuminmonth` as timestamp), - `d_date`, - `d_dayofweek`, - `d_month`, - `d_year`, - `d_yearmonthnum`, - `d_yearmonth`, - `d_daynuminweek`, - `d_daynuminmonth`, - `d_daynuminyear`, - `d_monthnuminyear`, - `d_weeknuminyear`, - `d_sellingseason`, - `d_lastdayinweekfl`, - `d_lastdayinmonthfl`, - `d_holidayfl`, - `d_weekdayfl` -FROM `dates_ext_n0` -PREHOOK: type: QUERY -PREHOOK: Input: default@dates_ext_n0 -PREHOOK: Output: default@dates_n0 -POSTHOOK: query: INSERT INTO `dates_n0` -SELECT - `d_datekey`, - cast(`d_year` || '-' || `d_monthnuminyear` || '-' || `d_daynuminmonth` as timestamp), - `d_date`, - `d_dayofweek`, - `d_month`, - `d_year`, - `d_yearmonthnum`, - `d_yearmonth`, - `d_daynuminweek`, - `d_daynuminmonth`, - `d_daynuminyear`, - `d_monthnuminyear`, - `d_weeknuminyear`, - `d_sellingseason`, - `d_lastdayinweekfl`, - `d_lastdayinmonthfl`, - `d_holidayfl`, - `d_weekdayfl` -FROM `dates_ext_n0` -POSTHOOK: type: QUERY -POSTHOOK: Input: default@dates_ext_n0 -POSTHOOK: Output: default@dates_n0 -POSTHOOK: Lineage: dates_n0.__time EXPRESSION [(dates_ext_n0)dates_ext_n0.FieldSchema(name:d_year, type:int, comment:null), (dates_ext_n0)dates_ext_n0.FieldSchema(name:d_monthnuminyear, type:int, comment:null), (dates_ext_n0)dates_ext_n0.FieldSchema(name:d_daynuminmonth, type:int, comment:null), ] -POSTHOOK: Lineage: dates_n0.d_date SIMPLE [(dates_ext_n0)dates_ext_n0.FieldSchema(name:d_date, type:string, comment:null), ] -POSTHOOK: Lineage: dates_n0.d_datekey SIMPLE [(dates_ext_n0)dates_ext_n0.FieldSchema(name:d_datekey, type:bigint, comment:null), ] -POSTHOOK: Lineage: dates_n0.d_daynuminmonth SIMPLE [(dates_ext_n0)dates_ext_n0.FieldSchema(name:d_daynuminmonth, type:int, comment:null), ] -POSTHOOK: Lineage: dates_n0.d_daynuminweek SIMPLE [(dates_ext_n0)dates_ext_n0.FieldSchema(name:d_daynuminweek, type:int, comment:null), ] -POSTHOOK: Lineage: dates_n0.d_daynuminyear SIMPLE [(dates_ext_n0)dates_ext_n0.FieldSchema(name:d_daynuminyear, type:int, comment:null), ] -POSTHOOK: Lineage: dates_n0.d_dayofweek SIMPLE [(dates_ext_n0)dates_ext_n0.FieldSchema(name:d_dayofweek, type:string, comment:null), ] -POSTHOOK: Lineage: dates_n0.d_holidayfl SIMPLE [(dates_ext_n0)dates_ext_n0.FieldSchema(name:d_holidayfl, type:int, comment:null), ] -POSTHOOK: Lineage: dates_n0.d_lastdayinmonthfl SIMPLE [(dates_ext_n0)dates_ext_n0.FieldSchema(name:d_lastdayinmonthfl, type:int, comment:null), ] -POSTHOOK: Lineage: dates_n0.d_lastdayinweekfl SIMPLE [(dates_ext_n0)dates_ext_n0.FieldSchema(name:d_lastdayinweekfl, type:int, comment:null), ] -POSTHOOK: Lineage: dates_n0.d_month SIMPLE [(dates_ext_n0)dates_ext_n0.FieldSchema(name:d_month, type:string, comment:null), ] -POSTHOOK: Lineage: dates_n0.d_monthnuminyear SIMPLE [(dates_ext_n0)dates_ext_n0.FieldSchema(name:d_monthnuminyear, type:int, comment:null), ] -POSTHOOK: Lineage: dates_n0.d_sellingseason SIMPLE [(dates_ext_n0)dates_ext_n0.FieldSchema(name:d_sellingseason, type:string, comment:null), ] -POSTHOOK: Lineage: dates_n0.d_weekdayfl SIMPLE [(dates_ext_n0)dates_ext_n0.FieldSchema(name:d_weekdayfl, type:int, comment:null), ] -POSTHOOK: Lineage: dates_n0.d_weeknuminyear SIMPLE [(dates_ext_n0)dates_ext_n0.FieldSchema(name:d_weeknuminyear, type:int, comment:null), ] -POSTHOOK: Lineage: dates_n0.d_year SIMPLE [(dates_ext_n0)dates_ext_n0.FieldSchema(name:d_year, type:int, comment:null), ] -POSTHOOK: Lineage: dates_n0.d_yearmonth SIMPLE [(dates_ext_n0)dates_ext_n0.FieldSchema(name:d_yearmonth, type:string, comment:null), ] -POSTHOOK: Lineage: dates_n0.d_yearmonthnum SIMPLE [(dates_ext_n0)dates_ext_n0.FieldSchema(name:d_yearmonthnum, type:int, comment:null), ] -PREHOOK: query: CREATE TABLE `ssb_part_ext_n0`( - `p_partkey` bigint, - `p_name` string, - `p_mfgr` string, - `p_category` string, - `p_brand1` string, - `p_color` string, - `p_type` string, - `p_size` int, - `p_container` string) -ROW FORMAT DELIMITED -FIELDS TERMINATED BY '|' -STORED AS TEXTFILE -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@ssb_part_ext_n0 -POSTHOOK: query: CREATE TABLE `ssb_part_ext_n0`( - `p_partkey` bigint, - `p_name` string, - `p_mfgr` string, - `p_category` string, - `p_brand1` string, - `p_color` string, - `p_type` string, - `p_size` int, - `p_container` string) -ROW FORMAT DELIMITED -FIELDS TERMINATED BY '|' -STORED AS TEXTFILE -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@ssb_part_ext_n0 -PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/ssb/part/' into table `ssb_part_ext_n0` -PREHOOK: type: LOAD -#### A masked pattern was here #### -PREHOOK: Output: default@ssb_part_ext_n0 -POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/ssb/part/' into table `ssb_part_ext_n0` -POSTHOOK: type: LOAD -#### A masked pattern was here #### -POSTHOOK: Output: default@ssb_part_ext_n0 -PREHOOK: query: CREATE TABLE `ssb_part_n0`( - `p_partkey` bigint, - `p_name` string, - `p_mfgr` string, - `p_category` string, - `p_brand1` string, - `p_color` string, - `p_type` string, - `p_size` int, - `p_container` string, - primary key (`p_partkey`) disable rely) -STORED AS ORC -TBLPROPERTIES ('transactional'='true') -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@ssb_part_n0 -POSTHOOK: query: CREATE TABLE `ssb_part_n0`( - `p_partkey` bigint, - `p_name` string, - `p_mfgr` string, - `p_category` string, - `p_brand1` string, - `p_color` string, - `p_type` string, - `p_size` int, - `p_container` string, - primary key (`p_partkey`) disable rely) -STORED AS ORC -TBLPROPERTIES ('transactional'='true') -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@ssb_part_n0 -PREHOOK: query: INSERT INTO `ssb_part_n0` -SELECT * FROM `ssb_part_ext_n0` -PREHOOK: type: QUERY -PREHOOK: Input: default@ssb_part_ext_n0 -PREHOOK: Output: default@ssb_part_n0 -POSTHOOK: query: INSERT INTO `ssb_part_n0` -SELECT * FROM `ssb_part_ext_n0` -POSTHOOK: type: QUERY -POSTHOOK: Input: default@ssb_part_ext_n0 -POSTHOOK: Output: default@ssb_part_n0 -POSTHOOK: Lineage: ssb_part_n0.p_brand1 SIMPLE [(ssb_part_ext_n0)ssb_part_ext_n0.FieldSchema(name:p_brand1, type:string, comment:null), ] -POSTHOOK: Lineage: ssb_part_n0.p_category SIMPLE [(ssb_part_ext_n0)ssb_part_ext_n0.FieldSchema(name:p_category, type:string, comment:null), ] -POSTHOOK: Lineage: ssb_part_n0.p_color SIMPLE [(ssb_part_ext_n0)ssb_part_ext_n0.FieldSchema(name:p_color, type:string, comment:null), ] -POSTHOOK: Lineage: ssb_part_n0.p_container SIMPLE [(ssb_part_ext_n0)ssb_part_ext_n0.FieldSchema(name:p_container, type:string, comment:null), ] -POSTHOOK: Lineage: ssb_part_n0.p_mfgr SIMPLE [(ssb_part_ext_n0)ssb_part_ext_n0.FieldSchema(name:p_mfgr, type:string, comment:null), ] -POSTHOOK: Lineage: ssb_part_n0.p_name SIMPLE [(ssb_part_ext_n0)ssb_part_ext_n0.FieldSchema(name:p_name, type:string, comment:null), ] -POSTHOOK: Lineage: ssb_part_n0.p_partkey SIMPLE [(ssb_part_ext_n0)ssb_part_ext_n0.FieldSchema(name:p_partkey, type:bigint, comment:null), ] -POSTHOOK: Lineage: ssb_part_n0.p_size SIMPLE [(ssb_part_ext_n0)ssb_part_ext_n0.FieldSchema(name:p_size, type:int, comment:null), ] -POSTHOOK: Lineage: ssb_part_n0.p_type SIMPLE [(ssb_part_ext_n0)ssb_part_ext_n0.FieldSchema(name:p_type, type:string, comment:null), ] -PREHOOK: query: CREATE TABLE `supplier_ext_n0`( - `s_suppkey` bigint, - `s_name` string, - `s_address` string, - `s_city` string, - `s_nation` string, - `s_region` string, - `s_phone` string) -ROW FORMAT DELIMITED -FIELDS TERMINATED BY '|' -STORED AS TEXTFILE -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@supplier_ext_n0 -POSTHOOK: query: CREATE TABLE `supplier_ext_n0`( - `s_suppkey` bigint, - `s_name` string, - `s_address` string, - `s_city` string, - `s_nation` string, - `s_region` string, - `s_phone` string) -ROW FORMAT DELIMITED -FIELDS TERMINATED BY '|' -STORED AS TEXTFILE -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@supplier_ext_n0 -PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/ssb/supplier/' into table `supplier_ext_n0` -PREHOOK: type: LOAD -#### A masked pattern was here #### -PREHOOK: Output: default@supplier_ext_n0 -POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/ssb/supplier/' into table `supplier_ext_n0` -POSTHOOK: type: LOAD -#### A masked pattern was here #### -POSTHOOK: Output: default@supplier_ext_n0 -PREHOOK: query: CREATE TABLE `supplier_n0`( - `s_suppkey` bigint, - `s_name` string, - `s_address` string, - `s_city` string, - `s_nation` string, - `s_region` string, - `s_phone` string, - primary key (`s_suppkey`) disable rely) -STORED AS ORC -TBLPROPERTIES ('transactional'='true') -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@supplier_n0 -POSTHOOK: query: CREATE TABLE `supplier_n0`( - `s_suppkey` bigint, - `s_name` string, - `s_address` string, - `s_city` string, - `s_nation` string, - `s_region` string, - `s_phone` string, - primary key (`s_suppkey`) disable rely) -STORED AS ORC -TBLPROPERTIES ('transactional'='true') -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@supplier_n0 -PREHOOK: query: INSERT INTO `supplier_n0` -SELECT * FROM `supplier_ext_n0` -PREHOOK: type: QUERY -PREHOOK: Input: default@supplier_ext_n0 -PREHOOK: Output: default@supplier_n0 -POSTHOOK: query: INSERT INTO `supplier_n0` -SELECT * FROM `supplier_ext_n0` -POSTHOOK: type: QUERY -POSTHOOK: Input: default@supplier_ext_n0 -POSTHOOK: Output: default@supplier_n0 -POSTHOOK: Lineage: supplier_n0.s_address SIMPLE [(supplier_ext_n0)supplier_ext_n0.FieldSchema(name:s_address, type:string, comment:null), ] -POSTHOOK: Lineage: supplier_n0.s_city SIMPLE [(supplier_ext_n0)supplier_ext_n0.FieldSchema(name:s_city, type:string, comment:null), ] -POSTHOOK: Lineage: supplier_n0.s_name SIMPLE [(supplier_ext_n0)supplier_ext_n0.FieldSchema(name:s_name, type:string, comment:null), ] -POSTHOOK: Lineage: supplier_n0.s_nation SIMPLE [(supplier_ext_n0)supplier_ext_n0.FieldSchema(name:s_nation, type:string, comment:null), ] -POSTHOOK: Lineage: supplier_n0.s_phone SIMPLE [(supplier_ext_n0)supplier_ext_n0.FieldSchema(name:s_phone, type:string, comment:null), ] -POSTHOOK: Lineage: supplier_n0.s_region SIMPLE [(supplier_ext_n0)supplier_ext_n0.FieldSchema(name:s_region, type:string, comment:null), ] -POSTHOOK: Lineage: supplier_n0.s_suppkey SIMPLE [(supplier_ext_n0)supplier_ext_n0.FieldSchema(name:s_suppkey, type:bigint, comment:null), ] -PREHOOK: query: CREATE TABLE `lineorder_ext_n0`( - `lo_orderkey` bigint, - `lo_linenumber` int, - `lo_custkey` bigint not null disable rely, - `lo_partkey` bigint not null disable rely, - `lo_suppkey` bigint not null disable rely, - `lo_orderdate` bigint not null disable rely, - `lo_ordpriority` string, - `lo_shippriority` string, - `lo_quantity` double, - `lo_extendedprice` double, - `lo_ordtotalprice` double, - `lo_discount` double, - `lo_revenue` double, - `lo_supplycost` double, - `lo_tax` double, - `lo_commitdate` bigint, - `lo_shipmode` string) -ROW FORMAT DELIMITED -FIELDS TERMINATED BY '|' -STORED AS TEXTFILE -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@lineorder_ext_n0 -POSTHOOK: query: CREATE TABLE `lineorder_ext_n0`( - `lo_orderkey` bigint, - `lo_linenumber` int, - `lo_custkey` bigint not null disable rely, - `lo_partkey` bigint not null disable rely, - `lo_suppkey` bigint not null disable rely, - `lo_orderdate` bigint not null disable rely, - `lo_ordpriority` string, - `lo_shippriority` string, - `lo_quantity` double, - `lo_extendedprice` double, - `lo_ordtotalprice` double, - `lo_discount` double, - `lo_revenue` double, - `lo_supplycost` double, - `lo_tax` double, - `lo_commitdate` bigint, - `lo_shipmode` string) -ROW FORMAT DELIMITED -FIELDS TERMINATED BY '|' -STORED AS TEXTFILE -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@lineorder_ext_n0 -PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/ssb/lineorder/' into table `lineorder_ext_n0` -PREHOOK: type: LOAD -#### A masked pattern was here #### -PREHOOK: Output: default@lineorder_ext_n0 -POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/ssb/lineorder/' into table `lineorder_ext_n0` -POSTHOOK: type: LOAD -#### A masked pattern was here #### -POSTHOOK: Output: default@lineorder_ext_n0 -PREHOOK: query: CREATE TABLE `lineorder_n0`( - `lo_orderkey` bigint, - `lo_linenumber` int, - `lo_custkey` bigint not null disable rely, - `lo_partkey` bigint not null disable rely, - `lo_suppkey` bigint not null disable rely, - `lo_orderdate` bigint not null disable rely, - `lo_ordpriority` string, - `lo_shippriority` string, - `lo_quantity` double, - `lo_extendedprice` double, - `lo_ordtotalprice` double, - `lo_discount` double, - `lo_revenue` double, - `lo_supplycost` double, - `lo_tax` double, - `lo_commitdate` bigint, - `lo_shipmode` string, - primary key (`lo_orderkey`) disable rely, - constraint fk1 foreign key (`lo_custkey`) references `customer_n0`(`c_custkey`) disable rely, - constraint fk2 foreign key (`lo_orderdate`) references `dates_n0`(`d_datekey`) disable rely, - constraint fk3 foreign key (`lo_partkey`) references `ssb_part_n0`(`p_partkey`) disable rely, - constraint fk4 foreign key (`lo_suppkey`) references `supplier_n0`(`s_suppkey`) disable rely) -STORED AS ORC -TBLPROPERTIES ('transactional'='true') -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@lineorder_n0 -POSTHOOK: query: CREATE TABLE `lineorder_n0`( - `lo_orderkey` bigint, - `lo_linenumber` int, - `lo_custkey` bigint not null disable rely, - `lo_partkey` bigint not null disable rely, - `lo_suppkey` bigint not null disable rely, - `lo_orderdate` bigint not null disable rely, - `lo_ordpriority` string, - `lo_shippriority` string, - `lo_quantity` double, - `lo_extendedprice` double, - `lo_ordtotalprice` double, - `lo_discount` double, - `lo_revenue` double, - `lo_supplycost` double, - `lo_tax` double, - `lo_commitdate` bigint, - `lo_shipmode` string, - primary key (`lo_orderkey`) disable rely, - constraint fk1 foreign key (`lo_custkey`) references `customer_n0`(`c_custkey`) disable rely, - constraint fk2 foreign key (`lo_orderdate`) references `dates_n0`(`d_datekey`) disable rely, - constraint fk3 foreign key (`lo_partkey`) references `ssb_part_n0`(`p_partkey`) disable rely, - constraint fk4 foreign key (`lo_suppkey`) references `supplier_n0`(`s_suppkey`) disable rely) -STORED AS ORC -TBLPROPERTIES ('transactional'='true') -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@lineorder_n0 -PREHOOK: query: INSERT INTO `lineorder_n0` -SELECT * FROM `lineorder_ext_n0` -PREHOOK: type: QUERY -PREHOOK: Input: default@lineorder_ext_n0 -PREHOOK: Output: default@lineorder_n0 -POSTHOOK: query: INSERT INTO `lineorder_n0` -SELECT * FROM `lineorder_ext_n0` -POSTHOOK: type: QUERY -POSTHOOK: Input: default@lineorder_ext_n0 -POSTHOOK: Output: default@lineorder_n0 -POSTHOOK: Lineage: lineorder_n0.lo_commitdate SIMPLE [(lineorder_ext_n0)lineorder_ext_n0.FieldSchema(name:lo_commitdate, type:bigint, comment:null), ] -POSTHOOK: Lineage: lineorder_n0.lo_custkey SIMPLE [(lineorder_ext_n0)lineorder_ext_n0.FieldSchema(name:lo_custkey, type:bigint, comment:null), ] -POSTHOOK: Lineage: lineorder_n0.lo_discount SIMPLE [(lineorder_ext_n0)lineorder_ext_n0.FieldSchema(name:lo_discount, type:double, comment:null), ] -POSTHOOK: Lineage: lineorder_n0.lo_extendedprice SIMPLE [(lineorder_ext_n0)lineorder_ext_n0.FieldSchema(name:lo_extendedprice, type:double, comment:null), ] -POSTHOOK: Lineage: lineorder_n0.lo_linenumber SIMPLE [(lineorder_ext_n0)lineorder_ext_n0.FieldSchema(name:lo_linenumber, type:int, comment:null), ] -POSTHOOK: Lineage: lineorder_n0.lo_orderdate SIMPLE [(lineorder_ext_n0)lineorder_ext_n0.FieldSchema(name:lo_orderdate, type:bigint, comment:null), ] -POSTHOOK: Lineage: lineorder_n0.lo_orderkey SIMPLE [(lineorder_ext_n0)lineorder_ext_n0.FieldSchema(name:lo_orderkey, type:bigint, comment:null), ] -POSTHOOK: Lineage: lineorder_n0.lo_ordpriority SIMPLE [(lineorder_ext_n0)lineorder_ext_n0.FieldSchema(name:lo_ordpriority, type:string, comment:null), ] -POSTHOOK: Lineage: lineorder_n0.lo_ordtotalprice SIMPLE [(lineorder_ext_n0)lineorder_ext_n0.FieldSchema(name:lo_ordtotalprice, type:double, comment:null), ] -POSTHOOK: Lineage: lineorder_n0.lo_partkey SIMPLE [(lineorder_ext_n0)lineorder_ext_n0.FieldSchema(name:lo_partkey, type:bigint, comment:null), ] -POSTHOOK: Lineage: lineorder_n0.lo_quantity SIMPLE [(lineorder_ext_n0)lineorder_ext_n0.FieldSchema(name:lo_quantity, type:double, comment:null), ] -POSTHOOK: Lineage: lineorder_n0.lo_revenue SIMPLE [(lineorder_ext_n0)lineorder_ext_n0.FieldSchema(name:lo_revenue, type:double, comment:null), ] -POSTHOOK: Lineage: lineorder_n0.lo_shipmode SIMPLE [(lineorder_ext_n0)lineorder_ext_n0.FieldSchema(name:lo_shipmode, type:string, comment:null), ] -POSTHOOK: Lineage: lineorder_n0.lo_shippriority SIMPLE [(lineorder_ext_n0)lineorder_ext_n0.FieldSchema(name:lo_shippriority, type:string, comment:null), ] -POSTHOOK: Lineage: lineorder_n0.lo_suppkey SIMPLE [(lineorder_ext_n0)lineorder_ext_n0.FieldSchema(name:lo_suppkey, type:bigint, comment:null), ] -POSTHOOK: Lineage: lineorder_n0.lo_supplycost SIMPLE [(lineorder_ext_n0)lineorder_ext_n0.FieldSchema(name:lo_supplycost, type:double, comment:null), ] -POSTHOOK: Lineage: lineorder_n0.lo_tax SIMPLE [(lineorder_ext_n0)lineorder_ext_n0.FieldSchema(name:lo_tax, type:double, comment:null), ] -PREHOOK: query: CREATE MATERIALIZED VIEW `ssb_mv_druid_100` -STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' -TBLPROPERTIES ("druid.segment.granularity" = "MONTH") -AS -SELECT - `__time` as `__time` , - cast(c_city as string) c_city, - cast(c_nation as string) c_nation, - cast(c_region as string) c_region, - c_mktsegment as c_mktsegment, - cast(d_weeknuminyear as string) d_weeknuminyear, - cast(d_year as string) d_year, - cast(d_yearmonth as string) d_yearmonth, - cast(d_yearmonthnum as string) d_yearmonthnum, - cast(p_brand1 as string) p_brand1, - cast(p_category as string) p_category, - cast(p_mfgr as string) p_mfgr, - p_type, - s_name, - cast(s_city as string) s_city, - cast(s_nation as string) s_nation, - cast(s_region as string) s_region, - cast(`lo_ordpriority` as string) lo_ordpriority, - cast(`lo_shippriority` as string) lo_shippriority, - `d_sellingseason` - `lo_shipmode`, - lo_revenue, - lo_supplycost , - lo_discount , - `lo_quantity`, - `lo_extendedprice`, - `lo_ordtotalprice`, - lo_extendedprice * lo_discount discounted_price, - lo_revenue - lo_supplycost net_revenue -FROM - customer_n0, dates_n0, lineorder_n0, ssb_part_n0, supplier_n0 -where - lo_orderdate = d_datekey - and lo_partkey = p_partkey - and lo_suppkey = s_suppkey - and lo_custkey = c_custkey -PREHOOK: type: CREATE_MATERIALIZED_VIEW -PREHOOK: Input: default@customer_n0 -PREHOOK: Input: default@dates_n0 -PREHOOK: Input: default@lineorder_n0 -PREHOOK: Input: default@ssb_part_n0 -PREHOOK: Input: default@supplier_n0 -PREHOOK: Output: database:default -PREHOOK: Output: default@ssb_mv_druid_100 -POSTHOOK: query: CREATE MATERIALIZED VIEW `ssb_mv_druid_100` -STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' -TBLPROPERTIES ("druid.segment.granularity" = "MONTH") -AS -SELECT - `__time` as `__time` , - cast(c_city as string) c_city, - cast(c_nation as string) c_nation, - cast(c_region as string) c_region, - c_mktsegment as c_mktsegment, - cast(d_weeknuminyear as string) d_weeknuminyear, - cast(d_year as string) d_year, - cast(d_yearmonth as string) d_yearmonth, - cast(d_yearmonthnum as string) d_yearmonthnum, - cast(p_brand1 as string) p_brand1, - cast(p_category as string) p_category, - cast(p_mfgr as string) p_mfgr, - p_type, - s_name, - cast(s_city as string) s_city, - cast(s_nation as string) s_nation, - cast(s_region as string) s_region, - cast(`lo_ordpriority` as string) lo_ordpriority, - cast(`lo_shippriority` as string) lo_shippriority, - `d_sellingseason` - `lo_shipmode`, - lo_revenue, - lo_supplycost , - lo_discount , - `lo_quantity`, - `lo_extendedprice`, - `lo_ordtotalprice`, - lo_extendedprice * lo_discount discounted_price, - lo_revenue - lo_supplycost net_revenue -FROM - customer_n0, dates_n0, lineorder_n0, ssb_part_n0, supplier_n0 -where - lo_orderdate = d_datekey - and lo_partkey = p_partkey - and lo_suppkey = s_suppkey - and lo_custkey = c_custkey -POSTHOOK: type: CREATE_MATERIALIZED_VIEW -POSTHOOK: Input: default@customer_n0 -POSTHOOK: Input: default@dates_n0 -POSTHOOK: Input: default@lineorder_n0 -POSTHOOK: Input: default@ssb_part_n0 -POSTHOOK: Input: default@supplier_n0 -POSTHOOK: Output: database:default -POSTHOOK: Output: default@ssb_mv_druid_100 -PREHOOK: query: EXPLAIN CBO -SELECT MONTH(`__time`) AS `mn___time_ok`, -CAST((MONTH(`__time`) - 1) / 3 + 1 AS BIGINT) AS `qr___time_ok`, -SUM(1) AS `sum_number_of_records_ok`, -YEAR(`__time`) AS `yr___time_ok` -FROM `ssb_mv_druid_100` -GROUP BY MONTH(`__time`), -CAST((MONTH(`__time`) - 1) / 3 + 1 AS BIGINT), -YEAR(`__time`) -PREHOOK: type: QUERY -PREHOOK: Input: default@ssb_mv_druid_100 -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN CBO -SELECT MONTH(`__time`) AS `mn___time_ok`, -CAST((MONTH(`__time`) - 1) / 3 + 1 AS BIGINT) AS `qr___time_ok`, -SUM(1) AS `sum_number_of_records_ok`, -YEAR(`__time`) AS `yr___time_ok` -FROM `ssb_mv_druid_100` -GROUP BY MONTH(`__time`), -CAST((MONTH(`__time`) - 1) / 3 + 1 AS BIGINT), -YEAR(`__time`) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@ssb_mv_druid_100 -POSTHOOK: Output: hdfs://### HDFS PATH ### -CBO PLAN: -HiveProject(mn___time_ok=[$0], qr___time_ok=[$1], sum_number_of_records_ok=[$3], yr___time_ok=[$2]) - DruidQuery(table=[[default, ssb_mv_druid_100]], intervals=[[1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z]], projects=[[MONTH(FLAG(MONTH), $0), CAST(+(/(CAST(-(MONTH(FLAG(MONTH), $0), 1)):DOUBLE, 3), 1)):BIGINT, YEAR(FLAG(YEAR), $0), 1]], groups=[{0, 1, 2}], aggs=[[sum($3)]]) - -PREHOOK: query: SELECT MONTH(`__time`) AS `mn___time_ok`, -CAST((MONTH(`__time`) - 1) / 3 + 1 AS BIGINT) AS `qr___time_ok`, -SUM(1) AS `sum_number_of_records_ok`, -YEAR(`__time`) AS `yr___time_ok` -FROM `ssb_mv_druid_100` -GROUP BY MONTH(`__time`), -CAST((MONTH(`__time`) - 1) / 3 + 1 AS BIGINT), -YEAR(`__time`) -PREHOOK: type: QUERY -PREHOOK: Input: default@ssb_mv_druid_100 -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: SELECT MONTH(`__time`) AS `mn___time_ok`, -CAST((MONTH(`__time`) - 1) / 3 + 1 AS BIGINT) AS `qr___time_ok`, -SUM(1) AS `sum_number_of_records_ok`, -YEAR(`__time`) AS `yr___time_ok` -FROM `ssb_mv_druid_100` -GROUP BY MONTH(`__time`), -CAST((MONTH(`__time`) - 1) / 3 + 1 AS BIGINT), -YEAR(`__time`) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@ssb_mv_druid_100 -POSTHOOK: Output: hdfs://### HDFS PATH ### -PREHOOK: query: EXPLAIN CBO -SELECT MONTH(`dates_n0`.`__time`) AS `mn___time_ok`, -CAST((MONTH(`dates_n0`.`__time`) - 1) / 3 + 1 AS BIGINT) AS `qr___time_ok`, -SUM(1) AS `sum_number_of_records_ok`, -YEAR(`dates_n0`.`__time`) AS `yr___time_ok` -FROM `lineorder_n0` `lineorder_n0` -JOIN `dates_n0` `dates_n0` ON (`lineorder_n0`.`lo_orderdate` = `dates_n0`.`d_datekey`) -JOIN `customer_n0` `customer_n0` ON (`lineorder_n0`.`lo_custkey` = `customer_n0`.`c_custkey`) -JOIN `supplier_n0` `supplier_n0` ON (`lineorder_n0`.`lo_suppkey` = `supplier_n0`.`s_suppkey`) -JOIN `ssb_part_n0` `ssb_part_n0` ON (`lineorder_n0`.`lo_partkey` = `ssb_part_n0`.`p_partkey`) -GROUP BY MONTH(`dates_n0`.`__time`), -CAST((MONTH(`dates_n0`.`__time`) - 1) / 3 + 1 AS BIGINT), -YEAR(`dates_n0`.`__time`) -PREHOOK: type: QUERY -PREHOOK: Input: default@customer_n0 -PREHOOK: Input: default@dates_n0 -PREHOOK: Input: default@lineorder_n0 -PREHOOK: Input: default@ssb_mv_druid_100 -PREHOOK: Input: default@ssb_part_n0 -PREHOOK: Input: default@supplier_n0 -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN CBO -SELECT MONTH(`dates_n0`.`__time`) AS `mn___time_ok`, -CAST((MONTH(`dates_n0`.`__time`) - 1) / 3 + 1 AS BIGINT) AS `qr___time_ok`, -SUM(1) AS `sum_number_of_records_ok`, -YEAR(`dates_n0`.`__time`) AS `yr___time_ok` -FROM `lineorder_n0` `lineorder_n0` -JOIN `dates_n0` `dates_n0` ON (`lineorder_n0`.`lo_orderdate` = `dates_n0`.`d_datekey`) -JOIN `customer_n0` `customer_n0` ON (`lineorder_n0`.`lo_custkey` = `customer_n0`.`c_custkey`) -JOIN `supplier_n0` `supplier_n0` ON (`lineorder_n0`.`lo_suppkey` = `supplier_n0`.`s_suppkey`) -JOIN `ssb_part_n0` `ssb_part_n0` ON (`lineorder_n0`.`lo_partkey` = `ssb_part_n0`.`p_partkey`) -GROUP BY MONTH(`dates_n0`.`__time`), -CAST((MONTH(`dates_n0`.`__time`) - 1) / 3 + 1 AS BIGINT), -YEAR(`dates_n0`.`__time`) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@customer_n0 -POSTHOOK: Input: default@dates_n0 -POSTHOOK: Input: default@lineorder_n0 -POSTHOOK: Input: default@ssb_mv_druid_100 -POSTHOOK: Input: default@ssb_part_n0 -POSTHOOK: Input: default@supplier_n0 -POSTHOOK: Output: hdfs://### HDFS PATH ### -CBO PLAN: -HiveProject(mn___time_ok=[$0], qr___time_ok=[$1], sum_number_of_records_ok=[$3], yr___time_ok=[$2]) - DruidQuery(table=[[default, ssb_mv_druid_100]], intervals=[[1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z]], projects=[[MONTH(FLAG(MONTH), $0), CAST(+(/(CAST(-(MONTH(FLAG(MONTH), $0), 1)):DOUBLE, 3), 1)):BIGINT, YEAR(FLAG(YEAR), $0), 1]], groups=[{0, 1, 2}], aggs=[[sum($3)]]) - -PREHOOK: query: SELECT MONTH(`dates_n0`.`__time`) AS `mn___time_ok`, -CAST((MONTH(`dates_n0`.`__time`) - 1) / 3 + 1 AS BIGINT) AS `qr___time_ok`, -SUM(1) AS `sum_number_of_records_ok`, -YEAR(`dates_n0`.`__time`) AS `yr___time_ok` -FROM `lineorder_n0` `lineorder_n0` -JOIN `dates_n0` `dates_n0` ON (`lineorder_n0`.`lo_orderdate` = `dates_n0`.`d_datekey`) -JOIN `customer_n0` `customer_n0` ON (`lineorder_n0`.`lo_custkey` = `customer_n0`.`c_custkey`) -JOIN `supplier_n0` `supplier_n0` ON (`lineorder_n0`.`lo_suppkey` = `supplier_n0`.`s_suppkey`) -JOIN `ssb_part_n0` `ssb_part_n0` ON (`lineorder_n0`.`lo_partkey` = `ssb_part_n0`.`p_partkey`) -GROUP BY MONTH(`dates_n0`.`__time`), -CAST((MONTH(`dates_n0`.`__time`) - 1) / 3 + 1 AS BIGINT), -YEAR(`dates_n0`.`__time`) -PREHOOK: type: QUERY -PREHOOK: Input: default@customer_n0 -PREHOOK: Input: default@dates_n0 -PREHOOK: Input: default@lineorder_n0 -PREHOOK: Input: default@ssb_mv_druid_100 -PREHOOK: Input: default@ssb_part_n0 -PREHOOK: Input: default@supplier_n0 -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: SELECT MONTH(`dates_n0`.`__time`) AS `mn___time_ok`, -CAST((MONTH(`dates_n0`.`__time`) - 1) / 3 + 1 AS BIGINT) AS `qr___time_ok`, -SUM(1) AS `sum_number_of_records_ok`, -YEAR(`dates_n0`.`__time`) AS `yr___time_ok` -FROM `lineorder_n0` `lineorder_n0` -JOIN `dates_n0` `dates_n0` ON (`lineorder_n0`.`lo_orderdate` = `dates_n0`.`d_datekey`) -JOIN `customer_n0` `customer_n0` ON (`lineorder_n0`.`lo_custkey` = `customer_n0`.`c_custkey`) -JOIN `supplier_n0` `supplier_n0` ON (`lineorder_n0`.`lo_suppkey` = `supplier_n0`.`s_suppkey`) -JOIN `ssb_part_n0` `ssb_part_n0` ON (`lineorder_n0`.`lo_partkey` = `ssb_part_n0`.`p_partkey`) -GROUP BY MONTH(`dates_n0`.`__time`), -CAST((MONTH(`dates_n0`.`__time`) - 1) / 3 + 1 AS BIGINT), -YEAR(`dates_n0`.`__time`) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@customer_n0 -POSTHOOK: Input: default@dates_n0 -POSTHOOK: Input: default@lineorder_n0 -POSTHOOK: Input: default@ssb_mv_druid_100 -POSTHOOK: Input: default@ssb_part_n0 -POSTHOOK: Input: default@supplier_n0 -POSTHOOK: Output: hdfs://### HDFS PATH ### -PREHOOK: query: DROP MATERIALIZED VIEW `ssb_mv_druid_100` -PREHOOK: type: DROP_MATERIALIZED_VIEW -PREHOOK: Input: default@ssb_mv_druid_100 -PREHOOK: Output: default@ssb_mv_druid_100 -POSTHOOK: query: DROP MATERIALIZED VIEW `ssb_mv_druid_100` -POSTHOOK: type: DROP_MATERIALIZED_VIEW -POSTHOOK: Input: default@ssb_mv_druid_100 -POSTHOOK: Output: default@ssb_mv_druid_100 diff --git a/ql/src/test/results/clientpositive/druid/druid_timeseries.q.out b/ql/src/test/results/clientpositive/druid/druid_timeseries.q.out deleted file mode 100644 index afbb2a12f63e..000000000000 --- a/ql/src/test/results/clientpositive/druid/druid_timeseries.q.out +++ /dev/null @@ -1,640 +0,0 @@ -PREHOOK: query: CREATE EXTERNAL TABLE druid_table_1_n3 -STORED BY 'org.apache.hadoop.hive.druid.QTestDruidStorageHandler' -TBLPROPERTIES ("druid.datasource" = "wikipedia") -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@druid_table_1_n3 -POSTHOOK: query: CREATE EXTERNAL TABLE druid_table_1_n3 -STORED BY 'org.apache.hadoop.hive.druid.QTestDruidStorageHandler' -TBLPROPERTIES ("druid.datasource" = "wikipedia") -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@druid_table_1_n3 -PREHOOK: query: EXPLAIN SELECT count(`__time`) from druid_table_1_n3 where `__time` >= '2010-01-01 00:00:00 UTC' AND `__time` <= '2012-03-01 00:00:00 UTC' OR added <= 0 -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_1_n3 -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN SELECT count(`__time`) from druid_table_1_n3 where `__time` >= '2010-01-01 00:00:00 UTC' AND `__time` <= '2012-03-01 00:00:00 UTC' OR added <= 0 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_1_n3 -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_table_1_n3 - properties: - druid.fieldNames $f0 - druid.fieldTypes bigint - druid.query.json {"queryType":"timeseries","dataSource":"wikipedia","descending":false,"granularity":"all","filter":{"type":"or","fields":[{"type":"bound","dimension":"added","upper":"0","upperStrict":false,"ordering":"numeric"},{"type":"bound","dimension":"__time","lower":"2010-01-01T00:00:00.000Z","lowerStrict":false,"upper":"2012-03-01T00:00:00.000Z","upperStrict":false,"ordering":"lexicographic","extractionFn":{"type":"timeFormat","format":"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'","timeZone":"UTC"}}]},"aggregations":[{"type":"count","name":"$f0"}],"intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"],"context":{"skipEmptyBuckets":false}} - druid.query.type timeseries - Select Operator - expressions: $f0 (type: bigint) - outputColumnNames: _col0 - ListSink - -PREHOOK: query: EXPLAIN SELECT count(`__time`) from druid_table_1_n3 where `__time` <= '2010-01-01 00:00:00 UTC' -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_1_n3 -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN SELECT count(`__time`) from druid_table_1_n3 where `__time` <= '2010-01-01 00:00:00 UTC' -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_1_n3 -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_table_1_n3 - properties: - druid.fieldNames $f0 - druid.fieldTypes bigint - druid.query.json {"queryType":"timeseries","dataSource":"wikipedia","descending":false,"granularity":"all","aggregations":[{"type":"count","name":"$f0"}],"intervals":["1900-01-01T00:00:00.000Z/2010-01-01T00:00:00.001Z"],"context":{"skipEmptyBuckets":false}} - druid.query.type timeseries - Select Operator - expressions: $f0 (type: bigint) - outputColumnNames: _col0 - ListSink - -PREHOOK: query: EXPLAIN -SELECT max(added), sum(variation) -FROM druid_table_1_n3 -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_1_n3 -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN -SELECT max(added), sum(variation) -FROM druid_table_1_n3 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_1_n3 -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_table_1_n3 - properties: - druid.fieldNames $f0,$f1 - druid.fieldTypes float,double - druid.query.json {"queryType":"timeseries","dataSource":"wikipedia","descending":false,"granularity":"all","aggregations":[{"type":"doubleMax","name":"$f0","fieldName":"added"},{"type":"doubleSum","name":"$f1","fieldName":"variation"}],"intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"],"context":{"skipEmptyBuckets":false}} - druid.query.type timeseries - Select Operator - expressions: $f0 (type: float), $f1 (type: double) - outputColumnNames: _col0, _col1 - ListSink - -PREHOOK: query: EXPLAIN -SELECT `__time`, max(added), sum(variation) -FROM druid_table_1_n3 -GROUP BY `__time` -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_1_n3 -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN -SELECT `__time`, max(added), sum(variation) -FROM druid_table_1_n3 -GROUP BY `__time` -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_1_n3 -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_table_1_n3 - properties: - druid.fieldNames extract,$f1,$f2 - druid.fieldTypes timestamp with local time zone,float,double - druid.query.json {"queryType":"groupBy","dataSource":"wikipedia","granularity":"all","dimensions":[{"type":"extraction","dimension":"__time","outputName":"extract","extractionFn":{"type":"timeFormat","format":"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'","timeZone":"UTC"}}],"limitSpec":{"type":"default"},"aggregations":[{"type":"doubleMax","name":"$f1","fieldName":"added"},{"type":"doubleSum","name":"$f2","fieldName":"variation"}],"intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"]} - druid.query.type groupBy - Select Operator - expressions: extract (type: timestamp with local time zone), $f1 (type: float), $f2 (type: double) - outputColumnNames: _col0, _col1, _col2 - ListSink - -PREHOOK: query: EXPLAIN -SELECT floor_year(`__time`), max(added), sum(variation) -FROM druid_table_1_n3 -GROUP BY floor_year(`__time`) -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_1_n3 -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN -SELECT floor_year(`__time`), max(added), sum(variation) -FROM druid_table_1_n3 -GROUP BY floor_year(`__time`) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_1_n3 -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_table_1_n3 - properties: - druid.fieldNames timestamp,$f1,$f2 - druid.fieldTypes timestamp with local time zone,float,double - druid.query.json {"queryType":"timeseries","dataSource":"wikipedia","descending":false,"granularity":{"type":"period","period":"P1Y","timeZone":"US/Pacific"},"aggregations":[{"type":"doubleMax","name":"$f1","fieldName":"added"},{"type":"doubleSum","name":"$f2","fieldName":"variation"}],"intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"],"context":{"skipEmptyBuckets":true}} - druid.query.type timeseries - Select Operator - expressions: timestamp (type: timestamp with local time zone), $f1 (type: float), $f2 (type: double) - outputColumnNames: _col0, _col1, _col2 - ListSink - -PREHOOK: query: EXPLAIN -SELECT floor_quarter(`__time`), max(added), sum(variation) -FROM druid_table_1_n3 -GROUP BY floor_quarter(`__time`) -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_1_n3 -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN -SELECT floor_quarter(`__time`), max(added), sum(variation) -FROM druid_table_1_n3 -GROUP BY floor_quarter(`__time`) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_1_n3 -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_table_1_n3 - properties: - druid.fieldNames timestamp,$f1,$f2 - druid.fieldTypes timestamp with local time zone,float,double - druid.query.json {"queryType":"timeseries","dataSource":"wikipedia","descending":false,"granularity":{"type":"period","period":"P3M","timeZone":"US/Pacific"},"aggregations":[{"type":"doubleMax","name":"$f1","fieldName":"added"},{"type":"doubleSum","name":"$f2","fieldName":"variation"}],"intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"],"context":{"skipEmptyBuckets":true}} - druid.query.type timeseries - Select Operator - expressions: timestamp (type: timestamp with local time zone), $f1 (type: float), $f2 (type: double) - outputColumnNames: _col0, _col1, _col2 - ListSink - -PREHOOK: query: EXPLAIN -SELECT floor_month(`__time`), max(added), sum(variation) -FROM druid_table_1_n3 -GROUP BY floor_month(`__time`) -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_1_n3 -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN -SELECT floor_month(`__time`), max(added), sum(variation) -FROM druid_table_1_n3 -GROUP BY floor_month(`__time`) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_1_n3 -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_table_1_n3 - properties: - druid.fieldNames timestamp,$f1,$f2 - druid.fieldTypes timestamp with local time zone,float,double - druid.query.json {"queryType":"timeseries","dataSource":"wikipedia","descending":false,"granularity":{"type":"period","period":"P1M","timeZone":"US/Pacific"},"aggregations":[{"type":"doubleMax","name":"$f1","fieldName":"added"},{"type":"doubleSum","name":"$f2","fieldName":"variation"}],"intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"],"context":{"skipEmptyBuckets":true}} - druid.query.type timeseries - Select Operator - expressions: timestamp (type: timestamp with local time zone), $f1 (type: float), $f2 (type: double) - outputColumnNames: _col0, _col1, _col2 - ListSink - -PREHOOK: query: EXPLAIN -SELECT floor_week(`__time`), max(added), sum(variation) -FROM druid_table_1_n3 -GROUP BY floor_week(`__time`) -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_1_n3 -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN -SELECT floor_week(`__time`), max(added), sum(variation) -FROM druid_table_1_n3 -GROUP BY floor_week(`__time`) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_1_n3 -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_table_1_n3 - properties: - druid.fieldNames timestamp,$f1,$f2 - druid.fieldTypes timestamp with local time zone,float,double - druid.query.json {"queryType":"timeseries","dataSource":"wikipedia","descending":false,"granularity":{"type":"period","period":"P1W","timeZone":"US/Pacific"},"aggregations":[{"type":"doubleMax","name":"$f1","fieldName":"added"},{"type":"doubleSum","name":"$f2","fieldName":"variation"}],"intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"],"context":{"skipEmptyBuckets":true}} - druid.query.type timeseries - Select Operator - expressions: timestamp (type: timestamp with local time zone), $f1 (type: float), $f2 (type: double) - outputColumnNames: _col0, _col1, _col2 - ListSink - -PREHOOK: query: EXPLAIN -SELECT floor_day(`__time`), max(added), sum(variation) -FROM druid_table_1_n3 -GROUP BY floor_day(`__time`) -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_1_n3 -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN -SELECT floor_day(`__time`), max(added), sum(variation) -FROM druid_table_1_n3 -GROUP BY floor_day(`__time`) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_1_n3 -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_table_1_n3 - properties: - druid.fieldNames timestamp,$f1,$f2 - druid.fieldTypes timestamp with local time zone,float,double - druid.query.json {"queryType":"timeseries","dataSource":"wikipedia","descending":false,"granularity":{"type":"period","period":"P1D","timeZone":"US/Pacific"},"aggregations":[{"type":"doubleMax","name":"$f1","fieldName":"added"},{"type":"doubleSum","name":"$f2","fieldName":"variation"}],"intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"],"context":{"skipEmptyBuckets":true}} - druid.query.type timeseries - Select Operator - expressions: timestamp (type: timestamp with local time zone), $f1 (type: float), $f2 (type: double) - outputColumnNames: _col0, _col1, _col2 - ListSink - -PREHOOK: query: EXPLAIN -SELECT floor_hour(`__time`), max(added), sum(variation) -FROM druid_table_1_n3 -GROUP BY floor_hour(`__time`) -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_1_n3 -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN -SELECT floor_hour(`__time`), max(added), sum(variation) -FROM druid_table_1_n3 -GROUP BY floor_hour(`__time`) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_1_n3 -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_table_1_n3 - properties: - druid.fieldNames timestamp,$f1,$f2 - druid.fieldTypes timestamp with local time zone,float,double - druid.query.json {"queryType":"timeseries","dataSource":"wikipedia","descending":false,"granularity":{"type":"period","period":"PT1H","timeZone":"US/Pacific"},"aggregations":[{"type":"doubleMax","name":"$f1","fieldName":"added"},{"type":"doubleSum","name":"$f2","fieldName":"variation"}],"intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"],"context":{"skipEmptyBuckets":true}} - druid.query.type timeseries - Select Operator - expressions: timestamp (type: timestamp with local time zone), $f1 (type: float), $f2 (type: double) - outputColumnNames: _col0, _col1, _col2 - ListSink - -PREHOOK: query: EXPLAIN -SELECT floor_minute(`__time`), max(added), sum(variation) -FROM druid_table_1_n3 -GROUP BY floor_minute(`__time`) -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_1_n3 -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN -SELECT floor_minute(`__time`), max(added), sum(variation) -FROM druid_table_1_n3 -GROUP BY floor_minute(`__time`) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_1_n3 -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_table_1_n3 - properties: - druid.fieldNames timestamp,$f1,$f2 - druid.fieldTypes timestamp with local time zone,float,double - druid.query.json {"queryType":"timeseries","dataSource":"wikipedia","descending":false,"granularity":{"type":"period","period":"PT1M","timeZone":"US/Pacific"},"aggregations":[{"type":"doubleMax","name":"$f1","fieldName":"added"},{"type":"doubleSum","name":"$f2","fieldName":"variation"}],"intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"],"context":{"skipEmptyBuckets":true}} - druid.query.type timeseries - Select Operator - expressions: timestamp (type: timestamp with local time zone), $f1 (type: float), $f2 (type: double) - outputColumnNames: _col0, _col1, _col2 - ListSink - -PREHOOK: query: EXPLAIN -SELECT floor_second(`__time`), max(added), sum(variation) -FROM druid_table_1_n3 -GROUP BY floor_second(`__time`) -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_1_n3 -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN -SELECT floor_second(`__time`), max(added), sum(variation) -FROM druid_table_1_n3 -GROUP BY floor_second(`__time`) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_1_n3 -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_table_1_n3 - properties: - druid.fieldNames timestamp,$f1,$f2 - druid.fieldTypes timestamp with local time zone,float,double - druid.query.json {"queryType":"timeseries","dataSource":"wikipedia","descending":false,"granularity":{"type":"period","period":"PT1S","timeZone":"US/Pacific"},"aggregations":[{"type":"doubleMax","name":"$f1","fieldName":"added"},{"type":"doubleSum","name":"$f2","fieldName":"variation"}],"intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"],"context":{"skipEmptyBuckets":true}} - druid.query.type timeseries - Select Operator - expressions: timestamp (type: timestamp with local time zone), $f1 (type: float), $f2 (type: double) - outputColumnNames: _col0, _col1, _col2 - ListSink - -PREHOOK: query: EXPLAIN -SELECT floor_hour(`__time`), max(added), sum(variation) -FROM druid_table_1_n3 -WHERE robot='1' -GROUP BY floor_hour(`__time`) -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_1_n3 -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN -SELECT floor_hour(`__time`), max(added), sum(variation) -FROM druid_table_1_n3 -WHERE robot='1' -GROUP BY floor_hour(`__time`) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_1_n3 -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_table_1_n3 - properties: - druid.fieldNames timestamp,$f1,$f2 - druid.fieldTypes timestamp with local time zone,float,double - druid.query.json {"queryType":"timeseries","dataSource":"wikipedia","descending":false,"granularity":{"type":"period","period":"PT1H","timeZone":"US/Pacific"},"filter":{"type":"selector","dimension":"robot","value":"1"},"aggregations":[{"type":"doubleMax","name":"$f1","fieldName":"added"},{"type":"doubleSum","name":"$f2","fieldName":"variation"}],"intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"],"context":{"skipEmptyBuckets":true}} - druid.query.type timeseries - Select Operator - expressions: timestamp (type: timestamp with local time zone), $f1 (type: float), $f2 (type: double) - outputColumnNames: _col0, _col1, _col2 - ListSink - -PREHOOK: query: EXPLAIN -SELECT floor_hour(`__time`), max(added), sum(variation) -FROM druid_table_1_n3 -WHERE floor_hour(`__time`) - BETWEEN CAST('2010-01-01 00:00:00' AS TIMESTAMP WITH LOCAL TIME ZONE) - AND CAST('2014-01-01 00:00:00' AS TIMESTAMP WITH LOCAL TIME ZONE) -GROUP BY floor_hour(`__time`) -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_1_n3 -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN -SELECT floor_hour(`__time`), max(added), sum(variation) -FROM druid_table_1_n3 -WHERE floor_hour(`__time`) - BETWEEN CAST('2010-01-01 00:00:00' AS TIMESTAMP WITH LOCAL TIME ZONE) - AND CAST('2014-01-01 00:00:00' AS TIMESTAMP WITH LOCAL TIME ZONE) -GROUP BY floor_hour(`__time`) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_1_n3 -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_table_1_n3 - properties: - druid.fieldNames timestamp,$f1,$f2 - druid.fieldTypes timestamp with local time zone,float,double - druid.query.json {"queryType":"timeseries","dataSource":"wikipedia","descending":false,"granularity":{"type":"period","period":"PT1H","timeZone":"US/Pacific"},"aggregations":[{"type":"doubleMax","name":"$f1","fieldName":"added"},{"type":"doubleSum","name":"$f2","fieldName":"variation"}],"intervals":["2010-01-01T08:00:00.000Z/2014-01-01T08:00:00.001Z"],"context":{"skipEmptyBuckets":true}} - druid.query.type timeseries - Select Operator - expressions: timestamp (type: timestamp with local time zone), $f1 (type: float), $f2 (type: double) - outputColumnNames: _col0, _col1, _col2 - ListSink - -PREHOOK: query: EXPLAIN -SELECT subq.h, subq.m, subq.s -FROM -( - SELECT floor_hour(`__time`) as h, max(added) as m, sum(variation) as s - FROM druid_table_1_n3 - GROUP BY floor_hour(`__time`) -) subq -WHERE subq.h BETWEEN CAST('2010-01-01 00:00:00' AS TIMESTAMP WITH LOCAL TIME ZONE) - AND CAST('2014-01-01 00:00:00' AS TIMESTAMP WITH LOCAL TIME ZONE) -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_1_n3 -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN -SELECT subq.h, subq.m, subq.s -FROM -( - SELECT floor_hour(`__time`) as h, max(added) as m, sum(variation) as s - FROM druid_table_1_n3 - GROUP BY floor_hour(`__time`) -) subq -WHERE subq.h BETWEEN CAST('2010-01-01 00:00:00' AS TIMESTAMP WITH LOCAL TIME ZONE) - AND CAST('2014-01-01 00:00:00' AS TIMESTAMP WITH LOCAL TIME ZONE) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_1_n3 -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_table_1_n3 - properties: - druid.fieldNames timestamp,$f1,$f2 - druid.fieldTypes timestamp with local time zone,float,double - druid.query.json {"queryType":"timeseries","dataSource":"wikipedia","descending":false,"granularity":{"type":"period","period":"PT1H","timeZone":"US/Pacific"},"aggregations":[{"type":"doubleMax","name":"$f1","fieldName":"added"},{"type":"doubleSum","name":"$f2","fieldName":"variation"}],"intervals":["2010-01-01T08:00:00.000Z/2014-01-01T08:00:00.001Z"],"context":{"skipEmptyBuckets":true}} - druid.query.type timeseries - Select Operator - expressions: timestamp (type: timestamp with local time zone), $f1 (type: float), $f2 (type: double) - outputColumnNames: _col0, _col1, _col2 - ListSink - -PREHOOK: query: EXPLAIN SELECT count(`__time`) from druid_table_1_n3 -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_1_n3 -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN SELECT count(`__time`) from druid_table_1_n3 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_1_n3 -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_table_1_n3 - properties: - druid.fieldNames $f0 - druid.fieldTypes bigint - druid.query.json {"queryType":"timeseries","dataSource":"wikipedia","descending":false,"granularity":"all","aggregations":[{"type":"count","name":"$f0"}],"intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"],"context":{"skipEmptyBuckets":false}} - druid.query.type timeseries - Select Operator - expressions: $f0 (type: bigint) - outputColumnNames: _col0 - ListSink - -PREHOOK: query: EXPLAIN SELECT count(`__time`) from druid_table_1_n3 where `__time` <= '2010-01-01 00:00:00 UTC' -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_1_n3 -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN SELECT count(`__time`) from druid_table_1_n3 where `__time` <= '2010-01-01 00:00:00 UTC' -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_1_n3 -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_table_1_n3 - properties: - druid.fieldNames $f0 - druid.fieldTypes bigint - druid.query.json {"queryType":"timeseries","dataSource":"wikipedia","descending":false,"granularity":"all","aggregations":[{"type":"count","name":"$f0"}],"intervals":["1900-01-01T00:00:00.000Z/2010-01-01T00:00:00.001Z"],"context":{"skipEmptyBuckets":false}} - druid.query.type timeseries - Select Operator - expressions: $f0 (type: bigint) - outputColumnNames: _col0 - ListSink - -PREHOOK: query: EXPLAIN SELECT count(`__time`) from druid_table_1_n3 where `__time` >= '2010-01-01 00:00:00' -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_1_n3 -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN SELECT count(`__time`) from druid_table_1_n3 where `__time` >= '2010-01-01 00:00:00' -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_1_n3 -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_table_1_n3 - properties: - druid.fieldNames $f0 - druid.fieldTypes bigint - druid.query.json {"queryType":"timeseries","dataSource":"wikipedia","descending":false,"granularity":"all","aggregations":[{"type":"count","name":"$f0"}],"intervals":["2010-01-01T08:00:00.000Z/3000-01-01T00:00:00.000Z"],"context":{"skipEmptyBuckets":false}} - druid.query.type timeseries - Select Operator - expressions: $f0 (type: bigint) - outputColumnNames: _col0 - ListSink - -PREHOOK: query: EXPLAIN SELECT count(`__time`) from druid_table_1_n3 where `__time` <= '2010-01-01 00:00:00' OR `__time` <= '2012-03-01 00:00:00' -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_1_n3 -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN SELECT count(`__time`) from druid_table_1_n3 where `__time` <= '2010-01-01 00:00:00' OR `__time` <= '2012-03-01 00:00:00' -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_1_n3 -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_table_1_n3 - properties: - druid.fieldNames $f0 - druid.fieldTypes bigint - druid.query.json {"queryType":"timeseries","dataSource":"wikipedia","descending":false,"granularity":"all","aggregations":[{"type":"count","name":"$f0"}],"intervals":["1900-01-01T00:00:00.000Z/2012-03-01T08:00:00.001Z"],"context":{"skipEmptyBuckets":false}} - druid.query.type timeseries - Select Operator - expressions: $f0 (type: bigint) - outputColumnNames: _col0 - ListSink - diff --git a/ql/src/test/results/clientpositive/druid/druid_timestamptz2.q.out b/ql/src/test/results/clientpositive/druid/druid_timestamptz2.q.out deleted file mode 100644 index d9690319ae43..000000000000 --- a/ql/src/test/results/clientpositive/druid/druid_timestamptz2.q.out +++ /dev/null @@ -1,209 +0,0 @@ -PREHOOK: query: CREATE database druid_test_dst -PREHOOK: type: CREATEDATABASE -PREHOOK: Output: database:druid_test_dst -POSTHOOK: query: CREATE database druid_test_dst -POSTHOOK: type: CREATEDATABASE -POSTHOOK: Output: database:druid_test_dst -PREHOOK: query: use druid_test_dst -PREHOOK: type: SWITCHDATABASE -PREHOOK: Input: database:druid_test_dst -POSTHOOK: query: use druid_test_dst -POSTHOOK: type: SWITCHDATABASE -POSTHOOK: Input: database:druid_test_dst -PREHOOK: query: create external table test_base_table(`timecolumn` timestamp, `interval_marker` string, `num_l` double) -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:druid_test_dst -PREHOOK: Output: druid_test_dst@test_base_table -POSTHOOK: query: create external table test_base_table(`timecolumn` timestamp, `interval_marker` string, `num_l` double) -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:druid_test_dst -POSTHOOK: Output: druid_test_dst@test_base_table -PREHOOK: query: insert into test_base_table values -('2015-03-08 00:00:00', 'i1-start', 4), -('2015-03-08 23:59:59', 'i1-end', 1), -('2015-03-09 00:00:00', 'i2-start', 4), -('2015-03-09 23:59:59', 'i2-end', 1), -('2015-03-10 00:00:00', 'i3-start', 2), -('2015-03-10 23:59:59', 'i3-end', 2) -PREHOOK: type: QUERY -PREHOOK: Input: _dummy_database@_dummy_table -PREHOOK: Output: druid_test_dst@test_base_table -POSTHOOK: query: insert into test_base_table values -('2015-03-08 00:00:00', 'i1-start', 4), -('2015-03-08 23:59:59', 'i1-end', 1), -('2015-03-09 00:00:00', 'i2-start', 4), -('2015-03-09 23:59:59', 'i2-end', 1), -('2015-03-10 00:00:00', 'i3-start', 2), -('2015-03-10 23:59:59', 'i3-end', 2) -POSTHOOK: type: QUERY -POSTHOOK: Input: _dummy_database@_dummy_table -POSTHOOK: Output: druid_test_dst@test_base_table -POSTHOOK: Lineage: test_base_table.interval_marker SCRIPT [] -POSTHOOK: Lineage: test_base_table.num_l SCRIPT [] -POSTHOOK: Lineage: test_base_table.timecolumn SCRIPT [] -PREHOOK: query: CREATE EXTERNAL TABLE druid_test_table_1 -STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' -TBLPROPERTIES ("druid.segment.granularity" = "DAY") -AS -select cast(`timecolumn` as timestamp with local time zone) as `__time`, `interval_marker`, `num_l` -FROM druid_test_dst.test_base_table -PREHOOK: type: CREATETABLE_AS_SELECT -PREHOOK: Input: druid_test_dst@test_base_table -PREHOOK: Output: database:druid_test_dst -PREHOOK: Output: druid_test_dst@druid_test_table_1 -POSTHOOK: query: CREATE EXTERNAL TABLE druid_test_table_1 -STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' -TBLPROPERTIES ("druid.segment.granularity" = "DAY") -AS -select cast(`timecolumn` as timestamp with local time zone) as `__time`, `interval_marker`, `num_l` -FROM druid_test_dst.test_base_table -POSTHOOK: type: CREATETABLE_AS_SELECT -POSTHOOK: Input: druid_test_dst@test_base_table -POSTHOOK: Output: database:druid_test_dst -POSTHOOK: Output: druid_test_dst@druid_test_table_1 -POSTHOOK: Lineage: druid_test_table_1.__time EXPRESSION [(test_base_table)test_base_table.FieldSchema(name:timecolumn, type:timestamp, comment:null), ] -POSTHOOK: Lineage: druid_test_table_1.interval_marker SIMPLE [(test_base_table)test_base_table.FieldSchema(name:interval_marker, type:string, comment:null), ] -POSTHOOK: Lineage: druid_test_table_1.num_l SIMPLE [(test_base_table)test_base_table.FieldSchema(name:num_l, type:double, comment:null), ] -PREHOOK: query: select * FROM druid_test_table_1 -PREHOOK: type: QUERY -PREHOOK: Input: druid_test_dst@druid_test_table_1 -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: select * FROM druid_test_table_1 -POSTHOOK: type: QUERY -POSTHOOK: Input: druid_test_dst@druid_test_table_1 -POSTHOOK: Output: hdfs://### HDFS PATH ### -2015-03-08 00:00:00.0 US/Pacific i1-start 4.0 -2015-03-08 23:59:59.0 US/Pacific i1-end 1.0 -2015-03-09 00:00:00.0 US/Pacific i2-start 4.0 -2015-03-09 23:59:59.0 US/Pacific i2-end 1.0 -2015-03-10 00:00:00.0 US/Pacific i3-start 2.0 -2015-03-10 23:59:59.0 US/Pacific i3-end 2.0 -PREHOOK: query: CREATE EXTERNAL TABLE druid_test_table_2 (`__time` timestamp with local time zone, `interval_marker` string, `num_l` double) -STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' -TBLPROPERTIES ("druid.segment.granularity" = "DAY") -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:druid_test_dst -PREHOOK: Output: druid_test_dst@druid_test_table_2 -POSTHOOK: query: CREATE EXTERNAL TABLE druid_test_table_2 (`__time` timestamp with local time zone, `interval_marker` string, `num_l` double) -STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' -TBLPROPERTIES ("druid.segment.granularity" = "DAY") -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:druid_test_dst -POSTHOOK: Output: druid_test_dst@druid_test_table_2 -PREHOOK: query: insert into druid_test_table_2 values -(cast('2015-03-08 00:00:00' as timestamp with local time zone), 'i1-start', 4), -(cast('2015-03-08 23:59:59' as timestamp with local time zone), 'i1-end', 1), -(cast('2015-03-09 00:00:00' as timestamp with local time zone), 'i2-start', 4), -(cast('2015-03-09 23:59:59' as timestamp with local time zone), 'i2-end', 1), -(cast('2015-03-10 00:00:00' as timestamp with local time zone), 'i3-start', 2), -(cast('2015-03-10 23:59:59' as timestamp with local time zone), 'i3-end', 2) -PREHOOK: type: QUERY -PREHOOK: Input: _dummy_database@_dummy_table -PREHOOK: Output: druid_test_dst@druid_test_table_2 -POSTHOOK: query: insert into druid_test_table_2 values -(cast('2015-03-08 00:00:00' as timestamp with local time zone), 'i1-start', 4), -(cast('2015-03-08 23:59:59' as timestamp with local time zone), 'i1-end', 1), -(cast('2015-03-09 00:00:00' as timestamp with local time zone), 'i2-start', 4), -(cast('2015-03-09 23:59:59' as timestamp with local time zone), 'i2-end', 1), -(cast('2015-03-10 00:00:00' as timestamp with local time zone), 'i3-start', 2), -(cast('2015-03-10 23:59:59' as timestamp with local time zone), 'i3-end', 2) -POSTHOOK: type: QUERY -POSTHOOK: Input: _dummy_database@_dummy_table -POSTHOOK: Output: druid_test_dst@druid_test_table_2 -PREHOOK: query: select * FROM druid_test_table_2 -PREHOOK: type: QUERY -PREHOOK: Input: druid_test_dst@druid_test_table_2 -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: select * FROM druid_test_table_2 -POSTHOOK: type: QUERY -POSTHOOK: Input: druid_test_dst@druid_test_table_2 -POSTHOOK: Output: hdfs://### HDFS PATH ### -2015-03-08 00:00:00.0 US/Pacific i1-start 4.0 -2015-03-08 23:59:59.0 US/Pacific i1-end 1.0 -2015-03-09 00:00:00.0 US/Pacific i2-start 4.0 -2015-03-09 23:59:59.0 US/Pacific i2-end 1.0 -2015-03-10 00:00:00.0 US/Pacific i3-start 2.0 -2015-03-10 23:59:59.0 US/Pacific i3-end 2.0 -PREHOOK: query: CREATE EXTERNAL TABLE druid_test_table_utc -STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' -TBLPROPERTIES ("druid.segment.granularity" = "DAY") -AS -select cast(`timecolumn` as timestamp with local time zone) as `__time`, `interval_marker`, `num_l` -FROM druid_test_dst.test_base_table -PREHOOK: type: CREATETABLE_AS_SELECT -PREHOOK: Input: druid_test_dst@test_base_table -PREHOOK: Output: database:druid_test_dst -PREHOOK: Output: druid_test_dst@druid_test_table_utc -POSTHOOK: query: CREATE EXTERNAL TABLE druid_test_table_utc -STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' -TBLPROPERTIES ("druid.segment.granularity" = "DAY") -AS -select cast(`timecolumn` as timestamp with local time zone) as `__time`, `interval_marker`, `num_l` -FROM druid_test_dst.test_base_table -POSTHOOK: type: CREATETABLE_AS_SELECT -POSTHOOK: Input: druid_test_dst@test_base_table -POSTHOOK: Output: database:druid_test_dst -POSTHOOK: Output: druid_test_dst@druid_test_table_utc -POSTHOOK: Lineage: druid_test_table_utc.__time EXPRESSION [(test_base_table)test_base_table.FieldSchema(name:timecolumn, type:timestamp, comment:null), ] -POSTHOOK: Lineage: druid_test_table_utc.interval_marker SIMPLE [(test_base_table)test_base_table.FieldSchema(name:interval_marker, type:string, comment:null), ] -POSTHOOK: Lineage: druid_test_table_utc.num_l SIMPLE [(test_base_table)test_base_table.FieldSchema(name:num_l, type:double, comment:null), ] -PREHOOK: query: select * FROM druid_test_table_utc -PREHOOK: type: QUERY -PREHOOK: Input: druid_test_dst@druid_test_table_utc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: select * FROM druid_test_table_utc -POSTHOOK: type: QUERY -POSTHOOK: Input: druid_test_dst@druid_test_table_utc -POSTHOOK: Output: hdfs://### HDFS PATH ### -2015-03-08 00:00:00.0 UTC i1-start 4.0 -2015-03-08 23:59:59.0 UTC i1-end 1.0 -2015-03-09 00:00:00.0 UTC i2-start 4.0 -2015-03-09 23:59:59.0 UTC i2-end 1.0 -2015-03-10 00:00:00.0 UTC i3-start 2.0 -2015-03-10 23:59:59.0 UTC i3-end 2.0 -PREHOOK: query: CREATE EXTERNAL TABLE druid_test_table_utc2 (`__time` timestamp with local time zone, `interval_marker` string, `num_l` double) -STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' -TBLPROPERTIES ("druid.segment.granularity" = "DAY") -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:druid_test_dst -PREHOOK: Output: druid_test_dst@druid_test_table_utc2 -POSTHOOK: query: CREATE EXTERNAL TABLE druid_test_table_utc2 (`__time` timestamp with local time zone, `interval_marker` string, `num_l` double) -STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' -TBLPROPERTIES ("druid.segment.granularity" = "DAY") -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:druid_test_dst -POSTHOOK: Output: druid_test_dst@druid_test_table_utc2 -PREHOOK: query: insert into druid_test_table_utc2 values -(cast('2015-03-08 00:00:00' as timestamp with local time zone), 'i1-start', 4), -(cast('2015-03-08 23:59:59' as timestamp with local time zone), 'i1-end', 1), -(cast('2015-03-09 00:00:00' as timestamp with local time zone), 'i2-start', 4), -(cast('2015-03-09 23:59:59' as timestamp with local time zone), 'i2-end', 1), -(cast('2015-03-10 00:00:00' as timestamp with local time zone), 'i3-start', 2), -(cast('2015-03-10 23:59:59' as timestamp with local time zone), 'i3-end', 2) -PREHOOK: type: QUERY -PREHOOK: Input: _dummy_database@_dummy_table -PREHOOK: Output: druid_test_dst@druid_test_table_utc2 -POSTHOOK: query: insert into druid_test_table_utc2 values -(cast('2015-03-08 00:00:00' as timestamp with local time zone), 'i1-start', 4), -(cast('2015-03-08 23:59:59' as timestamp with local time zone), 'i1-end', 1), -(cast('2015-03-09 00:00:00' as timestamp with local time zone), 'i2-start', 4), -(cast('2015-03-09 23:59:59' as timestamp with local time zone), 'i2-end', 1), -(cast('2015-03-10 00:00:00' as timestamp with local time zone), 'i3-start', 2), -(cast('2015-03-10 23:59:59' as timestamp with local time zone), 'i3-end', 2) -POSTHOOK: type: QUERY -POSTHOOK: Input: _dummy_database@_dummy_table -POSTHOOK: Output: druid_test_dst@druid_test_table_utc2 -PREHOOK: query: select * FROM druid_test_table_utc2 -PREHOOK: type: QUERY -PREHOOK: Input: druid_test_dst@druid_test_table_utc2 -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: select * FROM druid_test_table_utc2 -POSTHOOK: type: QUERY -POSTHOOK: Input: druid_test_dst@druid_test_table_utc2 -POSTHOOK: Output: hdfs://### HDFS PATH ### -2015-03-08 00:00:00.0 UTC i1-start 4.0 -2015-03-08 23:59:59.0 UTC i1-end 1.0 -2015-03-09 00:00:00.0 UTC i2-start 4.0 -2015-03-09 23:59:59.0 UTC i2-end 1.0 -2015-03-10 00:00:00.0 UTC i3-start 2.0 -2015-03-10 23:59:59.0 UTC i3-end 2.0 diff --git a/ql/src/test/results/clientpositive/druid/druid_topn.q.out b/ql/src/test/results/clientpositive/druid/druid_topn.q.out deleted file mode 100644 index 447cce11ce7e..000000000000 --- a/ql/src/test/results/clientpositive/druid/druid_topn.q.out +++ /dev/null @@ -1,374 +0,0 @@ -PREHOOK: query: CREATE EXTERNAL TABLE druid_table_1_n1 -STORED BY 'org.apache.hadoop.hive.druid.QTestDruidStorageHandler' -TBLPROPERTIES ("druid.datasource" = "wikipedia") -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@druid_table_1_n1 -POSTHOOK: query: CREATE EXTERNAL TABLE druid_table_1_n1 -STORED BY 'org.apache.hadoop.hive.druid.QTestDruidStorageHandler' -TBLPROPERTIES ("druid.datasource" = "wikipedia") -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@druid_table_1_n1 -PREHOOK: query: DESCRIBE FORMATTED druid_table_1_n1 -PREHOOK: type: DESCTABLE -PREHOOK: Input: default@druid_table_1_n1 -POSTHOOK: query: DESCRIBE FORMATTED druid_table_1_n1 -POSTHOOK: type: DESCTABLE -POSTHOOK: Input: default@druid_table_1_n1 -# col_name data_type comment -__time timestamp with local time zone from deserializer -robot string from deserializer -namespace string from deserializer -anonymous string from deserializer -unpatrolled string from deserializer -page string from deserializer -language string from deserializer -newpage string from deserializer -user string from deserializer -count float from deserializer -added float from deserializer -delta float from deserializer -variation float from deserializer -deleted float from deserializer - -# Detailed Table Information -Database: default -#### A masked pattern was here #### -Retention: 0 -#### A masked pattern was here #### -Table Type: EXTERNAL_TABLE -Table Parameters: - COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"__time\":\"true\",\"added\":\"true\",\"anonymous\":\"true\",\"count\":\"true\",\"deleted\":\"true\",\"delta\":\"true\",\"language\":\"true\",\"namespace\":\"true\",\"newpage\":\"true\",\"page\":\"true\",\"robot\":\"true\",\"unpatrolled\":\"true\",\"user\":\"true\",\"variation\":\"true\"}} - EXTERNAL TRUE - bucketing_version 2 - druid.datasource wikipedia - numFiles 0 - numRows 0 - rawDataSize 0 - storage_handler org.apache.hadoop.hive.druid.QTestDruidStorageHandler - totalSize 0 -#### A masked pattern was here #### - -# Storage Information -SerDe Library: org.apache.hadoop.hive.druid.QTestDruidSerDe -InputFormat: null -OutputFormat: null -Compressed: No -Num Buckets: -1 -Bucket Columns: [] -Sort Columns: [] -Storage Desc Params: - serialization.format 1 -PREHOOK: query: EXPLAIN -SELECT robot, max(added) as m, sum(variation) -FROM druid_table_1_n1 -GROUP BY robot -ORDER BY m DESC -LIMIT 100 -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_1_n1 -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN -SELECT robot, max(added) as m, sum(variation) -FROM druid_table_1_n1 -GROUP BY robot -ORDER BY m DESC -LIMIT 100 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_1_n1 -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_table_1_n1 - properties: - druid.fieldNames robot,$f1,$f2 - druid.fieldTypes string,float,double - druid.query.json {"queryType":"groupBy","dataSource":"wikipedia","granularity":"all","dimensions":[{"type":"default","dimension":"robot","outputName":"robot","outputType":"STRING"}],"limitSpec":{"type":"default","limit":100,"columns":[{"dimension":"$f1","direction":"descending","dimensionOrder":"numeric"}]},"aggregations":[{"type":"doubleMax","name":"$f1","fieldName":"added"},{"type":"doubleSum","name":"$f2","fieldName":"variation"}],"intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"]} - druid.query.type groupBy - Select Operator - expressions: robot (type: string), $f1 (type: float), $f2 (type: double) - outputColumnNames: _col0, _col1, _col2 - ListSink - -PREHOOK: query: EXPLAIN -SELECT robot, `__time`, max(added), sum(variation) as s -FROM druid_table_1_n1 -GROUP BY robot, `__time` -ORDER BY s DESC -LIMIT 100 -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_1_n1 -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN -SELECT robot, `__time`, max(added), sum(variation) as s -FROM druid_table_1_n1 -GROUP BY robot, `__time` -ORDER BY s DESC -LIMIT 100 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_1_n1 -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_table_1_n1 - properties: - druid.fieldNames extract,robot,$f2,$f3 - druid.fieldTypes timestamp with local time zone,string,float,double - druid.query.json {"queryType":"groupBy","dataSource":"wikipedia","granularity":"all","dimensions":[{"type":"extraction","dimension":"__time","outputName":"extract","extractionFn":{"type":"timeFormat","format":"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'","timeZone":"UTC"}},{"type":"default","dimension":"robot","outputName":"robot","outputType":"STRING"}],"limitSpec":{"type":"default","limit":100,"columns":[{"dimension":"$f3","direction":"descending","dimensionOrder":"numeric"}]},"aggregations":[{"type":"doubleMax","name":"$f2","fieldName":"added"},{"type":"doubleSum","name":"$f3","fieldName":"variation"}],"intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"]} - druid.query.type groupBy - Select Operator - expressions: robot (type: string), extract (type: timestamp with local time zone), $f2 (type: float), $f3 (type: double) - outputColumnNames: _col0, _col1, _col2, _col3 - ListSink - -PREHOOK: query: EXPLAIN -SELECT robot, floor_year(`__time`), max(added), sum(variation) as s -FROM druid_table_1_n1 -GROUP BY robot, floor_year(`__time`) -ORDER BY s DESC -LIMIT 10 -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_1_n1 -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN -SELECT robot, floor_year(`__time`), max(added), sum(variation) as s -FROM druid_table_1_n1 -GROUP BY robot, floor_year(`__time`) -ORDER BY s DESC -LIMIT 10 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_1_n1 -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_table_1_n1 - properties: - druid.fieldNames robot,floor_year,$f2,$f3 - druid.fieldTypes string,timestamp with local time zone,float,double - druid.query.json {"queryType":"groupBy","dataSource":"wikipedia","granularity":"all","dimensions":[{"type":"default","dimension":"robot","outputName":"robot","outputType":"STRING"},{"type":"extraction","dimension":"__time","outputName":"floor_year","extractionFn":{"type":"timeFormat","format":"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'","granularity":{"type":"period","period":"P1Y","timeZone":"US/Pacific"},"timeZone":"UTC","locale":"und"}}],"limitSpec":{"type":"default","limit":10,"columns":[{"dimension":"$f3","direction":"descending","dimensionOrder":"numeric"}]},"aggregations":[{"type":"doubleMax","name":"$f2","fieldName":"added"},{"type":"doubleSum","name":"$f3","fieldName":"variation"}],"intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"]} - druid.query.type groupBy - Select Operator - expressions: robot (type: string), floor_year (type: timestamp with local time zone), $f2 (type: float), $f3 (type: double) - outputColumnNames: _col0, _col1, _col2, _col3 - ListSink - -PREHOOK: query: EXPLAIN -SELECT robot, floor_month(`__time`), max(added), sum(variation) as s -FROM druid_table_1_n1 -GROUP BY robot, floor_month(`__time`) -ORDER BY s -LIMIT 10 -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_1_n1 -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN -SELECT robot, floor_month(`__time`), max(added), sum(variation) as s -FROM druid_table_1_n1 -GROUP BY robot, floor_month(`__time`) -ORDER BY s -LIMIT 10 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_1_n1 -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_table_1_n1 - properties: - druid.fieldNames robot,floor_month,$f2,$f3 - druid.fieldTypes string,timestamp with local time zone,float,double - druid.query.json {"queryType":"groupBy","dataSource":"wikipedia","granularity":"all","dimensions":[{"type":"default","dimension":"robot","outputName":"robot","outputType":"STRING"},{"type":"extraction","dimension":"__time","outputName":"floor_month","extractionFn":{"type":"timeFormat","format":"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'","granularity":{"type":"period","period":"P1M","timeZone":"US/Pacific"},"timeZone":"UTC","locale":"und"}}],"limitSpec":{"type":"default","limit":10,"columns":[{"dimension":"$f3","direction":"ascending","dimensionOrder":"numeric"}]},"aggregations":[{"type":"doubleMax","name":"$f2","fieldName":"added"},{"type":"doubleSum","name":"$f3","fieldName":"variation"}],"intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"]} - druid.query.type groupBy - Select Operator - expressions: robot (type: string), floor_month (type: timestamp with local time zone), $f2 (type: float), $f3 (type: double) - outputColumnNames: _col0, _col1, _col2, _col3 - ListSink - -PREHOOK: query: EXPLAIN -SELECT robot, floor_month(`__time`), max(added) as m, sum(variation) as s -FROM druid_table_1_n1 -GROUP BY robot, namespace, floor_month(`__time`) -ORDER BY s DESC, m DESC -LIMIT 10 -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_1_n1 -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN -SELECT robot, floor_month(`__time`), max(added) as m, sum(variation) as s -FROM druid_table_1_n1 -GROUP BY robot, namespace, floor_month(`__time`) -ORDER BY s DESC, m DESC -LIMIT 10 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_1_n1 -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_table_1_n1 - properties: - druid.fieldNames robot,namespace,floor_month,$f3,$f4 - druid.fieldTypes string,string,timestamp with local time zone,float,double - druid.query.json {"queryType":"groupBy","dataSource":"wikipedia","granularity":"all","dimensions":[{"type":"default","dimension":"robot","outputName":"robot","outputType":"STRING"},{"type":"default","dimension":"namespace","outputName":"namespace","outputType":"STRING"},{"type":"extraction","dimension":"__time","outputName":"floor_month","extractionFn":{"type":"timeFormat","format":"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'","granularity":{"type":"period","period":"P1M","timeZone":"US/Pacific"},"timeZone":"UTC","locale":"und"}}],"limitSpec":{"type":"default","limit":10,"columns":[{"dimension":"$f4","direction":"descending","dimensionOrder":"numeric"},{"dimension":"$f3","direction":"descending","dimensionOrder":"numeric"}]},"aggregations":[{"type":"doubleMax","name":"$f3","fieldName":"added"},{"type":"doubleSum","name":"$f4","fieldName":"variation"}],"intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"]} - druid.query.type groupBy - Select Operator - expressions: robot (type: string), floor_month (type: timestamp with local time zone), $f3 (type: float), $f4 (type: double) - outputColumnNames: _col0, _col1, _col2, _col3 - ListSink - -PREHOOK: query: EXPLAIN -SELECT robot, floor_month(`__time`), max(added) as m, sum(variation) as s -FROM druid_table_1_n1 -GROUP BY robot, namespace, floor_month(`__time`) -ORDER BY robot ASC, m DESC -LIMIT 10 -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_1_n1 -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN -SELECT robot, floor_month(`__time`), max(added) as m, sum(variation) as s -FROM druid_table_1_n1 -GROUP BY robot, namespace, floor_month(`__time`) -ORDER BY robot ASC, m DESC -LIMIT 10 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_1_n1 -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_table_1_n1 - properties: - druid.fieldNames robot,namespace,floor_month,$f3,$f4 - druid.fieldTypes string,string,timestamp with local time zone,float,double - druid.query.json {"queryType":"groupBy","dataSource":"wikipedia","granularity":"all","dimensions":[{"type":"default","dimension":"robot","outputName":"robot","outputType":"STRING"},{"type":"default","dimension":"namespace","outputName":"namespace","outputType":"STRING"},{"type":"extraction","dimension":"__time","outputName":"floor_month","extractionFn":{"type":"timeFormat","format":"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'","granularity":{"type":"period","period":"P1M","timeZone":"US/Pacific"},"timeZone":"UTC","locale":"und"}}],"limitSpec":{"type":"default","limit":10,"columns":[{"dimension":"robot","direction":"ascending","dimensionOrder":"lexicographic"},{"dimension":"$f3","direction":"descending","dimensionOrder":"numeric"}]},"aggregations":[{"type":"doubleMax","name":"$f3","fieldName":"added"},{"type":"doubleSum","name":"$f4","fieldName":"variation"}],"intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"]} - druid.query.type groupBy - Select Operator - expressions: robot (type: string), floor_month (type: timestamp with local time zone), $f3 (type: float), $f4 (type: double) - outputColumnNames: _col0, _col1, _col2, _col3 - ListSink - -PREHOOK: query: EXPLAIN -SELECT robot, floor_year(`__time`), max(added), sum(variation) as s -FROM druid_table_1_n1 -WHERE robot='1' -GROUP BY robot, floor_year(`__time`) -ORDER BY s -LIMIT 10 -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_1_n1 -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN -SELECT robot, floor_year(`__time`), max(added), sum(variation) as s -FROM druid_table_1_n1 -WHERE robot='1' -GROUP BY robot, floor_year(`__time`) -ORDER BY s -LIMIT 10 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_1_n1 -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_table_1_n1 - properties: - druid.fieldNames floor_year,$f1_0,$f2 - druid.fieldTypes timestamp with local time zone,float,double - druid.query.json {"queryType":"groupBy","dataSource":"wikipedia","granularity":"all","dimensions":[{"type":"extraction","dimension":"__time","outputName":"floor_year","extractionFn":{"type":"timeFormat","format":"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'","granularity":{"type":"period","period":"P1Y","timeZone":"US/Pacific"},"timeZone":"UTC","locale":"und"}}],"limitSpec":{"type":"default","limit":10,"columns":[{"dimension":"$f2","direction":"ascending","dimensionOrder":"numeric"}]},"filter":{"type":"selector","dimension":"robot","value":"1"},"aggregations":[{"type":"doubleMax","name":"$f1_0","fieldName":"added"},{"type":"doubleSum","name":"$f2","fieldName":"variation"}],"intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"]} - druid.query.type groupBy - Select Operator - expressions: '1' (type: string), floor_year (type: timestamp with local time zone), $f1_0 (type: float), $f2 (type: double) - outputColumnNames: _col0, _col1, _col2, _col3 - ListSink - -PREHOOK: query: EXPLAIN -SELECT robot, floor_hour(`__time`), max(added) as m, sum(variation) -FROM druid_table_1_n1 -WHERE floor_hour(`__time`) - BETWEEN CAST('2010-01-01 00:00:00' AS TIMESTAMP WITH LOCAL TIME ZONE) - AND CAST('2014-01-01 00:00:00' AS TIMESTAMP WITH LOCAL TIME ZONE) -GROUP BY robot, floor_hour(`__time`) -ORDER BY m -LIMIT 100 -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_1_n1 -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN -SELECT robot, floor_hour(`__time`), max(added) as m, sum(variation) -FROM druid_table_1_n1 -WHERE floor_hour(`__time`) - BETWEEN CAST('2010-01-01 00:00:00' AS TIMESTAMP WITH LOCAL TIME ZONE) - AND CAST('2014-01-01 00:00:00' AS TIMESTAMP WITH LOCAL TIME ZONE) -GROUP BY robot, floor_hour(`__time`) -ORDER BY m -LIMIT 100 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_1_n1 -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_table_1_n1 - properties: - druid.fieldNames robot,floor_hour,$f2,$f3 - druid.fieldTypes string,timestamp with local time zone,float,double - druid.query.json {"queryType":"groupBy","dataSource":"wikipedia","granularity":"all","dimensions":[{"type":"default","dimension":"robot","outputName":"robot","outputType":"STRING"},{"type":"extraction","dimension":"__time","outputName":"floor_hour","extractionFn":{"type":"timeFormat","format":"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'","granularity":{"type":"period","period":"PT1H","timeZone":"US/Pacific"},"timeZone":"UTC","locale":"und"}}],"limitSpec":{"type":"default","limit":100,"columns":[{"dimension":"$f2","direction":"ascending","dimensionOrder":"numeric"}]},"aggregations":[{"type":"doubleMax","name":"$f2","fieldName":"added"},{"type":"doubleSum","name":"$f3","fieldName":"variation"}],"intervals":["2010-01-01T08:00:00.000Z/2014-01-01T08:00:00.001Z"]} - druid.query.type groupBy - Select Operator - expressions: robot (type: string), floor_hour (type: timestamp with local time zone), $f2 (type: float), $f3 (type: double) - outputColumnNames: _col0, _col1, _col2, _col3 - ListSink - diff --git a/ql/src/test/results/clientpositive/druid/druidkafkamini_avro.q.out b/ql/src/test/results/clientpositive/druid/druidkafkamini_avro.q.out deleted file mode 100644 index d33dd4cbc202..000000000000 --- a/ql/src/test/results/clientpositive/druid/druidkafkamini_avro.q.out +++ /dev/null @@ -1,263 +0,0 @@ -PREHOOK: query: CREATE EXTERNAL TABLE druid_kafka_test_avro(`__time` timestamp , `page` string, `user` string, `language` string, - `country` string,`continent` string, `namespace` string, `newPage` boolean, `unpatrolled` boolean, - `anonymous` boolean, `robot` boolean, added int, deleted int, delta bigint) - STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' - TBLPROPERTIES ( - "druid.segment.granularity" = "MONTH", - "druid.query.granularity" = "MINUTE", - "kafka.bootstrap.servers" = "localhost:9092", - "kafka.topic" = "wiki_kafka_avro_table", - "druid.kafka.ingestion.useEarliestOffset" = "true", - "druid.kafka.ingestion.maxRowsInMemory" = "5", - "druid.kafka.ingestion.startDelay" = "PT1S", - "druid.kafka.ingestion.taskDuration" = "PT30S", - "druid.kafka.ingestion.period" = "PT5S", - "druid.kafka.ingestion.consumer.retries" = "2", - "druid.kafka.ingestion.reportParseExceptions" = "true", - "druid.timestamp.column" = "timestamp", - "druid.timestamp.format" = "MM/dd/yyyy HH:mm:ss", - "druid.parseSpec.format" = "avro", - 'avro.schema.literal'='{ - "type" : "record", - "name" : "Wikipedia", - "namespace" : "org.apache.hive.kafka", - "version": "1", - "fields" : [ { - "name" : "isrobot", - "type" : "boolean" - }, { - "name" : "channel", - "type" : "string" - }, { - "name" : "timestamp", - "type" : "string" - }, { - "name" : "flags", - "type" : "string" - }, { - "name" : "isunpatrolled", - "type" : "boolean" - }, { - "name" : "page", - "type" : "string" - }, { - "name" : "diffurl", - "type" : "string" - }, { - "name" : "added", - "type" : "long" - }, { - "name" : "comment", - "type" : "string" - }, { - "name" : "commentlength", - "type" : "long" - }, { - "name" : "isnew", - "type" : "boolean" - }, { - "name" : "isminor", - "type" : "boolean" - }, { - "name" : "delta", - "type" : "long" - }, { - "name" : "isanonymous", - "type" : "boolean" - }, { - "name" : "user", - "type" : "string" - }, { - "name" : "deltabucket", - "type" : "double" - }, { - "name" : "deleted", - "type" : "long" - }, { - "name" : "namespace", - "type" : "string" - } ] - }' - ) -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@druid_kafka_test_avro -POSTHOOK: query: CREATE EXTERNAL TABLE druid_kafka_test_avro(`__time` timestamp , `page` string, `user` string, `language` string, - `country` string,`continent` string, `namespace` string, `newPage` boolean, `unpatrolled` boolean, - `anonymous` boolean, `robot` boolean, added int, deleted int, delta bigint) - STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' - TBLPROPERTIES ( - "druid.segment.granularity" = "MONTH", - "druid.query.granularity" = "MINUTE", - "kafka.bootstrap.servers" = "localhost:9092", - "kafka.topic" = "wiki_kafka_avro_table", - "druid.kafka.ingestion.useEarliestOffset" = "true", - "druid.kafka.ingestion.maxRowsInMemory" = "5", - "druid.kafka.ingestion.startDelay" = "PT1S", - "druid.kafka.ingestion.taskDuration" = "PT30S", - "druid.kafka.ingestion.period" = "PT5S", - "druid.kafka.ingestion.consumer.retries" = "2", - "druid.kafka.ingestion.reportParseExceptions" = "true", - "druid.timestamp.column" = "timestamp", - "druid.timestamp.format" = "MM/dd/yyyy HH:mm:ss", - "druid.parseSpec.format" = "avro", - 'avro.schema.literal'='{ - "type" : "record", - "name" : "Wikipedia", - "namespace" : "org.apache.hive.kafka", - "version": "1", - "fields" : [ { - "name" : "isrobot", - "type" : "boolean" - }, { - "name" : "channel", - "type" : "string" - }, { - "name" : "timestamp", - "type" : "string" - }, { - "name" : "flags", - "type" : "string" - }, { - "name" : "isunpatrolled", - "type" : "boolean" - }, { - "name" : "page", - "type" : "string" - }, { - "name" : "diffurl", - "type" : "string" - }, { - "name" : "added", - "type" : "long" - }, { - "name" : "comment", - "type" : "string" - }, { - "name" : "commentlength", - "type" : "long" - }, { - "name" : "isnew", - "type" : "boolean" - }, { - "name" : "isminor", - "type" : "boolean" - }, { - "name" : "delta", - "type" : "long" - }, { - "name" : "isanonymous", - "type" : "boolean" - }, { - "name" : "user", - "type" : "string" - }, { - "name" : "deltabucket", - "type" : "double" - }, { - "name" : "deleted", - "type" : "long" - }, { - "name" : "namespace", - "type" : "string" - } ] - }' - ) -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@druid_kafka_test_avro -PREHOOK: query: ALTER TABLE druid_kafka_test_avro SET TBLPROPERTIES('druid.kafka.ingestion' = 'START') -PREHOOK: type: ALTERTABLE_PROPERTIES -PREHOOK: Input: default@druid_kafka_test_avro -PREHOOK: Output: default@druid_kafka_test_avro -POSTHOOK: query: ALTER TABLE druid_kafka_test_avro SET TBLPROPERTIES('druid.kafka.ingestion' = 'START') -POSTHOOK: type: ALTERTABLE_PROPERTIES -POSTHOOK: Input: default@druid_kafka_test_avro -POSTHOOK: Output: default@druid_kafka_test_avro -["default.druid_kafka_test_avro"] -PREHOOK: query: DESCRIBE druid_kafka_test_avro -PREHOOK: type: DESCTABLE -PREHOOK: Input: default@druid_kafka_test_avro -POSTHOOK: query: DESCRIBE druid_kafka_test_avro -POSTHOOK: type: DESCTABLE -POSTHOOK: Input: default@druid_kafka_test_avro -__time timestamp from deserializer -page string from deserializer -user string from deserializer -language string from deserializer -country string from deserializer -continent string from deserializer -namespace string from deserializer -newpage boolean from deserializer -unpatrolled boolean from deserializer -anonymous boolean from deserializer -robot boolean from deserializer -added int from deserializer -deleted int from deserializer -delta bigint from deserializer -PREHOOK: query: DESCRIBE EXTENDED druid_kafka_test_avro -PREHOOK: type: DESCTABLE -PREHOOK: Input: default@druid_kafka_test_avro -POSTHOOK: query: DESCRIBE EXTENDED druid_kafka_test_avro -POSTHOOK: type: DESCTABLE -POSTHOOK: Input: default@druid_kafka_test_avro -__time timestamp from deserializer -page string from deserializer -user string from deserializer -language string from deserializer -country string from deserializer -continent string from deserializer -namespace string from deserializer -newpage boolean from deserializer -unpatrolled boolean from deserializer -anonymous boolean from deserializer -robot boolean from deserializer -added int from deserializer -deleted int from deserializer -delta bigint from deserializer - -#### A masked pattern was here #### -StorageHandlerInfo -Druid Storage Handler Runtime Status for default.druid_kafka_test_avro -kafkaPartitions=1 -activeTasks=[] -publishingTasks=[] -#### A masked pattern was here #### -aggregateLag=0 -#### A masked pattern was here #### -PREHOOK: query: Select count(*) FROM druid_kafka_test_avro -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_kafka_test_avro -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: Select count(*) FROM druid_kafka_test_avro -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_kafka_test_avro -POSTHOOK: Output: hdfs://### HDFS PATH ### -11 -PREHOOK: query: Select page FROM druid_kafka_test_avro -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_kafka_test_avro -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: Select page FROM druid_kafka_test_avro -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_kafka_test_avro -POSTHOOK: Output: hdfs://### HDFS PATH ### -page is 0 -page is 100 -page is 200 -page is 300 -page is 400 -page is 500 -page is 600 -page is 700 -page is 800 -page is 900 -page is 1000 -PREHOOK: query: DROP TABLE druid_kafka_test_avro -PREHOOK: type: DROPTABLE -PREHOOK: Input: default@druid_kafka_test_avro -PREHOOK: Output: default@druid_kafka_test_avro -POSTHOOK: query: DROP TABLE druid_kafka_test_avro -POSTHOOK: type: DROPTABLE -POSTHOOK: Input: default@druid_kafka_test_avro -POSTHOOK: Output: default@druid_kafka_test_avro diff --git a/ql/src/test/results/clientpositive/druid/druidkafkamini_basic.q.out b/ql/src/test/results/clientpositive/druid/druidkafkamini_basic.q.out deleted file mode 100644 index 8d6ebe989f85..000000000000 --- a/ql/src/test/results/clientpositive/druid/druidkafkamini_basic.q.out +++ /dev/null @@ -1,767 +0,0 @@ -PREHOOK: query: CREATE EXTERNAL TABLE druid_kafka_test(`__time` timestamp, page string, `user` string, language string, added int, deleted int) - STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' - TBLPROPERTIES ( - "druid.segment.granularity" = "MONTH", - "druid.query.granularity" = "MINUTE", - "kafka.bootstrap.servers" = "localhost:9092", - "kafka.topic" = "test-topic", - "druid.kafka.ingestion.useEarliestOffset" = "true", - "druid.kafka.ingestion.maxRowsInMemory" = "5", - "druid.kafka.ingestion.startDelay" = "PT1S", - "druid.kafka.ingestion.taskDuration" = "PT60S", - "druid.kafka.ingestion.period" = "PT1S", - "druid.kafka.ingestion.consumer.retries" = "2" - ) -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@druid_kafka_test -POSTHOOK: query: CREATE EXTERNAL TABLE druid_kafka_test(`__time` timestamp, page string, `user` string, language string, added int, deleted int) - STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' - TBLPROPERTIES ( - "druid.segment.granularity" = "MONTH", - "druid.query.granularity" = "MINUTE", - "kafka.bootstrap.servers" = "localhost:9092", - "kafka.topic" = "test-topic", - "druid.kafka.ingestion.useEarliestOffset" = "true", - "druid.kafka.ingestion.maxRowsInMemory" = "5", - "druid.kafka.ingestion.startDelay" = "PT1S", - "druid.kafka.ingestion.taskDuration" = "PT60S", - "druid.kafka.ingestion.period" = "PT1S", - "druid.kafka.ingestion.consumer.retries" = "2" - ) -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@druid_kafka_test -PREHOOK: query: ALTER TABLE druid_kafka_test SET TBLPROPERTIES('druid.kafka.ingestion' = 'START') -PREHOOK: type: ALTERTABLE_PROPERTIES -PREHOOK: Input: default@druid_kafka_test -PREHOOK: Output: default@druid_kafka_test -POSTHOOK: query: ALTER TABLE druid_kafka_test SET TBLPROPERTIES('druid.kafka.ingestion' = 'START') -POSTHOOK: type: ALTERTABLE_PROPERTIES -POSTHOOK: Input: default@druid_kafka_test -POSTHOOK: Output: default@druid_kafka_test -["default.druid_kafka_test"] -PREHOOK: query: DESCRIBE druid_kafka_test -PREHOOK: type: DESCTABLE -PREHOOK: Input: default@druid_kafka_test -POSTHOOK: query: DESCRIBE druid_kafka_test -POSTHOOK: type: DESCTABLE -POSTHOOK: Input: default@druid_kafka_test -__time timestamp from deserializer -page string from deserializer -user string from deserializer -language string from deserializer -added int from deserializer -deleted int from deserializer -PREHOOK: query: DESCRIBE EXTENDED druid_kafka_test -PREHOOK: type: DESCTABLE -PREHOOK: Input: default@druid_kafka_test -POSTHOOK: query: DESCRIBE EXTENDED druid_kafka_test -POSTHOOK: type: DESCTABLE -POSTHOOK: Input: default@druid_kafka_test -__time timestamp from deserializer -page string from deserializer -user string from deserializer -language string from deserializer -added int from deserializer -deleted int from deserializer - -#### A masked pattern was here #### -StorageHandlerInfo -Druid Storage Handler Runtime Status for default.druid_kafka_test -kafkaPartitions=1 -activeTasks=[] -publishingTasks=[] -#### A masked pattern was here #### -aggregateLag=0 -#### A masked pattern was here #### -PREHOOK: query: Select count(*) FROM druid_kafka_test -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_kafka_test -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: Select count(*) FROM druid_kafka_test -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_kafka_test -POSTHOOK: Output: hdfs://### HDFS PATH ### -10 -PREHOOK: query: Select page FROM druid_kafka_test order by page -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_kafka_test -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: Select page FROM druid_kafka_test order by page -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_kafka_test -POSTHOOK: Output: hdfs://### HDFS PATH ### -Cherno Alpha -Cherno Alpha -Coyote Tango -Coyote Tango -Crimson Typhoon -Crimson Typhoon -Gypsy Danger -Gypsy Danger -Striker Eureka -Striker Eureka -PREHOOK: query: ALTER TABLE druid_kafka_test SET TBLPROPERTIES('druid.kafka.ingestion' = 'RESET') -PREHOOK: type: ALTERTABLE_PROPERTIES -PREHOOK: Input: default@druid_kafka_test -PREHOOK: Output: default@druid_kafka_test -POSTHOOK: query: ALTER TABLE druid_kafka_test SET TBLPROPERTIES('druid.kafka.ingestion' = 'RESET') -POSTHOOK: type: ALTERTABLE_PROPERTIES -POSTHOOK: Input: default@druid_kafka_test -POSTHOOK: Output: default@druid_kafka_test -PREHOOK: query: DESCRIBE druid_kafka_test -PREHOOK: type: DESCTABLE -PREHOOK: Input: default@druid_kafka_test -POSTHOOK: query: DESCRIBE druid_kafka_test -POSTHOOK: type: DESCTABLE -POSTHOOK: Input: default@druid_kafka_test -__time timestamp from deserializer -page string from deserializer -user string from deserializer -language string from deserializer -added int from deserializer -deleted int from deserializer -PREHOOK: query: DESCRIBE EXTENDED druid_kafka_test -PREHOOK: type: DESCTABLE -PREHOOK: Input: default@druid_kafka_test -POSTHOOK: query: DESCRIBE EXTENDED druid_kafka_test -POSTHOOK: type: DESCTABLE -POSTHOOK: Input: default@druid_kafka_test -__time timestamp from deserializer -page string from deserializer -user string from deserializer -language string from deserializer -added int from deserializer -deleted int from deserializer - -#### A masked pattern was here #### -StorageHandlerInfo -Druid Storage Handler Runtime Status for default.druid_kafka_test -kafkaPartitions=1 -activeTasks=[] -publishingTasks=[] -#### A masked pattern was here #### -aggregateLag=0 -#### A masked pattern was here #### -PREHOOK: query: Select count(*) FROM druid_kafka_test -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_kafka_test -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: Select count(*) FROM druid_kafka_test -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_kafka_test -POSTHOOK: Output: hdfs://### HDFS PATH ### -10 -PREHOOK: query: Select page FROM druid_kafka_test order by page -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_kafka_test -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: Select page FROM druid_kafka_test order by page -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_kafka_test -POSTHOOK: Output: hdfs://### HDFS PATH ### -Cherno Alpha -Cherno Alpha -Coyote Tango -Coyote Tango -Crimson Typhoon -Crimson Typhoon -Gypsy Danger -Gypsy Danger -Striker Eureka -Striker Eureka -PREHOOK: query: CREATE TABLE languages(shortname string, fullname string) -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@languages -POSTHOOK: query: CREATE TABLE languages(shortname string, fullname string) -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@languages -PREHOOK: query: INSERT INTO languages values -("en", "english"), -("ru", "russian") -PREHOOK: type: QUERY -PREHOOK: Input: _dummy_database@_dummy_table -PREHOOK: Output: default@languages -POSTHOOK: query: INSERT INTO languages values -("en", "english"), -("ru", "russian") -POSTHOOK: type: QUERY -POSTHOOK: Input: _dummy_database@_dummy_table -POSTHOOK: Output: default@languages -POSTHOOK: Lineage: languages.fullname SCRIPT [] -POSTHOOK: Lineage: languages.shortname SCRIPT [] -PREHOOK: query: EXPLAIN EXTENDED -SELECT a.fullname, b.`user` -FROM -( -(SELECT fullname, shortname -FROM languages) a -JOIN -(SELECT language, `user` -FROM druid_kafka_test) b - ON a.shortname = b.language -) -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_kafka_test -PREHOOK: Input: default@languages -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN EXTENDED -SELECT a.fullname, b.`user` -FROM -( -(SELECT fullname, shortname -FROM languages) a -JOIN -(SELECT language, `user` -FROM druid_kafka_test) b - ON a.shortname = b.language -) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_kafka_test -POSTHOOK: Input: default@languages -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Tez -#### A masked pattern was here #### - Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) -#### A masked pattern was here #### - Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: languages - filterExpr: shortname is not null (type: boolean) - Statistics: Num rows: 2 Data size: 354 Basic stats: COMPLETE Column stats: COMPLETE - GatherStats: false - Filter Operator - isSamplingPred: false - predicate: shortname is not null (type: boolean) - Statistics: Num rows: 2 Data size: 354 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: fullname (type: string), shortname (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 354 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - bucketingVersion: 2 - key expressions: _col1 (type: string) - null sort order: z - numBuckets: -1 - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 2 Data size: 354 Basic stats: COMPLETE Column stats: COMPLETE - tag: 0 - value expressions: _col0 (type: string) - auto parallelism: true - Execution mode: vectorized, llap - LLAP IO: no inputs - Path -> Alias: - hdfs://### HDFS PATH ### [languages] - Path -> Partition: - hdfs://### HDFS PATH ### - Partition - base file name: languages - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucket_count -1 - bucketing_version 2 - column.name.delimiter , - columns shortname,fullname - columns.types string:string -#### A masked pattern was here #### - location hdfs://### HDFS PATH ### - name default.languages - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucketing_version 2 - column.name.delimiter , - columns shortname,fullname - columns.comments - columns.types string:string -#### A masked pattern was here #### - location hdfs://### HDFS PATH ### - name default.languages - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.languages - name: default.languages - Truncated Path -> Alias: - /languages [languages] - Map 3 - Map Operator Tree: - TableScan - alias: druid_kafka_test - properties: - druid.fieldNames language,user - druid.fieldTypes string,string - druid.query.json {"queryType":"scan","dataSource":"default.druid_kafka_test","intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"],"filter":{"type":"not","field":{"type":"selector","dimension":"language","value":null}},"columns":["language","user"],"resultFormat":"compactedList"} - druid.query.type scan - Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE - GatherStats: false - Reduce Output Operator - bucketingVersion: 2 - key expressions: language (type: string) - null sort order: z - numBuckets: -1 - sort order: + - Map-reduce partition columns: language (type: string) - Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE - tag: 1 - value expressions: user (type: string) - auto parallelism: true - Execution mode: vectorized, llap - LLAP IO: no inputs - Path -> Alias: - hdfs://### HDFS PATH ### [druid_kafka_test] - Path -> Partition: - hdfs://### HDFS PATH ### - Partition - base file name: druid_kafka_test - input format: org.apache.hadoop.hive.druid.io.DruidQueryBasedInputFormat - output format: org.apache.hadoop.hive.druid.io.DruidOutputFormat - properties: - EXTERNAL TRUE - bucket_count -1 - bucketing_version 2 - column.name.delimiter , - columns __time,page,user,language,added,deleted - columns.types timestamp:string:string:string:int:int - druid.datasource default.druid_kafka_test - druid.fieldNames language,user - druid.fieldTypes string,string - druid.kafka.ingestion.consumer.retries 2 - druid.kafka.ingestion.maxRowsInMemory 5 - druid.kafka.ingestion.period PT1S - druid.kafka.ingestion.startDelay PT1S - druid.kafka.ingestion.taskDuration PT60S - druid.kafka.ingestion.useEarliestOffset true - druid.query.granularity MINUTE - druid.query.json {"queryType":"scan","dataSource":"default.druid_kafka_test","intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"],"filter":{"type":"not","field":{"type":"selector","dimension":"language","value":null}},"columns":["language","user"],"resultFormat":"compactedList"} - druid.query.type scan - druid.segment.granularity MONTH -#### A masked pattern was here #### - kafka.bootstrap.servers localhost:9092 - kafka.topic test-topic -#### A masked pattern was here #### - location hdfs://### HDFS PATH ### - name default.druid_kafka_test - serialization.format 1 - serialization.lib org.apache.hadoop.hive.druid.serde.DruidSerDe - storage_handler org.apache.hadoop.hive.druid.DruidStorageHandler - serde: org.apache.hadoop.hive.druid.serde.DruidSerDe - - input format: org.apache.hadoop.hive.druid.io.DruidQueryBasedInputFormat - output format: org.apache.hadoop.hive.druid.io.DruidOutputFormat - properties: - EXTERNAL TRUE - bucketing_version 2 - column.name.delimiter , - columns __time,page,user,language,added,deleted - columns.comments - columns.types timestamp:string:string:string:int:int - druid.datasource default.druid_kafka_test - druid.fieldNames language,user - druid.fieldTypes string,string - druid.kafka.ingestion.consumer.retries 2 - druid.kafka.ingestion.maxRowsInMemory 5 - druid.kafka.ingestion.period PT1S - druid.kafka.ingestion.startDelay PT1S - druid.kafka.ingestion.taskDuration PT60S - druid.kafka.ingestion.useEarliestOffset true - druid.query.granularity MINUTE - druid.query.json {"queryType":"scan","dataSource":"default.druid_kafka_test","intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"],"filter":{"type":"not","field":{"type":"selector","dimension":"language","value":null}},"columns":["language","user"],"resultFormat":"compactedList"} - druid.query.type scan - druid.segment.granularity MONTH -#### A masked pattern was here #### - kafka.bootstrap.servers localhost:9092 - kafka.topic test-topic -#### A masked pattern was here #### - location hdfs://### HDFS PATH ### - name default.druid_kafka_test - serialization.format 1 - serialization.lib org.apache.hadoop.hive.druid.serde.DruidSerDe - storage_handler org.apache.hadoop.hive.druid.DruidStorageHandler - serde: org.apache.hadoop.hive.druid.serde.DruidSerDe - name: default.druid_kafka_test - name: default.druid_kafka_test - Truncated Path -> Alias: - /druid_kafka_test [druid_kafka_test] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col1 (type: string) - 1 language (type: string) - outputColumnNames: _col0, _col3 - Position of Big Table: 0 - Statistics: Num rows: 2 Data size: 389 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col3 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 389 Basic stats: COMPLETE Column stats: NONE - File Output Operator - bucketingVersion: 2 - compressed: false - GlobalTableId: 0 - directory: hdfs://### HDFS PATH ### - NumFilesPerFileSink: 1 - Statistics: Num rows: 2 Data size: 389 Basic stats: COMPLETE Column stats: NONE - Stats Publishing Key Prefix: hdfs://### HDFS PATH ### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - bucketing_version -1 - columns _col0,_col1 - columns.types string:string - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: SELECT a.fullname, b.`user` -FROM -( -(SELECT fullname, shortname -FROM languages) a -JOIN -(SELECT language, `user` -FROM druid_kafka_test) b - ON a.shortname = b.language -) order by b.`user` -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_kafka_test -PREHOOK: Input: default@languages -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: SELECT a.fullname, b.`user` -FROM -( -(SELECT fullname, shortname -FROM languages) a -JOIN -(SELECT language, `user` -FROM druid_kafka_test) b - ON a.shortname = b.language -) order by b.`user` -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_kafka_test -POSTHOOK: Input: default@languages -POSTHOOK: Output: hdfs://### HDFS PATH ### -russian masterYi -russian masterYi -english nuclear -english nuclear -english speed -english speed -PREHOOK: query: EXPLAIN -SELECT language, -1 * (a + b) AS c -FROM ( - SELECT (sum(added)-sum(deleted)) / (count(*) * 3) AS a, sum(deleted) AS b, language - FROM druid_kafka_test - GROUP BY language) subq -ORDER BY c DESC -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_kafka_test -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN -SELECT language, -1 * (a + b) AS c -FROM ( - SELECT (sum(added)-sum(deleted)) / (count(*) * 3) AS a, sum(deleted) AS b, language - FROM druid_kafka_test - GROUP BY language) subq -ORDER BY c DESC -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_kafka_test -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_kafka_test - properties: - druid.fieldNames language,c - druid.fieldTypes string,double - druid.query.json {"queryType":"groupBy","dataSource":"default.druid_kafka_test","granularity":"all","dimensions":[{"type":"default","dimension":"language","outputName":"language","outputType":"STRING"}],"limitSpec":{"type":"default","columns":[{"dimension":"c","direction":"descending","dimensionOrder":"numeric"}]},"aggregations":[{"type":"longSum","name":"$f1","fieldName":"added"},{"type":"longSum","name":"$f2","fieldName":"deleted"},{"type":"count","name":"$f3"}],"postAggregations":[{"type":"expression","name":"c","expression":"(-1 * ((CAST((\"$f1\" - \"$f2\"), 'DOUBLE') / CAST((\"$f3\" * 3), 'DOUBLE')) + CAST(\"$f2\", 'DOUBLE')))"}],"intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"]} - druid.query.type groupBy - Select Operator - expressions: language (type: string), c (type: double) - outputColumnNames: _col0, _col1 - ListSink - -PREHOOK: query: EXPLAIN -SELECT language, `user`, sum(added) - sum(deleted) AS a -FROM druid_kafka_test -WHERE extract (week from `__time`) IN (10,11) -GROUP BY language, `user` -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_kafka_test -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN -SELECT language, `user`, sum(added) - sum(deleted) AS a -FROM druid_kafka_test -WHERE extract (week from `__time`) IN (10,11) -GROUP BY language, `user` -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_kafka_test -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_kafka_test - properties: - druid.fieldNames language,user,a - druid.fieldTypes string,string,bigint - druid.query.json {"queryType":"groupBy","dataSource":"default.druid_kafka_test","granularity":"all","dimensions":[{"type":"default","dimension":"user","outputName":"user","outputType":"STRING"},{"type":"default","dimension":"language","outputName":"language","outputType":"STRING"}],"limitSpec":{"type":"default"},"filter":{"type":"in","dimension":"__time","values":["10","11"],"extractionFn":{"type":"timeFormat","format":"w","timeZone":"UTC","locale":"en-US"}},"aggregations":[{"type":"longSum","name":"$f2","fieldName":"added"},{"type":"longSum","name":"$f3","fieldName":"deleted"}],"postAggregations":[{"type":"expression","name":"a","expression":"(\"$f2\" - \"$f3\")"}],"intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"]} - druid.query.type groupBy - Select Operator - expressions: language (type: string), user (type: string), a (type: bigint) - outputColumnNames: _col0, _col1, _col2 - ListSink - -PREHOOK: query: EXPLAIN -SELECT language, sum(deleted) / count(*) AS a -FROM druid_kafka_test -GROUP BY language -ORDER BY a DESC -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_kafka_test -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN -SELECT language, sum(deleted) / count(*) AS a -FROM druid_kafka_test -GROUP BY language -ORDER BY a DESC -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_kafka_test -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_kafka_test - properties: - druid.fieldNames language,a - druid.fieldTypes string,double - druid.query.json {"queryType":"groupBy","dataSource":"default.druid_kafka_test","granularity":"all","dimensions":[{"type":"default","dimension":"language","outputName":"language","outputType":"STRING"}],"limitSpec":{"type":"default","columns":[{"dimension":"a","direction":"descending","dimensionOrder":"numeric"}]},"aggregations":[{"type":"longSum","name":"$f1","fieldName":"deleted"},{"type":"count","name":"$f2"}],"postAggregations":[{"type":"expression","name":"a","expression":"(CAST(\"$f1\", 'DOUBLE') / CAST(\"$f2\", 'DOUBLE'))"}],"intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"]} - druid.query.type groupBy - Select Operator - expressions: language (type: string), a (type: double) - outputColumnNames: _col0, _col1 - ListSink - -PREHOOK: query: EXPLAIN -SELECT language, sum(added) / sum(deleted) AS a, - CASE WHEN sum(deleted)=0 THEN 1.0 ELSE sum(deleted) END AS b -FROM druid_kafka_test -GROUP BY language -ORDER BY a DESC -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_kafka_test -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN -SELECT language, sum(added) / sum(deleted) AS a, - CASE WHEN sum(deleted)=0 THEN 1.0 ELSE sum(deleted) END AS b -FROM druid_kafka_test -GROUP BY language -ORDER BY a DESC -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_kafka_test -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_kafka_test - properties: - druid.fieldNames language,a,b - druid.fieldTypes string,double,decimal(19,0) - druid.query.json {"queryType":"groupBy","dataSource":"default.druid_kafka_test","granularity":"all","dimensions":[{"type":"default","dimension":"language","outputName":"language","outputType":"STRING"}],"limitSpec":{"type":"default","columns":[{"dimension":"a","direction":"descending","dimensionOrder":"numeric"}]},"aggregations":[{"type":"longSum","name":"$f1","fieldName":"added"},{"type":"longSum","name":"$f2","fieldName":"deleted"}],"postAggregations":[{"type":"expression","name":"a","expression":"(CAST(\"$f1\", 'DOUBLE') / CAST(\"$f2\", 'DOUBLE'))"},{"type":"expression","name":"b","expression":"case_searched((\"$f2\" == 0),1,CAST(\"$f2\", 'DOUBLE'))"}],"intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"]} - druid.query.type groupBy - Select Operator - expressions: language (type: string), a (type: double), b (type: decimal(19,0)) - outputColumnNames: _col0, _col1, _col2 - ListSink - -PREHOOK: query: EXPLAIN -SELECT language, a, a - b as c -FROM ( - SELECT language, sum(added) + 100 AS a, sum(deleted) AS b - FROM druid_kafka_test - GROUP BY language) subq -ORDER BY a DESC -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_kafka_test -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN -SELECT language, a, a - b as c -FROM ( - SELECT language, sum(added) + 100 AS a, sum(deleted) AS b - FROM druid_kafka_test - GROUP BY language) subq -ORDER BY a DESC -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_kafka_test -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_kafka_test - properties: - druid.fieldNames language,a,c - druid.fieldTypes string,bigint,bigint - druid.query.json {"queryType":"groupBy","dataSource":"default.druid_kafka_test","granularity":"all","dimensions":[{"type":"default","dimension":"language","outputName":"language","outputType":"STRING"}],"limitSpec":{"type":"default","columns":[{"dimension":"a","direction":"descending","dimensionOrder":"numeric"}]},"aggregations":[{"type":"longSum","name":"$f1","fieldName":"added"},{"type":"longSum","name":"$f2","fieldName":"deleted"}],"postAggregations":[{"type":"expression","name":"a","expression":"(\"$f1\" + 100)"},{"type":"expression","name":"c","expression":"((\"$f1\" + 100) - \"$f2\")"}],"intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"]} - druid.query.type groupBy - Select Operator - expressions: language (type: string), a (type: bigint), c (type: bigint) - outputColumnNames: _col0, _col1, _col2 - ListSink - -PREHOOK: query: EXPLAIN -SELECT language, `user`, "A" -FROM ( - SELECT sum(added) - sum(deleted) AS a, language, `user` - FROM druid_kafka_test - GROUP BY language, `user` ) subq -ORDER BY "A" -LIMIT 5 -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_kafka_test -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN -SELECT language, `user`, "A" -FROM ( - SELECT sum(added) - sum(deleted) AS a, language, `user` - FROM druid_kafka_test - GROUP BY language, `user` ) subq -ORDER BY "A" -LIMIT 5 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_kafka_test -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_kafka_test - properties: - druid.fieldNames user,language - druid.fieldTypes string,string - druid.query.json {"queryType":"groupBy","dataSource":"default.druid_kafka_test","granularity":"all","dimensions":[{"type":"default","dimension":"user","outputName":"user","outputType":"STRING"},{"type":"default","dimension":"language","outputName":"language","outputType":"STRING"}],"limitSpec":{"type":"default","limit":5,"columns":[]},"aggregations":[],"intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"]} - druid.query.type groupBy - Select Operator - expressions: language (type: string), user (type: string), 'A' (type: string) - outputColumnNames: _col0, _col1, _col2 - ListSink - -PREHOOK: query: EXPLAIN -SELECT language, `user`, "A" -FROM ( - SELECT language, sum(added) + sum(deleted) AS a, `user` - FROM druid_kafka_test - GROUP BY language, `user`) subq -ORDER BY `user`, language -LIMIT 5 -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_kafka_test -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN -SELECT language, `user`, "A" -FROM ( - SELECT language, sum(added) + sum(deleted) AS a, `user` - FROM druid_kafka_test - GROUP BY language, `user`) subq -ORDER BY `user`, language -LIMIT 5 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_kafka_test -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_kafka_test - properties: - druid.fieldNames user,language - druid.fieldTypes string,string - druid.query.json {"queryType":"groupBy","dataSource":"default.druid_kafka_test","granularity":"all","dimensions":[{"type":"default","dimension":"user","outputName":"user","outputType":"STRING"},{"type":"default","dimension":"language","outputName":"language","outputType":"STRING"}],"limitSpec":{"type":"default","limit":5,"columns":[{"dimension":"user","direction":"ascending","dimensionOrder":"lexicographic"},{"dimension":"language","direction":"ascending","dimensionOrder":"lexicographic"}]},"aggregations":[],"intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"]} - druid.query.type groupBy - Select Operator - expressions: language (type: string), user (type: string), 'A' (type: string) - outputColumnNames: _col0, _col1, _col2 - ListSink - -PREHOOK: query: DROP TABLE druid_kafka_test -PREHOOK: type: DROPTABLE -PREHOOK: Input: default@druid_kafka_test -PREHOOK: Output: default@druid_kafka_test -POSTHOOK: query: DROP TABLE druid_kafka_test -POSTHOOK: type: DROPTABLE -POSTHOOK: Input: default@druid_kafka_test -POSTHOOK: Output: default@druid_kafka_test diff --git a/ql/src/test/results/clientpositive/druid/druidkafkamini_csv.q.out b/ql/src/test/results/clientpositive/druid/druidkafkamini_csv.q.out deleted file mode 100644 index 2f5817ae46b5..000000000000 --- a/ql/src/test/results/clientpositive/druid/druidkafkamini_csv.q.out +++ /dev/null @@ -1,138 +0,0 @@ -PREHOOK: query: CREATE EXTERNAL TABLE druid_kafka_test_csv(`__time` timestamp , `page` string, `user` string, `language` string, - `country` string,`continent` string, `namespace` string, `newpage` boolean, `unpatrolled` boolean, - `anonymous` boolean, `robot` boolean, added int, deleted int, delta bigint) - STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' - TBLPROPERTIES ( - "druid.segment.granularity" = "MONTH", - "druid.query.granularity" = "MINUTE", - "kafka.bootstrap.servers" = "localhost:9092", - "kafka.topic" = "wiki_kafka_csv", - "druid.kafka.ingestion.useEarliestOffset" = "true", - "druid.kafka.ingestion.maxRowsInMemory" = "5", - "druid.kafka.ingestion.startDelay" = "PT1S", - "druid.kafka.ingestion.taskDuration" = "PT30S", - "druid.kafka.ingestion.period" = "PT5S", - "druid.kafka.ingestion.consumer.retries" = "2", - "druid.kafka.ingestion.reportParseExceptions" = "true", - "druid.parseSpec.format" = "csv", - "druid.parseSpec.columns" = "__time,page,language,user,unpatrolled,newpage,robot,anonymous,namespace,continent,country,region,city,added,deleted,delta" - ) -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@druid_kafka_test_csv -POSTHOOK: query: CREATE EXTERNAL TABLE druid_kafka_test_csv(`__time` timestamp , `page` string, `user` string, `language` string, - `country` string,`continent` string, `namespace` string, `newpage` boolean, `unpatrolled` boolean, - `anonymous` boolean, `robot` boolean, added int, deleted int, delta bigint) - STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' - TBLPROPERTIES ( - "druid.segment.granularity" = "MONTH", - "druid.query.granularity" = "MINUTE", - "kafka.bootstrap.servers" = "localhost:9092", - "kafka.topic" = "wiki_kafka_csv", - "druid.kafka.ingestion.useEarliestOffset" = "true", - "druid.kafka.ingestion.maxRowsInMemory" = "5", - "druid.kafka.ingestion.startDelay" = "PT1S", - "druid.kafka.ingestion.taskDuration" = "PT30S", - "druid.kafka.ingestion.period" = "PT5S", - "druid.kafka.ingestion.consumer.retries" = "2", - "druid.kafka.ingestion.reportParseExceptions" = "true", - "druid.parseSpec.format" = "csv", - "druid.parseSpec.columns" = "__time,page,language,user,unpatrolled,newpage,robot,anonymous,namespace,continent,country,region,city,added,deleted,delta" - ) -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@druid_kafka_test_csv -PREHOOK: query: ALTER TABLE druid_kafka_test_csv SET TBLPROPERTIES('druid.kafka.ingestion' = 'START') -PREHOOK: type: ALTERTABLE_PROPERTIES -PREHOOK: Input: default@druid_kafka_test_csv -PREHOOK: Output: default@druid_kafka_test_csv -POSTHOOK: query: ALTER TABLE druid_kafka_test_csv SET TBLPROPERTIES('druid.kafka.ingestion' = 'START') -POSTHOOK: type: ALTERTABLE_PROPERTIES -POSTHOOK: Input: default@druid_kafka_test_csv -POSTHOOK: Output: default@druid_kafka_test_csv -["default.druid_kafka_test_csv"] -PREHOOK: query: DESCRIBE druid_kafka_test_csv -PREHOOK: type: DESCTABLE -PREHOOK: Input: default@druid_kafka_test_csv -POSTHOOK: query: DESCRIBE druid_kafka_test_csv -POSTHOOK: type: DESCTABLE -POSTHOOK: Input: default@druid_kafka_test_csv -__time timestamp from deserializer -page string from deserializer -user string from deserializer -language string from deserializer -country string from deserializer -continent string from deserializer -namespace string from deserializer -newpage boolean from deserializer -unpatrolled boolean from deserializer -anonymous boolean from deserializer -robot boolean from deserializer -added int from deserializer -deleted int from deserializer -delta bigint from deserializer -PREHOOK: query: DESCRIBE EXTENDED druid_kafka_test_csv -PREHOOK: type: DESCTABLE -PREHOOK: Input: default@druid_kafka_test_csv -POSTHOOK: query: DESCRIBE EXTENDED druid_kafka_test_csv -POSTHOOK: type: DESCTABLE -POSTHOOK: Input: default@druid_kafka_test_csv -__time timestamp from deserializer -page string from deserializer -user string from deserializer -language string from deserializer -country string from deserializer -continent string from deserializer -namespace string from deserializer -newpage boolean from deserializer -unpatrolled boolean from deserializer -anonymous boolean from deserializer -robot boolean from deserializer -added int from deserializer -deleted int from deserializer -delta bigint from deserializer - -#### A masked pattern was here #### -StorageHandlerInfo -Druid Storage Handler Runtime Status for default.druid_kafka_test_csv -kafkaPartitions=1 -activeTasks=[] -publishingTasks=[] -#### A masked pattern was here #### -aggregateLag=0 -#### A masked pattern was here #### -PREHOOK: query: Select count(*) FROM druid_kafka_test_csv -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_kafka_test_csv -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: Select count(*) FROM druid_kafka_test_csv -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_kafka_test_csv -POSTHOOK: Output: hdfs://### HDFS PATH ### -10 -PREHOOK: query: Select page FROM druid_kafka_test_csv -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_kafka_test_csv -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: Select page FROM druid_kafka_test_csv -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_kafka_test_csv -POSTHOOK: Output: hdfs://### HDFS PATH ### -Gypsy Danger -Striker Eureka -Cherno Alpha -Crimson Typhoon -Coyote Tango -Gypsy Danger -Striker Eureka -Cherno Alpha -Crimson Typhoon -Coyote Tango -PREHOOK: query: DROP TABLE druid_kafka_test_csv -PREHOOK: type: DROPTABLE -PREHOOK: Input: default@druid_kafka_test_csv -PREHOOK: Output: default@druid_kafka_test_csv -POSTHOOK: query: DROP TABLE druid_kafka_test_csv -POSTHOOK: type: DROPTABLE -POSTHOOK: Input: default@druid_kafka_test_csv -POSTHOOK: Output: default@druid_kafka_test_csv diff --git a/ql/src/test/results/clientpositive/druid/druidkafkamini_delimited.q.out b/ql/src/test/results/clientpositive/druid/druidkafkamini_delimited.q.out deleted file mode 100644 index 2331fba13ee5..000000000000 --- a/ql/src/test/results/clientpositive/druid/druidkafkamini_delimited.q.out +++ /dev/null @@ -1,140 +0,0 @@ -PREHOOK: query: CREATE EXTERNAL TABLE druid_kafka_test_delimited(`__time` timestamp , `page` string, `user` string, `language` string, - `country` string,`continent` string, `namespace` string, `newpage` boolean, `unpatrolled` boolean, - `anonymous` boolean, `robot` boolean, added int, deleted int, delta bigint) - STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' - TBLPROPERTIES ( - "druid.segment.granularity" = "MONTH", - "druid.query.granularity" = "MINUTE", - "kafka.bootstrap.servers" = "localhost:9092", - "kafka.topic" = "wiki_kafka_csv", - "druid.kafka.ingestion.useEarliestOffset" = "true", - "druid.kafka.ingestion.maxRowsInMemory" = "5", - "druid.kafka.ingestion.startDelay" = "PT1S", - "druid.kafka.ingestion.taskDuration" = "PT30S", - "druid.kafka.ingestion.period" = "PT5S", - "druid.kafka.ingestion.consumer.retries" = "2", - "druid.kafka.ingestion.reportParseExceptions" = "true", - "druid.parseSpec.format" = "delimited", - "druid.parseSpec.columns" = "__time,page,language,user,unpatrolled,newpage,robot,anonymous,namespace,continent,country,region,city,added,deleted,delta", - "druid.parseSpec.delimiter"="," - ) -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@druid_kafka_test_delimited -POSTHOOK: query: CREATE EXTERNAL TABLE druid_kafka_test_delimited(`__time` timestamp , `page` string, `user` string, `language` string, - `country` string,`continent` string, `namespace` string, `newpage` boolean, `unpatrolled` boolean, - `anonymous` boolean, `robot` boolean, added int, deleted int, delta bigint) - STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' - TBLPROPERTIES ( - "druid.segment.granularity" = "MONTH", - "druid.query.granularity" = "MINUTE", - "kafka.bootstrap.servers" = "localhost:9092", - "kafka.topic" = "wiki_kafka_csv", - "druid.kafka.ingestion.useEarliestOffset" = "true", - "druid.kafka.ingestion.maxRowsInMemory" = "5", - "druid.kafka.ingestion.startDelay" = "PT1S", - "druid.kafka.ingestion.taskDuration" = "PT30S", - "druid.kafka.ingestion.period" = "PT5S", - "druid.kafka.ingestion.consumer.retries" = "2", - "druid.kafka.ingestion.reportParseExceptions" = "true", - "druid.parseSpec.format" = "delimited", - "druid.parseSpec.columns" = "__time,page,language,user,unpatrolled,newpage,robot,anonymous,namespace,continent,country,region,city,added,deleted,delta", - "druid.parseSpec.delimiter"="," - ) -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@druid_kafka_test_delimited -PREHOOK: query: ALTER TABLE druid_kafka_test_delimited SET TBLPROPERTIES('druid.kafka.ingestion' = 'START') -PREHOOK: type: ALTERTABLE_PROPERTIES -PREHOOK: Input: default@druid_kafka_test_delimited -PREHOOK: Output: default@druid_kafka_test_delimited -POSTHOOK: query: ALTER TABLE druid_kafka_test_delimited SET TBLPROPERTIES('druid.kafka.ingestion' = 'START') -POSTHOOK: type: ALTERTABLE_PROPERTIES -POSTHOOK: Input: default@druid_kafka_test_delimited -POSTHOOK: Output: default@druid_kafka_test_delimited -["default.druid_kafka_test_delimited"] -PREHOOK: query: DESCRIBE druid_kafka_test_delimited -PREHOOK: type: DESCTABLE -PREHOOK: Input: default@druid_kafka_test_delimited -POSTHOOK: query: DESCRIBE druid_kafka_test_delimited -POSTHOOK: type: DESCTABLE -POSTHOOK: Input: default@druid_kafka_test_delimited -__time timestamp from deserializer -page string from deserializer -user string from deserializer -language string from deserializer -country string from deserializer -continent string from deserializer -namespace string from deserializer -newpage boolean from deserializer -unpatrolled boolean from deserializer -anonymous boolean from deserializer -robot boolean from deserializer -added int from deserializer -deleted int from deserializer -delta bigint from deserializer -PREHOOK: query: DESCRIBE EXTENDED druid_kafka_test_delimited -PREHOOK: type: DESCTABLE -PREHOOK: Input: default@druid_kafka_test_delimited -POSTHOOK: query: DESCRIBE EXTENDED druid_kafka_test_delimited -POSTHOOK: type: DESCTABLE -POSTHOOK: Input: default@druid_kafka_test_delimited -__time timestamp from deserializer -page string from deserializer -user string from deserializer -language string from deserializer -country string from deserializer -continent string from deserializer -namespace string from deserializer -newpage boolean from deserializer -unpatrolled boolean from deserializer -anonymous boolean from deserializer -robot boolean from deserializer -added int from deserializer -deleted int from deserializer -delta bigint from deserializer - -#### A masked pattern was here #### -StorageHandlerInfo -Druid Storage Handler Runtime Status for default.druid_kafka_test_delimited -kafkaPartitions=1 -activeTasks=[] -publishingTasks=[] -#### A masked pattern was here #### -aggregateLag=0 -#### A masked pattern was here #### -PREHOOK: query: Select count(*) FROM druid_kafka_test_delimited -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_kafka_test_delimited -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: Select count(*) FROM druid_kafka_test_delimited -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_kafka_test_delimited -POSTHOOK: Output: hdfs://### HDFS PATH ### -10 -PREHOOK: query: Select page FROM druid_kafka_test_delimited -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_kafka_test_delimited -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: Select page FROM druid_kafka_test_delimited -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_kafka_test_delimited -POSTHOOK: Output: hdfs://### HDFS PATH ### -"Gypsy Danger" -"Striker Eureka" -"Cherno Alpha" -"Crimson Typhoon" -"Coyote Tango" -"Gypsy Danger" -"Striker Eureka" -"Cherno Alpha" -"Crimson Typhoon" -"Coyote Tango" -PREHOOK: query: DROP TABLE druid_kafka_test_delimited -PREHOOK: type: DROPTABLE -PREHOOK: Input: default@druid_kafka_test_delimited -PREHOOK: Output: default@druid_kafka_test_delimited -POSTHOOK: query: DROP TABLE druid_kafka_test_delimited -POSTHOOK: type: DROPTABLE -POSTHOOK: Input: default@druid_kafka_test_delimited -POSTHOOK: Output: default@druid_kafka_test_delimited diff --git a/ql/src/test/results/clientpositive/druid/druidmini_dynamic_partition.q.out b/ql/src/test/results/clientpositive/druid/druidmini_dynamic_partition.q.out deleted file mode 100644 index 65f8cca39b79..000000000000 --- a/ql/src/test/results/clientpositive/druid/druidmini_dynamic_partition.q.out +++ /dev/null @@ -1,682 +0,0 @@ -PREHOOK: query: CREATE EXTERNAL TABLE druid_partitioned_table_0 - STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' - TBLPROPERTIES ( - "druid.segment.granularity" = "HOUR", - "druid.query.granularity" = "MINUTE", - "druid.segment.targetShardsPerGranularity" = "0" - ) - AS - SELECT cast (`ctimestamp1` as timestamp with local time zone) as `__time`, - cstring1, - cstring2, - cdouble, - cfloat, - ctinyint, - csmallint, - cint, - cbigint, - cboolean1, - cboolean2 - FROM alltypesorc where ctimestamp1 IS NOT NULL -PREHOOK: type: CREATETABLE_AS_SELECT -PREHOOK: Input: default@alltypesorc -PREHOOK: Output: database:default -PREHOOK: Output: default@druid_partitioned_table_0 -POSTHOOK: query: CREATE EXTERNAL TABLE druid_partitioned_table_0 - STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' - TBLPROPERTIES ( - "druid.segment.granularity" = "HOUR", - "druid.query.granularity" = "MINUTE", - "druid.segment.targetShardsPerGranularity" = "0" - ) - AS - SELECT cast (`ctimestamp1` as timestamp with local time zone) as `__time`, - cstring1, - cstring2, - cdouble, - cfloat, - ctinyint, - csmallint, - cint, - cbigint, - cboolean1, - cboolean2 - FROM alltypesorc where ctimestamp1 IS NOT NULL -POSTHOOK: type: CREATETABLE_AS_SELECT -POSTHOOK: Input: default@alltypesorc -POSTHOOK: Output: database:default -POSTHOOK: Output: default@druid_partitioned_table_0 -POSTHOOK: Lineage: druid_partitioned_table_0.__time EXPRESSION [(alltypesorc)alltypesorc.FieldSchema(name:ctimestamp1, type:timestamp, comment:null), ] -POSTHOOK: Lineage: druid_partitioned_table_0.cbigint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cbigint, type:bigint, comment:null), ] -POSTHOOK: Lineage: druid_partitioned_table_0.cboolean1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cboolean1, type:boolean, comment:null), ] -POSTHOOK: Lineage: druid_partitioned_table_0.cboolean2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cboolean2, type:boolean, comment:null), ] -POSTHOOK: Lineage: druid_partitioned_table_0.cdouble SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cdouble, type:double, comment:null), ] -POSTHOOK: Lineage: druid_partitioned_table_0.cfloat SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cfloat, type:float, comment:null), ] -POSTHOOK: Lineage: druid_partitioned_table_0.cint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] -POSTHOOK: Lineage: druid_partitioned_table_0.csmallint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:csmallint, type:smallint, comment:null), ] -POSTHOOK: Lineage: druid_partitioned_table_0.cstring1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ] -POSTHOOK: Lineage: druid_partitioned_table_0.cstring2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring2, type:string, comment:null), ] -POSTHOOK: Lineage: druid_partitioned_table_0.ctinyint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctinyint, type:tinyint, comment:null), ] -PREHOOK: query: EXPLAIN CREATE EXTERNAL TABLE druid_partitioned_table - STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' - TBLPROPERTIES ( - "druid.segment.granularity" = "HOUR", - "druid.query.granularity" = "MINUTE", - "druid.segment.targetShardsPerGranularity" = "6" - ) - AS - SELECT cast (`ctimestamp1` as timestamp with local time zone) as `__time`, - cstring1, - cstring2, - cdouble, - cfloat, - ctinyint, - csmallint, - cint, - cbigint, - cboolean1, - cboolean2 - FROM alltypesorc where ctimestamp1 IS NOT NULL -PREHOOK: type: CREATETABLE_AS_SELECT -PREHOOK: Input: default@alltypesorc -PREHOOK: Output: database:default -PREHOOK: Output: default@druid_partitioned_table -POSTHOOK: query: EXPLAIN CREATE EXTERNAL TABLE druid_partitioned_table - STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' - TBLPROPERTIES ( - "druid.segment.granularity" = "HOUR", - "druid.query.granularity" = "MINUTE", - "druid.segment.targetShardsPerGranularity" = "6" - ) - AS - SELECT cast (`ctimestamp1` as timestamp with local time zone) as `__time`, - cstring1, - cstring2, - cdouble, - cfloat, - ctinyint, - csmallint, - cint, - cbigint, - cboolean1, - cboolean2 - FROM alltypesorc where ctimestamp1 IS NOT NULL -POSTHOOK: type: CREATETABLE_AS_SELECT -POSTHOOK: Input: default@alltypesorc -POSTHOOK: Output: database:default -POSTHOOK: Output: default@druid_partitioned_table -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-2 depends on stages: Stage-1 - Stage-4 depends on stages: Stage-2, Stage-0 - Stage-3 depends on stages: Stage-4 - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Tez -#### A masked pattern was here #### - Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) -#### A masked pattern was here #### - Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: alltypesorc - filterExpr: ctimestamp1 is not null (type: boolean) - Statistics: Num rows: 12288 Data size: 2477090 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: ctimestamp1 is not null (type: boolean) - Statistics: Num rows: 9173 Data size: 1849230 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: CAST( ctimestamp1 AS timestamp with local time zone) (type: timestamp with local time zone), cstring1 (type: string), cstring2 (type: string), cdouble (type: double), cfloat (type: float), ctinyint (type: tinyint), csmallint (type: smallint), cint (type: int), cbigint (type: bigint), cboolean1 (type: boolean), cboolean2 (type: boolean) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10 - Statistics: Num rows: 9173 Data size: 1942190 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: timestamp with local time zone), _col1 (type: string), _col2 (type: string), _col3 (type: double), _col4 (type: float), _col5 (type: tinyint), _col6 (type: smallint), _col7 (type: int), _col8 (type: bigint), _col9 (type: boolean), _col10 (type: boolean), floor_hour(CAST( GenericUDFEpochMilli(_col0) AS TIMESTAMP)) (type: timestamp), (floor((1.0D / rand())) % 6) (type: bigint) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, __time_granularity, __druid_extra_partition_key - Statistics: Num rows: 9173 Data size: 1942190 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: __time_granularity (type: timestamp), __druid_extra_partition_key (type: bigint) - null sort order: aa - sort order: ++ - Map-reduce partition columns: __time_granularity (type: timestamp), __druid_extra_partition_key (type: bigint) - Statistics: Num rows: 9173 Data size: 1942190 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: timestamp with local time zone), _col1 (type: string), _col2 (type: string), _col3 (type: double), _col4 (type: float), _col5 (type: tinyint), _col6 (type: smallint), _col7 (type: int), _col8 (type: bigint), _col9 (type: boolean), _col10 (type: boolean) - Execution mode: llap - LLAP IO: all inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: timestamp with local time zone), VALUE._col1 (type: string), VALUE._col2 (type: string), VALUE._col3 (type: double), VALUE._col4 (type: float), VALUE._col5 (type: tinyint), VALUE._col6 (type: smallint), VALUE._col7 (type: int), VALUE._col8 (type: bigint), VALUE._col9 (type: boolean), VALUE._col10 (type: boolean), KEY.__time_granularity (type: timestamp), KEY.__druid_extra_partition_key (type: bigint) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, __time_granularity, __druid_extra_partition_key - Statistics: Num rows: 9173 Data size: 1942190 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 9173 Data size: 1942190 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.hive.druid.io.DruidQueryBasedInputFormat - output format: org.apache.hadoop.hive.druid.io.DruidOutputFormat - serde: org.apache.hadoop.hive.druid.serde.DruidSerDe - name: default.druid_partitioned_table - - Stage: Stage-2 - Dependency Collection - - Stage: Stage-4 - Create Table - columns: __time timestamp with local time zone, cstring1 string, cstring2 string, cdouble double, cfloat float, ctinyint tinyint, csmallint smallint, cint int, cbigint bigint, cboolean1 boolean, cboolean2 boolean - name: default.druid_partitioned_table - storage handler: org.apache.hadoop.hive.druid.DruidStorageHandler - table properties: - druid.query.granularity MINUTE - druid.segment.granularity HOUR - druid.segment.targetShardsPerGranularity 6 - external.table.purge true - isExternal: true - - Stage: Stage-3 - Stats Work - Basic Stats Work: - - Stage: Stage-0 - Move Operator - files: - hdfs directory: true - destination: hdfs://### HDFS PATH ### - -PREHOOK: query: CREATE EXTERNAL TABLE druid_partitioned_table -STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' -TBLPROPERTIES ( -"druid.segment.granularity" = "HOUR", -"druid.query.granularity" = "MINUTE", -"druid.segment.targetShardsPerGranularity" = "6" -) -AS -SELECT cast (`ctimestamp1` as timestamp with local time zone) as `__time`, - cstring1, - cstring2, - cdouble, - cfloat, - ctinyint, - csmallint, - cint, - cbigint, - cboolean1, - cboolean2 - FROM alltypesorc where ctimestamp1 IS NOT NULL -PREHOOK: type: CREATETABLE_AS_SELECT -PREHOOK: Input: default@alltypesorc -PREHOOK: Output: database:default -PREHOOK: Output: default@druid_partitioned_table -POSTHOOK: query: CREATE EXTERNAL TABLE druid_partitioned_table -STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' -TBLPROPERTIES ( -"druid.segment.granularity" = "HOUR", -"druid.query.granularity" = "MINUTE", -"druid.segment.targetShardsPerGranularity" = "6" -) -AS -SELECT cast (`ctimestamp1` as timestamp with local time zone) as `__time`, - cstring1, - cstring2, - cdouble, - cfloat, - ctinyint, - csmallint, - cint, - cbigint, - cboolean1, - cboolean2 - FROM alltypesorc where ctimestamp1 IS NOT NULL -POSTHOOK: type: CREATETABLE_AS_SELECT -POSTHOOK: Input: default@alltypesorc -POSTHOOK: Output: database:default -POSTHOOK: Output: default@druid_partitioned_table -POSTHOOK: Lineage: druid_partitioned_table.__time EXPRESSION [(alltypesorc)alltypesorc.FieldSchema(name:ctimestamp1, type:timestamp, comment:null), ] -POSTHOOK: Lineage: druid_partitioned_table.cbigint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cbigint, type:bigint, comment:null), ] -POSTHOOK: Lineage: druid_partitioned_table.cboolean1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cboolean1, type:boolean, comment:null), ] -POSTHOOK: Lineage: druid_partitioned_table.cboolean2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cboolean2, type:boolean, comment:null), ] -POSTHOOK: Lineage: druid_partitioned_table.cdouble SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cdouble, type:double, comment:null), ] -POSTHOOK: Lineage: druid_partitioned_table.cfloat SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cfloat, type:float, comment:null), ] -POSTHOOK: Lineage: druid_partitioned_table.cint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] -POSTHOOK: Lineage: druid_partitioned_table.csmallint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:csmallint, type:smallint, comment:null), ] -POSTHOOK: Lineage: druid_partitioned_table.cstring1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ] -POSTHOOK: Lineage: druid_partitioned_table.cstring2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring2, type:string, comment:null), ] -POSTHOOK: Lineage: druid_partitioned_table.ctinyint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctinyint, type:tinyint, comment:null), ] -PREHOOK: query: SELECT sum(cint), sum(cbigint) FROM druid_partitioned_table -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_partitioned_table -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: SELECT sum(cint), sum(cbigint) FROM druid_partitioned_table -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_partitioned_table -POSTHOOK: Output: hdfs://### HDFS PATH ### -1408069801800 10992545287 -PREHOOK: query: SELECT sum(cint), sum(cbigint) FROM druid_partitioned_table_0 -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_partitioned_table_0 -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: SELECT sum(cint), sum(cbigint) FROM druid_partitioned_table_0 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_partitioned_table_0 -POSTHOOK: Output: hdfs://### HDFS PATH ### -1408069801800 10992545287 -PREHOOK: query: SELECT floor_hour(cast(`ctimestamp1` as timestamp with local time zone)) as `__time`, - cstring1, - cstring2, - cdouble, - cfloat, - ctinyint, - csmallint, - cint, - cbigint, - cboolean1, - cboolean2 - FROM alltypesorc where ctimestamp1 IS NOT NULL order by `__time`, cstring2 DESC NULLS LAST, cstring1 DESC NULLS LAST LIMIT 10 -PREHOOK: type: QUERY -PREHOOK: Input: default@alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: SELECT floor_hour(cast(`ctimestamp1` as timestamp with local time zone)) as `__time`, - cstring1, - cstring2, - cdouble, - cfloat, - ctinyint, - csmallint, - cint, - cbigint, - cboolean1, - cboolean2 - FROM alltypesorc where ctimestamp1 IS NOT NULL order by `__time`, cstring2 DESC NULLS LAST, cstring1 DESC NULLS LAST LIMIT 10 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -1969-12-31 15:00:00.0 US/Pacific NULL yx36UAT823Cm -200.0 52.0 52 -200 NULL 2029007949 NULL true -1969-12-31 15:00:00.0 US/Pacific NULL yvcx4HYTT8tvAm6CNbXHaH -7196.0 40.0 40 -7196 NULL 437984126 NULL false -1969-12-31 15:00:00.0 US/Pacific NULL ysho54gMb 15601.0 -22.0 -22 15601 NULL 1553802956 NULL false -1969-12-31 15:00:00.0 US/Pacific NULL yqXw7J7 15601.0 44.0 44 15601 NULL 1265051089 NULL true -1969-12-31 15:00:00.0 US/Pacific NULL yfpFFQ0 15601.0 11.0 11 15601 NULL 11070716 NULL false -1969-12-31 15:00:00.0 US/Pacific NULL ySl6tu66m72erf1 -200.0 -50.0 -50 -200 NULL 824529348 NULL false -1969-12-31 15:00:00.0 US/Pacific NULL yPUM0wC54vq 15601.0 -8.0 -8 15601 NULL 1821279179 NULL true -1969-12-31 15:00:00.0 US/Pacific NULL yOgijYXi753GboFW7L1x65l -7196.0 -34.0 -34 -7196 NULL -1022931985 NULL false -1969-12-31 15:00:00.0 US/Pacific NULL yO7xKhbtrsBU147xuT0CF7Q 15601.0 47.0 47 15601 NULL 661404907 NULL true -1969-12-31 15:00:00.0 US/Pacific NULL yLB85lr145622oTRo -200.0 -41.0 -41 -200 NULL -260138227 NULL true -PREHOOK: query: EXPLAIN INSERT INTO TABLE druid_partitioned_table -SELECT cast (`ctimestamp2` as timestamp with local time zone) as `__time`, - cstring1, - cstring2, - cdouble, - cfloat, - ctinyint, - csmallint, - cint, - cbigint, - cboolean1, - cboolean2 - FROM alltypesorc where ctimestamp2 IS NOT NULL -PREHOOK: type: QUERY -PREHOOK: Input: default@alltypesorc -PREHOOK: Output: default@druid_partitioned_table -POSTHOOK: query: EXPLAIN INSERT INTO TABLE druid_partitioned_table -SELECT cast (`ctimestamp2` as timestamp with local time zone) as `__time`, - cstring1, - cstring2, - cdouble, - cfloat, - ctinyint, - csmallint, - cint, - cbigint, - cboolean1, - cboolean2 - FROM alltypesorc where ctimestamp2 IS NOT NULL -POSTHOOK: type: QUERY -POSTHOOK: Input: default@alltypesorc -POSTHOOK: Output: default@druid_partitioned_table -STAGE DEPENDENCIES: - Stage-0 is a root stage - Stage-1 is a root stage - Stage-2 is a root stage - Stage-4 depends on stages: Stage-2 - -STAGE PLANS: - Stage: Stage-0 - Unset Properties - table name: default.druid_partitioned_table - properties: - COLUMN_STATS_ACCURATE - - Stage: Stage-1 - Pre-Insert task - - Stage: Stage-2 - Tez -#### A masked pattern was here #### - Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) -#### A masked pattern was here #### - Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: alltypesorc - filterExpr: ctimestamp2 is not null (type: boolean) - Statistics: Num rows: 12288 Data size: 2477130 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: ctimestamp2 is not null (type: boolean) - Statistics: Num rows: 9174 Data size: 1849380 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: CAST( ctimestamp2 AS timestamp with local time zone) (type: timestamp with local time zone), cstring1 (type: string), cstring2 (type: string), cdouble (type: double), cfloat (type: float), ctinyint (type: tinyint), csmallint (type: smallint), cint (type: int), cbigint (type: bigint), cboolean1 (type: boolean), cboolean2 (type: boolean) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10 - Statistics: Num rows: 9174 Data size: 1942340 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: timestamp with local time zone), _col1 (type: string), _col2 (type: string), _col3 (type: double), _col4 (type: float), _col5 (type: tinyint), _col6 (type: smallint), _col7 (type: int), _col8 (type: bigint), _col9 (type: boolean), _col10 (type: boolean), floor_hour(CAST( GenericUDFEpochMilli(_col0) AS TIMESTAMP)) (type: timestamp), (floor((1.0D / rand())) % 6) (type: bigint) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, __time_granularity, __druid_extra_partition_key - Statistics: Num rows: 9174 Data size: 1942340 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: __time_granularity (type: timestamp), __druid_extra_partition_key (type: bigint) - null sort order: aa - sort order: ++ - Map-reduce partition columns: __time_granularity (type: timestamp), __druid_extra_partition_key (type: bigint) - Statistics: Num rows: 9174 Data size: 1942340 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: timestamp with local time zone), _col1 (type: string), _col2 (type: string), _col3 (type: double), _col4 (type: float), _col5 (type: tinyint), _col6 (type: smallint), _col7 (type: int), _col8 (type: bigint), _col9 (type: boolean), _col10 (type: boolean) - Execution mode: llap - LLAP IO: all inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: timestamp with local time zone), VALUE._col1 (type: string), VALUE._col2 (type: string), VALUE._col3 (type: double), VALUE._col4 (type: float), VALUE._col5 (type: tinyint), VALUE._col6 (type: smallint), VALUE._col7 (type: int), VALUE._col8 (type: bigint), VALUE._col9 (type: boolean), VALUE._col10 (type: boolean), KEY.__time_granularity (type: timestamp), KEY.__druid_extra_partition_key (type: bigint) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, __time_granularity, __druid_extra_partition_key - Statistics: Num rows: 9174 Data size: 1942340 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 9174 Data size: 1942340 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.hive.druid.io.DruidQueryBasedInputFormat - output format: org.apache.hadoop.hive.druid.io.DruidOutputFormat - serde: org.apache.hadoop.hive.druid.serde.DruidSerDe - name: default.druid_partitioned_table - - Stage: Stage-4 - Commit Insert Hook - -PREHOOK: query: INSERT INTO TABLE druid_partitioned_table -SELECT cast (`ctimestamp2` as timestamp with local time zone) as `__time`, - cstring1, - cstring2, - cdouble, - cfloat, - ctinyint, - csmallint, - cint, - cbigint, - cboolean1, - cboolean2 - FROM alltypesorc where ctimestamp2 IS NOT NULL -PREHOOK: type: QUERY -PREHOOK: Input: default@alltypesorc -PREHOOK: Output: default@druid_partitioned_table -POSTHOOK: query: INSERT INTO TABLE druid_partitioned_table -SELECT cast (`ctimestamp2` as timestamp with local time zone) as `__time`, - cstring1, - cstring2, - cdouble, - cfloat, - ctinyint, - csmallint, - cint, - cbigint, - cboolean1, - cboolean2 - FROM alltypesorc where ctimestamp2 IS NOT NULL -POSTHOOK: type: QUERY -POSTHOOK: Input: default@alltypesorc -POSTHOOK: Output: default@druid_partitioned_table -PREHOOK: query: SELECT sum(cint), sum(cbigint) FROM druid_partitioned_table -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_partitioned_table -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: SELECT sum(cint), sum(cbigint) FROM druid_partitioned_table -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_partitioned_table -POSTHOOK: Output: hdfs://### HDFS PATH ### -2857395071862 -1661313883124 -PREHOOK: query: EXPLAIN INSERT OVERWRITE TABLE druid_partitioned_table - SELECT cast (`ctimestamp1` as timestamp with local time zone) as `__time`, - cstring1, - cstring2, - cdouble, - cfloat, - ctinyint, - csmallint, - cint, - cbigint, - cboolean1, - cboolean2 - FROM alltypesorc where ctimestamp1 IS NOT NULL -PREHOOK: type: QUERY -PREHOOK: Input: default@alltypesorc -PREHOOK: Output: default@druid_partitioned_table -POSTHOOK: query: EXPLAIN INSERT OVERWRITE TABLE druid_partitioned_table - SELECT cast (`ctimestamp1` as timestamp with local time zone) as `__time`, - cstring1, - cstring2, - cdouble, - cfloat, - ctinyint, - csmallint, - cint, - cbigint, - cboolean1, - cboolean2 - FROM alltypesorc where ctimestamp1 IS NOT NULL -POSTHOOK: type: QUERY -POSTHOOK: Input: default@alltypesorc -POSTHOOK: Output: default@druid_partitioned_table -STAGE DEPENDENCIES: - Stage-0 is a root stage - Stage-1 is a root stage - Stage-2 is a root stage - Stage-4 depends on stages: Stage-2 - -STAGE PLANS: - Stage: Stage-0 - Unset Properties - table name: default.druid_partitioned_table - properties: - COLUMN_STATS_ACCURATE - - Stage: Stage-1 - Pre-Insert task - - Stage: Stage-2 - Tez -#### A masked pattern was here #### - Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) -#### A masked pattern was here #### - Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: alltypesorc - filterExpr: ctimestamp1 is not null (type: boolean) - Statistics: Num rows: 12288 Data size: 2477090 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: ctimestamp1 is not null (type: boolean) - Statistics: Num rows: 9173 Data size: 1849230 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: CAST( ctimestamp1 AS timestamp with local time zone) (type: timestamp with local time zone), cstring1 (type: string), cstring2 (type: string), cdouble (type: double), cfloat (type: float), ctinyint (type: tinyint), csmallint (type: smallint), cint (type: int), cbigint (type: bigint), cboolean1 (type: boolean), cboolean2 (type: boolean) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10 - Statistics: Num rows: 9173 Data size: 1942190 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: timestamp with local time zone), _col1 (type: string), _col2 (type: string), _col3 (type: double), _col4 (type: float), _col5 (type: tinyint), _col6 (type: smallint), _col7 (type: int), _col8 (type: bigint), _col9 (type: boolean), _col10 (type: boolean), floor_hour(CAST( GenericUDFEpochMilli(_col0) AS TIMESTAMP)) (type: timestamp), (floor((1.0D / rand())) % 6) (type: bigint) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, __time_granularity, __druid_extra_partition_key - Statistics: Num rows: 9173 Data size: 1942190 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: __time_granularity (type: timestamp), __druid_extra_partition_key (type: bigint) - null sort order: aa - sort order: ++ - Map-reduce partition columns: __time_granularity (type: timestamp), __druid_extra_partition_key (type: bigint) - Statistics: Num rows: 9173 Data size: 1942190 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: timestamp with local time zone), _col1 (type: string), _col2 (type: string), _col3 (type: double), _col4 (type: float), _col5 (type: tinyint), _col6 (type: smallint), _col7 (type: int), _col8 (type: bigint), _col9 (type: boolean), _col10 (type: boolean) - Execution mode: llap - LLAP IO: all inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: timestamp with local time zone), VALUE._col1 (type: string), VALUE._col2 (type: string), VALUE._col3 (type: double), VALUE._col4 (type: float), VALUE._col5 (type: tinyint), VALUE._col6 (type: smallint), VALUE._col7 (type: int), VALUE._col8 (type: bigint), VALUE._col9 (type: boolean), VALUE._col10 (type: boolean), KEY.__time_granularity (type: timestamp), KEY.__druid_extra_partition_key (type: bigint) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, __time_granularity, __druid_extra_partition_key - Statistics: Num rows: 9173 Data size: 1942190 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 9173 Data size: 1942190 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.hive.druid.io.DruidQueryBasedInputFormat - output format: org.apache.hadoop.hive.druid.io.DruidOutputFormat - serde: org.apache.hadoop.hive.druid.serde.DruidSerDe - name: default.druid_partitioned_table - - Stage: Stage-4 - Commit Insert Hook - is overwrite: true - -PREHOOK: query: INSERT OVERWRITE TABLE druid_partitioned_table - SELECT cast (`ctimestamp1` as timestamp with local time zone) as `__time`, - cstring1, - cstring2, - cdouble, - cfloat, - ctinyint, - csmallint, - cint, - cbigint, - cboolean1, - cboolean2 - FROM alltypesorc where ctimestamp1 IS NOT NULL -PREHOOK: type: QUERY -PREHOOK: Input: default@alltypesorc -PREHOOK: Output: default@druid_partitioned_table -POSTHOOK: query: INSERT OVERWRITE TABLE druid_partitioned_table - SELECT cast (`ctimestamp1` as timestamp with local time zone) as `__time`, - cstring1, - cstring2, - cdouble, - cfloat, - ctinyint, - csmallint, - cint, - cbigint, - cboolean1, - cboolean2 - FROM alltypesorc where ctimestamp1 IS NOT NULL -POSTHOOK: type: QUERY -POSTHOOK: Input: default@alltypesorc -POSTHOOK: Output: default@druid_partitioned_table -PREHOOK: query: SELECT sum(cint), sum(cbigint) FROM druid_partitioned_table -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_partitioned_table -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: SELECT sum(cint), sum(cbigint) FROM druid_partitioned_table -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_partitioned_table -POSTHOOK: Output: hdfs://### HDFS PATH ### -1408069801800 10992545287 -PREHOOK: query: CREATE EXTERNAL TABLE druid_max_size_partition - STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' - TBLPROPERTIES ( - "druid.segment.granularity" = "HOUR", - "druid.query.granularity" = "MINUTE" - ) - AS - SELECT cast (`ctimestamp1` as timestamp with local time zone) as `__time`, - cstring1, - cstring2, - cdouble, - cfloat, - ctinyint, - csmallint, - cint, - cbigint, - cboolean1, - cboolean2 - FROM alltypesorc where ctimestamp1 IS NOT NULL -PREHOOK: type: CREATETABLE_AS_SELECT -PREHOOK: Input: default@alltypesorc -PREHOOK: Output: database:default -PREHOOK: Output: default@druid_max_size_partition -POSTHOOK: query: CREATE EXTERNAL TABLE druid_max_size_partition - STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' - TBLPROPERTIES ( - "druid.segment.granularity" = "HOUR", - "druid.query.granularity" = "MINUTE" - ) - AS - SELECT cast (`ctimestamp1` as timestamp with local time zone) as `__time`, - cstring1, - cstring2, - cdouble, - cfloat, - ctinyint, - csmallint, - cint, - cbigint, - cboolean1, - cboolean2 - FROM alltypesorc where ctimestamp1 IS NOT NULL -POSTHOOK: type: CREATETABLE_AS_SELECT -POSTHOOK: Input: default@alltypesorc -POSTHOOK: Output: database:default -POSTHOOK: Output: default@druid_max_size_partition -POSTHOOK: Lineage: druid_max_size_partition.__time EXPRESSION [(alltypesorc)alltypesorc.FieldSchema(name:ctimestamp1, type:timestamp, comment:null), ] -POSTHOOK: Lineage: druid_max_size_partition.cbigint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cbigint, type:bigint, comment:null), ] -POSTHOOK: Lineage: druid_max_size_partition.cboolean1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cboolean1, type:boolean, comment:null), ] -POSTHOOK: Lineage: druid_max_size_partition.cboolean2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cboolean2, type:boolean, comment:null), ] -POSTHOOK: Lineage: druid_max_size_partition.cdouble SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cdouble, type:double, comment:null), ] -POSTHOOK: Lineage: druid_max_size_partition.cfloat SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cfloat, type:float, comment:null), ] -POSTHOOK: Lineage: druid_max_size_partition.cint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] -POSTHOOK: Lineage: druid_max_size_partition.csmallint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:csmallint, type:smallint, comment:null), ] -POSTHOOK: Lineage: druid_max_size_partition.cstring1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ] -POSTHOOK: Lineage: druid_max_size_partition.cstring2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring2, type:string, comment:null), ] -POSTHOOK: Lineage: druid_max_size_partition.ctinyint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctinyint, type:tinyint, comment:null), ] -PREHOOK: query: SELECT sum(cint), sum(cbigint) FROM druid_max_size_partition -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_max_size_partition -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: SELECT sum(cint), sum(cbigint) FROM druid_max_size_partition -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_max_size_partition -POSTHOOK: Output: hdfs://### HDFS PATH ### -1408069801800 10992545287 -PREHOOK: query: DROP TABLE druid_partitioned_table_0 -PREHOOK: type: DROPTABLE -PREHOOK: Input: default@druid_partitioned_table_0 -PREHOOK: Output: default@druid_partitioned_table_0 -POSTHOOK: query: DROP TABLE druid_partitioned_table_0 -POSTHOOK: type: DROPTABLE -POSTHOOK: Input: default@druid_partitioned_table_0 -POSTHOOK: Output: default@druid_partitioned_table_0 -PREHOOK: query: DROP TABLE druid_partitioned_table -PREHOOK: type: DROPTABLE -PREHOOK: Input: default@druid_partitioned_table -PREHOOK: Output: default@druid_partitioned_table -POSTHOOK: query: DROP TABLE druid_partitioned_table -POSTHOOK: type: DROPTABLE -POSTHOOK: Input: default@druid_partitioned_table -POSTHOOK: Output: default@druid_partitioned_table -PREHOOK: query: DROP TABLE druid_max_size_partition -PREHOOK: type: DROPTABLE -PREHOOK: Input: default@druid_max_size_partition -PREHOOK: Output: default@druid_max_size_partition -POSTHOOK: query: DROP TABLE druid_max_size_partition -POSTHOOK: type: DROPTABLE -POSTHOOK: Input: default@druid_max_size_partition -POSTHOOK: Output: default@druid_max_size_partition diff --git a/ql/src/test/results/clientpositive/druid/druidmini_expressions.q.out b/ql/src/test/results/clientpositive/druid/druidmini_expressions.q.out deleted file mode 100644 index e62a808e72eb..000000000000 --- a/ql/src/test/results/clientpositive/druid/druidmini_expressions.q.out +++ /dev/null @@ -1,2737 +0,0 @@ -PREHOOK: query: -- MATH AND STRING functions - -SELECT count(*) FROM druid_table_alltypesorc WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: -- MATH AND STRING functions - -SELECT count(*) FROM druid_table_alltypesorc WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -2081 -PREHOOK: query: SELECT count(*) FROM druid_table_alltypesorc WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: SELECT count(*) FROM druid_table_alltypesorc WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -7602 -PREHOOK: query: SELECT count(*) FROM druid_table_alltypesorc WHERE power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: SELECT count(*) FROM druid_table_alltypesorc WHERE power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -2081 -PREHOOK: query: SELECT SUM(cfloat + 1), CAST(SUM(cdouble + ctinyint) AS INTEGER), SUM(ctinyint) + 1 , CAST(SUM(csmallint) + SUM(cint) AS DOUBLE), SUM(cint), SUM(cbigint) -FROM druid_table_alltypesorc WHERE ceil(cfloat) > 0 AND floor(cdouble) * 2 < 1000 OR ln(cdouble) / log10(10) > 0 AND COS(cint) > 0 OR SIN(cdouble) > 1 -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: SELECT SUM(cfloat + 1), CAST(SUM(cdouble + ctinyint) AS INTEGER), SUM(ctinyint) + 1 , CAST(SUM(csmallint) + SUM(cint) AS DOUBLE), SUM(cint), SUM(cbigint) -FROM druid_table_alltypesorc WHERE ceil(cfloat) > 0 AND floor(cdouble) * 2 < 1000 OR ln(cdouble) / log10(10) > 0 AND COS(cint) > 0 OR SIN(cdouble) > 1 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -35881.09400522709 23378637 28946 7.57221208318E11 757217375155 68151649880 -PREHOOK: query: SELECT SUM(cfloat + 1), CAST(SUM(cdouble + ctinyint) AS INTEGER), SUM(ctinyint) + 1 , CAST(SUM(csmallint) + SUM(cint) AS DOUBLE), SUM(cint), SUM(cbigint) -FROM druid_table_alltypesorc WHERE ceil(cfloat) > 0 AND floor(cdouble) * 2 < 1000 -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: SELECT SUM(cfloat + 1), CAST(SUM(cdouble + ctinyint) AS INTEGER), SUM(ctinyint) + 1 , CAST(SUM(csmallint) + SUM(cint) AS DOUBLE), SUM(cint), SUM(cbigint) -FROM druid_table_alltypesorc WHERE ceil(cfloat) > 0 AND floor(cdouble) * 2 < 1000 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -78620.21200561523 -9973483 74628 3.54463845353E11 354473882701 92859255924 -PREHOOK: query: SELECT SUM(cfloat + 1), CAST(SUM(cdouble + ctinyint) AS INTEGER), SUM(ctinyint) + 1 , CAST(SUM(csmallint) + SUM(cint) AS DOUBLE), SUM(cint), SUM(cbigint) -FROM druid_table_alltypesorc WHERE ln(cdouble) / log10(10) > 0 AND COS(cint) > 0 OR SIN(cdouble) > 1 -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: SELECT SUM(cfloat + 1), CAST(SUM(cdouble + ctinyint) AS INTEGER), SUM(ctinyint) + 1 , CAST(SUM(csmallint) + SUM(cint) AS DOUBLE), SUM(cint), SUM(cbigint) -FROM druid_table_alltypesorc WHERE ln(cdouble) / log10(10) > 0 AND COS(cint) > 0 OR SIN(cdouble) > 1 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### --2195.4589961767197 27912403 -6517 5.33299976914E11 533291579903 13774723379 -PREHOOK: query: SELECT SUM(cfloat + 1), CAST(SUM(cdouble + ctinyint) AS INTEGER), SUM(ctinyint) + 1 , CAST(SUM(csmallint) + SUM(cint) AS DOUBLE), SUM(cint), SUM(cbigint) -FROM druid_table_alltypesorc WHERE SIN(cdouble) > 1 -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: SELECT SUM(cfloat + 1), CAST(SUM(cdouble + ctinyint) AS INTEGER), SUM(ctinyint) + 1 , CAST(SUM(csmallint) + SUM(cint) AS DOUBLE), SUM(cint), SUM(cbigint) -FROM druid_table_alltypesorc WHERE SIN(cdouble) > 1 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -0.0 0 1 0.0 0 0 -PREHOOK: query: SELECT cstring1 || '_'|| cstring2, substring(cstring2, 2, 3) as concat , upper(cstring2), lower(cstring1), SUM(cdouble) as s FROM druid_table_alltypesorc WHERE cstring1 IS NOT NULL AND cstring2 IS NOT NULL AND cstring2 like 'Y%' - GROUP BY cstring1 || '_'|| cstring2, substring(cstring2, 2, 3), upper(cstring2), lower(cstring1) ORDER BY concat DESC LIMIT 10 -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: SELECT cstring1 || '_'|| cstring2, substring(cstring2, 2, 3) as concat , upper(cstring2), lower(cstring1), SUM(cdouble) as s FROM druid_table_alltypesorc WHERE cstring1 IS NOT NULL AND cstring2 IS NOT NULL AND cstring2 like 'Y%' - GROUP BY cstring1 || '_'|| cstring2, substring(cstring2, 2, 3), upper(cstring2), lower(cstring1) ORDER BY concat DESC LIMIT 10 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -8Nj7qpHBTH1GUkMM1BXr2_YyROa06YMuK3C2eg85d yRO YYROA06YMUK3C2EG85D 8nj7qphbth1gukmm1bxr2 0.0 -k7rg3Vw6IpwU6_YyKI8Rb72WP5dP1BMSPoT yKI YYKI8RB72WP5DP1BMSPOT k7rg3vw6ipwu6 0.0 -b0r8g21X6I2TvvPj623IKR_YxSwHWr xSw YXSWHWR b0r8g21x6i2tvvpj623ikr 0.0 -ox4gTH52_YsjDHuPsD2 sjD YSJDHUPSD2 ox4gth52 0.0 -NEGa0N8MJ2dnn3MKAfl6u_Yr4e3n r4e YR4E3N nega0n8mj2dnn3mkafl6u 0.0 -767fOfF1Oj8fyOv6YFI16rM_YqdbA5 qdb YQDBA5 767foff1oj8fyov6yfi16rm 0.0 -kM4k0y1fqwton_YpK3CTDWEXOV pK3 YPK3CTDWEXOV km4k0y1fqwton 0.0 -TBI20Ba2YuO44754E2BM_YpB20i4 pB2 YPB20I4 tbi20ba2yuo44754e2bm 0.0 -jiqEpNs7qXo0y37_Ynnw5opXqf6BU nnw YNNW5OPXQF6BU jiqepns7qxo0y37 0.0 -TgS6dAlI2w4y_Ynh42DscA373RX27nBkft nh4 YNH42DSCA373RX27NBKFT tgs6dali2w4y 0.0 -PREHOOK: query: EXPLAIN SELECT count(*) FROM druid_table_alltypesorc WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN SELECT count(*) FROM druid_table_alltypesorc WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_table_alltypesorc - properties: - druid.fieldNames $f0 - druid.fieldTypes bigint - druid.query.json {"queryType":"timeseries","dataSource":"default.druid_table_alltypesorc","descending":false,"granularity":"all","filter":{"type":"and","fields":[{"type":"expression","expression":"(sqrt(abs(\"ctinyint\")) > 3)"},{"type":"expression","expression":"((pow(\"cfloat\",2) * pow(\"csmallint\",3)) > 1)"},{"type":"expression","expression":"(strlen(CAST(\"ctinyint\", 'STRING')) > 1)"},{"type":"expression","expression":"(strlen(CAST(\"ctinyint\", 'STRING')) < 10)"}]},"aggregations":[{"type":"count","name":"$f0"}],"intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"],"context":{"skipEmptyBuckets":false}} - druid.query.type timeseries - Select Operator - expressions: $f0 (type: bigint) - outputColumnNames: _col0 - ListSink - -PREHOOK: query: EXPLAIN SELECT SUM(cfloat + 1), CAST(SUM(cdouble + ctinyint) AS INTEGER), SUM(ctinyint) + 1 , CAST(SUM(csmallint) + SUM(cint) AS DOUBLE), SUM(cint), SUM(cbigint) - FROM druid_table_alltypesorc WHERE ceil(cfloat) > 0 AND floor(cdouble) * 2 < 1000 OR ln(cdouble) / log10(10) > 0 AND COS(cint) > 0 OR SIN(cdouble) > 1 -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN SELECT SUM(cfloat + 1), CAST(SUM(cdouble + ctinyint) AS INTEGER), SUM(ctinyint) + 1 , CAST(SUM(csmallint) + SUM(cint) AS DOUBLE), SUM(cint), SUM(cbigint) - FROM druid_table_alltypesorc WHERE ceil(cfloat) > 0 AND floor(cdouble) * 2 < 1000 OR ln(cdouble) / log10(10) > 0 AND COS(cint) > 0 OR SIN(cdouble) > 1 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_table_alltypesorc - properties: - druid.fieldNames $f0,_o__c1,_o__c2,_o__c3,$f4,$f5 - druid.fieldTypes double,int,bigint,double,bigint,bigint - druid.query.json {"queryType":"timeseries","dataSource":"default.druid_table_alltypesorc","descending":false,"granularity":"all","filter":{"type":"and","fields":[{"type":"or","fields":[{"type":"expression","expression":"((floor(\"cdouble\") * 2) < 1000)"},{"type":"expression","expression":"((log(\"cdouble\") / 1.0) > 0)"},{"type":"expression","expression":"(sin(\"cdouble\") > 1)"}]},{"type":"or","fields":[{"type":"and","fields":[{"type":"expression","expression":"(ceil(\"cfloat\") > 0)"},{"type":"expression","expression":"((floor(\"cdouble\") * 2) < 1000)"}]},{"type":"and","fields":[{"type":"expression","expression":"((log(\"cdouble\") / 1.0) > 0)"},{"type":"expression","expression":"(cos(\"cint\") > 0)"}]},{"type":"expression","expression":"(sin(\"cdouble\") > 1)"}]}]},"aggregations":[{"type":"doubleSum","name":"$f0","expression":"(\"cfloat\" + 1)"},{"type":"doubleSum","name":"$f1","expression":"(\"cdouble\" + CAST(\"ctinyint\", 'DOUBLE'))"},{"type":"longSum","name":"$f2","fieldName":"ctinyint"},{"type":"longSum","name":"$f3","fieldName":"csmallint"},{"type":"longSum","name":"$f4","fieldName":"cint"},{"type":"longSum","name":"$f5","fieldName":"cbigint"}],"postAggregations":[{"type":"expression","name":"_o__c1","expression":"CAST(\"$f1\", 'LONG')"},{"type":"expression","name":"_o__c2","expression":"(\"$f2\" + 1)"},{"type":"expression","name":"_o__c3","expression":"CAST((\"$f3\" + \"$f4\"), 'DOUBLE')"}],"intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"],"context":{"skipEmptyBuckets":false}} - druid.query.type timeseries - Select Operator - expressions: $f0 (type: double), _o__c1 (type: int), _o__c2 (type: bigint), _o__c3 (type: double), $f4 (type: bigint), $f5 (type: bigint) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - ListSink - -PREHOOK: query: EXPLAIN SELECT cstring1 || '_'|| cstring2, substring(cstring2, 2, 3) as concat , upper(cstring2), lower(cstring1), SUM(cdouble) as s FROM druid_table_alltypesorc WHERE cstring1 IS NOT NULL AND cstring2 IS NOT NULL AND cstring2 like 'Y%' - GROUP BY cstring1 || '_'|| cstring2, substring(cstring2, 2, 3), upper(cstring2), lower(cstring1) ORDER BY concat DESC LIMIT 10 -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN SELECT cstring1 || '_'|| cstring2, substring(cstring2, 2, 3) as concat , upper(cstring2), lower(cstring1), SUM(cdouble) as s FROM druid_table_alltypesorc WHERE cstring1 IS NOT NULL AND cstring2 IS NOT NULL AND cstring2 like 'Y%' - GROUP BY cstring1 || '_'|| cstring2, substring(cstring2, 2, 3), upper(cstring2), lower(cstring1) ORDER BY concat DESC LIMIT 10 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_table_alltypesorc - properties: - druid.fieldNames vc,vc0,vc1,vc2,$f4 - druid.fieldTypes string,string,string,string,double - druid.query.json {"queryType":"groupBy","dataSource":"default.druid_table_alltypesorc","granularity":"all","dimensions":[{"type":"default","dimension":"vc","outputName":"vc","outputType":"STRING"},{"type":"default","dimension":"vc0","outputName":"vc0","outputType":"STRING"},{"type":"default","dimension":"vc1","outputName":"vc1","outputType":"STRING"},{"type":"default","dimension":"vc2","outputName":"vc2","outputType":"STRING"}],"virtualColumns":[{"type":"expression","name":"vc","expression":"concat(concat(\"cstring1\",'_'),\"cstring2\")","outputType":"STRING"},{"type":"expression","name":"vc0","expression":"substring(\"cstring2\", 1, 3)","outputType":"STRING"},{"type":"expression","name":"vc1","expression":"upper(\"cstring2\")","outputType":"STRING"},{"type":"expression","name":"vc2","expression":"lower(\"cstring1\")","outputType":"STRING"}],"limitSpec":{"type":"default","limit":10,"columns":[{"dimension":"vc0","direction":"descending","dimensionOrder":"lexicographic"}]},"filter":{"type":"and","fields":[{"type":"expression","expression":"like(\"cstring2\",'Y%')"},{"type":"not","field":{"type":"selector","dimension":"cstring1","value":null}},{"type":"not","field":{"type":"selector","dimension":"cstring2","value":null}}]},"aggregations":[{"type":"doubleSum","name":"$f4","fieldName":"cdouble"}],"intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"]} - druid.query.type groupBy - Select Operator - expressions: vc (type: string), vc0 (type: string), vc1 (type: string), vc2 (type: string), $f4 (type: double) - outputColumnNames: _col0, _col1, _col2, _col3, _col4 - ListSink - -PREHOOK: query: explain extended select count(*) from (select `__time` from druid_table_alltypesorc limit 1) as src -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: explain extended select count(*) from (select `__time` from druid_table_alltypesorc limit 1) as src -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Tez -#### A masked pattern was here #### - Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) -#### A masked pattern was here #### - Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: druid_table_alltypesorc - properties: - druid.fieldNames vc - druid.fieldTypes int - druid.query.json {"queryType":"scan","dataSource":"default.druid_table_alltypesorc","intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"],"virtualColumns":[{"type":"expression","name":"vc","expression":"0","outputType":"LONG"}],"columns":["vc"],"resultFormat":"compactedList","limit":1} - druid.query.type scan - Statistics: Num rows: 9173 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - GatherStats: false - Select Operator - Statistics: Num rows: 9173 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Group By Operator - aggregations: count() - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: COMPLETE - Reduce Output Operator - bucketingVersion: 2 - null sort order: - numBuckets: -1 - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: COMPLETE - tag: -1 - value expressions: _col0 (type: bigint) - auto parallelism: false - Execution mode: vectorized, llap - LLAP IO: no inputs - Path -> Alias: - hdfs://### HDFS PATH ### [druid_table_alltypesorc] - Path -> Partition: - hdfs://### HDFS PATH ### - Partition - base file name: druid_table_alltypesorc - input format: org.apache.hadoop.hive.druid.io.DruidQueryBasedInputFormat - output format: org.apache.hadoop.hive.druid.io.DruidOutputFormat - properties: - EXTERNAL TRUE - bucket_count -1 - bucketing_version 2 - column.name.delimiter , - columns __time,cstring1,cstring2,cdouble,cfloat,ctinyint,csmallint,cint,cbigint,cboolean1,cboolean2,cintstring,cfloatstring,cdoublestring - columns.types timestamp with local time zone:string:string:double:float:tinyint:smallint:int:bigint:boolean:boolean:string:string:string - druid.datasource default.druid_table_alltypesorc - druid.fieldNames vc - druid.fieldTypes int - druid.query.granularity MINUTE - druid.query.json {"queryType":"scan","dataSource":"default.druid_table_alltypesorc","intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"],"virtualColumns":[{"type":"expression","name":"vc","expression":"0","outputType":"LONG"}],"columns":["vc"],"resultFormat":"compactedList","limit":1} - druid.query.type scan - druid.segment.granularity HOUR -#### A masked pattern was here #### - location hdfs://### HDFS PATH ### - name default.druid_table_alltypesorc - serialization.format 1 - serialization.lib org.apache.hadoop.hive.druid.serde.DruidSerDe - storage_handler org.apache.hadoop.hive.druid.DruidStorageHandler - serde: org.apache.hadoop.hive.druid.serde.DruidSerDe - - input format: org.apache.hadoop.hive.druid.io.DruidQueryBasedInputFormat - output format: org.apache.hadoop.hive.druid.io.DruidOutputFormat - properties: - EXTERNAL TRUE - bucketing_version 2 - column.name.delimiter , - columns __time,cstring1,cstring2,cdouble,cfloat,ctinyint,csmallint,cint,cbigint,cboolean1,cboolean2,cintstring,cfloatstring,cdoublestring - columns.comments - columns.types timestamp with local time zone:string:string:double:float:tinyint:smallint:int:bigint:boolean:boolean:string:string:string - druid.datasource default.druid_table_alltypesorc - druid.fieldNames vc - druid.fieldTypes int - druid.query.granularity MINUTE - druid.query.json {"queryType":"scan","dataSource":"default.druid_table_alltypesorc","intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"],"virtualColumns":[{"type":"expression","name":"vc","expression":"0","outputType":"LONG"}],"columns":["vc"],"resultFormat":"compactedList","limit":1} - druid.query.type scan - druid.segment.granularity HOUR -#### A masked pattern was here #### - location hdfs://### HDFS PATH ### - name default.druid_table_alltypesorc - serialization.format 1 - serialization.lib org.apache.hadoop.hive.druid.serde.DruidSerDe - storage_handler org.apache.hadoop.hive.druid.DruidStorageHandler - serde: org.apache.hadoop.hive.druid.serde.DruidSerDe - name: default.druid_table_alltypesorc - name: default.druid_table_alltypesorc - Truncated Path -> Alias: - /druid_table_alltypesorc [druid_table_alltypesorc] - Reducer 2 - Execution mode: vectorized, llap - Needs Tagging: false - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: COMPLETE - File Output Operator - bucketingVersion: 2 - compressed: false - GlobalTableId: 0 - directory: hdfs://### HDFS PATH ### - NumFilesPerFileSink: 1 - Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: COMPLETE - Stats Publishing Key Prefix: hdfs://### HDFS PATH ### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - bucketing_version -1 - columns _col0 - columns.types bigint - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: explain -SELECT `__time` -FROM druid_table_alltypesorc -WHERE (`__time` BETWEEN '1968-01-01 00:00:00' AND '1970-01-01 00:00:00') - OR (`__time` BETWEEN '1968-02-01 00:00:00' AND '1970-04-01 00:00:00') ORDER BY `__time` ASC LIMIT 10 -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: explain -SELECT `__time` -FROM druid_table_alltypesorc -WHERE (`__time` BETWEEN '1968-01-01 00:00:00' AND '1970-01-01 00:00:00') - OR (`__time` BETWEEN '1968-02-01 00:00:00' AND '1970-04-01 00:00:00') ORDER BY `__time` ASC LIMIT 10 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Tez -#### A masked pattern was here #### - Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) -#### A masked pattern was here #### - Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: druid_table_alltypesorc - properties: - druid.fieldNames vc - druid.fieldTypes timestamp with local time zone - druid.query.json {"queryType":"scan","dataSource":"default.druid_table_alltypesorc","intervals":["1968-01-01T08:00:00.000Z/1970-04-01T08:00:00.001Z"],"virtualColumns":[{"type":"expression","name":"vc","expression":"\"__time\"","outputType":"LONG"}],"columns":["vc"],"resultFormat":"compactedList"} - druid.query.type scan - Statistics: Num rows: 9173 Data size: 348640 Basic stats: COMPLETE Column stats: NONE - Top N Key Operator - sort order: + - keys: vc (type: timestamp with local time zone) - null sort order: z - Statistics: Num rows: 9173 Data size: 348640 Basic stats: COMPLETE Column stats: NONE - top n: 10 - Select Operator - expressions: vc (type: timestamp with local time zone) - outputColumnNames: _col0 - Statistics: Num rows: 9173 Data size: 348640 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: timestamp with local time zone) - null sort order: z - sort order: + - Statistics: Num rows: 9173 Data size: 348640 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: timestamp with local time zone) - outputColumnNames: _col0 - Statistics: Num rows: 9173 Data size: 348640 Basic stats: COMPLETE Column stats: NONE - Limit - Number of rows: 10 - Statistics: Num rows: 10 Data size: 380 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 10 Data size: 380 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: 10 - Processor Tree: - ListSink - -PREHOOK: query: SELECT `__time` -FROM druid_table_alltypesorc -WHERE (`__time` BETWEEN '1968-01-01 00:00:00' AND '1970-01-01 00:00:00') - OR (`__time` BETWEEN '1968-02-01 00:00:00' AND '1970-04-01 00:00:00') ORDER BY `__time` ASC LIMIT 10 -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: SELECT `__time` -FROM druid_table_alltypesorc -WHERE (`__time` BETWEEN '1968-01-01 00:00:00' AND '1970-01-01 00:00:00') - OR (`__time` BETWEEN '1968-02-01 00:00:00' AND '1970-04-01 00:00:00') ORDER BY `__time` ASC LIMIT 10 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -PREHOOK: query: explain -SELECT `__time` -FROM druid_table_alltypesorc -WHERE ('1968-01-01 00:00:00' <= `__time` AND `__time` <= '1970-01-01 00:00:00') - OR ('1968-02-01 00:00:00' <= `__time` AND `__time` <= '1970-04-01 00:00:00') ORDER BY `__time` ASC LIMIT 10 -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: explain -SELECT `__time` -FROM druid_table_alltypesorc -WHERE ('1968-01-01 00:00:00' <= `__time` AND `__time` <= '1970-01-01 00:00:00') - OR ('1968-02-01 00:00:00' <= `__time` AND `__time` <= '1970-04-01 00:00:00') ORDER BY `__time` ASC LIMIT 10 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Tez -#### A masked pattern was here #### - Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) -#### A masked pattern was here #### - Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: druid_table_alltypesorc - properties: - druid.fieldNames vc - druid.fieldTypes timestamp with local time zone - druid.query.json {"queryType":"scan","dataSource":"default.druid_table_alltypesorc","intervals":["1968-01-01T08:00:00.000Z/1970-04-01T08:00:00.001Z"],"virtualColumns":[{"type":"expression","name":"vc","expression":"\"__time\"","outputType":"LONG"}],"columns":["vc"],"resultFormat":"compactedList"} - druid.query.type scan - Statistics: Num rows: 9173 Data size: 348640 Basic stats: COMPLETE Column stats: NONE - Top N Key Operator - sort order: + - keys: vc (type: timestamp with local time zone) - null sort order: z - Statistics: Num rows: 9173 Data size: 348640 Basic stats: COMPLETE Column stats: NONE - top n: 10 - Select Operator - expressions: vc (type: timestamp with local time zone) - outputColumnNames: _col0 - Statistics: Num rows: 9173 Data size: 348640 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: timestamp with local time zone) - null sort order: z - sort order: + - Statistics: Num rows: 9173 Data size: 348640 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: timestamp with local time zone) - outputColumnNames: _col0 - Statistics: Num rows: 9173 Data size: 348640 Basic stats: COMPLETE Column stats: NONE - Limit - Number of rows: 10 - Statistics: Num rows: 10 Data size: 380 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 10 Data size: 380 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: 10 - Processor Tree: - ListSink - -PREHOOK: query: SELECT `__time` -FROM druid_table_alltypesorc -WHERE ('1968-01-01 00:00:00' <= `__time` AND `__time` <= '1970-01-01 00:00:00') - OR ('1968-02-01 00:00:00' <= `__time` AND `__time` <= '1970-04-01 00:00:00') ORDER BY `__time` ASC LIMIT 10 -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: SELECT `__time` -FROM druid_table_alltypesorc -WHERE ('1968-01-01 00:00:00' <= `__time` AND `__time` <= '1970-01-01 00:00:00') - OR ('1968-02-01 00:00:00' <= `__time` AND `__time` <= '1970-04-01 00:00:00') ORDER BY `__time` ASC LIMIT 10 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -PREHOOK: query: EXPLAIN select count(DISTINCT cstring2), sum(cdouble) FROM druid_table_alltypesorc GROUP BY `__time`, `cstring1` -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN select count(DISTINCT cstring2), sum(cdouble) FROM druid_table_alltypesorc GROUP BY `__time`, `cstring1` -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Tez -#### A masked pattern was here #### - Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) -#### A masked pattern was here #### - Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: druid_table_alltypesorc - properties: - druid.fieldNames extract,cstring1,cstring2,$f3 - druid.fieldTypes timestamp with local time zone,string,string,double - druid.query.json {"queryType":"groupBy","dataSource":"default.druid_table_alltypesorc","granularity":"all","dimensions":[{"type":"extraction","dimension":"__time","outputName":"extract","extractionFn":{"type":"timeFormat","format":"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'","timeZone":"UTC"}},{"type":"default","dimension":"cstring1","outputName":"cstring1","outputType":"STRING"},{"type":"default","dimension":"cstring2","outputName":"cstring2","outputType":"STRING"}],"limitSpec":{"type":"default"},"aggregations":[{"type":"doubleSum","name":"$f3","fieldName":"cdouble"}],"intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"]} - druid.query.type groupBy - Statistics: Num rows: 9173 Data size: 3625856 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(cstring2), sum($f3) - keys: extract (type: timestamp with local time zone), cstring1 (type: string) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 9173 Data size: 3625856 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: timestamp with local time zone), _col1 (type: string) - null sort order: zz - sort order: ++ - Map-reduce partition columns: _col0 (type: timestamp with local time zone), _col1 (type: string) - Statistics: Num rows: 9173 Data size: 3625856 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: bigint), _col3 (type: double) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0), sum(VALUE._col1) - keys: KEY._col0 (type: timestamp with local time zone), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 4586 Data size: 1812730 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col2 (type: bigint), _col3 (type: double) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 4586 Data size: 1812730 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 4586 Data size: 1812730 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: EXPLAIN select count(distinct cdouble), sum(cdouble) FROM druid_table_alltypesorc GROUP BY `__time`, `cstring1` -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN select count(distinct cdouble), sum(cdouble) FROM druid_table_alltypesorc GROUP BY `__time`, `cstring1` -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Tez -#### A masked pattern was here #### - Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) -#### A masked pattern was here #### - Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: druid_table_alltypesorc - properties: - druid.fieldNames extract,cstring1,cdouble,$f3 - druid.fieldTypes timestamp with local time zone,string,double,double - druid.query.json {"queryType":"groupBy","dataSource":"default.druid_table_alltypesorc","granularity":"all","dimensions":[{"type":"extraction","dimension":"__time","outputName":"extract","extractionFn":{"type":"timeFormat","format":"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'","timeZone":"UTC"}},{"type":"default","dimension":"cstring1","outputName":"cstring1","outputType":"STRING"},{"type":"default","dimension":"cdouble","outputName":"cdouble","outputType":"DOUBLE"}],"limitSpec":{"type":"default"},"aggregations":[{"type":"doubleSum","name":"$f3","fieldName":"cdouble"}],"intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"]} - druid.query.type groupBy - Statistics: Num rows: 9173 Data size: 2091840 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(cdouble), sum($f3) - keys: extract (type: timestamp with local time zone), cstring1 (type: string) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 9173 Data size: 2091840 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: timestamp with local time zone), _col1 (type: string) - null sort order: zz - sort order: ++ - Map-reduce partition columns: _col0 (type: timestamp with local time zone), _col1 (type: string) - Statistics: Num rows: 9173 Data size: 2091840 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: bigint), _col3 (type: double) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0), sum(VALUE._col1) - keys: KEY._col0 (type: timestamp with local time zone), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 4586 Data size: 1045805 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col2 (type: bigint), _col3 (type: double) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 4586 Data size: 1045805 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 4586 Data size: 1045805 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: EXPLAIN select count(distinct cstring2), sum(2 * cdouble) FROM druid_table_alltypesorc GROUP BY `__time`, `cstring1` -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN select count(distinct cstring2), sum(2 * cdouble) FROM druid_table_alltypesorc GROUP BY `__time`, `cstring1` -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Tez -#### A masked pattern was here #### - Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) -#### A masked pattern was here #### - Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: druid_table_alltypesorc - properties: - druid.fieldNames extract,cstring1,cstring2,$f3 - druid.fieldTypes timestamp with local time zone,string,string,double - druid.query.json {"queryType":"groupBy","dataSource":"default.druid_table_alltypesorc","granularity":"all","dimensions":[{"type":"extraction","dimension":"__time","outputName":"extract","extractionFn":{"type":"timeFormat","format":"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'","timeZone":"UTC"}},{"type":"default","dimension":"cstring1","outputName":"cstring1","outputType":"STRING"},{"type":"default","dimension":"cstring2","outputName":"cstring2","outputType":"STRING"}],"limitSpec":{"type":"default"},"aggregations":[{"type":"doubleSum","name":"$f3","expression":"(2 * \"cdouble\")"}],"intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"]} - druid.query.type groupBy - Statistics: Num rows: 9173 Data size: 3625856 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(cstring2), sum($f3) - keys: extract (type: timestamp with local time zone), cstring1 (type: string) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 9173 Data size: 3625856 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: timestamp with local time zone), _col1 (type: string) - null sort order: zz - sort order: ++ - Map-reduce partition columns: _col0 (type: timestamp with local time zone), _col1 (type: string) - Statistics: Num rows: 9173 Data size: 3625856 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: bigint), _col3 (type: double) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0), sum(VALUE._col1) - keys: KEY._col0 (type: timestamp with local time zone), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 4586 Data size: 1812730 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col2 (type: bigint), _col3 (type: double) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 4586 Data size: 1812730 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 4586 Data size: 1812730 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: EXPLAIN select count(distinct cstring2 || '_'|| cstring1), sum(cdouble) FROM druid_table_alltypesorc GROUP BY `__time`, `cstring1` -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN select count(distinct cstring2 || '_'|| cstring1), sum(cdouble) FROM druid_table_alltypesorc GROUP BY `__time`, `cstring1` -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Tez -#### A masked pattern was here #### - Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) -#### A masked pattern was here #### - Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: druid_table_alltypesorc - properties: - druid.fieldNames extract,cstring1,vc,$f3 - druid.fieldTypes timestamp with local time zone,string,string,double - druid.query.json {"queryType":"groupBy","dataSource":"default.druid_table_alltypesorc","granularity":"all","dimensions":[{"type":"extraction","dimension":"__time","outputName":"extract","extractionFn":{"type":"timeFormat","format":"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'","timeZone":"UTC"}},{"type":"default","dimension":"cstring1","outputName":"cstring1","outputType":"STRING"},{"type":"default","dimension":"vc","outputName":"vc","outputType":"STRING"}],"virtualColumns":[{"type":"expression","name":"vc","expression":"concat(concat(\"cstring2\",'_'),\"cstring1\")","outputType":"STRING"}],"limitSpec":{"type":"default"},"aggregations":[{"type":"doubleSum","name":"$f3","fieldName":"cdouble"}],"intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"]} - druid.query.type groupBy - Statistics: Num rows: 9173 Data size: 3625856 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(vc), sum($f3) - keys: extract (type: timestamp with local time zone), cstring1 (type: string) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 9173 Data size: 3625856 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: timestamp with local time zone), _col1 (type: string) - null sort order: zz - sort order: ++ - Map-reduce partition columns: _col0 (type: timestamp with local time zone), _col1 (type: string) - Statistics: Num rows: 9173 Data size: 3625856 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: bigint), _col3 (type: double) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0), sum(VALUE._col1) - keys: KEY._col0 (type: timestamp with local time zone), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 4586 Data size: 1812730 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col2 (type: bigint), _col3 (type: double) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 4586 Data size: 1812730 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 4586 Data size: 1812730 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: EXPLAIN select count(DISTINCT cstring2) FROM druid_table_alltypesorc -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN select count(DISTINCT cstring2) FROM druid_table_alltypesorc -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Tez -#### A masked pattern was here #### - Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) -#### A masked pattern was here #### - Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: druid_table_alltypesorc - properties: - druid.fieldNames cstring2 - druid.fieldTypes string - druid.query.json {"queryType":"groupBy","dataSource":"default.druid_table_alltypesorc","granularity":"all","dimensions":[{"type":"default","dimension":"cstring2","outputName":"cstring2","outputType":"STRING"}],"limitSpec":{"type":"default"},"aggregations":[],"intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"]} - druid.query.type groupBy - Statistics: Num rows: 9173 Data size: 1603744 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(cstring2) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Execution mode: vectorized, llap - LLAP IO: no inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: EXPLAIN select count(DISTINCT cstring2), sum(cdouble) FROM druid_table_alltypesorc -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN select count(DISTINCT cstring2), sum(cdouble) FROM druid_table_alltypesorc -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Tez -#### A masked pattern was here #### - Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) -#### A masked pattern was here #### - Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: druid_table_alltypesorc - properties: - druid.fieldNames cstring2,$f1 - druid.fieldTypes string,double - druid.query.json {"queryType":"groupBy","dataSource":"default.druid_table_alltypesorc","granularity":"all","dimensions":[{"type":"default","dimension":"cstring2","outputName":"cstring2","outputType":"STRING"}],"limitSpec":{"type":"default"},"aggregations":[{"type":"doubleSum","name":"$f1","fieldName":"cdouble"}],"intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"]} - druid.query.type groupBy - Statistics: Num rows: 9173 Data size: 1673472 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(cstring2), sum($f1) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 208 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 208 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint), _col1 (type: double) - Execution mode: vectorized, llap - LLAP IO: no inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0), sum(VALUE._col1) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 208 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 208 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: EXPLAIN select count(distinct cstring2 || '_'|| cstring1), sum(cdouble), min(cint) FROM druid_table_alltypesorc -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN select count(distinct cstring2 || '_'|| cstring1), sum(cdouble), min(cint) FROM druid_table_alltypesorc -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Tez -#### A masked pattern was here #### - Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) -#### A masked pattern was here #### - Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: druid_table_alltypesorc - properties: - druid.fieldNames vc,$f1,$f2 - druid.fieldTypes string,double,int - druid.query.json {"queryType":"groupBy","dataSource":"default.druid_table_alltypesorc","granularity":"all","dimensions":[{"type":"default","dimension":"vc","outputName":"vc","outputType":"STRING"}],"virtualColumns":[{"type":"expression","name":"vc","expression":"concat(concat(\"cstring2\",'_'),\"cstring1\")","outputType":"STRING"}],"limitSpec":{"type":"default"},"aggregations":[{"type":"doubleSum","name":"$f1","fieldName":"cdouble"},{"type":"longMin","name":"$f2","fieldName":"cint"}],"intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"]} - druid.query.type groupBy - Statistics: Num rows: 9173 Data size: 1708336 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(vc), sum($f1), min($f2) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint), _col1 (type: double), _col2 (type: int) - Execution mode: vectorized, llap - LLAP IO: no inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0), sum(VALUE._col1), min(VALUE._col2) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: EXPLAIN select count(*) from (select `__time` from druid_table_alltypesorc limit 1025) as src -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN select count(*) from (select `__time` from druid_table_alltypesorc limit 1025) as src -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Tez -#### A masked pattern was here #### - Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) -#### A masked pattern was here #### - Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: druid_table_alltypesorc - properties: - druid.fieldNames vc - druid.fieldTypes int - druid.query.json {"queryType":"scan","dataSource":"default.druid_table_alltypesorc","intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"],"virtualColumns":[{"type":"expression","name":"vc","expression":"0","outputType":"LONG"}],"columns":["vc"],"resultFormat":"compactedList","limit":1025} - druid.query.type scan - Statistics: Num rows: 9173 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Select Operator - Statistics: Num rows: 9173 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Group By Operator - aggregations: count() - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: COMPLETE - value expressions: _col0 (type: bigint) - Execution mode: vectorized, llap - LLAP IO: no inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: select count(DISTINCT cstring2), sum(cdouble) FROM druid_table_alltypesorc GROUP BY floor_year(`__time`) -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: select count(DISTINCT cstring2), sum(cdouble) FROM druid_table_alltypesorc GROUP BY floor_year(`__time`) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -6078 2.73086627938E7 -PREHOOK: query: select count(distinct cstring2), sum(2 * cdouble) FROM druid_table_alltypesorc GROUP BY floor_year(`__time`) -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: select count(distinct cstring2), sum(2 * cdouble) FROM druid_table_alltypesorc GROUP BY floor_year(`__time`) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -6078 5.46173255876E7 -PREHOOK: query: select count(DISTINCT cstring2) FROM druid_table_alltypesorc -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: select count(DISTINCT cstring2) FROM druid_table_alltypesorc -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -6078 -PREHOOK: query: explain select count(DISTINCT cstring1) FROM druid_table_alltypesorc -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: explain select count(DISTINCT cstring1) FROM druid_table_alltypesorc -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Tez -#### A masked pattern was here #### - Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) -#### A masked pattern was here #### - Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: druid_table_alltypesorc - properties: - druid.fieldNames cstring1 - druid.fieldTypes string - druid.query.json {"queryType":"groupBy","dataSource":"default.druid_table_alltypesorc","granularity":"all","dimensions":[{"type":"default","dimension":"cstring1","outputName":"cstring1","outputType":"STRING"}],"limitSpec":{"type":"default"},"aggregations":[],"intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"]} - druid.query.type groupBy - Statistics: Num rows: 9173 Data size: 1603744 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(cstring1) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Execution mode: vectorized, llap - LLAP IO: no inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: select count(DISTINCT cstring1) FROM druid_table_alltypesorc -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: select count(DISTINCT cstring1) FROM druid_table_alltypesorc -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -3025 -PREHOOK: query: select count(DISTINCT cstring2), sum(cdouble) FROM druid_table_alltypesorc -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: select count(DISTINCT cstring2), sum(cdouble) FROM druid_table_alltypesorc -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -6078 2.73086627938E7 -PREHOOK: query: select count(distinct cstring2 || '_'|| cstring1), sum(cdouble), min(cint) FROM druid_table_alltypesorc -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: select count(distinct cstring2 || '_'|| cstring1), sum(cdouble), min(cint) FROM druid_table_alltypesorc -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -6095 2.7308662793799996E7 -1073279343 -PREHOOK: query: select count(*) from (select `__time` from druid_table_alltypesorc limit 1025) as src -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: select count(*) from (select `__time` from druid_table_alltypesorc limit 1025) as src -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -1025 -PREHOOK: query: select count(*) from (select `__time` from druid_table_alltypesorc limit 200000) as src -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: select count(*) from (select `__time` from druid_table_alltypesorc limit 200000) as src -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -9173 -PREHOOK: query: select count(`__time`) from (select `__time` from druid_table_alltypesorc limit 200) as src -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: select count(`__time`) from (select `__time` from druid_table_alltypesorc limit 200) as src -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -200 -PREHOOK: query: select count(distinct `__time`) from druid_table_alltypesorc -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: select count(distinct `__time`) from druid_table_alltypesorc -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -2 -PREHOOK: query: select count(distinct `ctimestamp1`) from alltypesorc1 where ctimestamp1 IS NOT NULL -PREHOOK: type: QUERY -PREHOOK: Input: default@alltypesorc1 -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: select count(distinct `ctimestamp1`) from alltypesorc1 where ctimestamp1 IS NOT NULL -POSTHOOK: type: QUERY -POSTHOOK: Input: default@alltypesorc1 -POSTHOOK: Output: hdfs://### HDFS PATH ### -5523 -PREHOOK: query: explain select `timets` from (select `__time` as timets from druid_table_alltypesorc order by timets limit 10) as src order by `timets` -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: explain select `timets` from (select `__time` as timets from druid_table_alltypesorc order by timets limit 10) as src order by `timets` -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Tez -#### A masked pattern was here #### - Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) -#### A masked pattern was here #### - Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: druid_table_alltypesorc - properties: - druid.fieldNames vc - druid.fieldTypes timestamp with local time zone - druid.query.json {"queryType":"scan","dataSource":"default.druid_table_alltypesorc","intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"],"virtualColumns":[{"type":"expression","name":"vc","expression":"\"__time\"","outputType":"LONG"}],"columns":["vc"],"resultFormat":"compactedList"} - druid.query.type scan - Statistics: Num rows: 9173 Data size: 348640 Basic stats: COMPLETE Column stats: NONE - Top N Key Operator - sort order: + - keys: vc (type: timestamp with local time zone) - null sort order: z - Statistics: Num rows: 9173 Data size: 348640 Basic stats: COMPLETE Column stats: NONE - top n: 10 - Select Operator - expressions: vc (type: timestamp with local time zone) - outputColumnNames: _col0 - Statistics: Num rows: 9173 Data size: 348640 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: timestamp with local time zone) - null sort order: z - sort order: + - Statistics: Num rows: 9173 Data size: 348640 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: timestamp with local time zone) - outputColumnNames: _col0 - Statistics: Num rows: 9173 Data size: 348640 Basic stats: COMPLETE Column stats: NONE - Limit - Number of rows: 10 - Statistics: Num rows: 10 Data size: 380 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: timestamp with local time zone) - null sort order: z - sort order: + - Statistics: Num rows: 10 Data size: 380 Basic stats: COMPLETE Column stats: NONE - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: timestamp with local time zone) - outputColumnNames: _col0 - Statistics: Num rows: 10 Data size: 380 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 10 Data size: 380 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: explain select `timets` from (select cast(`__time` as timestamp ) as timets from druid_table_alltypesorc order by timets limit 10) as src order by `timets` -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: explain select `timets` from (select cast(`__time` as timestamp ) as timets from druid_table_alltypesorc order by timets limit 10) as src order by `timets` -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Tez -#### A masked pattern was here #### - Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) -#### A masked pattern was here #### - Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: druid_table_alltypesorc - properties: - druid.fieldNames vc - druid.fieldTypes timestamp - druid.query.json {"queryType":"scan","dataSource":"default.druid_table_alltypesorc","intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"],"virtualColumns":[{"type":"expression","name":"vc","expression":"timestamp_parse(timestamp_format(\"__time\",'yyyy-MM-dd\\u0027T\\u0027HH:mm:ss.SSS\\u0027Z\\u0027','US/Pacific'),'yyyy-MM-dd\\u0027T\\u0027HH:mm:ss.SSS\\u0027Z\\u0027','UTC')","outputType":"LONG"}],"columns":["vc"],"resultFormat":"compactedList"} - druid.query.type scan - Statistics: Num rows: 9173 Data size: 348640 Basic stats: COMPLETE Column stats: NONE - Top N Key Operator - sort order: + - keys: vc (type: timestamp) - null sort order: z - Statistics: Num rows: 9173 Data size: 348640 Basic stats: COMPLETE Column stats: NONE - top n: 10 - Select Operator - expressions: vc (type: timestamp) - outputColumnNames: _col0 - Statistics: Num rows: 9173 Data size: 348640 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: timestamp) - null sort order: z - sort order: + - Statistics: Num rows: 9173 Data size: 348640 Basic stats: COMPLETE Column stats: NONE - Execution mode: vectorized, llap - LLAP IO: no inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: timestamp) - outputColumnNames: _col0 - Statistics: Num rows: 9173 Data size: 348640 Basic stats: COMPLETE Column stats: NONE - Limit - Number of rows: 10 - Statistics: Num rows: 10 Data size: 380 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: timestamp) - null sort order: z - sort order: + - Statistics: Num rows: 10 Data size: 380 Basic stats: COMPLETE Column stats: NONE - Reducer 3 - Execution mode: vectorized, llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: timestamp) - outputColumnNames: _col0 - Statistics: Num rows: 10 Data size: 380 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 10 Data size: 380 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: select `timets_with_tz` from (select `__time` as timets_with_tz from druid_table_alltypesorc order by timets_with_tz limit 10) as src order by `timets_with_tz` -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: select `timets_with_tz` from (select `__time` as timets_with_tz from druid_table_alltypesorc order by timets_with_tz limit 10) as src order by `timets_with_tz` -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -PREHOOK: query: select `timets` from (select cast(`__time` as timestamp ) as timets from druid_table_alltypesorc order by timets limit 10) as src order by `timets` -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: select `timets` from (select cast(`__time` as timestamp ) as timets from druid_table_alltypesorc order by timets limit 10) as src order by `timets` -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -1969-12-31 15:59:00 -1969-12-31 15:59:00 -1969-12-31 15:59:00 -1969-12-31 15:59:00 -1969-12-31 15:59:00 -1969-12-31 15:59:00 -1969-12-31 15:59:00 -1969-12-31 15:59:00 -1969-12-31 15:59:00 -1969-12-31 15:59:00 -PREHOOK: query: explain select unix_timestamp(from_unixtime(1396681200)) from druid_table_alltypesorc limit 1 -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: explain select unix_timestamp(from_unixtime(1396681200)) from druid_table_alltypesorc limit 1 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_table_alltypesorc - properties: - druid.fieldNames vc - druid.fieldTypes bigint - druid.query.json {"queryType":"scan","dataSource":"default.druid_table_alltypesorc","intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"],"virtualColumns":[{"type":"expression","name":"vc","expression":"unix_timestamp(timestamp_format((1396681200 * '1000'),'yyyy-MM-dd HH:mm:ss','UTC'),'yyyy-MM-dd HH:mm:ss')","outputType":"LONG"}],"columns":["vc"],"resultFormat":"compactedList","limit":1} - druid.query.type scan - Select Operator - expressions: vc (type: bigint) - outputColumnNames: _col0 - ListSink - -PREHOOK: query: select unix_timestamp(from_unixtime(1396681200)) from druid_table_alltypesorc limit 1 -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: select unix_timestamp(from_unixtime(1396681200)) from druid_table_alltypesorc limit 1 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -1396681200 -PREHOOK: query: explain select unix_timestamp(`__time`) from druid_table_alltypesorc limit 1 -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: explain select unix_timestamp(`__time`) from druid_table_alltypesorc limit 1 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_table_alltypesorc - properties: - druid.fieldNames vc - druid.fieldTypes bigint - druid.query.json {"queryType":"scan","dataSource":"default.druid_table_alltypesorc","intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"],"virtualColumns":[{"type":"expression","name":"vc","expression":"div(\"__time\",1000)","outputType":"LONG"}],"columns":["vc"],"resultFormat":"compactedList","limit":1} - druid.query.type scan - Select Operator - expressions: vc (type: bigint) - outputColumnNames: _col0 - ListSink - -PREHOOK: query: select unix_timestamp(`__time`) from druid_table_alltypesorc limit 1 -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: select unix_timestamp(`__time`) from druid_table_alltypesorc limit 1 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### --60 -PREHOOK: query: explain select FROM_UNIXTIME(UNIX_TIMESTAMP(CAST(`__time` as timestamp ),'yyyy-MM-dd HH:mm:ss' ),'yyyy-MM-dd HH:mm:ss') -from druid_table_alltypesorc -GROUP BY FROM_UNIXTIME(UNIX_TIMESTAMP(CAST(`__time` as timestamp ),'yyyy-MM-dd HH:mm:ss' ),'yyyy-MM-dd HH:mm:ss') -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: explain select FROM_UNIXTIME(UNIX_TIMESTAMP(CAST(`__time` as timestamp ),'yyyy-MM-dd HH:mm:ss' ),'yyyy-MM-dd HH:mm:ss') -from druid_table_alltypesorc -GROUP BY FROM_UNIXTIME(UNIX_TIMESTAMP(CAST(`__time` as timestamp ),'yyyy-MM-dd HH:mm:ss' ),'yyyy-MM-dd HH:mm:ss') -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_table_alltypesorc - properties: - druid.fieldNames vc - druid.fieldTypes string - druid.query.json {"queryType":"groupBy","dataSource":"default.druid_table_alltypesorc","granularity":"all","dimensions":[{"type":"default","dimension":"vc","outputName":"vc","outputType":"STRING"}],"virtualColumns":[{"type":"expression","name":"vc","expression":"timestamp_format((div(timestamp_parse(timestamp_format(\"__time\",'yyyy-MM-dd\\u0027T\\u0027HH:mm:ss.SSS\\u0027Z\\u0027','US/Pacific'),'yyyy-MM-dd\\u0027T\\u0027HH:mm:ss.SSS\\u0027Z\\u0027','UTC'),1000) * '1000'),'yyyy-MM-dd HH:mm:ss','UTC')","outputType":"STRING"}],"limitSpec":{"type":"default"},"aggregations":[],"intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"]} - druid.query.type groupBy - Select Operator - expressions: vc (type: string) - outputColumnNames: _col0 - ListSink - -PREHOOK: query: select FROM_UNIXTIME(UNIX_TIMESTAMP (CAST(`__time` as timestamp ),'yyyy-MM-dd HH:mm:ss' ),'yyyy-MM-dd HH:mm:ss') -from druid_table_alltypesorc -GROUP BY FROM_UNIXTIME(UNIX_TIMESTAMP(CAST(`__time` as timestamp ),'yyyy-MM-dd HH:mm:ss' ),'yyyy-MM-dd HH:mm:ss') -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: select FROM_UNIXTIME(UNIX_TIMESTAMP (CAST(`__time` as timestamp ),'yyyy-MM-dd HH:mm:ss' ),'yyyy-MM-dd HH:mm:ss') -from druid_table_alltypesorc -GROUP BY FROM_UNIXTIME(UNIX_TIMESTAMP(CAST(`__time` as timestamp ),'yyyy-MM-dd HH:mm:ss' ),'yyyy-MM-dd HH:mm:ss') -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -1969-12-31 15:59:00 -1969-12-31 16:00:00 -PREHOOK: query: explain select TRUNC(cast(`__time` as timestamp), 'YY') from druid_table_alltypesorc GROUP BY TRUNC(cast(`__time` as timestamp), 'YY') -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: explain select TRUNC(cast(`__time` as timestamp), 'YY') from druid_table_alltypesorc GROUP BY TRUNC(cast(`__time` as timestamp), 'YY') -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_table_alltypesorc - properties: - druid.fieldNames vc - druid.fieldTypes string - druid.query.json {"queryType":"groupBy","dataSource":"default.druid_table_alltypesorc","granularity":"all","dimensions":[{"type":"default","dimension":"vc","outputName":"vc","outputType":"STRING"}],"virtualColumns":[{"type":"expression","name":"vc","expression":"timestamp_format(timestamp_floor(timestamp_parse(timestamp_format(\"__time\",'yyyy-MM-dd\\u0027T\\u0027HH:mm:ss.SSS\\u0027Z\\u0027','US/Pacific'),'yyyy-MM-dd\\u0027T\\u0027HH:mm:ss.SSS\\u0027Z\\u0027','UTC'),'P1Y','','UTC'),'yyyy-MM-dd','UTC')","outputType":"STRING"}],"limitSpec":{"type":"default"},"aggregations":[],"intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"]} - druid.query.type groupBy - Select Operator - expressions: vc (type: string) - outputColumnNames: _col0 - ListSink - -PREHOOK: query: select TRUNC(cast(`__time` as timestamp), 'YY') from druid_table_alltypesorc GROUP BY TRUNC(cast(`__time` as timestamp), 'YY') -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: select TRUNC(cast(`__time` as timestamp), 'YY') from druid_table_alltypesorc GROUP BY TRUNC(cast(`__time` as timestamp), 'YY') -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -1969-01-01 -PREHOOK: query: select TRUNC(cast(`__time` as timestamp), 'YEAR') from druid_table_alltypesorc GROUP BY TRUNC(cast(`__time` as timestamp), 'YEAR') -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: select TRUNC(cast(`__time` as timestamp), 'YEAR') from druid_table_alltypesorc GROUP BY TRUNC(cast(`__time` as timestamp), 'YEAR') -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -1969-01-01 -PREHOOK: query: select TRUNC(cast(`__time` as timestamp), 'YYYY') from druid_table_alltypesorc GROUP BY TRUNC(cast(`__time` as timestamp), 'YYYY') -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: select TRUNC(cast(`__time` as timestamp), 'YYYY') from druid_table_alltypesorc GROUP BY TRUNC(cast(`__time` as timestamp), 'YYYY') -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -1969-01-01 -PREHOOK: query: explain select TRUNC(cast(`__time` as timestamp), 'MONTH') from druid_table_alltypesorc GROUP BY TRUNC(cast(`__time` as timestamp), 'MONTH') -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: explain select TRUNC(cast(`__time` as timestamp), 'MONTH') from druid_table_alltypesorc GROUP BY TRUNC(cast(`__time` as timestamp), 'MONTH') -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_table_alltypesorc - properties: - druid.fieldNames vc - druid.fieldTypes string - druid.query.json {"queryType":"groupBy","dataSource":"default.druid_table_alltypesorc","granularity":"all","dimensions":[{"type":"default","dimension":"vc","outputName":"vc","outputType":"STRING"}],"virtualColumns":[{"type":"expression","name":"vc","expression":"timestamp_format(timestamp_floor(timestamp_parse(timestamp_format(\"__time\",'yyyy-MM-dd\\u0027T\\u0027HH:mm:ss.SSS\\u0027Z\\u0027','US/Pacific'),'yyyy-MM-dd\\u0027T\\u0027HH:mm:ss.SSS\\u0027Z\\u0027','UTC'),'P1M','','UTC'),'yyyy-MM-dd','UTC')","outputType":"STRING"}],"limitSpec":{"type":"default"},"aggregations":[],"intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"]} - druid.query.type groupBy - Select Operator - expressions: vc (type: string) - outputColumnNames: _col0 - ListSink - -PREHOOK: query: select TRUNC(cast(`__time` as timestamp), 'MONTH') from druid_table_alltypesorc GROUP BY TRUNC(cast(`__time` as timestamp), 'MONTH') -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: select TRUNC(cast(`__time` as timestamp), 'MONTH') from druid_table_alltypesorc GROUP BY TRUNC(cast(`__time` as timestamp), 'MONTH') -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -1969-12-01 -PREHOOK: query: select TRUNC(cast(`__time` as timestamp), 'MM') from druid_table_alltypesorc GROUP BY TRUNC(cast(`__time` as timestamp), 'MM') -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: select TRUNC(cast(`__time` as timestamp), 'MM') from druid_table_alltypesorc GROUP BY TRUNC(cast(`__time` as timestamp), 'MM') -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -1969-12-01 -PREHOOK: query: select TRUNC(cast(`__time` as timestamp), 'MON') from druid_table_alltypesorc GROUP BY TRUNC(cast(`__time` as timestamp), 'MON') -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: select TRUNC(cast(`__time` as timestamp), 'MON') from druid_table_alltypesorc GROUP BY TRUNC(cast(`__time` as timestamp), 'MON') -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -1969-12-01 -PREHOOK: query: explain select TRUNC(cast(`__time` as timestamp), 'QUARTER') from druid_table_alltypesorc GROUP BY TRUNC(cast(`__time` as timestamp), 'QUARTER') -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: explain select TRUNC(cast(`__time` as timestamp), 'QUARTER') from druid_table_alltypesorc GROUP BY TRUNC(cast(`__time` as timestamp), 'QUARTER') -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_table_alltypesorc - properties: - druid.fieldNames vc - druid.fieldTypes string - druid.query.json {"queryType":"groupBy","dataSource":"default.druid_table_alltypesorc","granularity":"all","dimensions":[{"type":"default","dimension":"vc","outputName":"vc","outputType":"STRING"}],"virtualColumns":[{"type":"expression","name":"vc","expression":"timestamp_format(timestamp_floor(timestamp_parse(timestamp_format(\"__time\",'yyyy-MM-dd\\u0027T\\u0027HH:mm:ss.SSS\\u0027Z\\u0027','US/Pacific'),'yyyy-MM-dd\\u0027T\\u0027HH:mm:ss.SSS\\u0027Z\\u0027','UTC'),'P3M','','UTC'),'yyyy-MM-dd','UTC')","outputType":"STRING"}],"limitSpec":{"type":"default"},"aggregations":[],"intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"]} - druid.query.type groupBy - Select Operator - expressions: vc (type: string) - outputColumnNames: _col0 - ListSink - -PREHOOK: query: select TRUNC(cast(`__time` as timestamp), 'QUARTER') from druid_table_alltypesorc GROUP BY TRUNC(cast(`__time` as timestamp), 'QUARTER') -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: select TRUNC(cast(`__time` as timestamp), 'QUARTER') from druid_table_alltypesorc GROUP BY TRUNC(cast(`__time` as timestamp), 'QUARTER') -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -1969-10-01 -PREHOOK: query: select TRUNC(cast(`__time` as timestamp), 'Q') from druid_table_alltypesorc GROUP BY TRUNC(cast(`__time` as timestamp), 'Q') -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: select TRUNC(cast(`__time` as timestamp), 'Q') from druid_table_alltypesorc GROUP BY TRUNC(cast(`__time` as timestamp), 'Q') -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -1969-10-01 -PREHOOK: query: explain select TO_DATE(`__time`) from druid_table_alltypesorc GROUP BY TO_DATE(`__time`) -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: explain select TO_DATE(`__time`) from druid_table_alltypesorc GROUP BY TO_DATE(`__time`) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_table_alltypesorc - properties: - druid.fieldNames vc - druid.fieldTypes date - druid.query.json {"queryType":"groupBy","dataSource":"default.druid_table_alltypesorc","granularity":"all","dimensions":[{"type":"default","dimension":"vc","outputName":"vc","outputType":"LONG"}],"virtualColumns":[{"type":"expression","name":"vc","expression":"timestamp_floor(\"__time\",'P1D','','US/Pacific')","outputType":"LONG"}],"limitSpec":{"type":"default"},"aggregations":[],"intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"]} - druid.query.type groupBy - Select Operator - expressions: vc (type: date) - outputColumnNames: _col0 - ListSink - -PREHOOK: query: select TO_DATE(`__time`) from druid_table_alltypesorc GROUP BY TO_DATE(`__time`) -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: select TO_DATE(`__time`) from druid_table_alltypesorc GROUP BY TO_DATE(`__time`) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -1969-12-31 -PREHOOK: query: EXPLAIN SELECT SUM((`druid_table_alias`.`cdouble` * `druid_table_alias`.`cdouble`)) AS `sum_calculation_4998925219892510720_ok`, - CAST(TRUNC(CAST(`druid_table_alias`.`__time` AS TIMESTAMP),'MM') AS DATE) AS `tmn___time_ok` -FROM `default`.`druid_table_alltypesorc` `druid_table_alias` -GROUP BY CAST(TRUNC(CAST(`druid_table_alias`.`__time` AS TIMESTAMP),'MM') AS DATE) -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN SELECT SUM((`druid_table_alias`.`cdouble` * `druid_table_alias`.`cdouble`)) AS `sum_calculation_4998925219892510720_ok`, - CAST(TRUNC(CAST(`druid_table_alias`.`__time` AS TIMESTAMP),'MM') AS DATE) AS `tmn___time_ok` -FROM `default`.`druid_table_alltypesorc` `druid_table_alias` -GROUP BY CAST(TRUNC(CAST(`druid_table_alias`.`__time` AS TIMESTAMP),'MM') AS DATE) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_table_alias - properties: - druid.fieldNames vc,$f1 - druid.fieldTypes date,double - druid.query.json {"queryType":"groupBy","dataSource":"default.druid_table_alltypesorc","granularity":"all","dimensions":[{"type":"default","dimension":"vc","outputName":"vc","outputType":"LONG"}],"virtualColumns":[{"type":"expression","name":"vc","expression":"timestamp_floor(timestamp_parse(timestamp_format(timestamp_floor(timestamp_parse(timestamp_format(\"__time\",'yyyy-MM-dd\\u0027T\\u0027HH:mm:ss.SSS\\u0027Z\\u0027','US/Pacific'),'yyyy-MM-dd\\u0027T\\u0027HH:mm:ss.SSS\\u0027Z\\u0027','UTC'),'P1M','','UTC'),'yyyy-MM-dd','UTC'),'','UTC'),'P1D','','UTC')","outputType":"LONG"}],"limitSpec":{"type":"default"},"aggregations":[{"type":"doubleSum","name":"$f1","expression":"(\"cdouble\" * \"cdouble\")"}],"intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"]} - druid.query.type groupBy - Select Operator - expressions: $f1 (type: double), vc (type: date) - outputColumnNames: _col0, _col1 - ListSink - -PREHOOK: query: SELECT SUM((`druid_table_alias`.`cdouble` * `druid_table_alias`.`cdouble`)) AS `sum_calculation_4998925219892510720_ok`, - CAST(TRUNC(CAST(`druid_table_alias`.`__time` AS TIMESTAMP),'MM') AS DATE) AS `tmn___time_ok` -FROM `default`.`druid_table_alltypesorc` `druid_table_alias` -GROUP BY CAST(TRUNC(CAST(`druid_table_alias`.`__time` AS TIMESTAMP),'MM') AS DATE) -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: SELECT SUM((`druid_table_alias`.`cdouble` * `druid_table_alias`.`cdouble`)) AS `sum_calculation_4998925219892510720_ok`, - CAST(TRUNC(CAST(`druid_table_alias`.`__time` AS TIMESTAMP),'MM') AS DATE) AS `tmn___time_ok` -FROM `default`.`druid_table_alltypesorc` `druid_table_alias` -GROUP BY CAST(TRUNC(CAST(`druid_table_alias`.`__time` AS TIMESTAMP),'MM') AS DATE) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -1.91216621994122E14 1969-12-01 -PREHOOK: query: explain SELECT DATE_ADD(cast(`__time` as date), CAST((cdouble / 1000) AS INT)) as date_1, DATE_SUB(cast(`__time` as date), CAST((cdouble / 1000) AS INT)) as date_2, cast(`__time` as date) as orig_date, CAST((cdouble / 1000) AS INT) as offset from druid_table_alltypesorc order by date_1, date_2 limit 3 -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: explain SELECT DATE_ADD(cast(`__time` as date), CAST((cdouble / 1000) AS INT)) as date_1, DATE_SUB(cast(`__time` as date), CAST((cdouble / 1000) AS INT)) as date_2, cast(`__time` as date) as orig_date, CAST((cdouble / 1000) AS INT) as offset from druid_table_alltypesorc order by date_1, date_2 limit 3 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Tez -#### A masked pattern was here #### - Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) -#### A masked pattern was here #### - Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: druid_table_alltypesorc - properties: - druid.fieldNames vc,vc0,vc1,vc2 - druid.fieldTypes date,date,date,int - druid.query.json {"queryType":"scan","dataSource":"default.druid_table_alltypesorc","intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"],"virtualColumns":[{"type":"expression","name":"vc","expression":"timestamp_shift(timestamp_floor(timestamp_parse(timestamp_format(\"__time\",'yyyy-MM-dd\\u0027T\\u0027HH:mm:ss.SSS\\u0027Z\\u0027','US/Pacific'),'yyyy-MM-dd\\u0027T\\u0027HH:mm:ss.SSS\\u0027Z\\u0027','UTC'),'P1D','','UTC'),'P1D',CAST((\"cdouble\" / 1000), 'LONG'),'UTC')","outputType":"LONG"},{"type":"expression","name":"vc0","expression":"timestamp_shift(timestamp_floor(timestamp_parse(timestamp_format(\"__time\",'yyyy-MM-dd\\u0027T\\u0027HH:mm:ss.SSS\\u0027Z\\u0027','US/Pacific'),'yyyy-MM-dd\\u0027T\\u0027HH:mm:ss.SSS\\u0027Z\\u0027','UTC'),'P1D','','UTC'),'P1D',-( CAST((\"cdouble\" / 1000), 'LONG') ),'UTC')","outputType":"LONG"},{"type":"expression","name":"vc1","expression":"timestamp_floor(timestamp_parse(timestamp_format(\"__time\",'yyyy-MM-dd\\u0027T\\u0027HH:mm:ss.SSS\\u0027Z\\u0027','US/Pacific'),'yyyy-MM-dd\\u0027T\\u0027HH:mm:ss.SSS\\u0027Z\\u0027','UTC'),'P1D','','UTC')","outputType":"LONG"},{"type":"expression","name":"vc2","expression":"CAST((\"cdouble\" / 1000), 'LONG')","outputType":"LONG"}],"columns":["vc","vc0","vc1","vc2"],"resultFormat":"compactedList"} - druid.query.type scan - Statistics: Num rows: 9173 Data size: 1499152 Basic stats: COMPLETE Column stats: NONE - Top N Key Operator - sort order: ++ - keys: vc (type: date), vc0 (type: date) - null sort order: zz - Statistics: Num rows: 9173 Data size: 1499152 Basic stats: COMPLETE Column stats: NONE - top n: 3 - Select Operator - expressions: vc (type: date), vc0 (type: date), vc1 (type: date), vc2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 9173 Data size: 1499152 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: date), _col1 (type: date) - null sort order: zz - sort order: ++ - Statistics: Num rows: 9173 Data size: 1499152 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: date), _col3 (type: int) - Execution mode: vectorized, llap - LLAP IO: no inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: date), KEY.reducesinkkey1 (type: date), VALUE._col0 (type: date), VALUE._col1 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 9173 Data size: 1499152 Basic stats: COMPLETE Column stats: NONE - Limit - Number of rows: 3 - Statistics: Num rows: 3 Data size: 489 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 3 Data size: 489 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: 3 - Processor Tree: - ListSink - -PREHOOK: query: SELECT DATE_ADD(cast(`__time` as date), CAST((cdouble / 1000) AS INT)) as date_1, DATE_SUB(cast(`__time` as date), CAST((cdouble / 1000) AS INT)) as date_2, cast(`__time` as date) as orig_date, CAST((cdouble / 1000) AS INT) as offset from druid_table_alltypesorc order by date_1, date_2 limit 3 -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: SELECT DATE_ADD(cast(`__time` as date), CAST((cdouble / 1000) AS INT)) as date_1, DATE_SUB(cast(`__time` as date), CAST((cdouble / 1000) AS INT)) as date_2, cast(`__time` as date) as orig_date, CAST((cdouble / 1000) AS INT) as offset from druid_table_alltypesorc order by date_1, date_2 limit 3 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -1969-12-15 1970-01-16 1969-12-31 -16 -1969-12-15 1970-01-16 1969-12-31 -16 -1969-12-15 1970-01-16 1969-12-31 -16 -PREHOOK: query: -- Boolean Values - - - EXPLAIN SELECT cboolean2, count(*) from druid_table_alltypesorc GROUP BY cboolean2 -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: -- Boolean Values - - - EXPLAIN SELECT cboolean2, count(*) from druid_table_alltypesorc GROUP BY cboolean2 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_table_alltypesorc - properties: - druid.fieldNames cboolean2,$f1 - druid.fieldTypes boolean,bigint - druid.query.json {"queryType":"groupBy","dataSource":"default.druid_table_alltypesorc","granularity":"all","dimensions":[{"type":"default","dimension":"cboolean2","outputName":"cboolean2","outputType":"LONG"}],"limitSpec":{"type":"default"},"aggregations":[{"type":"count","name":"$f1"}],"intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"]} - druid.query.type groupBy - Select Operator - expressions: cboolean2 (type: boolean), $f1 (type: bigint) - outputColumnNames: _col0, _col1 - ListSink - -PREHOOK: query: SELECT cboolean2, count(*) from druid_table_alltypesorc GROUP BY cboolean2 -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: SELECT cboolean2, count(*) from druid_table_alltypesorc GROUP BY cboolean2 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -false 6214 -true 2959 -PREHOOK: query: -- Expected results of this query are wrong due to https://issues.apache.org/jira/browse/CALCITE-2319 - -- It should get fixed once we upgrade calcite - SELECT ctinyint > 2, count(*) from druid_table_alltypesorc GROUP BY ctinyint > 2 -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: -- Expected results of this query are wrong due to https://issues.apache.org/jira/browse/CALCITE-2319 - -- It should get fixed once we upgrade calcite - SELECT ctinyint > 2, count(*) from druid_table_alltypesorc GROUP BY ctinyint > 2 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -false 4280 -true 4893 -PREHOOK: query: EXPLAIN SELECT ctinyint > 2, count(*) from druid_table_alltypesorc GROUP BY ctinyint > 2 -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN SELECT ctinyint > 2, count(*) from druid_table_alltypesorc GROUP BY ctinyint > 2 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_table_alltypesorc - properties: - druid.fieldNames vc,$f1 - druid.fieldTypes boolean,bigint - druid.query.json {"queryType":"groupBy","dataSource":"default.druid_table_alltypesorc","granularity":"all","dimensions":[{"type":"default","dimension":"vc","outputName":"vc","outputType":"LONG"}],"virtualColumns":[{"type":"expression","name":"vc","expression":"(\"ctinyint\" > 2)","outputType":"LONG"}],"limitSpec":{"type":"default"},"aggregations":[{"type":"count","name":"$f1"}],"intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"]} - druid.query.type groupBy - Select Operator - expressions: vc (type: boolean), $f1 (type: bigint) - outputColumnNames: _col0, _col1 - ListSink - -PREHOOK: query: EXPLAIN SELECT sum(cfloat) FROM druid_table_alltypesorc WHERE cstring1 != 'en' group by 1.011 -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN SELECT sum(cfloat) FROM druid_table_alltypesorc WHERE cstring1 != 'en' group by 1.011 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_table_alltypesorc - properties: - druid.fieldNames vc,$f1 - druid.fieldTypes boolean,double - druid.query.json {"queryType":"groupBy","dataSource":"default.druid_table_alltypesorc","granularity":"all","dimensions":[{"type":"default","dimension":"vc","outputName":"vc","outputType":"LONG"}],"virtualColumns":[{"type":"expression","name":"vc","expression":"1","outputType":"LONG"}],"limitSpec":{"type":"default"},"filter":{"type":"not","field":{"type":"selector","dimension":"cstring1","value":"en"}},"aggregations":[{"type":"doubleSum","name":"$f1","fieldName":"cfloat"}],"intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"]} - druid.query.type groupBy - Select Operator - expressions: $f1 (type: double) - outputColumnNames: _col0 - ListSink - -PREHOOK: query: SELECT sum(cfloat) FROM druid_table_alltypesorc WHERE cstring1 != 'en' group by 1.011 -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: SELECT sum(cfloat) FROM druid_table_alltypesorc WHERE cstring1 != 'en' group by 1.011 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### --39590.24699282646 -PREHOOK: query: EXPLAIN SELECT sum(cfloat) FROM druid_table_alltypesorc WHERE cstring1 != 'en' group by 1.011, 3.40 -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN SELECT sum(cfloat) FROM druid_table_alltypesorc WHERE cstring1 != 'en' group by 1.011, 3.40 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_table_alltypesorc - properties: - druid.fieldNames vc,$f1 - druid.fieldTypes boolean,double - druid.query.json {"queryType":"groupBy","dataSource":"default.druid_table_alltypesorc","granularity":"all","dimensions":[{"type":"default","dimension":"vc","outputName":"vc","outputType":"LONG"}],"virtualColumns":[{"type":"expression","name":"vc","expression":"1","outputType":"LONG"}],"limitSpec":{"type":"default"},"filter":{"type":"not","field":{"type":"selector","dimension":"cstring1","value":"en"}},"aggregations":[{"type":"doubleSum","name":"$f1","fieldName":"cfloat"}],"intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"]} - druid.query.type groupBy - Select Operator - expressions: $f1 (type: double) - outputColumnNames: _col0 - ListSink - -PREHOOK: query: SELECT sum(cfloat) FROM druid_table_alltypesorc WHERE cstring1 != 'en' group by 1.011, 3.40 -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: SELECT sum(cfloat) FROM druid_table_alltypesorc WHERE cstring1 != 'en' group by 1.011, 3.40 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### --39590.24699282646 -PREHOOK: query: EXPLAIN SELECT sum(cint) FROM druid_table_alltypesorc WHERE cfloat= 0.011 group by cfloat -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN SELECT sum(cint) FROM druid_table_alltypesorc WHERE cfloat= 0.011 group by cfloat -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_table_alltypesorc - properties: - druid.fieldNames vc,$f1 - druid.fieldTypes boolean,bigint - druid.query.json {"queryType":"groupBy","dataSource":"default.druid_table_alltypesorc","granularity":"all","dimensions":[{"type":"default","dimension":"vc","outputName":"vc","outputType":"LONG"}],"virtualColumns":[{"type":"expression","name":"vc","expression":"1","outputType":"LONG"}],"limitSpec":{"type":"default"},"filter":{"type":"bound","dimension":"cfloat","lower":"0.011","lowerStrict":false,"upper":"0.011","upperStrict":false,"ordering":"numeric"},"aggregations":[{"type":"longSum","name":"$f1","fieldName":"cint"}],"intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"]} - druid.query.type groupBy - Select Operator - expressions: $f1 (type: bigint) - outputColumnNames: _col0 - ListSink - -PREHOOK: query: SELECT sum(cint) FROM druid_table_alltypesorc WHERE cfloat= 0.011 group by cfloat -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: SELECT sum(cint) FROM druid_table_alltypesorc WHERE cfloat= 0.011 group by cfloat -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -PREHOOK: query: EXPLAIN SELECT cfloat, sum(cint) FROM druid_table_alltypesorc WHERE cfloat= 0.011 group by cfloat -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN SELECT cfloat, sum(cint) FROM druid_table_alltypesorc WHERE cfloat= 0.011 group by cfloat -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_table_alltypesorc - properties: - druid.fieldNames cfloat,$f1 - druid.fieldTypes float,bigint - druid.query.json {"queryType":"groupBy","dataSource":"default.druid_table_alltypesorc","granularity":"all","dimensions":[{"type":"default","dimension":"vc","outputName":"vc","outputType":"LONG"}],"virtualColumns":[{"type":"expression","name":"vc","expression":"1","outputType":"LONG"}],"limitSpec":{"type":"default"},"filter":{"type":"bound","dimension":"cfloat","lower":"0.011","lowerStrict":false,"upper":"0.011","upperStrict":false,"ordering":"numeric"},"aggregations":[{"type":"longSum","name":"$f1","fieldName":"cint"}],"postAggregations":[{"type":"expression","name":"cfloat","expression":"0.011"}],"intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"]} - druid.query.type groupBy - Select Operator - expressions: cfloat (type: float), $f1 (type: bigint) - outputColumnNames: _col0, _col1 - ListSink - -PREHOOK: query: SELECT cfloat, sum(cint) FROM druid_table_alltypesorc WHERE cfloat= 0.011 group by cfloat -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: SELECT cfloat, sum(cint) FROM druid_table_alltypesorc WHERE cfloat= 0.011 group by cfloat -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -PREHOOK: query: CREATE TABLE druid_table_n1 -STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' -TBLPROPERTIES ("druid.segment.granularity" = "HOUR", "druid.query.granularity" = "MINUTE") -AS - SELECT cast (current_timestamp() as timestamp with local time zone) as `__time`, -cast(datetime1 as string) as datetime1, -cast(date1 as string) as date1, -cast(time1 as string) as time1 -FROM TABLE ( -VALUES -('2004-04-09 22:20:14', '2004-04-09','22:20:14'), -('2004-04-04 22:50:16', '2004-04-04', '22:50:16'), -('2004-04-12 04:40:49', '2004-04-12', '04:40:49'), -('2004-04-11 00:00:00', '2004-04-11', null), -('00:00:00 18:58:41', null, '18:58:41')) as q (datetime1, date1, time1) -PREHOOK: type: CREATETABLE_AS_SELECT -PREHOOK: Input: _dummy_database@_dummy_table -PREHOOK: Output: database:default -PREHOOK: Output: default@druid_table_n1 -POSTHOOK: query: CREATE TABLE druid_table_n1 -STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' -TBLPROPERTIES ("druid.segment.granularity" = "HOUR", "druid.query.granularity" = "MINUTE") -AS - SELECT cast (current_timestamp() as timestamp with local time zone) as `__time`, -cast(datetime1 as string) as datetime1, -cast(date1 as string) as date1, -cast(time1 as string) as time1 -FROM TABLE ( -VALUES -('2004-04-09 22:20:14', '2004-04-09','22:20:14'), -('2004-04-04 22:50:16', '2004-04-04', '22:50:16'), -('2004-04-12 04:40:49', '2004-04-12', '04:40:49'), -('2004-04-11 00:00:00', '2004-04-11', null), -('00:00:00 18:58:41', null, '18:58:41')) as q (datetime1, date1, time1) -POSTHOOK: type: CREATETABLE_AS_SELECT -POSTHOOK: Input: _dummy_database@_dummy_table -POSTHOOK: Output: database:default -POSTHOOK: Output: default@druid_table_n1 -POSTHOOK: Lineage: druid_table_n1.__time SIMPLE [] -POSTHOOK: Lineage: druid_table_n1.date1 SCRIPT [] -POSTHOOK: Lineage: druid_table_n1.datetime1 SCRIPT [] -POSTHOOK: Lineage: druid_table_n1.time1 SCRIPT [] -PREHOOK: query: EXPLAIN SELECT TO_DATE(date1), TO_DATE(datetime1) FROM druid_table_n1 -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_n1 -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN SELECT TO_DATE(date1), TO_DATE(datetime1) FROM druid_table_n1 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_n1 -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_table_n1 - properties: - druid.fieldNames vc,vc0 - druid.fieldTypes date,date - druid.query.json {"queryType":"scan","dataSource":"default.druid_table_n1","intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"],"virtualColumns":[{"type":"expression","name":"vc","expression":"timestamp_floor(timestamp_parse(\"date1\",'','UTC'),'P1D','','UTC')","outputType":"LONG"},{"type":"expression","name":"vc0","expression":"timestamp_floor(timestamp_parse(\"datetime1\",'','UTC'),'P1D','','UTC')","outputType":"LONG"}],"columns":["vc","vc0"],"resultFormat":"compactedList"} - druid.query.type scan - Select Operator - expressions: vc (type: date), vc0 (type: date) - outputColumnNames: _col0, _col1 - ListSink - -PREHOOK: query: SELECT TO_DATE(date1), TO_DATE(datetime1) FROM druid_table_n1 -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_n1 -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: SELECT TO_DATE(date1), TO_DATE(datetime1) FROM druid_table_n1 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_n1 -POSTHOOK: Output: hdfs://### HDFS PATH ### -NULL NULL -2004-04-04 2004-04-04 -2004-04-09 2004-04-09 -2004-04-11 2004-04-11 -2004-04-12 2004-04-12 -PREHOOK: query: EXPLAIN select count(*) from (select `__time` from druid_table_alltypesorc limit 1025) as src -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN select count(*) from (select `__time` from druid_table_alltypesorc limit 1025) as src -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Tez -#### A masked pattern was here #### - Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) -#### A masked pattern was here #### - Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: druid_table_alltypesorc - properties: - druid.fieldNames vc - druid.fieldTypes int - druid.query.json {"queryType":"scan","dataSource":"default.druid_table_alltypesorc","intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"],"virtualColumns":[{"type":"expression","name":"vc","expression":"0","outputType":"LONG"}],"columns":["vc"],"resultFormat":"compactedList","limit":1025} - druid.query.type scan - Statistics: Num rows: 9173 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Select Operator - Statistics: Num rows: 9173 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Group By Operator - aggregations: count() - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: COMPLETE - value expressions: _col0 (type: bigint) - Execution mode: vectorized, llap - LLAP IO: no inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: select count(*) from (select `__time` from druid_table_alltypesorc limit 1025) as src -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: select count(*) from (select `__time` from druid_table_alltypesorc limit 1025) as src -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -1025 -PREHOOK: query: explain select `timets` from (select `__time` as timets from druid_table_alltypesorc order by timets limit 10) as src order by `timets` -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: explain select `timets` from (select `__time` as timets from druid_table_alltypesorc order by timets limit 10) as src order by `timets` -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Tez -#### A masked pattern was here #### - Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) -#### A masked pattern was here #### - Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: druid_table_alltypesorc - properties: - druid.fieldNames vc - druid.fieldTypes timestamp with local time zone - druid.query.json {"queryType":"scan","dataSource":"default.druid_table_alltypesorc","intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"],"virtualColumns":[{"type":"expression","name":"vc","expression":"\"__time\"","outputType":"LONG"}],"columns":["vc"],"resultFormat":"compactedList"} - druid.query.type scan - Statistics: Num rows: 9173 Data size: 348640 Basic stats: COMPLETE Column stats: NONE - Top N Key Operator - sort order: + - keys: vc (type: timestamp with local time zone) - null sort order: z - Statistics: Num rows: 9173 Data size: 348640 Basic stats: COMPLETE Column stats: NONE - top n: 10 - Select Operator - expressions: vc (type: timestamp with local time zone) - outputColumnNames: _col0 - Statistics: Num rows: 9173 Data size: 348640 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: timestamp with local time zone) - null sort order: z - sort order: + - Statistics: Num rows: 9173 Data size: 348640 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: timestamp with local time zone) - outputColumnNames: _col0 - Statistics: Num rows: 9173 Data size: 348640 Basic stats: COMPLETE Column stats: NONE - Limit - Number of rows: 10 - Statistics: Num rows: 10 Data size: 380 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: timestamp with local time zone) - null sort order: z - sort order: + - Statistics: Num rows: 10 Data size: 380 Basic stats: COMPLETE Column stats: NONE - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: timestamp with local time zone) - outputColumnNames: _col0 - Statistics: Num rows: 10 Data size: 380 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 10 Data size: 380 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: explain select `timets` from (select cast(`__time` as timestamp ) as timets from druid_table_alltypesorc order by timets limit 10) as src order by `timets` -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: explain select `timets` from (select cast(`__time` as timestamp ) as timets from druid_table_alltypesorc order by timets limit 10) as src order by `timets` -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Tez -#### A masked pattern was here #### - Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) -#### A masked pattern was here #### - Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: druid_table_alltypesorc - properties: - druid.fieldNames vc - druid.fieldTypes timestamp - druid.query.json {"queryType":"scan","dataSource":"default.druid_table_alltypesorc","intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"],"virtualColumns":[{"type":"expression","name":"vc","expression":"timestamp_parse(timestamp_format(\"__time\",'yyyy-MM-dd\\u0027T\\u0027HH:mm:ss.SSS\\u0027Z\\u0027','US/Pacific'),'yyyy-MM-dd\\u0027T\\u0027HH:mm:ss.SSS\\u0027Z\\u0027','UTC')","outputType":"LONG"}],"columns":["vc"],"resultFormat":"compactedList"} - druid.query.type scan - Statistics: Num rows: 9173 Data size: 348640 Basic stats: COMPLETE Column stats: NONE - Top N Key Operator - sort order: + - keys: vc (type: timestamp) - null sort order: z - Statistics: Num rows: 9173 Data size: 348640 Basic stats: COMPLETE Column stats: NONE - top n: 10 - Select Operator - expressions: vc (type: timestamp) - outputColumnNames: _col0 - Statistics: Num rows: 9173 Data size: 348640 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: timestamp) - null sort order: z - sort order: + - Statistics: Num rows: 9173 Data size: 348640 Basic stats: COMPLETE Column stats: NONE - Execution mode: vectorized, llap - LLAP IO: no inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: timestamp) - outputColumnNames: _col0 - Statistics: Num rows: 9173 Data size: 348640 Basic stats: COMPLETE Column stats: NONE - Limit - Number of rows: 10 - Statistics: Num rows: 10 Data size: 380 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: timestamp) - null sort order: z - sort order: + - Statistics: Num rows: 10 Data size: 380 Basic stats: COMPLETE Column stats: NONE - Reducer 3 - Execution mode: vectorized, llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: timestamp) - outputColumnNames: _col0 - Statistics: Num rows: 10 Data size: 380 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 10 Data size: 380 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: select `timets_with_tz` from (select `__time` as timets_with_tz from druid_table_alltypesorc order by timets_with_tz limit 10) as src order by `timets_with_tz` -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: select `timets_with_tz` from (select `__time` as timets_with_tz from druid_table_alltypesorc order by timets_with_tz limit 10) as src order by `timets_with_tz` -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -PREHOOK: query: select `timets` from (select cast(`__time` as timestamp ) as timets from druid_table_alltypesorc order by timets limit 10) as src order by `timets` -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: select `timets` from (select cast(`__time` as timestamp ) as timets from druid_table_alltypesorc order by timets limit 10) as src order by `timets` -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -1969-12-31 15:59:00 -1969-12-31 15:59:00 -1969-12-31 15:59:00 -1969-12-31 15:59:00 -1969-12-31 15:59:00 -1969-12-31 15:59:00 -1969-12-31 15:59:00 -1969-12-31 15:59:00 -1969-12-31 15:59:00 -1969-12-31 15:59:00 -PREHOOK: query: select count(cfloat) from (select `cfloat`, `cstring1` from druid_table_alltypesorc limit 1025) as src -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: select count(cfloat) from (select `cfloat`, `cstring1` from druid_table_alltypesorc limit 1025) as src -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -1025 -PREHOOK: query: select count(cstring1) from (select `cfloat`, `cstring1` from druid_table_alltypesorc limit 90000) as src -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: select count(cstring1) from (select `cfloat`, `cstring1` from druid_table_alltypesorc limit 90000) as src -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -6059 -PREHOOK: query: explain select count(cstring1) from (select `cfloat`, `cstring1`, `cint` from druid_table_alltypesorc limit 90000) as src -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: explain select count(cstring1) from (select `cfloat`, `cstring1`, `cint` from druid_table_alltypesorc limit 90000) as src -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Tez -#### A masked pattern was here #### - Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) -#### A masked pattern was here #### - Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: druid_table_alltypesorc - properties: - druid.fieldNames cstring1 - druid.fieldTypes string - druid.query.json {"queryType":"scan","dataSource":"default.druid_table_alltypesorc","intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"],"columns":["cstring1"],"resultFormat":"compactedList","limit":90000} - druid.query.type scan - Statistics: Num rows: 9173 Data size: 1603744 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(cstring1) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Execution mode: vectorized, llap - LLAP IO: no inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: select max(cint * cdouble) from (select `cfloat`, `cstring1`, `cint`, `cdouble` from druid_table_alltypesorc limit 90000) as src -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: select max(cint * cdouble) from (select `cfloat`, `cstring1`, `cint`, `cdouble` from druid_table_alltypesorc limit 90000) as src -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -5.16019886323666E15 -PREHOOK: query: explain select max(cint * cfloat) from (select `cfloat`, `cstring1`, `cint`, `cdouble` from druid_table_alltypesorc limit 90000) as src -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: explain select max(cint * cfloat) from (select `cfloat`, `cstring1`, `cint`, `cdouble` from druid_table_alltypesorc limit 90000) as src -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Tez -#### A masked pattern was here #### - Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) -#### A masked pattern was here #### - Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: druid_table_alltypesorc - properties: - druid.fieldNames cfloat,cint - druid.fieldTypes float,int - druid.query.json {"queryType":"scan","dataSource":"default.druid_table_alltypesorc","intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"],"columns":["cfloat","cint"],"resultFormat":"compactedList","limit":90000} - druid.query.type scan - Statistics: Num rows: 9173 Data size: 69728 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: (UDFToFloat(cint) * cfloat) (type: float) - outputColumnNames: _col0 - Statistics: Num rows: 9173 Data size: 69728 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: max(_col0) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: float) - Execution mode: vectorized, llap - LLAP IO: no inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Group By Operator - aggregations: max(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: explain select count(distinct `__time`, cint) from (select * from druid_table_alltypesorc) as src -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: explain select count(distinct `__time`, cint) from (select * from druid_table_alltypesorc) as src -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Tez -#### A masked pattern was here #### - Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) -#### A masked pattern was here #### - Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: druid_table_alltypesorc - properties: - druid.fieldNames extract,cint - druid.fieldTypes timestamp with local time zone,int - druid.query.json {"queryType":"groupBy","dataSource":"default.druid_table_alltypesorc","granularity":"all","dimensions":[{"type":"extraction","dimension":"__time","outputName":"extract","extractionFn":{"type":"timeFormat","format":"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'","timeZone":"UTC"}},{"type":"default","dimension":"cint","outputName":"cint","outputType":"LONG"}],"limitSpec":{"type":"default"},"aggregations":[],"intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"]} - druid.query.type groupBy - Statistics: Num rows: 9173 Data size: 34864 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: cint (type: int) - outputColumnNames: cint - Statistics: Num rows: 9173 Data size: 34864 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(cint) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Execution mode: vectorized, llap - LLAP IO: no inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: select count(distinct `__time`, cint) from (select * from druid_table_alltypesorc) as src -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: select count(distinct `__time`, cint) from (select * from druid_table_alltypesorc) as src -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -3027 diff --git a/ql/src/test/results/clientpositive/druid/druidmini_extractTime.q.out b/ql/src/test/results/clientpositive/druid/druidmini_extractTime.q.out deleted file mode 100644 index 7b80b05b1fe0..000000000000 --- a/ql/src/test/results/clientpositive/druid/druidmini_extractTime.q.out +++ /dev/null @@ -1,886 +0,0 @@ -PREHOOK: query: SELECT EXTRACT(SECOND from `__time`) FROM druid_table_alltypesorc -WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 -AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 GROUP BY EXTRACT(SECOND from `__time`) -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: SELECT EXTRACT(SECOND from `__time`) FROM druid_table_alltypesorc -WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 -AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 GROUP BY EXTRACT(SECOND from `__time`) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -0 -PREHOOK: query: EXPLAIN SELECT EXTRACT(SECOND from `__time`) FROM druid_table_alltypesorc -WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 -AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 GROUP BY EXTRACT(SECOND from `__time`) -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN SELECT EXTRACT(SECOND from `__time`) FROM druid_table_alltypesorc -WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 -AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 GROUP BY EXTRACT(SECOND from `__time`) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_table_alltypesorc - properties: - druid.fieldNames extract_second - druid.fieldTypes int - druid.query.json {"queryType":"groupBy","dataSource":"default.druid_table_alltypesorc","granularity":"all","dimensions":[{"type":"extraction","dimension":"__time","outputName":"extract_second","extractionFn":{"type":"timeFormat","format":"s","timeZone":"US/Pacific","locale":"en-US"}}],"limitSpec":{"type":"default"},"filter":{"type":"and","fields":[{"type":"expression","expression":"(sqrt(abs(\"ctinyint\")) > 3)"},{"type":"expression","expression":"((pow(\"cfloat\",2) * pow(\"csmallint\",3)) > 1)"},{"type":"expression","expression":"(strlen(CAST(\"ctinyint\", 'STRING')) > 1)"},{"type":"expression","expression":"(strlen(CAST(\"ctinyint\", 'STRING')) < 10)"}]},"aggregations":[],"intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"]} - druid.query.type groupBy - Select Operator - expressions: extract_second (type: int) - outputColumnNames: _col0 - ListSink - -PREHOOK: query: SELECT EXTRACT(MINUTE from `__time`) FROM druid_table_alltypesorc -WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 -AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 GROUP BY EXTRACT(MINUTE from `__time`) -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: SELECT EXTRACT(MINUTE from `__time`) FROM druid_table_alltypesorc -WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 -AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 GROUP BY EXTRACT(MINUTE from `__time`) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -0 -59 -PREHOOK: query: EXPLAIN SELECT EXTRACT(MINUTE from `__time`) FROM druid_table_alltypesorc -WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 -AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 GROUP BY EXTRACT(MINUTE from `__time`) -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN SELECT EXTRACT(MINUTE from `__time`) FROM druid_table_alltypesorc -WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 -AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 GROUP BY EXTRACT(MINUTE from `__time`) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_table_alltypesorc - properties: - druid.fieldNames extract_minute - druid.fieldTypes int - druid.query.json {"queryType":"groupBy","dataSource":"default.druid_table_alltypesorc","granularity":"all","dimensions":[{"type":"extraction","dimension":"__time","outputName":"extract_minute","extractionFn":{"type":"timeFormat","format":"m","timeZone":"US/Pacific","locale":"en-US"}}],"limitSpec":{"type":"default"},"filter":{"type":"and","fields":[{"type":"expression","expression":"(sqrt(abs(\"ctinyint\")) > 3)"},{"type":"expression","expression":"((pow(\"cfloat\",2) * pow(\"csmallint\",3)) > 1)"},{"type":"expression","expression":"(strlen(CAST(\"ctinyint\", 'STRING')) > 1)"},{"type":"expression","expression":"(strlen(CAST(\"ctinyint\", 'STRING')) < 10)"}]},"aggregations":[],"intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"]} - druid.query.type groupBy - Select Operator - expressions: extract_minute (type: int) - outputColumnNames: _col0 - ListSink - -PREHOOK: query: SELECT EXTRACT(HOUR from `__time`) FROM druid_table_alltypesorc -WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 -AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 GROUP BY EXTRACT(HOUR from `__time`) -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: SELECT EXTRACT(HOUR from `__time`) FROM druid_table_alltypesorc -WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 -AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 GROUP BY EXTRACT(HOUR from `__time`) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -15 -16 -PREHOOK: query: EXPLAIN SELECT EXTRACT(HOUR from `__time`) FROM druid_table_alltypesorc -WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 -AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 GROUP BY EXTRACT(HOUR from `__time`) -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN SELECT EXTRACT(HOUR from `__time`) FROM druid_table_alltypesorc -WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 -AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 GROUP BY EXTRACT(HOUR from `__time`) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_table_alltypesorc - properties: - druid.fieldNames extract_hour - druid.fieldTypes int - druid.query.json {"queryType":"groupBy","dataSource":"default.druid_table_alltypesorc","granularity":"all","dimensions":[{"type":"extraction","dimension":"__time","outputName":"extract_hour","extractionFn":{"type":"timeFormat","format":"H","timeZone":"US/Pacific","locale":"en-US"}}],"limitSpec":{"type":"default"},"filter":{"type":"and","fields":[{"type":"expression","expression":"(sqrt(abs(\"ctinyint\")) > 3)"},{"type":"expression","expression":"((pow(\"cfloat\",2) * pow(\"csmallint\",3)) > 1)"},{"type":"expression","expression":"(strlen(CAST(\"ctinyint\", 'STRING')) > 1)"},{"type":"expression","expression":"(strlen(CAST(\"ctinyint\", 'STRING')) < 10)"}]},"aggregations":[],"intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"]} - druid.query.type groupBy - Select Operator - expressions: extract_hour (type: int) - outputColumnNames: _col0 - ListSink - -PREHOOK: query: SELECT EXTRACT(DAY from `__time`) FROM druid_table_alltypesorc -WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 -AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 GROUP BY EXTRACT(DAY from `__time`) -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: SELECT EXTRACT(DAY from `__time`) FROM druid_table_alltypesorc -WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 -AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 GROUP BY EXTRACT(DAY from `__time`) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -31 -PREHOOK: query: EXPLAIN SELECT EXTRACT(DAY from `__time`) FROM druid_table_alltypesorc -WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 -AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 GROUP BY EXTRACT(DAY from `__time`) -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN SELECT EXTRACT(DAY from `__time`) FROM druid_table_alltypesorc -WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 -AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 GROUP BY EXTRACT(DAY from `__time`) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_table_alltypesorc - properties: - druid.fieldNames extract_day - druid.fieldTypes int - druid.query.json {"queryType":"groupBy","dataSource":"default.druid_table_alltypesorc","granularity":"all","dimensions":[{"type":"extraction","dimension":"__time","outputName":"extract_day","extractionFn":{"type":"timeFormat","format":"d","timeZone":"US/Pacific","locale":"en-US"}}],"limitSpec":{"type":"default"},"filter":{"type":"and","fields":[{"type":"expression","expression":"(sqrt(abs(\"ctinyint\")) > 3)"},{"type":"expression","expression":"((pow(\"cfloat\",2) * pow(\"csmallint\",3)) > 1)"},{"type":"expression","expression":"(strlen(CAST(\"ctinyint\", 'STRING')) > 1)"},{"type":"expression","expression":"(strlen(CAST(\"ctinyint\", 'STRING')) < 10)"}]},"aggregations":[],"intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"]} - druid.query.type groupBy - Select Operator - expressions: extract_day (type: int) - outputColumnNames: _col0 - ListSink - -PREHOOK: query: SELECT EXTRACT(WEEK from `__time`) FROM druid_table_alltypesorc -WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 -AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 GROUP BY EXTRACT(WEEK from `__time`) -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: SELECT EXTRACT(WEEK from `__time`) FROM druid_table_alltypesorc -WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 -AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 GROUP BY EXTRACT(WEEK from `__time`) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -1 -PREHOOK: query: EXPLAIN SELECT EXTRACT(WEEK from `__time`) FROM druid_table_alltypesorc -WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 -AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 GROUP BY EXTRACT(WEEK from `__time`) -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN SELECT EXTRACT(WEEK from `__time`) FROM druid_table_alltypesorc -WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 -AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 GROUP BY EXTRACT(WEEK from `__time`) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_table_alltypesorc - properties: - druid.fieldNames extract_week - druid.fieldTypes int - druid.query.json {"queryType":"groupBy","dataSource":"default.druid_table_alltypesorc","granularity":"all","dimensions":[{"type":"extraction","dimension":"__time","outputName":"extract_week","extractionFn":{"type":"timeFormat","format":"w","timeZone":"US/Pacific","locale":"en-US"}}],"limitSpec":{"type":"default"},"filter":{"type":"and","fields":[{"type":"expression","expression":"(sqrt(abs(\"ctinyint\")) > 3)"},{"type":"expression","expression":"((pow(\"cfloat\",2) * pow(\"csmallint\",3)) > 1)"},{"type":"expression","expression":"(strlen(CAST(\"ctinyint\", 'STRING')) > 1)"},{"type":"expression","expression":"(strlen(CAST(\"ctinyint\", 'STRING')) < 10)"}]},"aggregations":[],"intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"]} - druid.query.type groupBy - Select Operator - expressions: extract_week (type: int) - outputColumnNames: _col0 - ListSink - -PREHOOK: query: SELECT EXTRACT(MONTH from `__time`) FROM druid_table_alltypesorc -WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 -AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 GROUP BY EXTRACT(MONTH from `__time`) -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: SELECT EXTRACT(MONTH from `__time`) FROM druid_table_alltypesorc -WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 -AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 GROUP BY EXTRACT(MONTH from `__time`) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -12 -PREHOOK: query: EXPLAIN SELECT EXTRACT(MONTH from `__time`) FROM druid_table_alltypesorc -WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 -AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 GROUP BY EXTRACT(MONTH from `__time`) -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN SELECT EXTRACT(MONTH from `__time`) FROM druid_table_alltypesorc -WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 -AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 GROUP BY EXTRACT(MONTH from `__time`) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_table_alltypesorc - properties: - druid.fieldNames extract_month - druid.fieldTypes int - druid.query.json {"queryType":"groupBy","dataSource":"default.druid_table_alltypesorc","granularity":"all","dimensions":[{"type":"extraction","dimension":"__time","outputName":"extract_month","extractionFn":{"type":"timeFormat","format":"M","timeZone":"US/Pacific","locale":"en-US"}}],"limitSpec":{"type":"default"},"filter":{"type":"and","fields":[{"type":"expression","expression":"(sqrt(abs(\"ctinyint\")) > 3)"},{"type":"expression","expression":"((pow(\"cfloat\",2) * pow(\"csmallint\",3)) > 1)"},{"type":"expression","expression":"(strlen(CAST(\"ctinyint\", 'STRING')) > 1)"},{"type":"expression","expression":"(strlen(CAST(\"ctinyint\", 'STRING')) < 10)"}]},"aggregations":[],"intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"]} - druid.query.type groupBy - Select Operator - expressions: extract_month (type: int) - outputColumnNames: _col0 - ListSink - -PREHOOK: query: SELECT EXTRACT(QUARTER from `__time`) FROM druid_table_alltypesorc -WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 -AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 GROUP BY EXTRACT(QUARTER from `__time`) -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: SELECT EXTRACT(QUARTER from `__time`) FROM druid_table_alltypesorc -WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 -AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 GROUP BY EXTRACT(QUARTER from `__time`) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -4 -PREHOOK: query: EXPLAIN SELECT EXTRACT(QUARTER from `__time`) FROM druid_table_alltypesorc -WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 -AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 GROUP BY EXTRACT(QUARTER from `__time`) -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN SELECT EXTRACT(QUARTER from `__time`) FROM druid_table_alltypesorc -WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 -AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 GROUP BY EXTRACT(QUARTER from `__time`) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_table_alltypesorc - properties: - druid.fieldNames vc - druid.fieldTypes int - druid.query.json {"queryType":"groupBy","dataSource":"default.druid_table_alltypesorc","granularity":"all","dimensions":[{"type":"default","dimension":"vc","outputName":"vc","outputType":"LONG"}],"virtualColumns":[{"type":"expression","name":"vc","expression":"timestamp_extract(\"__time\",'QUARTER','US/Pacific')","outputType":"LONG"}],"limitSpec":{"type":"default"},"filter":{"type":"and","fields":[{"type":"expression","expression":"(sqrt(abs(\"ctinyint\")) > 3)"},{"type":"expression","expression":"((pow(\"cfloat\",2) * pow(\"csmallint\",3)) > 1)"},{"type":"expression","expression":"(strlen(CAST(\"ctinyint\", 'STRING')) > 1)"},{"type":"expression","expression":"(strlen(CAST(\"ctinyint\", 'STRING')) < 10)"}]},"aggregations":[],"intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"]} - druid.query.type groupBy - Select Operator - expressions: vc (type: int) - outputColumnNames: _col0 - ListSink - -PREHOOK: query: SELECT EXTRACT(YEAR from `__time`) FROM druid_table_alltypesorc -WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 -AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 GROUP BY EXTRACT(YEAR from `__time`) -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: SELECT EXTRACT(YEAR from `__time`) FROM druid_table_alltypesorc -WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 -AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 GROUP BY EXTRACT(YEAR from `__time`) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -1969 -PREHOOK: query: EXPLAIN SELECT EXTRACT(YEAR from `__time`) FROM druid_table_alltypesorc -WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 -AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 GROUP BY EXTRACT(YEAR from `__time`) -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN SELECT EXTRACT(YEAR from `__time`) FROM druid_table_alltypesorc -WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 -AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 GROUP BY EXTRACT(YEAR from `__time`) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_table_alltypesorc - properties: - druid.fieldNames extract_year - druid.fieldTypes int - druid.query.json {"queryType":"groupBy","dataSource":"default.druid_table_alltypesorc","granularity":"all","dimensions":[{"type":"extraction","dimension":"__time","outputName":"extract_year","extractionFn":{"type":"timeFormat","format":"yyyy","timeZone":"US/Pacific","locale":"en-US"}}],"limitSpec":{"type":"default"},"filter":{"type":"and","fields":[{"type":"expression","expression":"(sqrt(abs(\"ctinyint\")) > 3)"},{"type":"expression","expression":"((pow(\"cfloat\",2) * pow(\"csmallint\",3)) > 1)"},{"type":"expression","expression":"(strlen(CAST(\"ctinyint\", 'STRING')) > 1)"},{"type":"expression","expression":"(strlen(CAST(\"ctinyint\", 'STRING')) < 10)"}]},"aggregations":[],"intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"]} - druid.query.type groupBy - Select Operator - expressions: extract_year (type: int) - outputColumnNames: _col0 - ListSink - -PREHOOK: query: EXPLAIN SELECT EXTRACT(SECOND from `__time`) FROM druid_table_alltypesorc WHERE EXTRACT(SECOND from `__time`) = 0 LIMIT 1 -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN SELECT EXTRACT(SECOND from `__time`) FROM druid_table_alltypesorc WHERE EXTRACT(SECOND from `__time`) = 0 LIMIT 1 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_table_alltypesorc - properties: - druid.fieldNames vc - druid.fieldTypes int - druid.query.json {"queryType":"scan","dataSource":"default.druid_table_alltypesorc","intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"],"filter":{"type":"bound","dimension":"__time","lower":"0","lowerStrict":false,"upper":"0","upperStrict":false,"ordering":"numeric","extractionFn":{"type":"timeFormat","format":"s","timeZone":"US/Pacific","locale":"en-US"}},"virtualColumns":[{"type":"expression","name":"vc","expression":"0","outputType":"LONG"}],"columns":["vc"],"resultFormat":"compactedList","limit":1} - druid.query.type scan - Select Operator - expressions: vc (type: int) - outputColumnNames: _col0 - ListSink - -PREHOOK: query: SELECT EXTRACT(SECOND from `__time`) FROM druid_table_alltypesorc WHERE EXTRACT(SECOND from `__time`) = 0 LIMIT 1 -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: SELECT EXTRACT(SECOND from `__time`) FROM druid_table_alltypesorc WHERE EXTRACT(SECOND from `__time`) = 0 LIMIT 1 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -0 -PREHOOK: query: EXPLAIN SELECT EXTRACT(MINUTE from `__time`) FROM druid_table_alltypesorc -WHERE EXTRACT(MINUTE from `__time`) >= 0 LIMIT 2 -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN SELECT EXTRACT(MINUTE from `__time`) FROM druid_table_alltypesorc -WHERE EXTRACT(MINUTE from `__time`) >= 0 LIMIT 2 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_table_alltypesorc - properties: - druid.fieldNames vc - druid.fieldTypes int - druid.query.json {"queryType":"scan","dataSource":"default.druid_table_alltypesorc","intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"],"filter":{"type":"bound","dimension":"__time","lower":"0","lowerStrict":false,"ordering":"numeric","extractionFn":{"type":"timeFormat","format":"m","timeZone":"US/Pacific","locale":"en-US"}},"virtualColumns":[{"type":"expression","name":"vc","expression":"timestamp_extract(\"__time\",'MINUTE','US/Pacific')","outputType":"LONG"}],"columns":["vc"],"resultFormat":"compactedList","limit":2} - druid.query.type scan - Select Operator - expressions: vc (type: int) - outputColumnNames: _col0 - ListSink - -PREHOOK: query: SELECT EXTRACT(MINUTE from `__time`) as minute FROM druid_table_alltypesorc - WHERE EXTRACT(MINUTE from `__time`) >= 0 order by minute LIMIT 2 -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: SELECT EXTRACT(MINUTE from `__time`) as minute FROM druid_table_alltypesorc - WHERE EXTRACT(MINUTE from `__time`) >= 0 order by minute LIMIT 2 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -0 -0 -PREHOOK: query: EXPLAIN SELECT EXTRACT(HOUR from `__time`) FROM druid_table_alltypesorc -WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 -AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 LIMIT 1 -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN SELECT EXTRACT(HOUR from `__time`) FROM druid_table_alltypesorc -WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 -AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 LIMIT 1 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_table_alltypesorc - properties: - druid.fieldNames vc - druid.fieldTypes int - druid.query.json {"queryType":"scan","dataSource":"default.druid_table_alltypesorc","intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"],"filter":{"type":"and","fields":[{"type":"expression","expression":"(sqrt(abs(\"ctinyint\")) > 3)"},{"type":"expression","expression":"((pow(\"cfloat\",2) * pow(\"csmallint\",3)) > 1)"},{"type":"expression","expression":"(strlen(CAST(\"ctinyint\", 'STRING')) > 1)"},{"type":"expression","expression":"(strlen(CAST(\"ctinyint\", 'STRING')) < 10)"}]},"virtualColumns":[{"type":"expression","name":"vc","expression":"timestamp_extract(\"__time\",'HOUR','US/Pacific')","outputType":"LONG"}],"columns":["vc"],"resultFormat":"compactedList","limit":1} - druid.query.type scan - Select Operator - expressions: vc (type: int) - outputColumnNames: _col0 - ListSink - -PREHOOK: query: SELECT EXTRACT(HOUR from `__time`) FROM druid_table_alltypesorc -WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 -AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 LIMIT 1 -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: SELECT EXTRACT(HOUR from `__time`) FROM druid_table_alltypesorc -WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 -AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 LIMIT 1 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -15 -PREHOOK: query: EXPLAIN SELECT EXTRACT(DAY from `__time`), EXTRACT(DAY from `__time`) DIV 7 AS WEEK, SUBSTRING(CAST(CAST(`__time` AS DATE) AS STRING), 9, 2) AS day_str -FROM druid_table_alltypesorc WHERE SUBSTRING(CAST(CAST(`__time` AS DATE) AS STRING), 9, 2) = 31 LIMIT 1 -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN SELECT EXTRACT(DAY from `__time`), EXTRACT(DAY from `__time`) DIV 7 AS WEEK, SUBSTRING(CAST(CAST(`__time` AS DATE) AS STRING), 9, 2) AS day_str -FROM druid_table_alltypesorc WHERE SUBSTRING(CAST(CAST(`__time` AS DATE) AS STRING), 9, 2) = 31 LIMIT 1 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_table_alltypesorc - properties: - druid.fieldNames vc,vc0,vc1 - druid.fieldTypes int,bigint,string - druid.query.json {"queryType":"scan","dataSource":"default.druid_table_alltypesorc","intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"],"filter":{"type":"expression","expression":"(CAST(substring(timestamp_format(timestamp_floor(timestamp_parse(timestamp_format(\"__time\",'yyyy-MM-dd\\u0027T\\u0027HH:mm:ss.SSS\\u0027Z\\u0027','US/Pacific'),'yyyy-MM-dd\\u0027T\\u0027HH:mm:ss.SSS\\u0027Z\\u0027','UTC'),'P1D','','UTC'),'yyyy-MM-dd','UTC'), 8, 2), 'DOUBLE') == 31)"},"virtualColumns":[{"type":"expression","name":"vc","expression":"timestamp_extract(\"__time\",'DAY','US/Pacific')","outputType":"LONG"},{"type":"expression","name":"vc0","expression":"div(timestamp_extract(\"__time\",'DAY','US/Pacific'),7)","outputType":"LONG"},{"type":"expression","name":"vc1","expression":"substring(timestamp_format(timestamp_floor(timestamp_parse(timestamp_format(\"__time\",'yyyy-MM-dd\\u0027T\\u0027HH:mm:ss.SSS\\u0027Z\\u0027','US/Pacific'),'yyyy-MM-dd\\u0027T\\u0027HH:mm:ss.SSS\\u0027Z\\u0027','UTC'),'P1D','','UTC'),'yyyy-MM-dd','UTC'), 8, 2)","outputType":"STRING"}],"columns":["vc","vc0","vc1"],"resultFormat":"compactedList","limit":1} - druid.query.type scan - Select Operator - expressions: vc (type: int), vc0 (type: bigint), vc1 (type: string) - outputColumnNames: _col0, _col1, _col2 - ListSink - -PREHOOK: query: SELECT EXTRACT(DAY from `__time`) , EXTRACT(DAY from `__time`) DIV 7 AS WEEK, SUBSTRING(CAST(CAST(`__time` AS DATE) AS STRING), 9, 2) AS dar_str -FROM druid_table_alltypesorc WHERE SUBSTRING(CAST(CAST(`__time` AS DATE) AS STRING), 9, 2) = 31 LIMIT 1 -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: SELECT EXTRACT(DAY from `__time`) , EXTRACT(DAY from `__time`) DIV 7 AS WEEK, SUBSTRING(CAST(CAST(`__time` AS DATE) AS STRING), 9, 2) AS dar_str -FROM druid_table_alltypesorc WHERE SUBSTRING(CAST(CAST(`__time` AS DATE) AS STRING), 9, 2) = 31 LIMIT 1 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -31 4 31 -PREHOOK: query: EXPLAIN SELECT EXTRACT(WEEK from `__time`) FROM druid_table_alltypesorc WHERE EXTRACT(WEEK from `__time`) >= 1 -AND EXTRACT(WEEK from `__time`) DIV 4 + 1 = 1 LIMIT 1 -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN SELECT EXTRACT(WEEK from `__time`) FROM druid_table_alltypesorc WHERE EXTRACT(WEEK from `__time`) >= 1 -AND EXTRACT(WEEK from `__time`) DIV 4 + 1 = 1 LIMIT 1 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_table_alltypesorc - properties: - druid.fieldNames vc - druid.fieldTypes int - druid.query.json {"queryType":"scan","dataSource":"default.druid_table_alltypesorc","intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"],"filter":{"type":"and","fields":[{"type":"bound","dimension":"__time","lower":"1","lowerStrict":false,"ordering":"numeric","extractionFn":{"type":"timeFormat","format":"w","timeZone":"US/Pacific","locale":"en-US"}},{"type":"expression","expression":"((div(timestamp_extract(\"__time\",'WEEK','US/Pacific'),4) + 1) == 1)"}]},"virtualColumns":[{"type":"expression","name":"vc","expression":"timestamp_extract(\"__time\",'WEEK','US/Pacific')","outputType":"LONG"}],"columns":["vc"],"resultFormat":"compactedList","limit":1} - druid.query.type scan - Select Operator - expressions: vc (type: int) - outputColumnNames: _col0 - ListSink - -PREHOOK: query: SELECT EXTRACT(WEEK from `__time`) FROM druid_table_alltypesorc WHERE EXTRACT(WEEK from `__time`) >= 1 -AND EXTRACT(WEEK from `__time`) DIV 4 + 1 = 1 LIMIT 1 -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: SELECT EXTRACT(WEEK from `__time`) FROM druid_table_alltypesorc WHERE EXTRACT(WEEK from `__time`) >= 1 -AND EXTRACT(WEEK from `__time`) DIV 4 + 1 = 1 LIMIT 1 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -1 -PREHOOK: query: EXPLAIN SELECT EXTRACT(MONTH FROM `__time`) / 4 + 1, EXTRACT(MONTH FROM `__time`), SUBSTRING(CAST(CAST(`__time` AS DATE) AS STRING), 6, 2) as month_str FROM druid_table_alltypesorc -WHERE EXTRACT(MONTH FROM `__time`) / 4 + 1 = 4 AND EXTRACT(MONTH FROM `__time`) BETWEEN 11 AND 12 LIMIT 1 -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN SELECT EXTRACT(MONTH FROM `__time`) / 4 + 1, EXTRACT(MONTH FROM `__time`), SUBSTRING(CAST(CAST(`__time` AS DATE) AS STRING), 6, 2) as month_str FROM druid_table_alltypesorc -WHERE EXTRACT(MONTH FROM `__time`) / 4 + 1 = 4 AND EXTRACT(MONTH FROM `__time`) BETWEEN 11 AND 12 LIMIT 1 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_table_alltypesorc - properties: - druid.fieldNames vc,vc0,vc1 - druid.fieldTypes double,int,string - druid.query.json {"queryType":"scan","dataSource":"default.druid_table_alltypesorc","intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"],"filter":{"type":"and","fields":[{"type":"expression","expression":"(((CAST(timestamp_extract(\"__time\",'MONTH','US/Pacific'), 'DOUBLE') / 4) + 1) == 4)"},{"type":"bound","dimension":"__time","lower":"11","lowerStrict":false,"upper":"12","upperStrict":false,"ordering":"numeric","extractionFn":{"type":"timeFormat","format":"M","timeZone":"US/Pacific","locale":"en-US"}}]},"virtualColumns":[{"type":"expression","name":"vc","expression":"4","outputType":"DOUBLE"},{"type":"expression","name":"vc0","expression":"timestamp_extract(\"__time\",'MONTH','US/Pacific')","outputType":"LONG"},{"type":"expression","name":"vc1","expression":"substring(timestamp_format(timestamp_floor(timestamp_parse(timestamp_format(\"__time\",'yyyy-MM-dd\\u0027T\\u0027HH:mm:ss.SSS\\u0027Z\\u0027','US/Pacific'),'yyyy-MM-dd\\u0027T\\u0027HH:mm:ss.SSS\\u0027Z\\u0027','UTC'),'P1D','','UTC'),'yyyy-MM-dd','UTC'), 5, 2)","outputType":"STRING"}],"columns":["vc","vc0","vc1"],"resultFormat":"compactedList","limit":1} - druid.query.type scan - Select Operator - expressions: vc (type: double), vc0 (type: int), vc1 (type: string) - outputColumnNames: _col0, _col1, _col2 - ListSink - -PREHOOK: query: SELECT EXTRACT(MONTH FROM `__time`) / 4 + 1, EXTRACT(MONTH FROM `__time`), SUBSTRING(CAST(CAST(`__time` AS DATE) AS STRING), 6, 2) as month_str FROM druid_table_alltypesorc - WHERE EXTRACT(MONTH FROM `__time`) / 4 + 1 = 4 AND EXTRACT(MONTH FROM `__time`) BETWEEN 11 AND 12 LIMIT 1 -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: SELECT EXTRACT(MONTH FROM `__time`) / 4 + 1, EXTRACT(MONTH FROM `__time`), SUBSTRING(CAST(CAST(`__time` AS DATE) AS STRING), 6, 2) as month_str FROM druid_table_alltypesorc - WHERE EXTRACT(MONTH FROM `__time`) / 4 + 1 = 4 AND EXTRACT(MONTH FROM `__time`) BETWEEN 11 AND 12 LIMIT 1 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -4.0 12 12 -PREHOOK: query: EXPLAIN SELECT EXTRACT(QUARTER from `__time`), EXTRACT(MONTH FROM `__time`) / 4 + 1 as q_number FROM druid_table_alltypesorc WHERE EXTRACT(QUARTER from `__time`) >= 4 - AND EXTRACT(MONTH FROM `__time`) / 4 + 1 = 4 LIMIT 1 -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN SELECT EXTRACT(QUARTER from `__time`), EXTRACT(MONTH FROM `__time`) / 4 + 1 as q_number FROM druid_table_alltypesorc WHERE EXTRACT(QUARTER from `__time`) >= 4 - AND EXTRACT(MONTH FROM `__time`) / 4 + 1 = 4 LIMIT 1 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_table_alltypesorc - properties: - druid.fieldNames vc,vc0 - druid.fieldTypes int,double - druid.query.json {"queryType":"scan","dataSource":"default.druid_table_alltypesorc","intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"],"filter":{"type":"and","fields":[{"type":"expression","expression":"(timestamp_extract(\"__time\",'QUARTER','US/Pacific') >= 4)"},{"type":"expression","expression":"(((CAST(timestamp_extract(\"__time\",'MONTH','US/Pacific'), 'DOUBLE') / 4) + 1) == 4)"}]},"virtualColumns":[{"type":"expression","name":"vc","expression":"timestamp_extract(\"__time\",'QUARTER','US/Pacific')","outputType":"LONG"},{"type":"expression","name":"vc0","expression":"4","outputType":"DOUBLE"}],"columns":["vc","vc0"],"resultFormat":"compactedList","limit":1} - druid.query.type scan - Select Operator - expressions: vc (type: int), vc0 (type: double) - outputColumnNames: _col0, _col1 - ListSink - -PREHOOK: query: SELECT EXTRACT(QUARTER from `__time`), EXTRACT(MONTH FROM `__time`) / 4 + 1 as q_number FROM druid_table_alltypesorc WHERE EXTRACT(QUARTER from `__time`) >= 4 - AND EXTRACT(MONTH FROM `__time`) / 4 + 1 = 4 LIMIT 1 -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: SELECT EXTRACT(QUARTER from `__time`), EXTRACT(MONTH FROM `__time`) / 4 + 1 as q_number FROM druid_table_alltypesorc WHERE EXTRACT(QUARTER from `__time`) >= 4 - AND EXTRACT(MONTH FROM `__time`) / 4 + 1 = 4 LIMIT 1 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -4 4.0 -PREHOOK: query: EXPLAIN SELECT EXTRACT(YEAR from `__time`), SUBSTRING(CAST(CAST(`__time` AS DATE) AS STRING), 1, 4) AS year_str FROM druid_table_alltypesorc WHERE EXTRACT(YEAR from `__time`) >= 1969 -AND CAST(EXTRACT(YEAR from `__time`) as STRING) = '1969' LIMIT 1 -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN SELECT EXTRACT(YEAR from `__time`), SUBSTRING(CAST(CAST(`__time` AS DATE) AS STRING), 1, 4) AS year_str FROM druid_table_alltypesorc WHERE EXTRACT(YEAR from `__time`) >= 1969 -AND CAST(EXTRACT(YEAR from `__time`) as STRING) = '1969' LIMIT 1 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_table_alltypesorc - properties: - druid.fieldNames vc,vc0 - druid.fieldTypes int,string - druid.query.json {"queryType":"scan","dataSource":"default.druid_table_alltypesorc","intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"],"filter":{"type":"and","fields":[{"type":"bound","dimension":"__time","lower":"1969-01-01T08:00:00.000Z","lowerStrict":false,"ordering":"lexicographic","extractionFn":{"type":"timeFormat","format":"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'","timeZone":"UTC"}},{"type":"expression","expression":"(CAST(timestamp_extract(\"__time\",'YEAR','US/Pacific'), 'STRING') == '1969')"}]},"virtualColumns":[{"type":"expression","name":"vc","expression":"timestamp_extract(\"__time\",'YEAR','US/Pacific')","outputType":"LONG"},{"type":"expression","name":"vc0","expression":"substring(timestamp_format(timestamp_floor(timestamp_parse(timestamp_format(\"__time\",'yyyy-MM-dd\\u0027T\\u0027HH:mm:ss.SSS\\u0027Z\\u0027','US/Pacific'),'yyyy-MM-dd\\u0027T\\u0027HH:mm:ss.SSS\\u0027Z\\u0027','UTC'),'P1D','','UTC'),'yyyy-MM-dd','UTC'), 0, 4)","outputType":"STRING"}],"columns":["vc","vc0"],"resultFormat":"compactedList","limit":1} - druid.query.type scan - Select Operator - expressions: vc (type: int), vc0 (type: string) - outputColumnNames: _col0, _col1 - ListSink - -PREHOOK: query: SELECT EXTRACT(YEAR from `__time`), SUBSTRING(CAST(CAST(`__time` AS DATE) AS STRING), 1, 4) as year_str FROM druid_table_alltypesorc WHERE EXTRACT(YEAR from `__time`) >= 1969 -AND CAST(EXTRACT(YEAR from `__time`) as STRING) = '1969' LIMIT 1 -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: SELECT EXTRACT(YEAR from `__time`), SUBSTRING(CAST(CAST(`__time` AS DATE) AS STRING), 1, 4) as year_str FROM druid_table_alltypesorc WHERE EXTRACT(YEAR from `__time`) >= 1969 -AND CAST(EXTRACT(YEAR from `__time`) as STRING) = '1969' LIMIT 1 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -1969 1969 -PREHOOK: query: explain SELECT CAST(`__time` AS TIMESTAMP) AS `x_time`, SUM(cfloat) FROM druid_table_alltypesorc GROUP BY CAST(`__time` AS TIMESTAMP) ORDER BY `x_time` LIMIT 5 -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: explain SELECT CAST(`__time` AS TIMESTAMP) AS `x_time`, SUM(cfloat) FROM druid_table_alltypesorc GROUP BY CAST(`__time` AS TIMESTAMP) ORDER BY `x_time` LIMIT 5 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_table_alltypesorc - properties: - druid.fieldNames extract,$f1 - druid.fieldTypes timestamp,double - druid.query.json {"queryType":"groupBy","dataSource":"default.druid_table_alltypesorc","granularity":"all","dimensions":[{"type":"extraction","dimension":"__time","outputName":"extract","extractionFn":{"type":"timeFormat","format":"yyyy-MM-dd HH:mm:ss","timeZone":"US/Pacific","locale":"en"}}],"limitSpec":{"type":"default","limit":5,"columns":[{"dimension":"extract","direction":"ascending","dimensionOrder":"lexicographic"}]},"aggregations":[{"type":"doubleSum","name":"$f1","fieldName":"cfloat"}],"intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"]} - druid.query.type groupBy - Select Operator - expressions: extract (type: timestamp), $f1 (type: double) - outputColumnNames: _col0, _col1 - ListSink - -PREHOOK: query: SELECT CAST(`__time` AS TIMESTAMP) AS `x_time`, SUM(cfloat) FROM druid_table_alltypesorc GROUP BY CAST(`__time` AS TIMESTAMP) ORDER BY `x_time` LIMIT 5 -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: SELECT CAST(`__time` AS TIMESTAMP) AS `x_time`, SUM(cfloat) FROM druid_table_alltypesorc GROUP BY CAST(`__time` AS TIMESTAMP) ORDER BY `x_time` LIMIT 5 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -1969-12-31 15:59:00 -4532.570000171661 -1969-12-31 16:00:00 -35057.6769926548 -PREHOOK: query: explain SELECT CAST(`__time` AS DATE) AS `x_date`, SUM(cfloat) FROM druid_table_alltypesorc GROUP BY CAST(`__time` AS DATE) ORDER BY `x_date` LIMIT 5 -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: explain SELECT CAST(`__time` AS DATE) AS `x_date`, SUM(cfloat) FROM druid_table_alltypesorc GROUP BY CAST(`__time` AS DATE) ORDER BY `x_date` LIMIT 5 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_table_alltypesorc - properties: - druid.fieldNames vc,$f1 - druid.fieldTypes date,double - druid.query.json {"queryType":"groupBy","dataSource":"default.druid_table_alltypesorc","granularity":"all","dimensions":[{"type":"default","dimension":"vc","outputName":"vc","outputType":"LONG"}],"virtualColumns":[{"type":"expression","name":"vc","expression":"timestamp_floor(timestamp_parse(timestamp_format(\"__time\",'yyyy-MM-dd\\u0027T\\u0027HH:mm:ss.SSS\\u0027Z\\u0027','US/Pacific'),'yyyy-MM-dd\\u0027T\\u0027HH:mm:ss.SSS\\u0027Z\\u0027','UTC'),'P1D','','UTC')","outputType":"LONG"}],"limitSpec":{"type":"default","limit":5,"columns":[{"dimension":"vc","direction":"ascending","dimensionOrder":"lexicographic"}]},"aggregations":[{"type":"doubleSum","name":"$f1","fieldName":"cfloat"}],"intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"]} - druid.query.type groupBy - Select Operator - expressions: vc (type: date), $f1 (type: double) - outputColumnNames: _col0, _col1 - ListSink - -PREHOOK: query: SELECT CAST(`__time` AS DATE) AS `x_date`, SUM(cfloat) FROM druid_table_alltypesorc GROUP BY CAST(`__time` AS DATE) ORDER BY `x_date` LIMIT 5 -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: SELECT CAST(`__time` AS DATE) AS `x_date`, SUM(cfloat) FROM druid_table_alltypesorc GROUP BY CAST(`__time` AS DATE) ORDER BY `x_date` LIMIT 5 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -1969-12-31 -39590.24699282646 -PREHOOK: query: SELECT CAST(`__time` AS DATE) AS `x_date` FROM druid_table_alltypesorc ORDER BY `x_date` LIMIT 5 -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: SELECT CAST(`__time` AS DATE) AS `x_date` FROM druid_table_alltypesorc ORDER BY `x_date` LIMIT 5 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -1969-12-31 -1969-12-31 -1969-12-31 -1969-12-31 -1969-12-31 -PREHOOK: query: create table test_extract_from_string_base_table(`timecolumn` timestamp, `date_c` string, `timestamp_c` string, `metric_c` double) -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@test_extract_from_string_base_table -POSTHOOK: query: create table test_extract_from_string_base_table(`timecolumn` timestamp, `date_c` string, `timestamp_c` string, `metric_c` double) -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@test_extract_from_string_base_table -PREHOOK: query: insert into test_extract_from_string_base_table values ('2015-03-08 00:00:00', '2015-03-10', '2015-03-08 05:30:20', 5.0) -PREHOOK: type: QUERY -PREHOOK: Input: _dummy_database@_dummy_table -PREHOOK: Output: default@test_extract_from_string_base_table -POSTHOOK: query: insert into test_extract_from_string_base_table values ('2015-03-08 00:00:00', '2015-03-10', '2015-03-08 05:30:20', 5.0) -POSTHOOK: type: QUERY -POSTHOOK: Input: _dummy_database@_dummy_table -POSTHOOK: Output: default@test_extract_from_string_base_table -POSTHOOK: Lineage: test_extract_from_string_base_table.date_c SCRIPT [] -POSTHOOK: Lineage: test_extract_from_string_base_table.metric_c SCRIPT [] -POSTHOOK: Lineage: test_extract_from_string_base_table.timecolumn SCRIPT [] -POSTHOOK: Lineage: test_extract_from_string_base_table.timestamp_c SCRIPT [] -PREHOOK: query: CREATE EXTERNAL TABLE druid_test_extract_from_string_table -STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' -TBLPROPERTIES ("druid.segment.granularity" = "DAY") -AS select -cast(`timecolumn` as timestamp with local time zone) as `__time`, `date_c`, `timestamp_c`, `metric_c` -FROM test_extract_from_string_base_table -PREHOOK: type: CREATETABLE_AS_SELECT -PREHOOK: Input: default@test_extract_from_string_base_table -PREHOOK: Output: database:default -PREHOOK: Output: default@druid_test_extract_from_string_table -POSTHOOK: query: CREATE EXTERNAL TABLE druid_test_extract_from_string_table -STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' -TBLPROPERTIES ("druid.segment.granularity" = "DAY") -AS select -cast(`timecolumn` as timestamp with local time zone) as `__time`, `date_c`, `timestamp_c`, `metric_c` -FROM test_extract_from_string_base_table -POSTHOOK: type: CREATETABLE_AS_SELECT -POSTHOOK: Input: default@test_extract_from_string_base_table -POSTHOOK: Output: database:default -POSTHOOK: Output: default@druid_test_extract_from_string_table -POSTHOOK: Lineage: druid_test_extract_from_string_table.__time EXPRESSION [(test_extract_from_string_base_table)test_extract_from_string_base_table.FieldSchema(name:timecolumn, type:timestamp, comment:null), ] -POSTHOOK: Lineage: druid_test_extract_from_string_table.date_c SIMPLE [(test_extract_from_string_base_table)test_extract_from_string_base_table.FieldSchema(name:date_c, type:string, comment:null), ] -POSTHOOK: Lineage: druid_test_extract_from_string_table.metric_c SIMPLE [(test_extract_from_string_base_table)test_extract_from_string_base_table.FieldSchema(name:metric_c, type:double, comment:null), ] -POSTHOOK: Lineage: druid_test_extract_from_string_table.timestamp_c SIMPLE [(test_extract_from_string_base_table)test_extract_from_string_base_table.FieldSchema(name:timestamp_c, type:string, comment:null), ] -PREHOOK: query: explain select -year(date_c), month(date_c),day(date_c), -year(timestamp_c), month(timestamp_c),day(timestamp_c), hour(timestamp_c), minute (timestamp_c), second (timestamp_c) -from druid_test_extract_from_string_table -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_test_extract_from_string_table -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: explain select -year(date_c), month(date_c),day(date_c), -year(timestamp_c), month(timestamp_c),day(timestamp_c), hour(timestamp_c), minute (timestamp_c), second (timestamp_c) -from druid_test_extract_from_string_table -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_test_extract_from_string_table -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_test_extract_from_string_table - properties: - druid.fieldNames vc,vc0,vc1,vc2,vc3,vc4,vc5,vc6,vc7 - druid.fieldTypes int,int,int,int,int,int,int,int,int - druid.query.json {"queryType":"scan","dataSource":"default.druid_test_extract_from_string_table","intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"],"virtualColumns":[{"type":"expression","name":"vc","expression":"timestamp_extract(timestamp_floor(timestamp_parse(\"date_c\",'','UTC'),'P1D','','UTC'),'YEAR','UTC')","outputType":"LONG"},{"type":"expression","name":"vc0","expression":"timestamp_extract(timestamp_floor(timestamp_parse(\"date_c\",'','UTC'),'P1D','','UTC'),'MONTH','UTC')","outputType":"LONG"},{"type":"expression","name":"vc1","expression":"timestamp_extract(timestamp_floor(timestamp_parse(\"date_c\",'','UTC'),'P1D','','UTC'),'DAY','UTC')","outputType":"LONG"},{"type":"expression","name":"vc2","expression":"timestamp_extract(timestamp_floor(timestamp_parse(\"timestamp_c\",'','UTC'),'P1D','','UTC'),'YEAR','UTC')","outputType":"LONG"},{"type":"expression","name":"vc3","expression":"timestamp_extract(timestamp_floor(timestamp_parse(\"timestamp_c\",'','UTC'),'P1D','','UTC'),'MONTH','UTC')","outputType":"LONG"},{"type":"expression","name":"vc4","expression":"timestamp_extract(timestamp_floor(timestamp_parse(\"timestamp_c\",'','UTC'),'P1D','','UTC'),'DAY','UTC')","outputType":"LONG"},{"type":"expression","name":"vc5","expression":"timestamp_extract(timestamp_parse(\"timestamp_c\",'','UTC'),'HOUR','UTC')","outputType":"LONG"},{"type":"expression","name":"vc6","expression":"timestamp_extract(timestamp_parse(\"timestamp_c\",'','UTC'),'MINUTE','UTC')","outputType":"LONG"},{"type":"expression","name":"vc7","expression":"timestamp_extract(timestamp_parse(\"timestamp_c\",'','UTC'),'SECOND','UTC')","outputType":"LONG"}],"columns":["vc","vc0","vc1","vc2","vc3","vc4","vc5","vc6","vc7"],"resultFormat":"compactedList"} - druid.query.type scan - Select Operator - expressions: vc (type: int), vc0 (type: int), vc1 (type: int), vc2 (type: int), vc3 (type: int), vc4 (type: int), vc5 (type: int), vc6 (type: int), vc7 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 - ListSink - -PREHOOK: query: select year(date_c), month(date_c), day(date_c), -year(timestamp_c), month(timestamp_c), day(timestamp_c), hour(timestamp_c), minute (timestamp_c), second (timestamp_c) -from druid_test_extract_from_string_table -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_test_extract_from_string_table -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: select year(date_c), month(date_c), day(date_c), -year(timestamp_c), month(timestamp_c), day(timestamp_c), hour(timestamp_c), minute (timestamp_c), second (timestamp_c) -from druid_test_extract_from_string_table -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_test_extract_from_string_table -POSTHOOK: Output: hdfs://### HDFS PATH ### -2015 3 10 2015 3 8 5 30 20 -PREHOOK: query: DROP TABLE druid_test_extract_from_string_table -PREHOOK: type: DROPTABLE -PREHOOK: Input: default@druid_test_extract_from_string_table -PREHOOK: Output: default@druid_test_extract_from_string_table -POSTHOOK: query: DROP TABLE druid_test_extract_from_string_table -POSTHOOK: type: DROPTABLE -POSTHOOK: Input: default@druid_test_extract_from_string_table -POSTHOOK: Output: default@druid_test_extract_from_string_table -PREHOOK: query: DROP TABLE test_extract_from_string_base_table -PREHOOK: type: DROPTABLE -PREHOOK: Input: default@test_extract_from_string_base_table -PREHOOK: Output: default@test_extract_from_string_base_table -POSTHOOK: query: DROP TABLE test_extract_from_string_base_table -POSTHOOK: type: DROPTABLE -POSTHOOK: Input: default@test_extract_from_string_base_table -POSTHOOK: Output: default@test_extract_from_string_base_table diff --git a/ql/src/test/results/clientpositive/druid/druidmini_floorTime.q.out b/ql/src/test/results/clientpositive/druid/druidmini_floorTime.q.out deleted file mode 100644 index 9502b522a922..000000000000 --- a/ql/src/test/results/clientpositive/druid/druidmini_floorTime.q.out +++ /dev/null @@ -1,692 +0,0 @@ -PREHOOK: query: SELECT floor(`__time` to SECOND) FROM druid_table_alltypesorc -WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 -AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 GROUP BY floor(`__time` to SECOND) -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: SELECT floor(`__time` to SECOND) FROM druid_table_alltypesorc -WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 -AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 GROUP BY floor(`__time` to SECOND) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 16:00:00.0 US/Pacific -PREHOOK: query: EXPLAIN SELECT floor(`__time` to SECOND) FROM druid_table_alltypesorc -WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 -AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 GROUP BY floor(`__time` to SECOND) -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN SELECT floor(`__time` to SECOND) FROM druid_table_alltypesorc -WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 -AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 GROUP BY floor(`__time` to SECOND) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_table_alltypesorc - properties: - druid.fieldNames timestamp - druid.fieldTypes timestamp with local time zone - druid.query.json {"queryType":"timeseries","dataSource":"default.druid_table_alltypesorc","descending":false,"granularity":{"type":"period","period":"PT1S","timeZone":"US/Pacific"},"filter":{"type":"and","fields":[{"type":"expression","expression":"(sqrt(abs(\"ctinyint\")) > 3)"},{"type":"expression","expression":"((pow(\"cfloat\",2) * pow(\"csmallint\",3)) > 1)"},{"type":"expression","expression":"(strlen(CAST(\"ctinyint\", 'STRING')) > 1)"},{"type":"expression","expression":"(strlen(CAST(\"ctinyint\", 'STRING')) < 10)"}]},"aggregations":[],"intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"],"context":{"skipEmptyBuckets":true}} - druid.query.type timeseries - Select Operator - expressions: timestamp (type: timestamp with local time zone) - outputColumnNames: _col0 - ListSink - -PREHOOK: query: SELECT floor(`__time` to MINUTE) FROM druid_table_alltypesorc -WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 -AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 GROUP BY floor(`__time` to MINUTE) -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: SELECT floor(`__time` to MINUTE) FROM druid_table_alltypesorc -WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 -AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 GROUP BY floor(`__time` to MINUTE) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 16:00:00.0 US/Pacific -PREHOOK: query: EXPLAIN SELECT floor(`__time` to MINUTE) FROM druid_table_alltypesorc -WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 -AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 GROUP BY floor(`__time` to MINUTE) -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN SELECT floor(`__time` to MINUTE) FROM druid_table_alltypesorc -WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 -AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 GROUP BY floor(`__time` to MINUTE) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_table_alltypesorc - properties: - druid.fieldNames timestamp - druid.fieldTypes timestamp with local time zone - druid.query.json {"queryType":"timeseries","dataSource":"default.druid_table_alltypesorc","descending":false,"granularity":{"type":"period","period":"PT1M","timeZone":"US/Pacific"},"filter":{"type":"and","fields":[{"type":"expression","expression":"(sqrt(abs(\"ctinyint\")) > 3)"},{"type":"expression","expression":"((pow(\"cfloat\",2) * pow(\"csmallint\",3)) > 1)"},{"type":"expression","expression":"(strlen(CAST(\"ctinyint\", 'STRING')) > 1)"},{"type":"expression","expression":"(strlen(CAST(\"ctinyint\", 'STRING')) < 10)"}]},"aggregations":[],"intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"],"context":{"skipEmptyBuckets":true}} - druid.query.type timeseries - Select Operator - expressions: timestamp (type: timestamp with local time zone) - outputColumnNames: _col0 - ListSink - -PREHOOK: query: SELECT floor(`__time` to HOUR) FROM druid_table_alltypesorc -WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 -AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 GROUP BY floor(`__time` to HOUR) -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: SELECT floor(`__time` to HOUR) FROM druid_table_alltypesorc -WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 -AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 GROUP BY floor(`__time` to HOUR) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -1969-12-31 15:00:00.0 US/Pacific -1969-12-31 16:00:00.0 US/Pacific -PREHOOK: query: EXPLAIN SELECT floor(`__time` to HOUR) FROM druid_table_alltypesorc -WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 -AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 GROUP BY floor(`__time` to HOUR) -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN SELECT floor(`__time` to HOUR) FROM druid_table_alltypesorc -WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 -AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 GROUP BY floor(`__time` to HOUR) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_table_alltypesorc - properties: - druid.fieldNames timestamp - druid.fieldTypes timestamp with local time zone - druid.query.json {"queryType":"timeseries","dataSource":"default.druid_table_alltypesorc","descending":false,"granularity":{"type":"period","period":"PT1H","timeZone":"US/Pacific"},"filter":{"type":"and","fields":[{"type":"expression","expression":"(sqrt(abs(\"ctinyint\")) > 3)"},{"type":"expression","expression":"((pow(\"cfloat\",2) * pow(\"csmallint\",3)) > 1)"},{"type":"expression","expression":"(strlen(CAST(\"ctinyint\", 'STRING')) > 1)"},{"type":"expression","expression":"(strlen(CAST(\"ctinyint\", 'STRING')) < 10)"}]},"aggregations":[],"intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"],"context":{"skipEmptyBuckets":true}} - druid.query.type timeseries - Select Operator - expressions: timestamp (type: timestamp with local time zone) - outputColumnNames: _col0 - ListSink - -PREHOOK: query: SELECT EXTRACT(DAY from `__time`) FROM druid_table_alltypesorc -WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 -AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 GROUP BY EXTRACT(DAY from `__time`) -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: SELECT EXTRACT(DAY from `__time`) FROM druid_table_alltypesorc -WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 -AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 GROUP BY EXTRACT(DAY from `__time`) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -31 -PREHOOK: query: EXPLAIN SELECT EXTRACT(DAY from `__time`) FROM druid_table_alltypesorc -WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 -AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 GROUP BY EXTRACT(DAY from `__time`) -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN SELECT EXTRACT(DAY from `__time`) FROM druid_table_alltypesorc -WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 -AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 GROUP BY EXTRACT(DAY from `__time`) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_table_alltypesorc - properties: - druid.fieldNames extract_day - druid.fieldTypes int - druid.query.json {"queryType":"groupBy","dataSource":"default.druid_table_alltypesorc","granularity":"all","dimensions":[{"type":"extraction","dimension":"__time","outputName":"extract_day","extractionFn":{"type":"timeFormat","format":"d","timeZone":"US/Pacific","locale":"en-US"}}],"limitSpec":{"type":"default"},"filter":{"type":"and","fields":[{"type":"expression","expression":"(sqrt(abs(\"ctinyint\")) > 3)"},{"type":"expression","expression":"((pow(\"cfloat\",2) * pow(\"csmallint\",3)) > 1)"},{"type":"expression","expression":"(strlen(CAST(\"ctinyint\", 'STRING')) > 1)"},{"type":"expression","expression":"(strlen(CAST(\"ctinyint\", 'STRING')) < 10)"}]},"aggregations":[],"intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"]} - druid.query.type groupBy - Select Operator - expressions: extract_day (type: int) - outputColumnNames: _col0 - ListSink - -PREHOOK: query: SELECT EXTRACT(WEEK from `__time`) FROM druid_table_alltypesorc -WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 -AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 GROUP BY EXTRACT(WEEK from `__time`) -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: SELECT EXTRACT(WEEK from `__time`) FROM druid_table_alltypesorc -WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 -AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 GROUP BY EXTRACT(WEEK from `__time`) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -1 -PREHOOK: query: EXPLAIN SELECT EXTRACT(WEEK from `__time`) FROM druid_table_alltypesorc -WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 -AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 GROUP BY EXTRACT(WEEK from `__time`) -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN SELECT EXTRACT(WEEK from `__time`) FROM druid_table_alltypesorc -WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 -AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 GROUP BY EXTRACT(WEEK from `__time`) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_table_alltypesorc - properties: - druid.fieldNames extract_week - druid.fieldTypes int - druid.query.json {"queryType":"groupBy","dataSource":"default.druid_table_alltypesorc","granularity":"all","dimensions":[{"type":"extraction","dimension":"__time","outputName":"extract_week","extractionFn":{"type":"timeFormat","format":"w","timeZone":"US/Pacific","locale":"en-US"}}],"limitSpec":{"type":"default"},"filter":{"type":"and","fields":[{"type":"expression","expression":"(sqrt(abs(\"ctinyint\")) > 3)"},{"type":"expression","expression":"((pow(\"cfloat\",2) * pow(\"csmallint\",3)) > 1)"},{"type":"expression","expression":"(strlen(CAST(\"ctinyint\", 'STRING')) > 1)"},{"type":"expression","expression":"(strlen(CAST(\"ctinyint\", 'STRING')) < 10)"}]},"aggregations":[],"intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"]} - druid.query.type groupBy - Select Operator - expressions: extract_week (type: int) - outputColumnNames: _col0 - ListSink - -PREHOOK: query: SELECT EXTRACT(MONTH from `__time`) FROM druid_table_alltypesorc -WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 -AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 GROUP BY EXTRACT(MONTH from `__time`) -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: SELECT EXTRACT(MONTH from `__time`) FROM druid_table_alltypesorc -WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 -AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 GROUP BY EXTRACT(MONTH from `__time`) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -12 -PREHOOK: query: EXPLAIN SELECT EXTRACT(MONTH from `__time`) FROM druid_table_alltypesorc -WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 -AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 GROUP BY EXTRACT(MONTH from `__time`) -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN SELECT EXTRACT(MONTH from `__time`) FROM druid_table_alltypesorc -WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 -AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 GROUP BY EXTRACT(MONTH from `__time`) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_table_alltypesorc - properties: - druid.fieldNames extract_month - druid.fieldTypes int - druid.query.json {"queryType":"groupBy","dataSource":"default.druid_table_alltypesorc","granularity":"all","dimensions":[{"type":"extraction","dimension":"__time","outputName":"extract_month","extractionFn":{"type":"timeFormat","format":"M","timeZone":"US/Pacific","locale":"en-US"}}],"limitSpec":{"type":"default"},"filter":{"type":"and","fields":[{"type":"expression","expression":"(sqrt(abs(\"ctinyint\")) > 3)"},{"type":"expression","expression":"((pow(\"cfloat\",2) * pow(\"csmallint\",3)) > 1)"},{"type":"expression","expression":"(strlen(CAST(\"ctinyint\", 'STRING')) > 1)"},{"type":"expression","expression":"(strlen(CAST(\"ctinyint\", 'STRING')) < 10)"}]},"aggregations":[],"intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"]} - druid.query.type groupBy - Select Operator - expressions: extract_month (type: int) - outputColumnNames: _col0 - ListSink - -PREHOOK: query: SELECT EXTRACT(QUARTER from `__time`) FROM druid_table_alltypesorc -WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 -AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 GROUP BY EXTRACT(QUARTER from `__time`) -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: SELECT EXTRACT(QUARTER from `__time`) FROM druid_table_alltypesorc -WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 -AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 GROUP BY EXTRACT(QUARTER from `__time`) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -4 -PREHOOK: query: EXPLAIN SELECT EXTRACT(QUARTER from `__time`) FROM druid_table_alltypesorc -WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 -AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 GROUP BY EXTRACT(QUARTER from `__time`) -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN SELECT EXTRACT(QUARTER from `__time`) FROM druid_table_alltypesorc -WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 -AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 GROUP BY EXTRACT(QUARTER from `__time`) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_table_alltypesorc - properties: - druid.fieldNames vc - druid.fieldTypes int - druid.query.json {"queryType":"groupBy","dataSource":"default.druid_table_alltypesorc","granularity":"all","dimensions":[{"type":"default","dimension":"vc","outputName":"vc","outputType":"LONG"}],"virtualColumns":[{"type":"expression","name":"vc","expression":"timestamp_extract(\"__time\",'QUARTER','US/Pacific')","outputType":"LONG"}],"limitSpec":{"type":"default"},"filter":{"type":"and","fields":[{"type":"expression","expression":"(sqrt(abs(\"ctinyint\")) > 3)"},{"type":"expression","expression":"((pow(\"cfloat\",2) * pow(\"csmallint\",3)) > 1)"},{"type":"expression","expression":"(strlen(CAST(\"ctinyint\", 'STRING')) > 1)"},{"type":"expression","expression":"(strlen(CAST(\"ctinyint\", 'STRING')) < 10)"}]},"aggregations":[],"intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"]} - druid.query.type groupBy - Select Operator - expressions: vc (type: int) - outputColumnNames: _col0 - ListSink - -PREHOOK: query: SELECT EXTRACT(YEAR from `__time`) FROM druid_table_alltypesorc -WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 -AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 GROUP BY EXTRACT(YEAR from `__time`) -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: SELECT EXTRACT(YEAR from `__time`) FROM druid_table_alltypesorc -WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 -AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 GROUP BY EXTRACT(YEAR from `__time`) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -1969 -PREHOOK: query: EXPLAIN SELECT EXTRACT(YEAR from `__time`) FROM druid_table_alltypesorc -WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 -AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 GROUP BY EXTRACT(YEAR from `__time`) -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN SELECT EXTRACT(YEAR from `__time`) FROM druid_table_alltypesorc -WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 -AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 GROUP BY EXTRACT(YEAR from `__time`) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_table_alltypesorc - properties: - druid.fieldNames extract_year - druid.fieldTypes int - druid.query.json {"queryType":"groupBy","dataSource":"default.druid_table_alltypesorc","granularity":"all","dimensions":[{"type":"extraction","dimension":"__time","outputName":"extract_year","extractionFn":{"type":"timeFormat","format":"yyyy","timeZone":"US/Pacific","locale":"en-US"}}],"limitSpec":{"type":"default"},"filter":{"type":"and","fields":[{"type":"expression","expression":"(sqrt(abs(\"ctinyint\")) > 3)"},{"type":"expression","expression":"((pow(\"cfloat\",2) * pow(\"csmallint\",3)) > 1)"},{"type":"expression","expression":"(strlen(CAST(\"ctinyint\", 'STRING')) > 1)"},{"type":"expression","expression":"(strlen(CAST(\"ctinyint\", 'STRING')) < 10)"}]},"aggregations":[],"intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"]} - druid.query.type groupBy - Select Operator - expressions: extract_year (type: int) - outputColumnNames: _col0 - ListSink - -PREHOOK: query: EXPLAIN SELECT EXTRACT(SECOND from `__time`) FROM druid_table_alltypesorc WHERE EXTRACT(SECOND from `__time`) = 0 LIMIT 1 -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN SELECT EXTRACT(SECOND from `__time`) FROM druid_table_alltypesorc WHERE EXTRACT(SECOND from `__time`) = 0 LIMIT 1 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_table_alltypesorc - properties: - druid.fieldNames vc - druid.fieldTypes int - druid.query.json {"queryType":"scan","dataSource":"default.druid_table_alltypesorc","intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"],"filter":{"type":"bound","dimension":"__time","lower":"0","lowerStrict":false,"upper":"0","upperStrict":false,"ordering":"numeric","extractionFn":{"type":"timeFormat","format":"s","timeZone":"US/Pacific","locale":"en-US"}},"virtualColumns":[{"type":"expression","name":"vc","expression":"0","outputType":"LONG"}],"columns":["vc"],"resultFormat":"compactedList","limit":1} - druid.query.type scan - Select Operator - expressions: vc (type: int) - outputColumnNames: _col0 - ListSink - -PREHOOK: query: SELECT EXTRACT(SECOND from `__time`) FROM druid_table_alltypesorc WHERE EXTRACT(SECOND from `__time`) = 0 LIMIT 1 -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: SELECT EXTRACT(SECOND from `__time`) FROM druid_table_alltypesorc WHERE EXTRACT(SECOND from `__time`) = 0 LIMIT 1 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -0 -PREHOOK: query: EXPLAIN SELECT EXTRACT(MINUTE from `__time`) FROM druid_table_alltypesorc -WHERE EXTRACT(MINUTE from `__time`) >= 0 LIMIT 2 -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN SELECT EXTRACT(MINUTE from `__time`) FROM druid_table_alltypesorc -WHERE EXTRACT(MINUTE from `__time`) >= 0 LIMIT 2 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_table_alltypesorc - properties: - druid.fieldNames vc - druid.fieldTypes int - druid.query.json {"queryType":"scan","dataSource":"default.druid_table_alltypesorc","intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"],"filter":{"type":"bound","dimension":"__time","lower":"0","lowerStrict":false,"ordering":"numeric","extractionFn":{"type":"timeFormat","format":"m","timeZone":"US/Pacific","locale":"en-US"}},"virtualColumns":[{"type":"expression","name":"vc","expression":"timestamp_extract(\"__time\",'MINUTE','US/Pacific')","outputType":"LONG"}],"columns":["vc"],"resultFormat":"compactedList","limit":2} - druid.query.type scan - Select Operator - expressions: vc (type: int) - outputColumnNames: _col0 - ListSink - -PREHOOK: query: SELECT EXTRACT(MINUTE from `__time`) as minute FROM druid_table_alltypesorc - WHERE EXTRACT(MINUTE from `__time`) >= 0 order by minute LIMIT 2 -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: SELECT EXTRACT(MINUTE from `__time`) as minute FROM druid_table_alltypesorc - WHERE EXTRACT(MINUTE from `__time`) >= 0 order by minute LIMIT 2 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -0 -0 -PREHOOK: query: EXPLAIN SELECT EXTRACT(HOUR from `__time`) FROM druid_table_alltypesorc -WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 -AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 LIMIT 1 -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN SELECT EXTRACT(HOUR from `__time`) FROM druid_table_alltypesorc -WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 -AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 LIMIT 1 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_table_alltypesorc - properties: - druid.fieldNames vc - druid.fieldTypes int - druid.query.json {"queryType":"scan","dataSource":"default.druid_table_alltypesorc","intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"],"filter":{"type":"and","fields":[{"type":"expression","expression":"(sqrt(abs(\"ctinyint\")) > 3)"},{"type":"expression","expression":"((pow(\"cfloat\",2) * pow(\"csmallint\",3)) > 1)"},{"type":"expression","expression":"(strlen(CAST(\"ctinyint\", 'STRING')) > 1)"},{"type":"expression","expression":"(strlen(CAST(\"ctinyint\", 'STRING')) < 10)"}]},"virtualColumns":[{"type":"expression","name":"vc","expression":"timestamp_extract(\"__time\",'HOUR','US/Pacific')","outputType":"LONG"}],"columns":["vc"],"resultFormat":"compactedList","limit":1} - druid.query.type scan - Select Operator - expressions: vc (type: int) - outputColumnNames: _col0 - ListSink - -PREHOOK: query: SELECT EXTRACT(HOUR from `__time`) FROM druid_table_alltypesorc -WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 -AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 LIMIT 1 -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: SELECT EXTRACT(HOUR from `__time`) FROM druid_table_alltypesorc -WHERE character_length(CAST(ctinyint AS STRING)) > 1 AND char_length(CAST(ctinyint AS STRING)) < 10 -AND power(cfloat, 2) * pow(csmallint, 3) > 1 AND SQRT(ABS(ctinyint)) > 3 LIMIT 1 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -15 -PREHOOK: query: EXPLAIN SELECT EXTRACT(DAY from `__time`), EXTRACT(DAY from `__time`) DIV 7 AS WEEK, SUBSTRING(CAST(CAST(`__time` AS DATE) AS STRING), 9, 2) AS day_str -FROM druid_table_alltypesorc WHERE SUBSTRING(CAST(CAST(`__time` AS DATE) AS STRING), 9, 2) = 31 LIMIT 1 -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN SELECT EXTRACT(DAY from `__time`), EXTRACT(DAY from `__time`) DIV 7 AS WEEK, SUBSTRING(CAST(CAST(`__time` AS DATE) AS STRING), 9, 2) AS day_str -FROM druid_table_alltypesorc WHERE SUBSTRING(CAST(CAST(`__time` AS DATE) AS STRING), 9, 2) = 31 LIMIT 1 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_table_alltypesorc - properties: - druid.fieldNames vc,vc0,vc1 - druid.fieldTypes int,bigint,string - druid.query.json {"queryType":"scan","dataSource":"default.druid_table_alltypesorc","intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"],"filter":{"type":"expression","expression":"(CAST(substring(timestamp_format(timestamp_floor(timestamp_parse(timestamp_format(\"__time\",'yyyy-MM-dd\\u0027T\\u0027HH:mm:ss.SSS\\u0027Z\\u0027','US/Pacific'),'yyyy-MM-dd\\u0027T\\u0027HH:mm:ss.SSS\\u0027Z\\u0027','UTC'),'P1D','','UTC'),'yyyy-MM-dd','UTC'), 8, 2), 'DOUBLE') == 31)"},"virtualColumns":[{"type":"expression","name":"vc","expression":"timestamp_extract(\"__time\",'DAY','US/Pacific')","outputType":"LONG"},{"type":"expression","name":"vc0","expression":"div(timestamp_extract(\"__time\",'DAY','US/Pacific'),7)","outputType":"LONG"},{"type":"expression","name":"vc1","expression":"substring(timestamp_format(timestamp_floor(timestamp_parse(timestamp_format(\"__time\",'yyyy-MM-dd\\u0027T\\u0027HH:mm:ss.SSS\\u0027Z\\u0027','US/Pacific'),'yyyy-MM-dd\\u0027T\\u0027HH:mm:ss.SSS\\u0027Z\\u0027','UTC'),'P1D','','UTC'),'yyyy-MM-dd','UTC'), 8, 2)","outputType":"STRING"}],"columns":["vc","vc0","vc1"],"resultFormat":"compactedList","limit":1} - druid.query.type scan - Select Operator - expressions: vc (type: int), vc0 (type: bigint), vc1 (type: string) - outputColumnNames: _col0, _col1, _col2 - ListSink - -PREHOOK: query: SELECT EXTRACT(DAY from `__time`) , EXTRACT(DAY from `__time`) DIV 7 AS WEEK, SUBSTRING(CAST(CAST(`__time` AS DATE) AS STRING), 9, 2) AS dar_str -FROM druid_table_alltypesorc WHERE SUBSTRING(CAST(CAST(`__time` AS DATE) AS STRING), 9, 2) = 31 LIMIT 1 -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: SELECT EXTRACT(DAY from `__time`) , EXTRACT(DAY from `__time`) DIV 7 AS WEEK, SUBSTRING(CAST(CAST(`__time` AS DATE) AS STRING), 9, 2) AS dar_str -FROM druid_table_alltypesorc WHERE SUBSTRING(CAST(CAST(`__time` AS DATE) AS STRING), 9, 2) = 31 LIMIT 1 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -31 4 31 -PREHOOK: query: EXPLAIN SELECT EXTRACT(WEEK from `__time`) FROM druid_table_alltypesorc WHERE EXTRACT(WEEK from `__time`) >= 1 -AND EXTRACT(WEEK from `__time`) DIV 4 + 1 = 1 LIMIT 1 -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN SELECT EXTRACT(WEEK from `__time`) FROM druid_table_alltypesorc WHERE EXTRACT(WEEK from `__time`) >= 1 -AND EXTRACT(WEEK from `__time`) DIV 4 + 1 = 1 LIMIT 1 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_table_alltypesorc - properties: - druid.fieldNames vc - druid.fieldTypes int - druid.query.json {"queryType":"scan","dataSource":"default.druid_table_alltypesorc","intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"],"filter":{"type":"and","fields":[{"type":"bound","dimension":"__time","lower":"1","lowerStrict":false,"ordering":"numeric","extractionFn":{"type":"timeFormat","format":"w","timeZone":"US/Pacific","locale":"en-US"}},{"type":"expression","expression":"((div(timestamp_extract(\"__time\",'WEEK','US/Pacific'),4) + 1) == 1)"}]},"virtualColumns":[{"type":"expression","name":"vc","expression":"timestamp_extract(\"__time\",'WEEK','US/Pacific')","outputType":"LONG"}],"columns":["vc"],"resultFormat":"compactedList","limit":1} - druid.query.type scan - Select Operator - expressions: vc (type: int) - outputColumnNames: _col0 - ListSink - -PREHOOK: query: SELECT EXTRACT(WEEK from `__time`) FROM druid_table_alltypesorc WHERE EXTRACT(WEEK from `__time`) >= 1 -AND EXTRACT(WEEK from `__time`) DIV 4 + 1 = 1 LIMIT 1 -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: SELECT EXTRACT(WEEK from `__time`) FROM druid_table_alltypesorc WHERE EXTRACT(WEEK from `__time`) >= 1 -AND EXTRACT(WEEK from `__time`) DIV 4 + 1 = 1 LIMIT 1 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -1 -PREHOOK: query: EXPLAIN SELECT EXTRACT(MONTH FROM `__time`) / 4 + 1, EXTRACT(MONTH FROM `__time`), SUBSTRING(CAST(CAST(`__time` AS DATE) AS STRING), 6, 2) as month_str FROM druid_table_alltypesorc -WHERE EXTRACT(MONTH FROM `__time`) / 4 + 1 = 4 AND EXTRACT(MONTH FROM `__time`) BETWEEN 11 AND 12 LIMIT 1 -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN SELECT EXTRACT(MONTH FROM `__time`) / 4 + 1, EXTRACT(MONTH FROM `__time`), SUBSTRING(CAST(CAST(`__time` AS DATE) AS STRING), 6, 2) as month_str FROM druid_table_alltypesorc -WHERE EXTRACT(MONTH FROM `__time`) / 4 + 1 = 4 AND EXTRACT(MONTH FROM `__time`) BETWEEN 11 AND 12 LIMIT 1 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_table_alltypesorc - properties: - druid.fieldNames vc,vc0,vc1 - druid.fieldTypes double,int,string - druid.query.json {"queryType":"scan","dataSource":"default.druid_table_alltypesorc","intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"],"filter":{"type":"and","fields":[{"type":"expression","expression":"(((CAST(timestamp_extract(\"__time\",'MONTH','US/Pacific'), 'DOUBLE') / 4) + 1) == 4)"},{"type":"bound","dimension":"__time","lower":"11","lowerStrict":false,"upper":"12","upperStrict":false,"ordering":"numeric","extractionFn":{"type":"timeFormat","format":"M","timeZone":"US/Pacific","locale":"en-US"}}]},"virtualColumns":[{"type":"expression","name":"vc","expression":"4","outputType":"DOUBLE"},{"type":"expression","name":"vc0","expression":"timestamp_extract(\"__time\",'MONTH','US/Pacific')","outputType":"LONG"},{"type":"expression","name":"vc1","expression":"substring(timestamp_format(timestamp_floor(timestamp_parse(timestamp_format(\"__time\",'yyyy-MM-dd\\u0027T\\u0027HH:mm:ss.SSS\\u0027Z\\u0027','US/Pacific'),'yyyy-MM-dd\\u0027T\\u0027HH:mm:ss.SSS\\u0027Z\\u0027','UTC'),'P1D','','UTC'),'yyyy-MM-dd','UTC'), 5, 2)","outputType":"STRING"}],"columns":["vc","vc0","vc1"],"resultFormat":"compactedList","limit":1} - druid.query.type scan - Select Operator - expressions: vc (type: double), vc0 (type: int), vc1 (type: string) - outputColumnNames: _col0, _col1, _col2 - ListSink - -PREHOOK: query: SELECT EXTRACT(MONTH FROM `__time`) / 4 + 1, EXTRACT(MONTH FROM `__time`), SUBSTRING(CAST(CAST(`__time` AS DATE) AS STRING), 6, 2) as month_str FROM druid_table_alltypesorc - WHERE EXTRACT(MONTH FROM `__time`) / 4 + 1 = 4 AND EXTRACT(MONTH FROM `__time`) BETWEEN 11 AND 12 LIMIT 1 -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: SELECT EXTRACT(MONTH FROM `__time`) / 4 + 1, EXTRACT(MONTH FROM `__time`), SUBSTRING(CAST(CAST(`__time` AS DATE) AS STRING), 6, 2) as month_str FROM druid_table_alltypesorc - WHERE EXTRACT(MONTH FROM `__time`) / 4 + 1 = 4 AND EXTRACT(MONTH FROM `__time`) BETWEEN 11 AND 12 LIMIT 1 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -4.0 12 12 -PREHOOK: query: EXPLAIN SELECT EXTRACT(QUARTER from `__time`), EXTRACT(MONTH FROM `__time`) / 4 + 1 as q_number FROM druid_table_alltypesorc WHERE EXTRACT(QUARTER from `__time`) >= 4 - AND EXTRACT(MONTH FROM `__time`) / 4 + 1 = 4 LIMIT 1 -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN SELECT EXTRACT(QUARTER from `__time`), EXTRACT(MONTH FROM `__time`) / 4 + 1 as q_number FROM druid_table_alltypesorc WHERE EXTRACT(QUARTER from `__time`) >= 4 - AND EXTRACT(MONTH FROM `__time`) / 4 + 1 = 4 LIMIT 1 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_table_alltypesorc - properties: - druid.fieldNames vc,vc0 - druid.fieldTypes int,double - druid.query.json {"queryType":"scan","dataSource":"default.druid_table_alltypesorc","intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"],"filter":{"type":"and","fields":[{"type":"expression","expression":"(timestamp_extract(\"__time\",'QUARTER','US/Pacific') >= 4)"},{"type":"expression","expression":"(((CAST(timestamp_extract(\"__time\",'MONTH','US/Pacific'), 'DOUBLE') / 4) + 1) == 4)"}]},"virtualColumns":[{"type":"expression","name":"vc","expression":"timestamp_extract(\"__time\",'QUARTER','US/Pacific')","outputType":"LONG"},{"type":"expression","name":"vc0","expression":"4","outputType":"DOUBLE"}],"columns":["vc","vc0"],"resultFormat":"compactedList","limit":1} - druid.query.type scan - Select Operator - expressions: vc (type: int), vc0 (type: double) - outputColumnNames: _col0, _col1 - ListSink - -PREHOOK: query: SELECT EXTRACT(QUARTER from `__time`), EXTRACT(MONTH FROM `__time`) / 4 + 1 as q_number FROM druid_table_alltypesorc WHERE EXTRACT(QUARTER from `__time`) >= 4 - AND EXTRACT(MONTH FROM `__time`) / 4 + 1 = 4 LIMIT 1 -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: SELECT EXTRACT(QUARTER from `__time`), EXTRACT(MONTH FROM `__time`) / 4 + 1 as q_number FROM druid_table_alltypesorc WHERE EXTRACT(QUARTER from `__time`) >= 4 - AND EXTRACT(MONTH FROM `__time`) / 4 + 1 = 4 LIMIT 1 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -4 4.0 -PREHOOK: query: EXPLAIN SELECT EXTRACT(YEAR from `__time`), SUBSTRING(CAST(CAST(`__time` AS DATE) AS STRING), 1, 4) AS year_str FROM druid_table_alltypesorc WHERE EXTRACT(YEAR from `__time`) >= 1969 -AND CAST(EXTRACT(YEAR from `__time`) as STRING) = '1969' LIMIT 1 -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN SELECT EXTRACT(YEAR from `__time`), SUBSTRING(CAST(CAST(`__time` AS DATE) AS STRING), 1, 4) AS year_str FROM druid_table_alltypesorc WHERE EXTRACT(YEAR from `__time`) >= 1969 -AND CAST(EXTRACT(YEAR from `__time`) as STRING) = '1969' LIMIT 1 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_table_alltypesorc - properties: - druid.fieldNames vc,vc0 - druid.fieldTypes int,string - druid.query.json {"queryType":"scan","dataSource":"default.druid_table_alltypesorc","intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"],"filter":{"type":"and","fields":[{"type":"bound","dimension":"__time","lower":"1969-01-01T08:00:00.000Z","lowerStrict":false,"ordering":"lexicographic","extractionFn":{"type":"timeFormat","format":"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'","timeZone":"UTC"}},{"type":"expression","expression":"(CAST(timestamp_extract(\"__time\",'YEAR','US/Pacific'), 'STRING') == '1969')"}]},"virtualColumns":[{"type":"expression","name":"vc","expression":"timestamp_extract(\"__time\",'YEAR','US/Pacific')","outputType":"LONG"},{"type":"expression","name":"vc0","expression":"substring(timestamp_format(timestamp_floor(timestamp_parse(timestamp_format(\"__time\",'yyyy-MM-dd\\u0027T\\u0027HH:mm:ss.SSS\\u0027Z\\u0027','US/Pacific'),'yyyy-MM-dd\\u0027T\\u0027HH:mm:ss.SSS\\u0027Z\\u0027','UTC'),'P1D','','UTC'),'yyyy-MM-dd','UTC'), 0, 4)","outputType":"STRING"}],"columns":["vc","vc0"],"resultFormat":"compactedList","limit":1} - druid.query.type scan - Select Operator - expressions: vc (type: int), vc0 (type: string) - outputColumnNames: _col0, _col1 - ListSink - -PREHOOK: query: SELECT EXTRACT(YEAR from `__time`), SUBSTRING(CAST(CAST(`__time` AS DATE) AS STRING), 1, 4) as year_str FROM druid_table_alltypesorc WHERE EXTRACT(YEAR from `__time`) >= 1969 -AND CAST(EXTRACT(YEAR from `__time`) as STRING) = '1969' LIMIT 1 -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: SELECT EXTRACT(YEAR from `__time`), SUBSTRING(CAST(CAST(`__time` AS DATE) AS STRING), 1, 4) as year_str FROM druid_table_alltypesorc WHERE EXTRACT(YEAR from `__time`) >= 1969 -AND CAST(EXTRACT(YEAR from `__time`) as STRING) = '1969' LIMIT 1 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -1969 1969 diff --git a/ql/src/test/results/clientpositive/druid/druidmini_mv.q.out b/ql/src/test/results/clientpositive/druid/druidmini_mv.q.out deleted file mode 100644 index 734431a01743..000000000000 --- a/ql/src/test/results/clientpositive/druid/druidmini_mv.q.out +++ /dev/null @@ -1,741 +0,0 @@ -PREHOOK: query: CREATE TABLE cmv_basetable_n2 -STORED AS orc -TBLPROPERTIES ('transactional'='true') -AS -SELECT cast(current_timestamp() AS timestamp) AS t, - cast(a AS int) AS a, - cast(b AS varchar(256)) AS b, - cast(userid AS varchar(256)) AS userid, - cast(c AS double) AS c, - cast(d AS int) AS d -FROM TABLE ( - VALUES - (1, 'alfred', 'alfred', 10.30, 2), - (2, 'bob', 'bob', 3.14, 3), - (2, 'bonnie', 'bonnie', 172342.2, 3), - (3, 'calvin', 'calvin', 978.76, 3), - (3, 'charlie', 'charlie_a', 9.8, 1), - (3, 'charlie', 'charlie_b', 15.8, 1)) as q (a, b, userid, c, d) -PREHOOK: type: CREATETABLE_AS_SELECT -PREHOOK: Input: _dummy_database@_dummy_table -PREHOOK: Output: database:default -PREHOOK: Output: default@cmv_basetable_n2 -POSTHOOK: query: CREATE TABLE cmv_basetable_n2 -STORED AS orc -TBLPROPERTIES ('transactional'='true') -AS -SELECT cast(current_timestamp() AS timestamp) AS t, - cast(a AS int) AS a, - cast(b AS varchar(256)) AS b, - cast(userid AS varchar(256)) AS userid, - cast(c AS double) AS c, - cast(d AS int) AS d -FROM TABLE ( - VALUES - (1, 'alfred', 'alfred', 10.30, 2), - (2, 'bob', 'bob', 3.14, 3), - (2, 'bonnie', 'bonnie', 172342.2, 3), - (3, 'calvin', 'calvin', 978.76, 3), - (3, 'charlie', 'charlie_a', 9.8, 1), - (3, 'charlie', 'charlie_b', 15.8, 1)) as q (a, b, userid, c, d) -POSTHOOK: type: CREATETABLE_AS_SELECT -POSTHOOK: Input: _dummy_database@_dummy_table -POSTHOOK: Output: database:default -POSTHOOK: Output: default@cmv_basetable_n2 -POSTHOOK: Lineage: cmv_basetable_n2.a SCRIPT [] -POSTHOOK: Lineage: cmv_basetable_n2.b SCRIPT [] -POSTHOOK: Lineage: cmv_basetable_n2.c SCRIPT [] -POSTHOOK: Lineage: cmv_basetable_n2.d SCRIPT [] -POSTHOOK: Lineage: cmv_basetable_n2.t SIMPLE [] -POSTHOOK: Lineage: cmv_basetable_n2.userid SCRIPT [] -PREHOOK: query: CREATE MATERIALIZED VIEW cmv_mat_view_n2 -STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' -TBLPROPERTIES ("druid.segment.granularity" = "HOUR") -AS -SELECT cast(t AS timestamp with local time zone) as `__time`, a, b, c, userid -FROM cmv_basetable_n2 -WHERE a = 2 -PREHOOK: type: CREATE_MATERIALIZED_VIEW -PREHOOK: Input: default@cmv_basetable_n2 -PREHOOK: Output: database:default -PREHOOK: Output: default@cmv_mat_view_n2 -POSTHOOK: query: CREATE MATERIALIZED VIEW cmv_mat_view_n2 -STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' -TBLPROPERTIES ("druid.segment.granularity" = "HOUR") -AS -SELECT cast(t AS timestamp with local time zone) as `__time`, a, b, c, userid -FROM cmv_basetable_n2 -WHERE a = 2 -POSTHOOK: type: CREATE_MATERIALIZED_VIEW -POSTHOOK: Input: default@cmv_basetable_n2 -POSTHOOK: Output: database:default -POSTHOOK: Output: default@cmv_mat_view_n2 -PREHOOK: query: SELECT a, b, c FROM cmv_mat_view_n2 -PREHOOK: type: QUERY -PREHOOK: Input: default@cmv_mat_view_n2 -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: SELECT a, b, c FROM cmv_mat_view_n2 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@cmv_mat_view_n2 -POSTHOOK: Output: hdfs://### HDFS PATH ### -2 bob 3.14 -2 bonnie 172342.2 -PREHOOK: query: SHOW TBLPROPERTIES cmv_mat_view_n2 -PREHOOK: type: SHOW_TBLPROPERTIES -POSTHOOK: query: SHOW TBLPROPERTIES cmv_mat_view_n2 -POSTHOOK: type: SHOW_TBLPROPERTIES -COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} -bucketing_version 2 -druid.datasource default.cmv_mat_view_n2 -druid.segment.granularity HOUR -numFiles 0 -numFilesErasureCoded 0 -numRows 2 -rawDataSize 0 -storage_handler org.apache.hadoop.hive.druid.DruidStorageHandler -totalSize #Masked# -#### A masked pattern was here #### -PREHOOK: query: CREATE MATERIALIZED VIEW IF NOT EXISTS cmv_mat_view2_n0 -STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' -TBLPROPERTIES ("druid.segment.granularity" = "HOUR") -AS -SELECT cast(t AS timestamp with local time zone) as `__time`, a, b, c, userid -FROM cmv_basetable_n2 -WHERE a = 3 -PREHOOK: type: CREATE_MATERIALIZED_VIEW -PREHOOK: Input: default@cmv_basetable_n2 -PREHOOK: Output: database:default -PREHOOK: Output: default@cmv_mat_view2_n0 -POSTHOOK: query: CREATE MATERIALIZED VIEW IF NOT EXISTS cmv_mat_view2_n0 -STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' -TBLPROPERTIES ("druid.segment.granularity" = "HOUR") -AS -SELECT cast(t AS timestamp with local time zone) as `__time`, a, b, c, userid -FROM cmv_basetable_n2 -WHERE a = 3 -POSTHOOK: type: CREATE_MATERIALIZED_VIEW -POSTHOOK: Input: default@cmv_basetable_n2 -POSTHOOK: Output: database:default -POSTHOOK: Output: default@cmv_mat_view2_n0 -PREHOOK: query: SELECT a, c FROM cmv_mat_view2_n0 -PREHOOK: type: QUERY -PREHOOK: Input: default@cmv_mat_view2_n0 -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: SELECT a, c FROM cmv_mat_view2_n0 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@cmv_mat_view2_n0 -POSTHOOK: Output: hdfs://### HDFS PATH ### -3 15.8 -3 9.8 -3 978.76 -PREHOOK: query: SHOW TBLPROPERTIES cmv_mat_view2_n0 -PREHOOK: type: SHOW_TBLPROPERTIES -POSTHOOK: query: SHOW TBLPROPERTIES cmv_mat_view2_n0 -POSTHOOK: type: SHOW_TBLPROPERTIES -COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} -bucketing_version 2 -druid.datasource default.cmv_mat_view2_n0 -druid.segment.granularity HOUR -numFiles 0 -numFilesErasureCoded 0 -numRows 3 -rawDataSize 0 -storage_handler org.apache.hadoop.hive.druid.DruidStorageHandler -totalSize #Masked# -#### A masked pattern was here #### -PREHOOK: query: EXPLAIN -SELECT a, c -FROM cmv_basetable_n2 -WHERE a = 3 -PREHOOK: type: QUERY -PREHOOK: Input: default@cmv_basetable_n2 -PREHOOK: Input: default@cmv_mat_view2_n0 -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN -SELECT a, c -FROM cmv_basetable_n2 -WHERE a = 3 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@cmv_basetable_n2 -POSTHOOK: Input: default@cmv_mat_view2_n0 -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: cmv_mat_view2_n0 - properties: - druid.fieldNames a,c - druid.fieldTypes int,double - druid.query.json {"queryType":"scan","dataSource":"default.cmv_mat_view2_n0","intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"],"columns":["a","c"],"resultFormat":"compactedList"} - druid.query.type scan - Select Operator - expressions: a (type: int), c (type: double) - outputColumnNames: _col0, _col1 - ListSink - -PREHOOK: query: SELECT a, c -FROM cmv_basetable_n2 -WHERE a = 3 -PREHOOK: type: QUERY -PREHOOK: Input: default@cmv_basetable_n2 -PREHOOK: Input: default@cmv_mat_view2_n0 -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: SELECT a, c -FROM cmv_basetable_n2 -WHERE a = 3 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@cmv_basetable_n2 -POSTHOOK: Input: default@cmv_mat_view2_n0 -POSTHOOK: Output: hdfs://### HDFS PATH ### -3 15.8 -3 9.8 -3 978.76 -Warning: Shuffle Join MERGEJOIN[10][tables = [cmv_mat_view2_n0, $hdt$_0]] in Stage 'Reducer 2' is a cross product -PREHOOK: query: EXPLAIN -SELECT * FROM ( - (SELECT a, c FROM cmv_basetable_n2 WHERE a = 3) table1 - JOIN - (SELECT a, c FROM cmv_basetable_n2 WHERE d = 3) table2 - ON table1.a = table2.a) -PREHOOK: type: QUERY -PREHOOK: Input: default@cmv_basetable_n2 -PREHOOK: Input: default@cmv_mat_view2_n0 -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN -SELECT * FROM ( - (SELECT a, c FROM cmv_basetable_n2 WHERE a = 3) table1 - JOIN - (SELECT a, c FROM cmv_basetable_n2 WHERE d = 3) table2 - ON table1.a = table2.a) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@cmv_basetable_n2 -POSTHOOK: Input: default@cmv_mat_view2_n0 -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Tez -#### A masked pattern was here #### - Edges: - Reducer 2 <- Map 1 (XPROD_EDGE), Map 3 (XPROD_EDGE) -#### A masked pattern was here #### - Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: cmv_basetable_n2 - filterExpr: ((a = 3) and (d = 3)) (type: boolean) - Statistics: Num rows: 6 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: ((a = 3) and (d = 3)) (type: boolean) - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: c (type: double) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: double) - Execution mode: vectorized, llap - LLAP IO: may be used (ACID table) - Map 3 - Map Operator Tree: - TableScan - alias: cmv_mat_view2_n0 - properties: - druid.fieldNames a,c - druid.fieldTypes int,double - druid.query.json {"queryType":"scan","dataSource":"default.cmv_mat_view2_n0","intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"],"columns":["a","c"],"resultFormat":"compactedList"} - druid.query.type scan - Statistics: Num rows: 3 Data size: 36 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 3 Data size: 36 Basic stats: COMPLETE Column stats: NONE - value expressions: a (type: int), c (type: double) - Execution mode: vectorized, llap - LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 - 1 - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 3 Data size: 63 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: double), _col0 (type: int), _col2 (type: double) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 3 Data size: 63 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 3 Data size: 63 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -Warning: Shuffle Join MERGEJOIN[10][tables = [cmv_mat_view2_n0, $hdt$_0]] in Stage 'Reducer 2' is a cross product -PREHOOK: query: SELECT * FROM ( - (SELECT a, c FROM cmv_basetable_n2 WHERE a = 3) table1 - JOIN - (SELECT a, c FROM cmv_basetable_n2 WHERE d = 3) table2 - ON table1.a = table2.a) -PREHOOK: type: QUERY -PREHOOK: Input: default@cmv_basetable_n2 -PREHOOK: Input: default@cmv_mat_view2_n0 -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: SELECT * FROM ( - (SELECT a, c FROM cmv_basetable_n2 WHERE a = 3) table1 - JOIN - (SELECT a, c FROM cmv_basetable_n2 WHERE d = 3) table2 - ON table1.a = table2.a) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@cmv_basetable_n2 -POSTHOOK: Input: default@cmv_mat_view2_n0 -POSTHOOK: Output: hdfs://### HDFS PATH ### -3 15.8 3 978.76 -3 9.8 3 978.76 -3 978.76 3 978.76 -PREHOOK: query: INSERT INTO cmv_basetable_n2 VALUES - (cast(current_timestamp() AS timestamp), 3, 'charlie', 'charlie_c', 15.8, 1) -PREHOOK: type: QUERY -PREHOOK: Input: _dummy_database@_dummy_table -PREHOOK: Output: default@cmv_basetable_n2 -POSTHOOK: query: INSERT INTO cmv_basetable_n2 VALUES - (cast(current_timestamp() AS timestamp), 3, 'charlie', 'charlie_c', 15.8, 1) -POSTHOOK: type: QUERY -POSTHOOK: Input: _dummy_database@_dummy_table -POSTHOOK: Output: default@cmv_basetable_n2 -POSTHOOK: Lineage: cmv_basetable_n2.a SCRIPT [] -POSTHOOK: Lineage: cmv_basetable_n2.b SCRIPT [] -POSTHOOK: Lineage: cmv_basetable_n2.c SCRIPT [] -POSTHOOK: Lineage: cmv_basetable_n2.d SCRIPT [] -POSTHOOK: Lineage: cmv_basetable_n2.t SCRIPT [] -POSTHOOK: Lineage: cmv_basetable_n2.userid SCRIPT [] -Warning: Shuffle Join MERGEJOIN[13][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product -PREHOOK: query: EXPLAIN -SELECT * FROM ( - (SELECT a, c FROM cmv_basetable_n2 WHERE a = 3) table1 - JOIN - (SELECT a, c FROM cmv_basetable_n2 WHERE d = 3) table2 - ON table1.a = table2.a) -PREHOOK: type: QUERY -PREHOOK: Input: default@cmv_basetable_n2 -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN -SELECT * FROM ( - (SELECT a, c FROM cmv_basetable_n2 WHERE a = 3) table1 - JOIN - (SELECT a, c FROM cmv_basetable_n2 WHERE d = 3) table2 - ON table1.a = table2.a) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@cmv_basetable_n2 -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Tez -#### A masked pattern was here #### - Edges: - Reducer 2 <- Map 1 (XPROD_EDGE), Map 3 (XPROD_EDGE) -#### A masked pattern was here #### - Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: cmv_basetable_n2 - filterExpr: (a = 3) (type: boolean) - Statistics: Num rows: 7 Data size: 84 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (a = 3) (type: boolean) - Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: c (type: double) - outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: double) - Execution mode: vectorized, llap - LLAP IO: may be used (ACID table) - Map 3 - Map Operator Tree: - TableScan - alias: cmv_basetable_n2 - filterExpr: ((d = 3) and (a = 3)) (type: boolean) - Statistics: Num rows: 7 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: ((d = 3) and (a = 3)) (type: boolean) - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: c (type: double) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: double) - Execution mode: vectorized, llap - LLAP IO: may be used (ACID table) - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 - 1 - outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: 3 (type: int), _col0 (type: double), 3 (type: int), _col1 (type: double) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 2 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 2 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -Warning: Shuffle Join MERGEJOIN[13][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product -PREHOOK: query: SELECT * FROM ( - (SELECT a, c FROM cmv_basetable_n2 WHERE a = 3) table1 - JOIN - (SELECT a, c FROM cmv_basetable_n2 WHERE d = 3) table2 - ON table1.a = table2.a) -PREHOOK: type: QUERY -PREHOOK: Input: default@cmv_basetable_n2 -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: SELECT * FROM ( - (SELECT a, c FROM cmv_basetable_n2 WHERE a = 3) table1 - JOIN - (SELECT a, c FROM cmv_basetable_n2 WHERE d = 3) table2 - ON table1.a = table2.a) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@cmv_basetable_n2 -POSTHOOK: Output: hdfs://### HDFS PATH ### -3 15.8 3 978.76 -3 15.8 3 978.76 -3 9.8 3 978.76 -3 978.76 3 978.76 -PREHOOK: query: EXPLAIN -ALTER MATERIALIZED VIEW cmv_mat_view2_n0 REBUILD -PREHOOK: type: ALTER_MATERIALIZED_VIEW_REBUILD -PREHOOK: Input: default@cmv_basetable_n2 -PREHOOK: Input: default@cmv_mat_view2_n0 -PREHOOK: Output: default@cmv_mat_view2_n0 -POSTHOOK: query: EXPLAIN -ALTER MATERIALIZED VIEW cmv_mat_view2_n0 REBUILD -POSTHOOK: type: ALTER_MATERIALIZED_VIEW_REBUILD -POSTHOOK: Input: default@cmv_basetable_n2 -POSTHOOK: Input: default@cmv_mat_view2_n0 -POSTHOOK: Output: default@cmv_mat_view2_n0 -STAGE DEPENDENCIES: - Stage-0 is a root stage - Stage-4 depends on stages: Stage-0, Stage-1, Stage-2 - Stage-1 is a root stage - Stage-2 is a root stage - Stage-5 depends on stages: Stage-2 - -STAGE PLANS: - Stage: Stage-0 - Unset Properties - table name: default.cmv_mat_view2_n0 - properties: - COLUMN_STATS_ACCURATE - - Stage: Stage-4 - Materialized View Update - name: default.cmv_mat_view2_n0 - update creation metadata: true - - Stage: Stage-1 - Pre-Insert task - - Stage: Stage-2 - Tez -#### A masked pattern was here #### - Edges: - Map 1 <- Union 2 (CONTAINS) - Map 4 <- Union 2 (CONTAINS) - Reducer 3 <- Union 2 (SIMPLE_EDGE) -#### A masked pattern was here #### - Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: cmv_basetable_n2 - filterExpr: ((a = 3) and (ROW__ID.writeid > 1L)) (type: boolean) - Statistics: Num rows: 7 Data size: 1652 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: ((a = 3) and (ROW__ID.writeid > 1L)) (type: boolean) - Statistics: Num rows: 1 Data size: 236 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: CAST( t AS timestamp with local time zone) (type: timestamp with local time zone), 3 (type: int), b (type: varchar(256)), c (type: double), userid (type: varchar(256)) - outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 1 Data size: 236 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: timestamp with local time zone), _col1 (type: int), _col2 (type: varchar(256)), _col3 (type: double), _col4 (type: varchar(256)), floor_hour(CAST( GenericUDFEpochMilli(_col0) AS TIMESTAMP)) (type: timestamp) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, __time_granularity - Statistics: Num rows: 4 Data size: 1104 Basic stats: COMPLETE Column stats: PARTIAL - Reduce Output Operator - key expressions: __time_granularity (type: timestamp) - null sort order: a - sort order: + - Map-reduce partition columns: __time_granularity (type: timestamp) - Statistics: Num rows: 4 Data size: 1104 Basic stats: COMPLETE Column stats: PARTIAL - value expressions: _col0 (type: timestamp with local time zone), _col1 (type: int), _col2 (type: varchar(256)), _col3 (type: double), _col4 (type: varchar(256)) - Execution mode: llap - LLAP IO: may be used (ACID table) - Map 4 - Map Operator Tree: - TableScan - alias: cmv_mat_view2_n0 - Statistics: Num rows: 3 Data size: 2196 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: vc (type: timestamp with local time zone), a (type: int), b (type: varchar(256)), c (type: double), userid (type: varchar(256)) - outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 3 Data size: 2196 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: timestamp with local time zone), _col1 (type: int), _col2 (type: varchar(256)), _col3 (type: double), _col4 (type: varchar(256)), floor_hour(CAST( GenericUDFEpochMilli(_col0) AS TIMESTAMP)) (type: timestamp) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, __time_granularity - Statistics: Num rows: 4 Data size: 1104 Basic stats: COMPLETE Column stats: PARTIAL - Reduce Output Operator - key expressions: __time_granularity (type: timestamp) - null sort order: a - sort order: + - Map-reduce partition columns: __time_granularity (type: timestamp) - Statistics: Num rows: 4 Data size: 1104 Basic stats: COMPLETE Column stats: PARTIAL - value expressions: _col0 (type: timestamp with local time zone), _col1 (type: int), _col2 (type: varchar(256)), _col3 (type: double), _col4 (type: varchar(256)) - Execution mode: llap - LLAP IO: no inputs - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: timestamp with local time zone), VALUE._col1 (type: int), VALUE._col2 (type: varchar(256)), VALUE._col3 (type: double), VALUE._col4 (type: varchar(256)), KEY.__time_granularity (type: timestamp) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, __time_granularity - Statistics: Num rows: 4 Data size: 1104 Basic stats: COMPLETE Column stats: PARTIAL - File Output Operator - compressed: false - Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 4 Data size: 1104 Basic stats: COMPLETE Column stats: PARTIAL - table: - input format: org.apache.hadoop.hive.druid.io.DruidQueryBasedInputFormat - output format: org.apache.hadoop.hive.druid.io.DruidOutputFormat - serde: org.apache.hadoop.hive.druid.serde.DruidSerDe - name: default.cmv_mat_view2_n0 - Union 2 - Vertex: Union 2 - - Stage: Stage-5 - Commit Insert Hook - is overwrite: true - -PREHOOK: query: ALTER MATERIALIZED VIEW cmv_mat_view2_n0 REBUILD -PREHOOK: type: ALTER_MATERIALIZED_VIEW_REBUILD -PREHOOK: Input: default@cmv_basetable_n2 -PREHOOK: Input: default@cmv_mat_view2_n0 -PREHOOK: Output: default@cmv_mat_view2_n0 -POSTHOOK: query: ALTER MATERIALIZED VIEW cmv_mat_view2_n0 REBUILD -POSTHOOK: type: ALTER_MATERIALIZED_VIEW_REBUILD -POSTHOOK: Input: default@cmv_basetable_n2 -POSTHOOK: Input: default@cmv_mat_view2_n0 -POSTHOOK: Output: default@cmv_mat_view2_n0 -PREHOOK: query: SHOW TBLPROPERTIES cmv_mat_view2_n0 -PREHOOK: type: SHOW_TBLPROPERTIES -POSTHOOK: query: SHOW TBLPROPERTIES cmv_mat_view2_n0 -POSTHOOK: type: SHOW_TBLPROPERTIES -bucketing_version 2 -druid.datasource default.cmv_mat_view2_n0 -druid.segment.granularity HOUR -#### A masked pattern was here #### -numFiles 0 -numFilesErasureCoded 0 -numRows 3 -rawDataSize 0 -storage_handler org.apache.hadoop.hive.druid.DruidStorageHandler -totalSize #Masked# -#### A masked pattern was here #### -Warning: Shuffle Join MERGEJOIN[10][tables = [cmv_mat_view2_n0, $hdt$_0]] in Stage 'Reducer 2' is a cross product -PREHOOK: query: EXPLAIN -SELECT * FROM ( - (SELECT a, c FROM cmv_basetable_n2 WHERE a = 3) table1 - JOIN - (SELECT a, c FROM cmv_basetable_n2 WHERE d = 3) table2 - ON table1.a = table2.a) -PREHOOK: type: QUERY -PREHOOK: Input: default@cmv_basetable_n2 -PREHOOK: Input: default@cmv_mat_view2_n0 -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN -SELECT * FROM ( - (SELECT a, c FROM cmv_basetable_n2 WHERE a = 3) table1 - JOIN - (SELECT a, c FROM cmv_basetable_n2 WHERE d = 3) table2 - ON table1.a = table2.a) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@cmv_basetable_n2 -POSTHOOK: Input: default@cmv_mat_view2_n0 -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Tez -#### A masked pattern was here #### - Edges: - Reducer 2 <- Map 1 (XPROD_EDGE), Map 3 (XPROD_EDGE) -#### A masked pattern was here #### - Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: cmv_basetable_n2 - filterExpr: ((a = 3) and (d = 3)) (type: boolean) - Statistics: Num rows: 7 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: ((a = 3) and (d = 3)) (type: boolean) - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: c (type: double) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: double) - Execution mode: vectorized, llap - LLAP IO: may be used (ACID table) - Map 3 - Map Operator Tree: - TableScan - alias: cmv_mat_view2_n0 - properties: - druid.fieldNames a,c - druid.fieldTypes int,double - druid.query.json {"queryType":"scan","dataSource":"default.cmv_mat_view2_n0","intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"],"columns":["a","c"],"resultFormat":"compactedList"} - druid.query.type scan - Statistics: Num rows: 3 Data size: 36 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 3 Data size: 36 Basic stats: COMPLETE Column stats: NONE - value expressions: a (type: int), c (type: double) - Execution mode: vectorized, llap - LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 - 1 - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 3 Data size: 63 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: double), _col0 (type: int), _col2 (type: double) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 3 Data size: 63 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 3 Data size: 63 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -Warning: Shuffle Join MERGEJOIN[10][tables = [cmv_mat_view2_n0, $hdt$_0]] in Stage 'Reducer 2' is a cross product -PREHOOK: query: SELECT * FROM ( - (SELECT a, c FROM cmv_basetable_n2 WHERE a = 3) table1 - JOIN - (SELECT a, c FROM cmv_basetable_n2 WHERE d = 3) table2 - ON table1.a = table2.a) -PREHOOK: type: QUERY -PREHOOK: Input: default@cmv_basetable_n2 -PREHOOK: Input: default@cmv_mat_view2_n0 -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: SELECT * FROM ( - (SELECT a, c FROM cmv_basetable_n2 WHERE a = 3) table1 - JOIN - (SELECT a, c FROM cmv_basetable_n2 WHERE d = 3) table2 - ON table1.a = table2.a) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@cmv_basetable_n2 -POSTHOOK: Input: default@cmv_mat_view2_n0 -POSTHOOK: Output: hdfs://### HDFS PATH ### -3 15.8 3 978.76 -3 15.8 3 978.76 -3 9.8 3 978.76 -3 978.76 3 978.76 -PREHOOK: query: DROP MATERIALIZED VIEW cmv_mat_view_n2 -PREHOOK: type: DROP_MATERIALIZED_VIEW -PREHOOK: Input: default@cmv_mat_view_n2 -PREHOOK: Output: default@cmv_mat_view_n2 -POSTHOOK: query: DROP MATERIALIZED VIEW cmv_mat_view_n2 -POSTHOOK: type: DROP_MATERIALIZED_VIEW -POSTHOOK: Input: default@cmv_mat_view_n2 -POSTHOOK: Output: default@cmv_mat_view_n2 -PREHOOK: query: DROP MATERIALIZED VIEW cmv_mat_view2_n0 -PREHOOK: type: DROP_MATERIALIZED_VIEW -PREHOOK: Input: default@cmv_mat_view2_n0 -PREHOOK: Output: default@cmv_mat_view2_n0 -POSTHOOK: query: DROP MATERIALIZED VIEW cmv_mat_view2_n0 -POSTHOOK: type: DROP_MATERIALIZED_VIEW -POSTHOOK: Input: default@cmv_mat_view2_n0 -POSTHOOK: Output: default@cmv_mat_view2_n0 -PREHOOK: query: DROP TABLE cmv_basetable_n2 -PREHOOK: type: DROPTABLE -PREHOOK: Input: default@cmv_basetable_n2 -PREHOOK: Output: default@cmv_basetable_n2 -POSTHOOK: query: DROP TABLE cmv_basetable_n2 -POSTHOOK: type: DROPTABLE -POSTHOOK: Input: default@cmv_basetable_n2 -POSTHOOK: Output: default@cmv_basetable_n2 diff --git a/ql/src/test/results/clientpositive/druid/druidmini_semijoin_reduction_all_types.q.out b/ql/src/test/results/clientpositive/druid/druidmini_semijoin_reduction_all_types.q.out deleted file mode 100644 index 0dd2295b73ee..000000000000 --- a/ql/src/test/results/clientpositive/druid/druidmini_semijoin_reduction_all_types.q.out +++ /dev/null @@ -1,2033 +0,0 @@ -PREHOOK: query: DROP TABLE IF EXISTS alltypesorc_small -PREHOOK: type: DROPTABLE -POSTHOOK: query: DROP TABLE IF EXISTS alltypesorc_small -POSTHOOK: type: DROPTABLE -PREHOOK: query: CREATE TABLE alltypesorc_small( - ctinyint TINYINT, - csmallint SMALLINT, - cint INT, - cbigint BIGINT, - cfloat FLOAT, - cdouble DOUBLE, - cstring1 STRING, - cstring2 STRING, - ctimestamp1 TIMESTAMP, - cboolean1 BOOLEAN, - cboolean2 BOOLEAN) - STORED AS ORC -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@alltypesorc_small -POSTHOOK: query: CREATE TABLE alltypesorc_small( - ctinyint TINYINT, - csmallint SMALLINT, - cint INT, - cbigint BIGINT, - cfloat FLOAT, - cdouble DOUBLE, - cstring1 STRING, - cstring2 STRING, - ctimestamp1 TIMESTAMP, - cboolean1 BOOLEAN, - cboolean2 BOOLEAN) - STORED AS ORC -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@alltypesorc_small -PREHOOK: query: Insert into table alltypesorc_small -Select ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, cast(`__time` as timestamp), cboolean1, cboolean2 from druid_table_alltypesorc where cstring2 like '%a%' and cstring1 like '%a%' -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: default@alltypesorc_small -POSTHOOK: query: Insert into table alltypesorc_small -Select ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, cast(`__time` as timestamp), cboolean1, cboolean2 from druid_table_alltypesorc where cstring2 like '%a%' and cstring1 like '%a%' -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: default@alltypesorc_small -POSTHOOK: Lineage: alltypesorc_small.cbigint SIMPLE [(druid_table_alltypesorc)druid_table_alltypesorc.FieldSchema(name:cbigint, type:bigint, comment:from deserializer), ] -POSTHOOK: Lineage: alltypesorc_small.cboolean1 SIMPLE [(druid_table_alltypesorc)druid_table_alltypesorc.FieldSchema(name:cboolean1, type:boolean, comment:from deserializer), ] -POSTHOOK: Lineage: alltypesorc_small.cboolean2 SIMPLE [(druid_table_alltypesorc)druid_table_alltypesorc.FieldSchema(name:cboolean2, type:boolean, comment:from deserializer), ] -POSTHOOK: Lineage: alltypesorc_small.cdouble SIMPLE [(druid_table_alltypesorc)druid_table_alltypesorc.FieldSchema(name:cdouble, type:double, comment:from deserializer), ] -POSTHOOK: Lineage: alltypesorc_small.cfloat SIMPLE [(druid_table_alltypesorc)druid_table_alltypesorc.FieldSchema(name:cfloat, type:float, comment:from deserializer), ] -POSTHOOK: Lineage: alltypesorc_small.cint SIMPLE [(druid_table_alltypesorc)druid_table_alltypesorc.FieldSchema(name:cint, type:int, comment:from deserializer), ] -POSTHOOK: Lineage: alltypesorc_small.csmallint SIMPLE [(druid_table_alltypesorc)druid_table_alltypesorc.FieldSchema(name:csmallint, type:smallint, comment:from deserializer), ] -POSTHOOK: Lineage: alltypesorc_small.cstring1 SIMPLE [(druid_table_alltypesorc)druid_table_alltypesorc.FieldSchema(name:cstring1, type:string, comment:from deserializer), ] -POSTHOOK: Lineage: alltypesorc_small.cstring2 SIMPLE [(druid_table_alltypesorc)druid_table_alltypesorc.FieldSchema(name:cstring2, type:string, comment:from deserializer), ] -POSTHOOK: Lineage: alltypesorc_small.ctimestamp1 SIMPLE [(druid_table_alltypesorc)druid_table_alltypesorc.FieldSchema(name:vc, type:timestamp, comment:from deserializer), ] -POSTHOOK: Lineage: alltypesorc_small.ctinyint SIMPLE [(druid_table_alltypesorc)druid_table_alltypesorc.FieldSchema(name:ctinyint, type:tinyint, comment:from deserializer), ] -PREHOOK: query: Select count(*) from alltypesorc_small -PREHOOK: type: QUERY -PREHOOK: Input: default@alltypesorc_small -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: Select count(*) from alltypesorc_small -POSTHOOK: type: QUERY -POSTHOOK: Input: default@alltypesorc_small -POSTHOOK: Output: hdfs://### HDFS PATH ### -69 -PREHOOK: query: Select count(*) from druid_table_alltypesorc -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: Select count(*) from druid_table_alltypesorc -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -9173 -PREHOOK: query: DESCRIBE druid_table_alltypesorc -PREHOOK: type: DESCTABLE -PREHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: query: DESCRIBE druid_table_alltypesorc -POSTHOOK: type: DESCTABLE -POSTHOOK: Input: default@druid_table_alltypesorc -__time timestamp with local time zone from deserializer -cstring1 string from deserializer -cstring2 string from deserializer -cdouble double from deserializer -cfloat float from deserializer -ctinyint tinyint from deserializer -csmallint smallint from deserializer -cint int from deserializer -cbigint bigint from deserializer -cboolean1 boolean from deserializer -cboolean2 boolean from deserializer -cintstring string from deserializer -cfloatstring string from deserializer -cdoublestring string from deserializer -PREHOOK: query: DESCRIBE alltypesorc_small -PREHOOK: type: DESCTABLE -PREHOOK: Input: default@alltypesorc_small -POSTHOOK: query: DESCRIBE alltypesorc_small -POSTHOOK: type: DESCTABLE -POSTHOOK: Input: default@alltypesorc_small -ctinyint tinyint -csmallint smallint -cint int -cbigint bigint -cfloat float -cdouble double -cstring1 string -cstring2 string -ctimestamp1 timestamp -cboolean1 boolean -cboolean2 boolean -PREHOOK: query: EXPLAIN select count(*) from alltypesorc_small join druid_table_alltypesorc on (alltypesorc_small.cstring1 = druid_table_alltypesorc.cstring1) -PREHOOK: type: QUERY -PREHOOK: Input: default@alltypesorc_small -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN select count(*) from alltypesorc_small join druid_table_alltypesorc on (alltypesorc_small.cstring1 = druid_table_alltypesorc.cstring1) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@alltypesorc_small -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Tez -#### A masked pattern was here #### - Edges: - Map 5 <- Reducer 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) - Reducer 4 <- Map 1 (CUSTOM_SIMPLE_EDGE) -#### A masked pattern was here #### - Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: alltypesorc_small - filterExpr: cstring1 is not null (type: boolean) - Statistics: Num rows: 69 Data size: 6969 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: cstring1 is not null (type: boolean) - Statistics: Num rows: 69 Data size: 6969 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: cstring1 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 69 Data size: 6969 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 69 Data size: 6969 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 69 Data size: 6969 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=69) - minReductionHashAggr: 0.98550725 - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary) - Execution mode: vectorized, llap - LLAP IO: all inputs - Map 5 - Map Operator Tree: - TableScan - alias: druid_table_alltypesorc - filterExpr: (cstring1 BETWEEN DynamicValue(RS_4_alltypesorc_small_cstring1_min) AND DynamicValue(RS_4_alltypesorc_small_cstring1_max) and in_bloom_filter(cstring1, DynamicValue(RS_4_alltypesorc_small_cstring1_bloom_filter))) (type: boolean) - properties: - druid.fieldNames cstring1 - druid.fieldTypes string - druid.query.json {"queryType":"scan","dataSource":{"type":"table","name":"default.druid_table_alltypesorc"},"intervals":{"type":"LegacySegmentSpec","intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"]},"virtualColumns":[],"resultFormat":"compactedList","batchSize":20480,"limit":9223372036854775807,"order":"none","filter":{"type":"and","fields":[{"type":"and","fields":[{"type":"bound","dimension":"cstring1","lower":"DynamicValue(RS_4_alltypesorc_small_cstring1_min)","upper":"DynamicValue(RS_4_alltypesorc_small_cstring1_max)","lowerStrict":false,"upperStrict":false,"extractionFn":null,"ordering":{"type":"lexicographic"}},{"type":"bloom","dimension":"cstring1","bloomKFilter":"BAAAAAgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA","extractionFn":null}]},{"type":"not","field":{"type":"selector","dimension":"cstring1","value":null,"extractionFn":null}}]},"columns":["cstring1"],"legacy":null,"context":null,"descending":false,"granularity":{"type":"all"}} - druid.query.type scan - Statistics: Num rows: 9173 Data size: 1603744 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (cstring1 BETWEEN DynamicValue(RS_4_alltypesorc_small_cstring1_min) AND DynamicValue(RS_4_alltypesorc_small_cstring1_max) and in_bloom_filter(cstring1, DynamicValue(RS_4_alltypesorc_small_cstring1_bloom_filter))) (type: boolean) - Statistics: Num rows: 9173 Data size: 1603744 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: cstring1 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: cstring1 (type: string) - Statistics: Num rows: 9173 Data size: 1603744 Basic stats: COMPLETE Column stats: NONE - Execution mode: vectorized, llap - LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 cstring1 (type: string) - Statistics: Num rows: 10090 Data size: 1764118 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Reducer 3 - Execution mode: vectorized, llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 - Execution mode: vectorized, llap - Reduce Operator Tree: - Group By Operator - aggregations: min(VALUE._col0), max(VALUE._col1), bloom_filter(VALUE._col2, expectedEntries=69) - mode: final - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary) - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: select count(*) from alltypesorc_small join druid_table_alltypesorc on (alltypesorc_small.cstring1 = druid_table_alltypesorc.cstring1) -PREHOOK: type: QUERY -PREHOOK: Input: default@alltypesorc_small -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: select count(*) from alltypesorc_small join druid_table_alltypesorc on (alltypesorc_small.cstring1 = druid_table_alltypesorc.cstring1) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@alltypesorc_small -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -69 -PREHOOK: query: select count(*) from alltypesorc_small join druid_table_alltypesorc on (alltypesorc_small.cstring1 = druid_table_alltypesorc.cstring1) -PREHOOK: type: QUERY -PREHOOK: Input: default@alltypesorc_small -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: select count(*) from alltypesorc_small join druid_table_alltypesorc on (alltypesorc_small.cstring1 = druid_table_alltypesorc.cstring1) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@alltypesorc_small -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -69 -PREHOOK: query: EXPLAIN select count(*) from alltypesorc_small join druid_table_alltypesorc on (alltypesorc_small.ctinyint = druid_table_alltypesorc.ctinyint) -PREHOOK: type: QUERY -PREHOOK: Input: default@alltypesorc_small -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN select count(*) from alltypesorc_small join druid_table_alltypesorc on (alltypesorc_small.ctinyint = druid_table_alltypesorc.ctinyint) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@alltypesorc_small -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Tez -#### A masked pattern was here #### - Edges: - Map 5 <- Reducer 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) - Reducer 4 <- Map 1 (CUSTOM_SIMPLE_EDGE) -#### A masked pattern was here #### - Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: alltypesorc_small - filterExpr: ctinyint is not null (type: boolean) - Statistics: Num rows: 69 Data size: 276 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: ctinyint is not null (type: boolean) - Statistics: Num rows: 69 Data size: 276 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: ctinyint (type: tinyint) - outputColumnNames: _col0 - Statistics: Num rows: 69 Data size: 276 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: tinyint) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: tinyint) - Statistics: Num rows: 69 Data size: 276 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: tinyint) - outputColumnNames: _col0 - Statistics: Num rows: 69 Data size: 276 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=3) - minReductionHashAggr: 0.98550725 - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: tinyint), _col1 (type: tinyint), _col2 (type: binary) - Execution mode: vectorized, llap - LLAP IO: all inputs - Map 5 - Map Operator Tree: - TableScan - alias: druid_table_alltypesorc - filterExpr: (ctinyint BETWEEN DynamicValue(RS_4_alltypesorc_small_ctinyint_min) AND DynamicValue(RS_4_alltypesorc_small_ctinyint_max) and in_bloom_filter(ctinyint, DynamicValue(RS_4_alltypesorc_small_ctinyint_bloom_filter))) (type: boolean) - properties: - druid.fieldNames ctinyint - druid.fieldTypes tinyint - druid.query.json {"queryType":"scan","dataSource":{"type":"table","name":"default.druid_table_alltypesorc"},"intervals":{"type":"LegacySegmentSpec","intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"]},"virtualColumns":[],"resultFormat":"compactedList","batchSize":20480,"limit":9223372036854775807,"order":"none","filter":{"type":"and","fields":[{"type":"and","fields":[{"type":"bound","dimension":"ctinyint","lower":"DynamicValue(RS_4_alltypesorc_small_ctinyint_min)","upper":"DynamicValue(RS_4_alltypesorc_small_ctinyint_max)","lowerStrict":false,"upperStrict":false,"extractionFn":null,"ordering":{"type":"numeric"}},{"type":"bloom","dimension":"ctinyint","bloomKFilter":"BAAAAAgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA","extractionFn":null}]},{"type":"not","field":{"type":"selector","dimension":"ctinyint","value":null,"extractionFn":null}}]},"columns":["ctinyint"],"legacy":null,"context":null,"descending":false,"granularity":{"type":"all"}} - druid.query.type scan - Statistics: Num rows: 9173 Data size: 34864 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (ctinyint BETWEEN DynamicValue(RS_4_alltypesorc_small_ctinyint_min) AND DynamicValue(RS_4_alltypesorc_small_ctinyint_max) and in_bloom_filter(ctinyint, DynamicValue(RS_4_alltypesorc_small_ctinyint_bloom_filter))) (type: boolean) - Statistics: Num rows: 9173 Data size: 34864 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: ctinyint (type: tinyint) - null sort order: z - sort order: + - Map-reduce partition columns: ctinyint (type: tinyint) - Statistics: Num rows: 9173 Data size: 34864 Basic stats: COMPLETE Column stats: NONE - Execution mode: vectorized, llap - LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: tinyint) - 1 ctinyint (type: tinyint) - Statistics: Num rows: 10090 Data size: 38350 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Reducer 3 - Execution mode: vectorized, llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 - Execution mode: vectorized, llap - Reduce Operator Tree: - Group By Operator - aggregations: min(VALUE._col0), max(VALUE._col1), bloom_filter(VALUE._col2, expectedEntries=3) - mode: final - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: tinyint), _col1 (type: tinyint), _col2 (type: binary) - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: select count(*) from alltypesorc_small join druid_table_alltypesorc on (alltypesorc_small.ctinyint = druid_table_alltypesorc.ctinyint) -PREHOOK: type: QUERY -PREHOOK: Input: default@alltypesorc_small -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: select count(*) from alltypesorc_small join druid_table_alltypesorc on (alltypesorc_small.ctinyint = druid_table_alltypesorc.ctinyint) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@alltypesorc_small -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -73895 -PREHOOK: query: select count(*) from alltypesorc_small join druid_table_alltypesorc on (alltypesorc_small.ctinyint = druid_table_alltypesorc.ctinyint) -PREHOOK: type: QUERY -PREHOOK: Input: default@alltypesorc_small -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: select count(*) from alltypesorc_small join druid_table_alltypesorc on (alltypesorc_small.ctinyint = druid_table_alltypesorc.ctinyint) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@alltypesorc_small -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -73895 -PREHOOK: query: EXPLAIN select count(*) from alltypesorc_small join druid_table_alltypesorc on (alltypesorc_small.csmallint = druid_table_alltypesorc.csmallint) -PREHOOK: type: QUERY -PREHOOK: Input: default@alltypesorc_small -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN select count(*) from alltypesorc_small join druid_table_alltypesorc on (alltypesorc_small.csmallint = druid_table_alltypesorc.csmallint) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@alltypesorc_small -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Tez -#### A masked pattern was here #### - Edges: - Map 5 <- Reducer 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) - Reducer 4 <- Map 1 (CUSTOM_SIMPLE_EDGE) -#### A masked pattern was here #### - Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: alltypesorc_small - filterExpr: csmallint is not null (type: boolean) - Statistics: Num rows: 69 Data size: 276 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: csmallint is not null (type: boolean) - Statistics: Num rows: 69 Data size: 276 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: csmallint (type: smallint) - outputColumnNames: _col0 - Statistics: Num rows: 69 Data size: 276 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: smallint) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: smallint) - Statistics: Num rows: 69 Data size: 276 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: smallint) - outputColumnNames: _col0 - Statistics: Num rows: 69 Data size: 276 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=1) - minReductionHashAggr: 0.98550725 - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: smallint), _col1 (type: smallint), _col2 (type: binary) - Execution mode: vectorized, llap - LLAP IO: all inputs - Map 5 - Map Operator Tree: - TableScan - alias: druid_table_alltypesorc - filterExpr: (csmallint BETWEEN DynamicValue(RS_4_alltypesorc_small_csmallint_min) AND DynamicValue(RS_4_alltypesorc_small_csmallint_max) and in_bloom_filter(csmallint, DynamicValue(RS_4_alltypesorc_small_csmallint_bloom_filter))) (type: boolean) - properties: - druid.fieldNames csmallint - druid.fieldTypes smallint - druid.query.json {"queryType":"scan","dataSource":{"type":"table","name":"default.druid_table_alltypesorc"},"intervals":{"type":"LegacySegmentSpec","intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"]},"virtualColumns":[],"resultFormat":"compactedList","batchSize":20480,"limit":9223372036854775807,"order":"none","filter":{"type":"and","fields":[{"type":"and","fields":[{"type":"bound","dimension":"csmallint","lower":"DynamicValue(RS_4_alltypesorc_small_csmallint_min)","upper":"DynamicValue(RS_4_alltypesorc_small_csmallint_max)","lowerStrict":false,"upperStrict":false,"extractionFn":null,"ordering":{"type":"numeric"}},{"type":"bloom","dimension":"csmallint","bloomKFilter":"BAAAAAgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA","extractionFn":null}]},{"type":"not","field":{"type":"selector","dimension":"csmallint","value":null,"extractionFn":null}}]},"columns":["csmallint"],"legacy":null,"context":null,"descending":false,"granularity":{"type":"all"}} - druid.query.type scan - Statistics: Num rows: 9173 Data size: 34864 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (csmallint BETWEEN DynamicValue(RS_4_alltypesorc_small_csmallint_min) AND DynamicValue(RS_4_alltypesorc_small_csmallint_max) and in_bloom_filter(csmallint, DynamicValue(RS_4_alltypesorc_small_csmallint_bloom_filter))) (type: boolean) - Statistics: Num rows: 9173 Data size: 34864 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: csmallint (type: smallint) - null sort order: z - sort order: + - Map-reduce partition columns: csmallint (type: smallint) - Statistics: Num rows: 9173 Data size: 34864 Basic stats: COMPLETE Column stats: NONE - Execution mode: vectorized, llap - LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: smallint) - 1 csmallint (type: smallint) - Statistics: Num rows: 10090 Data size: 38350 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Reducer 3 - Execution mode: vectorized, llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 - Execution mode: vectorized, llap - Reduce Operator Tree: - Group By Operator - aggregations: min(VALUE._col0), max(VALUE._col1), bloom_filter(VALUE._col2, expectedEntries=1) - mode: final - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: smallint), _col1 (type: smallint), _col2 (type: binary) - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: select count(*) from alltypesorc_small join druid_table_alltypesorc on (alltypesorc_small.csmallint = druid_table_alltypesorc.csmallint) -PREHOOK: type: QUERY -PREHOOK: Input: default@alltypesorc_small -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: select count(*) from alltypesorc_small join druid_table_alltypesorc on (alltypesorc_small.csmallint = druid_table_alltypesorc.csmallint) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@alltypesorc_small -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -211968 -PREHOOK: query: select count(*) from alltypesorc_small join druid_table_alltypesorc on (alltypesorc_small.csmallint = druid_table_alltypesorc.csmallint) -PREHOOK: type: QUERY -PREHOOK: Input: default@alltypesorc_small -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: select count(*) from alltypesorc_small join druid_table_alltypesorc on (alltypesorc_small.csmallint = druid_table_alltypesorc.csmallint) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@alltypesorc_small -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -211968 -PREHOOK: query: EXPLAIN select count(*) from alltypesorc_small join druid_table_alltypesorc on (alltypesorc_small.cint = druid_table_alltypesorc.cint) -PREHOOK: type: QUERY -PREHOOK: Input: default@alltypesorc_small -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN select count(*) from alltypesorc_small join druid_table_alltypesorc on (alltypesorc_small.cint = druid_table_alltypesorc.cint) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@alltypesorc_small -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Tez -#### A masked pattern was here #### - Edges: - Map 5 <- Reducer 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) - Reducer 4 <- Map 1 (CUSTOM_SIMPLE_EDGE) -#### A masked pattern was here #### - Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: alltypesorc_small - filterExpr: cint is not null (type: boolean) - Statistics: Num rows: 69 Data size: 276 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: cint is not null (type: boolean) - Statistics: Num rows: 69 Data size: 276 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: cint (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 69 Data size: 276 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 69 Data size: 276 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 69 Data size: 276 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=69) - minReductionHashAggr: 0.98550725 - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: binary) - Execution mode: vectorized, llap - LLAP IO: all inputs - Map 5 - Map Operator Tree: - TableScan - alias: druid_table_alltypesorc - filterExpr: (cint BETWEEN DynamicValue(RS_4_alltypesorc_small_cint_min) AND DynamicValue(RS_4_alltypesorc_small_cint_max) and in_bloom_filter(cint, DynamicValue(RS_4_alltypesorc_small_cint_bloom_filter))) (type: boolean) - properties: - druid.fieldNames cint - druid.fieldTypes int - druid.query.json {"queryType":"scan","dataSource":{"type":"table","name":"default.druid_table_alltypesorc"},"intervals":{"type":"LegacySegmentSpec","intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"]},"virtualColumns":[],"resultFormat":"compactedList","batchSize":20480,"limit":9223372036854775807,"order":"none","filter":{"type":"and","fields":[{"type":"and","fields":[{"type":"bound","dimension":"cint","lower":"DynamicValue(RS_4_alltypesorc_small_cint_min)","upper":"DynamicValue(RS_4_alltypesorc_small_cint_max)","lowerStrict":false,"upperStrict":false,"extractionFn":null,"ordering":{"type":"numeric"}},{"type":"bloom","dimension":"cint","bloomKFilter":"BAAAAAgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA","extractionFn":null}]},{"type":"not","field":{"type":"selector","dimension":"cint","value":null,"extractionFn":null}}]},"columns":["cint"],"legacy":null,"context":null,"descending":false,"granularity":{"type":"all"}} - druid.query.type scan - Statistics: Num rows: 9173 Data size: 34864 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (cint BETWEEN DynamicValue(RS_4_alltypesorc_small_cint_min) AND DynamicValue(RS_4_alltypesorc_small_cint_max) and in_bloom_filter(cint, DynamicValue(RS_4_alltypesorc_small_cint_bloom_filter))) (type: boolean) - Statistics: Num rows: 9173 Data size: 34864 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: cint (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: cint (type: int) - Statistics: Num rows: 9173 Data size: 34864 Basic stats: COMPLETE Column stats: NONE - Execution mode: vectorized, llap - LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: int) - 1 cint (type: int) - Statistics: Num rows: 10090 Data size: 38350 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Reducer 3 - Execution mode: vectorized, llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 - Execution mode: vectorized, llap - Reduce Operator Tree: - Group By Operator - aggregations: min(VALUE._col0), max(VALUE._col1), bloom_filter(VALUE._col2, expectedEntries=69) - mode: final - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: binary) - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: select count(*) from alltypesorc_small join druid_table_alltypesorc on (alltypesorc_small.cint = druid_table_alltypesorc.cint) -PREHOOK: type: QUERY -PREHOOK: Input: default@alltypesorc_small -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: select count(*) from alltypesorc_small join druid_table_alltypesorc on (alltypesorc_small.cint = druid_table_alltypesorc.cint) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@alltypesorc_small -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -69 -PREHOOK: query: select count(*) from alltypesorc_small join druid_table_alltypesorc on (alltypesorc_small.cint = druid_table_alltypesorc.cint) -PREHOOK: type: QUERY -PREHOOK: Input: default@alltypesorc_small -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: select count(*) from alltypesorc_small join druid_table_alltypesorc on (alltypesorc_small.cint = druid_table_alltypesorc.cint) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@alltypesorc_small -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -69 -PREHOOK: query: EXPLAIN select count(*) from alltypesorc_small join druid_table_alltypesorc on (alltypesorc_small.cbigint = druid_table_alltypesorc.cbigint) -PREHOOK: type: QUERY -PREHOOK: Input: default@alltypesorc_small -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN select count(*) from alltypesorc_small join druid_table_alltypesorc on (alltypesorc_small.cbigint = druid_table_alltypesorc.cbigint) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@alltypesorc_small -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Tez -#### A masked pattern was here #### - Edges: - Map 5 <- Reducer 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) - Reducer 4 <- Map 1 (CUSTOM_SIMPLE_EDGE) -#### A masked pattern was here #### - Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: alltypesorc_small - filterExpr: cbigint is not null (type: boolean) - Statistics: Num rows: 69 Data size: 552 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: cbigint is not null (type: boolean) - Statistics: Num rows: 69 Data size: 552 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: cbigint (type: bigint) - outputColumnNames: _col0 - Statistics: Num rows: 69 Data size: 552 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: bigint) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: bigint) - Statistics: Num rows: 69 Data size: 552 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: bigint) - outputColumnNames: _col0 - Statistics: Num rows: 69 Data size: 552 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=69) - minReductionHashAggr: 0.98550725 - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: binary) - Execution mode: vectorized, llap - LLAP IO: all inputs - Map 5 - Map Operator Tree: - TableScan - alias: druid_table_alltypesorc - filterExpr: (cbigint BETWEEN DynamicValue(RS_4_alltypesorc_small_cbigint_min) AND DynamicValue(RS_4_alltypesorc_small_cbigint_max) and in_bloom_filter(cbigint, DynamicValue(RS_4_alltypesorc_small_cbigint_bloom_filter))) (type: boolean) - properties: - druid.fieldNames cbigint - druid.fieldTypes bigint - druid.query.json {"queryType":"scan","dataSource":{"type":"table","name":"default.druid_table_alltypesorc"},"intervals":{"type":"LegacySegmentSpec","intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"]},"virtualColumns":[],"resultFormat":"compactedList","batchSize":20480,"limit":9223372036854775807,"order":"none","filter":{"type":"and","fields":[{"type":"and","fields":[{"type":"bound","dimension":"cbigint","lower":"DynamicValue(RS_4_alltypesorc_small_cbigint_min)","upper":"DynamicValue(RS_4_alltypesorc_small_cbigint_max)","lowerStrict":false,"upperStrict":false,"extractionFn":null,"ordering":{"type":"numeric"}},{"type":"bloom","dimension":"cbigint","bloomKFilter":"BAAAAAgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA","extractionFn":null}]},{"type":"not","field":{"type":"selector","dimension":"cbigint","value":null,"extractionFn":null}}]},"columns":["cbigint"],"legacy":null,"context":null,"descending":false,"granularity":{"type":"all"}} - druid.query.type scan - Statistics: Num rows: 9173 Data size: 69728 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (cbigint BETWEEN DynamicValue(RS_4_alltypesorc_small_cbigint_min) AND DynamicValue(RS_4_alltypesorc_small_cbigint_max) and in_bloom_filter(cbigint, DynamicValue(RS_4_alltypesorc_small_cbigint_bloom_filter))) (type: boolean) - Statistics: Num rows: 9173 Data size: 69728 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: cbigint (type: bigint) - null sort order: z - sort order: + - Map-reduce partition columns: cbigint (type: bigint) - Statistics: Num rows: 9173 Data size: 69728 Basic stats: COMPLETE Column stats: NONE - Execution mode: vectorized, llap - LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: bigint) - 1 cbigint (type: bigint) - Statistics: Num rows: 10090 Data size: 76700 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Reducer 3 - Execution mode: vectorized, llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 - Execution mode: vectorized, llap - Reduce Operator Tree: - Group By Operator - aggregations: min(VALUE._col0), max(VALUE._col1), bloom_filter(VALUE._col2, expectedEntries=69) - mode: final - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: binary) - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: select count(*) from alltypesorc_small join druid_table_alltypesorc on (alltypesorc_small.cbigint = druid_table_alltypesorc.cbigint) -PREHOOK: type: QUERY -PREHOOK: Input: default@alltypesorc_small -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: select count(*) from alltypesorc_small join druid_table_alltypesorc on (alltypesorc_small.cbigint = druid_table_alltypesorc.cbigint) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@alltypesorc_small -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -69 -PREHOOK: query: select count(*) from alltypesorc_small join druid_table_alltypesorc on (alltypesorc_small.cbigint = druid_table_alltypesorc.cbigint) -PREHOOK: type: QUERY -PREHOOK: Input: default@alltypesorc_small -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: select count(*) from alltypesorc_small join druid_table_alltypesorc on (alltypesorc_small.cbigint = druid_table_alltypesorc.cbigint) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@alltypesorc_small -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -69 -PREHOOK: query: EXPLAIN select count(*) from alltypesorc_small join druid_table_alltypesorc on (alltypesorc_small.cfloat = druid_table_alltypesorc.cfloat) -PREHOOK: type: QUERY -PREHOOK: Input: default@alltypesorc_small -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN select count(*) from alltypesorc_small join druid_table_alltypesorc on (alltypesorc_small.cfloat = druid_table_alltypesorc.cfloat) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@alltypesorc_small -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Tez -#### A masked pattern was here #### - Edges: - Map 5 <- Reducer 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) - Reducer 4 <- Map 1 (CUSTOM_SIMPLE_EDGE) -#### A masked pattern was here #### - Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: alltypesorc_small - filterExpr: cfloat is not null (type: boolean) - Statistics: Num rows: 69 Data size: 276 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: cfloat is not null (type: boolean) - Statistics: Num rows: 69 Data size: 276 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: cfloat (type: float) - outputColumnNames: _col0 - Statistics: Num rows: 69 Data size: 276 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: float) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: float) - Statistics: Num rows: 69 Data size: 276 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: float) - outputColumnNames: _col0 - Statistics: Num rows: 69 Data size: 276 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=3) - minReductionHashAggr: 0.98550725 - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: float), _col1 (type: float), _col2 (type: binary) - Execution mode: vectorized, llap - LLAP IO: all inputs - Map 5 - Map Operator Tree: - TableScan - alias: druid_table_alltypesorc - filterExpr: (cfloat BETWEEN DynamicValue(RS_4_alltypesorc_small_cfloat_min) AND DynamicValue(RS_4_alltypesorc_small_cfloat_max) and in_bloom_filter(cfloat, DynamicValue(RS_4_alltypesorc_small_cfloat_bloom_filter))) (type: boolean) - properties: - druid.fieldNames cfloat - druid.fieldTypes float - druid.query.json {"queryType":"scan","dataSource":{"type":"table","name":"default.druid_table_alltypesorc"},"intervals":{"type":"LegacySegmentSpec","intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"]},"virtualColumns":[],"resultFormat":"compactedList","batchSize":20480,"limit":9223372036854775807,"order":"none","filter":{"type":"not","field":{"type":"selector","dimension":"cfloat","value":null,"extractionFn":null}},"columns":["cfloat"],"legacy":null,"context":null,"descending":false,"granularity":{"type":"all"}} - druid.query.type scan - Statistics: Num rows: 9173 Data size: 34864 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (cfloat BETWEEN DynamicValue(RS_4_alltypesorc_small_cfloat_min) AND DynamicValue(RS_4_alltypesorc_small_cfloat_max) and in_bloom_filter(cfloat, DynamicValue(RS_4_alltypesorc_small_cfloat_bloom_filter))) (type: boolean) - Statistics: Num rows: 9173 Data size: 34864 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: cfloat (type: float) - null sort order: z - sort order: + - Map-reduce partition columns: cfloat (type: float) - Statistics: Num rows: 9173 Data size: 34864 Basic stats: COMPLETE Column stats: NONE - Execution mode: vectorized, llap - LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: float) - 1 cfloat (type: float) - Statistics: Num rows: 10090 Data size: 38350 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Reducer 3 - Execution mode: vectorized, llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 - Execution mode: vectorized, llap - Reduce Operator Tree: - Group By Operator - aggregations: min(VALUE._col0), max(VALUE._col1), bloom_filter(VALUE._col2, expectedEntries=3) - mode: final - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: float), _col1 (type: float), _col2 (type: binary) - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: select count(*) from alltypesorc_small join druid_table_alltypesorc on (alltypesorc_small.cfloat = druid_table_alltypesorc.cfloat) -PREHOOK: type: QUERY -PREHOOK: Input: default@alltypesorc_small -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: select count(*) from alltypesorc_small join druid_table_alltypesorc on (alltypesorc_small.cfloat = druid_table_alltypesorc.cfloat) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@alltypesorc_small -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -73895 -PREHOOK: query: select count(*) from alltypesorc_small join druid_table_alltypesorc on (alltypesorc_small.cfloat = druid_table_alltypesorc.cfloat) -PREHOOK: type: QUERY -PREHOOK: Input: default@alltypesorc_small -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: select count(*) from alltypesorc_small join druid_table_alltypesorc on (alltypesorc_small.cfloat = druid_table_alltypesorc.cfloat) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@alltypesorc_small -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -73895 -PREHOOK: query: EXPLAIN select count(*) from alltypesorc_small join druid_table_alltypesorc on (alltypesorc_small.cdouble = druid_table_alltypesorc.cdouble) -PREHOOK: type: QUERY -PREHOOK: Input: default@alltypesorc_small -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN select count(*) from alltypesorc_small join druid_table_alltypesorc on (alltypesorc_small.cdouble = druid_table_alltypesorc.cdouble) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@alltypesorc_small -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Tez -#### A masked pattern was here #### - Edges: - Map 5 <- Reducer 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) - Reducer 4 <- Map 1 (CUSTOM_SIMPLE_EDGE) -#### A masked pattern was here #### - Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: alltypesorc_small - filterExpr: cdouble is not null (type: boolean) - Statistics: Num rows: 69 Data size: 552 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: cdouble is not null (type: boolean) - Statistics: Num rows: 69 Data size: 552 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: cdouble (type: double) - outputColumnNames: _col0 - Statistics: Num rows: 69 Data size: 552 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: double) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: double) - Statistics: Num rows: 69 Data size: 552 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: double) - outputColumnNames: _col0 - Statistics: Num rows: 69 Data size: 552 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=1) - minReductionHashAggr: 0.98550725 - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: double), _col1 (type: double), _col2 (type: binary) - Execution mode: vectorized, llap - LLAP IO: all inputs - Map 5 - Map Operator Tree: - TableScan - alias: druid_table_alltypesorc - filterExpr: (cdouble BETWEEN DynamicValue(RS_4_alltypesorc_small_cdouble_min) AND DynamicValue(RS_4_alltypesorc_small_cdouble_max) and in_bloom_filter(cdouble, DynamicValue(RS_4_alltypesorc_small_cdouble_bloom_filter))) (type: boolean) - properties: - druid.fieldNames cdouble - druid.fieldTypes double - druid.query.json {"queryType":"scan","dataSource":{"type":"table","name":"default.druid_table_alltypesorc"},"intervals":{"type":"LegacySegmentSpec","intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"]},"virtualColumns":[],"resultFormat":"compactedList","batchSize":20480,"limit":9223372036854775807,"order":"none","filter":{"type":"and","fields":[{"type":"and","fields":[{"type":"bound","dimension":"cdouble","lower":"DynamicValue(RS_4_alltypesorc_small_cdouble_min)","upper":"DynamicValue(RS_4_alltypesorc_small_cdouble_max)","lowerStrict":false,"upperStrict":false,"extractionFn":null,"ordering":{"type":"numeric"}},{"type":"bloom","dimension":"cdouble","bloomKFilter":"BAAAAAgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA","extractionFn":null}]},{"type":"not","field":{"type":"selector","dimension":"cdouble","value":null,"extractionFn":null}}]},"columns":["cdouble"],"legacy":null,"context":null,"descending":false,"granularity":{"type":"all"}} - druid.query.type scan - Statistics: Num rows: 9173 Data size: 69728 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (cdouble BETWEEN DynamicValue(RS_4_alltypesorc_small_cdouble_min) AND DynamicValue(RS_4_alltypesorc_small_cdouble_max) and in_bloom_filter(cdouble, DynamicValue(RS_4_alltypesorc_small_cdouble_bloom_filter))) (type: boolean) - Statistics: Num rows: 9173 Data size: 69728 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: cdouble (type: double) - null sort order: z - sort order: + - Map-reduce partition columns: cdouble (type: double) - Statistics: Num rows: 9173 Data size: 69728 Basic stats: COMPLETE Column stats: NONE - Execution mode: vectorized, llap - LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: double) - 1 cdouble (type: double) - Statistics: Num rows: 10090 Data size: 76700 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Reducer 3 - Execution mode: vectorized, llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 - Execution mode: vectorized, llap - Reduce Operator Tree: - Group By Operator - aggregations: min(VALUE._col0), max(VALUE._col1), bloom_filter(VALUE._col2, expectedEntries=1) - mode: final - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: double), _col1 (type: double), _col2 (type: binary) - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: select count(*) from alltypesorc_small join druid_table_alltypesorc on (alltypesorc_small.cdouble = druid_table_alltypesorc.cdouble) -PREHOOK: type: QUERY -PREHOOK: Input: default@alltypesorc_small -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: select count(*) from alltypesorc_small join druid_table_alltypesorc on (alltypesorc_small.cdouble = druid_table_alltypesorc.cdouble) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@alltypesorc_small -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -211968 -PREHOOK: query: EXPLAIN select count(*) from alltypesorc_small join druid_table_alltypesorc on (alltypesorc_small.ctimestamp1 = druid_table_alltypesorc.`__time`) -PREHOOK: type: QUERY -PREHOOK: Input: default@alltypesorc_small -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN select count(*) from alltypesorc_small join druid_table_alltypesorc on (alltypesorc_small.ctimestamp1 = druid_table_alltypesorc.`__time`) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@alltypesorc_small -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Tez -#### A masked pattern was here #### - Edges: - Map 5 <- Reducer 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) - Reducer 4 <- Map 1 (CUSTOM_SIMPLE_EDGE) -#### A masked pattern was here #### - Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: alltypesorc_small - filterExpr: CAST( ctimestamp1 AS timestamp with local time zone) is not null (type: boolean) - Statistics: Num rows: 69 Data size: 2760 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: CAST( ctimestamp1 AS timestamp with local time zone) is not null (type: boolean) - Statistics: Num rows: 69 Data size: 2760 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: CAST( ctimestamp1 AS timestamp with local time zone) (type: timestamp with local time zone) - outputColumnNames: _col0 - Statistics: Num rows: 69 Data size: 2760 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: timestamp with local time zone) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: timestamp with local time zone) - Statistics: Num rows: 69 Data size: 2760 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: timestamp with local time zone) - outputColumnNames: _col0 - Statistics: Num rows: 69 Data size: 2760 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=69) - minReductionHashAggr: 0.98550725 - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 120 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 120 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: timestamp with local time zone), _col1 (type: timestamp with local time zone), _col2 (type: binary) - Execution mode: llap - LLAP IO: all inputs - Map 5 - Map Operator Tree: - TableScan - alias: druid_table_alltypesorc - filterExpr: (vc BETWEEN DynamicValue(RS_4_alltypesorc_small_ctimestamp1_min) AND DynamicValue(RS_4_alltypesorc_small_ctimestamp1_max) and in_bloom_filter(vc, DynamicValue(RS_4_alltypesorc_small_ctimestamp1_bloom_filter))) (type: boolean) - properties: - druid.fieldNames vc - druid.fieldTypes timestamp with local time zone - druid.query.json {"queryType":"scan","dataSource":{"type":"table","name":"default.druid_table_alltypesorc"},"intervals":{"type":"LegacySegmentSpec","intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"]},"virtualColumns":[{"type":"expression","name":"vc","expression":"\"__time\"","outputType":"LONG"}],"resultFormat":"compactedList","batchSize":20480,"limit":9223372036854775807,"order":"none","filter":null,"columns":["vc"],"legacy":null,"context":null,"descending":false,"granularity":{"type":"all"}} - druid.query.type scan - Statistics: Num rows: 9173 Data size: 348640 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (vc BETWEEN DynamicValue(RS_4_alltypesorc_small_ctimestamp1_min) AND DynamicValue(RS_4_alltypesorc_small_ctimestamp1_max) and in_bloom_filter(vc, DynamicValue(RS_4_alltypesorc_small_ctimestamp1_bloom_filter))) (type: boolean) - Statistics: Num rows: 9173 Data size: 348640 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: vc (type: timestamp with local time zone) - null sort order: z - sort order: + - Map-reduce partition columns: vc (type: timestamp with local time zone) - Statistics: Num rows: 9173 Data size: 348640 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: timestamp with local time zone) - 1 vc (type: timestamp with local time zone) - Statistics: Num rows: 10090 Data size: 383504 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Reducer 3 - Execution mode: vectorized, llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: min(VALUE._col0), max(VALUE._col1), bloom_filter(VALUE._col2, expectedEntries=69) - mode: final - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 120 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 120 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: timestamp with local time zone), _col1 (type: timestamp with local time zone), _col2 (type: binary) - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: select count(*) from alltypesorc_small join druid_table_alltypesorc on (alltypesorc_small.ctimestamp1 = cast(druid_table_alltypesorc.`__time` as timestamp)) -PREHOOK: type: QUERY -PREHOOK: Input: default@alltypesorc_small -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: select count(*) from alltypesorc_small join druid_table_alltypesorc on (alltypesorc_small.ctimestamp1 = cast(druid_table_alltypesorc.`__time` as timestamp)) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@alltypesorc_small -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -434493 -PREHOOK: query: select count(*) from alltypesorc_small join druid_table_alltypesorc on (alltypesorc_small.ctimestamp1 = cast(druid_table_alltypesorc.`__time` as timestamp)) -PREHOOK: type: QUERY -PREHOOK: Input: default@alltypesorc_small -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: select count(*) from alltypesorc_small join druid_table_alltypesorc on (alltypesorc_small.ctimestamp1 = cast(druid_table_alltypesorc.`__time` as timestamp)) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@alltypesorc_small -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -434493 -PREHOOK: query: EXPLAIN select count(*) from alltypesorc_small join druid_table_alltypesorc on (alltypesorc_small.cboolean1 = druid_table_alltypesorc.cboolean1) -PREHOOK: type: QUERY -PREHOOK: Input: default@alltypesorc_small -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN select count(*) from alltypesorc_small join druid_table_alltypesorc on (alltypesorc_small.cboolean1 = druid_table_alltypesorc.cboolean1) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@alltypesorc_small -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Tez -#### A masked pattern was here #### - Edges: - Map 5 <- Reducer 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) - Reducer 4 <- Map 1 (CUSTOM_SIMPLE_EDGE) -#### A masked pattern was here #### - Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: alltypesorc_small - filterExpr: cboolean1 is not null (type: boolean) - Statistics: Num rows: 69 Data size: 276 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: cboolean1 is not null (type: boolean) - Statistics: Num rows: 69 Data size: 276 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: cboolean1 (type: boolean) - outputColumnNames: _col0 - Statistics: Num rows: 69 Data size: 276 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: boolean) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: boolean) - Statistics: Num rows: 69 Data size: 276 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: boolean) - outputColumnNames: _col0 - Statistics: Num rows: 69 Data size: 276 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=1) - minReductionHashAggr: 0.98550725 - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: boolean), _col1 (type: boolean), _col2 (type: binary) - Execution mode: vectorized, llap - LLAP IO: all inputs - Map 5 - Map Operator Tree: - TableScan - alias: druid_table_alltypesorc - filterExpr: (cboolean1 BETWEEN DynamicValue(RS_4_alltypesorc_small_cboolean1_min) AND DynamicValue(RS_4_alltypesorc_small_cboolean1_max) and in_bloom_filter(cboolean1, DynamicValue(RS_4_alltypesorc_small_cboolean1_bloom_filter))) (type: boolean) - properties: - druid.fieldNames cboolean1 - druid.fieldTypes boolean - druid.query.json {"queryType":"scan","dataSource":{"type":"table","name":"default.druid_table_alltypesorc"},"intervals":{"type":"LegacySegmentSpec","intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"]},"virtualColumns":[],"resultFormat":"compactedList","batchSize":20480,"limit":9223372036854775807,"order":"none","filter":{"type":"not","field":{"type":"selector","dimension":"cboolean1","value":null,"extractionFn":null}},"columns":["cboolean1"],"legacy":null,"context":null,"descending":false,"granularity":{"type":"all"}} - druid.query.type scan - Statistics: Num rows: 9173 Data size: 34864 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (cboolean1 BETWEEN DynamicValue(RS_4_alltypesorc_small_cboolean1_min) AND DynamicValue(RS_4_alltypesorc_small_cboolean1_max) and in_bloom_filter(cboolean1, DynamicValue(RS_4_alltypesorc_small_cboolean1_bloom_filter))) (type: boolean) - Statistics: Num rows: 9173 Data size: 34864 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: cboolean1 (type: boolean) - null sort order: z - sort order: + - Map-reduce partition columns: cboolean1 (type: boolean) - Statistics: Num rows: 9173 Data size: 34864 Basic stats: COMPLETE Column stats: NONE - Execution mode: vectorized, llap - LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: boolean) - 1 cboolean1 (type: boolean) - Statistics: Num rows: 10090 Data size: 38350 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Reducer 3 - Execution mode: vectorized, llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 - Execution mode: vectorized, llap - Reduce Operator Tree: - Group By Operator - aggregations: min(VALUE._col0), max(VALUE._col1), bloom_filter(VALUE._col2, expectedEntries=1) - mode: final - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: boolean), _col1 (type: boolean), _col2 (type: binary) - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: select count(*) from alltypesorc_small join druid_table_alltypesorc on (alltypesorc_small.cboolean1 = druid_table_alltypesorc.cboolean1) -PREHOOK: type: QUERY -PREHOOK: Input: default@alltypesorc_small -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: select count(*) from alltypesorc_small join druid_table_alltypesorc on (alltypesorc_small.cboolean1 = druid_table_alltypesorc.cboolean1) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@alltypesorc_small -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -418071 -PREHOOK: query: select count(*) from alltypesorc_small join druid_table_alltypesorc on (alltypesorc_small.cboolean1 = druid_table_alltypesorc.cboolean1) -PREHOOK: type: QUERY -PREHOOK: Input: default@alltypesorc_small -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: select count(*) from alltypesorc_small join druid_table_alltypesorc on (alltypesorc_small.cboolean1 = druid_table_alltypesorc.cboolean1) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@alltypesorc_small -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -418071 -PREHOOK: query: EXPLAIN select count(*) from alltypesorc_small join druid_table_alltypesorc on (cast(alltypesorc_small.cint as string) = druid_table_alltypesorc.cintstring) -PREHOOK: type: QUERY -PREHOOK: Input: default@alltypesorc_small -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN select count(*) from alltypesorc_small join druid_table_alltypesorc on (cast(alltypesorc_small.cint as string) = druid_table_alltypesorc.cintstring) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@alltypesorc_small -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Tez -#### A masked pattern was here #### - Edges: - Map 5 <- Reducer 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) - Reducer 4 <- Map 1 (CUSTOM_SIMPLE_EDGE) -#### A masked pattern was here #### - Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: alltypesorc_small - filterExpr: CAST( cint AS STRING) is not null (type: boolean) - Statistics: Num rows: 69 Data size: 276 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: CAST( cint AS STRING) is not null (type: boolean) - Statistics: Num rows: 69 Data size: 276 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: CAST( cint AS STRING) (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 69 Data size: 12696 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 69 Data size: 12696 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 69 Data size: 12696 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=69) - minReductionHashAggr: 0.98550725 - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary) - Execution mode: vectorized, llap - LLAP IO: all inputs - Map 5 - Map Operator Tree: - TableScan - alias: druid_table_alltypesorc - filterExpr: (cintstring BETWEEN DynamicValue(RS_4_alltypesorc_small_cint_min) AND DynamicValue(RS_4_alltypesorc_small_cint_max) and in_bloom_filter(cintstring, DynamicValue(RS_4_alltypesorc_small_cint_bloom_filter))) (type: boolean) - properties: - druid.fieldNames cintstring - druid.fieldTypes string - druid.query.json {"queryType":"scan","dataSource":{"type":"table","name":"default.druid_table_alltypesorc"},"intervals":{"type":"LegacySegmentSpec","intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"]},"virtualColumns":[],"resultFormat":"compactedList","batchSize":20480,"limit":9223372036854775807,"order":"none","filter":{"type":"and","fields":[{"type":"and","fields":[{"type":"bound","dimension":"cintstring","lower":"DynamicValue(RS_4_alltypesorc_small_cint_min)","upper":"DynamicValue(RS_4_alltypesorc_small_cint_max)","lowerStrict":false,"upperStrict":false,"extractionFn":null,"ordering":{"type":"lexicographic"}},{"type":"bloom","dimension":"cintstring","bloomKFilter":"BAAAAAgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA","extractionFn":null}]},{"type":"not","field":{"type":"selector","dimension":"cintstring","value":null,"extractionFn":null}}]},"columns":["cintstring"],"legacy":null,"context":null,"descending":false,"granularity":{"type":"all"}} - druid.query.type scan - Statistics: Num rows: 9173 Data size: 1603744 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (cintstring BETWEEN DynamicValue(RS_4_alltypesorc_small_cint_min) AND DynamicValue(RS_4_alltypesorc_small_cint_max) and in_bloom_filter(cintstring, DynamicValue(RS_4_alltypesorc_small_cint_bloom_filter))) (type: boolean) - Statistics: Num rows: 9173 Data size: 1603744 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: cintstring (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: cintstring (type: string) - Statistics: Num rows: 9173 Data size: 1603744 Basic stats: COMPLETE Column stats: NONE - Execution mode: vectorized, llap - LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 cintstring (type: string) - Statistics: Num rows: 10090 Data size: 1764118 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Reducer 3 - Execution mode: vectorized, llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 - Execution mode: vectorized, llap - Reduce Operator Tree: - Group By Operator - aggregations: min(VALUE._col0), max(VALUE._col1), bloom_filter(VALUE._col2, expectedEntries=69) - mode: final - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary) - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: select count(*) from alltypesorc_small join druid_table_alltypesorc on (cast(alltypesorc_small.cint as string) = druid_table_alltypesorc.cintstring) -PREHOOK: type: QUERY -PREHOOK: Input: default@alltypesorc_small -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: select count(*) from alltypesorc_small join druid_table_alltypesorc on (cast(alltypesorc_small.cint as string) = druid_table_alltypesorc.cintstring) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@alltypesorc_small -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -69 -PREHOOK: query: select count(*) from alltypesorc_small join druid_table_alltypesorc on (cast(alltypesorc_small.cint as string) = druid_table_alltypesorc.cintstring) -PREHOOK: type: QUERY -PREHOOK: Input: default@alltypesorc_small -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: select count(*) from alltypesorc_small join druid_table_alltypesorc on (cast(alltypesorc_small.cint as string) = druid_table_alltypesorc.cintstring) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@alltypesorc_small -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -69 -PREHOOK: query: EXPLAIN select count(*) from alltypesorc_small join druid_table_alltypesorc on (cast(alltypesorc_small.cdouble as string) = druid_table_alltypesorc.cdoublestring) -PREHOOK: type: QUERY -PREHOOK: Input: default@alltypesorc_small -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN select count(*) from alltypesorc_small join druid_table_alltypesorc on (cast(alltypesorc_small.cdouble as string) = druid_table_alltypesorc.cdoublestring) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@alltypesorc_small -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Tez -#### A masked pattern was here #### - Edges: - Map 5 <- Reducer 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) - Reducer 4 <- Map 1 (CUSTOM_SIMPLE_EDGE) -#### A masked pattern was here #### - Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: alltypesorc_small - filterExpr: CAST( cdouble AS STRING) is not null (type: boolean) - Statistics: Num rows: 69 Data size: 552 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: CAST( cdouble AS STRING) is not null (type: boolean) - Statistics: Num rows: 69 Data size: 552 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: CAST( cdouble AS STRING) (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 69 Data size: 12696 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 69 Data size: 12696 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 69 Data size: 12696 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=1) - minReductionHashAggr: 0.98550725 - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary) - Execution mode: vectorized, llap - LLAP IO: all inputs - Map 5 - Map Operator Tree: - TableScan - alias: druid_table_alltypesorc - filterExpr: (cdoublestring BETWEEN DynamicValue(RS_4_alltypesorc_small_cdouble_min) AND DynamicValue(RS_4_alltypesorc_small_cdouble_max) and in_bloom_filter(cdoublestring, DynamicValue(RS_4_alltypesorc_small_cdouble_bloom_filter))) (type: boolean) - properties: - druid.fieldNames cdoublestring - druid.fieldTypes string - druid.query.json {"queryType":"scan","dataSource":{"type":"table","name":"default.druid_table_alltypesorc"},"intervals":{"type":"LegacySegmentSpec","intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"]},"virtualColumns":[],"resultFormat":"compactedList","batchSize":20480,"limit":9223372036854775807,"order":"none","filter":{"type":"and","fields":[{"type":"and","fields":[{"type":"bound","dimension":"cdoublestring","lower":"DynamicValue(RS_4_alltypesorc_small_cdouble_min)","upper":"DynamicValue(RS_4_alltypesorc_small_cdouble_max)","lowerStrict":false,"upperStrict":false,"extractionFn":null,"ordering":{"type":"lexicographic"}},{"type":"bloom","dimension":"cdoublestring","bloomKFilter":"BAAAAAgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA","extractionFn":null}]},{"type":"not","field":{"type":"selector","dimension":"cdoublestring","value":null,"extractionFn":null}}]},"columns":["cdoublestring"],"legacy":null,"context":null,"descending":false,"granularity":{"type":"all"}} - druid.query.type scan - Statistics: Num rows: 9173 Data size: 1603744 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (cdoublestring BETWEEN DynamicValue(RS_4_alltypesorc_small_cdouble_min) AND DynamicValue(RS_4_alltypesorc_small_cdouble_max) and in_bloom_filter(cdoublestring, DynamicValue(RS_4_alltypesorc_small_cdouble_bloom_filter))) (type: boolean) - Statistics: Num rows: 9173 Data size: 1603744 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: cdoublestring (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: cdoublestring (type: string) - Statistics: Num rows: 9173 Data size: 1603744 Basic stats: COMPLETE Column stats: NONE - Execution mode: vectorized, llap - LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 cdoublestring (type: string) - Statistics: Num rows: 10090 Data size: 1764118 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Reducer 3 - Execution mode: vectorized, llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 - Execution mode: vectorized, llap - Reduce Operator Tree: - Group By Operator - aggregations: min(VALUE._col0), max(VALUE._col1), bloom_filter(VALUE._col2, expectedEntries=1) - mode: final - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary) - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: select count(*) from alltypesorc_small join druid_table_alltypesorc on (cast(alltypesorc_small.cdouble as string) = druid_table_alltypesorc.cdoublestring) -PREHOOK: type: QUERY -PREHOOK: Input: default@alltypesorc_small -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: select count(*) from alltypesorc_small join druid_table_alltypesorc on (cast(alltypesorc_small.cdouble as string) = druid_table_alltypesorc.cdoublestring) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@alltypesorc_small -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -0 -PREHOOK: query: select count(*) from alltypesorc_small join druid_table_alltypesorc on (cast(alltypesorc_small.cdouble as string) = druid_table_alltypesorc.cdoublestring) -PREHOOK: type: QUERY -PREHOOK: Input: default@alltypesorc_small -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: select count(*) from alltypesorc_small join druid_table_alltypesorc on (cast(alltypesorc_small.cdouble as string) = druid_table_alltypesorc.cdoublestring) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@alltypesorc_small -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -0 -PREHOOK: query: EXPLAIN select count(*) from alltypesorc_small join druid_table_alltypesorc on (cast(alltypesorc_small.cfloat as string) = druid_table_alltypesorc.cfloatstring) -PREHOOK: type: QUERY -PREHOOK: Input: default@alltypesorc_small -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN select count(*) from alltypesorc_small join druid_table_alltypesorc on (cast(alltypesorc_small.cfloat as string) = druid_table_alltypesorc.cfloatstring) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@alltypesorc_small -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Tez -#### A masked pattern was here #### - Edges: - Map 5 <- Reducer 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) - Reducer 4 <- Map 1 (CUSTOM_SIMPLE_EDGE) -#### A masked pattern was here #### - Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: alltypesorc_small - filterExpr: CAST( cfloat AS STRING) is not null (type: boolean) - Statistics: Num rows: 69 Data size: 276 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: CAST( cfloat AS STRING) is not null (type: boolean) - Statistics: Num rows: 69 Data size: 276 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: CAST( cfloat AS STRING) (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 69 Data size: 12696 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 69 Data size: 12696 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 69 Data size: 12696 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=3) - minReductionHashAggr: 0.98550725 - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary) - Execution mode: vectorized, llap - LLAP IO: all inputs - Map 5 - Map Operator Tree: - TableScan - alias: druid_table_alltypesorc - filterExpr: (cfloatstring BETWEEN DynamicValue(RS_4_alltypesorc_small_cfloat_min) AND DynamicValue(RS_4_alltypesorc_small_cfloat_max) and in_bloom_filter(cfloatstring, DynamicValue(RS_4_alltypesorc_small_cfloat_bloom_filter))) (type: boolean) - properties: - druid.fieldNames cfloatstring - druid.fieldTypes string - druid.query.json {"queryType":"scan","dataSource":{"type":"table","name":"default.druid_table_alltypesorc"},"intervals":{"type":"LegacySegmentSpec","intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"]},"virtualColumns":[],"resultFormat":"compactedList","batchSize":20480,"limit":9223372036854775807,"order":"none","filter":{"type":"and","fields":[{"type":"and","fields":[{"type":"bound","dimension":"cfloatstring","lower":"DynamicValue(RS_4_alltypesorc_small_cfloat_min)","upper":"DynamicValue(RS_4_alltypesorc_small_cfloat_max)","lowerStrict":false,"upperStrict":false,"extractionFn":null,"ordering":{"type":"lexicographic"}},{"type":"bloom","dimension":"cfloatstring","bloomKFilter":"BAAAAAgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA","extractionFn":null}]},{"type":"not","field":{"type":"selector","dimension":"cfloatstring","value":null,"extractionFn":null}}]},"columns":["cfloatstring"],"legacy":null,"context":null,"descending":false,"granularity":{"type":"all"}} - druid.query.type scan - Statistics: Num rows: 9173 Data size: 1603744 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (cfloatstring BETWEEN DynamicValue(RS_4_alltypesorc_small_cfloat_min) AND DynamicValue(RS_4_alltypesorc_small_cfloat_max) and in_bloom_filter(cfloatstring, DynamicValue(RS_4_alltypesorc_small_cfloat_bloom_filter))) (type: boolean) - Statistics: Num rows: 9173 Data size: 1603744 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: cfloatstring (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: cfloatstring (type: string) - Statistics: Num rows: 9173 Data size: 1603744 Basic stats: COMPLETE Column stats: NONE - Execution mode: vectorized, llap - LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 cfloatstring (type: string) - Statistics: Num rows: 10090 Data size: 1764118 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Reducer 3 - Execution mode: vectorized, llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 - Execution mode: vectorized, llap - Reduce Operator Tree: - Group By Operator - aggregations: min(VALUE._col0), max(VALUE._col1), bloom_filter(VALUE._col2, expectedEntries=3) - mode: final - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary) - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: select count(*) from alltypesorc_small join druid_table_alltypesorc on (cast(alltypesorc_small.cfloat as string) = druid_table_alltypesorc.cfloatstring) -PREHOOK: type: QUERY -PREHOOK: Input: default@alltypesorc_small -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: select count(*) from alltypesorc_small join druid_table_alltypesorc on (cast(alltypesorc_small.cfloat as string) = druid_table_alltypesorc.cfloatstring) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@alltypesorc_small -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -73895 -PREHOOK: query: select count(*) from alltypesorc_small join druid_table_alltypesorc on (cast(alltypesorc_small.cfloat as string) = druid_table_alltypesorc.cfloatstring) -PREHOOK: type: QUERY -PREHOOK: Input: default@alltypesorc_small -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: select count(*) from alltypesorc_small join druid_table_alltypesorc on (cast(alltypesorc_small.cfloat as string) = druid_table_alltypesorc.cfloatstring) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@alltypesorc_small -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -73895 diff --git a/ql/src/test/results/clientpositive/druid/druidmini_test1.q.out b/ql/src/test/results/clientpositive/druid/druidmini_test1.q.out deleted file mode 100644 index a0d7e1e43c8c..000000000000 --- a/ql/src/test/results/clientpositive/druid/druidmini_test1.q.out +++ /dev/null @@ -1,960 +0,0 @@ -PREHOOK: query: explain select count(*) FROM druid_table_alltypesorc -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: explain select count(*) FROM druid_table_alltypesorc -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_table_alltypesorc - properties: - druid.fieldNames $f0 - druid.fieldTypes bigint - druid.query.json {"queryType":"timeseries","dataSource":"default.druid_table_alltypesorc","descending":false,"granularity":"all","aggregations":[{"type":"count","name":"$f0"}],"intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"],"context":{"skipEmptyBuckets":false}} - druid.query.type timeseries - Select Operator - expressions: $f0 (type: bigint) - outputColumnNames: _col0 - ListSink - -PREHOOK: query: SELECT count(*) FROM druid_table_alltypesorc -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: SELECT count(*) FROM druid_table_alltypesorc -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -9173 -PREHOOK: query: EXPLAIN SELECT floor_year(`__time`), SUM(cfloat), SUM(cdouble), SUM(ctinyint), SUM(csmallint),SUM(cint), SUM(cbigint) -FROM druid_table_alltypesorc GROUP BY floor_year(`__time`) -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN SELECT floor_year(`__time`), SUM(cfloat), SUM(cdouble), SUM(ctinyint), SUM(csmallint),SUM(cint), SUM(cbigint) -FROM druid_table_alltypesorc GROUP BY floor_year(`__time`) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_table_alltypesorc - properties: - druid.fieldNames timestamp,$f1,$f2,$f3,$f4,$f5,$f6 - druid.fieldTypes timestamp with local time zone,double,double,bigint,bigint,bigint,bigint - druid.query.json {"queryType":"timeseries","dataSource":"default.druid_table_alltypesorc","descending":false,"granularity":{"type":"period","period":"P1Y","timeZone":"US/Pacific"},"aggregations":[{"type":"doubleSum","name":"$f1","fieldName":"cfloat"},{"type":"doubleSum","name":"$f2","fieldName":"cdouble"},{"type":"longSum","name":"$f3","fieldName":"ctinyint"},{"type":"longSum","name":"$f4","fieldName":"csmallint"},{"type":"longSum","name":"$f5","fieldName":"cint"},{"type":"longSum","name":"$f6","fieldName":"cbigint"}],"intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"],"context":{"skipEmptyBuckets":true}} - druid.query.type timeseries - Select Operator - expressions: timestamp (type: timestamp with local time zone), $f1 (type: double), $f2 (type: double), $f3 (type: bigint), $f4 (type: bigint), $f5 (type: bigint), $f6 (type: bigint) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - ListSink - -PREHOOK: query: SELECT floor_year(`__time`), SUM(cfloat), SUM(cdouble), SUM(ctinyint), SUM(csmallint),SUM(cint), SUM(cbigint) -FROM druid_table_alltypesorc GROUP BY floor_year(`__time`) -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: SELECT floor_year(`__time`), SUM(cfloat), SUM(cdouble), SUM(ctinyint), SUM(csmallint),SUM(cint), SUM(cbigint) -FROM druid_table_alltypesorc GROUP BY floor_year(`__time`) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -1969-01-01 00:00:00.0 US/Pacific -39590.24699282646 2.73086627938E7 -39967 7781089 1408069801800 10992545287 -PREHOOK: query: EXPLAIN SELECT floor_year(`__time`), MIN(cfloat), MIN(cdouble), MIN(ctinyint), MIN(csmallint),MIN(cint), MIN(cbigint) -FROM druid_table_alltypesorc GROUP BY floor_year(`__time`) -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN SELECT floor_year(`__time`), MIN(cfloat), MIN(cdouble), MIN(ctinyint), MIN(csmallint),MIN(cint), MIN(cbigint) -FROM druid_table_alltypesorc GROUP BY floor_year(`__time`) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_table_alltypesorc - properties: - druid.fieldNames timestamp,$f1,$f2,$f3,$f4,$f5,$f6 - druid.fieldTypes timestamp with local time zone,float,double,tinyint,smallint,int,bigint - druid.query.json {"queryType":"timeseries","dataSource":"default.druid_table_alltypesorc","descending":false,"granularity":{"type":"period","period":"P1Y","timeZone":"US/Pacific"},"aggregations":[{"type":"doubleMin","name":"$f1","fieldName":"cfloat"},{"type":"doubleMin","name":"$f2","fieldName":"cdouble"},{"type":"longMin","name":"$f3","fieldName":"ctinyint"},{"type":"longMin","name":"$f4","fieldName":"csmallint"},{"type":"longMin","name":"$f5","fieldName":"cint"},{"type":"longMin","name":"$f6","fieldName":"cbigint"}],"intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"],"context":{"skipEmptyBuckets":true}} - druid.query.type timeseries - Select Operator - expressions: timestamp (type: timestamp with local time zone), $f1 (type: float), $f2 (type: double), $f3 (type: tinyint), $f4 (type: smallint), $f5 (type: int), $f6 (type: bigint) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - ListSink - -PREHOOK: query: SELECT floor_year(`__time`), MIN(cfloat), MIN(cdouble), MIN(ctinyint), MIN(csmallint),MIN(cint), MIN(cbigint) -FROM druid_table_alltypesorc GROUP BY floor_year(`__time`) -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: SELECT floor_year(`__time`), MIN(cfloat), MIN(cdouble), MIN(ctinyint), MIN(csmallint),MIN(cint), MIN(cbigint) -FROM druid_table_alltypesorc GROUP BY floor_year(`__time`) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -1969-01-01 00:00:00.0 US/Pacific -64.0 -16373.0 -64 -16373 -1073279343 -2147311592 -PREHOOK: query: EXPLAIN SELECT floor_year(`__time`), MAX(cfloat), MAX(cdouble), MAX(ctinyint), MAX(csmallint),MAX(cint), MAX(cbigint) -FROM druid_table_alltypesorc GROUP BY floor_year(`__time`) -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN SELECT floor_year(`__time`), MAX(cfloat), MAX(cdouble), MAX(ctinyint), MAX(csmallint),MAX(cint), MAX(cbigint) -FROM druid_table_alltypesorc GROUP BY floor_year(`__time`) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_table_alltypesorc - properties: - druid.fieldNames timestamp,$f1,$f2,$f3,$f4,$f5,$f6 - druid.fieldTypes timestamp with local time zone,float,double,tinyint,smallint,int,bigint - druid.query.json {"queryType":"timeseries","dataSource":"default.druid_table_alltypesorc","descending":false,"granularity":{"type":"period","period":"P1Y","timeZone":"US/Pacific"},"aggregations":[{"type":"doubleMax","name":"$f1","fieldName":"cfloat"},{"type":"doubleMax","name":"$f2","fieldName":"cdouble"},{"type":"longMax","name":"$f3","fieldName":"ctinyint"},{"type":"longMax","name":"$f4","fieldName":"csmallint"},{"type":"longMax","name":"$f5","fieldName":"cint"},{"type":"longMax","name":"$f6","fieldName":"cbigint"}],"intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"],"context":{"skipEmptyBuckets":true}} - druid.query.type timeseries - Select Operator - expressions: timestamp (type: timestamp with local time zone), $f1 (type: float), $f2 (type: double), $f3 (type: tinyint), $f4 (type: smallint), $f5 (type: int), $f6 (type: bigint) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - ListSink - -PREHOOK: query: SELECT floor_year(`__time`), MAX(cfloat), MAX(cdouble), MAX(ctinyint), MAX(csmallint),MAX(cint), MAX(cbigint) -FROM druid_table_alltypesorc GROUP BY floor_year(`__time`) -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: SELECT floor_year(`__time`), MAX(cfloat), MAX(cdouble), MAX(ctinyint), MAX(csmallint),MAX(cint), MAX(cbigint) -FROM druid_table_alltypesorc GROUP BY floor_year(`__time`) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -1969-01-01 00:00:00.0 US/Pacific 79.553 9763215.5639 62 16370 1073680599 2145498388 -PREHOOK: query: EXPLAIN SELECT cstring1, SUM(cdouble) as s FROM druid_table_alltypesorc GROUP BY cstring1 ORDER BY s ASC LIMIT 10 -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN SELECT cstring1, SUM(cdouble) as s FROM druid_table_alltypesorc GROUP BY cstring1 ORDER BY s ASC LIMIT 10 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_table_alltypesorc - properties: - druid.fieldNames cstring1,$f1 - druid.fieldTypes string,double - druid.query.json {"queryType":"groupBy","dataSource":"default.druid_table_alltypesorc","granularity":"all","dimensions":[{"type":"default","dimension":"cstring1","outputName":"cstring1","outputType":"STRING"}],"limitSpec":{"type":"default","limit":10,"columns":[{"dimension":"$f1","direction":"ascending","dimensionOrder":"numeric"}]},"aggregations":[{"type":"doubleSum","name":"$f1","fieldName":"cdouble"}],"intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"]} - druid.query.type groupBy - Select Operator - expressions: cstring1 (type: string), $f1 (type: double) - outputColumnNames: _col0, _col1 - ListSink - -PREHOOK: query: SELECT cstring1, SUM(cdouble) as s FROM druid_table_alltypesorc GROUP BY cstring1 ORDER BY s ASC LIMIT 10 -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: SELECT cstring1, SUM(cdouble) as s FROM druid_table_alltypesorc GROUP BY cstring1 ORDER BY s ASC LIMIT 10 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -1cGVWH7n1QU -596096.6799999999 -821UdmGbkEf4j -14161.826999999997 -00iT08 0.0 -02v8WnLuYDos3Cq 0.0 -yv1js 0.0 -02VRbSC5I 0.0 -014ILGhXxNY7g02hl0Xw 0.0 -02vDyIVT752 0.0 -00PafC7v 0.0 -ytpx1RL8F2I 0.0 -PREHOOK: query: EXPLAIN SELECT cstring2, MAX(cdouble) FROM druid_table_alltypesorc GROUP BY cstring2 ORDER BY cstring2 ASC LIMIT 10 -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN SELECT cstring2, MAX(cdouble) FROM druid_table_alltypesorc GROUP BY cstring2 ORDER BY cstring2 ASC LIMIT 10 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_table_alltypesorc - properties: - druid.fieldNames cstring2,$f1 - druid.fieldTypes string,double - druid.query.json {"queryType":"groupBy","dataSource":"default.druid_table_alltypesorc","granularity":"all","dimensions":[{"type":"default","dimension":"cstring2","outputName":"cstring2","outputType":"STRING"}],"limitSpec":{"type":"default","limit":10,"columns":[{"dimension":"cstring2","direction":"ascending","dimensionOrder":"lexicographic"}]},"aggregations":[{"type":"doubleMax","name":"$f1","fieldName":"cdouble"}],"intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"]} - druid.query.type groupBy - Select Operator - expressions: cstring2 (type: string), $f1 (type: double) - outputColumnNames: _col0, _col1 - ListSink - -PREHOOK: query: SELECT cstring2, MAX(cdouble) FROM druid_table_alltypesorc GROUP BY cstring2 ORDER BY cstring2 ASC LIMIT 10 -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: SELECT cstring2, MAX(cdouble) FROM druid_table_alltypesorc GROUP BY cstring2 ORDER BY cstring2 ASC LIMIT 10 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -NULL 9763215.5639 -0034fkcXMQI3 15601.0 -004J8y 0.0 -00GNm -200.0 -00GW4dnb6Wgj52 -200.0 -00PBhB1Iefgk 0.0 -00d5kr1wEB7evExG 15601.0 -00qccwt8n 0.0 -017fFeQ3Gcsa83Xj2Vo0 0.0 -01EfkvNk6mjG44uxs 0.0 -PREHOOK: query: EXPLAIN -SELECT `__time` -FROM druid_table_alltypesorc ORDER BY `__time` ASC LIMIT 10 -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN -SELECT `__time` -FROM druid_table_alltypesorc ORDER BY `__time` ASC LIMIT 10 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Tez -#### A masked pattern was here #### - Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) -#### A masked pattern was here #### - Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: druid_table_alltypesorc - properties: - druid.fieldNames vc - druid.fieldTypes timestamp with local time zone - druid.query.json {"queryType":"scan","dataSource":"default.druid_table_alltypesorc","intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"],"virtualColumns":[{"type":"expression","name":"vc","expression":"\"__time\"","outputType":"LONG"}],"columns":["vc"],"resultFormat":"compactedList"} - druid.query.type scan - Statistics: Num rows: 9173 Data size: 348640 Basic stats: COMPLETE Column stats: NONE - Top N Key Operator - sort order: + - keys: vc (type: timestamp with local time zone) - null sort order: z - Statistics: Num rows: 9173 Data size: 348640 Basic stats: COMPLETE Column stats: NONE - top n: 10 - Select Operator - expressions: vc (type: timestamp with local time zone) - outputColumnNames: _col0 - Statistics: Num rows: 9173 Data size: 348640 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: timestamp with local time zone) - null sort order: z - sort order: + - Statistics: Num rows: 9173 Data size: 348640 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: timestamp with local time zone) - outputColumnNames: _col0 - Statistics: Num rows: 9173 Data size: 348640 Basic stats: COMPLETE Column stats: NONE - Limit - Number of rows: 10 - Statistics: Num rows: 10 Data size: 380 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 10 Data size: 380 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: 10 - Processor Tree: - ListSink - -PREHOOK: query: SELECT `__time` -FROM druid_table_alltypesorc ORDER BY `__time` ASC LIMIT 10 -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: SELECT `__time` -FROM druid_table_alltypesorc ORDER BY `__time` ASC LIMIT 10 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -PREHOOK: query: EXPLAIN -SELECT `__time` -FROM druid_table_alltypesorc -WHERE `__time` < '1970-03-01 00:00:00' ORDER BY `__time` ASC LIMIT 10 -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN -SELECT `__time` -FROM druid_table_alltypesorc -WHERE `__time` < '1970-03-01 00:00:00' ORDER BY `__time` ASC LIMIT 10 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Tez -#### A masked pattern was here #### - Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) -#### A masked pattern was here #### - Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: druid_table_alltypesorc - properties: - druid.fieldNames vc - druid.fieldTypes timestamp with local time zone - druid.query.json {"queryType":"scan","dataSource":"default.druid_table_alltypesorc","intervals":["1900-01-01T00:00:00.000Z/1970-03-01T08:00:00.000Z"],"virtualColumns":[{"type":"expression","name":"vc","expression":"\"__time\"","outputType":"LONG"}],"columns":["vc"],"resultFormat":"compactedList"} - druid.query.type scan - Statistics: Num rows: 9173 Data size: 348640 Basic stats: COMPLETE Column stats: NONE - Top N Key Operator - sort order: + - keys: vc (type: timestamp with local time zone) - null sort order: z - Statistics: Num rows: 9173 Data size: 348640 Basic stats: COMPLETE Column stats: NONE - top n: 10 - Select Operator - expressions: vc (type: timestamp with local time zone) - outputColumnNames: _col0 - Statistics: Num rows: 9173 Data size: 348640 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: timestamp with local time zone) - null sort order: z - sort order: + - Statistics: Num rows: 9173 Data size: 348640 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: timestamp with local time zone) - outputColumnNames: _col0 - Statistics: Num rows: 9173 Data size: 348640 Basic stats: COMPLETE Column stats: NONE - Limit - Number of rows: 10 - Statistics: Num rows: 10 Data size: 380 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 10 Data size: 380 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: 10 - Processor Tree: - ListSink - -PREHOOK: query: SELECT `__time` -FROM druid_table_alltypesorc -WHERE `__time` < '1970-03-01 00:00:00' ORDER BY `__time` ASC LIMIT 10 -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: SELECT `__time` -FROM druid_table_alltypesorc -WHERE `__time` < '1970-03-01 00:00:00' ORDER BY `__time` ASC LIMIT 10 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -PREHOOK: query: EXPLAIN -SELECT `__time` -FROM druid_table_alltypesorc -WHERE `__time` >= '1968-01-01 00:00:00' AND `__time` <= '1970-03-01 00:00:00' ORDER BY `__time` ASC LIMIT 10 -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN -SELECT `__time` -FROM druid_table_alltypesorc -WHERE `__time` >= '1968-01-01 00:00:00' AND `__time` <= '1970-03-01 00:00:00' ORDER BY `__time` ASC LIMIT 10 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Tez -#### A masked pattern was here #### - Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) -#### A masked pattern was here #### - Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: druid_table_alltypesorc - properties: - druid.fieldNames vc - druid.fieldTypes timestamp with local time zone - druid.query.json {"queryType":"scan","dataSource":"default.druid_table_alltypesorc","intervals":["1968-01-01T08:00:00.000Z/1970-03-01T08:00:00.001Z"],"virtualColumns":[{"type":"expression","name":"vc","expression":"\"__time\"","outputType":"LONG"}],"columns":["vc"],"resultFormat":"compactedList"} - druid.query.type scan - Statistics: Num rows: 9173 Data size: 348640 Basic stats: COMPLETE Column stats: NONE - Top N Key Operator - sort order: + - keys: vc (type: timestamp with local time zone) - null sort order: z - Statistics: Num rows: 9173 Data size: 348640 Basic stats: COMPLETE Column stats: NONE - top n: 10 - Select Operator - expressions: vc (type: timestamp with local time zone) - outputColumnNames: _col0 - Statistics: Num rows: 9173 Data size: 348640 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: timestamp with local time zone) - null sort order: z - sort order: + - Statistics: Num rows: 9173 Data size: 348640 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: timestamp with local time zone) - outputColumnNames: _col0 - Statistics: Num rows: 9173 Data size: 348640 Basic stats: COMPLETE Column stats: NONE - Limit - Number of rows: 10 - Statistics: Num rows: 10 Data size: 380 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 10 Data size: 380 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: 10 - Processor Tree: - ListSink - -PREHOOK: query: SELECT `__time` -FROM druid_table_alltypesorc -WHERE `__time` >= '1968-01-01 00:00:00' AND `__time` <= '1970-03-01 00:00:00' ORDER BY `__time` ASC LIMIT 10 -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: SELECT `__time` -FROM druid_table_alltypesorc -WHERE `__time` >= '1968-01-01 00:00:00' AND `__time` <= '1970-03-01 00:00:00' ORDER BY `__time` ASC LIMIT 10 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -PREHOOK: query: EXPLAIN -SELECT `__time` -FROM druid_table_alltypesorc -WHERE `__time` >= '1968-01-01 00:00:00' AND `__time` <= '1970-03-01 00:00:00' - AND `__time` < '2011-01-01 00:00:00' ORDER BY `__time` ASC LIMIT 10 -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN -SELECT `__time` -FROM druid_table_alltypesorc -WHERE `__time` >= '1968-01-01 00:00:00' AND `__time` <= '1970-03-01 00:00:00' - AND `__time` < '2011-01-01 00:00:00' ORDER BY `__time` ASC LIMIT 10 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Tez -#### A masked pattern was here #### - Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) -#### A masked pattern was here #### - Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: druid_table_alltypesorc - properties: - druid.fieldNames vc - druid.fieldTypes timestamp with local time zone - druid.query.json {"queryType":"scan","dataSource":"default.druid_table_alltypesorc","intervals":["1968-01-01T08:00:00.000Z/1970-03-01T08:00:00.001Z"],"virtualColumns":[{"type":"expression","name":"vc","expression":"\"__time\"","outputType":"LONG"}],"columns":["vc"],"resultFormat":"compactedList"} - druid.query.type scan - Statistics: Num rows: 9173 Data size: 348640 Basic stats: COMPLETE Column stats: NONE - Top N Key Operator - sort order: + - keys: vc (type: timestamp with local time zone) - null sort order: z - Statistics: Num rows: 9173 Data size: 348640 Basic stats: COMPLETE Column stats: NONE - top n: 10 - Select Operator - expressions: vc (type: timestamp with local time zone) - outputColumnNames: _col0 - Statistics: Num rows: 9173 Data size: 348640 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: timestamp with local time zone) - null sort order: z - sort order: + - Statistics: Num rows: 9173 Data size: 348640 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: timestamp with local time zone) - outputColumnNames: _col0 - Statistics: Num rows: 9173 Data size: 348640 Basic stats: COMPLETE Column stats: NONE - Limit - Number of rows: 10 - Statistics: Num rows: 10 Data size: 380 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 10 Data size: 380 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: 10 - Processor Tree: - ListSink - -PREHOOK: query: SELECT `__time` -FROM druid_table_alltypesorc -WHERE `__time` >= '1968-01-01 00:00:00' AND `__time` <= '1970-03-01 00:00:00' - AND `__time` < '2011-01-01 00:00:00' ORDER BY `__time` ASC LIMIT 10 -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: SELECT `__time` -FROM druid_table_alltypesorc -WHERE `__time` >= '1968-01-01 00:00:00' AND `__time` <= '1970-03-01 00:00:00' - AND `__time` < '2011-01-01 00:00:00' ORDER BY `__time` ASC LIMIT 10 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -PREHOOK: query: EXPLAIN -SELECT `__time` -FROM druid_table_alltypesorc -WHERE `__time` BETWEEN '1968-01-01 00:00:00' AND '1970-01-01 00:00:00' ORDER BY `__time` ASC LIMIT 10 -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN -SELECT `__time` -FROM druid_table_alltypesorc -WHERE `__time` BETWEEN '1968-01-01 00:00:00' AND '1970-01-01 00:00:00' ORDER BY `__time` ASC LIMIT 10 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Tez -#### A masked pattern was here #### - Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) -#### A masked pattern was here #### - Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: druid_table_alltypesorc - properties: - druid.fieldNames vc - druid.fieldTypes timestamp with local time zone - druid.query.json {"queryType":"scan","dataSource":"default.druid_table_alltypesorc","intervals":["1968-01-01T08:00:00.000Z/1970-01-01T08:00:00.001Z"],"virtualColumns":[{"type":"expression","name":"vc","expression":"\"__time\"","outputType":"LONG"}],"columns":["vc"],"resultFormat":"compactedList"} - druid.query.type scan - Statistics: Num rows: 9173 Data size: 348640 Basic stats: COMPLETE Column stats: NONE - Top N Key Operator - sort order: + - keys: vc (type: timestamp with local time zone) - null sort order: z - Statistics: Num rows: 9173 Data size: 348640 Basic stats: COMPLETE Column stats: NONE - top n: 10 - Select Operator - expressions: vc (type: timestamp with local time zone) - outputColumnNames: _col0 - Statistics: Num rows: 9173 Data size: 348640 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: timestamp with local time zone) - null sort order: z - sort order: + - Statistics: Num rows: 9173 Data size: 348640 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: timestamp with local time zone) - outputColumnNames: _col0 - Statistics: Num rows: 9173 Data size: 348640 Basic stats: COMPLETE Column stats: NONE - Limit - Number of rows: 10 - Statistics: Num rows: 10 Data size: 380 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 10 Data size: 380 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: 10 - Processor Tree: - ListSink - -PREHOOK: query: SELECT `__time` -FROM druid_table_alltypesorc -WHERE `__time` BETWEEN '1968-01-01 00:00:00' AND '1970-01-01 00:00:00' ORDER BY `__time` ASC LIMIT 10 -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: SELECT `__time` -FROM druid_table_alltypesorc -WHERE `__time` BETWEEN '1968-01-01 00:00:00' AND '1970-01-01 00:00:00' ORDER BY `__time` ASC LIMIT 10 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -PREHOOK: query: EXPLAIN -SELECT `__time` -FROM druid_table_alltypesorc -WHERE (`__time` BETWEEN '1968-01-01 00:00:00' AND '1970-01-01 00:00:00') - OR (`__time` BETWEEN '1968-02-01 00:00:00' AND '1970-04-01 00:00:00') ORDER BY `__time` ASC LIMIT 10 -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN -SELECT `__time` -FROM druid_table_alltypesorc -WHERE (`__time` BETWEEN '1968-01-01 00:00:00' AND '1970-01-01 00:00:00') - OR (`__time` BETWEEN '1968-02-01 00:00:00' AND '1970-04-01 00:00:00') ORDER BY `__time` ASC LIMIT 10 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Tez -#### A masked pattern was here #### - Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) -#### A masked pattern was here #### - Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: druid_table_alltypesorc - properties: - druid.fieldNames vc - druid.fieldTypes timestamp with local time zone - druid.query.json {"queryType":"scan","dataSource":"default.druid_table_alltypesorc","intervals":["1968-01-01T08:00:00.000Z/1970-04-01T08:00:00.001Z"],"virtualColumns":[{"type":"expression","name":"vc","expression":"\"__time\"","outputType":"LONG"}],"columns":["vc"],"resultFormat":"compactedList"} - druid.query.type scan - Statistics: Num rows: 9173 Data size: 348640 Basic stats: COMPLETE Column stats: NONE - Top N Key Operator - sort order: + - keys: vc (type: timestamp with local time zone) - null sort order: z - Statistics: Num rows: 9173 Data size: 348640 Basic stats: COMPLETE Column stats: NONE - top n: 10 - Select Operator - expressions: vc (type: timestamp with local time zone) - outputColumnNames: _col0 - Statistics: Num rows: 9173 Data size: 348640 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: timestamp with local time zone) - null sort order: z - sort order: + - Statistics: Num rows: 9173 Data size: 348640 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: timestamp with local time zone) - outputColumnNames: _col0 - Statistics: Num rows: 9173 Data size: 348640 Basic stats: COMPLETE Column stats: NONE - Limit - Number of rows: 10 - Statistics: Num rows: 10 Data size: 380 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 10 Data size: 380 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: 10 - Processor Tree: - ListSink - -PREHOOK: query: SELECT `__time` -FROM druid_table_alltypesorc -WHERE (`__time` BETWEEN '1968-01-01 00:00:00' AND '1970-01-01 00:00:00') - OR (`__time` BETWEEN '1968-02-01 00:00:00' AND '1970-04-01 00:00:00') ORDER BY `__time` ASC LIMIT 10 -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: SELECT `__time` -FROM druid_table_alltypesorc -WHERE (`__time` BETWEEN '1968-01-01 00:00:00' AND '1970-01-01 00:00:00') - OR (`__time` BETWEEN '1968-02-01 00:00:00' AND '1970-04-01 00:00:00') ORDER BY `__time` ASC LIMIT 10 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -PREHOOK: query: explain select (cstring1 is null ) AS is_null, (cint is not null ) as isnotnull FROM druid_table_alltypesorc -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: explain select (cstring1 is null ) AS is_null, (cint is not null ) as isnotnull FROM druid_table_alltypesorc -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_table_alltypesorc - properties: - druid.fieldNames vc,vc0 - druid.fieldTypes boolean,boolean - druid.query.json {"queryType":"scan","dataSource":"default.druid_table_alltypesorc","intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"],"virtualColumns":[{"type":"expression","name":"vc","expression":"isnull(\"cstring1\")","outputType":"LONG"},{"type":"expression","name":"vc0","expression":"notnull(\"cint\")","outputType":"LONG"}],"columns":["vc","vc0"],"resultFormat":"compactedList"} - druid.query.type scan - Select Operator - expressions: vc (type: boolean), vc0 (type: boolean) - outputColumnNames: _col0, _col1 - ListSink - -PREHOOK: query: explain select substring(to_date(`__time`), 4) from druid_table_alltypesorc limit 5 -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: explain select substring(to_date(`__time`), 4) from druid_table_alltypesorc limit 5 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_table_alltypesorc - properties: - druid.fieldNames vc - druid.fieldTypes string - druid.query.json {"queryType":"scan","dataSource":"default.druid_table_alltypesorc","intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"],"virtualColumns":[{"type":"expression","name":"vc","expression":"substring(timestamp_format(timestamp_floor(\"__time\",'P1D','','US/Pacific'),'yyyy-MM-dd','UTC'), 3, -1)","outputType":"STRING"}],"columns":["vc"],"resultFormat":"compactedList","limit":5} - druid.query.type scan - Select Operator - expressions: vc (type: string) - outputColumnNames: _col0 - ListSink - -PREHOOK: query: select substring(to_date(`__time`), 4) from druid_table_alltypesorc limit 5 -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: select substring(to_date(`__time`), 4) from druid_table_alltypesorc limit 5 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -9-12-31 -9-12-31 -9-12-31 -9-12-31 -9-12-31 -PREHOOK: query: explain select substring(cast(to_date(`__time`) as string), 4) from druid_table_alltypesorc limit 5 -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: explain select substring(cast(to_date(`__time`) as string), 4) from druid_table_alltypesorc limit 5 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_table_alltypesorc - properties: - druid.fieldNames vc - druid.fieldTypes string - druid.query.json {"queryType":"scan","dataSource":"default.druid_table_alltypesorc","intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"],"virtualColumns":[{"type":"expression","name":"vc","expression":"substring(timestamp_format(timestamp_floor(\"__time\",'P1D','','US/Pacific'),'yyyy-MM-dd','UTC'), 3, -1)","outputType":"STRING"}],"columns":["vc"],"resultFormat":"compactedList","limit":5} - druid.query.type scan - Select Operator - expressions: vc (type: string) - outputColumnNames: _col0 - ListSink - -PREHOOK: query: select substring(cast(to_date(`__time`) as string), 4) from druid_table_alltypesorc limit 5 -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: select substring(cast(to_date(`__time`) as string), 4) from druid_table_alltypesorc limit 5 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -9-12-31 -9-12-31 -9-12-31 -9-12-31 -9-12-31 diff --git a/ql/src/test/results/clientpositive/druid/druidmini_test_alter.q.out b/ql/src/test/results/clientpositive/druid/druidmini_test_alter.q.out deleted file mode 100644 index 0627cfdcf527..000000000000 --- a/ql/src/test/results/clientpositive/druid/druidmini_test_alter.q.out +++ /dev/null @@ -1,210 +0,0 @@ -PREHOOK: query: CREATE EXTERNAL TABLE druid_alltypesorc_n0 -STORED BY 'org.apache.hadoop.hive.druid.QTestDruidStorageHandlerToAddFaultyHost' -TBLPROPERTIES ("druid.segment.granularity" = "HOUR", "druid.query.granularity" = "MINUTE") -AS - SELECT cast (`ctimestamp2` as timestamp with local time zone) as `__time`, -cstring1, -cdouble, -cfloat, -ctinyint, -csmallint, -cint, -cbigint, -cboolean1 -FROM alltypesorc where ctimestamp2 IS NOT NULL -PREHOOK: type: CREATETABLE_AS_SELECT -PREHOOK: Input: default@alltypesorc -PREHOOK: Output: database:default -PREHOOK: Output: default@druid_alltypesorc_n0 -POSTHOOK: query: CREATE EXTERNAL TABLE druid_alltypesorc_n0 -STORED BY 'org.apache.hadoop.hive.druid.QTestDruidStorageHandlerToAddFaultyHost' -TBLPROPERTIES ("druid.segment.granularity" = "HOUR", "druid.query.granularity" = "MINUTE") -AS - SELECT cast (`ctimestamp2` as timestamp with local time zone) as `__time`, -cstring1, -cdouble, -cfloat, -ctinyint, -csmallint, -cint, -cbigint, -cboolean1 -FROM alltypesorc where ctimestamp2 IS NOT NULL -POSTHOOK: type: CREATETABLE_AS_SELECT -POSTHOOK: Input: default@alltypesorc -POSTHOOK: Output: database:default -POSTHOOK: Output: default@druid_alltypesorc_n0 -POSTHOOK: Lineage: druid_alltypesorc_n0.__time EXPRESSION [(alltypesorc)alltypesorc.FieldSchema(name:ctimestamp2, type:timestamp, comment:null), ] -POSTHOOK: Lineage: druid_alltypesorc_n0.cbigint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cbigint, type:bigint, comment:null), ] -POSTHOOK: Lineage: druid_alltypesorc_n0.cboolean1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cboolean1, type:boolean, comment:null), ] -POSTHOOK: Lineage: druid_alltypesorc_n0.cdouble SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cdouble, type:double, comment:null), ] -POSTHOOK: Lineage: druid_alltypesorc_n0.cfloat SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cfloat, type:float, comment:null), ] -POSTHOOK: Lineage: druid_alltypesorc_n0.cint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] -POSTHOOK: Lineage: druid_alltypesorc_n0.csmallint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:csmallint, type:smallint, comment:null), ] -POSTHOOK: Lineage: druid_alltypesorc_n0.cstring1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ] -POSTHOOK: Lineage: druid_alltypesorc_n0.ctinyint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctinyint, type:tinyint, comment:null), ] -PREHOOK: query: DESCRIBE druid_alltypesorc_n0 -PREHOOK: type: DESCTABLE -PREHOOK: Input: default@druid_alltypesorc_n0 -POSTHOOK: query: DESCRIBE druid_alltypesorc_n0 -POSTHOOK: type: DESCTABLE -POSTHOOK: Input: default@druid_alltypesorc_n0 -__time timestamp with local time zone from deserializer -cstring1 string from deserializer -cdouble double from deserializer -cfloat float from deserializer -ctinyint tinyint from deserializer -csmallint smallint from deserializer -cint int from deserializer -cbigint bigint from deserializer -cboolean1 boolean from deserializer -PREHOOK: query: DESCRIBE extended druid_alltypesorc_n0 -PREHOOK: type: DESCTABLE -PREHOOK: Input: default@druid_alltypesorc_n0 -POSTHOOK: query: DESCRIBE extended druid_alltypesorc_n0 -POSTHOOK: type: DESCTABLE -POSTHOOK: Input: default@druid_alltypesorc_n0 -__time timestamp with local time zone from deserializer -cstring1 string from deserializer -cdouble double from deserializer -cfloat float from deserializer -ctinyint tinyint from deserializer -csmallint smallint from deserializer -cint int from deserializer -cbigint bigint from deserializer -cboolean1 boolean from deserializer - -#### A masked pattern was here #### -PREHOOK: query: SELECT COUNT(*) FROM druid_alltypesorc_n0 -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_alltypesorc_n0 -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: SELECT COUNT(*) FROM druid_alltypesorc_n0 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_alltypesorc_n0 -POSTHOOK: Output: hdfs://### HDFS PATH ### -3033 -PREHOOK: query: ALTER TABLE druid_alltypesorc_n0 ADD COLUMNS (cstring2 string, cboolean2 boolean, cint2 int) -PREHOOK: type: ALTERTABLE_ADDCOLS -PREHOOK: Input: default@druid_alltypesorc_n0 -PREHOOK: Output: default@druid_alltypesorc_n0 -POSTHOOK: query: ALTER TABLE druid_alltypesorc_n0 ADD COLUMNS (cstring2 string, cboolean2 boolean, cint2 int) -POSTHOOK: type: ALTERTABLE_ADDCOLS -POSTHOOK: Input: default@druid_alltypesorc_n0 -POSTHOOK: Output: default@druid_alltypesorc_n0 -PREHOOK: query: DESCRIBE druid_alltypesorc_n0 -PREHOOK: type: DESCTABLE -PREHOOK: Input: default@druid_alltypesorc_n0 -POSTHOOK: query: DESCRIBE druid_alltypesorc_n0 -POSTHOOK: type: DESCTABLE -POSTHOOK: Input: default@druid_alltypesorc_n0 -__time timestamp with local time zone from deserializer -cstring1 string from deserializer -cdouble double from deserializer -cfloat float from deserializer -ctinyint tinyint from deserializer -csmallint smallint from deserializer -cint int from deserializer -cbigint bigint from deserializer -cboolean1 boolean from deserializer -cstring2 string from deserializer -cboolean2 boolean from deserializer -cint2 int from deserializer -PREHOOK: query: DESCRIBE extended druid_alltypesorc_n0 -PREHOOK: type: DESCTABLE -PREHOOK: Input: default@druid_alltypesorc_n0 -POSTHOOK: query: DESCRIBE extended druid_alltypesorc_n0 -POSTHOOK: type: DESCTABLE -POSTHOOK: Input: default@druid_alltypesorc_n0 -__time timestamp with local time zone from deserializer -cstring1 string from deserializer -cdouble double from deserializer -cfloat float from deserializer -ctinyint tinyint from deserializer -csmallint smallint from deserializer -cint int from deserializer -cbigint bigint from deserializer -cboolean1 boolean from deserializer -cstring2 string from deserializer -cboolean2 boolean from deserializer -cint2 int from deserializer - -#### A masked pattern was here #### -PREHOOK: query: SELECT COUNT(*) FROM druid_alltypesorc_n0 WHERE cstring2 IS NOT NULL -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_alltypesorc_n0 -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: SELECT COUNT(*) FROM druid_alltypesorc_n0 WHERE cstring2 IS NOT NULL -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_alltypesorc_n0 -POSTHOOK: Output: hdfs://### HDFS PATH ### -0 -PREHOOK: query: INSERT INTO TABLE druid_alltypesorc_n0 - SELECT cast (`ctimestamp1` as timestamp with local time zone) as `__time`, -cstring1, -cdouble, -cfloat, -ctinyint, -csmallint, -cint, -cbigint, -cboolean1, -cstring2, -cboolean2, -cint -FROM alltypesorc where ctimestamp1 IS NOT NULL -PREHOOK: type: QUERY -PREHOOK: Input: default@alltypesorc -PREHOOK: Output: default@druid_alltypesorc_n0 -POSTHOOK: query: INSERT INTO TABLE druid_alltypesorc_n0 - SELECT cast (`ctimestamp1` as timestamp with local time zone) as `__time`, -cstring1, -cdouble, -cfloat, -ctinyint, -csmallint, -cint, -cbigint, -cboolean1, -cstring2, -cboolean2, -cint -FROM alltypesorc where ctimestamp1 IS NOT NULL -POSTHOOK: type: QUERY -POSTHOOK: Input: default@alltypesorc -POSTHOOK: Output: default@druid_alltypesorc_n0 -PREHOOK: query: SELECT COUNT(*) FROM druid_alltypesorc_n0 -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_alltypesorc_n0 -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: SELECT COUNT(*) FROM druid_alltypesorc_n0 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_alltypesorc_n0 -POSTHOOK: Output: hdfs://### HDFS PATH ### -9138 -PREHOOK: query: SELECT COUNT(*) FROM druid_alltypesorc_n0 WHERE cstring2 IS NULL -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_alltypesorc_n0 -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: SELECT COUNT(*) FROM druid_alltypesorc_n0 WHERE cstring2 IS NULL -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_alltypesorc_n0 -POSTHOOK: Output: hdfs://### HDFS PATH ### -3041 -PREHOOK: query: SELECT COUNT(*) FROM druid_alltypesorc_n0 WHERE cstring2 IS NOT NULL -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_alltypesorc_n0 -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: SELECT COUNT(*) FROM druid_alltypesorc_n0 WHERE cstring2 IS NOT NULL -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_alltypesorc_n0 -POSTHOOK: Output: hdfs://### HDFS PATH ### -6097 -PREHOOK: query: DROP TABLE druid_alltypesorc_n0 -PREHOOK: type: DROPTABLE -PREHOOK: Input: default@druid_alltypesorc_n0 -PREHOOK: Output: default@druid_alltypesorc_n0 -POSTHOOK: query: DROP TABLE druid_alltypesorc_n0 -POSTHOOK: type: DROPTABLE -POSTHOOK: Input: default@druid_alltypesorc_n0 -POSTHOOK: Output: default@druid_alltypesorc_n0 diff --git a/ql/src/test/results/clientpositive/druid/druidmini_test_insert.q.out b/ql/src/test/results/clientpositive/druid/druidmini_test_insert.q.out deleted file mode 100644 index bb309e51d8ec..000000000000 --- a/ql/src/test/results/clientpositive/druid/druidmini_test_insert.q.out +++ /dev/null @@ -1,875 +0,0 @@ -PREHOOK: query: CREATE EXTERNAL TABLE druid_alltypesorc -STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' -TBLPROPERTIES ("druid.segment.granularity" = "HOUR", "druid.query.granularity" = "MINUTE") -AS -SELECT cast (`ctimestamp2` as timestamp with local time zone) as `__time`, - cstring1, - cstring2, - cdouble, - cfloat, - ctinyint, - csmallint, - cint, - cbigint, - cboolean1, - cboolean2 - FROM alltypesorc where ctimestamp2 IS NOT NULL -PREHOOK: type: CREATETABLE_AS_SELECT -PREHOOK: Input: default@alltypesorc -PREHOOK: Output: database:default -PREHOOK: Output: default@druid_alltypesorc -POSTHOOK: query: CREATE EXTERNAL TABLE druid_alltypesorc -STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' -TBLPROPERTIES ("druid.segment.granularity" = "HOUR", "druid.query.granularity" = "MINUTE") -AS -SELECT cast (`ctimestamp2` as timestamp with local time zone) as `__time`, - cstring1, - cstring2, - cdouble, - cfloat, - ctinyint, - csmallint, - cint, - cbigint, - cboolean1, - cboolean2 - FROM alltypesorc where ctimestamp2 IS NOT NULL -POSTHOOK: type: CREATETABLE_AS_SELECT -POSTHOOK: Input: default@alltypesorc -POSTHOOK: Output: database:default -POSTHOOK: Output: default@druid_alltypesorc -POSTHOOK: Lineage: druid_alltypesorc.__time EXPRESSION [(alltypesorc)alltypesorc.FieldSchema(name:ctimestamp2, type:timestamp, comment:null), ] -POSTHOOK: Lineage: druid_alltypesorc.cbigint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cbigint, type:bigint, comment:null), ] -POSTHOOK: Lineage: druid_alltypesorc.cboolean1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cboolean1, type:boolean, comment:null), ] -POSTHOOK: Lineage: druid_alltypesorc.cboolean2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cboolean2, type:boolean, comment:null), ] -POSTHOOK: Lineage: druid_alltypesorc.cdouble SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cdouble, type:double, comment:null), ] -POSTHOOK: Lineage: druid_alltypesorc.cfloat SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cfloat, type:float, comment:null), ] -POSTHOOK: Lineage: druid_alltypesorc.cint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] -POSTHOOK: Lineage: druid_alltypesorc.csmallint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:csmallint, type:smallint, comment:null), ] -POSTHOOK: Lineage: druid_alltypesorc.cstring1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ] -POSTHOOK: Lineage: druid_alltypesorc.cstring2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring2, type:string, comment:null), ] -POSTHOOK: Lineage: druid_alltypesorc.ctinyint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctinyint, type:tinyint, comment:null), ] -PREHOOK: query: SELECT COUNT(*) FROM druid_alltypesorc -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: SELECT COUNT(*) FROM druid_alltypesorc -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -6057 -PREHOOK: query: INSERT INTO TABLE druid_alltypesorc -SELECT cast (`ctimestamp1` as timestamp with local time zone) as `__time`, - cstring1, - cstring2, - cdouble, - cfloat, - ctinyint, - csmallint, - cint, - cbigint, - cboolean1, - cboolean2 - FROM alltypesorc where ctimestamp1 IS NOT NULL -PREHOOK: type: QUERY -PREHOOK: Input: default@alltypesorc -PREHOOK: Output: default@druid_alltypesorc -POSTHOOK: query: INSERT INTO TABLE druid_alltypesorc -SELECT cast (`ctimestamp1` as timestamp with local time zone) as `__time`, - cstring1, - cstring2, - cdouble, - cfloat, - ctinyint, - csmallint, - cint, - cbigint, - cboolean1, - cboolean2 - FROM alltypesorc where ctimestamp1 IS NOT NULL -POSTHOOK: type: QUERY -POSTHOOK: Input: default@alltypesorc -POSTHOOK: Output: default@druid_alltypesorc -PREHOOK: query: analyze table druid_alltypesorc compute statistics for columns -PREHOOK: type: ANALYZE_TABLE -PREHOOK: Input: default@druid_alltypesorc -PREHOOK: Output: default@druid_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: analyze table druid_alltypesorc compute statistics for columns -POSTHOOK: type: ANALYZE_TABLE -POSTHOOK: Input: default@druid_alltypesorc -POSTHOOK: Output: default@druid_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -PREHOOK: query: SELECT COUNT(*) FROM druid_alltypesorc -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: SELECT COUNT(*) FROM druid_alltypesorc -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -12162 -PREHOOK: query: INSERT OVERWRITE TABLE druid_alltypesorc -SELECT cast (`ctimestamp1` as timestamp with local time zone) as `__time`, - cstring1, - cstring2, - cdouble, - cfloat, - ctinyint, - csmallint, - cint, - cbigint, - cboolean1, - cboolean2 - FROM alltypesorc where ctimestamp1 IS NOT NULL -PREHOOK: type: QUERY -PREHOOK: Input: default@alltypesorc -PREHOOK: Output: default@druid_alltypesorc -POSTHOOK: query: INSERT OVERWRITE TABLE druid_alltypesorc -SELECT cast (`ctimestamp1` as timestamp with local time zone) as `__time`, - cstring1, - cstring2, - cdouble, - cfloat, - ctinyint, - csmallint, - cint, - cbigint, - cboolean1, - cboolean2 - FROM alltypesorc where ctimestamp1 IS NOT NULL -POSTHOOK: type: QUERY -POSTHOOK: Input: default@alltypesorc -POSTHOOK: Output: default@druid_alltypesorc -PREHOOK: query: SELECT COUNT(*) FROM druid_alltypesorc -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: SELECT COUNT(*) FROM druid_alltypesorc -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -6105 -PREHOOK: query: DROP TABLE druid_alltypesorc -PREHOOK: type: DROPTABLE -PREHOOK: Input: default@druid_alltypesorc -PREHOOK: Output: default@druid_alltypesorc -POSTHOOK: query: DROP TABLE druid_alltypesorc -POSTHOOK: type: DROPTABLE -POSTHOOK: Input: default@druid_alltypesorc -POSTHOOK: Output: default@druid_alltypesorc -PREHOOK: query: -- Test create then insert - - create database druid_test_create_then_insert -PREHOOK: type: CREATEDATABASE -PREHOOK: Output: database:druid_test_create_then_insert -POSTHOOK: query: -- Test create then insert - - create database druid_test_create_then_insert -POSTHOOK: type: CREATEDATABASE -POSTHOOK: Output: database:druid_test_create_then_insert -PREHOOK: query: use druid_test_create_then_insert -PREHOOK: type: SWITCHDATABASE -PREHOOK: Input: database:druid_test_create_then_insert -POSTHOOK: query: use druid_test_create_then_insert -POSTHOOK: type: SWITCHDATABASE -POSTHOOK: Input: database:druid_test_create_then_insert -PREHOOK: query: create table test_table_n9(`timecolumn` timestamp, `userid` string, `num_l` float) -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:druid_test_create_then_insert -PREHOOK: Output: druid_test_create_then_insert@test_table_n9 -POSTHOOK: query: create table test_table_n9(`timecolumn` timestamp, `userid` string, `num_l` float) -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:druid_test_create_then_insert -POSTHOOK: Output: druid_test_create_then_insert@test_table_n9 -PREHOOK: query: insert into test_table_n9 values ('2015-01-08 00:00:00', 'i1-start', 4) -PREHOOK: type: QUERY -PREHOOK: Input: _dummy_database@_dummy_table -PREHOOK: Output: druid_test_create_then_insert@test_table_n9 -POSTHOOK: query: insert into test_table_n9 values ('2015-01-08 00:00:00', 'i1-start', 4) -POSTHOOK: type: QUERY -POSTHOOK: Input: _dummy_database@_dummy_table -POSTHOOK: Output: druid_test_create_then_insert@test_table_n9 -POSTHOOK: Lineage: test_table_n9.num_l SCRIPT [] -POSTHOOK: Lineage: test_table_n9.timecolumn SCRIPT [] -POSTHOOK: Lineage: test_table_n9.userid SCRIPT [] -PREHOOK: query: insert into test_table_n9 values ('2015-01-08 23:59:59', 'i1-end', 1) -PREHOOK: type: QUERY -PREHOOK: Input: _dummy_database@_dummy_table -PREHOOK: Output: druid_test_create_then_insert@test_table_n9 -POSTHOOK: query: insert into test_table_n9 values ('2015-01-08 23:59:59', 'i1-end', 1) -POSTHOOK: type: QUERY -POSTHOOK: Input: _dummy_database@_dummy_table -POSTHOOK: Output: druid_test_create_then_insert@test_table_n9 -POSTHOOK: Lineage: test_table_n9.num_l SCRIPT [] -POSTHOOK: Lineage: test_table_n9.timecolumn SCRIPT [] -POSTHOOK: Lineage: test_table_n9.userid SCRIPT [] -PREHOOK: query: CREATE EXTERNAL TABLE druid_table_n1 (`__time` timestamp with local time zone, `userid` string, `num_l` float) - STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' - TBLPROPERTIES ("druid.segment.granularity" = "DAY") -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:druid_test_create_then_insert -PREHOOK: Output: druid_test_create_then_insert@druid_table_n1 -POSTHOOK: query: CREATE EXTERNAL TABLE druid_table_n1 (`__time` timestamp with local time zone, `userid` string, `num_l` float) - STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' - TBLPROPERTIES ("druid.segment.granularity" = "DAY") -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:druid_test_create_then_insert -POSTHOOK: Output: druid_test_create_then_insert@druid_table_n1 -PREHOOK: query: INSERT INTO TABLE druid_table_n1 - select cast(`timecolumn` as timestamp with local time zone) as `__time`, `userid`, `num_l` FROM test_table_n9 -PREHOOK: type: QUERY -PREHOOK: Input: druid_test_create_then_insert@test_table_n9 -PREHOOK: Output: druid_test_create_then_insert@druid_table_n1 -POSTHOOK: query: INSERT INTO TABLE druid_table_n1 - select cast(`timecolumn` as timestamp with local time zone) as `__time`, `userid`, `num_l` FROM test_table_n9 -POSTHOOK: type: QUERY -POSTHOOK: Input: druid_test_create_then_insert@test_table_n9 -POSTHOOK: Output: druid_test_create_then_insert@druid_table_n1 -PREHOOK: query: select count(*) FROM druid_table_n1 -PREHOOK: type: QUERY -PREHOOK: Input: druid_test_create_then_insert@druid_table_n1 -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: select count(*) FROM druid_table_n1 -POSTHOOK: type: QUERY -POSTHOOK: Input: druid_test_create_then_insert@druid_table_n1 -POSTHOOK: Output: hdfs://### HDFS PATH ### -2 -PREHOOK: query: DROP TABLE test_table_n9 -PREHOOK: type: DROPTABLE -PREHOOK: Input: druid_test_create_then_insert@test_table_n9 -PREHOOK: Output: druid_test_create_then_insert@test_table_n9 -POSTHOOK: query: DROP TABLE test_table_n9 -POSTHOOK: type: DROPTABLE -POSTHOOK: Input: druid_test_create_then_insert@test_table_n9 -POSTHOOK: Output: druid_test_create_then_insert@test_table_n9 -PREHOOK: query: DROP TABLE druid_table_n1 -PREHOOK: type: DROPTABLE -PREHOOK: Input: druid_test_create_then_insert@druid_table_n1 -PREHOOK: Output: druid_test_create_then_insert@druid_table_n1 -POSTHOOK: query: DROP TABLE druid_table_n1 -POSTHOOK: type: DROPTABLE -POSTHOOK: Input: druid_test_create_then_insert@druid_table_n1 -POSTHOOK: Output: druid_test_create_then_insert@druid_table_n1 -PREHOOK: query: DROP DATABASE druid_test_create_then_insert -PREHOOK: type: DROPDATABASE -PREHOOK: Input: database:druid_test_create_then_insert -PREHOOK: Output: database:druid_test_create_then_insert -POSTHOOK: query: DROP DATABASE druid_test_create_then_insert -POSTHOOK: type: DROPDATABASE -POSTHOOK: Input: database:druid_test_create_then_insert -POSTHOOK: Output: database:druid_test_create_then_insert -PREHOOK: query: create database druid_test_dst -PREHOOK: type: CREATEDATABASE -PREHOOK: Output: database:druid_test_dst -POSTHOOK: query: create database druid_test_dst -POSTHOOK: type: CREATEDATABASE -POSTHOOK: Output: database:druid_test_dst -PREHOOK: query: use druid_test_dst -PREHOOK: type: SWITCHDATABASE -PREHOOK: Input: database:druid_test_dst -POSTHOOK: query: use druid_test_dst -POSTHOOK: type: SWITCHDATABASE -POSTHOOK: Input: database:druid_test_dst -PREHOOK: query: create table test_base_table(`timecolumn` timestamp, `userid` string, `num_l` float) -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:druid_test_dst -PREHOOK: Output: druid_test_dst@test_base_table -POSTHOOK: query: create table test_base_table(`timecolumn` timestamp, `userid` string, `num_l` float) -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:druid_test_dst -POSTHOOK: Output: druid_test_dst@test_base_table -PREHOOK: query: insert into test_base_table values ('2015-03-08 00:00:00', 'i1-start', 4) -PREHOOK: type: QUERY -PREHOOK: Input: _dummy_database@_dummy_table -PREHOOK: Output: druid_test_dst@test_base_table -POSTHOOK: query: insert into test_base_table values ('2015-03-08 00:00:00', 'i1-start', 4) -POSTHOOK: type: QUERY -POSTHOOK: Input: _dummy_database@_dummy_table -POSTHOOK: Output: druid_test_dst@test_base_table -POSTHOOK: Lineage: test_base_table.num_l SCRIPT [] -POSTHOOK: Lineage: test_base_table.timecolumn SCRIPT [] -POSTHOOK: Lineage: test_base_table.userid SCRIPT [] -PREHOOK: query: insert into test_base_table values ('2015-03-08 23:59:59', 'i1-end', 1) -PREHOOK: type: QUERY -PREHOOK: Input: _dummy_database@_dummy_table -PREHOOK: Output: druid_test_dst@test_base_table -POSTHOOK: query: insert into test_base_table values ('2015-03-08 23:59:59', 'i1-end', 1) -POSTHOOK: type: QUERY -POSTHOOK: Input: _dummy_database@_dummy_table -POSTHOOK: Output: druid_test_dst@test_base_table -POSTHOOK: Lineage: test_base_table.num_l SCRIPT [] -POSTHOOK: Lineage: test_base_table.timecolumn SCRIPT [] -POSTHOOK: Lineage: test_base_table.userid SCRIPT [] -PREHOOK: query: insert into test_base_table values ('2015-03-09 00:00:00', 'i2-start', 4) -PREHOOK: type: QUERY -PREHOOK: Input: _dummy_database@_dummy_table -PREHOOK: Output: druid_test_dst@test_base_table -POSTHOOK: query: insert into test_base_table values ('2015-03-09 00:00:00', 'i2-start', 4) -POSTHOOK: type: QUERY -POSTHOOK: Input: _dummy_database@_dummy_table -POSTHOOK: Output: druid_test_dst@test_base_table -POSTHOOK: Lineage: test_base_table.num_l SCRIPT [] -POSTHOOK: Lineage: test_base_table.timecolumn SCRIPT [] -POSTHOOK: Lineage: test_base_table.userid SCRIPT [] -PREHOOK: query: insert into test_base_table values ('2015-03-09 23:59:59', 'i2-end', 1) -PREHOOK: type: QUERY -PREHOOK: Input: _dummy_database@_dummy_table -PREHOOK: Output: druid_test_dst@test_base_table -POSTHOOK: query: insert into test_base_table values ('2015-03-09 23:59:59', 'i2-end', 1) -POSTHOOK: type: QUERY -POSTHOOK: Input: _dummy_database@_dummy_table -POSTHOOK: Output: druid_test_dst@test_base_table -POSTHOOK: Lineage: test_base_table.num_l SCRIPT [] -POSTHOOK: Lineage: test_base_table.timecolumn SCRIPT [] -POSTHOOK: Lineage: test_base_table.userid SCRIPT [] -PREHOOK: query: insert into test_base_table values ('2015-03-10 00:00:00', 'i3-start', 2) -PREHOOK: type: QUERY -PREHOOK: Input: _dummy_database@_dummy_table -PREHOOK: Output: druid_test_dst@test_base_table -POSTHOOK: query: insert into test_base_table values ('2015-03-10 00:00:00', 'i3-start', 2) -POSTHOOK: type: QUERY -POSTHOOK: Input: _dummy_database@_dummy_table -POSTHOOK: Output: druid_test_dst@test_base_table -POSTHOOK: Lineage: test_base_table.num_l SCRIPT [] -POSTHOOK: Lineage: test_base_table.timecolumn SCRIPT [] -POSTHOOK: Lineage: test_base_table.userid SCRIPT [] -PREHOOK: query: insert into test_base_table values ('2015-03-10 23:59:59', 'i3-end', 2) -PREHOOK: type: QUERY -PREHOOK: Input: _dummy_database@_dummy_table -PREHOOK: Output: druid_test_dst@test_base_table -POSTHOOK: query: insert into test_base_table values ('2015-03-10 23:59:59', 'i3-end', 2) -POSTHOOK: type: QUERY -POSTHOOK: Input: _dummy_database@_dummy_table -POSTHOOK: Output: druid_test_dst@test_base_table -POSTHOOK: Lineage: test_base_table.num_l SCRIPT [] -POSTHOOK: Lineage: test_base_table.timecolumn SCRIPT [] -POSTHOOK: Lineage: test_base_table.userid SCRIPT [] -PREHOOK: query: CREATE EXTERNAL TABLE druid_test_table_n9 -STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' -TBLPROPERTIES ("druid.segment.granularity" = "DAY") -AS -select cast(`timecolumn` as timestamp with local time zone) as `__time`, `userid`, `num_l` FROM test_base_table -PREHOOK: type: CREATETABLE_AS_SELECT -PREHOOK: Input: druid_test_dst@test_base_table -PREHOOK: Output: database:druid_test_dst -PREHOOK: Output: druid_test_dst@druid_test_table_n9 -POSTHOOK: query: CREATE EXTERNAL TABLE druid_test_table_n9 -STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' -TBLPROPERTIES ("druid.segment.granularity" = "DAY") -AS -select cast(`timecolumn` as timestamp with local time zone) as `__time`, `userid`, `num_l` FROM test_base_table -POSTHOOK: type: CREATETABLE_AS_SELECT -POSTHOOK: Input: druid_test_dst@test_base_table -POSTHOOK: Output: database:druid_test_dst -POSTHOOK: Output: druid_test_dst@druid_test_table_n9 -POSTHOOK: Lineage: druid_test_table_n9.__time EXPRESSION [(test_base_table)test_base_table.FieldSchema(name:timecolumn, type:timestamp, comment:null), ] -POSTHOOK: Lineage: druid_test_table_n9.num_l SIMPLE [(test_base_table)test_base_table.FieldSchema(name:num_l, type:float, comment:null), ] -POSTHOOK: Lineage: druid_test_table_n9.userid SIMPLE [(test_base_table)test_base_table.FieldSchema(name:userid, type:string, comment:null), ] -PREHOOK: query: select * FROM druid_test_table_n9 -PREHOOK: type: QUERY -PREHOOK: Input: druid_test_dst@druid_test_table_n9 -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: select * FROM druid_test_table_n9 -POSTHOOK: type: QUERY -POSTHOOK: Input: druid_test_dst@druid_test_table_n9 -POSTHOOK: Output: hdfs://### HDFS PATH ### -2015-03-08 00:00:00.0 US/Pacific i1-start 4.0 -2015-03-08 23:59:59.0 US/Pacific i1-end 1.0 -2015-03-09 00:00:00.0 US/Pacific i2-start 4.0 -2015-03-09 23:59:59.0 US/Pacific i2-end 1.0 -2015-03-10 00:00:00.0 US/Pacific i3-start 2.0 -2015-03-10 23:59:59.0 US/Pacific i3-end 2.0 -PREHOOK: query: select * from druid_test_table_n9 where `__time` = cast('2015-03-08 00:00:00' as timestamp with local time zone) -PREHOOK: type: QUERY -PREHOOK: Input: druid_test_dst@druid_test_table_n9 -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: select * from druid_test_table_n9 where `__time` = cast('2015-03-08 00:00:00' as timestamp with local time zone) -POSTHOOK: type: QUERY -POSTHOOK: Input: druid_test_dst@druid_test_table_n9 -POSTHOOK: Output: hdfs://### HDFS PATH ### -2015-03-08 00:00:00.0 US/Pacific i1-start 4.0 -PREHOOK: query: select * from druid_test_table_n9 where `__time` = cast('2015-03-08 23:59:59' as timestamp with local time zone) -PREHOOK: type: QUERY -PREHOOK: Input: druid_test_dst@druid_test_table_n9 -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: select * from druid_test_table_n9 where `__time` = cast('2015-03-08 23:59:59' as timestamp with local time zone) -POSTHOOK: type: QUERY -POSTHOOK: Input: druid_test_dst@druid_test_table_n9 -POSTHOOK: Output: hdfs://### HDFS PATH ### -2015-03-08 23:59:59.0 US/Pacific i1-end 1.0 -PREHOOK: query: select * from druid_test_table_n9 where `__time` = cast('2015-03-09 00:00:00' as timestamp with local time zone) -PREHOOK: type: QUERY -PREHOOK: Input: druid_test_dst@druid_test_table_n9 -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: select * from druid_test_table_n9 where `__time` = cast('2015-03-09 00:00:00' as timestamp with local time zone) -POSTHOOK: type: QUERY -POSTHOOK: Input: druid_test_dst@druid_test_table_n9 -POSTHOOK: Output: hdfs://### HDFS PATH ### -2015-03-09 00:00:00.0 US/Pacific i2-start 4.0 -PREHOOK: query: select * from druid_test_table_n9 where `__time` = cast('2015-03-09 23:59:59' as timestamp with local time zone) -PREHOOK: type: QUERY -PREHOOK: Input: druid_test_dst@druid_test_table_n9 -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: select * from druid_test_table_n9 where `__time` = cast('2015-03-09 23:59:59' as timestamp with local time zone) -POSTHOOK: type: QUERY -POSTHOOK: Input: druid_test_dst@druid_test_table_n9 -POSTHOOK: Output: hdfs://### HDFS PATH ### -2015-03-09 23:59:59.0 US/Pacific i2-end 1.0 -PREHOOK: query: select * from druid_test_table_n9 where `__time` = cast('2015-03-10 00:00:00' as timestamp with local time zone) -PREHOOK: type: QUERY -PREHOOK: Input: druid_test_dst@druid_test_table_n9 -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: select * from druid_test_table_n9 where `__time` = cast('2015-03-10 00:00:00' as timestamp with local time zone) -POSTHOOK: type: QUERY -POSTHOOK: Input: druid_test_dst@druid_test_table_n9 -POSTHOOK: Output: hdfs://### HDFS PATH ### -2015-03-10 00:00:00.0 US/Pacific i3-start 2.0 -PREHOOK: query: select * from druid_test_table_n9 where `__time` = cast('2015-03-10 23:59:59' as timestamp with local time zone) -PREHOOK: type: QUERY -PREHOOK: Input: druid_test_dst@druid_test_table_n9 -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: select * from druid_test_table_n9 where `__time` = cast('2015-03-10 23:59:59' as timestamp with local time zone) -POSTHOOK: type: QUERY -POSTHOOK: Input: druid_test_dst@druid_test_table_n9 -POSTHOOK: Output: hdfs://### HDFS PATH ### -2015-03-10 23:59:59.0 US/Pacific i3-end 2.0 -PREHOOK: query: explain select * from druid_test_table_n9 where `__time` = cast('2015-03-08 00:00:00' as timestamp with local time zone) -PREHOOK: type: QUERY -PREHOOK: Input: druid_test_dst@druid_test_table_n9 -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: explain select * from druid_test_table_n9 where `__time` = cast('2015-03-08 00:00:00' as timestamp with local time zone) -POSTHOOK: type: QUERY -POSTHOOK: Input: druid_test_dst@druid_test_table_n9 -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_test_table_n9 - properties: - druid.fieldNames userid,num_l - druid.fieldTypes string,float - druid.query.json {"queryType":"scan","dataSource":"druid_test_dst.druid_test_table_n9","intervals":["2015-03-08T08:00:00.000Z/2015-03-08T08:00:00.001Z"],"columns":["userid","num_l"],"resultFormat":"compactedList"} - druid.query.type scan - Select Operator - expressions: TIMESTAMPLOCALTZ'2015-03-08 00:00:00.0 US/Pacific' (type: timestamp with local time zone), userid (type: string), num_l (type: float) - outputColumnNames: _col0, _col1, _col2 - ListSink - -PREHOOK: query: explain select * from druid_test_table_n9 where `__time` = cast('2015-03-08 23:59:59' as timestamp with local time zone) -PREHOOK: type: QUERY -PREHOOK: Input: druid_test_dst@druid_test_table_n9 -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: explain select * from druid_test_table_n9 where `__time` = cast('2015-03-08 23:59:59' as timestamp with local time zone) -POSTHOOK: type: QUERY -POSTHOOK: Input: druid_test_dst@druid_test_table_n9 -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_test_table_n9 - properties: - druid.fieldNames userid,num_l - druid.fieldTypes string,float - druid.query.json {"queryType":"scan","dataSource":"druid_test_dst.druid_test_table_n9","intervals":["2015-03-09T06:59:59.000Z/2015-03-09T06:59:59.001Z"],"columns":["userid","num_l"],"resultFormat":"compactedList"} - druid.query.type scan - Select Operator - expressions: TIMESTAMPLOCALTZ'2015-03-08 23:59:59.0 US/Pacific' (type: timestamp with local time zone), userid (type: string), num_l (type: float) - outputColumnNames: _col0, _col1, _col2 - ListSink - -PREHOOK: query: explain select * from druid_test_table_n9 where `__time` = cast('2015-03-09 00:00:00' as timestamp with local time zone) -PREHOOK: type: QUERY -PREHOOK: Input: druid_test_dst@druid_test_table_n9 -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: explain select * from druid_test_table_n9 where `__time` = cast('2015-03-09 00:00:00' as timestamp with local time zone) -POSTHOOK: type: QUERY -POSTHOOK: Input: druid_test_dst@druid_test_table_n9 -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_test_table_n9 - properties: - druid.fieldNames userid,num_l - druid.fieldTypes string,float - druid.query.json {"queryType":"scan","dataSource":"druid_test_dst.druid_test_table_n9","intervals":["2015-03-09T07:00:00.000Z/2015-03-09T07:00:00.001Z"],"columns":["userid","num_l"],"resultFormat":"compactedList"} - druid.query.type scan - Select Operator - expressions: TIMESTAMPLOCALTZ'2015-03-09 00:00:00.0 US/Pacific' (type: timestamp with local time zone), userid (type: string), num_l (type: float) - outputColumnNames: _col0, _col1, _col2 - ListSink - -PREHOOK: query: explain select * from druid_test_table_n9 where `__time` = cast('2015-03-09 23:59:59' as timestamp with local time zone) -PREHOOK: type: QUERY -PREHOOK: Input: druid_test_dst@druid_test_table_n9 -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: explain select * from druid_test_table_n9 where `__time` = cast('2015-03-09 23:59:59' as timestamp with local time zone) -POSTHOOK: type: QUERY -POSTHOOK: Input: druid_test_dst@druid_test_table_n9 -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_test_table_n9 - properties: - druid.fieldNames userid,num_l - druid.fieldTypes string,float - druid.query.json {"queryType":"scan","dataSource":"druid_test_dst.druid_test_table_n9","intervals":["2015-03-10T06:59:59.000Z/2015-03-10T06:59:59.001Z"],"columns":["userid","num_l"],"resultFormat":"compactedList"} - druid.query.type scan - Select Operator - expressions: TIMESTAMPLOCALTZ'2015-03-09 23:59:59.0 US/Pacific' (type: timestamp with local time zone), userid (type: string), num_l (type: float) - outputColumnNames: _col0, _col1, _col2 - ListSink - -PREHOOK: query: explain select * from druid_test_table_n9 where `__time` = cast('2015-03-10 00:00:00' as timestamp with local time zone) -PREHOOK: type: QUERY -PREHOOK: Input: druid_test_dst@druid_test_table_n9 -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: explain select * from druid_test_table_n9 where `__time` = cast('2015-03-10 00:00:00' as timestamp with local time zone) -POSTHOOK: type: QUERY -POSTHOOK: Input: druid_test_dst@druid_test_table_n9 -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_test_table_n9 - properties: - druid.fieldNames userid,num_l - druid.fieldTypes string,float - druid.query.json {"queryType":"scan","dataSource":"druid_test_dst.druid_test_table_n9","intervals":["2015-03-10T07:00:00.000Z/2015-03-10T07:00:00.001Z"],"columns":["userid","num_l"],"resultFormat":"compactedList"} - druid.query.type scan - Select Operator - expressions: TIMESTAMPLOCALTZ'2015-03-10 00:00:00.0 US/Pacific' (type: timestamp with local time zone), userid (type: string), num_l (type: float) - outputColumnNames: _col0, _col1, _col2 - ListSink - -PREHOOK: query: explain select * from druid_test_table_n9 where `__time` = cast('2015-03-10 23:59:59' as timestamp with local time zone) -PREHOOK: type: QUERY -PREHOOK: Input: druid_test_dst@druid_test_table_n9 -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: explain select * from druid_test_table_n9 where `__time` = cast('2015-03-10 23:59:59' as timestamp with local time zone) -POSTHOOK: type: QUERY -POSTHOOK: Input: druid_test_dst@druid_test_table_n9 -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_test_table_n9 - properties: - druid.fieldNames userid,num_l - druid.fieldTypes string,float - druid.query.json {"queryType":"scan","dataSource":"druid_test_dst.druid_test_table_n9","intervals":["2015-03-11T06:59:59.000Z/2015-03-11T06:59:59.001Z"],"columns":["userid","num_l"],"resultFormat":"compactedList"} - druid.query.type scan - Select Operator - expressions: TIMESTAMPLOCALTZ'2015-03-10 23:59:59.0 US/Pacific' (type: timestamp with local time zone), userid (type: string), num_l (type: float) - outputColumnNames: _col0, _col1, _col2 - ListSink - -PREHOOK: query: select * from druid_test_table_n9 where `__time` = cast('2015-03-08 00:00:00' as timestamp ) -PREHOOK: type: QUERY -PREHOOK: Input: druid_test_dst@druid_test_table_n9 -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: select * from druid_test_table_n9 where `__time` = cast('2015-03-08 00:00:00' as timestamp ) -POSTHOOK: type: QUERY -POSTHOOK: Input: druid_test_dst@druid_test_table_n9 -POSTHOOK: Output: hdfs://### HDFS PATH ### -2015-03-08 00:00:00.0 US/Pacific i1-start 4.0 -PREHOOK: query: select * from druid_test_table_n9 where `__time` = cast('2015-03-08 23:59:59' as timestamp ) -PREHOOK: type: QUERY -PREHOOK: Input: druid_test_dst@druid_test_table_n9 -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: select * from druid_test_table_n9 where `__time` = cast('2015-03-08 23:59:59' as timestamp ) -POSTHOOK: type: QUERY -POSTHOOK: Input: druid_test_dst@druid_test_table_n9 -POSTHOOK: Output: hdfs://### HDFS PATH ### -2015-03-08 23:59:59.0 US/Pacific i1-end 1.0 -PREHOOK: query: select * from druid_test_table_n9 where `__time` = cast('2015-03-09 00:00:00' as timestamp ) -PREHOOK: type: QUERY -PREHOOK: Input: druid_test_dst@druid_test_table_n9 -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: select * from druid_test_table_n9 where `__time` = cast('2015-03-09 00:00:00' as timestamp ) -POSTHOOK: type: QUERY -POSTHOOK: Input: druid_test_dst@druid_test_table_n9 -POSTHOOK: Output: hdfs://### HDFS PATH ### -2015-03-09 00:00:00.0 US/Pacific i2-start 4.0 -PREHOOK: query: select * from druid_test_table_n9 where `__time` = cast('2015-03-09 23:59:59' as timestamp ) -PREHOOK: type: QUERY -PREHOOK: Input: druid_test_dst@druid_test_table_n9 -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: select * from druid_test_table_n9 where `__time` = cast('2015-03-09 23:59:59' as timestamp ) -POSTHOOK: type: QUERY -POSTHOOK: Input: druid_test_dst@druid_test_table_n9 -POSTHOOK: Output: hdfs://### HDFS PATH ### -2015-03-09 23:59:59.0 US/Pacific i2-end 1.0 -PREHOOK: query: select * from druid_test_table_n9 where `__time` = cast('2015-03-10 00:00:00' as timestamp ) -PREHOOK: type: QUERY -PREHOOK: Input: druid_test_dst@druid_test_table_n9 -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: select * from druid_test_table_n9 where `__time` = cast('2015-03-10 00:00:00' as timestamp ) -POSTHOOK: type: QUERY -POSTHOOK: Input: druid_test_dst@druid_test_table_n9 -POSTHOOK: Output: hdfs://### HDFS PATH ### -2015-03-10 00:00:00.0 US/Pacific i3-start 2.0 -PREHOOK: query: select * from druid_test_table_n9 where `__time` = cast('2015-03-10 23:59:59' as timestamp ) -PREHOOK: type: QUERY -PREHOOK: Input: druid_test_dst@druid_test_table_n9 -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: select * from druid_test_table_n9 where `__time` = cast('2015-03-10 23:59:59' as timestamp ) -POSTHOOK: type: QUERY -POSTHOOK: Input: druid_test_dst@druid_test_table_n9 -POSTHOOK: Output: hdfs://### HDFS PATH ### -2015-03-10 23:59:59.0 US/Pacific i3-end 2.0 -PREHOOK: query: EXPLAIN select * from druid_test_table_n9 where `__time` = cast('2015-03-08 00:00:00' as timestamp ) -PREHOOK: type: QUERY -PREHOOK: Input: druid_test_dst@druid_test_table_n9 -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN select * from druid_test_table_n9 where `__time` = cast('2015-03-08 00:00:00' as timestamp ) -POSTHOOK: type: QUERY -POSTHOOK: Input: druid_test_dst@druid_test_table_n9 -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_test_table_n9 - properties: - druid.fieldNames userid,num_l - druid.fieldTypes string,float - druid.query.json {"queryType":"scan","dataSource":"druid_test_dst.druid_test_table_n9","intervals":["2015-03-08T08:00:00.000Z/2015-03-08T08:00:00.001Z"],"columns":["userid","num_l"],"resultFormat":"compactedList"} - druid.query.type scan - Select Operator - expressions: TIMESTAMPLOCALTZ'2015-03-08 00:00:00.0 US/Pacific' (type: timestamp with local time zone), userid (type: string), num_l (type: float) - outputColumnNames: _col0, _col1, _col2 - ListSink - -PREHOOK: query: EXPLAIN select * from druid_test_table_n9 where `__time` = cast('2015-03-08 23:59:59' as timestamp ) -PREHOOK: type: QUERY -PREHOOK: Input: druid_test_dst@druid_test_table_n9 -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN select * from druid_test_table_n9 where `__time` = cast('2015-03-08 23:59:59' as timestamp ) -POSTHOOK: type: QUERY -POSTHOOK: Input: druid_test_dst@druid_test_table_n9 -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_test_table_n9 - properties: - druid.fieldNames userid,num_l - druid.fieldTypes string,float - druid.query.json {"queryType":"scan","dataSource":"druid_test_dst.druid_test_table_n9","intervals":["2015-03-09T06:59:59.000Z/2015-03-09T06:59:59.001Z"],"columns":["userid","num_l"],"resultFormat":"compactedList"} - druid.query.type scan - Select Operator - expressions: TIMESTAMPLOCALTZ'2015-03-08 23:59:59.0 US/Pacific' (type: timestamp with local time zone), userid (type: string), num_l (type: float) - outputColumnNames: _col0, _col1, _col2 - ListSink - -PREHOOK: query: EXPLAIN select * from druid_test_table_n9 where `__time` = cast('2015-03-09 00:00:00' as timestamp ) -PREHOOK: type: QUERY -PREHOOK: Input: druid_test_dst@druid_test_table_n9 -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN select * from druid_test_table_n9 where `__time` = cast('2015-03-09 00:00:00' as timestamp ) -POSTHOOK: type: QUERY -POSTHOOK: Input: druid_test_dst@druid_test_table_n9 -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_test_table_n9 - properties: - druid.fieldNames userid,num_l - druid.fieldTypes string,float - druid.query.json {"queryType":"scan","dataSource":"druid_test_dst.druid_test_table_n9","intervals":["2015-03-09T07:00:00.000Z/2015-03-09T07:00:00.001Z"],"columns":["userid","num_l"],"resultFormat":"compactedList"} - druid.query.type scan - Select Operator - expressions: TIMESTAMPLOCALTZ'2015-03-09 00:00:00.0 US/Pacific' (type: timestamp with local time zone), userid (type: string), num_l (type: float) - outputColumnNames: _col0, _col1, _col2 - ListSink - -PREHOOK: query: EXPLAIN select * from druid_test_table_n9 where `__time` = cast('2015-03-09 23:59:59' as timestamp ) -PREHOOK: type: QUERY -PREHOOK: Input: druid_test_dst@druid_test_table_n9 -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN select * from druid_test_table_n9 where `__time` = cast('2015-03-09 23:59:59' as timestamp ) -POSTHOOK: type: QUERY -POSTHOOK: Input: druid_test_dst@druid_test_table_n9 -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_test_table_n9 - properties: - druid.fieldNames userid,num_l - druid.fieldTypes string,float - druid.query.json {"queryType":"scan","dataSource":"druid_test_dst.druid_test_table_n9","intervals":["2015-03-10T06:59:59.000Z/2015-03-10T06:59:59.001Z"],"columns":["userid","num_l"],"resultFormat":"compactedList"} - druid.query.type scan - Select Operator - expressions: TIMESTAMPLOCALTZ'2015-03-09 23:59:59.0 US/Pacific' (type: timestamp with local time zone), userid (type: string), num_l (type: float) - outputColumnNames: _col0, _col1, _col2 - ListSink - -PREHOOK: query: EXPLAIN select * from druid_test_table_n9 where `__time` = cast('2015-03-10 00:00:00' as timestamp ) -PREHOOK: type: QUERY -PREHOOK: Input: druid_test_dst@druid_test_table_n9 -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN select * from druid_test_table_n9 where `__time` = cast('2015-03-10 00:00:00' as timestamp ) -POSTHOOK: type: QUERY -POSTHOOK: Input: druid_test_dst@druid_test_table_n9 -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_test_table_n9 - properties: - druid.fieldNames userid,num_l - druid.fieldTypes string,float - druid.query.json {"queryType":"scan","dataSource":"druid_test_dst.druid_test_table_n9","intervals":["2015-03-10T07:00:00.000Z/2015-03-10T07:00:00.001Z"],"columns":["userid","num_l"],"resultFormat":"compactedList"} - druid.query.type scan - Select Operator - expressions: TIMESTAMPLOCALTZ'2015-03-10 00:00:00.0 US/Pacific' (type: timestamp with local time zone), userid (type: string), num_l (type: float) - outputColumnNames: _col0, _col1, _col2 - ListSink - -PREHOOK: query: EXPLAIN select * from druid_test_table_n9 where `__time` = cast('2015-03-10 23:59:59' as timestamp ) -PREHOOK: type: QUERY -PREHOOK: Input: druid_test_dst@druid_test_table_n9 -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN select * from druid_test_table_n9 where `__time` = cast('2015-03-10 23:59:59' as timestamp ) -POSTHOOK: type: QUERY -POSTHOOK: Input: druid_test_dst@druid_test_table_n9 -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_test_table_n9 - properties: - druid.fieldNames userid,num_l - druid.fieldTypes string,float - druid.query.json {"queryType":"scan","dataSource":"druid_test_dst.druid_test_table_n9","intervals":["2015-03-11T06:59:59.000Z/2015-03-11T06:59:59.001Z"],"columns":["userid","num_l"],"resultFormat":"compactedList"} - druid.query.type scan - Select Operator - expressions: TIMESTAMPLOCALTZ'2015-03-10 23:59:59.0 US/Pacific' (type: timestamp with local time zone), userid (type: string), num_l (type: float) - outputColumnNames: _col0, _col1, _col2 - ListSink - -PREHOOK: query: insert into druid_test_table_n9 values -('2015-03-12 00:00:00', 'i4-1', 4), -('2015-03-12 00:00:00', 'i4-1', 2), -('2015-03-12 00:00:00', 'i4-1', 1) -PREHOOK: type: QUERY -PREHOOK: Input: _dummy_database@_dummy_table -PREHOOK: Output: druid_test_dst@druid_test_table_n9 -POSTHOOK: query: insert into druid_test_table_n9 values -('2015-03-12 00:00:00', 'i4-1', 4), -('2015-03-12 00:00:00', 'i4-1', 2), -('2015-03-12 00:00:00', 'i4-1', 1) -POSTHOOK: type: QUERY -POSTHOOK: Input: _dummy_database@_dummy_table -POSTHOOK: Output: druid_test_dst@druid_test_table_n9 -PREHOOK: query: select * from druid_test_table_n9 where `__time` = cast('2015-03-12 00:00:00' as timestamp ) -PREHOOK: type: QUERY -PREHOOK: Input: druid_test_dst@druid_test_table_n9 -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: select * from druid_test_table_n9 where `__time` = cast('2015-03-12 00:00:00' as timestamp ) -POSTHOOK: type: QUERY -POSTHOOK: Input: druid_test_dst@druid_test_table_n9 -POSTHOOK: Output: hdfs://### HDFS PATH ### -2015-03-12 00:00:00.0 US/Pacific i4-1 4.0 -2015-03-12 00:00:00.0 US/Pacific i4-1 2.0 -2015-03-12 00:00:00.0 US/Pacific i4-1 1.0 -PREHOOK: query: DROP TABLE test_base_table -PREHOOK: type: DROPTABLE -PREHOOK: Input: druid_test_dst@test_base_table -PREHOOK: Output: druid_test_dst@test_base_table -POSTHOOK: query: DROP TABLE test_base_table -POSTHOOK: type: DROPTABLE -POSTHOOK: Input: druid_test_dst@test_base_table -POSTHOOK: Output: druid_test_dst@test_base_table -PREHOOK: query: DROP TABLE druid_test_table_n9 -PREHOOK: type: DROPTABLE -PREHOOK: Input: druid_test_dst@druid_test_table_n9 -PREHOOK: Output: druid_test_dst@druid_test_table_n9 -POSTHOOK: query: DROP TABLE druid_test_table_n9 -POSTHOOK: type: DROPTABLE -POSTHOOK: Input: druid_test_dst@druid_test_table_n9 -POSTHOOK: Output: druid_test_dst@druid_test_table_n9 -PREHOOK: query: drop database druid_test_dst -PREHOOK: type: DROPDATABASE -PREHOOK: Input: database:druid_test_dst -PREHOOK: Output: database:druid_test_dst -POSTHOOK: query: drop database druid_test_dst -POSTHOOK: type: DROPDATABASE -POSTHOOK: Input: database:druid_test_dst -POSTHOOK: Output: database:druid_test_dst diff --git a/ql/src/test/results/clientpositive/druid/druidmini_test_ts.q.out b/ql/src/test/results/clientpositive/druid/druidmini_test_ts.q.out deleted file mode 100644 index ea893b9ed540..000000000000 --- a/ql/src/test/results/clientpositive/druid/druidmini_test_ts.q.out +++ /dev/null @@ -1,556 +0,0 @@ -PREHOOK: query: SELECT count(*) FROM druid_table_alltypesorc -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: SELECT count(*) FROM druid_table_alltypesorc -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -9173 -PREHOOK: query: SELECT floor_year(`__time`), SUM(cfloat), SUM(cdouble), SUM(ctinyint), SUM(csmallint),SUM(cint), SUM(cbigint) -FROM druid_table_alltypesorc GROUP BY floor_year(`__time`) -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: SELECT floor_year(`__time`), SUM(cfloat), SUM(cdouble), SUM(ctinyint), SUM(csmallint),SUM(cint), SUM(cbigint) -FROM druid_table_alltypesorc GROUP BY floor_year(`__time`) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -1969-01-01 00:00:00.0 US/Pacific -39590.24699282646 2.73086627938E7 -39967 7781089 1408069801800 10992545287 -PREHOOK: query: SELECT floor_year(`__time`), MIN(cfloat), MIN(cdouble), MIN(ctinyint), MIN(csmallint),MIN(cint), MIN(cbigint) -FROM druid_table_alltypesorc GROUP BY floor_year(`__time`) -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: SELECT floor_year(`__time`), MIN(cfloat), MIN(cdouble), MIN(ctinyint), MIN(csmallint),MIN(cint), MIN(cbigint) -FROM druid_table_alltypesorc GROUP BY floor_year(`__time`) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -1969-01-01 00:00:00.0 US/Pacific -64.0 -16373.0 -64 -16373 -1073279343 -2147311592 -PREHOOK: query: SELECT floor_year(`__time`), MAX(cfloat), MAX(cdouble), MAX(ctinyint), MAX(csmallint),MAX(cint), MAX(cbigint) -FROM druid_table_alltypesorc GROUP BY floor_year(`__time`) -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: SELECT floor_year(`__time`), MAX(cfloat), MAX(cdouble), MAX(ctinyint), MAX(csmallint),MAX(cint), MAX(cbigint) -FROM druid_table_alltypesorc GROUP BY floor_year(`__time`) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -1969-01-01 00:00:00.0 US/Pacific 79.553 9763215.5639 62 16370 1073680599 2145498388 -PREHOOK: query: SELECT cstring1, SUM(cdouble) as s FROM druid_table_alltypesorc GROUP BY cstring1 ORDER BY s ASC LIMIT 10 -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: SELECT cstring1, SUM(cdouble) as s FROM druid_table_alltypesorc GROUP BY cstring1 ORDER BY s ASC LIMIT 10 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -1cGVWH7n1QU -596096.6799999999 -821UdmGbkEf4j -14161.826999999997 -00iT08 0.0 -02v8WnLuYDos3Cq 0.0 -yv1js 0.0 -02VRbSC5I 0.0 -014ILGhXxNY7g02hl0Xw 0.0 -02vDyIVT752 0.0 -00PafC7v 0.0 -ytpx1RL8F2I 0.0 -PREHOOK: query: SELECT cstring2, MAX(cdouble) FROM druid_table_alltypesorc GROUP BY cstring2 ORDER BY cstring2 ASC LIMIT 10 -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: SELECT cstring2, MAX(cdouble) FROM druid_table_alltypesorc GROUP BY cstring2 ORDER BY cstring2 ASC LIMIT 10 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -NULL 9763215.5639 -0034fkcXMQI3 15601.0 -004J8y 0.0 -00GNm -200.0 -00GW4dnb6Wgj52 -200.0 -00PBhB1Iefgk 0.0 -00d5kr1wEB7evExG 15601.0 -00qccwt8n 0.0 -017fFeQ3Gcsa83Xj2Vo0 0.0 -01EfkvNk6mjG44uxs 0.0 -PREHOOK: query: SELECT `__time` -FROM druid_table_alltypesorc ORDER BY `__time` ASC LIMIT 10 -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: SELECT `__time` -FROM druid_table_alltypesorc ORDER BY `__time` ASC LIMIT 10 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -PREHOOK: query: SELECT `__time` -FROM druid_table_alltypesorc -WHERE `__time` < '1970-03-01 00:00:00' ORDER BY `__time` ASC LIMIT 10 -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: SELECT `__time` -FROM druid_table_alltypesorc -WHERE `__time` < '1970-03-01 00:00:00' ORDER BY `__time` ASC LIMIT 10 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -PREHOOK: query: SELECT `__time` -FROM druid_table_alltypesorc -WHERE `__time` >= '1968-01-01 00:00:00' AND `__time` <= '1970-03-01 00:00:00' ORDER BY `__time` ASC LIMIT 10 -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: SELECT `__time` -FROM druid_table_alltypesorc -WHERE `__time` >= '1968-01-01 00:00:00' AND `__time` <= '1970-03-01 00:00:00' ORDER BY `__time` ASC LIMIT 10 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -PREHOOK: query: SELECT `__time` -FROM druid_table_alltypesorc -WHERE `__time` >= '1968-01-01 00:00:00' AND `__time` <= '1970-03-01 00:00:00' - AND `__time` < '2011-01-01 00:00:00' ORDER BY `__time` ASC LIMIT 10 -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: SELECT `__time` -FROM druid_table_alltypesorc -WHERE `__time` >= '1968-01-01 00:00:00' AND `__time` <= '1970-03-01 00:00:00' - AND `__time` < '2011-01-01 00:00:00' ORDER BY `__time` ASC LIMIT 10 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -PREHOOK: query: SELECT `__time` -FROM druid_table_alltypesorc -WHERE `__time` BETWEEN '1968-01-01 00:00:00' AND '1970-01-01 00:00:00' ORDER BY `__time` ASC LIMIT 10 -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: SELECT `__time` -FROM druid_table_alltypesorc -WHERE `__time` BETWEEN '1968-01-01 00:00:00' AND '1970-01-01 00:00:00' ORDER BY `__time` ASC LIMIT 10 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -PREHOOK: query: SELECT `__time` -FROM druid_table_alltypesorc -WHERE (`__time` BETWEEN '1968-01-01 00:00:00' AND '1970-01-01 00:00:00') - OR (`__time` BETWEEN '1968-02-01 00:00:00' AND '1970-04-01 00:00:00') ORDER BY `__time` ASC LIMIT 10 -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: SELECT `__time` -FROM druid_table_alltypesorc -WHERE (`__time` BETWEEN '1968-01-01 00:00:00' AND '1970-01-01 00:00:00') - OR (`__time` BETWEEN '1968-02-01 00:00:00' AND '1970-04-01 00:00:00') ORDER BY `__time` ASC LIMIT 10 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -1969-12-31 15:59:00.0 US/Pacific -PREHOOK: query: EXPLAIN -SELECT `__time` -FROM druid_table_alltypesorc -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN -SELECT `__time` -FROM druid_table_alltypesorc -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_table_alltypesorc - properties: - druid.fieldNames vc - druid.fieldTypes timestamp with local time zone - druid.query.json {"queryType":"scan","dataSource":"default.druid_table_alltypesorc","intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"],"virtualColumns":[{"type":"expression","name":"vc","expression":"\"__time\"","outputType":"LONG"}],"columns":["vc"],"resultFormat":"compactedList"} - druid.query.type scan - Select Operator - expressions: vc (type: timestamp with local time zone) - outputColumnNames: _col0 - ListSink - -PREHOOK: query: EXPLAIN -SELECT `__time` -FROM druid_table_alltypesorc -WHERE `__time` < '2012-03-01 00:00:00' -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN -SELECT `__time` -FROM druid_table_alltypesorc -WHERE `__time` < '2012-03-01 00:00:00' -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_table_alltypesorc - properties: - druid.fieldNames vc - druid.fieldTypes timestamp with local time zone - druid.query.json {"queryType":"scan","dataSource":"default.druid_table_alltypesorc","intervals":["1900-01-01T00:00:00.000Z/2012-03-01T08:00:00.000Z"],"virtualColumns":[{"type":"expression","name":"vc","expression":"\"__time\"","outputType":"LONG"}],"columns":["vc"],"resultFormat":"compactedList"} - druid.query.type scan - Select Operator - expressions: vc (type: timestamp with local time zone) - outputColumnNames: _col0 - ListSink - -PREHOOK: query: EXPLAIN -SELECT `__time` -FROM druid_table_alltypesorc -WHERE `__time` >= '2010-01-01 00:00:00' AND `__time` <= '2012-03-01 00:00:00' -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN -SELECT `__time` -FROM druid_table_alltypesorc -WHERE `__time` >= '2010-01-01 00:00:00' AND `__time` <= '2012-03-01 00:00:00' -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_table_alltypesorc - properties: - druid.fieldNames vc - druid.fieldTypes timestamp with local time zone - druid.query.json {"queryType":"scan","dataSource":"default.druid_table_alltypesorc","intervals":["2010-01-01T08:00:00.000Z/2012-03-01T08:00:00.001Z"],"virtualColumns":[{"type":"expression","name":"vc","expression":"\"__time\"","outputType":"LONG"}],"columns":["vc"],"resultFormat":"compactedList"} - druid.query.type scan - Select Operator - expressions: vc (type: timestamp with local time zone) - outputColumnNames: _col0 - ListSink - -PREHOOK: query: EXPLAIN -SELECT `__time` -FROM druid_table_alltypesorc -WHERE `__time` >= '2010-01-01 00:00:00' AND `__time` <= '2012-03-01 00:00:00' - AND `__time` < '2011-01-01 00:00:00' -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN -SELECT `__time` -FROM druid_table_alltypesorc -WHERE `__time` >= '2010-01-01 00:00:00' AND `__time` <= '2012-03-01 00:00:00' - AND `__time` < '2011-01-01 00:00:00' -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_table_alltypesorc - properties: - druid.fieldNames vc - druid.fieldTypes timestamp with local time zone - druid.query.json {"queryType":"scan","dataSource":"default.druid_table_alltypesorc","intervals":["2010-01-01T08:00:00.000Z/2011-01-01T08:00:00.000Z"],"virtualColumns":[{"type":"expression","name":"vc","expression":"\"__time\"","outputType":"LONG"}],"columns":["vc"],"resultFormat":"compactedList"} - druid.query.type scan - Select Operator - expressions: vc (type: timestamp with local time zone) - outputColumnNames: _col0 - ListSink - -PREHOOK: query: EXPLAIN -SELECT `__time` -FROM druid_table_alltypesorc -WHERE `__time` BETWEEN '2010-01-01 00:00:00' AND '2011-01-01 00:00:00' -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN -SELECT `__time` -FROM druid_table_alltypesorc -WHERE `__time` BETWEEN '2010-01-01 00:00:00' AND '2011-01-01 00:00:00' -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_table_alltypesorc - properties: - druid.fieldNames vc - druid.fieldTypes timestamp with local time zone - druid.query.json {"queryType":"scan","dataSource":"default.druid_table_alltypesorc","intervals":["2010-01-01T08:00:00.000Z/2011-01-01T08:00:00.001Z"],"virtualColumns":[{"type":"expression","name":"vc","expression":"\"__time\"","outputType":"LONG"}],"columns":["vc"],"resultFormat":"compactedList"} - druid.query.type scan - Select Operator - expressions: vc (type: timestamp with local time zone) - outputColumnNames: _col0 - ListSink - -PREHOOK: query: EXPLAIN -SELECT `__time` -FROM druid_table_alltypesorc -WHERE (`__time` BETWEEN '2010-01-01 00:00:00' AND '2011-01-01 00:00:00') - OR (`__time` BETWEEN '2012-01-01 00:00:00' AND '2013-01-01 00:00:00') -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN -SELECT `__time` -FROM druid_table_alltypesorc -WHERE (`__time` BETWEEN '2010-01-01 00:00:00' AND '2011-01-01 00:00:00') - OR (`__time` BETWEEN '2012-01-01 00:00:00' AND '2013-01-01 00:00:00') -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_table_alltypesorc - properties: - druid.fieldNames vc - druid.fieldTypes timestamp with local time zone - druid.query.json {"queryType":"scan","dataSource":"default.druid_table_alltypesorc","intervals":["2010-01-01T08:00:00.000Z/2011-01-01T08:00:00.001Z","2012-01-01T08:00:00.000Z/2013-01-01T08:00:00.001Z"],"virtualColumns":[{"type":"expression","name":"vc","expression":"\"__time\"","outputType":"LONG"}],"columns":["vc"],"resultFormat":"compactedList"} - druid.query.type scan - Select Operator - expressions: vc (type: timestamp with local time zone) - outputColumnNames: _col0 - ListSink - -PREHOOK: query: EXPLAIN -SELECT `__time` -FROM druid_table_alltypesorc -WHERE (`__time` BETWEEN '2010-01-01 00:00:00' AND '2011-01-01 00:00:00') - OR (`__time` BETWEEN '2010-06-01 00:00:00' AND '2012-01-01 00:00:00') -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN -SELECT `__time` -FROM druid_table_alltypesorc -WHERE (`__time` BETWEEN '2010-01-01 00:00:00' AND '2011-01-01 00:00:00') - OR (`__time` BETWEEN '2010-06-01 00:00:00' AND '2012-01-01 00:00:00') -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_table_alltypesorc - properties: - druid.fieldNames vc - druid.fieldTypes timestamp with local time zone - druid.query.json {"queryType":"scan","dataSource":"default.druid_table_alltypesorc","intervals":["2010-01-01T08:00:00.000Z/2012-01-01T08:00:00.001Z"],"virtualColumns":[{"type":"expression","name":"vc","expression":"\"__time\"","outputType":"LONG"}],"columns":["vc"],"resultFormat":"compactedList"} - druid.query.type scan - Select Operator - expressions: vc (type: timestamp with local time zone) - outputColumnNames: _col0 - ListSink - -PREHOOK: query: EXPLAIN -SELECT `__time` -FROM druid_table_alltypesorc -WHERE `__time` IN ('2010-01-01 00:00:00','2011-01-01 00:00:00') -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN -SELECT `__time` -FROM druid_table_alltypesorc -WHERE `__time` IN ('2010-01-01 00:00:00','2011-01-01 00:00:00') -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_table_alltypesorc - properties: - druid.fieldNames vc - druid.fieldTypes timestamp with local time zone - druid.query.json {"queryType":"scan","dataSource":"default.druid_table_alltypesorc","intervals":["2010-01-01T08:00:00.000Z/2010-01-01T08:00:00.001Z","2011-01-01T08:00:00.000Z/2011-01-01T08:00:00.001Z"],"virtualColumns":[{"type":"expression","name":"vc","expression":"\"__time\"","outputType":"LONG"}],"columns":["vc"],"resultFormat":"compactedList"} - druid.query.type scan - Select Operator - expressions: vc (type: timestamp with local time zone) - outputColumnNames: _col0 - ListSink - -PREHOOK: query: EXPLAIN -SELECT `__time`, cstring2 -FROM druid_table_alltypesorc -WHERE cstring2 = 'user1' AND `__time` IN ('2010-01-01 00:00:00','2011-01-01 00:00:00') -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN -SELECT `__time`, cstring2 -FROM druid_table_alltypesorc -WHERE cstring2 = 'user1' AND `__time` IN ('2010-01-01 00:00:00','2011-01-01 00:00:00') -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_table_alltypesorc - properties: - druid.fieldNames vc,vc0 - druid.fieldTypes timestamp with local time zone,string - druid.query.json {"queryType":"scan","dataSource":"default.druid_table_alltypesorc","intervals":["2010-01-01T08:00:00.000Z/2010-01-01T08:00:00.001Z","2011-01-01T08:00:00.000Z/2011-01-01T08:00:00.001Z"],"filter":{"type":"selector","dimension":"cstring2","value":"user1"},"virtualColumns":[{"type":"expression","name":"vc","expression":"\"__time\"","outputType":"LONG"},{"type":"expression","name":"vc0","expression":"'user1'","outputType":"STRING"}],"columns":["vc","vc0"],"resultFormat":"compactedList"} - druid.query.type scan - Select Operator - expressions: vc (type: timestamp with local time zone), vc0 (type: string) - outputColumnNames: _col0, _col1 - ListSink - -PREHOOK: query: EXPLAIN -SELECT `__time`, cstring2 -FROM druid_table_alltypesorc -WHERE cstring2 = 'user1' OR `__time` IN ('2010-01-01 00:00:00','2011-01-01 00:00:00') -PREHOOK: type: QUERY -PREHOOK: Input: default@druid_table_alltypesorc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: EXPLAIN -SELECT `__time`, cstring2 -FROM druid_table_alltypesorc -WHERE cstring2 = 'user1' OR `__time` IN ('2010-01-01 00:00:00','2011-01-01 00:00:00') -POSTHOOK: type: QUERY -POSTHOOK: Input: default@druid_table_alltypesorc -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: druid_table_alltypesorc - properties: - druid.fieldNames vc,cstring2 - druid.fieldTypes timestamp with local time zone,string - druid.query.json {"queryType":"scan","dataSource":"default.druid_table_alltypesorc","intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"],"filter":{"type":"or","fields":[{"type":"in","dimension":"__time","values":["2010-01-01T08:00:00.000Z","2011-01-01T08:00:00.000Z"],"extractionFn":{"type":"timeFormat","format":"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'","timeZone":"UTC"}},{"type":"selector","dimension":"cstring2","value":"user1"}]},"virtualColumns":[{"type":"expression","name":"vc","expression":"\"__time\"","outputType":"LONG"}],"columns":["vc","cstring2"],"resultFormat":"compactedList"} - druid.query.type scan - Select Operator - expressions: vc (type: timestamp with local time zone), cstring2 (type: string) - outputColumnNames: _col0, _col1 - ListSink -