Hive commonly used sql optimization parameters

hive optimization parameters
--HIVE_CONFIG=
set hive.exec.parallel=true;
set mapred.max.split.size=50000000; -- Determine the maximum file size processed by each map, the unit is B, that is, how much to adjust the mapper First determine the size of a mapper processing, the default is 256m or 256000000
set mapred.min.split.size.per.node=50000000; -- The smallest file size that can be processed in a node
set mapred.min.split.size.per. rack=50000000; -- The smallest file size that can be processed in the rack
set hive.exec.reducers.bytes.per.reducer=50000000; -- The amount of data processed by each reduce, default 1GB 1073741824 
set hive.hadoop.supports .splittable.combineinputformat=true; --Whether to combine small input files to reduce the number of mappers
set hive.vectorized.execution.enabled = true; --Vector query, 1024 rows of data will be formed each time the data is processed One batch is processed instead of row by row, which can significantly improve the execution speed
. set hive.vectorized.execution.reduce.enabled = true; -- vector query, each time the data is processed, 1024 rows of data will be formed into a batch for processing processing, rather than line-by-line, which can significantly speed up execution
set mapreduce.map.memory.mb=5120; --Adjust the use of memory
set mapreduce.reduce.memory.mb=5120; --Adjust the use of memory
 

Guess you like

Origin http://43.154.161.224:23101/article/api/json?id=324982368&siteId=291194637