== Analyzed Logical Plan ==
count: bigint
Aggregate [count(1) AS count#2143214L]
+- TypedFilter <function1>, interface org.apache.spark.sql.Row, [StructField(ip,StringType,true), StructField(userAgentOrig,StringType,true), StructField(session,StringType,true), StructField(userid,StringType,true), StructField(runnerType,StringType,true), StructField(OS,StringType,true), StructField(browser,StringType,true), StructField(DevBrand,StringType,true), StructField(DevMode,StringType,true), StructField(appVersion,StringType,true), StructField(reqCount,LongType,false), StructField(bizErrorCount,LongType,true), StructField(sysErrorCount,LongType,true), StructField(maxRespTime,DoubleType,true), StructField(minRespTime,DoubleType,true), StructField(avgRespTime,DoubleType,true), StructField(in,DoubleType,true), StructField(out,DoubleType,true), StructField(epoch,IntegerType,false), StructField(nginxInTime,StringType,true)], createexternalrow(ip#2143051.toString, userAgentOrig#2143078.toString, session#2143052.toString, userid#2143053.toString, runnerType#2143054.toString, OS#2143055.toString, browser#2143056.toString, DevBrand#2143057.toString, DevMode#2143058.toString, appVersion#2143059.toString, reqCount#2143123L, bizErrorCount#2143124L, sysErrorCount#2143125L, maxRespTime#2143126, minRespTime#2143127, avgRespTime#2143128, in#2143129, out#2143130, epoch#2143067, nginxInTime#2143050.toString, StructField(ip,StringType,true), StructField(userAgentOrig,StringType,true), StructField(session,StringType,true), StructField(userid,StringType,true), ... 16 more fields)
+- Aggregate [ip#2143051, userid#2143053, userAgentOrig#2143078, epoch#2143067, nginxInTime#2143050, runnerType#2143054, os#2143055, browser#2143056, devBrand#2143057, devMode#2143058, appVersion#2143059, session#2143052], [ip#2143051, userAgentOrig#2143078, session#2143052, userid#2143053, runnerType#2143054, OS#2143055, browser#2143056, DevBrand#2143057, DevMode#2143058, appVersion#2143059, count(request#2143060) AS reqCount#2143123L, sum(bizError#2143062L) AS bizErrorCount#2143124L, sum(sysError#2143063L) AS sysErrorCount#2143125L, max(responseTime#2143064) AS maxRespTime#2143126, min(responseTime#2143064) AS minRespTime#2143127, avg(responseTime#2143064) AS avgRespTime#2143128, sum(flowIn#2143065) AS in#2143129, sum(flowOut#2143066) AS out#2143130, epoch#2143067, nginxInTime#2143050]
+- SubqueryAlias ngxlogstatistic
+- TypedFilter <function1>, interface org.apache.spark.sql.Row, [StructField(nginxInTime,StringType,true), StructField(ip,StringType,true), StructField(session,StringType,true), StructField(userid,StringType,true), StructField(runnerType,StringType,true), StructField(OS,StringType,true), StructField(browser,StringType,true), StructField(DevBrand,StringType,true), StructField(DevMode,StringType,true), StructField(appVersion,StringType,true), StructField(request,StringType,true), StructField(isStatic,BooleanType,false), StructField(bizError,LongType,false), StructField(sysError,LongType,false), StructField(responseTime,DoubleType,false), StructField(flowIn,DoubleType,false), StructField(flowOut,DoubleType,false), StructField(epoch,IntegerType,false), StructField(hostname,StringType,true), StructField(port,IntegerType,false), StructField(protocol,StringType,true), StructField(orginalUrl,StringType,true), StructField(method,StringType,true), StructField(path,StringType,true), ... 11 more fields], createexternalrow(nginxInTime#2143050.toString, ip#2143051.toString, session#2143052.toString, userid#2143053.toString, runnerType#2143054.toString, OS#2143055.toString, browser#2143056.toString, DevBrand#2143057.toString, DevMode#2143058.toString, appVersion#2143059.toString, request#2143060.toString, isStatic#2143061, bizError#2143062L, sysError#2143063L, responseTime#2143064, flowIn#2143065, flowOut#2143066, epoch#2143067, hostname#2143068.toString, port#2143069, protocol#2143070.toString, orginalUrl#2143071.toString, method#2143072.toString, path#2143073.toString, ... 46 more fields)
+- Project [nginxInTime#2143014 AS nginxInTime#2143050, ip#2143015 AS ip#2143051, session#2143016 AS session#2143052, userid#2143017 AS userid#2143053, runnerType#2143018 AS runnerType#2143054, OS#2143019 AS OS#2143055, browser#2143020 AS browser#2143056, DevBrand#2143021 AS DevBrand#2143057, DevMode#2143022 AS DevMode#2143058, appVersion#2143023 AS appVersion#2143059, request#2143024 AS request#2143060, isStatic#2143025 AS isStatic#2143061, bizError#2143026L AS bizError#2143062L, sysError#2143027L AS sysError#2143063L, responseTime#2143028 AS responseTime#2143064, flowIn#2143029 AS flowIn#2143065, flowOut#2143030 AS flowOut#2143066, epoch#2143031 AS epoch#2143067, hostname#2143032 AS hostname#2143068, port#2143033 AS port#2143069, protocol#2143034 AS protocol#2143070, orginalUrl#2143035 AS orginalUrl#2143071, method#2143036 AS method#2143072, path#2143037 AS path#2143073, ... 11 more fields]
+- SerializeFromObject [staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, assertnotnull(assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true])).nginxInTime, true) AS nginxInTime#2143014, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, assertnotnull(assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true])).ip, true) AS ip#2143015, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, assertnotnull(assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true])).session, true) AS session#2143016, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, assertnotnull(assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true])).userid, true) AS userid#2143017, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, assertnotnull(assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true])).runnerType, true) AS runnerType#2143018, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, assertnotnull(assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true])).OS, true) AS OS#2143019, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, assertnotnull(assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true])).browser, true) AS browser#2143020, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, assertnotnull(assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true])).DevBrand, true) AS DevBrand#2143021, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, assertnotnull(assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true])).DevMode, true) AS DevMode#2143022, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, assertnotnull(assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true])).appVersion, true) AS appVersion#2143023, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, assertnotnull(assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true])).request, true) AS request#2143024, assertnotnull(assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true])).isStatic AS isStatic#2143025, assertnotnull(assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true])).bizError AS bizError#2143026L, assertnotnull(assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true])).sysError AS sysError#2143027L, assertnotnull(assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true])).responseTime AS responseTime#2143028, assertnotnull(assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true])).flowIn AS flowIn#2143029, assertnotnull(assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true])).flowOut AS flowOut#2143030, assertnotnull(assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true])).epoch AS epoch#2143031, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, assertnotnull(assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true])).hostname, true) AS hostname#2143032, assertnotnull(assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true])).port AS port#2143033, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, assertnotnull(assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true])).protocol, true) AS protocol#2143034, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, assertnotnull(assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true])).orginalUrl, true) AS orginalUrl#2143035, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, assertnotnull(assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true])).method, true) AS method#2143036, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, assertnotnull(assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true])).path, true) AS path#2143037, ... 11 more fields]
+- ExternalRDD [obj#2143013]
== Optimized Logical Plan ==
Aggregate [count(1) AS count#2143214L]
+- Project
+- TypedFilter <function1>, interface org.apache.spark.sql.Row, [StructField(ip,StringType,true), StructField(userAgentOrig,StringType,true), StructField(session,StringType,true), StructField(userid,StringType,true), StructField(runnerType,StringType,true), StructField(OS,StringType,true), StructField(browser,StringType,true), StructField(DevBrand,StringType,true), StructField(DevMode,StringType,true), StructField(appVersion,StringType,true), StructField(reqCount,LongType,false), StructField(bizErrorCount,LongType,true), StructField(sysErrorCount,LongType,true), StructField(maxRespTime,DoubleType,true), StructField(minRespTime,DoubleType,true), StructField(avgRespTime,DoubleType,true), StructField(in,DoubleType,true), StructField(out,DoubleType,true), StructField(epoch,IntegerType,false), StructField(nginxInTime,StringType,true)], createexternalrow(ip#2143015.toString, userAgentOrig#2143078.toString, session#2143016.toString, userid#2143017.toString, runnerType#2143018.toString, OS#2143019.toString, browser#2143020.toString, DevBrand#2143021.toString, DevMode#2143022.toString, appVersion#2143023.toString, reqCount#2143123L, bizErrorCount#2143124L, sysErrorCount#2143125L, maxRespTime#2143126, minRespTime#2143127, avgRespTime#2143128, in#2143129, out#2143130, epoch#2143031, nginxInTime#2143014.toString, StructField(ip,StringType,true), StructField(userAgentOrig,StringType,true), StructField(session,StringType,true), StructField(userid,StringType,true), ... 16 more fields)
+- Aggregate [ip#2143015, userid#2143017, userAgentOrig#2143078, epoch#2143031, nginxInTime#2143014, runnerType#2143018, OS#2143019, browser#2143020, DevBrand#2143021, DevMode#2143022, appVersion#2143023, session#2143016], [ip#2143015, userAgentOrig#2143078, session#2143016, userid#2143017, runnerType#2143018, OS#2143019, browser#2143020, DevBrand#2143021, DevMode#2143022, appVersion#2143023, count(request#2143024) AS reqCount#2143123L, sum(bizError#2143026L) AS bizErrorCount#2143124L, sum(sysError#2143027L) AS sysErrorCount#2143125L, max(responseTime#2143028) AS maxRespTime#2143126, min(responseTime#2143028) AS minRespTime#2143127, avg(responseTime#2143028) AS avgRespTime#2143128, sum(flowIn#2143029) AS in#2143129, sum(flowOut#2143030) AS out#2143130, epoch#2143031, nginxInTime#2143014]
+- Project [nginxInTime#2143014, ip#2143015, session#2143016, userid#2143017, runnerType#2143018, OS#2143019, browser#2143020, DevBrand#2143021, DevMode#2143022, appVersion#2143023, request#2143024, bizError#2143026L, sysError#2143027L, responseTime#2143028, flowIn#2143029, flowOut#2143030, epoch#2143031, userAgentOrig#2143078]
+- TypedFilter <function1>, interface org.apache.spark.sql.Row, [StructField(nginxInTime,StringType,true), StructField(ip,StringType,true), StructField(session,StringType,true), StructField(userid,StringType,true), StructField(runnerType,StringType,true), StructField(OS,StringType,true), StructField(browser,StringType,true), StructField(DevBrand,StringType,true), StructField(DevMode,StringType,true), StructField(appVersion,StringType,true), StructField(request,StringType,true), StructField(isStatic,BooleanType,false), StructField(bizError,LongType,false), StructField(sysError,LongType,false), StructField(responseTime,DoubleType,false), StructField(flowIn,DoubleType,false), StructField(flowOut,DoubleType,false), StructField(epoch,IntegerType,false), StructField(hostname,StringType,true), StructField(port,IntegerType,false), StructField(protocol,StringType,true), StructField(orginalUrl,StringType,true), StructField(method,StringType,true), StructField(path,StringType,true), ... 11 more fields], createexternalrow(nginxInTime#2143014.toString, ip#2143015.toString, session#2143016.toString, userid#2143017.toString, runnerType#2143018.toString, OS#2143019.toString, browser#2143020.toString, DevBrand#2143021.toString, DevMode#2143022.toString, appVersion#2143023.toString, request#2143024.toString, isStatic#2143025, bizError#2143026L, sysError#2143027L, responseTime#2143028, flowIn#2143029, flowOut#2143030, epoch#2143031, hostname#2143032.toString, port#2143033, protocol#2143034.toString, orginalUrl#2143035.toString, method#2143036.toString, path#2143037.toString, ... 46 more fields)
+- Project [nginxInTime#2143014, ip#2143015, session#2143016, userid#2143017, runnerType#2143018, OS#2143019, browser#2143020, DevBrand#2143021, DevMode#2143022, appVersion#2143023, request#2143024, isStatic#2143025, bizError#2143026L, sysError#2143027L, responseTime#2143028, flowIn#2143029, flowOut#2143030, epoch#2143031, hostname#2143032, port#2143033, protocol#2143034, orginalUrl#2143035, method#2143036, path#2143037, ... 11 more fields]
+- SerializeFromObject [staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true]).nginxInTime, true) AS nginxInTime#2143014, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true]).ip, true) AS ip#2143015, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true]).session, true) AS session#2143016, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true]).userid, true) AS userid#2143017, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true]).runnerType, true) AS runnerType#2143018, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true]).OS, true) AS OS#2143019, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true]).browser, true) AS browser#2143020, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true]).DevBrand, true) AS DevBrand#2143021, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true]).DevMode, true) AS DevMode#2143022, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true]).appVersion, true) AS appVersion#2143023, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true]).request, true) AS request#2143024, assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true]).isStatic AS isStatic#2143025, assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true]).bizError AS bizError#2143026L, assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true]).sysError AS sysError#2143027L, assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true]).responseTime AS responseTime#2143028, assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true]).flowIn AS flowIn#2143029, assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true]).flowOut AS flowOut#2143030, assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true]).epoch AS epoch#2143031, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true]).hostname, true) AS hostname#2143032, assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true]).port AS port#2143033, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true]).protocol, true) AS protocol#2143034, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true]).orginalUrl, true) AS orginalUrl#2143035, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true]).method, true) AS method#2143036, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true]).path, true) AS path#2143037, ... 11 more fields]
+- ExternalRDD [obj#2143013]
== Physical Plan ==
*HashAggregate(keys=[], functions=[count(1)], output=[count#2143214L])
+- Exchange SinglePartition
+- *HashAggregate(keys=[], functions=[partial_count(1)], output=[count#2143218L])
+- *Project
+- *Filter <function1>.apply
+- *HashAggregate(keys=[ip#2143015, userid#2143017, userAgentOrig#2143078, epoch#2143031, nginxInTime#2143014, runnerType#2143018, OS#2143019, browser#2143020, DevBrand#2143021, DevMode#2143022, appVersion#2143023, session#2143016], functions=[count(request#2143024), sum(bizError#2143026L), sum(sysError#2143027L), max(responseTime#2143028), min(responseTime#2143028), avg(responseTime#2143028), sum(flowIn#2143029), sum(flowOut#2143030)], output=[ip#2143015, userAgentOrig#2143078, session#2143016, userid#2143017, runnerType#2143018, OS#2143019, browser#2143020, DevBrand#2143021, DevMode#2143022, appVersion#2143023, reqCount#2143123L, bizErrorCount#2143124L, sysErrorCount#2143125L, maxRespTime#2143126, minRespTime#2143127, avgRespTime#2143128, in#2143129, out#2143130, epoch#2143031, nginxInTime#2143014])
+- Exchange hashpartitioning(ip#2143015, userid#2143017, userAgentOrig#2143078, epoch#2143031, nginxInTime#2143014, runnerType#2143018, OS#2143019, browser#2143020, DevBrand#2143021, DevMode#2143022, appVersion#2143023, session#2143016, 200)
+- *HashAggregate(keys=[ip#2143015, userid#2143017, userAgentOrig#2143078, epoch#2143031, nginxInTime#2143014, runnerType#2143018, OS#2143019, browser#2143020, DevBrand#2143021, DevMode#2143022, appVersion#2143023, session#2143016], functions=[partial_count(request#2143024), partial_sum(bizError#2143026L), partial_sum(sysError#2143027L), partial_max(responseTime#2143028), partial_min(responseTime#2143028), partial_avg(responseTime#2143028), partial_sum(flowIn#2143029), partial_sum(flowOut#2143030)], output=[ip#2143015, userid#2143017, userAgentOrig#2143078, epoch#2143031, nginxInTime#2143014, runnerType#2143018, OS#2143019, browser#2143020, DevBrand#2143021, DevMode#2143022, appVersion#2143023, session#2143016, count#2143228L, sum#2143229L, sum#2143230L, max#2143231, min#2143232, sum#2143233, count#2143234L, sum#2143235, sum#2143236])
+- *Project [nginxInTime#2143014, ip#2143015, session#2143016, userid#2143017, runnerType#2143018, OS#2143019, browser#2143020, DevBrand#2143021, DevMode#2143022, appVersion#2143023, request#2143024, bizError#2143026L, sysError#2143027L, responseTime#2143028, flowIn#2143029, flowOut#2143030, epoch#2143031, userAgentOrig#2143078]
+- *Filter <function1>.apply
+- *Project [nginxInTime#2143014, ip#2143015, session#2143016, userid#2143017, runnerType#2143018, OS#2143019, browser#2143020, DevBrand#2143021, DevMode#2143022, appVersion#2143023, request#2143024, isStatic#2143025, bizError#2143026L, sysError#2143027L, responseTime#2143028, flowIn#2143029, flowOut#2143030, epoch#2143031, hostname#2143032, port#2143033, protocol#2143034, orginalUrl#2143035, method#2143036, path#2143037, ... 11 more fields]
+- *SerializeFromObject [staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true]).nginxInTime, true) AS nginxInTime#2143014, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true]).ip, true) AS ip#2143015, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true]).session, true) AS session#2143016, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true]).userid, true) AS userid#2143017, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true]).runnerType, true) AS runnerType#2143018, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true]).OS, true) AS OS#2143019, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true]).browser, true) AS browser#2143020, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true]).DevBrand, true) AS DevBrand#2143021, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true]).DevMode, true) AS DevMode#2143022, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true]).appVersion, true) AS appVersion#2143023, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true]).request, true) AS request#2143024, assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true]).isStatic AS isStatic#2143025, assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true]).bizError AS bizError#2143026L, assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true]).sysError AS sysError#2143027L, assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true]).responseTime AS responseTime#2143028, assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true]).flowIn AS flowIn#2143029, assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true]).flowOut AS flowOut#2143030, assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true]).epoch AS epoch#2143031, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true]).hostname, true) AS hostname#2143032, assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true]).port AS port#2143033, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true]).protocol, true) AS protocol#2143034, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true]).orginalUrl, true) AS orginalUrl#2143035, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true]).method, true) AS method#2143036, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true]).path, true) AS path#2143037, ... 11 more fields]
+- Scan ExternalRDDScan[obj#2143013],org.apache.spark.sql.execution.SparkPlanInfo@2b2b349c,1532311567320)
18/07/23 10:06:07 ERROR scheduler.LiveListenerBus: SparkListenerBus has already stopped! Dropping event SparkListenerSQLExecutionEnd(33263,1532311567415)
18/07/23 10:06:09 ERROR scheduler.LiveListenerBus: SparkListenerBus has already stopped! Dropping event WrappedStreamingListenerEvent(StreamingListenerOutputOperationCompleted(OutputOperationInfo(1532311550000 ms,0,foreachRDD at NginxLogStatistic.scala:169,org.apache.spark.streaming.dstream.DStream.foreachRDD(DStream.scala:635)
com.sznongfu.cn.online.statistic.NginxLogStatistic$.process(NginxLogStatistic.scala:169)
com.sznongfu.cn.center.StartCenter$.main(StartCenter.scala:51)
com.sznongfu.cn.center.StartCenter.main(StartCenter.scala)
sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
java.lang.reflect.Method.invoke(Method.java:498)
org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:755)
org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:180)
org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:205)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:119)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala),Some(1532311567050),Some(1532311567415),Some(org.apache.spark.sql.catalyst.errors.package$TreeNodeException: execute, tree:
Exchange SinglePartition
+- *HashAggregate(keys=[], functions=[partial_count(1)], output=[count#2143218L])
+- *Project
+- *Filter <function1>.apply
+- *HashAggregate(keys=[ip#2143015, userid#2143017, userAgentOrig#2143078, epoch#2143031, nginxInTime#2143014, runnerType#2143018, OS#2143019, browser#2143020, DevBrand#2143021, DevMode#2143022, appVersion#2143023, session#2143016], functions=[count(request#2143024), sum(bizError#2143026L), sum(sysError#2143027L), max(responseTime#2143028), min(responseTime#2143028), avg(responseTime#2143028), sum(flowIn#2143029), sum(flowOut#2143030)], output=[ip#2143015, userAgentOrig#2143078, session#2143016, userid#2143017, runnerType#2143018, OS#2143019, browser#2143020, DevBrand#2143021, DevMode#2143022, appVersion#2143023, reqCount#2143123L, bizErrorCount#2143124L, sysErrorCount#2143125L, maxRespTime#2143126, minRespTime#2143127, avgRespTime#2143128, in#2143129, out#2143130, epoch#2143031, nginxInTime#2143014])
+- Exchange hashpartitioning(ip#2143015, userid#2143017, userAgentOrig#2143078, epoch#2143031, nginxInTime#2143014, runnerType#2143018, OS#2143019, browser#2143020, DevBrand#2143021, DevMode#2143022, appVersion#2143023, session#2143016, 200)
+- *HashAggregate(keys=[ip#2143015, userid#2143017, userAgentOrig#2143078, epoch#2143031, nginxInTime#2143014, runnerType#2143018, OS#2143019, browser#2143020, DevBrand#2143021, DevMode#2143022, appVersion#2143023, session#2143016], functions=[partial_count(request#2143024), partial_sum(bizError#2143026L), partial_sum(sysError#2143027L), partial_max(responseTime#2143028), partial_min(responseTime#2143028), partial_avg(responseTime#2143028), partial_sum(flowIn#2143029), partial_sum(flowOut#2143030)], output=[ip#2143015, userid#2143017, userAgentOrig#2143078, epoch#2143031, nginxInTime#2143014, runnerType#2143018, OS#2143019, browser#2143020, DevBrand#2143021, DevMode#2143022, appVersion#2143023, session#2143016, count#2143228L, sum#2143229L, sum#2143230L, max#2143231, min#2143232, sum#2143233, count#2143234L, sum#2143235, sum#2143236])
+- *Project [nginxInTime#2143014, ip#2143015, session#2143016, userid#2143017, runnerType#2143018, OS#2143019, browser#2143020, DevBrand#2143021, DevMode#2143022, appVersion#2143023, request#2143024, bizError#2143026L, sysError#2143027L, responseTime#2143028, flowIn#2143029, flowOut#2143030, epoch#2143031, userAgentOrig#2143078]
+- *Filter <function1>.apply
+- *Project [nginxInTime#2143014, ip#2143015, session#2143016, userid#2143017, runnerType#2143018, OS#2143019, browser#2143020, DevBrand#2143021, DevMode#2143022, appVersion#2143023, request#2143024, isStatic#2143025, bizError#2143026L, sysError#2143027L, responseTime#2143028, flowIn#2143029, flowOut#2143030, epoch#2143031, hostname#2143032, port#2143033, protocol#2143034, orginalUrl#2143035, method#2143036, path#2143037, ... 11 more fields]
+- *SerializeFromObject [staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true]).nginxInTime, true) AS nginxInTime#2143014, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true]).ip, true) AS ip#2143015, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true]).session, true) AS session#2143016, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true]).userid, true) AS userid#2143017, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true]).runnerType, true) AS runnerType#2143018, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true]).OS, true) AS OS#2143019, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true]).browser, true) AS browser#2143020, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true]).DevBrand, true) AS DevBrand#2143021, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true]).DevMode, true) AS DevMode#2143022, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true]).appVersion, true) AS appVersion#2143023, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true]).request, true) AS request#2143024, assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true]).isStatic AS isStatic#2143025, assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true]).bizError AS bizError#2143026L, assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true]).sysError AS sysError#2143027L, assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true]).responseTime AS responseTime#2143028, assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true]).flowIn AS flowIn#2143029, assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true]).flowOut AS flowOut#2143030, assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true]).epoch AS epoch#2143031, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true]).hostname, true) AS hostname#2143032, assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true]).port AS port#2143033, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true]).protocol, true) AS protocol#2143034, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true]).orginalUrl, true) AS orginalUrl#2143035, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true]).method, true) AS method#2143036, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true]).path, true) AS path#2143037, ... 11 more fields]
+- Scan ExternalRDDScan[obj#2143013]
at org.apache.spark.sql.catalyst.errors.package$.attachTree(package.scala:56)
at org.apache.spark.sql.execution.exchange.ShuffleExchange.doExecute(ShuffleExchange.scala:115)
at org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$1.apply(SparkPlan.scala:117)
at org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$1.apply(SparkPlan.scala:117)
at org.apache.spark.sql.execution.SparkPlan$$anonfun$executeQuery$1.apply(SparkPlan.scala:138)
at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:135)
at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:116)
at org.apache.spark.sql.execution.InputAdapter.inputRDDs(WholeStageCodegenExec.scala:252)
at org.apache.spark.sql.execution.aggregate.HashAggregateExec.inputRDDs(HashAggregateExec.scala:141)
at org.apache.spark.sql.execution.WholeStageCodegenExec.doExecute(WholeStageCodegenExec.scala:386)
at org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$1.apply(SparkPlan.scala:117)
at org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$1.apply(SparkPlan.scala:117)
at org.apache.spark.sql.execution.SparkPlan$$anonfun$executeQuery$1.apply(SparkPlan.scala:138)
at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:135)
at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:116)
at org.apache.spark.sql.execution.SparkPlan.getByteArrayRdd(SparkPlan.scala:228)
at org.apache.spark.sql.execution.SparkPlan.executeCollect(SparkPlan.scala:275)
at org.apache.spark.sql.Dataset$$anonfun$count$1.apply(Dataset.scala:2430)
at org.apache.spark.sql.Dataset$$anonfun$count$1.apply(Dataset.scala:2429)
at org.apache.spark.sql.Dataset$$anonfun$55.apply(Dataset.scala:2837)
at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:65)
at org.apache.spark.sql.Dataset.withAction(Dataset.scala:2836)
at org.apache.spark.sql.Dataset.count(Dataset.scala:2429)
at com.sznongfu.cn.online.statistic.NginxLogStatistic$$anonfun$process$1.apply(NginxLogStatistic.scala:238)
at com.sznongfu.cn.online.statistic.NginxLogStatistic$$anonfun$process$1.apply(NginxLogStatistic.scala:169)
at org.apache.spark.streaming.dstream.ForEachDStream$$anonfun$1$$anonfun$apply$mcV$sp$1.apply$mcV$sp(ForEachDStream.scala:51)
at org.apache.spark.streaming.dstream.ForEachDStream$$anonfun$1$$anonfun$apply$mcV$sp$1.apply(ForEachDStream.scala:51)
at org.apache.spark.streaming.dstream.ForEachDStream$$anonfun$1$$anonfun$apply$mcV$sp$1.apply(ForEachDStream.scala:51)
at org.apache.spark.streaming.dstream.DStream.createRDDWithLocalProperties(DStream.scala:416)
at org.apache.spark.streaming.dstream.ForEachDStream$$anonfun$1.apply$mcV$sp(ForEachDStream.scala:50)
at org.apache.spark.streaming.dstream.ForEachDStream$$anonfun$1.apply(ForEachDStream.scala:50)
at org.apache.spark.streaming.dstream.ForEachDStream$$anonfun$1.apply(ForEachDStream.scala:50)
at scala.util.Try$.apply(Try.scala:192)
at org.apache.spark.streaming.scheduler.Job.run(Job.scala:39)
at org.apache.spark.streaming.scheduler.JobScheduler$JobHandler$$anonfun$run$1.apply$mcV$sp(JobScheduler.scala:257)
at org.apache.spark.streaming.scheduler.JobScheduler$JobHandler$$anonfun$run$1.apply(JobScheduler.scala:257)
at org.apache.spark.streaming.scheduler.JobScheduler$JobHandler$$anonfun$run$1.apply(JobScheduler.scala:257)
at scala.util.DynamicVariable.withValue(DynamicVariable.scala:58)
at org.apache.spark.streaming.scheduler.JobScheduler$JobHandler.run(JobScheduler.scala:256)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.spark.sql.catalyst.errors.package$TreeNodeException: execute, tree:
Exchange hashpartitioning(ip#2143015, userid#2143017, userAgentOrig#2143078, epoch#2143031, nginxInTime#2143014, runnerType#2143018, OS#2143019, browser#2143020, DevBrand#2143021, DevMode#2143022, appVersion#2143023, session#2143016, 200)
+- *HashAggregate(keys=[ip#2143015, userid#2143017, userAgentOrig#2143078, epoch#2143031, nginxInTime#2143014, runnerType#2143018, OS#2143019, browser#2143020, DevBrand#2143021, DevMode#2143022, appVersion#2143023, session#2143016], functions=[partial_count(request#2143024), partial_sum(bizError#2143026L), partial_sum(sysError#2143027L), partial_max(responseTime#2143028), partial_min(responseTime#2143028), partial_avg(responseTime#2143028), partial_sum(flowIn#2143029), partial_sum(flowOut#2143030)], output=[ip#2143015, userid#2143017, userAgentOrig#2143078, epoch#2143031, nginxInTime#2143014, runnerType#2143018, OS#2143019, browser#2143020, DevBrand#2143021, DevMode#2143022, appVersion#2143023, session#2143016, count#2143228L, sum#2143229L, sum#2143230L, max#2143231, min#2143232, sum#2143233, count#2143234L, sum#2143235, sum#2143236])
+- *Project [nginxInTime#2143014, ip#2143015, session#2143016, userid#2143017, runnerType#2143018, OS#2143019, browser#2143020, DevBrand#2143021, DevMode#2143022, appVersion#2143023, request#2143024, bizError#2143026L, sysError#2143027L, responseTime#2143028, flowIn#2143029, flowOut#2143030, epoch#2143031, userAgentOrig#2143078]
+- *Filter <function1>.apply
+- *Project [nginxInTime#2143014, ip#2143015, session#2143016, userid#2143017, runnerType#2143018, OS#2143019, browser#2143020, DevBrand#2143021, DevMode#2143022, appVersion#2143023, request#2143024, isStatic#2143025, bizError#2143026L, sysError#2143027L, responseTime#2143028, flowIn#2143029, flowOut#2143030, epoch#2143031, hostname#2143032, port#2143033, protocol#2143034, orginalUrl#2143035, method#2143036, path#2143037, ... 11 more fields]
+- *SerializeFromObject [staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true]).nginxInTime, true) AS nginxInTime#2143014, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true]).ip, true) AS ip#2143015, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true]).session, true) AS session#2143016, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true]).userid, true) AS userid#2143017, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true]).runnerType, true) AS runnerType#2143018, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true]).OS, true) AS OS#2143019, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true]).browser, true) AS browser#2143020, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true]).DevBrand, true) AS DevBrand#2143021, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true]).DevMode, true) AS DevMode#2143022, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true]).appVersion, true) AS appVersion#2143023, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true]).request, true) AS request#2143024, assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true]).isStatic AS isStatic#2143025, assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true]).bizError AS bizError#2143026L, assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true]).sysError AS sysError#2143027L, assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true]).responseTime AS responseTime#2143028, assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true]).flowIn AS flowIn#2143029, assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true]).flowOut AS flowOut#2143030, assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true]).epoch AS epoch#2143031, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true]).hostname, true) AS hostname#2143032, assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true]).port AS port#2143033, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true]).protocol, true) AS protocol#2143034, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true]).orginalUrl, true) AS orginalUrl#2143035, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true]).method, true) AS method#2143036, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, assertnotnull(input[0, com.sznongfu.cn.online.model.NginxLog, true]).path, true) AS path#2143037, ... 11 more fields]
+- Scan ExternalRDDScan[obj#2143013]
at org.apache.spark.sql.catalyst.errors.package$.attachTree(package.scala:56)
at org.apache.spark.sql.execution.exchange.ShuffleExchange.doExecute(ShuffleExchange.scala:115)
at org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$1.apply(SparkPlan.scala:117)
at org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$1.apply(SparkPlan.scala:117)
at org.apache.spark.sql.execution.SparkPlan$$anonfun$executeQuery$1.apply(SparkPlan.scala:138)
at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:135)
at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:116)
at org.apache.spark.sql.execution.InputAdapter.inputRDDs(WholeStageCodegenExec.scala:252)
at org.apache.spark.sql.execution.aggregate.HashAggregateExec.inputRDDs(HashAggregateExec.scala:141)
at org.apache.spark.sql.execution.FilterExec.inputRDDs(basicPhysicalOperators.scala:124)
at org.apache.spark.sql.execution.ProjectExec.inputRDDs(basicPhysicalOperators.scala:42)
at org.apache.spark.sql.execution.aggregate.HashAggregateExec.inputRDDs(HashAggregateExec.scala:141)
at org.apache.spark.sql.execution.WholeStageCodegenExec.doExecute(WholeStageCodegenExec.scala:386)
at org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$1.apply(SparkPlan.scala:117)
at org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$1.apply(SparkPlan.scala:117)
at org.apache.spark.sql.execution.SparkPlan$$anonfun$executeQuery$1.apply(SparkPlan.scala:138)
at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:135)
at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:116)
at org.apache.spark.sql.execution.exchange.ShuffleExchange.prepareShuffleDependency(ShuffleExchange.scala:88)
at org.apache.spark.sql.execution.exchange.ShuffleExchange$$anonfun$doExecute$1.apply(ShuffleExchange.scala:124)
at org.apache.spark.sql.execution.exchange.ShuffleExchange$$anonfun$doExecute$1.apply(ShuffleExchange.scala:115)
at org.apache.spark.sql.catalyst.errors.package$.attachTree(package.scala:52)
... 43 more
Caused by: java.lang.NullPointerException
at org.apache.spark.sql.execution.exchange.ShuffleExchange$.needToCopyObjectsBeforeShuffle(ShuffleExchange.scala:162)
at org.apache.spark.sql.execution.exchange.ShuffleExchange$.prepareShuffleDependency(ShuffleExchange.scala:246)
at org.apache.spark.sql.execution.exchange.ShuffleExchange.prepareShuffleDependency(ShuffleExchange.scala:87)
at org.apache.spark.sql.execution.exchange.ShuffleExchange$$anonfun$doExecute$1.apply(ShuffleExchange.scala:124)
at org.apache.spark.sql.execution.exchange.ShuffleExchange$$anonfun$doExecute$1.apply(ShuffleExchange.scala:115)
at org.apache.spark.sql.catalyst.errors.package$.attachTree(package.scala:52)
... 66 more
))))
18/07/23 10:06:09 INFO scheduler.JobScheduler: Finished job streaming job 1532311550000 ms.0 from job set of time 15323115500
原因:
内存提前回收,导致mongo 未正常入库,约定停止应用(stop sparkSession),报nullpointException
解决:
入库之后,进行内存回收,list.clean()
错误二:
18/07/23 11:46:56 INFO scheduler.TaskSetManager: Starting task 0.0 in stage 10.0 (TID 633, 10.26.51.89, executor 0, partition 0, NODE_LOCAL, 4730 bytes)
18/07/23 11:46:57 INFO storage.BlockManagerInfo: Added broadcast_9_piece0 in memory on 10.26.51.89:44547 (size: 3.7 KB, free: 366.2 MB)
18/07/23 11:46:57 INFO spark.MapOutputTrackerMasterEndpoint: Asked to send map output locations for shuffle 6 to 10.26.51.89:59416
18/07/23 11:46:57 INFO spark.MapOutputTrackerMaster: Size of output statuses for shuffle 6 is 198 bytes
Exception in thread "streaming-job-executor-0" java.lang.Error: java.lang.InterruptedException
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1148)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:748)
Caused by: java.lang.InterruptedException
at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:998)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304)
at scala.concurrent.impl.Promise$DefaultPromise.tryAwait(Promise.scala:202)
at scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:218)
at scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:153)
at org.apache.spark.util.ThreadUtils$.awaitReady(ThreadUtils.scala:222)
原因:
executor-cores 不足,导致并发度不高
解决:
增加executor-cores数量
