磁盘空间不足导致HiveSession创建失败

问题描述

Hive的服务器（Linux）的磁盘空间不足，会导致与hive建立session失败。最终导致使用到hive服务的Spark任务和Hive任务失败。

堆栈关键字

org.apache.hive.service.cli.HiveSQLException: Failed to open new session: java.lang.RuntimeException: java.lang.RuntimeException: java.lang.RuntimeException: java.io.IOException: No space left on device。

具体错误报文：

[2024-06-28 00:04:28,193][ERROR][From MonitorThreadFactory's Monitor-Worker-3][com.suning.base.monitor.Spark2Monitor:47] Spark2Monitor Exception.
org.apache.hive.service.cli.HiveSQLException: Failed to open new session: java.lang.RuntimeException: java.lang.RuntimeException: java.lang.RuntimeException: java.io.IOException: No space left on deviceat org.apache.hive.jdbc.Utils.verifySuccess(Utils.java:256) ~[hive-jdbc-1.2.1.jar:1.2.1]at org.apache.hive.jdbc.Utils.verifySuccess(Utils.java:247) ~[hive-jdbc-1.2.1.jar:1.2.1]at org.apache.hive.jdbc.HiveConnection.openSession(HiveConnection.java:586) ~[hive-jdbc-1.2.1.jar:1.2.1]at org.apache.hive.jdbc.HiveConnection.<init>(HiveConnection.java:192) ~[hive-jdbc-1.2.1.jar:1.2.1]at org.apache.hive.jdbc.HiveDriver.connect(HiveDriver.java:105) ~[hive-jdbc-1.2.1.jar:1.2.1]at java.sql.DriverManager.getConnection(DriverManager.java:664) ~[?:1.8.0_131]at java.sql.DriverManager.getConnection(DriverManager.java:247) ~[?:1.8.0_131]at com.suning.base.components.hive.HiveClientHelper.getConnection(HiveClientHelper.java:40) ~[base-monitoring-service-1.1.8.jar:1.1.8]at com.suning.base.components.hive.HiveClientHelper.getConnection(HiveClientHelper.java:36) ~[base-monitoring-service-1.1.8.jar:1.1.8]at com.suning.base.components.hive.HiveClientHelper.getConnection(HiveClientHelper.java:32) ~[base-monitoring-service-1.1.8.jar:1.1.8]at com.suning.base.monitor.Spark2Monitor.monitor(Spark2Monitor.java:38) ~[base-monitoring-service-1.1.8.jar:1.1.8]at com.suning.base.BaseMonitoringService$1.lambda$run$0(BaseMonitoringService.java:264) ~[base-monitoring-service-1.1.8.jar:1.1.8]at java.util.concurrent.FutureTask.run(FutureTask.java:266) [?:1.8.0_131]at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) [?:1.8.0_131]at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) [?:1.8.0_131]at java.lang.Thread.run(Thread.java:748) [?:1.8.0_131]
Caused by: org.apache.hive.service.cli.HiveSQLException: Failed to open new session: java.lang.RuntimeException: java.lang.RuntimeException: java.lang.RuntimeException: java.io.IOException: No space left on deviceat org.apache.hive.service.cli.session.SessionManager.openSession(SessionManager.java:266) ~[hive-service-1.2.1.jar:1.2.1]at org.apache.hive.service.cli.CLIService.openSessionWithImpersonation(CLIService.java:204) ~[hive-service-1.2.1.jar:1.2.1]at org.apache.hive.service.cli.thrift.ThriftCLIService.getSessionHandle(ThriftCLIService.java:404) ~[hive-service-1.2.1.jar:1.2.1]at org.apache.hive.service.cli.thrift.ThriftCLIService.OpenSession(ThriftCLIService.java:299) ~[hive-service-1.2.1.jar:1.2.1]at org.apache.hive.service.cli.thrift.TCLIService$Processor$OpenSession.getResult(TCLIService.java:1253) ~[hive-service-1.2.1.jar:1.2.1]at org.apache.hive.service.cli.thrift.TCLIService$Processor$OpenSession.getResult(TCLIService.java:1238) ~[hive-service-1.2.1.jar:1.2.1]at org.apache.thrift.ProcessFunction.process(ProcessFunction.java:39) ~[libthrift-0.9.2.jar:0.9.2]at org.apache.thrift.TBaseProcessor.process(TBaseProcessor.java:39) ~[libthrift-0.9.2.jar:0.9.2]at org.apache.hive.service.auth.TSetIpAddressProcessor.process(TSetIpAddressProcessor.java:56) ~[hive-service-1.2.1.jar:1.2.1]at org.apache.thrift.server.TThreadPoolServer$WorkerProcess.run(TThreadPoolServer.java:285) ~[libthrift-0.9.2.jar:0.9.2]at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_131]at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_131]... 1 more
Caused by: java.lang.RuntimeException: java.lang.RuntimeException: java.lang.RuntimeException: java.io.IOException: No space left on deviceat org.apache.hive.service.cli.session.HiveSessionProxy.invoke(HiveSessionProxy.java:83) ~[hive-service-1.2.1.jar:1.2.1]at org.apache.hive.service.cli.session.HiveSessionProxy.access$000(HiveSessionProxy.java:36) ~[hive-service-1.2.1.jar:1.2.1]at org.apache.hive.service.cli.session.HiveSessionProxy$1.run(HiveSessionProxy.java:63) ~[hive-service-1.2.1.jar:1.2.1]at java.security.AccessController.doPrivileged(Native Method) ~[?:1.8.0_131]at javax.security.auth.Subject.doAs(Subject.java:422) ~[?:1.8.0_131]at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1836) ~[hadoop-common-2.7.3.jar:?]at org.apache.hive.service.cli.session.HiveSessionProxy.invoke(HiveSessionProxy.java:59) ~[hive-service-1.2.1.jar:1.2.1]at com.sun.proxy.$Proxy20.open(Unknown Source) ~[?:?]at org.apache.hive.service.cli.session.SessionManager.openSession(SessionManager.java:258) ~[hive-service-1.2.1.jar:1.2.1]at org.apache.hive.service.cli.CLIService.openSessionWithImpersonation(CLIService.java:204) ~[hive-service-1.2.1.jar:1.2.1]at org.apache.hive.service.cli.thrift.ThriftCLIService.getSessionHandle(ThriftCLIService.java:404) ~[hive-service-1.2.1.jar:1.2.1]at org.apache.hive.service.cli.thrift.ThriftCLIService.OpenSession(ThriftCLIService.java:299) ~[hive-service-1.2.1.jar:1.2.1]at org.apache.hive.service.cli.thrift.TCLIService$Processor$OpenSession.getResult(TCLIService.java:1253) ~[hive-service-1.2.1.jar:1.2.1]at org.apache.hive.service.cli.thrift.TCLIService$Processor$OpenSession.getResult(TCLIService.java:1238) ~[hive-service-1.2.1.jar:1.2.1]at org.apache.thrift.ProcessFunction.process(ProcessFunction.java:39) ~[libthrift-0.9.2.jar:0.9.2]at org.apache.thrift.TBaseProcessor.process(TBaseProcessor.java:39) ~[libthrift-0.9.2.jar:0.9.2]at org.apache.hive.service.auth.TSetIpAddressProcessor.process(TSetIpAddressProcessor.java:56) ~[hive-service-1.2.1.jar:1.2.1]at org.apache.thrift.server.TThreadPoolServer$WorkerProcess.run(TThreadPoolServer.java:285) ~[libthrift-0.9.2.jar:0.9.2]at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_131]at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_131]... 1 more
Caused by: java.lang.RuntimeException: java.lang.RuntimeException: java.io.IOException: No space left on deviceat org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:502) ~[?:?]at org.apache.hive.service.cli.session.HiveSessionImpl.open(HiveSessionImpl.java:147) ~[hive-service-1.2.1.jar:1.2.1]at sun.reflect.GeneratedMethodAccessor42.invoke(Unknown Source) ~[?:?]at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_131]at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_131]at org.apache.hive.service.cli.session.HiveSessionProxy.invoke(HiveSessionProxy.java:78) ~[hive-service-1.2.1.jar:1.2.1]at org.apache.hive.service.cli.session.HiveSessionProxy.access$000(HiveSessionProxy.java:36) ~[hive-service-1.2.1.jar:1.2.1]at org.apache.hive.service.cli.session.HiveSessionProxy$1.run(HiveSessionProxy.java:63) ~[hive-service-1.2.1.jar:1.2.1]at java.security.AccessController.doPrivileged(Native Method) ~[?:1.8.0_131]at javax.security.auth.Subject.doAs(Subject.java:422) ~[?:1.8.0_131]at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1836) ~[hadoop-common-2.7.3.jar:?]at org.apache.hive.service.cli.session.HiveSessionProxy.invoke(HiveSessionProxy.java:59) ~[hive-service-1.2.1.jar:1.2.1]at com.sun.proxy.$Proxy20.open(Unknown Source) ~[?:?]at org.apache.hive.service.cli.session.SessionManager.openSession(SessionManager.java:258) ~[hive-service-1.2.1.jar:1.2.1]at org.apache.hive.service.cli.CLIService.openSessionWithImpersonation(CLIService.java:204) ~[hive-service-1.2.1.jar:1.2.1]at org.apache.hive.service.cli.thrift.ThriftCLIService.getSessionHandle(ThriftCLIService.java:404) ~[hive-service-1.2.1.jar:1.2.1]at org.apache.hive.service.cli.thrift.ThriftCLIService.OpenSession(ThriftCLIService.java:299) ~[hive-service-1.2.1.jar:1.2.1]at org.apache.hive.service.cli.thrift.TCLIService$Processor$OpenSession.getResult(TCLIService.java:1253) ~[hive-service-1.2.1.jar:1.2.1]at org.apache.hive.service.cli.thrift.TCLIService$Processor$OpenSession.getResult(TCLIService.java:1238) ~[hive-service-1.2.1.jar:1.2.1]at org.apache.thrift.ProcessFunction.process(ProcessFunction.java:39) ~[libthrift-0.9.2.jar:0.9.2]at org.apache.thrift.TBaseProcessor.process(TBaseProcessor.java:39) ~[libthrift-0.9.2.jar:0.9.2]at org.apache.hive.service.auth.TSetIpAddressProcessor.process(TSetIpAddressProcessor.java:56) ~[hive-service-1.2.1.jar:1.2.1]at org.apache.thrift.server.TThreadPoolServer$WorkerProcess.run(TThreadPoolServer.java:285) ~[libthrift-0.9.2.jar:0.9.2]at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_131]at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_131]... 1 more
Caused by: java.lang.RuntimeException: java.io.IOException: No space left on deviceat org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:495) ~[?:?]at org.apache.hive.service.cli.session.HiveSessionImpl.open(HiveSessionImpl.java:147) ~[hive-service-1.2.1.jar:1.2.1]at sun.reflect.GeneratedMethodAccessor42.invoke(Unknown Source) ~[?:?]at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_131]at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_131]at org.apache.hive.service.cli.session.HiveSessionProxy.invoke(HiveSessionProxy.java:78) ~[hive-service-1.2.1.jar:1.2.1]at org.apache.hive.service.cli.session.HiveSessionProxy.access$000(HiveSessionProxy.java:36) ~[hive-service-1.2.1.jar:1.2.1]at org.apache.hive.service.cli.session.HiveSessionProxy$1.run(HiveSessionProxy.java:63) ~[hive-service-1.2.1.jar:1.2.1]at java.security.AccessController.doPrivileged(Native Method) ~[?:1.8.0_131]at javax.security.auth.Subject.doAs(Subject.java:422) ~[?:1.8.0_131]at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1836) ~[hadoop-common-2.7.3.jar:?]at org.apache.hive.service.cli.session.HiveSessionProxy.invoke(HiveSessionProxy.java:59) ~[hive-service-1.2.1.jar:1.2.1]at com.sun.proxy.$Proxy20.open(Unknown Source) ~[?:?]at org.apache.hive.service.cli.session.SessionManager.openSession(SessionManager.java:258) ~[hive-service-1.2.1.jar:1.2.1]at org.apache.hive.service.cli.CLIService.openSessionWithImpersonation(CLIService.java:204) ~[hive-service-1.2.1.jar:1.2.1]at org.apache.hive.service.cli.thrift.ThriftCLIService.getSessionHandle(ThriftCLIService.java:404) ~[hive-service-1.2.1.jar:1.2.1]at org.apache.hive.service.cli.thrift.ThriftCLIService.OpenSession(ThriftCLIService.java:299) ~[hive-service-1.2.1.jar:1.2.1]at org.apache.hive.service.cli.thrift.TCLIService$Processor$OpenSession.getResult(TCLIService.java:1253) ~[hive-service-1.2.1.jar:1.2.1]at org.apache.hive.service.cli.thrift.TCLIService$Processor$OpenSession.getResult(TCLIService.java:1238) ~[hive-service-1.2.1.jar:1.2.1]at org.apache.thrift.ProcessFunction.process(ProcessFunction.java:39) ~[libthrift-0.9.2.jar:0.9.2]at org.apache.thrift.TBaseProcessor.process(TBaseProcessor.java:39) ~[libthrift-0.9.2.jar:0.9.2]at org.apache.hive.service.auth.TSetIpAddressProcessor.process(TSetIpAddressProcessor.java:56) ~[hive-service-1.2.1.jar:1.2.1]at org.apache.thrift.server.TThreadPoolServer$WorkerProcess.run(TThreadPoolServer.java:285) ~[libthrift-0.9.2.jar:0.9.2]at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_131]at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_131]... 1 more
Caused by: java.io.IOException: No space left on deviceat java.io.UnixFileSystem.createFileExclusively(Native Method) ~[?:1.8.0_131]at java.io.File.createTempFile(File.java:2024) ~[?:1.8.0_131]at org.apache.hadoop.hive.ql.session.SessionState.createTempFile(SessionState.java:798) ~[?:?]at org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:493) ~[?:?]at org.apache.hive.service.cli.session.HiveSessionImpl.open(HiveSessionImpl.java:147) ~[hive-service-1.2.1.jar:1.2.1]at sun.reflect.GeneratedMethodAccessor42.invoke(Unknown Source) ~[?:?]at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_131]at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_131]at org.apache.hive.service.cli.session.HiveSessionProxy.invoke(HiveSessionProxy.java:78) ~[hive-service-1.2.1.jar:1.2.1]at org.apache.hive.service.cli.session.HiveSessionProxy.access$000(HiveSessionProxy.java:36) ~[hive-service-1.2.1.jar:1.2.1]at org.apache.hive.service.cli.session.HiveSessionProxy$1.run(HiveSessionProxy.java:63) ~[hive-service-1.2.1.jar:1.2.1]at java.security.AccessController.doPrivileged(Native Method) ~[?:1.8.0_131]at javax.security.auth.Subject.doAs(Subject.java:422) ~[?:1.8.0_131]at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1836) ~[hadoop-common-2.7.3.jar:?]at org.apache.hive.service.cli.session.HiveSessionProxy.invoke(HiveSessionProxy.java:59) ~[hive-service-1.2.1.jar:1.2.1]at com.sun.proxy.$Proxy20.open(Unknown Source) ~[?:?]at org.apache.hive.service.cli.session.SessionManager.openSession(SessionManager.java:258) ~[hive-service-1.2.1.jar:1.2.1]at org.apache.hive.service.cli.CLIService.openSessionWithImpersonation(CLIService.java:204) ~[hive-service-1.2.1.jar:1.2.1]at org.apache.hive.service.cli.thrift.ThriftCLIService.getSessionHandle(ThriftCLIService.java:404) ~[hive-service-1.2.1.jar:1.2.1]at org.apache.hive.service.cli.thrift.ThriftCLIService.OpenSession(ThriftCLIService.java:299) ~[hive-service-1.2.1.jar:1.2.1]at org.apache.hive.service.cli.thrift.TCLIService$Processor$OpenSession.getResult(TCLIService.java:1253) ~[hive-service-1.2.1.jar:1.2.1]at org.apache.hive.service.cli.thrift.TCLIService$Processor$OpenSession.getResult(TCLIService.java:1238) ~[hive-service-1.2.1.jar:1.2.1]at org.apache.thrift.ProcessFunction.process(ProcessFunction.java:39) ~[libthrift-0.9.2.jar:0.9.2]at org.apache.thrift.TBaseProcessor.process(TBaseProcessor.java:39) ~[libthrift-0.9.2.jar:0.9.2]at org.apache.hive.service.auth.TSetIpAddressProcessor.process(TSetIpAddressProcessor.java:56) ~[hive-service-1.2.1.jar:1.2.1]at org.apache.thrift.server.TThreadPoolServer$WorkerProcess.run(TThreadPoolServer.java:285) ~[libthrift-0.9.2.jar:0.9.2]at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_131]at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_131]... 1 more

处理方案

确认服务器的磁盘使用情况

命令 df -h /opt

結果

[root@bigdata03-demo ~]# df -h /opt
文件系统                    容量  已用  可用 已用% 挂载点
/dev/mapper/systemvg-optlv   33G   23G   11G   69% /opt

寻找存储最大的目录和文件

查询子目录的大小，子目录层级一级，命令du -h --max-depth=1 /opt

[root@bigdata03-demo ~]# du -h --max-depth=1 /opt
618M    /opt/zookeeper
371M    /opt/java
1.6G    /opt/hive
8.3M    /opt/sparkLocal
3.8G    /opt/spark
76M     /opt/promes
48K     /opt/flinkLocal
172M    /opt/monitor
22G     /opt

确认删除文件

命令 rm -f 文件绝对路径