HIVE - Internal error processing abort_txn

로그내용

2023-02-28T19:36:04,684 INFO [aff416f0-8cb9-4382-8dc1-fa123e9d6418 HiveServer2-Handler-Pool: Thread-99]: metastore.RetryingMetaStoreClient (:()) - RetryingMetaStoreClient trying reconnect as hive (auth:SIMPLE)
2023-02-28T19:36:04,685 INFO [aff416f0-8cb9-4382-8dc1-fa123e9d6418 HiveServer2-Handler-Pool: Thread-99]: metastore.HiveMetaStoreClient (:()) - Closed a connection to metastore, current connections: 6
2023-02-28T19:36:04,685 INFO [aff416f0-8cb9-4382-8dc1-fa123e9d6418 HiveServer2-Handler-Pool: Thread-99]: metastore.HiveMetaStoreClient (:()) - Trying to connect to metastore with URI thrift://hostname:9083
2023-02-28T19:36:04,685 INFO [aff416f0-8cb9-4382-8dc1-fa123e9d6418 HiveServer2-Handler-Pool: Thread-99]: metastore.HiveMetaStoreClient (:()) - Opened a connection to metastore, current connections: 7
2023-02-28T19:36:04,687 INFO [aff416f0-8cb9-4382-8dc1-fa123e9d6418 HiveServer2-Handler-Pool: Thread-99]: metastore.HiveMetaStoreClient (:()) - Connected to metastore.
2023-02-28T19:36:04,691 WARN [aff416f0-8cb9-4382-8dc1-fa123e9d6418 HiveServer2-Handler-Pool: Thread-99]: metastore.RetryingMetaStoreClient (:()) - MetaStoreClient lost connection. Attempting to reconnect (3 of 24) after 5s. rollbackTxn
org.apache.thrift.TApplicationException: Internal error processing abort_txn
at org.apache.thrift.TApplicationException.read(TApplicationException.java:111) ~[hive-exec-3.1.3000.7.1.4.0-203.jar:3.1.3000.7.1.4.0-203]
at org.apache.thrift.TServiceClient.receiveBase(TServiceClient.java:79) ~[hive-exec-3.1.3000.7.1.4.0-203.jar:3.1.3000.7.1.4.0-203]
at.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$Client.recv_abort_txn(ThriftHiveMetastore.java:5609) ~[hive-exec-3.1.3000.7.1.4.0-203.jar:3.1.3000.7.1.4.0-203] at org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$Client.abort_txn(ThriftHiveMetastore.java:5596) ~[hive-exec-3.1.3000.7.1.4.0-203.jar:3.1.3000.7.1.4.0-203]
at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.rollbackTxn(HiveMetaStoreClient.java:3125) ~[hive-exec-3.1.3000.7.1.4.0-203.jar:3.1.3000.7.1.4.0-203]
at sun.reflect.GeneratedMethodAccessor31.invoke(Unknown Source) ~[?:?]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.invoke(RetryingMetaStoreClient.java:213) ~[hive-exec-3.1.3000.7.1.4.0-203.jar:3.1.3000.7.1.4.0-203]
at com.sun.proxy.$Proxy55.rollbackTxn(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor31.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275] at org.apache.hadoop.hive.metastore.HiveMetaStoreClient$SynchronizedHandler.invoke(HiveMetaStoreClient.java:3428) ~[hive-exec-3.1.3000.7.1.4.0-203.jar:3.1.3000.7.1.4.0-203]
at com.sun.proxy.$Proxy55.rollbackTxn(Unknown Source) ~[?:?] at org.apache.hadoop.hive.ql.lockmgr.DbTxnManager.rollbackTxn(DbTxnManager.java:606) ~[hive-exec-3.1.3000.7.1.4.0-203.jar:3.1.3000.7.1.4.0-203] at org.apache.hadoop.hive.ql.Driver.releaseLocksAndCommitOrRollback(Driver.java:456) ~[hive-exec-3.1.3000.7.1.4.0-203.jar:3.1.3000.7.1.4.0-203] at org.apache.hadoop.hive.ql.Driver.releaseLocksAndCommitOrRollback(Driver.java:419) ~[hive-exec-3.1.3000.7.1.4.0-203.jar:3.1.3000.7.1.4.0-203] at org.apache.hadoop.hive.ql.Driver.compileInternal(Driver.java:608) ~[hive-exec-3.1.3000.7.1.4.0-203.jar:3.1.3000.7.1.4.0-203] at org.apache.hadoop.hive.ql.Driver.compileAndRespond(Driver.java:551) ~[hive-exec-3.1.3000.7.1.4.0-203.jar:3.1.3000.7.1.4.0-203] at org.apache.hadoop.hive.ql.Driver.compileAndRespond(Driver.java:545) ~[hive-exec-3.1.3000.7.1.4.0-203.jar:3.1.3000.7.1.4.0-203] at org.apache.hadoop.hive.ql.reexec.ReExecDriver.compileAndRespond(ReExecDriver.java:127) ~[hive-exec-3.1.3000.7.1.4.0-203.jar:3.1.3000.7.1.4.0-203] at org.apache.hive.service.cli.operation.SQLOperation.prepare(SQLOperation.java:199) ~[hive-service-3.1.3000.7.1.4.0-203.jar:3.1.3000.7.1.4.0-203] at org.apache.hive.service.cli.operation.SQLOperation.runInternal(SQLOperation.java:260) ~[hive-service-3.1.3000.7.1.4.0-203.jar:3.1.3000.7.1.4.0-203] at org.apache.hive.service.cli.operation.Operation.run(Operation.java:274) ~[hive-service-3.1.3000.7.1.4.0-203.jar:3.1.3000.7.1.4.0-203] at org.apache.hive.service.cli.session.HiveSessionImpl.executeStatementInternal(HiveSessionImpl.java:565) ~[hive-service-3.1.3000.7.1.4.0-203.jar:3.1.3000.7.1.4.0-203] at org.apache.hive.service.cli.session.HiveSessionImpl.executeStatementAsync(HiveSessionImpl.java:551) ~[hive-service-3.1.3000.7.1.4.0-203.jar:3.1.3000.7.1.4.0-203] at org.apache.hive.service.cli.CLIService.executeStatementAsync(CLIService.java:315) ~[hive-service-3.1.3000.7.1.4.0-203.jar:3.1.3000.7.1.4.0-203] at org.apache.hive.service.cli.thrift.ThriftCLIService.ExecuteStatement(ThriftCLIService.java:567) ~[hive-service-3.1.3000.7.1.4.0-203.jar:3.1.3000.7.1.4.0-203] at org.apache.hive.service.rpc.thrift.TCLIService$Processor$ExecuteStatement.getResult(TCLIService.java:1557) ~[hive-exec-3.1.3000.7.1.4.0-203.jar:3.1.3000.7.1.4.0-203] at org.apache.hive.service.rpc.thrift.TCLIService$Processor$ExecuteStatement.getResult(TCLIService.java:1542) ~[hive-exec-3.1.3000.7.1.4.0-203.jar:3.1.3000.7.1.4.0-203] at org.apache.thrift.ProcessFunction.process(ProcessFunction.java:39) ~[hive-exec-3.1.3000.7.1.4.0-203.jar:3.1.3000.7.1.4.0-203] at org.apache.thrift.TBaseProcessor.process(TBaseProcessor.java:39) ~[hive-exec-3.1.3000.7.1.4.0-203.jar:3.1.3000.7.1.4.0-203] at org.apache.hive.service.auth.TSetIpAddressProcessor.process(TSetIpAddressProcessor.java:56) ~[hive-service-3.1.3000.7.1.4.0-203.jar:3.1.3000.7.1.4.0-203] at org.apache.thrift.server.TThreadPoolServer$WorkerProcess.run(TThreadPoolServer.java:286) ~[hive-exec-3.1.3000.7.1.4.0-203.jar:3.1.3000.7.1.4.0-203]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]

조치

psql -U hive
hive=> reindex index tc_txnid_index;
REINDEX
hive=> reindex index hl_txnid_index;
REINDEX

Hive 재기동 필요없음.
reindex 실행 시 영향도 파악 필요.