Have a standalone Oracle coherence cluster with 6 Data nodes.
Recently migrated from 12.1.2.0.5 to 12.2.1.4.0. There are no code or configuration changes but time taken for upsert operation have significantly increased. Frequent Thread Dumps are being logged by service guardian When trying to recover the unresponsive threads.
Only issue i believe is cache is using interceptor to intercept the change events and push the changes messages to another cache in cluster But this use to work fine in 12.1.2.0.5 so wondering what has changed?
One of the threaddumps which do not indicate any deadlock.
Thread in bold are indefinitely waiting and become unresponsive
"Recovery Thread" id=203 State:RUNNABLE
at sun.management.ThreadImpl.dumpThreads0(Native Method)
at sun.management.ThreadImpl.dumpAllThreads(ThreadImpl.java:454)
at com.oracle.common.util.Threads.getThreadDump(Threads.java:76)
at com.oracle.common.util.Threads.getThreadDump(Threads.java:27)
at com.tangosol.net.GuardSupport.getThreadDump(GuardSupport.java:837)
at com.tangosol.net.GuardSupport.logStackTraces(GuardSupport.java:826)
at com.tangosol.internal.net.cluster.DefaultServiceFailurePolicy.onGuardableRecovery(DefaultServiceFailurePolicy.java:44)
at com.tangosol.coherence.component.util.daemon.queueProcessor.service.Grid$WrapperGuardable.recover(Grid.CDB:14)
at com.tangosol.net.GuardSupport$Context$1.run(GuardSupport.java:674)
at java.lang.Thread.run(Thread.java:748)
"Recovery Thread" id=202 State:RUNNABLE
at com.tangosol.coherence.component.net.message.requestMessage.DistributedCacheRequest.getDescription(DistributedCacheRequest.CDB:3)
at com.tangosol.coherence.component.net.Message.toString(Message.CDB:90)
at com.tangosol.coherence.component.net.Message.toString(Message.CDB:1)
at java.lang.String.valueOf(String.java:2994)
at com.tangosol.coherence.component.util.DaemonPool$WrapperTask.getTaskId(DaemonPool.CDB:4)
at com.tangosol.coherence.component.util.DaemonPool$Daemon$Guard.recover(DaemonPool.CDB:16)
at com.tangosol.internal.net.cluster.DefaultServiceFailurePolicy.onGuardableRecovery(DefaultServiceFailurePolicy.java:53)
at com.tangosol.coherence.component.util.daemon.queueProcessor.service.Grid$WrapperGuardable.recover(Grid.CDB:14)
at com.tangosol.net.GuardSupport$Context$1.run(GuardSupport.java:674)
at java.lang.Thread.run(Thread.java:748)
"pool-1-thread-3" id=198 State:WAITING
at sun.misc.Unsafe.park(Native Method)
- waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@27a4503c
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:748)
"pool-1-thread-2" id=197 State:WAITING
at sun.misc.Unsafe.park(Native Method)
- waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@27a4503c
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:748)
"pool-1-thread-1" id=196 State:WAITING
at sun.misc.Unsafe.park(Native Method)
- waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@27a4503c
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:748)
"Request Timer" id=127 State:WAITING
at java.lang.Object.wait(Native Method)
- waiting on java.util.TaskQueue@6152d488
at java.lang.Object.wait(Object.java:502)
at java.util.TimerThread.mainLoop(Timer.java:526)
at java.util.TimerThread.run(Timer.java:505)
"C3P0PooledConnectionPoolManager[identityToken->2tf8iaab1o6b0wy1tyiq1b|7ad1475f]-HelperThread-#2" id=124 State:TIMED_WAITING
at java.lang.Object.wait(Native Method)
- waiting on com.mchange.v2.async.ThreadPoolAsynchronousRunner@656d4c7a
at com.mchange.v2.async.ThreadPoolAsynchronousRunner$PoolThread.run(ThreadPoolAsynchronousRunner.java:683)
"C3P0PooledConnectionPoolManager[identityToken->2tf8iaab1o6b0wy1tyiq1b|7ad1475f]-HelperThread-#1" id=123 State:TIMED_WAITING
at java.lang.Object.wait(Native Method)
- waiting on com.mchange.v2.async.ThreadPoolAsynchronousRunner@656d4c7a
at com.mchange.v2.async.ThreadPoolAsynchronousRunner$PoolThread.run(ThreadPoolAsynchronousRunner.java:683)
"C3P0PooledConnectionPoolManager[identityToken->2tf8iaab1o6b0wy1tyiq1b|7ad1475f]-HelperThread-#0" id=122 State:TIMED_WAITING
at java.lang.Object.wait(Native Method)
- waiting on com.mchange.v2.async.ThreadPoolAsynchronousRunner@656d4c7a
at com.mchange.v2.async.ThreadPoolAsynchronousRunner$PoolThread.run(ThreadPoolAsynchronousRunner.java:683)
"C3P0PooledConnectionPoolManager[identityToken->2tf8iaab1o6b0wy1tyiq1b|7ad1475f]-AdminTaskTimer" id=121 State:TIMED_WAITING
at java.lang.Object.wait(Native Method)
- waiting on java.util.TaskQueue@206fdeb3
at java.util.TimerThread.mainLoop(Timer.java:552)
at java.util.TimerThread.run(Timer.java:505)
"oldTradeEvictor-1" id=118 State:TIMED_WAITING
at sun.misc.Unsafe.park(Native Method)
- waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@3387dd0d
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:748)
"simonResetScheduler-1" id=117 State:TIMED_WAITING
at sun.misc.Unsafe.park(Native Method)
- waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@73240a30
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:748)
"simonOutputScheduler-1" id=116 State:TIMED_WAITING
at sun.misc.Unsafe.park(Native Method)
- waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@235a458
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:748)
"Context_2_ConsumerDispatcher" id=115 State:WAITING
at sun.misc.Unsafe.park(Native Method)
- waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@2eedc0b6
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:403)
at com.solacesystems.jcsmp.protocol.nio.impl.ConsumerNotificationDispatcher.eventLoop(ConsumerNotificationDispatcher.java:80)
at com.solacesystems.jcsmp.protocol.nio.impl.ConsumerNotificationDispatcher.run(ConsumerNotificationDispatcher.java:100)
at java.lang.Thread.run(Thread.java:748)
"Thread-9" id=114 State:TIMED_WAITING
at java.lang.Thread.sleep(Native Method)
at com.rbc.solace.impl.SolaceSessionImpl$HeartBeatThread.run(SolaceSessionImpl.java:422)
"HBThread" id=113 State:WAITING
at sun.misc.Unsafe.park(Native Method)
- waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@238f1fd3
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at com.rbc.solace.impl.SolaceSessionImpl$MonitorPublishedTopicsThread.run(SolaceSessionImpl.java:351)
"Context_2_ReactorThread" id=112 State:RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
- locked sun.nio.ch.Util$3@7029e418
- locked java.util.Collections$UnmodifiableSet@5f118921
- locked sun.nio.ch.EPollSelectorImpl@1fb90a05
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at com.solacesystems.jcsmp.protocol.nio.impl.SyncEventDispatcherReactor.eventLoop(SyncEventDispatcherReactor.java:153)
at com.solacesystems.jcsmp.protocol.nio.impl.SyncEventDispatcherReactor$SEDReactorThread.run(SyncEventDispatcherReactor.java:338)
at java.lang.Thread.run(Thread.java:748)
"ServiceMonitor" id=110 State:TIMED_WAITING
at java.lang.Thread.sleep(Native Method)
at java.lang.Thread.sleep(Thread.java:340)
at com.oracle.common.base.Blocking.sleep(Blocking.java:126)
at com.oracle.common.base.Blocking.sleep(Blocking.java:111)
at com.tangosol.net.SimpleServiceMonitor.monitorServices(SimpleServiceMonitor.java:199)
at com.tangosol.net.SimpleServiceMonitor$1.run(SimpleServiceMonitor.java:169)
at java.lang.Thread.run(Thread.java:748)
"InvocationServiceWorker:0x0000:16" id=109 State:TIMED_WAITING
at sun.misc.Unsafe.park(Native Method)
- waiting on com.oracle.common.util.ConcurrentAssociationPile@4493efa6
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at com.oracle.common.base.Blocking.parkNanos(Blocking.java:157)
at com.oracle.common.base.ConcurrentNotifier.park(ConcurrentNotifier.java:149)
at com.oracle.common.base.ConcurrentNotifier.await(ConcurrentNotifier.java:66)
at com.tangosol.coherence.component.util.Daemon.onWait(Daemon.CDB:13)
at com.tangosol.coherence.component.util.DaemonPool$Daemon.onWait(DaemonPool.CDB:3)
at com.tangosol.coherence.component.util.daemon.queueProcessor.service.Grid$DaemonPool$Daemon.onWait(Grid.CDB:3)
at com.tangosol.coherence.component.util.Daemon.run(Daemon.CDB:43)
at java.lang.Thread.run(Thread.java:748)
"Invocation:InvocationService" id=108 State:TIMED_WAITING
at sun.misc.Unsafe.park(Native Method)
- waiting on com.oracle.common.base.SingleWaiterMultiNotifier@7b25da08
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at com.oracle.common.base.Blocking.parkNanos(Blocking.java:157)
at com.oracle.common.base.SingleWaiterMultiNotifier.await(SingleWaiterMultiNotifier.java:54)
at com.tangosol.coherence.component.util.queue.SingleConsumerQueue.await(SingleConsumerQueue.CDB:1)
at com.tangosol.coherence.component.util.Daemon.onWait(Daemon.CDB:13)
at com.tangosol.coherence.component.util.daemon.QueueProcessor.onWait(QueueProcessor.CDB:3)
at com.tangosol.coherence.component.util.daemon.queueProcessor.service.Grid.onWait(Grid.CDB:6)
at com.tangosol.coherence.component.util.Daemon.run(Daemon.CDB:43)
at java.lang.Thread.run(Thread.java:748)
"DistributedCacheWorker:0x0000:16" id=107 State:TIMED_WAITING
at sun.misc.Unsafe.park(Native Method)
- waiting on com.oracle.common.util.ConcurrentAssociationPile@282e0ccc
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at com.oracle.common.base.Blocking.parkNanos(Blocking.java:157)
at com.oracle.common.base.ConcurrentNotifier.park(ConcurrentNotifier.java:149)
at com.oracle.common.base.ConcurrentNotifier.await(ConcurrentNotifier.java:66)
at com.tangosol.coherence.component.util.Daemon.onWait(Daemon.CDB:13)
at com.tangosol.coherence.component.util.DaemonPool$Daemon.onWait(DaemonPool.CDB:3)
at com.tangosol.coherence.component.util.daemon.queueProcessor.service.Grid$DaemonPool$Daemon.onWait(Grid.CDB:3)
at com.tangosol.coherence.component.util.Daemon.run(Daemon.CDB:43)
at java.lang.Thread.run(Thread.java:748)
"DistributedCache:EventDispatcher" id=106 State:TIMED_WAITING
at java.lang.Object.wait(Native Method)
- waiting on com.tangosol.coherence.component.util.daemon.queueProcessor.Service$EventDispatcher$Queue@7f90c2ee
at java.lang.Object.wait(Object.java:460)
at com.oracle.common.base.Blocking.wait(Blocking.java:97)
at com.oracle.common.base.Blocking.wait(Blocking.java:80)
at com.tangosol.coherence.component.util.Queue.await(Queue.CDB:9)
at com.tangosol.coherence.component.util.Daemon.onWait(Daemon.CDB:13)
at com.tangosol.coherence.component.util.daemon.QueueProcessor.onWait(QueueProcessor.CDB:3)
at com.tangosol.coherence.component.util.daemon.queueProcessor.Service$EventDispatcher.onWait(Service.CDB:7)
at com.tangosol.coherence.component.util.Daemon.run(Daemon.CDB:43)
at java.lang.Thread.run(Thread.java:748)
"DistributedCache" id=105 State:TIMED_WAITING
at sun.misc.Unsafe.park(Native Method)
- waiting on com.oracle.common.base.SingleWaiterMultiNotifier@2425fc65
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at com.oracle.common.base.Blocking.parkNanos(Blocking.java:157)
at com.oracle.common.base.SingleWaiterMultiNotifier.await(SingleWaiterMultiNotifier.java:54)
at com.tangosol.coherence.component.util.queue.SingleConsumerQueue.await(SingleConsumerQueue.CDB:1)
at com.tangosol.coherence.component.util.Daemon.onWait(Daemon.CDB:13)
at com.tangosol.coherence.component.util.daemon.QueueProcessor.onWait(QueueProcessor.CDB:3)
at com.tangosol.coherence.component.util.daemon.queueProcessor.service.Grid.onWait(Grid.CDB:6)
at com.tangosol.coherence.component.util.Daemon.run(Daemon.CDB:43)
at java.lang.Thread.run(Thread.java:748)
"DistributedCache:UtilityService:EventDispatcher" id=104 State:TIMED_WAITING
at java.lang.Object.wait(Native Method)
- waiting on com.tangosol.coherence.component.util.daemon.queueProcessor.Service$EventDispatcher$Queue@71859773
at java.lang.Object.wait(Object.java:460)
at com.oracle.common.base.Blocking.wait(Blocking.java:97)
at com.oracle.common.base.Blocking.wait(Blocking.java:80)
at com.tangosol.coherence.component.util.Queue.await(Queue.CDB:9)
at com.tangosol.coherence.component.util.Daemon.onWait(Daemon.CDB:13)
at com.tangosol.coherence.component.util.daemon.QueueProcessor.onWait(QueueProcessor.CDB:3)
at com.tangosol.coherence.component.util.daemon.queueProcessor.Service$EventDispatcher.onWait(Service.CDB:7)
at com.tangosol.coherence.component.util.Daemon.run(Daemon.CDB:43)
at java.lang.Thread.run(Thread.java:748)
"DistributedCache:UtilityService" id=103 State:TIMED_WAITING
at sun.misc.Unsafe.park(Native Method)
- waiting on com.oracle.common.base.SingleWaiterMultiNotifier@2995bde3
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at com.oracle.common.base.Blocking.parkNanos(Blocking.java:157)
at com.oracle.common.base.SingleWaiterMultiNotifier.await(SingleWaiterMultiNotifier.java:54)
at com.tangosol.coherence.component.util.queue.SingleConsumerQueue.await(SingleConsumerQueue.CDB:1)
at com.tangosol.coherence.component.util.Daemon.onWait(Daemon.CDB:13)
at com.tangosol.coherence.component.util.daemon.QueueProcessor.onWait(QueueProcessor.CDB:3)
at com.tangosol.coherence.component.util.daemon.queueProcessor.service.Grid.onWait(Grid.CDB:6)
at com.tangosol.coherence.component.util.Daemon.run(Daemon.CDB:43)
at java.lang.Thread.run(Thread.java:748)
"GTMMappingDataServiceWorker:3" id=102 State:TIMED_WAITING
at sun.misc.Unsafe.park(Native Method)
- waiting on com.oracle.common.util.ConcurrentAssociationPile@50eb8d8d
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at com.oracle.common.base.Blocking.parkNanos(Blocking.java:157)
at com.oracle.common.base.ConcurrentNotifier.park(ConcurrentNotifier.java:149)
at com.oracle.common.base.ConcurrentNotifier.await(ConcurrentNotifier.java:66)
at com.tangosol.coherence.component.util.Daemon.onWait(Daemon.CDB:13)
at com.tangosol.coherence.component.util.DaemonPool$Daemon.onWait(DaemonPool.CDB:3)
at com.tangosol.coherence.component.util.daemon.queueProcessor.service.Grid$DaemonPool$Daemon.onWait(Grid.CDB:3)
at com.tangosol.coherence.component.util.Daemon.run(Daemon.CDB:43)
at java.lang.Thread.run(Thread.java:748)
"GTMMappingDataServiceWorker:2" id=101 State:TIMED_WAITING
at sun.misc.Unsafe.park(Native Method)
- waiting on com.oracle.common.util.ConcurrentAssociationPile@5b268521
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at com.oracle.common.base.Blocking.parkNanos(Blocking.java:157)
at com.oracle.common.base.ConcurrentNotifier.park(ConcurrentNotifier.java:149)
at com.oracle.common.base.ConcurrentNotifier.await(ConcurrentNotifier.java:66)
at com.tangosol.coherence.component.util.Daemon.onWait(Daemon.CDB:13)
at com.tangosol.coherence.component.util.DaemonPool$Daemon.onWait(DaemonPool.CDB:3)
at com.tangosol.coherence.component.util.daemon.queueProcessor.service.Grid$DaemonPool$Daemon.onWait(Grid.CDB:3)
at com.tangosol.coherence.component.util.Daemon.run(Daemon.CDB:43)
at java.lang.Thread.run(Thread.java:748)
"GTMMappingDataServiceWorker:1" id=100 State:TIMED_WAITING
at sun.misc.Unsafe.park(Native Method)
- waiting on com.oracle.common.util.ConcurrentAssociationPile@645af39
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at com.oracle.common.base.Blocking.parkNanos(Blocking.java:157)
at com.oracle.common.base.ConcurrentNotifier.park(ConcurrentNotifier.java:149)
at com.oracle.common.base.ConcurrentNotifier.await(ConcurrentNotifier.java:66)
at com.tangosol.coherence.component.util.Daemon.onWait(Daemon.CDB:13)
at com.tangosol.coherence.component.util.DaemonPool$Daemon.onWait(DaemonPool.CDB:3)
at com.tangosol.coherence.component.util.daemon.queueProcessor.service.Grid$DaemonPool$Daemon.onWait(Grid.CDB:3)
at com.tangosol.coherence.component.util.Daemon.run(Daemon.CDB:43)
at java.lang.Thread.run(Thread.java:748)
"GTMMappingDataServiceWorker:0" id=99 State:TIMED_WAITING
at sun.misc.Unsafe.park(Native Method)
- waiting on com.oracle.common.util.ConcurrentAssociationPile@2bb60b7
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at com.oracle.common.base.Blocking.parkNanos(Blocking.java:157)
at com.oracle.common.base.ConcurrentNotifier.park(ConcurrentNotifier.java:149)
at com.oracle.common.base.ConcurrentNotifier.await(ConcurrentNotifier.java:66)
at com.tangosol.coherence.component.util.Daemon.onWait(Daemon.CDB:13)
at com.tangosol.coherence.component.util.DaemonPool$Daemon.onWait(DaemonPool.CDB:3)
at com.tangosol.coherence.component.util.daemon.queueProcessor.service.Grid$DaemonPool$Daemon.onWait(Grid.CDB:3)
at com.tangosol.coherence.component.util.Daemon.run(Daemon.CDB:43)
at java.lang.Thread.run(Thread.java:748)
"DistributedCache:GTMMappingDataService:EventDispatcher" id=98 State:TIMED_WAITING
at java.lang.Object.wait(Native Method)
- waiting on com.tangosol.coherence.component.util.daemon.queueProcessor.Service$EventDispatcher$Queue@eb00288
at java.lang.Object.wait(Object.java:460)
at com.oracle.common.base.Blocking.wait(Blocking.java:97)
at com.oracle.common.base.Blocking.wait(Blocking.java:80)
at com.tangosol.coherence.component.util.Queue.await(Queue.CDB:9)
at com.tangosol.coherence.component.util.Daemon.onWait(Daemon.CDB:13)
at com.tangosol.coherence.component.util.daemon.QueueProcessor.onWait(QueueProcessor.CDB:3)
at com.tangosol.coherence.component.util.daemon.queueProcessor.Service$EventDispatcher.onWait(Service.CDB:7)
at com.tangosol.coherence.component.util.Daemon.run(Daemon.CDB:43)
at java.lang.Thread.run(Thread.java:748)
"DistributedCache:GTMMappingDataService" id=97 State:TIMED_WAITING
at sun.misc.Unsafe.park(Native Method)
- waiting on com.oracle.common.base.SingleWaiterMultiNotifier@615091f2
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at com.oracle.common.base.Blocking.parkNanos(Blocking.java:157)
at com.oracle.common.base.SingleWaiterMultiNotifier.await(SingleWaiterMultiNotifier.java:54)
at com.tangosol.coherence.component.util.queue.SingleConsumerQueue.await(SingleConsumerQueue.CDB:1)
at com.tangosol.coherence.component.util.Daemon.onWait(Daemon.CDB:13)
at com.tangosol.coherence.component.util.daemon.QueueProcessor.onWait(QueueProcessor.CDB:3)
at com.tangosol.coherence.component.util.daemon.queueProcessor.service.Grid.onWait(Grid.CDB:6)
at com.tangosol.coherence.component.util.Daemon.run(Daemon.CDB:43)
at java.lang.Thread.run(Thread.java:748)
"GTMTradeDataServiceWorker:23" id=96 State:TIMED_WAITING
at sun.misc.Unsafe.park(Native Method)
- waiting on com.oracle.common.util.ConcurrentAssociationPile@4ac7c686
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at com.oracle.common.base.Blocking.parkNanos(Blocking.java:157)
at com.oracle.common.base.ConcurrentNotifier.park(ConcurrentNotifier.java:149)
at com.oracle.common.base.ConcurrentNotifier.await(ConcurrentNotifier.java:66)
at com.tangosol.coherence.component.util.Daemon.onWait(Daemon.CDB:13)
at com.tangosol.coherence.component.util.DaemonPool$Daemon.onWait(DaemonPool.CDB:3)
at com.tangosol.coherence.component.util.daemon.queueProcessor.service.Grid$DaemonPool$Daemon.onWait(Grid.CDB:3)
at com.tangosol.coherence.component.util.Daemon.run(Daemon.CDB:43)
at java.lang.Thread.run(Thread.java:748)
"GTMTradeDataServiceWorker:22" id=95 State:TIMED_WAITING
at sun.misc.Unsafe.park(Native Method)
- waiting on com.oracle.common.util.ConcurrentAssociationPile@13a0b9b1
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at com.oracle.common.base.Blocking.parkNanos(Blocking.java:157)
at com.oracle.common.base.ConcurrentNotifier.park(ConcurrentNotifier.java:149)
at com.oracle.common.base.ConcurrentNotifier.await(ConcurrentNotifier.java:66)
at com.tangosol.coherence.component.util.Daemon.onWait(Daemon.CDB:13)
at com.tangosol.coherence.component.util.DaemonPool$Daemon.onWait(DaemonPool.CDB:3)
at com.tangosol.coherence.component.util.daemon.queueProcessor.service.Grid$DaemonPool$Daemon.onWait(Grid.CDB:3)
at com.tangosol.coherence.component.util.Daemon.run(Daemon.CDB:43)
at java.lang.Thread.run(Thread.java:748)
"GTMTradeDataServiceWorker:21" id=94 State:TIMED_WAITING
at sun.misc.Unsafe.park(Native Method)
- waiting on com.oracle.common.util.ConcurrentAssociationPile@1d0a5d0f
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at com.oracle.common.base.Blocking.parkNanos(Blocking.java:157)
at com.oracle.common.base.ConcurrentNotifier.park(ConcurrentNotifier.java:149)
at com.oracle.common.base.ConcurrentNotifier.await(ConcurrentNotifier.java:66)
at com.tangosol.coherence.component.util.Daemon.onWait(Daemon.CDB:13)
at com.tangosol.coherence.component.util.DaemonPool$Daemon.onWait(DaemonPool.CDB:3)
at com.tangosol.coherence.component.util.daemon.queueProcessor.service.Grid$DaemonPool$Daemon.onWait(Grid.CDB:3)
at com.tangosol.coherence.component.util.Daemon.run(Daemon.CDB:43)
at java.lang.Thread.run(Thread.java:748)
"GTMTradeDataServiceWorker:20" id=93 State:TIMED_WAITING
at sun.misc.Unsafe.park(Native Method)
- waiting on com.oracle.common.util.ConcurrentAssociationPile@7830805d
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at com.oracle.common.base.Blocking.parkNanos(Blocking.java:157)
at com.oracle.common.base.ConcurrentNotifier.park(ConcurrentNotifier.java:149)
at com.oracle.common.base.ConcurrentNotifier.await(ConcurrentNotifier.java:66)
at com.tangosol.coherence.component.util.Daemon.onWait(Daemon.CDB:13)
at com.tangosol.coherence.component.util.DaemonPool$Daemon.onWait(DaemonPool.CDB:3)
at com.tangosol.coherence.component.util.daemon.queueProcessor.service.Grid$DaemonPool$Daemon.onWait(Grid.CDB:3)
at com.tangosol.coherence.component.util.Daemon.run(Daemon.CDB:43)
at java.lang.Thread.run(Thread.java:748)
"GTMTradeDataServiceWorker:19" id=92 State:TIMED_WAITING
at sun.misc.Unsafe.park(Native Method)
- waiting on com.oracle.common.util.ConcurrentAssociationPile@c379a2b
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at com.oracle.common.base.Blocking.parkNanos(Blocking.java:157)
at com.oracle.common.base.ConcurrentNotifier.park(ConcurrentNotifier.java:149)
at com.oracle.common.base.ConcurrentNotifier.await(ConcurrentNotifier.java:66)
at com.tangosol.coherence.component.util.Daemon.onWait(Daemon.CDB:13)
at com.tangosol.coherence.component.util.DaemonPool$Daemon.onWait(DaemonPool.CDB:3)
at com.tangosol.coherence.component.util.daemon.queueProcessor.service.Grid$DaemonPool$Daemon.onWait(Grid.CDB:3)
at com.tangosol.coherence.component.util.Daemon.run(Daemon.CDB:43)
at java.lang.Thread.run(Thread.java:748)
"GTMTradeDataServiceWorker:18" id=91 State:TIMED_WAITING
at sun.misc.Unsafe.park(Native Method)
- waiting on com.oracle.common.util.ConcurrentAssociationPile@1a4cc72d
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at com.oracle.common.base.Blocking.parkNanos(Blocking.java:157)
at com.oracle.common.base.ConcurrentNotifier.park(ConcurrentNotifier.java:149)
at com.oracle.common.base.ConcurrentNotifier.await(ConcurrentNotifier.java:66)
at com.tangosol.coherence.component.util.Daemon.onWait(Daemon.CDB:13)
at com.tangosol.coherence.component.util.DaemonPool$Daemon.onWait(DaemonPool.CDB:3)
at com.tangosol.coherence.component.util.daemon.queueProcessor.service.Grid$DaemonPool$Daemon.onWait(Grid.CDB:3)
at com.tangosol.coherence.component.util.Daemon.run(Daemon.CDB:43)
at java.lang.Thread.run(Thread.java:748)
"GTMTradeDataServiceWorker:17" id=90 State:TIMED_WAITING
at sun.misc.Unsafe.park(Native Method)
- waiting on com.oracle.common.util.ConcurrentAssociationPile@55e1034e
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at com.oracle.common.base.Blocking.parkNanos(Blocking.java:157)
at com.oracle.common.base.ConcurrentNotifier.park(ConcurrentNotifier.java:149)
at com.oracle.common.base.ConcurrentNotifier.await(ConcurrentNotifier.java:66)
at com.tangosol.coherence.component.util.Daemon.onWait(Daemon.CDB:13)
at com.tangosol.coherence.component.util.DaemonPool$Daemon.onWait(DaemonPool.CDB:3)
at com.tangosol.coherence.component.util.daemon.queueProcessor.service.Grid$DaemonPool$Daemon.onWait(Grid.CDB:3)
at com.tangosol.coherence.component.util.Daemon.run(Daemon.CDB:43)
at java.lang.Thread.run(Thread.java:748)
"GTMTradeDataServiceWorker:16" id=89 State:TIMED_WAITING
at sun.misc.Unsafe.park(Native Method)
- waiting on com.oracle.common.util.ConcurrentAssociationPile@29af53c7
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at com.oracle.common.base.Blocking.parkNanos(Blocking.java:157)
at com.oracle.common.base.ConcurrentNotifier.park(ConcurrentNotifier.java:149)
at com.oracle.common.base.ConcurrentNotifier.await(ConcurrentNotifier.java:66)
at com.tangosol.coherence.component.util.Daemon.onWait(Daemon.CDB:13)
at com.tangosol.coherence.component.util.DaemonPool$Daemon.onWait(DaemonPool.CDB:3)
at com.tangosol.coherence.component.util.daemon.queueProcessor.service.Grid$DaemonPool$Daemon.onWait(Grid.CDB:3)
at com.tangosol.coherence.component.util.Daemon.run(Daemon.CDB:43)
at java.lang.Thread.run(Thread.java:748)
"GTMTradeDataServiceWorker:15" id=88 State:TIMED_WAITING
at sun.misc.Unsafe.park(Native Method)
- waiting on com.oracle.common.util.ConcurrentAssociationPile@19f9721b
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at com.oracle.common.base.Blocking.parkNanos(Blocking.java:157)
at com.oracle.common.base.ConcurrentNotifier.park(ConcurrentNotifier.java:149)
at com.oracle.common.base.ConcurrentNotifier.await(ConcurrentNotifier.java:66)
at com.tangosol.coherence.component.util.Daemon.onWait(Daemon.CDB:13)
at com.tangosol.coherence.component.util.DaemonPool$Daemon.onWait(DaemonPool.CDB:3)
at com.tangosol.coherence.component.util.daemon.queueProcessor.service.Grid$DaemonPool$Daemon.onWait(Grid.CDB:3)
at com.tangosol.coherence.component.util.Daemon.run(Daemon.CDB:43)
at java.lang.Thread.run(Thread.java:748)
"GTMTradeDataServiceWorker:14" id=87 State:TIMED_WAITING
at sun.misc.Unsafe.park(Native Method)
- waiting on com.oracle.common.util.ConcurrentAssociationPile@65f0bba6
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at com.oracle.common.base.Blocking.parkNanos(Blocking.java:157)
at com.oracle.common.base.ConcurrentNotifier.park(ConcurrentNotifier.java:149)
at com.oracle.common.base.ConcurrentNotifier.await(ConcurrentNotifier.java:66)
at com.tangosol.coherence.component.util.Daemon.onWait(Daemon.CDB:13)
at com.tangosol.coherence.component.util.DaemonPool$Daemon.onWait(DaemonPool.CDB:3)
at com.tangosol.coherence.component.util.daemon.queueProcessor.service.Grid$DaemonPool$Daemon.onWait(Grid.CDB:3)
at com.tangosol.coherence.component.util.Daemon.run(Daemon.CDB:43)
at java.lang.Thread.run(Thread.java:748)
"GTMTradeDataServiceWorker:13" id=86 State:TIMED_WAITING
at sun.misc.Unsafe.park(Native Method)
- waiting on com.oracle.common.util.ConcurrentAssociationPile@38562f25
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at com.oracle.common.base.Blocking.parkNanos(Blocking.java:157)
at com.oracle.common.base.ConcurrentNotifier.park(ConcurrentNotifier.java:149)
at com.oracle.common.base.ConcurrentNotifier.await(ConcurrentNotifier.java:66)
at com.tangosol.coherence.component.util.Daemon.onWait(Daemon.CDB:13)
at com.tangosol.coherence.component.util.DaemonPool$Daemon.onWait(DaemonPool.CDB:3)
at com.tangosol.coherence.component.util.daemon.queueProcessor.service.Grid$DaemonPool$Daemon.onWait(Grid.CDB:3)
at com.tangosol.coherence.component.util.Daemon.run(Daemon.CDB:43)
at java.lang.Thread.run(Thread.java:748)
"GTMTradeDataServiceWorker:12" id=85 State:TIMED_WAITING
at sun.misc.Unsafe.park(Native Method)
- waiting on com.oracle.common.util.ConcurrentAssociationPile@6b346283
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at com.oracle.common.base.Blocking.parkNanos(Blocking.java:157)
at com.oracle.common.base.ConcurrentNotifier.park(ConcurrentNotifier.java:149)
at com.oracle.common.base.ConcurrentNotifier.await(ConcurrentNotifier.java:66)
at com.tangosol.coherence.component.util.Daemon.onWait(Daemon.CDB:13)
at com.tangosol.coherence.component.util.DaemonPool$Daemon.onWait(DaemonPool.CDB:3)
at com.tangosol.coherence.component.util.daemon.queueProcessor.service.Grid$DaemonPool$Daemon.onWait(Grid.CDB:3)
at com.tangosol.coherence.component.util.Daemon.run(Daemon.CDB:43)
at java.lang.Thread.run(Thread.java:748)
"GTMTradeDataServiceWorker:11" id=84 State:TIMED_WAITING
at sun.misc.Unsafe.park(Native Method)
- waiting on com.oracle.common.util.ConcurrentAssociationPile@390fbc54
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at com.oracle.common.base.Blocking.parkNanos(Blocking.java:157)
at com.oracle.common.base.ConcurrentNotifier.park(ConcurrentNotifier.java:149)
at com.oracle.common.base.ConcurrentNotifier.await(ConcurrentNotifier.java:66)
at com.tangosol.coherence.component.util.Daemon.onWait(Daemon.CDB:13)
at com.tangosol.coherence.component.util.DaemonPool$Daemon.onWait(DaemonPool.CDB:3)
at com.tangosol.coherence.component.util.daemon.queueProcessor.service.Grid$DaemonPool$Daemon.onWait(Grid.CDB:3)
at com.tangosol.coherence.component.util.Daemon.run(Daemon.CDB:43)
at java.lang.Thread.run(Thread.java:748)
"GTMTradeDataServiceWorker:10" id=83 State:TIMED_WAITING
at sun.misc.Unsafe.park(Native Method)
- waiting on com.oracle.common.util.ConcurrentAssociationPile@59420d58
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at com.oracle.common.base.Blocking.parkNanos(Blocking.java:157)
at com.oracle.common.base.ConcurrentNotifier.park(ConcurrentNotifier.java:149)
at com.oracle.common.base.ConcurrentNotifier.await(ConcurrentNotifier.java:66)
at com.tangosol.coherence.component.util.Daemon.onWait(Daemon.CDB:13)
at com.tangosol.coherence.component.util.DaemonPool$Daemon.onWait(DaemonPool.CDB:3)
at com.tangosol.coherence.component.util.daemon.queueProcessor.service.Grid$DaemonPool$Daemon.onWait(Grid.CDB:3)
at com.tangosol.coherence.component.util.Daemon.run(Daemon.CDB:43)
at java.lang.Thread.run(Thread.java:748)
"GTMTradeDataServiceWorker:9" id=82 State:TIMED_WAITING
at sun.misc.Unsafe.park(Native Method)
- waiting on com.oracle.common.util.ConcurrentAssociationPile@50643e52
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at com.oracle.common.base.Blocking.parkNanos(Blocking.java:157)
at com.oracle.common.base.ConcurrentNotifier.park(ConcurrentNotifier.java:149)
at com.oracle.common.base.ConcurrentNotifier.await(ConcurrentNotifier.java:66)
at com.tangosol.coherence.component.util.Daemon.onWait(Daemon.CDB:13)
at com.tangosol.coherence.component.util.DaemonPool$Daemon.onWait(DaemonPool.CDB:3)
at com.tangosol.coherence.component.util.daemon.queueProcessor.service.Grid$DaemonPool$Daemon.onWait(Grid.CDB:3)
at com.tangosol.coherence.component.util.Daemon.run(Daemon.CDB:43)
at java.lang.Thread.run(Thread.java:748)
"GTMTradeDataServiceWorker:8" id=81 State:TIMED_WAITING
at sun.misc.Unsafe.park(Native Method)
- waiting on com.oracle.common.util.ConcurrentAssociationPile@180c7ea7
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at com.oracle.common.base.Blocking.parkNanos(Blocking.java:157)
at com.oracle.common.base.ConcurrentNotifier.park(ConcurrentNotifier.java:149)
at com.oracle.common.base.ConcurrentNotifier.await(ConcurrentNotifier.java:66)
at com.tangosol.coherence.component.util.Daemon.onWait(Daemon.CDB:13)
at com.tangosol.coherence.component.util.DaemonPool$Daemon.onWait(DaemonPool.CDB:3)
at com.tangosol.coherence.component.util.daemon.queueProcessor.service.Grid$DaemonPool$Daemon.onWait(Grid.CDB:3)
at com.tangosol.coherence.component.util.Daemon.run(Daemon.CDB:43)
at java.lang.Thread.run(Thread.java:748)
"GTMTradeDataServiceWorker:7" id=80 State:TIMED_WAITING
at sun.misc.Unsafe.park(Native Method)
- waiting on com.oracle.common.util.ConcurrentAssociationPile@4ac7c686
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at com.oracle.common.base.Blocking.parkNanos(Blocking.java:157)
at com.oracle.common.base.ConcurrentNotifier.park(ConcurrentNotifier.java:149)
at com.oracle.common.base.ConcurrentNotifier.await(ConcurrentNotifier.java:66)
at com.tangosol.coherence.component.util.Daemon.onWait(Daemon.CDB:13)
at com.tangosol.coherence.component.util.DaemonPool$Daemon.onWait(DaemonPool.CDB:3)
at com.tangosol.coherence.component.util.daemon.queueProcessor.service.Grid$DaemonPool$Daemon.onWait(Grid.CDB:3)
at com.tangosol.coherence.component.util.Daemon.run(Daemon.CDB:43)
at java.lang.Thread.run(Thread.java:748)
"GTMTradeDataServiceWorker:6" id=79 State:TIMED_WAITING
at sun.misc.Unsafe.park(Native Method)
- waiting on com.oracle.common.util.ConcurrentAssociationPile@13a0b9b1
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at com.oracle.common.base.Blocking.parkNanos(Blocking.java:157)
at com.oracle.common.base.ConcurrentNotifier.park(ConcurrentNotifier.java:149)
at com.oracle.common.base.ConcurrentNotifier.await(ConcurrentNotifier.java:66)
at com.tangosol.coherence.component.util.Daemon.onWait(Daemon.CDB:13)
at com.tangosol.coherence.component.util.DaemonPool$Daemon.onWait(DaemonPool.CDB:3)
at com.tangosol.coherence.component.util.daemon.queueProcessor.service.Grid$DaemonPool$Daemon.onWait(Grid.CDB:3)
at com.tangosol.coherence.component.util.Daemon.run(Daemon.CDB:43)
at java.lang.Thread.run(Thread.java:748)
"GTMTradeDataServiceWorker:5" id=78 State:TIMED_WAITING
at sun.misc.Unsafe.park(Native Method)
- waiting on com.oracle.common.util.ConcurrentAssociationPile@1d0a5d0f
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at com.oracle.common.base.Blocking.parkNanos(Blocking.java:157)
at com.oracle.common.base.ConcurrentNotifier.park(ConcurrentNotifier.java:149)
at com.oracle.common.base.ConcurrentNotifier.await(ConcurrentNotifier.java:66)
at com.tangosol.coherence.component.util.Daemon.onWait(Daemon.CDB:13)
at com.tangosol.coherence.component.util.DaemonPool$Daemon.onWait(DaemonPool.CDB:3)
at com.tangosol.coherence.component.util.daemon.queueProcessor.service.Grid$DaemonPool$Daemon.onWait(Grid.CDB:3)
at com.tangosol.coherence.component.util.Daemon.run(Daemon.CDB:43)
at java.lang.Thread.run(Thread.java:748)
"GTMTradeDataServiceWorker:4" id=77 State:TIMED_WAITING
at sun.misc.Unsafe.park(Native Method)
- waiting on com.oracle.common.util.ConcurrentAssociationPile@7830805d
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at com.oracle.common.base.Blocking.parkNanos(Blocking.java:157)
at com.oracle.common.base.ConcurrentNotifier.park(ConcurrentNotifier.java:149)
at com.oracle.common.base.ConcurrentNotifier.await(ConcurrentNotifier.java:66)
at com.tangosol.coherence.component.util.Daemon.onWait(Daemon.CDB:13)
at com.tangosol.coherence.component.util.DaemonPool$Daemon.onWait(DaemonPool.CDB:3)
at com.tangosol.coherence.component.util.daemon.queueProcessor.service.Grid$DaemonPool$Daemon.onWait(Grid.CDB:3)
at com.tangosol.coherence.component.util.Daemon.run(Daemon.CDB:43)
at java.lang.Thread.run(Thread.java:748)
"GTMTradeDataServiceWorker:3" id=76 State:TIMED_WAITING
at sun.misc.Unsafe.park(Native Method)
- waiting on com.oracle.common.util.ConcurrentAssociationPile@c379a2b
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at com.oracle.common.base.Blocking.parkNanos(Blocking.java:157)
at com.oracle.common.base.ConcurrentNotifier.park(ConcurrentNotifier.java:149)
at com.oracle.common.base.ConcurrentNotifier.await(ConcurrentNotifier.java:66)
at com.tangosol.coherence.component.util.Daemon.onWait(Daemon.CDB:13)
at com.tangosol.coherence.component.util.DaemonPool$Daemon.onWait(DaemonPool.CDB:3)
at com.tangosol.coherence.component.util.daemon.queueProcessor.service.Grid$DaemonPool$Daemon.onWait(Grid.CDB:3)
at com.tangosol.coherence.component.util.Daemon.run(Daemon.CDB:43)
at java.lang.Thread.run(Thread.java:748)
"GTMTradeDataServiceWorker:2" id=75 State:TIMED_WAITING
at sun.misc.Unsafe.park(Native Method)
- waiting on com.oracle.common.util.ConcurrentAssociationPile@1a4cc72d
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at com.oracle.common.base.Blocking.parkNanos(Blocking.java:157)
at com.oracle.common.base.ConcurrentNotifier.park(ConcurrentNotifier.java:149)
at com.oracle.common.base.ConcurrentNotifier.await(ConcurrentNotifier.java:66)
at com.tangosol.coherence.component.util.Daemon.onWait(Daemon.CDB:13)
at com.tangosol.coherence.component.util.DaemonPool$Daemon.onWait(DaemonPool.CDB:3)
at com.tangosol.coherence.component.util.daemon.queueProcessor.service.Grid$DaemonPool$Daemon.onWait(Grid.CDB:3)
at com.tangosol.coherence.component.util.Daemon.run(Daemon.CDB:43)
at java.lang.Thread.run(Thread.java:748)
"GTMTradeDataServiceWorker:1" id=74 State:TIMED_WAITING
at sun.misc.Unsafe.park(Native Method)
- waiting on com.oracle.common.util.ConcurrentAssociationPile@55e1034e
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at com.oracle.common.base.Blocking.parkNanos(Blocking.java:157)
at com.oracle.common.base.ConcurrentNotifier.park(ConcurrentNotifier.java:149)
at com.oracle.common.base.ConcurrentNotifier.await(ConcurrentNotifier.java:66)
at com.tangosol.coherence.component.util.Daemon.onWait(Daemon.CDB:13)
at com.tangosol.coherence.component.util.DaemonPool$Daemon.onWait(DaemonPool.CDB:3)
at com.tangosol.coherence.component.util.daemon.queueProcessor.service.Grid$DaemonPool$Daemon.onWait(Grid.CDB:3)
at com.tangosol.coherence.component.util.Daemon.run(Daemon.CDB:43)
at java.lang.Thread.run(Thread.java:748)
"GTMTradeDataServiceWorker:0" id=73 State:TIMED_WAITING
at sun.misc.Unsafe.park(Native Method)
- waiting on com.oracle.common.util.ConcurrentAssociationPile@29af53c7
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at com.oracle.common.base.Blocking.parkNanos(Blocking.java:157)
at com.oracle.common.base.ConcurrentNotifier.park(ConcurrentNotifier.java:149)
at com.oracle.common.base.ConcurrentNotifier.await(ConcurrentNotifier.java:66)
at com.tangosol.coherence.component.util.Daemon.onWait(Daemon.CDB:13)
at com.tangosol.coherence.component.util.DaemonPool$Daemon.onWait(DaemonPool.CDB:3)
at com.tangosol.coherence.component.util.daemon.queueProcessor.service.Grid$DaemonPool$Daemon.onWait(Grid.CDB:3)
at com.tangosol.coherence.component.util.Daemon.run(Daemon.CDB:43)
at java.lang.Thread.run(Thread.java:748)
"DistributedCache:GTMTradeDataService:EventDispatcher" id=72 State:TIMED_WAITING
at java.lang.Object.wait(Native Method)
- waiting on com.tangosol.coherence.component.util.daemon.queueProcessor.Service$EventDispatcher$Queue@5c259cee
at java.lang.Object.wait(Object.java:460)
at com.oracle.common.base.Blocking.wait(Blocking.java:97)
at com.oracle.common.base.Blocking.wait(Blocking.java:80)
at com.tangosol.coherence.component.util.Queue.await(Queue.CDB:9)
at com.tangosol.coherence.component.util.Daemon.onWait(Daemon.CDB:13)
at com.tangosol.coherence.component.util.daemon.QueueProcessor.onWait(QueueProcessor.CDB:3)
at com.tangosol.coherence.component.util.daemon.queueProcessor.Service$EventDispatcher.onWait(Service.CDB:7)
at com.tangosol.coherence.component.util.Daemon.run(Daemon.CDB:43)
at java.lang.Thread.run(Thread.java:748)
"DistributedCache:GTMTradeDataService" id=71 State:TIMED_WAITING
at sun.misc.Unsafe.park(Native Method)
- waiting on com.oracle.common.base.SingleWaiterMultiNotifier@15c472ff
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at com.oracle.common.base.Blocking.parkNanos(Blocking.java:157)
at com.oracle.common.base.SingleWaiterMultiNotifier.await(SingleWaiterMultiNotifier.java:54)
at com.tangosol.coherence.component.util.queue.SingleConsumerQueue.await(SingleConsumerQueue.CDB:1)
at com.tangosol.coherence.component.util.Daemon.onWait(Daemon.CDB:13)
at com.tangosol.coherence.component.util.daemon.QueueProcessor.onWait(QueueProcessor.CDB:3)
at com.tangosol.coherence.component.util.daemon.queueProcessor.service.Grid.onWait(Grid.CDB:6)
at com.tangosol.coherence.component.util.Daemon.run(Daemon.CDB:43)
at java.lang.Thread.run(Thread.java:748)
"GTMRefDataServiceWorker:23" id=70 State:TIMED_WAITING
at sun.misc.Unsafe.park(Native Method)
- waiting on com.oracle.common.util.ConcurrentAssociationPile@64a358bb
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at com.oracle.common.base.Blocking.parkNanos(Blocking.java:157)
at com.oracle.common.base.ConcurrentNotifier.park(ConcurrentNotifier.java:149)
at com.oracle.common.base.ConcurrentNotifier.await(ConcurrentNotifier.java:66)
at com.tangosol.coherence.component.util.Daemon.onWait(Daemon.CDB:13)
at com.tangosol.coherence.component.util.DaemonPool$Daemon.onWait(DaemonPool.CDB:3)
at com.tangosol.coherence.component.util.daemon.queueProcessor.service.Grid$DaemonPool$Daemon.onWait(Grid.CDB:3)
at com.tangosol.coherence.component.util.Daemon.run(Daemon.CDB:43)
at java.lang.Thread.run(Thread.java:748)
"GTMRefDataServiceWorker:22" id=69 State:TIMED_WAITING
at sun.misc.Unsafe.park(Native Method)
- waiting on com.oracle.common.util.ConcurrentAssociationPile@1fa1ebce
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at com.oracle.common.base.Blocking.parkNanos(Blocking.java:157)
at com.oracle.common.base.ConcurrentNotifier.park(ConcurrentNotifier.java:149)
at com.oracle.common.base.ConcurrentNotifier.await(ConcurrentNotifier.java:66)
at com.tangosol.coherence.component.util.Daemon.onWait(Daemon.CDB:13)
at com.tangosol.coherence.component.util.DaemonPool$Daemon.onWait(DaemonPool.CDB:3)
at com.tangosol.coherence.component.util.daemon.queueProcessor.service.Grid$DaemonPool$Daemon.onWait(Grid.CDB:3)
at com.tangosol.coherence.component.util.Daemon.run(Daemon.CDB:43)
at java.lang.Thread.run(Thread.java:748)
"GTMRefDataServiceWorker:21" id=68 State:TIMED_WAITING
at sun.misc.Unsafe.park(Native Method)
- waiting on com.oracle.common.util.ConcurrentAssociationPile@6f13d638
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at com.oracle.common.base.Blocking.parkNanos(Blocking.java:157)
at com.oracle.common.base.ConcurrentNotifier.park(ConcurrentNotifier.java:149)
at com.oracle.common.base.ConcurrentNotifier.await(ConcurrentNotifier.java:66)
at com.tangosol.coherence.component.util.Daemon.onWait(Daemon.CDB:13)
at com.tangosol.coherence.component.util.DaemonPool$Daemon.onWait(DaemonPool.CDB:3)
at com.tangosol.coherence.component.util.daemon.queueProcessor.service.Grid$DaemonPool$Daemon.onWait(Grid.CDB:3)
at com.tangosol.coherence.component.util.Daemon.run(Daemon.CDB:43)
at java.lang.Thread.run(Thread.java:748)
"GTMRefDataServiceWorker:20" id=67 State:TIMED_WAITING
at sun.misc.Unsafe.park(Native Method)
- waiting on com.oracle.common.util.ConcurrentAssociationPile@660410c3
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at com.oracle.common.base.Blocking.parkNanos(Blocking.java:157)
at com.oracle.common.base.ConcurrentNotifier.park(ConcurrentNotifier.java:149)
at com.oracle.common.base.ConcurrentNotifier.await(ConcurrentNotifier.java:66)
at com.tangosol.coherence.component.util.Daemon.onWait(Daemon.CDB:13)
at com.tangosol.coherence.component.util.DaemonPool$Daemon.onWait(DaemonPool.CDB:3)
at com.tangosol.coherence.component.util.daemon.queueProcessor.service.Grid$DaemonPool$Daemon.onWait(Grid.CDB:3)
at com.tangosol.coherence.component.util.Daemon.run(Daemon.CDB:43)
at java.lang.Thread.run(Thread.java:748)
"GTMRefDataServiceWorker:19" id=66 State:WAITING
at sun.misc.Unsafe.park(Native Method)
- waiting on com.oracle.common.base.SingleWaiterCooperativeNotifier@2ce97d95
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at com.oracle.common.base.Blocking.parkNanos(Blocking.java:153)
at com.oracle.common.base.Blocking.park(Blocking.java:137)
at com.oracle.common.base.SingleWaiterMultiNotifier.await(SingleWaiterMultiNotifier.java:50)
at com.oracle.common.base.SingleWaiterCooperativeNotifier.await(SingleWaiterCooperativeNotifier.java:49)
at com.tangosol.coherence.component.net.Poll.waitCompletion(Poll.CDB:6)
at com.tangosol.coherence.component.util.daemon.queueProcessor.service.Grid.poll(Grid.CDB:34)
at com.tangosol.coherence.component.util.daemon.queueProcessor.service.Grid.poll(Grid.CDB:1)
at com.tangosol.coherence.component.util.daemon.queueProcessor.service.grid.partitionedService.PartitionedCache$BinaryMap.put(PartitionedCache.CDB:32)
at com.tangosol.coherence.component.util.daemon.queueProcessor.service.grid.partitionedService.PartitionedCache$BinaryMap.put(PartitionedCache.CDB:1)
at com.oracle.common.collections.ConverterCollections$ConverterMap.put(ConverterCollections.java:1549)
at com.tangosol.coherence.component.util.daemon.queueProcessor.service.grid.partitionedService.PartitionedCache$ViewMap.put(PartitionedCache.CDB:1)
at com.tangosol.coherence.component.util.SafeNamedCache.put(SafeNamedCache.CDB:1)
at com.rbc.fi.tm.coherence.event.CodexPayloadFormatInterceptor.process(CodexPayloadFormatInterceptor.java:41)
at com.rbc.fi.tm.coherence.event.AbstractDataTransferEventInterceptor.process(AbstractDataTransferEventInterceptor.java:20)
at com.rbc.fi.tm.coherence.event.AbstractEventInterceptor.onEvent(AbstractEventInterceptor.java:46)
at com.rbc.fi.tm.coherence.event.AbstractEventInterceptor.onEvent(AbstractEventInterceptor.java:18)
at com.tangosol.net.events.internal.NamedEventInterceptor.onEvent(NamedEventInterceptor.java:261)
at com.tangosol.net.events.internal.AbstractEvent.nextInterceptor(AbstractEvent.java:116)
at com.tangosol.net.events.internal.AbstractEvent.dispatch(AbstractEvent.java:154)
at com.tangosol.net.events.internal.AbstractEventDispatcher$4.proceed(AbstractEventDispatcher.java:270)
at com.tangosol.coherence.component.util.daemon.queueProcessor.service.grid.partitionedService.PartitionedCache$EventsHelper.onEntryChanging(PartitionedCache.CDB:22)
at com.tangosol.coherence.component.util.daemon.queueProcessor.service.grid.partitionedService.PartitionedCache$Storage.processInterceptors(PartitionedCache.CDB:13)
at com.tangosol.coherence.component.util.daemon.queueProcessor.service.grid.partitionedService.PartitionedCache$InvocationContext.processInterceptors(PartitionedCache.CDB:44)
at com.tangosol.coherence.component.util.daemon.queueProcessor.service.grid.partitionedService.PartitionedCache$InvocationContext.processInterceptors(PartitionedCache.CDB:1)
at com.tangosol.coherence.component.util.daemon.queueProcessor.service.grid.partitionedService.PartitionedCache$Storage.get(PartitionedCache.CDB:16)
at com.tangosol.coherence.component.util.daemon.queueProcessor.service.grid.partitionedService.PartitionedCache.onGetRequest(PartitionedCache.CDB:33)
at com.tangosol.coherence.component.util.daemon.queueProcessor.service.grid.partitionedService.PartitionedCache$GetRequest.run(PartitionedCache.CDB:1)
at com.tangosol.coherence.component.util.DaemonPool$WrapperTask.run(DaemonPool.CDB:1)
at com.tangosol.coherence.component.util.DaemonPool$WrapperTask.run(DaemonPool.CDB:32)
at com.tangosol.coherence.component.util.daemon.queueProcessor.service.grid.PartitionedService$DaemonPool$WrapperTask.run(PartitionedService.CDB:1)
at com.tangosol.coherence.component.util.DaemonPool$Daemon.onNotify(DaemonPool.CDB:66)
at com.tangosol.coherence.component.util.Daemon.run(Daemon.CDB:54)
at java.lang.Thread.run(Thread.java:748)
"GTMRefDataServiceWorker:18" id=65 State:TIMED_WAITING
at sun.misc.Unsafe.park(Native Method)
- waiting on com.oracle.common.util.ConcurrentAssociationPile@59f612af
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at com.oracle.common.base.Blocking.parkNanos(Blocking.java:157)
at com.oracle.common.base.ConcurrentNotifier.park(ConcurrentNotifier.java:149)
at com.oracle.common.base.ConcurrentNotifier.await(ConcurrentNotifier.java:66)
at com.tangosol.coherence.component.util.Daemon.onWait(Daemon.CDB:13)
at com.tangosol.coherence.component.util.DaemonPool$Daemon.onWait(DaemonPool.CDB:3)
at com.tangosol.coherence.component.util.daemon.queueProcessor.service.Grid$DaemonPool$Daemon.onWait(Grid.CDB:3)
at com.tangosol.coherence.component.util.Daemon.run(Daemon.CDB:43)
at java.lang.Thread.run(Thread.java:748)
"GTMRefDataServiceWorker:17" id=64 State:TIMED_WAITING
at sun.misc.Unsafe.park(Native Method)
- waiting on com.oracle.common.util.ConcurrentAssociationPile@27eeffaa
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at com.oracle.common.base.Blocking.parkNanos(Blocking.java:157)
at com.oracle.common.base.ConcurrentNotifier.park(ConcurrentNotifier.java:149)
at com.oracle.common.base.ConcurrentNotifier.await(ConcurrentNotifier.java:66)
at com.tangosol.coherence.component.util.Daemon.onWait(Daemon.CDB:13)
at com.tangosol.coherence.component.util.DaemonPool$Daemon.onWait(DaemonPool.CDB:3)
at com.tangosol.coherence.component.util.daemon.queueProcessor.service.Grid$DaemonPool$Daemon.onWait(Grid.CDB:3)
at com.tangosol.coherence.component.util.Daemon.run(Daemon.CDB:43)
at java.lang.Thread.run(Thread.java:748)
"GTMRefDataServiceWorker:16" id=63 State:TIMED_WAITING
at sun.misc.Unsafe.park(Native Method)
- waiting on com.oracle.common.util.ConcurrentAssociationPile@37819f02
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at com.oracle.common.base.Blocking.parkNanos(Blocking.java:157)
at com.oracle.common.base.ConcurrentNotifier.park(ConcurrentNotifier.java:149)
at com.oracle.common.base.ConcurrentNotifier.await(ConcurrentNotifier.java:66)
at com.tangosol.coherence.component.util.Daemon.onWait(Daemon.CDB:13)
at com.tangosol.coherence.component.util.DaemonPool$Daemon.onWait(DaemonPool.CDB:3)
at com.tangosol.coherence.component.util.daemon.queueProcessor.service.Grid$DaemonPool$Daemon.onWait(Grid.CDB:3)
at com.tangosol.coherence.component.util.Daemon.run(Daemon.CDB:43)
at java.lang.Thread.run(Thread.java:748)
"GTMRefDataServiceWorker:15" id=62 State:TIMED_WAITING
at sun.misc.Unsafe.park(Native Method)
- waiting on com.oracle.common.util.ConcurrentAssociationPile@1e04eec9
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at com.oracle.common.base.Blocking.parkNanos(Blocking.java:157)
at com.oracle.common.base.ConcurrentNotifier.park(ConcurrentNotifier.java:149)
at com.oracle.common.base.ConcurrentNotifier.await(ConcurrentNotifier.java:66)
at com.tangosol.coherence.component.util.Daemon.onWait(Daemon.CDB:13)
at com.tangosol.coherence.component.util.DaemonPool$Daemon.onWait(DaemonPool.CDB:3)
at com.tangosol.coherence.component.util.daemon.queueProcessor.service.Grid$DaemonPool$Daemon.onWait(Grid.CDB:3)
at com.tangosol.coherence.component.util.Daemon.run(Daemon.CDB:43)
at java.lang.Thread.run(Thread.java:748)
"GTMRefDataServiceWorker:14" id=61 State:WAITING
at sun.misc.Unsafe.park(Native Method)
- waiting on com.oracle.common.base.SingleWaiterCooperativeNotifier@64108599
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at com.oracle.common.base.Blocking.parkNanos(Blocking.java:153)
at com.oracle.common.base.Blocking.park(Blocking.java:137)
at com.oracle.common.base.SingleWaiterMultiNotifier.await(SingleWaiterMultiNotifier.java:50)
at com.oracle.common.base.SingleWaiterCooperativeNotifier.await(SingleWaiterCooperativeNotifier.java:49)
at com.tangosol.coherence.component.net.Poll.waitCompletion(Poll.CDB:6)
at com.tangosol.coherence.component.util.daemon.queueProcessor.service.Grid.poll(Grid.CDB:34)
at com.tangosol.coherence.component.util.daemon.queueProcessor.service.Grid.poll(Grid.CDB:1)
at com.tangosol.coherence.component.util.daemon.queueProcessor.service.grid.partitionedService.PartitionedCache$BinaryMap.get(PartitionedCache.CDB:28)
at com.oracle.common.collections.ConverterCollections$ConverterMap.get(ConverterCollections.java:1539)
at com.tangosol.coherence.component.util.daemon.queueProcessor.service.grid.partitionedService.PartitionedCache$ViewMap.get(PartitionedCache.CDB:1)
at com.tangosol.coherence.component.util.SafeNamedCache.get(SafeNamedCache.CDB:1)
at com.rbc.fi.tm.coherence.cachestore.ShreddedDataCacheLoader.load(ShreddedDataCacheLoader.java:40)
at com.rbc.fi.tm.coherence.cachestore.ShreddedDataCacheLoader.loadAll(ShreddedDataCacheLoader.java:68)
at com.tangosol.net.cache.ReadWriteBackingMap$CacheLoaderCacheStore.loadAll(ReadWriteBackingMap.java:6468)
at com.tangosol.net.cache.ReadWriteBackingMap$CacheStoreWrapper.loadAllInternal(ReadWriteBackingMap.java:5975)
at com.tangosol.net.cache.ReadWriteBackingMap$StoreWrapper.loadAll(ReadWriteBackingMap.java:5053)
at com.tangosol.net.cache.ReadWriteBackingMap.getAll(ReadWriteBackingMap.java:1143)
at com.tangosol.coherence.component.util.daemon.queueProcessor.service.grid.partitionedService.PartitionedCache$Storage.getAll(PartitionedCache.CDB:13)
at com.tangosol.coherence.component.util.daemon.queueProcessor.service.grid.partitionedService.PartitionedCache.onGetAllRequest(PartitionedCache.CDB:59)
at com.tangosol.coherence.component.util.daemon.queueProcessor.service.grid.partitionedService.PartitionedCache$GetAllRequest.run(PartitionedCache.CDB:1)
at com.tangosol.coherence.component.util.DaemonPool$WrapperTask.run(DaemonPool.CDB:1)
at com.tangosol.coherence.component.util.DaemonPool$WrapperTask.run(DaemonPool.CDB:32)
at com.tangosol.coherence.component.util.daemon.queueProcessor.service.grid.PartitionedService$DaemonPool$WrapperTask.run(PartitionedService.CDB:1)
at com.tangosol.coherence.component.util.DaemonPool$Daemon.onNotify(DaemonPool.CDB:66)
at com.tangosol.coherence.component.util.Daemon.run(Daemon.CDB:54)
at java.lang.Thread.run(Thread.java:748)
"GTMRefDataServiceWorker:13" id=60 State:TIMED_WAITING
at sun.misc.Unsafe.park(Native Method)
- waiting on com.oracle.common.util.ConcurrentAssociationPile@2fd82cd3
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at com.oracle.common.base.Blocking.parkNanos(Blocking.java:157)
at com.oracle.common.base.ConcurrentNotifier.park(ConcurrentNotifier.java:149)
at com.oracle.common.base.ConcurrentNotifier.await(ConcurrentNotifier.java:66)
at com.tangosol.coherence.component.util.Daemon.onWait(Daemon.CDB:13)
at com.tangosol.coherence.component.util.DaemonPool$Daemon.onWait(DaemonPool.CDB:3)
at com.tangosol.coherence.component.util.daemon.queueProcessor.service.Grid$DaemonPool$Daemon.onWait(Grid.CDB:3)
at com.tangosol.coherence.component.util.Daemon.run(Daemon.CDB:43)
at java.lang.Thread.run(Thread.java:748)