I have a 2 node cluster, each node running 2 App Servers. I have a third node on which the Rational License Server runs. The cluster has been running for 2 weeks with no restarts. But now, only one of the nodes in the cluster appears to be able to communicate with the license server. The other node is unable to communicate with the license server OR is unable to get a license (I am not sure which)

Any pointers on how to recover from these errors?

Here are the kind of exceptions I see in the ramDebug.log & SystemOut.log

java.lang.NullPointerException
at com.ibm.ram.license.LicenseManager$ServerDown.process(LicenseManager.java:2233)
at com.ibm.ram.license.LicenseManager$3.messageReceived(LicenseManager.java:1808)
at com.ibm.ram.license.LicenseManager$3.messageReceived(LicenseManager.java:1)
at com.ibm.ram.cache.NotificationManager$Notifier.messageReceived(NotificationManager.java:841)
at com.ibm.ram.cache.WebsphereNotificationManager$WebsphereNotifier.processMessage(WebsphereNotificationManager.java:290)
at sun.reflect.GeneratedMethodAccessor119.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:618)
at com.ibm.websphere.asynchbeans.JMXBroadcaster$JMXListenerTriple.run(JMXBroadcaster.java:152)
at com.ibm.ws.asynchbeans.J2EEContext$RunProxy.run(J2EEContext.java:261)
at java.security.AccessController.doPrivileged(AccessController.java:214)
at javax.security.auth.Subject.doAs(Subject.java:495)
at com.ibm.websphere.security.auth.WSSubject.doAs(WSSubject.java:118)
at com.ibm.ws.asynchbeans.J2EEContext$DoAsProxy.run(J2EEContext.java:328)
at java.security.AccessController.doPrivileged(AccessController.java:241)
at com.ibm.ws.asynchbeans.J2EEContext.run(J2EEContext.java:757)
at com.ibm.websphere.asynchbeans.JMXBroadcaster$JMXListenerTriple.fireEvent(JMXBroadcaster.java:119)
at com.ibm.websphere.asynchbeans.JMXBroadcaster.fireEvent(JMXBroadcaster.java:414)
at com.ibm.websphere.asynchbeans.JMXBroadcaster$TriggerProxy.invoke(JMXBroadcaster.java:348)
at $Proxy34.processMessage(Unknown Source)
at com.ibm.ram.cache.WebsphereNotificationManager$MessageWork.run(WebsphereNotificationManager.java:326)
at com.ibm.ws.asynchbeans.J2EEContext$RunProxy.run(J2EEContext.java:261)
at java.security.AccessController.doPrivileged(AccessController.java:214)
at javax.security.auth.Subject.doAs(Subject.java:495)
at com.ibm.websphere.security.auth.WSSubject.doAs(WSSubject.java:118)
at com.ibm.ws.asynchbeans.J2EEContext$DoAsProxy.run(J2EEContext.java:328)
at java.security.AccessController.doPrivileged(AccessController.java:241)
at com.ibm.ws.asynchbeans.J2EEContext.run(J2EEContext.java:757)
at com.ibm.ws.asynchbeans.WorkWithExecutionContextImpl.go(WorkWithExecutionContextImpl.java:218)
at com.ibm.ws.asynchbeans.ABWorkItemImpl.run(ABWorkItemImpl.java:158)
at com.ibm.ws.util.ThreadPool$Worker.run(ThreadPool.java:1497)




java.lang.RuntimeException: Time out on request for license. Server not responding.
at com.ibm.ws.webservices.engine.WebServicesFault.makeFault(WebServicesFault.java:192)
at com.ibm.ws.webservices.engine.dispatchers.java.SessionDispatcher.invoke(SessionDispatcher.java:213)
at com.ibm.ws.webservices.engine.PivotHandlerWrapper.invoke(PivotHandlerWrapper.java:248)
at com.ibm.ws.webservices.engine.handlers.jaxrpc.JAXRPCHandler.invoke(JAXRPCHandler.java:152)
at com.ibm.ws.webservices.engine.handlers.WrappedHandler.invoke(WrappedHandler.java:64)
at com.ibm.ws.webservices.engine.PivotHandlerWrapper.invoke(PivotHandlerWrapper.java:248)
at com.ibm.ws.webservices.engine.PivotHandlerWrapper.invoke(PivotHandlerWrapper.java:248)
at com.ibm.ws.webservices.engine.WebServicesEngine.invoke(WebServicesEngine.java:336)
at com.ibm.ws.webservices.engine.transport.http.WebServicesServlet.doPost(WebServicesServlet.java:840)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:763)
at com.ibm.ws.webservices.engine.transport.http.WebServicesServletBase.service(WebServicesServletBase.java:341)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:856)
at com.ibm.ws.webcontainer.servlet.ServletWrapper.service(ServletWrapper.java:1152)
at com.ibm.ws.webcontainer.servlet.ServletWrapper.service(ServletWrapper.java:1087)
at com.ibm.ws.webcontainer.filter.WebAppFilterChain.doFilter(WebAppFilterChain.java:118)
at com.ibm.ws.webcontainer.filter.WebAppFilterChain._doFilter(WebAppFilterChain.java:87)
at com.ibm.ws.webcontainer.filter.WebAppFilterManager.doFilter(WebAppFilterManager.java:837)
at com.ibm.ws.webcontainer.filter.WebAppFilterManager.doFilter(WebAppFilterManager.java:680)
at com.ibm.ws.webcontainer.servlet.ServletWrapper.handleRequest(ServletWrapper.java:588)
at com.ibm.ws.wswebcontainer.servlet.ServletWrapper.handleRequest(ServletWrapper.java:525)
at com.ibm.ws.webcontainer.webapp.WebAppRequestDispatcher.forward(WebAppRequestDispatcher.java:321)
at com.ibm.ram.repository.web.ws.RAMSecure.doPost(RAMSecure.java:48)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:763)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:856)
at com.ibm.ws.webcontainer.servlet.ServletWrapper.service(ServletWrapper.java:1152)
at com.ibm.ws.webcontainer.servlet.ServletWrapper.service(ServletWrapper.java:1087)
at com.ibm.ws.webcontainer.filter.WebAppFilterChain.doFilter(WebAppFilterChain.java:145)
at com.ibm.ram.repository.web.RAMWSServletFilter.doFilter(RAMWSServletFilter.java:411)
at com.ibm.ws.webcontainer.filter.FilterInstanceWrapper.doFilter(FilterInstanceWrapper.java:190)
at com.ibm.ws.webcontainer.filter.WebAppFilterChain.doFilter(WebAppFilterChain.java:130)
at com.ibm.ws.webcontainer.filter.WebAppFilterChain._doFilter(WebAppFilterChain.java:87)
at com.ibm.ws.webcontainer.filter.WebAppFilterManager.doFilter(WebAppFilterManager.java:837)
at com.ibm.ws.webcontainer.filter.WebAppFilterManager.doFilter(WebAppFilterManager.java:680)
at com.ibm.ws.webcontainer.servlet.ServletWrapper.handleRequest(ServletWrapper.java:588)
at com.ibm.ws.wswebcontainer.servlet.ServletWrapper.handleRequest(ServletWrapper.java:525)
at com.ibm.ws.webcontainer.servlet.CacheServletWrapper.handleRequest(CacheServletWrapper.java:90)
at com.ibm.ws.webcontainer.WebContainer.handleRequest(WebContainer.java:751)
at com.ibm.ws.wswebcontainer.WebContainer.handleRequest(WebContainer.java:1478)
at com.ibm.ws.webcontainer.channel.WCChannelLink.ready(WCChannelLink.java:126)
at com.ibm.ws.http.channel.inbound.impl.HttpInboundLink.handleDiscrimination(HttpInboundLink.java:458)
at com.ibm.ws.http.channel.inbound.impl.HttpInboundLink.handleNewInformation(HttpInboundLink.java:387)
at com.ibm.ws.http.channel.inbound.impl.HttpInboundLink.ready(HttpInboundLink.java:267)
at com.ibm.ws.tcp.channel.impl.NewConnectionInitialReadCallback.sendToDiscriminators(NewConnectionInitialReadCallback.java:214)
at com.ibm.ws.tcp.channel.impl.NewConnectionInitialReadCallback.complete(NewConnectionInitialReadCallback.java:113)
at com.ibm.ws.tcp.channel.impl.AioReadCompletionListener.futureCompleted(AioReadCompletionListener.java:165)
at com.ibm.io.async.AbstractAsyncFuture.invokeCallback(AbstractAsyncFuture.java:217)
at com.ibm.io.async.AsyncChannelFuture$1.run(AsyncChannelFuture.java:205)
at com.ibm.ws.util.ThreadPool$Worker.run(ThreadPool.java:1497)



Caused by: java.lang.RuntimeException: Time out on request for license. Server not responding.
at com.ibm.ram.license.LicenseManager.allocateLicense(LicenseManager.java:1611)
at com.ibm.ram.repository.security.SecurityManager$PromoteLicensePolicy.allocateLicense(SecurityManager.java:2119)
at com.ibm.ram.repository.security.AbstractUser.allocateLicense(AbstractUser.java:585)
at com.ibm.ram.repository.core.RepositorySessionHandler.allocateLicense(RepositorySessionHandler.java:381)
at com.ibm.ram.repository.assets.AssetManager.assertReadDetails(AssetManager.java:2862)
at com.ibm.ram.repository.assets.AssetManager.primGetAsset(AssetManager.java:1814)
at com.ibm.ram.repository.assets.AssetManager.getAsset(AssetManager.java:1954)
at com.ibm.ram.repository.web.ws.core.RAM1Impl.getAsset(RAM1Impl.java:269)
at com.ibm.ram.repository.web.ws.core.RAM1Impl.getAssetByID(RAM1Impl.java:254)
at com.ibm.ram.repository.web.ws.was.RAM1SoapBindingImpl.getAssetByID(RAM1SoapBindingImpl.java:1302)
at sun.reflect.GeneratedMethodAccessor161.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:618)
at com.ibm.ws.webservices.engine.dispatchers.java.JavaDispatcher.invokeMethod(JavaDispatcher.java:178)
at com.ibm.ws.webservices.engine.dispatchers.java.JavaDispatcher.invokeOperation(JavaDispatcher.java:141)
at com.ibm.ws.webservices.engine.dispatchers.SoapRPCProcessor.processRequestResponse(SoapRPCProcessor.java:481)
at com.ibm.ws.webservices.engine.dispatchers.SoapRPCProcessor.processMessage(SoapRPCProcessor.java:427)
at com.ibm.ws.webservices.engine.dispatchers.BasicDispatcher.processMessage(BasicDispatcher.java:134)
at com.ibm.ws.webservices.engine.dispatchers.java.SessionDispatcher.invoke(SessionDispatcher.java:204)
... 46 more