com.ibm.team.reports.service.internal.ReportRestService.postRenderQuery
by testman, 16:26:53:321 running time
java.lang.Object.wait(Native Method)
java.lang.Object.wait(Object.java:231)
java.lang.Thread.join(Thread.java:742)
java.lang.Thread.join(Thread.java:701)
org.eclipse.birt.data.engine.executor.DataSourceQuery.execute(DataSourceQuery.java:697)
org.eclipse.birt.data.engine.impl.PreparedOdaDSQuery$OdaDSQueryExecutor.executeOdiQuery(PreparedOdaDSQuery.java:331)
org.eclipse.birt.data.engine.impl.QueryExecutor.execute(QueryExecutor.java:945)
org.eclipse.birt.data.engine.impl.ServiceForQueryResults.executeQuery(ServiceForQueryResults.java:233)
org.eclipse.birt.data.engine.impl.QueryResults.getResultIterator(QueryResults.java:160)
org.eclipse.birt.report.engine.data.dte.QueryResultSet.(QueryResultSet.java:92)
org.eclipse.birt.report.engine.data.dte.DteDataEngine.doExecuteQuery(DteDataEngine.java:129)
org.eclipse.birt.report.engine.data.dte.AbstractDataEngine.execute(AbstractDataEngine.java:253)
org.eclipse.birt.report.engine.executor.ExecutionContext.executeQuery(ExecutionContext.java:1755)
org.eclipse.birt.report.engine.executor.QueryItemExecutor.executeQuery(QueryItemExecutor.java:77)
org.eclipse.birt.report.engine.executor.DynamicTextItemExecutor.execute(DynamicTextItemExecutor.java:73)
org.eclipse.birt.report.engine.internal.executor.dup.SuppressDuplicateItemExecutor.execute(SuppressDuplicateItemExecutor.java:42)
org.eclipse.birt.report.engine.internal.executor.wrap.WrappedReportItemExecutor.execute(WrappedReportItemExecutor.java:45)
org.eclipse.birt.report.engine.internal.executor.l18n.LocalizedReportItemExecutor.execute(LocalizedReportItemExecutor.java:33)
org.eclipse.birt.report.engine.layout.html.HTMLBlockStackingLM.layoutNodes(HTMLBlockStackingLM.java:63)
org.eclipse.birt.report.engine.layout.html.HTMLPageLM.layout(HTMLPageLM.java:90)
org.eclipse.birt.report.engine.layout.html.HTMLReportLayoutEngine.layout(HTMLReportLayoutEngine.java:101)
org.eclipse.birt.report.engine.api.impl.RunAndRenderTask.doRun(RunAndRenderTask.java:151)
org.eclipse.birt.report.engine.api.impl.RunAndRenderTask.run(RunAndRenderTask.java:72)
com.ibm.team.reports.service.birt.internal.BirtReportEngineService.internalGetRenderedReport(BirtReportEngineService.java:672)
com.ibm.team.reports.service.birt.internal.BirtReportEngineService.getRenderedReport(BirtReportEngineService.java:299)
sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:79)
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
java.lang.reflect.Method.invoke(Method.java:618)
org.eclipse.soda.sat.core.internal.record.ExportProxyServiceRecord.invoke(ExportProxyServiceRecord.java:370)
org.eclipse.soda.sat.core.internal.record.ExportProxyServiceRecord.access$0(ExportProxyServiceRecord.java:356)
org.eclipse.soda.sat.core.internal.record.ExportProxyServiceRecord$ExportedServiceInvocationHandler.invoke(ExportProxyServiceRecord.java:56)
$Proxy371.getRenderedReport(Unknown Source)
com.ibm.team.reports.service.internal.InternalReportService.getRenderedReport(InternalReportService.java:1098)
sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:79)
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
java.lang.reflect.Method.invoke(Method.java:618)
org.eclipse.soda.sat.core.internal.record.ExportProxyServiceRecord.invoke(ExportProxyServiceRecord.java:370)
org.eclipse.soda.sat.core.internal.record.ExportProxyServiceRecord.access$0(ExportProxyServiceRecord.java:356)
org.eclipse.soda.sat.core.internal.record.ExportProxyServiceRecord$ExportedServiceInvocationHandler.invoke(ExportProxyServiceRecord.java:56)
$Proxy214.getRenderedReport(Unknown Source)
com.ibm.team.reports.service.internal.ReportRestService$1.run(ReportRestService.java:421)
com.ibm.team.reports.service.internal.ReportRestService.getRenderedReport(ReportRestService.java:428)
com.ibm.team.reports.service.internal.ReportRestService.getRenderedQuery(ReportRestService.java:361)
com.ibm.team.reports.service.internal.ReportRestService.postRenderQuery(ReportRestService.java:507)
sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:79)
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
java.lang.reflect.Method.invoke(Method.java:618)
org.eclipse.soda.sat.core.internal.record.ExportProxyServiceRecord.invoke(ExportProxyServiceRecord.java:370)
org.eclipse.soda.sat.core.internal.record.ExportProxyServiceRecord.access$0(ExportProxyServiceRecord.java:356)
org.eclipse.soda.sat.core.internal.record.ExportProxyServiceRecord$ExportedServiceInvocationHandler.invoke(ExportProxyServiceRecord.java:56)
$Proxy218.postRenderQuery(Unknown Source)
sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:79)
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
java.lang.reflect.Method.invoke(Method.java:618)
com.ibm.team.repository.servlet.AbstractTeamServerServlet.doModelledRestService(AbstractTeamServerServlet.java:573)
com.ibm.team.repository.servlet.AbstractTeamServerServlet.handleRequest2(AbstractTeamServerServlet.java:1773)
com.ibm.team.repository.servlet.AbstractTeamServerServlet.handleRequest(AbstractTeamServerServlet.java:1644)
com.ibm.team.repository.servlet.AbstractTeamServerServlet.service(AbstractTeamServerServlet.java:1557)
javax.servlet.http.HttpServlet.service(HttpServlet.java:729)
org.eclipse.equinox.http.registry.internal.ServletManager$ServletWrapper.service(ServletManager.java:180)
org.eclipse.equinox.http.servlet.internal.ServletRegistration.handleRequest(ServletRegistration.java:90)
org.eclipse.equinox.http.servlet.internal.ProxyServlet.processAlias(ProxyServlet.java:111)
org.eclipse.equinox.http.servlet.internal.ProxyServlet.service(ProxyServlet.java:75)
javax.servlet.http.HttpServlet.service(HttpServlet.java:729)
org.eclipse.equinox.servletbridge.BridgeServlet.service(BridgeServlet.java:121)
com.ibm.team.repository.server.servletbridge.JazzServlet.service(JazzServlet.java:54)
javax.servlet.http.HttpServlet.service(HttpServlet.java:729)
org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:269)
org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:188)
org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:213)
org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:172)
org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:525)
org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:127)
org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:117)
org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:108)
org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:174)
org.apache.coyote.http11.Http11Processor.process(Http11Processor.java:873)
org.apache.coyote.http11.Http11BaseProtocol$Http11ConnectionHandler.processConnection(Http11BaseProtocol.java:665)
org.apache.tomcat.util.net.PoolTcpEndpoint.processSocket(PoolTcpEndpoint.java:528)
org.apache.tomcat.util.net.LeaderFollowerWorkerThread.runIt(LeaderFollowerWorkerThread.java:81)
org.apache.tomcat.util.threads.ThreadPool$ControlRunnable.run(ThreadPool.java:689)
java.lang.Thread.run(Thread.java:811)

com.ibm.team.datawarehouse.service.internal.InternalDataWarehouseService.read
by testman, 16:26:47:505 running time
org.apache.derby.impl.store.raw.data.StoredPage.readRecordFromArray(Unknown Source)
org.apache.derby.impl.store.raw.data.StoredPage.restoreRecordFromSlot(Unknown Source)
org.apache.derby.impl.store.raw.data.BasePage.fetchFromSlot(Unknown Source)
org.apache.derby.impl.store.access.conglomerate.GenericConglomerateController.fetch(Unknown Source)
org.apache.derby.impl.sql.execute.IndexRowToBaseRowResultSet.getNextRowCore(Unknown Source)
org.apache.derby.impl.sql.execute.ProjectRestrictResultSet.getNextRowCore(Unknown Source)
org.apache.derby.impl.sql.execute.NestedLoopJoinResultSet.getNextRowCore(Unknown Source)
org.apache.derby.impl.sql.execute.NestedLoopJoinResultSet.getNextRowCore(Unknown Source)
org.apache.derby.impl.sql.execute.NestedLoopJoinResultSet.getNextRowCore(Unknown Source)
org.apache.derby.impl.sql.execute.ProjectRestrictResultSet.getNextRowCore(Unknown Source)
org.apache.derby.impl.sql.execute.GroupedAggregateResultSet.getRowFromResultSet(Unknown Source)
org.apache.derby.impl.sql.execute.GroupedAggregateResultSet.getNextRowFromRS(Unknown Source)
org.apache.derby.impl.sql.execute.GroupedAggregateResultSet.loadSorter(Unknown Source)
org.apache.derby.impl.sql.execute.GroupedAggregateResultSet.openCore(Unknown Source)
org.apache.derby.impl.sql.execute.ProjectRestrictResultSet.openCore(Unknown Source)
org.apache.derby.impl.sql.execute.SortResultSet.openCore(Unknown Source)
org.apache.derby.impl.sql.execute.ProjectRestrictResultSet.openCore(Unknown Source)
org.apache.derby.impl.sql.execute.BasicNoPutResultSetImpl.open(Unknown Source)
org.apache.derby.impl.sql.GenericPreparedStatement.execute(Unknown Source)
org.apache.derby.impl.jdbc.EmbedStatement.executeStatement(Unknown Source)
org.apache.derby.impl.jdbc.EmbedStatement.execute(Unknown Source)
org.apache.derby.impl.jdbc.EmbedStatement.executeQuery(Unknown Source)
com.ibm.team.repository.service.internal.db.jdbcwrappers.stat.StatementStatWrapper.executeQuery(StatementStatWrapper.java:199)
com.ibm.team.repository.service.internal.db.jdbcwrappers.errlog.StatementErrLogWrapper.executeQuery(StatementErrLogWrapper.java:191)
com.ibm.team.repository.service.internal.db.jdbcwrappers.leak.StatementLeakWrapper.executeQuery(StatementLeakWrapper.java:78)
com.ibm.team.reports.service.oda.internal.ServiceQuery$2.run(ServiceQuery.java:879)
com.ibm.team.datawarehouse.service.internal.InternalDataWarehouseService$5.run(InternalDataWarehouseService.java:509)
com.ibm.team.repository.service.internal.rdb.RepositoryDatabase$Transaction.run(RepositoryDatabase.java:466)
com.ibm.team.repository.service.internal.rdb.RepositoryDatabase$1.run(RepositoryDatabase.java:292)
com.ibm.team.repository.service.internal.rdb.ConnectionPoolService.withCurrentConnection(ConnectionPoolService.java:331)
sun.reflect.GeneratedMethodAccessor65.invoke(Unknown Source)
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
java.lang.reflect.Method.invoke(Method.java:618)
org.eclipse.soda.sat.core.internal.record.ExportProxyServiceRecord.invoke(ExportProxyServiceRecord.java:370)
org.eclipse.soda.sat.core.internal.record.ExportProxyServiceRecord.access$0(ExportProxyServiceRecord.java:356)
org.eclipse.soda.sat.core.internal.record.ExportProxyServiceRecord$ExportedServiceInvocationHandler.invoke(ExportProxyServiceRecord.java:56)
$Proxy22.withCurrentConnection(Unknown Source)
com.ibm.team.repository.service.internal.rdb.RepositoryDatabase.runTransaction(RepositoryDatabase.java:288)
com.ibm.team.repository.service.internal.rdb.RepositoryDatabase.runInReadUncommittedTransaction(RepositoryDatabase.java:255)
com.ibm.team.repository.service.internal.TransactionService.runReadUncommittedTransaction(TransactionService.java:84)
sun.reflect.GeneratedMethodAccessor124.invoke(Unknown Source)
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
java.lang.reflect.Method.invoke(Method.java:618)
org.eclipse.soda.sat.core.internal.record.ExportProxyServiceRecord.invoke(ExportProxyServiceRecord.java:370)
org.eclipse.soda.sat.core.internal.record.ExportProxyServiceRecord.access$0(ExportProxyServiceRecord.java:356)
org.eclipse.soda.sat.core.internal.record.ExportProxyServiceRecord$ExportedServiceInvocationHandler.invoke(ExportProxyServiceRecord.java:56)
$Proxy44.runReadUncommittedTransaction(Unknown Source)
com.ibm.team.datawarehouse.service.internal.InternalDataWarehouseService.read(InternalDataWarehouseService.java:504)
sun.reflect.GeneratedMethodAccessor426.invoke(Unknown Source)
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
java.lang.reflect.Method.invoke(Method.java:618)
org.eclipse.soda.sat.core.internal.record.ExportProxyServiceRecord.invoke(ExportProxyServiceRecord.java:370)
org.eclipse.soda.sat.core.internal.record.ExportProxyServiceRecord.access$0(ExportProxyServiceRecord.java:356)
org.eclipse.soda.sat.core.internal.record.ExportProxyServiceRecord$ExportedServiceInvocationHandler.invoke(ExportProxyServiceRecord.java:56)
$Proxy82.read(Unknown Source)
com.ibm.team.reports.service.oda.internal.ServiceQuery.getDatawarehouseValues(ServiceQuery.java:866)
com.ibm.team.reports.service.oda.internal.ServiceQuery.getValues(ServiceQuery.java:146)
com.ibm.team.reports.common.oda.generic.GenericQuery.executeQuery(GenericQuery.java:76)
org.eclipse.datatools.connectivity.oda.consumer.helper.OdaQuery.doExecuteQuery(OdaQuery.java:443)
org.eclipse.datatools.connectivity.oda.consumer.helper.OdaQuery.executeQuery(OdaQuery.java:408)
org.eclipse.birt.data.engine.odaconsumer.PreparedStatement.execute(PreparedStatement.java:546)
org.eclipse.birt.data.engine.executor.DataSourceQuery$OdaQueryExecutor.run(DataSourceQuery.java:929)
java.lang.Thread.run(Thread.java:811)

Thing is this happens on our test server, but not on our production server (thankfully). The main difference between the two is the database. The test server has the default trial setup (Derby) while the production server has SQL Server. Also Production is ifix5 while Test is ifix7.

Right now after creating a few simple successful queries I have 8 services running at over 30 minutes with cpu at 100% and I can't even bring up a dashboard now - I get the "Connection error. The server is offline or your internet connection is down" message due to the services maxing out the cpu.

I restored my ifix5 backup and CANNOT make the problem occur. Every time I restore my ifix7 backup I can make the problem occur. This appears to be a serious problem with ifix7. Will it be fixed in ifix8???

CORRECTION!!! The problem is occurring on ifix5 as well. Not sure why it didn't at first. Now when I log in to the web interface and it tries to bring up the dashboard (the default page), the queries on the dashboard bog down the server and I get the "connection error" message in the browser. This is with only one user. The server is completely useless now and I can't figure out what's causing it, nor can I kill the runaway services from within RTC. Kill command on the Tomcat server required. There must be a solution to this or no one would be able to use the application.

This may be a tad early to call, but I may have resolved this issue. While no UI errors were generated in the queries that caused the runaway services, it may be related to an earlier problem I resolved by eliminating invalid enumeration selections. Details posted here:

https://jazz.net/forums/viewtopic.php?t=18609&highlight=&sid=798cae97e261ce667cd1b269e9914494

It would appear the REST API could be more robust in dealing with invalid data.

I need to test for another week to be sure, but we can't break it ever since fixing the invalid selections this morning, which in this case were caused by a deleted Priority setting (oddly enough, the example I used months ago).

Note that I also had to clear temp files to make this fix complete:

rm -r /opt/app/jazz/server/tomcat/temp/*

rm -r /opt/app/jazz/server/tomcat/work