ORM, crash under load [solved]

We are having a serious problem with one of our sites. It started to crash after restarting within some hours/some days. The traffic is around 10 req/s. It seems to be related to hibernate session handling, but I don’t have a clue what happens exactly and why. The site technically does not crash, lucee still responds but each request which runs and calls CBHelper.prepareUIRequest() runs into a timeout
For me it looks like that Lucee can’t handle too many Orm object.new() calls?!
Anybody experienced this before, any ideas?

(it runs on Lucee 5.3.2.77 and Contentbox 4.0.0+560)

Here comes the stack trace:

java.lang.Object.wait(Native Method)
lucee.commons.io.SystemUtil.wait(SystemUtil.java:680)
lucee.runtime.db.DatasourceConnectionPool.getDatasourceConnection(DatasourceConnectionPool.java:73)
sun.reflect.GeneratedMethodAccessor68.invoke(Unknown Source)
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
java.lang.reflect.Method.invoke(Method.java:498)
org.lucee.extension.orm.hibernate.CommonUtil.getDatasourceConnection(CommonUtil.java:710)
org.lucee.extension.orm.hibernate.HibernateORMSession.<init>(HibernateORMSession.java:63)
org.lucee.extension.orm.hibernate.HibernateORMEngine.createSession(HibernateORMEngine.java:77)
lucee.runtime.PageContextImpl.getORMSession(PageContextImpl.java:3351)
lucee.runtime.orm.ORMUtil.getSession(ORMUtil.java:58)
lucee.runtime.orm.ORMUtil.getSession(ORMUtil.java:54)
lucee.runtime.functions.orm.EntityNew.call(EntityNew.java:43)
lucee.runtime.functions.orm.EntityNew.call(EntityNew.java:39)
models.baseormservice_cfc$cf.udfCall1(/cborm/models/BaseORMService.cfc:332)
models.baseormservice_cfc$cf.udfCall(/cborm/models/BaseORMService.cfc)
lucee.runtime.type.UDFImpl.implementation(UDFImpl.java:106)
lucee.runtime.type.UDFImpl._call(UDFImpl.java:342)
lucee.runtime.type.UDFImpl.callWithNamedValues(UDFImpl.java:205)
lucee.runtime.ComponentImpl._call(ComponentImpl.java:683)
lucee.runtime.ComponentImpl._call(ComponentImpl.java:578)
lucee.runtime.SuperComponent.callWithNamedValues(SuperComponent.java:93)
lucee.runtime.util.VariableUtilImpl.callFunctionWithNamedValues(VariableUtilImpl.java:846)
lucee.runtime.PageContextImpl.getFunctionWithNamedValues(PageContextImpl.java:1726)
models.virtualentityservice_cfc$cf.udfCall1(/cborm/models/VirtualEntityService.cfc:114)
models.virtualentityservice_cfc$cf.udfCall(/cborm/models/VirtualEntityService.cfc)
lucee.runtime.type.UDFImpl.implementation(UDFImpl.java:106)
lucee.runtime.type.UDFImpl._call(UDFImpl.java:342)
lucee.runtime.type.UDFImpl.call(UDFImpl.java:215)
lucee.runtime.ComponentImpl._call(ComponentImpl.java:682)
lucee.runtime.ComponentImpl._call(ComponentImpl.java:570)
lucee.runtime.ComponentImpl.call(ComponentImpl.java:1918)
lucee.runtime.util.VariableUtilImpl.callFunctionWithoutNamedValues(VariableUtilImpl.java:768)
lucee.runtime.PageContextImpl.getFunction(PageContextImpl.java:1707)
models.security.securityservice_cfc$cf.udfCall1(/contentbox/models/security/SecurityService.cfc:124)
models.security.securityservice_cfc$cf.udfCall(/contentbox/models/security/SecurityService.cfc)
lucee.runtime.type.UDFImpl.implementation(UDFImpl.java:106)
lucee.runtime.type.UDFImpl._call(UDFImpl.java:342)
lucee.runtime.type.UDFImpl.call(UDFImpl.java:215)
lucee.runtime.ComponentImpl._call(ComponentImpl.java:682)
lucee.runtime.ComponentImpl._call(ComponentImpl.java:570)
lucee.runtime.ComponentImpl.call(ComponentImpl.java:1918)
lucee.runtime.util.VariableUtilImpl.callFunctionWithoutNamedValues(VariableUtilImpl.java:768)
lucee.runtime.PageContextImpl.getFunction(PageContextImpl.java:1707)
models.system.cbhelper_cfc$cf.udfCall4(/contentbox/models/system/CBHelper.cfc:366)
models.system.cbhelper_cfc$cf.udfCall(/contentbox/models/system/CBHelper.cfc)
lucee.runtime.type.UDFImpl.implementation(UDFImpl.java:106)
lucee.runtime.type.UDFImpl._call(UDFImpl.java:342)
lucee.runtime.type.UDFImpl.call(UDFImpl.java:215)
lucee.runtime.ComponentImpl._call(ComponentImpl.java:682)
lucee.runtime.ComponentImpl._call(ComponentImpl.java:570)
lucee.runtime.ComponentImpl.call(ComponentImpl.java:1918)
lucee.runtime.util.VariableUtilImpl.callFunctionWithoutNamedValues(VariableUtilImpl.java:768)
lucee.runtime.PageContextImpl.getFunction(PageContextImpl.java:1707)
modules.theme.interceptors.layout_cfc$cf.udfCall(/cbapp/modules/theme/interceptors/Layout.cfc:50)
lucee.runtime.type.UDFImpl.implementation(UDFImpl.java:106)
lucee.runtime.type.UDFImpl._call(UDFImpl.java:342)
lucee.runtime.type.UDFImpl.callWithNamedValues(UDFImpl.java:205)
lucee.runtime.ComponentImpl._call(ComponentImpl.java:683)
lucee.runtime.ComponentImpl._call(ComponentImpl.java:570)
lucee.runtime.ComponentImpl.callWithNamedValues(ComponentImpl.java:1937)
lucee.runtime.util.VariableUtilImpl.callFunctionWithNamedValues(VariableUtilImpl.java:880)
lucee.runtime.functions.dynamicEvaluation.Invoke.call(Invoke.java:49)
system.web.context.interceptorstate_cfc$cf.udfCall2(/coldbox/system/web/context/InterceptorState.cfc:445)
system.web.context.interceptorstate_cfc$cf.udfCall(/coldbox/system/web/context/InterceptorState.cfc)
lucee.runtime.type.UDFImpl.implementation(UDFImpl.java:106)
lucee.runtime.type.UDFImpl._call(UDFImpl.java:342)
lucee.runtime.type.UDFImpl.callWithNamedValues(UDFImpl.java:205)
lucee.runtime.type.scope.UndefinedImpl.callWithNamedValues(UndefinedImpl.java:778)
lucee.runtime.util.VariableUtilImpl.callFunctionWithNamedValues(VariableUtilImpl.java:846)
lucee.runtime.PageContextImpl.getFunctionWithNamedValues(PageContextImpl.java:1726)
system.web.context.interceptorstate_cfc$cf.udfCall1(/coldbox/system/web/context/InterceptorState.cfc:314)
system.web.context.interceptorstate_cfc$cf.udfCall(/coldbox/system/web/context/InterceptorState.cfc)
lucee.runtime.type.UDFImpl.implementation(UDFImpl.java:106)
lucee.runtime.type.UDFImpl._call(UDFImpl.java:342)
lucee.runtime.type.UDFImpl.callWithNamedValues(UDFImpl.java:205)
lucee.runtime.type.scope.UndefinedImpl.callWithNamedValues(UndefinedImpl.java:778)
lucee.runtime.util.VariableUtilImpl.callFunctionWithNamedValues(VariableUtilImpl.java:846)
lucee.runtime.PageContextImpl.getFunctionWithNamedValues(PageContextImpl.java:1726)
system.web.context.interceptorstate_cfc$cf.udfCall1(/coldbox/system/web/context/InterceptorState.cfc:140)
system.web.context.interceptorstate_cfc$cf.udfCall(/coldbox/system/web/context/InterceptorState.cfc)
lucee.runtime.type.UDFImpl.implementation(UDFImpl.java:106)
lucee.runtime.type.UDFImpl._call(UDFImpl.java:342)
lucee.runtime.type.UDFImpl.callWithNamedValues(UDFImpl.java:205)
lucee.runtime.ComponentImpl._call(ComponentImpl.java:683)
lucee.runtime.ComponentImpl._call(ComponentImpl.java:570)
lucee.runtime.ComponentImpl.callWithNamedValues(ComponentImpl.java:1937)
lucee.runtime.util.VariableUtilImpl.callFunctionWithNamedValues(VariableUtilImpl.java:846)
lucee.runtime.PageContextImpl.getFunctionWithNamedValues(PageContextImpl.java:1726)
system.web.services.interceptorservice_cfc$cf.udfCall1(/coldbox/system/web/services/InterceptorService.cfc:154)
system.web.services.interceptorservice_cfc$cf.udfCall(/coldbox/system/web/services/InterceptorService.cfc)
lucee.runtime.type.UDFImpl.implementation(UDFImpl.java:106)
lucee.runtime.type.UDFImpl._call(UDFImpl.java:342)
lucee.runtime.type.UDFImpl.call(UDFImpl.java:215)
lucee.runtime.ComponentImpl._call(ComponentImpl.java:682)
lucee.runtime.ComponentImpl._call(ComponentImpl.java:570)
lucee.runtime.ComponentImpl.call(ComponentImpl.java:1918)
lucee.runtime.util.VariableUtilImpl.callFunctionWithoutNamedValues(VariableUtilImpl.java:768)
lucee.runtime.PageContextImpl.getFunction(PageContextImpl.java:1707)
coldbox.system.bootstrap_cfc$cf.udfCall1(/coldbox/system/Bootstrap.cfc:204)
coldbox.system.bootstrap_cfc$cf.udfCall(/coldbox/system/Bootstrap.cfc)
lucee.runtime.type.UDFImpl.implementation(UDFImpl.java:106)
lucee.runtime.type.UDFImpl._call(UDFImpl.java:342)
lucee.runtime.type.UDFImpl.call(UDFImpl.java:215)
lucee.runtime.type.scope.UndefinedImpl.call(UndefinedImpl.java:765)
lucee.runtime.util.VariableUtilImpl.callFunctionWithoutNamedValues(VariableUtilImpl.java:768)
lucee.runtime.PageContextImpl.getFunction(PageContextImpl.java:1707)
coldbox.system.bootstrap_cfc$cf.udfCall1(/coldbox/system/Bootstrap.cfc:467)
coldbox.system.bootstrap_cfc$cf.udfCall(/coldbox/system/Bootstrap.cfc)
lucee.runtime.type.UDFImpl.implementation(UDFImpl.java:106)
lucee.runtime.type.UDFImpl._call(UDFImpl.java:342)
lucee.runtime.type.UDFImpl.call(UDFImpl.java:215)
lucee.runtime.ComponentImpl._call(ComponentImpl.java:682)
lucee.runtime.ComponentImpl._call(ComponentImpl.java:570)
lucee.runtime.ComponentImpl.call(ComponentImpl.java:1918)
lucee.runtime.util.VariableUtilImpl.callFunctionWithoutNamedValues(VariableUtilImpl.java:768)
lucee.runtime.PageContextImpl.getFunction(PageContextImpl.java:1707)
application_cfc$cf.udfCall(/Application.cfc:179)
lucee.runtime.type.UDFImpl.implementation(UDFImpl.java:106)
lucee.runtime.type.UDFImpl._call(UDFImpl.java:342)
lucee.runtime.type.UDFImpl.call(UDFImpl.java:215)
lucee.runtime.ComponentImpl._call(ComponentImpl.java:682)
lucee.runtime.ComponentImpl._call(ComponentImpl.java:570)
lucee.runtime.ComponentImpl.call(ComponentImpl.java:1918)
lucee.runtime.listener.ModernAppListener.call(ModernAppListener.java:436)
lucee.runtime.listener.ModernAppListener._onRequest(ModernAppListener.java:132)
lucee.runtime.listener.MixedAppListener.onRequest(MixedAppListener.java:42)
lucee.runtime.PageContextImpl.execute(PageContextImpl.java:2409)
lucee.runtime.PageContextImpl._execute(PageContextImpl.java:2399)
lucee.runtime.PageContextImpl.executeCFML(PageContextImpl.java:2374)
lucee.runtime.engine.Request.exe(Request.java:43)
lucee.runtime.engine.CFMLEngineImpl._service(CFMLEngineImpl.java:1037)
lucee.runtime.engine.CFMLEngineImpl.serviceCFML(CFMLEngineImpl.java:983)
lucee.loader.engine.CFMLEngineWrapper.serviceCFML(CFMLEngineWrapper.java:102)
lucee.loader.servlet.CFMLServlet.service(CFMLServlet.java:51)
javax.servlet.http.HttpServlet.service(HttpServlet.java:742)
org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:231)
org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
org.apache.tomcat.websocket.server.WsFilter.doFilter(WsFilter.java:52)
org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)
org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
sun.reflect.GeneratedMethodAccessor79.invoke(Unknown Source)
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
java.lang.reflect.Method.invoke(Method.java:498)
com.intergral.fusionreactor.j2ee.filterchain.WrappedFilterChain.doFilter(WrappedFilterChain.java:134)
com.intergral.fusionreactor.j2ee.filter.FusionReactorRequestHandler.doNext(FusionReactorRequestHandler.java:764)
com.intergral.fusionreactor.j2ee.filter.FusionReactorRequestHandler.doHttpServletRequest(FusionReactorRequestHandler.java:344)
com.intergral.fusionreactor.j2ee.filter.FusionReactorRequestHandler.doFusionRequest(FusionReactorRequestHandler.java:207)
com.intergral.fusionreactor.j2ee.filter.FusionReactorRequestHandler.handle(FusionReactorRequestHandler.java:801)
com.intergral.fusionreactor.j2ee.filter.FusionReactorCoreFilter.doFilter(FusionReactorCoreFilter.java:36)
sun.reflect.GeneratedMethodAccessor78.invoke(Unknown Source)
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
java.lang.reflect.Method.invoke(Method.java:498)
com.intergral.fusionreactor.j2ee.filterchain.WrappedFilterChain.doFilter(WrappedFilterChain.java:71)
sun.reflect.GeneratedMethodAccessor77.invoke(Unknown Source)
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
java.lang.reflect.Method.invoke(Method.java:498)
com.intergral.fusionreactor.agent.filter.FusionReactorStaticFilter.doFilter(FusionReactorStaticFilter.java:54)
com.intergral.fusionreactor.agent.pointcuts.NewFilterChainPointCut$1.invoke(NewFilterChainPointCut.java:41)
org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java)
org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:198)
org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:96)
org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:496)
org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:140)
org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:81)
org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:87)
org.apache.catalina.valves.RemoteIpValve.invoke(RemoteIpValve.java:677)
org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:342)
org.apache.coyote.http11.Http11Processor.service(Http11Processor.java:803)
org.apache.coyote.AbstractProcessorLight.process(AbstractProcessorLight.java:66)
org.apache.coyote.AbstractProtocol$ConnectionHandler.process(AbstractProtocol.java:790)
org.apache.tomcat.util.net.NioEndpoint$SocketProcessor.doRun(NioEndpoint.java:1468)
org.apache.tomcat.util.net.SocketProcessorBase.run(SocketProcessorBase.java:49)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61)
java.lang.Thread.run(Thread.java:748)

We finally found the reason, the datasource had a connection limit of 100. After increasing it everything turned back to normal

1 Like