-
Bug
-
Resolution: Incomplete
-
Major
-
None
-
RHEL 5 64 bit, Tomcat 6.0.29
After running for a few hours, the Hudson server start showing the main Java process eating 100% of all 8 CPU cores on the server and it continues to do so endless until I restart it. Thread dumps taken over 11 hours apart show several threads handling the same GET requests for the same builds' consoleText URL:
"Handling GET /hudson/job/Core_Build/9773/consoleText : http-8080-36" daemon prio=10 tid=0x000000005310b800 nid=0x32fe runnable [0x000000004572f000]
java.lang.Thread.State: RUNNABLE
at java.io.FilterInputStream.skip(FilterInputStream.java:125)
at hudson.util.IOUtils.skip(IOUtils.java:68)
at hudson.console.ConsoleNote.skip(ConsoleNote.java:244)
at hudson.console.PlainTextConsoleOutputStream.eol(PlainTextConsoleOutputStream.java:68)
at hudson.console.LineTransformationOutputStream.eol(LineTransformationOutputStream.java:60)
at hudson.console.LineTransformationOutputStream.write(LineTransformationOutputStream.java:56)
at hudson.console.LineTransformationOutputStream.write(LineTransformationOutputStream.java:74)
at org.apache.commons.io.output.ProxyOutputStream.write(ProxyOutputStream.java:70)
at org.apache.commons.io.output.CountingOutputStream.write(CountingOutputStream.java:71)
at org.kohsuke.stapler.framework.io.LargeText.writeLogTo(LargeText.java:145)
at hudson.console.AnnotatedLargeText.writeLogTo(AnnotatedLargeText.java:148)
at hudson.model.Run.doConsoleText(Run.java:1577)
at sun.reflect.GeneratedMethodAccessor663.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at org.kohsuke.stapler.Function$InstanceFunction.invoke(Function.java:259)
at org.kohsuke.stapler.Function.bindAndInvoke(Function.java:126)
at org.kohsuke.stapler.Function.bindAndInvokeAndServeResponse(Function.java:65)
at org.kohsuke.stapler.MetaClass$1.doDispatch(MetaClass.java:79)
at org.kohsuke.stapler.NameBasedDispatcher.dispatch(NameBasedDispatcher.java:30)
at org.kohsuke.stapler.Stapler.invoke(Stapler.java:537)
at org.kohsuke.stapler.MetaClass$13.dispatch(MetaClass.java:359)
at org.kohsuke.stapler.Stapler.invoke(Stapler.java:537)
at org.kohsuke.stapler.MetaClass$7.doDispatch(MetaClass.java:219)
at org.kohsuke.stapler.NameBasedDispatcher.dispatch(NameBasedDispatcher.java:30)
at org.kohsuke.stapler.Stapler.invoke(Stapler.java:537)
at org.kohsuke.stapler.Stapler.invoke(Stapler.java:453)
at org.kohsuke.stapler.Stapler.service(Stapler.java:135)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:717)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:290)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)
at hudson.util.PluginServletFilter$1.doFilter(PluginServletFilter.java:94)
at hudson.plugins.greenballs.GreenBallFilter.doFilter(GreenBallFilter.java:51)
at hudson.util.PluginServletFilter$1.doFilter(PluginServletFilter.java:97)
at hudson.plugins.audit_trail.AuditTrailFilter.doFilter(AuditTrailFilter.java:64)
at hudson.util.PluginServletFilter$1.doFilter(PluginServletFilter.java:97)
at hudson.util.PluginServletFilter.doFilter(PluginServletFilter.java:86)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:235)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)
at hudson.security.csrf.CrumbFilter.doFilter(CrumbFilter.java:47)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:235)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)
at hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.java:84)
at hudson.security.UnwrapSecurityExceptionFilter.doFilter(UnwrapSecurityExceptionFilter.java:51)
at hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.java:87)
at org.acegisecurity.ui.ExceptionTranslationFilter.doFilter(ExceptionTranslationFilter.java:166)
at hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.java:87)
at org.acegisecurity.providers.anonymous.AnonymousProcessingFilter.doFilter(AnonymousProcessingFilter.java:125)
at hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.java:87)
at org.acegisecurity.ui.rememberme.RememberMeProcessingFilter.doFilter(RememberMeProcessingFilter.java:135)
at hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.java:87)
at org.acegisecurity.ui.AbstractProcessingFilter.doFilter(AbstractProcessingFilter.java:271)
at hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.java:87)
at org.acegisecurity.ui.basicauth.BasicProcessingFilter.doFilter(BasicProcessingFilter.java:173)
at hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.java:87)
at org.acegisecurity.context.HttpSessionContextIntegrationFilter.doFilter(HttpSessionContextIntegrationFilter.java:249)
at hudson.security.HttpSessionContextIntegrationFilter2.doFilter(HttpSessionContextIntegrationFilter2.java:66)
at hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.java:87)
at hudson.security.ChainedServletFilter.doFilter(ChainedServletFilter.java:76)
at hudson.security.HudsonFilter.doFilter(HudsonFilter.java:164)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:235)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)
at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:233)
at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:191)
at org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:433)
at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:128)
at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:102)
at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:109)
at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:286)
at org.apache.coyote.http11.Http11Processor.process(Http11Processor.java:845)
at org.apache.coyote.http11.Http11Protocol$Http11ConnectionHandler.process(Http11Protocol.java:583)
at org.apache.tomcat.util.net.JIoEndpoint$Worker.run(JIoEndpoint.java:447)
at java.lang.Thread.run(Thread.java:619)
Locked ownable synchronizers:
- None
Full thread dumps pasted:
First dump: http://hudson.pastebin.com/HpB68hVs
Second dump: (~11 hours later) http://hudson.pastebin.com/eDA2WhnE