-
Bug
-
Resolution: Unresolved
-
Minor
-
None
-
Jenkins Version: 2.176.3
Plugins:
{code}
workflow-cps,2.73
bouncycastle-api,2.17
cloudbees-folder,6.9
structs,1.20
junit,1.28
antisamy-markup-formatter,1.5
github-api,1.95
pam-auth,1.5.1
pipeline-stage-step,2.3
windows-slaves,1.4
ssh-credentials,1.17.1
display-url-api,2.3.2
mailer,1.24
rebuild,1.31
git,3.11.0
token-macro,2.8
external-monitor-job,1.7
icon-shim,2.0.3
github,1.29.4
matrix-auth,2.4.2
git-client,2.8.0
script-security,1.62
matrix-project,1.14
build-timeout,1.19
credentials,2.2.1
git-server,1.8
workflow-step-api,2.20
workflow-cps-global-lib,2.15
plain-credentials,1.5
credentials-binding,1.20
mapdb-api,1.0.9.0
timestamper,1.10
workflow-api,2.36
workflow-basic-steps,2.18
branch-api,2.5.4
resource-disposer,0.13
subversion,2.12.2
ws-cleanup,0.37
ldap,1.20
ant,1.9
gradle,1.33
pipeline-milestone-step,1.3.1
workflow-support,3.3
workflow-multibranch,2.21
pipeline-build-step,2.9
jquery-detached,1.2.1
durable-task,1.30
pipeline-input-step,2.10
ssh-slaves,1.29.4
ace-editor,1.1
workflow-scm-step,2.9
scm-api,2.6.3
email-ext,2.66
workflow-job,2.33
pipeline-graph-analysis,1.10
pipeline-rest-api,2.12
handlebars,1.1.1
momentjs,1.1.1
workflow-aggregator,2.6
pipeline-stage-view,2.12
docker-workflow,1.19
pipeline-model-definition,1.3.9
config-file-provider,3.6.2
metrics,4.0.2.5
workflow-durable-task-step,2.31
modernstatus,1.2
pipeline-model-extensions,1.3.9
maven-repo-cleaner,1.2
pipeline-github-lib,1.0
github-organization-folder,1.6
job-dsl,1.74
configurationslicing,1.47
jquery,1.12.4-0
saferestart,0.3
dashboard-view,2.11
javadoc,1.5
jobConfigHistory,2.23.1
pipeline-model-api,1.3.9
github-oauth,0.33
disk-usage,0.28
log-parser,2.1
monitoring,1.79.0
mesos,0.18.1
variant,1.3
jira,3.0.8
docker-plugin,1.1.7
ssh-agent,1.17
ghprb,1.42.0
scriptler,2.9
pipeline-stage-tags-metadata,1.3.9
maven-plugin,3.4
github-branch-source,2.5.5
authentication-tokens,1.3
docker-commons,1.15
pipeline-model-declarative-agent,1.1.1
build-failure-analyzer,1.22.0
configure-job-column-plugin,1.0
dropdown-viewstabbar-plugin,1.7
blueocean-pipeline-editor,1.18.1
nodelabelparameter,1.7.2
uno-choice,1.5.3
docker-java-api,3.0.14
blueocean-rest,1.18.1
blueocean-pipeline-scm-api,1.18.1
blueocean-git-pipeline,1.18.1
blueocean-config,1.18.1
cloudbees-bitbucket-branch-source,2.4.5
blueocean-jira,1.18.1
sse-gateway,1.19
blueocean-core-js,1.18.1
htmlpublisher,1.18
simple-theme-plugin,0.5.1
blueocean-autofavorite,1.2.4
jdk-tool,1.3
blueocean-web,1.18.1
blueocean-pipeline-api-impl,1.18.1
blueocean-bitbucket-pipeline,1.18.1
blueocean-personalization,1.18.1
command-launcher,1.3
blueocean,1.18.1
blueocean-commons,1.18.1
build-monitor-plugin,1.12+build.201809061734
apache-httpcomponents-client-4-api,4.5.5-3.0
mercurial,2.8
blueocean-jwt,1.18.1
jackson2-api,2.9.9.1
jenkins-design-language,1.18.1
blueocean-events,1.18.1
pubsub-light,1.12
blueocean-i18n,1.18.1
favorite,2.3.2
blueocean-rest-impl,1.18.1
blueocean-github-pipeline,1.18.1
jsch,0.1.55.1
handy-uri-templates-2-api,2.1.7-1.0
blueocean-dashboard,1.18.1
blueocean-display-url,2.3.0
trilead-api,1.0.3
lockable-resources,2.5
prometheus,2.0.0
github-scm-trait-notification-context,1.1
artifactory,3.3.2
ivy,2.1
basic-branch-build-strategies,1.3.2
http_request,1.8.23
pipeline-github,2.5
plot,2.1.1
pipeline-utility-steps,2.3.0
blueocean-executor-info,1.18.1
{code}Jenkins Version: 2.176.3 Plugins: {code} workflow-cps,2.73 bouncycastle-api,2.17 cloudbees-folder,6.9 structs,1.20 junit,1.28 antisamy-markup-formatter,1.5 github-api,1.95 pam-auth,1.5.1 pipeline-stage-step,2.3 windows-slaves,1.4 ssh-credentials,1.17.1 display-url-api,2.3.2 mailer,1.24 rebuild,1.31 git,3.11.0 token-macro,2.8 external-monitor-job,1.7 icon-shim,2.0.3 github,1.29.4 matrix-auth,2.4.2 git-client,2.8.0 script-security,1.62 matrix-project,1.14 build-timeout,1.19 credentials,2.2.1 git-server,1.8 workflow-step-api,2.20 workflow-cps-global-lib,2.15 plain-credentials,1.5 credentials-binding,1.20 mapdb-api,1.0.9.0 timestamper,1.10 workflow-api,2.36 workflow-basic-steps,2.18 branch-api,2.5.4 resource-disposer,0.13 subversion,2.12.2 ws-cleanup,0.37 ldap,1.20 ant,1.9 gradle,1.33 pipeline-milestone-step,1.3.1 workflow-support,3.3 workflow-multibranch,2.21 pipeline-build-step,2.9 jquery-detached,1.2.1 durable-task,1.30 pipeline-input-step,2.10 ssh-slaves,1.29.4 ace-editor,1.1 workflow-scm-step,2.9 scm-api,2.6.3 email-ext,2.66 workflow-job,2.33 pipeline-graph-analysis,1.10 pipeline-rest-api,2.12 handlebars,1.1.1 momentjs,1.1.1 workflow-aggregator,2.6 pipeline-stage-view,2.12 docker-workflow,1.19 pipeline-model-definition,1.3.9 config-file-provider,3.6.2 metrics,4.0.2.5 workflow-durable-task-step,2.31 modernstatus,1.2 pipeline-model-extensions,1.3.9 maven-repo-cleaner,1.2 pipeline-github-lib,1.0 github-organization-folder,1.6 job-dsl,1.74 configurationslicing,1.47 jquery,1.12.4-0 saferestart,0.3 dashboard-view,2.11 javadoc,1.5 jobConfigHistory,2.23.1 pipeline-model-api,1.3.9 github-oauth,0.33 disk-usage,0.28 log-parser,2.1 monitoring,1.79.0 mesos,0.18.1 variant,1.3 jira,3.0.8 docker-plugin,1.1.7 ssh-agent,1.17 ghprb,1.42.0 scriptler,2.9 pipeline-stage-tags-metadata,1.3.9 maven-plugin,3.4 github-branch-source,2.5.5 authentication-tokens,1.3 docker-commons,1.15 pipeline-model-declarative-agent,1.1.1 build-failure-analyzer,1.22.0 configure-job-column-plugin,1.0 dropdown-viewstabbar-plugin,1.7 blueocean-pipeline-editor,1.18.1 nodelabelparameter,1.7.2 uno-choice,1.5.3 docker-java-api,3.0.14 blueocean-rest,1.18.1 blueocean-pipeline-scm-api,1.18.1 blueocean-git-pipeline,1.18.1 blueocean-config,1.18.1 cloudbees-bitbucket-branch-source,2.4.5 blueocean-jira,1.18.1 sse-gateway,1.19 blueocean-core-js,1.18.1 htmlpublisher,1.18 simple-theme-plugin,0.5.1 blueocean-autofavorite,1.2.4 jdk-tool,1.3 blueocean-web,1.18.1 blueocean-pipeline-api-impl,1.18.1 blueocean-bitbucket-pipeline,1.18.1 blueocean-personalization,1.18.1 command-launcher,1.3 blueocean,1.18.1 blueocean-commons,1.18.1 build-monitor-plugin,1.12+build.201809061734 apache-httpcomponents-client-4-api,4.5.5-3.0 mercurial,2.8 blueocean-jwt,1.18.1 jackson2-api,2.9.9.1 jenkins-design-language,1.18.1 blueocean-events,1.18.1 pubsub-light,1.12 blueocean-i18n,1.18.1 favorite,2.3.2 blueocean-rest-impl,1.18.1 blueocean-github-pipeline,1.18.1 jsch,0.1.55.1 handy-uri-templates-2-api,2.1.7-1.0 blueocean-dashboard,1.18.1 blueocean-display-url,2.3.0 trilead-api,1.0.3 lockable-resources,2.5 prometheus,2.0.0 github-scm-trait-notification-context,1.1 artifactory,3.3.2 ivy,2.1 basic-branch-build-strategies,1.3.2 http_request,1.8.23 pipeline-github,2.5 plot,2.1.1 pipeline-utility-steps,2.3.0 blueocean-executor-info,1.18.1 {code}
We're seeing a deadlock on agents when the ZipInstaller tool tries to check whether the tool needs to be installed again or not.
A relevant thread dump:
"org.jenkinsci.plugins.workflow.steps.SynchronousNonBlockingStepExecution [#30]" - Thread t@99095 java.lang.Thread.State: RUNNABLE at java.net.SocketInputStream.socketRead0(Native Method) at java.net.SocketInputStream.socketRead(SocketInputStream.java:116) at java.net.SocketInputStream.read(SocketInputStream.java:170) at java.net.SocketInputStream.read(SocketInputStream.java:141) at sun.security.ssl.InputRecord.readFully(InputRecord.java:465) at sun.security.ssl.InputRecord.read(InputRecord.java:503) at sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:973) at sun.security.ssl.SSLSocketImpl.readDataRecord(SSLSocketImpl.java:930) at sun.security.ssl.AppInputStream.read(AppInputStream.java:105) at java.io.BufferedInputStream.fill(BufferedInputStream.java:246) at java.io.BufferedInputStream.read1(BufferedInputStream.java:286) at java.io.BufferedInputStream.read(BufferedInputStream.java:345) at sun.net.www.http.HttpClient.parseHTTPHeader(HttpClient.java:704) at sun.net.www.http.HttpClient.parseHTTP(HttpClient.java:647) at sun.net.www.protocol.http.HttpURLConnection.getInputStream0(HttpURLConnection.java:1569) at sun.net.www.protocol.http.HttpURLConnection.getInputStream(HttpURLConnection.java:1474) at java.net.HttpURLConnection.getResponseCode(HttpURLConnection.java:480) at sun.net.www.protocol.https.HttpsURLConnectionImpl.getResponseCode(HttpsURLConnectionImpl.java:338) at hudson.FilePath.installIfNecessaryFrom(FilePath.java:874) at hudson.FilePath.installIfNecessaryFrom(FilePath.java:846) at hudson.tools.ZipExtractionInstaller.performInstallation(ZipExtractionInstaller.java:83) at hudson.tools.InstallerTranslator.getToolHome(InstallerTranslator.java:69) at hudson.tools.ToolLocationNodeProperty.getToolHome(ToolLocationNodeProperty.java:109) at hudson.tools.ToolInstallation.translateFor(ToolInstallation.java:206) at hudson.model.JDK.forNode(JDK.java:147) at hudson.model.JDK.forNode(JDK.java:60) at org.jenkinsci.plugins.workflow.steps.ToolStep$Execution.run(ToolStep.java:152) at org.jenkinsci.plugins.workflow.steps.ToolStep$Execution.run(ToolStep.java:133) at org.jenkinsci.plugins.workflow.steps.SynchronousNonBlockingStepExecution.lambda$start$0(SynchronousNonBlockingStepExecution.java:47) at org.jenkinsci.plugins.workflow.steps.SynchronousNonBlockingStepExecution$$Lambda$271/291985104.run(Unknown Source) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745)
We believe this comes down to the connection getResponseCode() call waiting indefinitely, from here: https://github.com/jenkinsci/jenkins/blob/master/core/src/main/java/hudson/FilePath.java#L876
We're using the ZipInstaller to install a JDK from our artifactory repository (instead of the Oracle JDK website).
Proposal
Add a timeout around the getResponseCode() section (or further down), just to prevent that connection from hanging indefinitely.
I have a screenshot of a heap dump where we found what URL was getting stuck within our infrastructure.