This issue reported via [Cloudbees Support|https://support.cloudbees.com/hc/en-us/requests/179894], but reported here as it likely affects others.

Summary: Scanner aborts at first error

On the master, an nfs problem prohibits the job scanner from cleaning up an inactive PR. Unfortunately once it hits the error, it exits and does not continue to attempt to clean up the other PRs. A single error on a heavily used repo causes a significant growth in disk space, jobs, and overhead.  We have seen a similar problem if other causes prevent removal like a directory without permission for Jenkins.

DESIRED Behavior: Continue removing other obsolete jobs

Log

Will remove PR-41524
FATAL: Failed to recompute children of Team » REDACTED » pipeline
jenkins.util.io.CompositeIOException: Unable to delete '/var/jenkins_home/jobs/team/jobs/REDACTED/pipeline/branches/PR-41524'. Tried 3 times (of a maximum of 3) waiting 0.1 sec between attempts.
at jenkins.util.io.PathRemover.forceRemoveRecursive(PathRemover.java:99)
at hudson.Util.deleteRecursive(Util.java:294)
at hudson.Util.deleteRecursive(Util.java:283)
at hudson.model.AbstractItem.performDelete(AbstractItem.java:798)
at org.jenkinsci.plugins.workflow.job.WorkflowJob.performDelete(WorkflowJob.java:650)
at hudson.model.AbstractItem.delete(AbstractItem.java:782)
at hudson.model.Job.delete(Job.java:677)
at com.cloudbees.hudson.plugins.folder.computed.ComputedFolder.updateChildren(ComputedFolder.java:290)
at com.cloudbees.hudson.plugins.folder.computed.FolderComputation.run(FolderComputation.java:164)
at jenkins.branch.MultiBranchProject$BranchIndexing.run(MultiBranchProject.java:1032)
at hudson.model.ResourceController.execute(ResourceController.java:97)
at hudson.model.Executor.run(Executor.java:428)
Suppressed: jenkins.util.io.CompositeIOException: Unable to remove file /var/jenkins_home/jobs/team/jobs/REDACTED/pipeline/branches/PR-41524/builds/2/.nfs0000000002280a9400001dfd
at jenkins.util.io.PathRemover.removeOrMakeRemovableThenRemove(PathRemover.java:252)
at jenkins.util.io.PathRemover.tryRemoveFile(PathRemover.java:205)
at jenkins.util.io.PathRemover.tryRemoveRecursive(PathRemover.java:216)
at jenkins.util.io.PathRemover.tryRemoveDirectoryContents(PathRemover.java:226)
at jenkins.util.io.PathRemover.tryRemoveRecursive(PathRemover.java:215)
at jenkins.util.io.PathRemover.tryRemoveDirectoryContents(PathRemover.java:226)
at jenkins.util.io.PathRemover.tryRemoveRecursive(PathRemover.java:215)
at jenkins.util.io.PathRemover.tryRemoveDirectoryContents(PathRemover.java:226)
at jenkins.util.io.PathRemover.tryRemoveRecursive(PathRemover.java:215)
at jenkins.util.io.PathRemover.forceRemoveRecursive(PathRemover.java:96)
... 11 more
Suppressed: java.nio.file.FileSystemException: /var/jenkins_home/jobs/team/jobs/REDACTED/pipeline/branches/PR-41524/builds/2/.nfs0000000002280a9400001dfd: Device or resource busy
at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91)
at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102)
at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107)
at sun.nio.fs.UnixFileSystemProvider.implDelete(UnixFileSystemProvider.java:244)
at sun.nio.fs.AbstractFileSystemProvider.deleteIfExists(AbstractFileSystemProvider.java:108)
at java.nio.file.Files.deleteIfExists(Files.java:1165)
at jenkins.util.io.PathRemover.removeOrMakeRemovableThenRemove(PathRemover.java:237)
... 20 more
Suppressed: java.nio.file.FileSystemException: /var/jenkins_home/jobs/team/jobs/REDACTED/pipeline/branches/PR-41524/builds/2/.nfs0000000002280a9400001dfd: Device or resource busy
at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91)
at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102)
at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107)
at sun.nio.fs.UnixFileSystemProvider.implDelete(UnixFileSystemProvider.java:244)
at sun.nio.fs.AbstractFileSystemProvider.deleteIfExists(AbstractFileSystemProvider.java:108)
at java.nio.file.Files.deleteIfExists(Files.java:1165)
at jenkins.util.io.PathRemover.removeOrMakeRemovableThenRemove(PathRemover.java:241)
... 20 more
Suppressed: jenkins.util.io.CompositeIOException: Unable to remove directory /var/jenkins_home/jobs/team/jobs/REDACTED/pipeline/branches/PR-41524/builds/2 with directory contents: [/var/jenkins_home/jobs/team/REDACTED/jobs/pipeline/branches/PR-41524/builds/2/.nfs0000000002280a9400001dfd]
at jenkins.util.io.PathRemover.removeOrMakeRemovableThenRemove(PathRemover.java:250)
at jenkins.util.io.PathRemover.tryRemoveFile(PathRemover.java:205)
at jenkins.util.io.PathRemover.tryRemoveRecursive(PathRemover.java:216)
at jenkins.util.io.PathRemover.tryRemoveDirectoryContents(PathRemover.java:226)
at jenkins.util.io.PathRemover.tryRemoveRecursive(PathRemover.java:215)
at jenkins.util.io.PathRemover.tryRemoveDirectoryContents(PathRemover.java:226)
at jenkins.util.io.PathRemover.tryRemoveRecursive(PathRemover.java:215)
at jenkins.util.io.PathRemover.forceRemoveRecursive(PathRemover.java:96)
... 11 more
Suppressed: java.nio.file.DirectoryNotEmptyException: /var/jenkins_home/jobs/team/jobs/REDACTED/pipeline/branches/PR-41524/builds/2
at sun.nio.fs.UnixFileSystemProvider.implDelete(UnixFileSystemProvider.java:242)
at sun.nio.fs.AbstractFileSystemProvider.deleteIfExists(AbstractFileSystemProvider.java:108)
at java.nio.file.Files.deleteIfExists(Files.java:1165)
at jenkins.util.io.PathRemover.removeOrMakeRemovableThenRemove(PathRemover.java:237)
... 18 more
Suppressed: java.nio.file.DirectoryNotEmptyException: /var/jenkins_home/jobs/team/jobs/REDACTED/pipeline/branches/PR-41524/builds/2
at sun.nio.fs.UnixFileSystemProvider.implDelete(UnixFileSystemProvider.java:242)
at sun.nio.fs.AbstractFileSystemProvider.deleteIfExists(AbstractFileSystemProvider.java:108)
at java.nio.file.Files.deleteIfExists(Files.java:1165)
at jenkins.util.io.PathRemover.removeOrMakeRemovableThenRemove(PathRemover.java:241)
... 18 more
Suppressed: jenkins.util.io.CompositeIOException: Unable to remove directory /var/jenkins_home/jobs/team/jobs/REDACTED/pipeline/branches/PR-41524/builds with directory contents: [/var/jenkins_home/jobs/team/jobs/REDACTED/pipeline/branches/PR-41524/builds/2]
at jenkins.util.io.PathRemover.removeOrMakeRemovableThenRemove(PathRemover.java:250)
at jenkins.util.io.PathRemover.tryRemoveFile(PathRemover.java:205)
at jenkins.util.io.PathRemover.tryRemoveRecursive(PathRemover.java:216)
at jenkins.util.io.PathRemover.tryRemoveDirectoryContents(PathRemover.java:226)
at jenkins.util.io.PathRemover.tryRemoveRecursive(PathRemover.java:215)
at jenkins.util.io.PathRemover.forceRemoveRecursive(PathRemover.java:96)
... 11 more
Suppressed: java.nio.file.DirectoryNotEmptyException: /var/jenkins_home/jobs/team/jobs/REDACTED/pipeline/branches/PR-41524/builds
at sun.nio.fs.UnixFileSystemProvider.implDelete(UnixFileSystemProvider.java:242)
at sun.nio.fs.AbstractFileSystemProvider.deleteIfExists(AbstractFileSystemProvider.java:108)
at java.nio.file.Files.deleteIfExists(Files.java:1165)
at jenkins.util.io.PathRemover.removeOrMakeRemovableThenRemove(PathRemover.java:237)
... 16 more
Suppressed: java.nio.file.DirectoryNotEmptyException: /var/jenkins_home/jobs/team/jobs/REDACTED/pipeline/branches/PR-41524/builds
at sun.nio.fs.UnixFileSystemProvider.implDelete(UnixFileSystemProvider.java:242)
at sun.nio.fs.AbstractFileSystemProvider.deleteIfExists(AbstractFileSystemProvider.java:108)
at java.nio.file.Files.deleteIfExists(Files.java:1165)
at jenkins.util.io.PathRemover.removeOrMakeRemovableThenRemove(PathRemover.java:241)
... 16 more
Suppressed: jenkins.util.io.CompositeIOException: Unable to remove directory /var/jenkins_home/jobs/team/jobs/REDACTED/pipeline/branches/PR-41524 with directory contents: [/var/jenkins_home/jobs/team/jobs/REDACTED/pipeline/branches/PR-41524/builds]
at jenkins.util.io.PathRemover.removeOrMakeRemovableThenRemove(PathRemover.java:250)
at jenkins.util.io.PathRemover.tryRemoveFile(PathRemover.java:205)
at jenkins.util.io.PathRemover.tryRemoveRecursive(PathRemover.java:216)
at jenkins.util.io.PathRemover.forceRemoveRecursive(PathRemover.java:96)
... 11 more
Suppressed: java.nio.file.DirectoryNotEmptyException: /var/jenkins_home/jobs/team/jobs/REDACTED/pipeline/branches/PR-41524
at sun.nio.fs.UnixFileSystemProvider.implDelete(UnixFileSystemProvider.java:242)
at sun.nio.fs.AbstractFileSystemProvider.deleteIfExists(AbstractFileSystemProvider.java:108)
at java.nio.file.Files.deleteIfExists(Files.java:1165)
at jenkins.util.io.PathRemover.removeOrMakeRemovableThenRemove(PathRemover.java:237)
... 14 more
Suppressed: java.nio.file.DirectoryNotEmptyException: /var/jenkins_home/jobs/team/jobs/REDACTED/pipeline/branches/PR-41524
at sun.nio.fs.UnixFileSystemProvider.implDelete(UnixFileSystemProvider.java:242)
at sun.nio.fs.AbstractFileSystemProvider.deleteIfExists(AbstractFileSystemProvider.java:108)
at java.nio.file.Files.deleteIfExists(Files.java:1165)
at jenkins.util.io.PathRemover.removeOrMakeRemovableThenRemove(PathRemover.java:241)
... 14 more