Puppet agent using 100% CPU, on a single node for an hour
Puppet seems to peg the CPU when I try to do a recursive deploy of a directory into a folder that already has a substantial number of files.
The the provider used by the user resource on my systems does not appear to deploy the files in /etc/skel into a newly provisioned directory. So it seemed to me that I could use this recursive deploy to populate the directory like so.
# password is in the vault
user { "myuser":
ensure => 'present',
password => 'guessmypassword',
home => '/home/myuser',
shell => '/bin/bash',
}
file { '/home/myuser':
ensure => directory,
owner => 'myuser',
group => 'myuser',
mode => 0755,
require => User['myuser'],
}
file { '/home/myuser/.ssh':
ensure => directory,
owner => 'myuser',
group => 'myuser',
mode => 0700,
require => User['myuser'],
}
file { 'myuser_skeleton':
path => '/home/myuser',
source => '/etc/skel',
owner => 'myuser',
group => 'myuser',
recurse => true,
replace => false,
}
Where the problem seems to show up is that user, on one of the systems, had a svn working copy of a project with about 5,000 files. A puppet run on that system seems to sit at 100% for 40 minutes.
I ran puppet under strace, during the lockup it seems to keep calling gettimeofday() over 4000 times for some files, then it moves onto the next and repeats.
munmap(0xb6ffc000, 4096) = 0
lstat64("/home/myuser/ptlib/include/ptlib/Nucleus++/ptlib/.svn/tmp/props", {st_mode=S_IFDIR|0755, st_size=4096, ...}) = 0
stat64("/home/myuser", {st_mode=S_IFDIR|0755, st_size=4096, ...}) = 0
stat64("/etc/localtime", {st_mode=S_IFREG|0644, st_size=2819, ...}) = 0
lstat64("/home/myuser/ptlib/src/ptlib/common/.svn/prop-base/sfile.cxx.svn-base", {st_mode=S_IFREG|0444, st_size=109, ...}) = 0
stat64("/usr/local/lib/site_ruby/1.8/digest/md5.rb", 0xbff645f0) = -1 ENOENT (No such file or directory)
stat64("/usr/local/lib/site_ruby/1.8/digest/md5.so", 0xbff645f0) = -1 ENOENT (No such file or directory)
stat64("/usr/local/lib/site_ruby/1.8/i486-linux/digest/md5.rb", 0xbff645f0) = -1 ENOENT (No such file or directory)
stat64("/usr/local/lib/site_ruby/1.8/i486-linux/digest/md5.so", 0xbff645f0) = -1 ENOENT (No such file or directory)
stat64("/usr/local/lib/site_ruby/1.8/i386-linux/digest/md5.rb", 0xbff645f0) = -1 ENOENT (No such file or directory)
stat64("/usr/local/lib/site_ruby/1.8/i386-linux/digest/md5.so", 0xbff645f0) = -1 ENOENT (No such file or directory)
stat64("/usr/local/lib/site_ruby/digest/md5.rb", 0xbff645f0) = -1 ENOENT (No such file or directory)
stat64("/usr/local/lib/site_ruby/digest/md5.so", 0xbff645f0) = -1 ENOENT (No such file or directory)
stat64("/usr/lib/ruby/vendor_ruby/1.8/digest/md5.rb", 0xbff645f0) = -1 ENOENT (No such file or directory)
stat64("/usr/lib/ruby/vendor_ruby/1.8/digest/md5.so", 0xbff645f0) = -1 ENOENT (No such file or directory)
stat64("/usr/lib/ruby/vendor_ruby/1.8/i486-linux/digest/md5.rb", 0xbff645f0) = -1 ENOENT (No such file or directory)
stat64("/usr/lib/ruby/vendor_ruby/1.8/i486-linux/digest/md5.so", 0xbff645f0) = -1 ENOENT (No such file or directory)
stat64("/usr/lib/ruby/vendor_ruby/digest/md5.rb", 0xbff645f0) = -1 ENOENT (No such file or directory)
stat64("/usr/lib/ruby/vendor_ruby/digest/md5.so", 0xbff645f0) = -1 ENOENT (No such file or directory)
stat64("/usr/lib/ruby/1.8/digest/md5.rb", 0xbff645f0) = -1 ENOENT (No such file or directory)
stat64("/usr/lib/ruby/1.8/digest/md5.so", 0xbff645f0) = -1 ENOENT (No such file or directory)
stat64("/usr/lib/ruby/1.8/i486-linux/digest/md5.rb", 0xbff645f0) = -1 ENOENT (No such file or directory)
stat64("/usr/lib/ruby/1.8/i486-linux/digest/md5.so", {st_mode=S_IFREG|0644, st_size=6848, ...}) = 0
open("/usr/lib/ruby/1.8/i486-linux/digest/md5.so", O_RDONLY|O_LARGEFILE) = 6
fstat64(6, {st_mode=S_IFREG|0644, st_size=6848, ...}) = 0
close(6) = 0
open("/home/myuser/ptlib/src/ptlib/common/.svn/prop-base/sfile.cxx.svn-base", O_RDONLY|O_LARGEFILE) = 6
fstat64(6, {st_mode=S_IFREG|0444, st_size=109, ...}) = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb6ffc000
read(6, "K 17\nCVSRevisionNumber\nV 3\n1.3\nK"..., 4096) = 109
read(6, "", 4096) = 0
close(6) = 0
munmap(0xb6ffc000, 4096) = 0
lstat64("/home/myuser/ptlib/plugins/vidinput_bsd/.svn/props", {st_mode=S_IFDIR|0755, st_size=4096, ...}) = 0
stat64("/home/myuser", {st_mode=S_IFDIR|0755, st_size=4096, ...}) = 0
stat64("/etc/localtime", {st_mode=S_IFREG|0644, st_size=2819, ...}) = 0
gettimeofday({1319573116, 304890}, NULL) = 0
gettimeofday({1319573116, 304951}, NULL) = 0
gettimeofday({1319573116, 304993}, NULL) = 0
gettimeofday({1319573116, 305034}, NULL) = 0
... repeats ~4000 times.
gettimeofday({1319573116, 305075}, NULL) = 0
gettimeofday({1319573116, 305116}, NULL) = 0
gettimeofday({1319573116, 305158}, NULL) = 0
gettimeofday({1319573116, 305199}, NULL) = 0
gettimeofday({1319573116, 305245}, NULL) = 0
brk(0xbbf1000) = 0xbbf1000
brk(0xbc12000) = 0xbc12000
brk(0xbc33000) = 0xbc33000
brk(0xbc55000) = 0xbc55000
brk(0xbc76000) = 0xbc76000
brk(0xbc98000) = 0xbc98000
... repeats 200 times
stat64("/etc/skel", {st_mode=S_IFDIR|0755, st_size=4096, ...}) = 0
lstat64("/etc/skel", {st_mode=S_IFDIR|0755, st_size=4096, ...}) = 0
lstat64("/etc/skel", {st_mode=S_IFDIR|0755, st_size=4096, ...}) = 0
open("/etc/skel", O_RDONLY|O_NONBLOCK|O_LARGEFILE|O_DIRECTORY|O_CLOEXEC) = 6
brk(0xc334000) = 0xc334000
lseek(6, 0, SEEK_SET) = 0
getdents64(6, /* 5 entries */, 32768) = 144
getdents64(6, /* 0 entries */, 32768) = 0
close(6) = 0
... and one
Use recurse => remote instead of recurse => true -- this will scan only the files on the server side instead of the client.
remote — Descends recursively into the remote directory but not the local directory. Allows copying of a few files into a directory containing many unmanaged files without scanning all the local files.
Extract taken from: https://puppet.com/docs/puppet/latest/types/file.html