diff options
Diffstat (limited to 'lib')
125 files changed, 3724 insertions, 323 deletions
diff --git a/lib/puppet.rb b/lib/puppet.rb index 0329e5b04..9105f7f62 100644 --- a/lib/puppet.rb +++ b/lib/puppet.rb @@ -24,7 +24,7 @@ require 'puppet/util/run_mode' # it's also a place to find top-level commands like 'debug' module Puppet - PUPPETVERSION = '2.7.9' + PUPPETVERSION = '2.7.10' def Puppet.version PUPPETVERSION diff --git a/lib/puppet/agent.rb b/lib/puppet/agent.rb index 47dd44a0e..b684470e3 100644 --- a/lib/puppet/agent.rb +++ b/lib/puppet/agent.rb @@ -8,6 +8,9 @@ class Puppet::Agent require 'puppet/agent/locker' include Puppet::Agent::Locker + require 'puppet/agent/disabler' + include Puppet::Agent::Disabler + attr_reader :client_class, :client, :splayed # Just so we can specify that we are "the" instance. @@ -31,6 +34,10 @@ class Puppet::Agent Puppet.notice "Run of #{client_class} already in progress; skipping" return end + if disabled? + Puppet.notice "Skipping run of #{client_class}; administratively disabled: #{disable_message}" + return + end result = nil block_run = Puppet::Application.controlled_run do splay diff --git a/lib/puppet/agent/disabler.rb b/lib/puppet/agent/disabler.rb new file mode 100644 index 000000000..34ab94bbf --- /dev/null +++ b/lib/puppet/agent/disabler.rb @@ -0,0 +1,27 @@ +require 'puppet/util/anonymous_filelock' + +module Puppet::Agent::Disabler + # Let the daemon run again, freely in the filesystem. + def enable + disable_lockfile.unlock + end + + # Stop the daemon from making any catalog runs. + def disable(msg='') + disable_lockfile.lock(msg) + end + + def disable_lockfile + @disable_lockfile ||= Puppet::Util::AnonymousFilelock.new(lockfile_path+".disabled") + + @disable_lockfile + end + + def disabled? + disable_lockfile.locked? + end + + def disable_message + disable_lockfile.message + end +end diff --git a/lib/puppet/agent/locker.rb b/lib/puppet/agent/locker.rb index 98f5b38d9..d41b12547 100644 --- a/lib/puppet/agent/locker.rb +++ b/lib/puppet/agent/locker.rb @@ -3,16 +3,6 @@ require 'puppet/util/pidlock' # Break out the code related to locking the agent. This module is just # included into the agent, but having it here makes it easier to test. module Puppet::Agent::Locker - # Let the daemon run again, freely in the filesystem. - def enable - lockfile.unlock(:anonymous => true) - end - - # Stop the daemon from making any catalog runs. - def disable - lockfile.lock(:anonymous => true) - end - # Yield if we get a lock, else do nothing. Return # true/false depending on whether we get the lock. def lock diff --git a/lib/puppet/application.rb b/lib/puppet/application.rb index b7cb1169d..f6dac3911 100644 --- a/lib/puppet/application.rb +++ b/lib/puppet/application.rb @@ -263,12 +263,15 @@ class Application end def initialize(command_line = nil) + require 'puppet/util/command_line' @command_line = command_line || Puppet::Util::CommandLine.new set_run_mode self.class.run_mode @options = {} require 'puppet' + require 'puppet/util/instrumentation' + Puppet::Util::Instrumentation.init end # WARNING: This is a totally scary, frightening, and nasty internal API. We diff --git a/lib/puppet/application/agent.rb b/lib/puppet/application/agent.rb index a3854a8fd..055c36df4 100644 --- a/lib/puppet/application/agent.rb +++ b/lib/puppet/application/agent.rb @@ -39,7 +39,12 @@ class Puppet::Application::Agent < Puppet::Application end option("--centrallogging") - option("--disable") + + option("--disable [MESSAGE]") do |message| + options[:disable] = true + options[:disable_message] = message + end + option("--enable") option("--debug","-d") option("--fqdn FQDN","-f") @@ -101,7 +106,7 @@ similar), or run interactively for testing purposes. USAGE ----- puppet agent [--certname <name>] [-D|--daemonize|--no-daemonize] - [-d|--debug] [--detailed-exitcodes] [--digest <digest>] [--disable] [--enable] + [-d|--debug] [--detailed-exitcodes] [--digest <digest>] [--disable [message]] [--enable] [--fingerprint] [-h|--help] [-l|--logdest syslog|<file>|console] [--no-client] [--noop] [-o|--onetime] [--serve <handler>] [-t|--test] [-v|--verbose] [-V|--version] [-w|--waitforcert <seconds>] @@ -205,6 +210,9 @@ configuration options can also be generated by running puppet agent with not want the central configuration to override the local state until everything is tested and committed. + Disable can also take an optional message that will be reported by the + 'puppet agent' at the next disabled run. + 'puppet agent' uses the same lock file while it is running, so no more than one 'puppet agent' process is working at a time. @@ -339,6 +347,8 @@ Copyright (c) 2011 Puppet Labs, LLC Licensed under the Apache 2.0 License Puppet.err detail.to_s end + @daemon.stop(:exit => false) + if not report exit(1) elsif options[:detailed_exitcodes] then @@ -384,7 +394,7 @@ Copyright (c) 2011 Puppet Labs, LLC Licensed under the Apache 2.0 License if options[:enable] agent.enable elsif options[:disable] - agent.disable + agent.disable(options[:disable_message] || 'reason not specified') end exit(0) end @@ -416,6 +426,35 @@ Copyright (c) 2011 Puppet Labs, LLC Licensed under the Apache 2.0 License cert = @host.wait_for_cert(waitforcert) unless options[:fingerprint] end + def setup_agent + # We need tomake the client either way, we just don't start it + # if --no-client is set. + require 'puppet/agent' + require 'puppet/configurer' + @agent = Puppet::Agent.new(Puppet::Configurer) + + enable_disable_client(@agent) if options[:enable] or options[:disable] + + @daemon.agent = agent if options[:client] + + # It'd be nice to daemonize later, but we have to daemonize before the + # waitforcert happens. + @daemon.daemonize if Puppet[:daemonize] + + setup_host + + @objects = [] + + # This has to go after the certs are dealt with. + if Puppet[:listen] + unless Puppet[:onetime] + setup_listen + else + Puppet.notice "Ignoring --listen on onetime run" + end + end + end + def setup setup_test if options[:test] @@ -460,31 +499,10 @@ Copyright (c) 2011 Puppet Labs, LLC Licensed under the Apache 2.0 License Puppet::Resource::Catalog.indirection.cache_class = :yaml - # We need tomake the client either way, we just don't start it - # if --no-client is set. - require 'puppet/agent' - require 'puppet/configurer' - @agent = Puppet::Agent.new(Puppet::Configurer) - - enable_disable_client(@agent) if options[:enable] or options[:disable] - - @daemon.agent = agent if options[:client] - - # It'd be nice to daemonize later, but we have to daemonize before the - # waitforcert happens. - @daemon.daemonize if Puppet[:daemonize] - - setup_host - - @objects = [] - - # This has to go after the certs are dealt with. - if Puppet[:listen] - unless Puppet[:onetime] - setup_listen - else - Puppet.notice "Ignoring --listen on onetime run" - end + unless options[:fingerprint] + setup_agent + else + setup_host end end end diff --git a/lib/puppet/application/apply.rb b/lib/puppet/application/apply.rb index 5df00d76b..e9b4e7a24 100644 --- a/lib/puppet/application/apply.rb +++ b/lib/puppet/application/apply.rb @@ -14,6 +14,14 @@ class Puppet::Application::Apply < Puppet::Application option("--detailed-exitcodes") option("--apply catalog", "-a catalog") do |arg| + Puppet.warning <<EOM +--apply is deprecated and will be removed in the future. Please +use 'puppet apply --catalog <catalog>'. +EOM + options[:catalog] = arg + end + + option("--catalog catalog", "-c catalog") do |arg| options[:catalog] = arg end @@ -46,7 +54,7 @@ USAGE ----- puppet apply [-h|--help] [-V|--version] [-d|--debug] [-v|--verbose] [-e|--execute] [--detailed-exitcodes] [-l|--logdest <file>] - [--apply <catalog>] <file> + [--apply <catalog>] [--catalog <catalog>] <file> DESCRIPTION @@ -107,6 +115,11 @@ configuration options can also be generated by running puppet with * --apply: Apply a JSON catalog (such as one generated with 'puppet master --compile'). You can + either specify a JSON file or pipe in JSON from standard input. Deprecated, please + use --catalog instead. + +* --catalog: + Apply a JSON catalog (such as one generated with 'puppet master --compile'). You can either specify a JSON file or pipe in JSON from standard input. @@ -114,6 +127,7 @@ EXAMPLE ------- $ puppet apply -l /tmp/manifest.log manifest.pp $ puppet apply --modulepath=/root/dev/modules -e "include ntpd::server" + $ puppet apply --catalog catalog.json AUTHOR diff --git a/lib/puppet/application/cert.rb b/lib/puppet/application/cert.rb index 7285d8869..7b1431146 100644 --- a/lib/puppet/application/cert.rb +++ b/lib/puppet/application/cert.rb @@ -17,11 +17,11 @@ class Puppet::Application::Cert < Puppet::Application @subcommand = (sub == :clean ? :destroy : sub) end - option("--clean", "-c") do + option("--clean", "-c") do |arg| self.subcommand = "destroy" end - option("--all", "-a") do + option("--all", "-a") do |arg| @all = true end @@ -29,7 +29,7 @@ class Puppet::Application::Cert < Puppet::Application @digest = arg end - option("--signed", "-s") do + option("--signed", "-s") do |arg| @signed = true end @@ -39,7 +39,7 @@ class Puppet::Application::Cert < Puppet::Application require 'puppet/ssl/certificate_authority/interface' Puppet::SSL::CertificateAuthority::Interface::INTERFACE_METHODS.reject {|m| m == :destroy }.each do |method| - option("--#{method.to_s.gsub('_','-')}", "-#{method.to_s[0,1]}") do + option("--#{method.to_s.gsub('_','-')}", "-#{method.to_s[0,1]}") do |arg| self.subcommand = method end end @@ -48,7 +48,7 @@ class Puppet::Application::Cert < Puppet::Application options[:allow_dns_alt_names] = value end - option("--verbose", "-v") do + option("--verbose", "-v") do |arg| Puppet::Util::Log.level = :info end diff --git a/lib/puppet/application/instrumentation_data.rb b/lib/puppet/application/instrumentation_data.rb new file mode 100644 index 000000000..e4f86f196 --- /dev/null +++ b/lib/puppet/application/instrumentation_data.rb @@ -0,0 +1,4 @@ +require 'puppet/application/indirection_base' + +class Puppet::Application::Instrumentation_data < Puppet::Application::IndirectionBase +end diff --git a/lib/puppet/application/instrumentation_listener.rb b/lib/puppet/application/instrumentation_listener.rb new file mode 100644 index 000000000..64029b5c9 --- /dev/null +++ b/lib/puppet/application/instrumentation_listener.rb @@ -0,0 +1,4 @@ +require 'puppet/application/indirection_base' + +class Puppet::Application::Instrumentation_listener < Puppet::Application::IndirectionBase +end diff --git a/lib/puppet/application/instrumentation_probe.rb b/lib/puppet/application/instrumentation_probe.rb new file mode 100644 index 000000000..b31f95c45 --- /dev/null +++ b/lib/puppet/application/instrumentation_probe.rb @@ -0,0 +1,4 @@ +require 'puppet/application/indirection_base' + +class Puppet::Application::Instrumentation_probe < Puppet::Application::IndirectionBase +end diff --git a/lib/puppet/configurer.rb b/lib/puppet/configurer.rb index 83b5a9ad4..477363730 100644 --- a/lib/puppet/configurer.rb +++ b/lib/puppet/configurer.rb @@ -178,9 +178,17 @@ class Puppet::Configurer end def save_last_run_summary(report) - Puppet::Util::FileLocking.writelock(Puppet[:lastrunfile], 0660) do |file| - file.print YAML.dump(report.raw_summary) - end + last_run = Puppet.settings.setting(:lastrunfile) + last_run.create = true # force file creation + + resource = last_run.to_resource + resource[:content] = YAML.dump(report.raw_summary) + + catalog = Puppet::Resource::Catalog.new("last_run_file") + catalog.add_resource(resource) + ral = catalog.to_ral + ral.host_config = false + ral.apply rescue => detail puts detail.backtrace if Puppet[:trace] Puppet.err "Could not save last run local report: #{detail}" diff --git a/lib/puppet/configurer/downloader.rb b/lib/puppet/configurer/downloader.rb index b3696201a..54075ee7e 100644 --- a/lib/puppet/configurer/downloader.rb +++ b/lib/puppet/configurer/downloader.rb @@ -63,14 +63,16 @@ class Puppet::Configurer::Downloader private + require 'sys/admin' if Puppet.features.microsoft_windows? + def default_arguments { :path => path, :recurse => true, :source => source, :tag => name, - :owner => Process.uid, - :group => Process.gid, + :owner => Puppet.features.microsoft_windows? ? Sys::Admin.get_login : Process.uid, + :group => Puppet.features.microsoft_windows? ? 'S-1-0-0' : Process.gid, :purge => true, :force => true, :backup => false, diff --git a/lib/puppet/configurer/fact_handler.rb b/lib/puppet/configurer/fact_handler.rb index 803495773..7d18aa9f2 100644 --- a/lib/puppet/configurer/fact_handler.rb +++ b/lib/puppet/configurer/fact_handler.rb @@ -15,7 +15,6 @@ module Puppet::Configurer::FactHandler # finding facts and the 'rest' terminus for caching them. Thus, we'll # compile them and then "cache" them on the server. begin - reload_facter facts = Puppet::Node::Facts.indirection.find(Puppet[:node_name_value]) unless Puppet[:node_name_fact].empty? Puppet[:node_name_value] = facts.values[Puppet[:node_name_fact]] @@ -54,24 +53,4 @@ module Puppet::Configurer::FactHandler Puppet::Configurer::Downloader.new("fact", Puppet[:factdest], Puppet[:factsource], Puppet[:factsignore]).evaluate end - - # Clear out all of the loaded facts and reload them from disk. - # NOTE: This is clumsy and shouldn't be required for later (1.5.x) versions - # of Facter. - def reload_facter - Facter.clear - - # Reload everything. - if Facter.respond_to? :loadfacts - Facter.loadfacts - elsif Facter.respond_to? :load - Facter.load - else - Puppet.warning "You should upgrade your version of Facter to at least 1.3.8" - end - - # This loads all existing facts and any new ones. We have to remove and - # reload because there's no way to unload specific facts. - Puppet::Node::Facts::Facter.load_fact_plugins - end end diff --git a/lib/puppet/defaults.rb b/lib/puppet/defaults.rb index 078923754..26ee43c14 100644 --- a/lib/puppet/defaults.rb +++ b/lib/puppet/defaults.rb @@ -199,6 +199,10 @@ module Puppet essentially means that you can't have any code outside of a node, class, or definition other than in the site manifest."] ) + Puppet.setdefaults(:module_tool, + :module_repository => ['http://forge.puppetlabs.com', "The module repository"], + :module_working_dir => ['$vardir/puppet-module', "The directory into which module tool data is stored"] + ) hostname = Facter["hostname"].value domain = Facter["domain"].value @@ -705,11 +709,11 @@ EOT "Whether to send reports after every transaction." ], :lastrunfile => { :default => "$statedir/last_run_summary.yaml", - :mode => 0660, + :mode => 0644, :desc => "Where puppet agent stores the last run report summary in yaml format." }, :lastrunreport => { :default => "$statedir/last_run_report.yaml", - :mode => 0660, + :mode => 0644, :desc => "Where puppet agent stores the last run report in yaml format." }, :graph => [false, "Whether to create dot graph files for the different diff --git a/lib/puppet/face/instrumentation_data.rb b/lib/puppet/face/instrumentation_data.rb new file mode 100644 index 000000000..1abf7cd00 --- /dev/null +++ b/lib/puppet/face/instrumentation_data.rb @@ -0,0 +1,28 @@ +require 'puppet/indirector/face' + +Puppet::Indirector::Face.define(:instrumentation_data, '0.0.1') do + copyright "Puppet Labs", 2011 + license "Apache 2 license; see COPYING" + + summary "Manage instrumentation listener accumulated data." + description <<-EOT + This subcommand allows to retrieve the various listener data. + EOT + + get_action(:destroy).summary "Invalid for this subcommand." + get_action(:save).summary "Invalid for this subcommand." + get_action(:search).summary "Invalid for this subcommand." + + find = get_action(:find) + find.summary "Retrieve listener data." + find.render_as = :pson + find.returns <<-EOT + The data of an instrumentation listener + EOT + find.examples <<-EOT + Retrieve listener data: + + $ puppet instrumentation_data find performance --terminus rest + EOT + +end diff --git a/lib/puppet/face/instrumentation_listener.rb b/lib/puppet/face/instrumentation_listener.rb new file mode 100644 index 000000000..de4a742a1 --- /dev/null +++ b/lib/puppet/face/instrumentation_listener.rb @@ -0,0 +1,96 @@ +require 'puppet/indirector/face' + +Puppet::Indirector::Face.define(:instrumentation_listener, '0.0.1') do + copyright "Puppet Labs", 2011 + license "Apache 2 license; see COPYING" + + summary "Manage instrumentation listeners." + description <<-EOT + This subcommand enables/disables or list instrumentation listeners. + EOT + + get_action(:destroy).summary "Invalid for this subcommand." + + find = get_action(:find) + find.summary "Retrieve a single listener." + find.render_as = :pson + find.returns <<-EOT + The status of an instrumentation listener + EOT + find.examples <<-EOT + Retrieve a given listener: + + $ puppet instrumentation_listener find performance --terminus rest + EOT + + search = get_action(:search) + search.summary "Retrieve all instrumentation listeners statuses." + search.arguments "<dummy_text>" + search.render_as = :pson + search.returns <<-EOT + The statuses of all instrumentation listeners + EOT + search.short_description <<-EOT + This retrieves all instrumentation listeners + EOT + search.notes <<-EOT + Although this action always returns all instrumentation listeners, it requires a dummy search + key; this is a known bug. + EOT + search.examples <<-EOT + Retrieve the state of the listeners running in the remote puppet master: + + $ puppet instrumentation_listener search x --terminus rest + EOT + + def manage(name, activate) + Puppet::Util::Instrumentation::Listener.indirection.terminus_class = :rest + listener = Puppet::Face[:instrumentation_listener, '0.0.1'].find(name) + if listener + listener.enabled = activate + Puppet::Face[:instrumentation_listener, '0.0.1'].save(listener) + end + end + + action :enable do + summary "Enable a given instrumentation listener." + arguments "<listener>" + returns "Nothing." + description <<-EOT + Enable a given instrumentation listener. After being enabled the listener + will start receiving instrumentation notifications from the probes if those + are enabled. + EOT + examples <<-EOT + Enable the "performance" listener in the running master: + + $ puppet instrumentation_listener enable performance --terminus rest + EOT + + when_invoked do |name, options| + manage(name, true) + end + end + + action :disable do + summary "Disable a given instrumentation listener." + arguments "<listener>" + returns "Nothing." + description <<-EOT + Disable a given instrumentation listener. After being disabled the listener + will stop receiving instrumentation notifications from the probes. + EOT + examples <<-EOT + Disable the "performance" listener in the running master: + + $ puppet instrumentation_listener disable performance --terminus rest + EOT + + when_invoked do |name, options| + manage(name, false) + end + end + + get_action(:save).summary "API only: modify an instrumentation listener status." + get_action(:save).arguments "<listener>" +end diff --git a/lib/puppet/face/instrumentation_probe.rb b/lib/puppet/face/instrumentation_probe.rb new file mode 100644 index 000000000..52b331cb3 --- /dev/null +++ b/lib/puppet/face/instrumentation_probe.rb @@ -0,0 +1,77 @@ +require 'puppet/indirector/face' + +Puppet::Indirector::Face.define(:instrumentation_probe, '0.0.1') do + copyright "Puppet Labs", 2011 + license "Apache 2 license; see COPYING" + + summary "Manage instrumentation probes." + description <<-EOT + This subcommand enables/disables or list instrumentation listeners. + EOT + + get_action(:find).summary "Invalid for this subcommand." + + search = get_action(:search) + search.summary "Retrieve all probe statuses." + search.arguments "<dummy_text>" + search.render_as = :pson + search.returns <<-EOT + The statuses of all instrumentation probes + EOT + search.short_description <<-EOT + This retrieves all instrumentation probes + EOT + search.notes <<-EOT + Although this action always returns all instrumentation probes, it requires a dummy search + key; this is a known bug. + EOT + search.examples <<-EOT + Retrieve the state of the probes running in the remote puppet master: + + $ puppet instrumentation_probe search x --terminus rest + EOT + + action :enable do + summary "Enable all instrumentation probes." + arguments "<dummy>" + returns "Nothing." + description <<-EOT + Enable all instrumentation probes. After being enabled, all enabled listeners + will start receiving instrumentation notifications from the probes. + EOT + examples <<-EOT + Enable the probes for the running master: + + $ puppet instrumentation_probe enable x --terminus rest + EOT + + when_invoked do |name, options| + Puppet::Face[:instrumentation_probe, '0.0.1'].save(nil) + end + end + + action :disable do + summary "Disable all instrumentation probes." + arguments "<dummy>" + returns "Nothing." + description <<-EOT + Disable all instrumentation probes. After being disabled, no listeners + will receive instrumentation notifications. + EOT + examples <<-EOT + Disable the probes for the running master: + + $ puppet instrumentation_probe disable x --terminus rest + EOT + + when_invoked do |name, options| + Puppet::Face[:instrumentation_probe, '0.0.1'].destroy(nil) + end + end + + get_action(:save).summary "API only: enable all instrumentation probes." + get_action(:save).arguments "<dummy>" + + get_action(:destroy).summary "API only: disable all instrumentation probes." + get_action(:destroy).arguments "<dummy>" +end diff --git a/lib/puppet/face/module/build.rb b/lib/puppet/face/module/build.rb new file mode 100644 index 000000000..227363448 --- /dev/null +++ b/lib/puppet/face/module/build.rb @@ -0,0 +1,31 @@ +Puppet::Face.define(:module, '1.0.0') do + action(:build) do + summary "Build a module release package." + description <<-EOT + Build a module release archive file by processing the Modulefile in the + module directory. The release archive file will be stored in the pkg + directory of the module directory. + EOT + + returns "Pathname object representing the path to the release archive." + + examples <<-EOT + Build a module release: + + $ puppet module build puppetlabs-apache + notice: Building /Users/kelseyhightower/puppetlabs-apache for release + puppetlabs-apache/pkg/puppetlabs-apache-0.0.1.tar.gz + EOT + + arguments "<path>" + + when_invoked do |path, options| + Puppet::Module::Tool::Applications::Builder.run(path, options) + end + + when_rendering :console do |return_value| + # Get the string representation of the Pathname object. + return_value.to_s + end + end +end diff --git a/lib/puppet/face/module/changes.rb b/lib/puppet/face/module/changes.rb new file mode 100644 index 000000000..026661107 --- /dev/null +++ b/lib/puppet/face/module/changes.rb @@ -0,0 +1,38 @@ +Puppet::Face.define(:module, '1.0.0') do + action(:changes) do + summary "Show modified files of an installed module." + description <<-EOT + Show files that have been modified after installation of a given module + by comparing the on-disk md5 checksum of each file against the module's + metadata. + EOT + + returns "Array of strings representing paths of modified files." + + examples <<-EOT + Show modified files of an installed module: + + $ puppet module changes /etc/puppet/modules/vcsrepo/ + warning: 1 files modified + lib/puppet/provider/vcsrepo.rb + EOT + + arguments "<path>" + + when_invoked do |path, options| + root_path = Puppet::Module::Tool.find_module_root(path) + Puppet::Module::Tool::Applications::Checksummer.run(root_path, options) + end + + when_rendering :console do |return_value| + if return_value.empty? + Puppet.notice "No modified files" + else + Puppet.warning "#{return_value.size} files modified" + end + return_value.map do |changed_file| + "#{changed_file}" + end.join("\n") + end + end +end diff --git a/lib/puppet/face/module/clean.rb b/lib/puppet/face/module/clean.rb new file mode 100644 index 000000000..637263057 --- /dev/null +++ b/lib/puppet/face/module/clean.rb @@ -0,0 +1,30 @@ +Puppet::Face.define(:module, '1.0.0') do + action(:clean) do + summary "Clean the module download cache." + description <<-EOT + Clean the module download cache. + EOT + + returns <<-EOT + Return a status Hash: + + { :status => "success", :msg => "Cleaned module cache." } + EOT + + examples <<-EOT + Clean the module download cache: + + $ puppet module clean + Cleaned module cache. + EOT + + when_invoked do |options| + Puppet::Module::Tool::Applications::Cleaner.run(options) + end + + when_rendering :console do |return_value| + # Print the status message to the console. + return_value[:msg] + end + end +end diff --git a/lib/puppet/face/module/generate.rb b/lib/puppet/face/module/generate.rb new file mode 100644 index 000000000..b9dc354bf --- /dev/null +++ b/lib/puppet/face/module/generate.rb @@ -0,0 +1,40 @@ +Puppet::Face.define(:module, '1.0.0') do + action(:generate) do + summary "Generate boilerplate for a new module." + description <<-EOT + Generate boilerplate for a new module by creating a directory + pre-populated with a directory structure and files recommended for + Puppet best practices. + EOT + + returns "Array of Pathname objects representing paths of generated files." + + examples <<-EOT + Generate a new module in the current directory: + + $ puppet module generate puppetlabs-ssh + notice: Generating module at /Users/kelseyhightower/puppetlabs-ssh + puppetlabs-ssh + puppetlabs-ssh/tests + puppetlabs-ssh/tests/init.pp + puppetlabs-ssh/spec + puppetlabs-ssh/spec/spec_helper.rb + puppetlabs-ssh/spec/spec.opts + puppetlabs-ssh/README + puppetlabs-ssh/Modulefile + puppetlabs-ssh/metadata.json + puppetlabs-ssh/manifests + puppetlabs-ssh/manifests/init.pp + EOT + + arguments "<name>" + + when_invoked do |name, options| + Puppet::Module::Tool::Applications::Generator.run(name, options) + end + + when_rendering :console do |return_value| + return_value.map {|f| f.to_s }.join("\n") + end + end +end diff --git a/lib/puppet/face/module/install.rb b/lib/puppet/face/module/install.rb new file mode 100644 index 000000000..8f95ff485 --- /dev/null +++ b/lib/puppet/face/module/install.rb @@ -0,0 +1,83 @@ +Puppet::Face.define(:module, '1.0.0') do + action(:install) do + summary "Install a module from a repository or release archive." + description <<-EOT + Install a module from a release archive file on-disk or by downloading + one from a repository. Unpack the archive into the install directory + specified by the --install-dir option, which defaults to + #{Puppet.settings[:modulepath].split(File::PATH_SEPARATOR).first} + EOT + + returns "Pathname object representing the path to the installed module." + + examples <<-EOT + Install a module from the default repository: + + $ puppet module install puppetlabs/vcsrepo + notice: Installing puppetlabs-vcsrepo-0.0.4.tar.gz to /etc/puppet/modules/vcsrepo + /etc/puppet/modules/vcsrepo + + Install a specific module version from a repository: + + $ puppet module install puppetlabs/vcsrepo -v 0.0.4 + notice: Installing puppetlabs-vcsrepo-0.0.4.tar.gz to /etc/puppet/modules/vcsrepo + /etc/puppet/modules/vcsrepo + + Install a module into a specific directory: + + $ puppet module install puppetlabs/vcsrepo --install-dir=/usr/share/puppet/modules + notice: Installing puppetlabs-vcsrepo-0.0.4.tar.gz to /usr/share/puppet/modules/vcsrepo + /usr/share/puppet/modules/vcsrepo + + Install a module from a release archive: + + $ puppet module install puppetlabs-vcsrepo-0.0.4.tar.gz + notice: Installing puppetlabs-vcsrepo-0.0.4.tar.gz to /etc/puppet/modules/vcsrepo + /etc/puppet/modules/vcsrepo + EOT + + arguments "<name>" + + option "--force", "-f" do + summary "Force overwrite of existing module, if any." + description <<-EOT + Force overwrite of existing module, if any. + EOT + end + + option "--install-dir=", "-i=" do + default_to { Puppet.settings[:modulepath].split(File::PATH_SEPARATOR).first } + summary "The directory into which modules are installed." + description <<-EOT + The directory into which modules are installed, defaults to the first + directory in the modulepath. + EOT + end + + option "--module-repository=", "-r=" do + default_to { Puppet.settings[:module_repository] } + summary "Module repository to use." + description <<-EOT + Module repository to use. + EOT + end + + option "--version=", "-v=" do + summary "Module version to install." + description <<-EOT + Module version to install, can be a requirement string, eg '>= 1.0.3', + defaults to latest version. + EOT + end + + when_invoked do |name, options| + Puppet::Module::Tool::Applications::Installer.run(name, options) + end + + when_rendering :console do |return_value| + # Get the string representation of the Pathname object and print it to + # the console. + return_value.to_s + end + end +end diff --git a/lib/puppet/face/module/list.rb b/lib/puppet/face/module/list.rb new file mode 100644 index 000000000..772990069 --- /dev/null +++ b/lib/puppet/face/module/list.rb @@ -0,0 +1,64 @@ +Puppet::Face.define(:module, '1.0.0') do + action(:list) do + summary "List installed modules" + description <<-HEREDOC + List puppet modules from a specific environment, specified modulepath or + default to listing modules in the default modulepath: + #{Puppet.settings[:modulepath]} + HEREDOC + returns "hash of paths to module objects" + + option "--env ENVIRONMENT" do + summary "Which environments' modules to list" + end + + option "--modulepath MODULEPATH" do + summary "Which directories to look for modules in" + end + + examples <<-EOT + List installed modules: + + $ puppet module list + /etc/puppet/modules + bacula (0.0.2) + /usr/share/puppet/modules + apache (0.0.3) + bacula (0.0.1) + + List installed modules from a specified environment: + + $ puppet module list --env 'test' + /tmp/puppet/modules + rrd (0.0.2) + + List installed modules from a specified modulepath: + + $ puppet module list --modulepath /tmp/facts1:/tmp/facts2 + /tmp/facts1 + stdlib + /tmp/facts2 + nginx (1.0.0) + EOT + + when_invoked do |options| + Puppet[:modulepath] = options[:modulepath] if options[:modulepath] + environment = Puppet::Node::Environment.new(options[:env]) + + environment.modules_by_path + end + + when_rendering :console do |modules_by_path| + output = '' + modules_by_path.each do |path, modules| + output << "#{path}\n" + modules.each do |mod| + version_string = mod.version ? "(#{mod.version})" : '' + output << " #{mod.name} #{version_string}\n" + end + end + output + end + + end +end diff --git a/lib/puppet/face/module/search.rb b/lib/puppet/face/module/search.rb new file mode 100644 index 000000000..fdc2c983d --- /dev/null +++ b/lib/puppet/face/module/search.rb @@ -0,0 +1,66 @@ +Puppet::Face.define(:module, '1.0.0') do + action(:search) do + summary "Search a repository for a module." + description <<-EOT + Search a repository for modules whose names match a specific substring. + EOT + + returns "Array of module metadata hashes" + + examples <<-EOT + Search the default repository for a module: + + $ puppet module search puppetlabs + notice: Searching http://forge.puppetlabs.com + notice: 24 found. + puppetlabs/apache (0.0.3) + puppetlabs/collectd (0.0.1) + puppetlabs/ruby (0.0.1) + puppetlabs/vcsrepo (0.0.4) + puppetlabs/gcc (0.0.3) + puppetlabs/passenger (0.0.2) + puppetlabs/DeveloperBootstrap (0.0.5) + jeffmccune/tomcat (1.0.1) + puppetlabs/motd (1.0.0) + puppetlabs/lvm (0.1.0) + puppetlabs/rabbitmq (1.0.4) + puppetlabs/prosvc_repo (1.0.1) + puppetlabs/stdlib (2.2.0) + puppetlabs/java (0.1.5) + puppetlabs/activemq (0.1.6) + puppetlabs/mcollective (0.1.8) + puppetlabs/git (0.0.2) + puppetlabs/ntp (0.0.4) + puppetlabs/nginx (0.0.1) + puppetlabs/cloud_provisioner (0.6.0rc1) + puppetlabs/mrepo (0.1.1) + puppetlabs/f5 (0.1.0) + puppetlabs/firewall (0.0.3) + puppetlabs/bprobe (0.0.3) + EOT + + arguments "<term>" + + option "--module-repository=", "-r=" do + default_to { Puppet.settings[:module_repository] } + summary "Module repository to use." + description <<-EOT + Module repository to use. + EOT + end + + when_invoked do |term, options| + Puppet.notice "Searching #{options[:module_repository]}" + Puppet::Module::Tool::Applications::Searcher.run(term, options) + end + + when_rendering :console do |return_value| + Puppet.notice "#{return_value.size} found." + return_value.map do |match| + # We reference the full_name here when referring to the full_module_name, + # because full_name is what is returned from the forge API call. + "#{match['full_name']} (#{match['version']})" + end.join("\n") + end + end +end diff --git a/lib/puppet/face/module/uninstall.rb b/lib/puppet/face/module/uninstall.rb new file mode 100644 index 000000000..19c983ea6 --- /dev/null +++ b/lib/puppet/face/module/uninstall.rb @@ -0,0 +1,50 @@ +Puppet::Face.define(:module, '1.0.0') do + action(:uninstall) do + summary "Uninstall a puppet module." + description <<-EOT + Uninstall a puppet module from the modulepath or a specific + target directory which defaults to + #{Puppet.settings[:modulepath].split(File::PATH_SEPARATOR).join(', ')}. + EOT + + returns "Array of strings representing paths of uninstalled files." + + examples <<-EOT + Uninstall a module from all directories in the modulepath: + + $ puppet module uninstall ssh + Removed /etc/puppet/modules/ssh + + Uninstall a module from a specific directory: + + $ puppet module uninstall --target-directory /usr/share/puppet/modules ssh + Removed /usr/share/puppet/modules/ssh + EOT + + arguments "<name>" + + option "--target-directory=", "-t=" do + default_to { Puppet.settings[:modulepath].split(File::PATH_SEPARATOR) } + summary "The target directory to search from modules." + description <<-EOT + The target directory to search for modules. + EOT + end + + when_invoked do |name, options| + + if options[:target_directory].is_a?(Array) + options[:target_directories] = options[:target_directory] + else + options[:target_directories] = [ options[:target_directory] ] + end + options.delete(:target_directory) + + Puppet::Module::Tool::Applications::Uninstaller.run(name, options) + end + + when_rendering :console do |removed_modules| + removed_modules.map { |path| "Removed #{path}" }.join('\n') + end + end +end diff --git a/lib/puppet/face/node/clean.rb b/lib/puppet/face/node/clean.rb index d2852de04..a4df1bfaf 100644 --- a/lib/puppet/face/node/clean.rb +++ b/lib/puppet/face/node/clean.rb @@ -54,10 +54,7 @@ Puppet::Face.define(:node, '0.0.1') do clean_cached_facts(node) clean_cached_node(node) clean_reports(node) - - # This is roughly functional, but seems to introduce order-dependent test - # failures; this can be re-added when those issues are resolved. - # clean_storeconfigs(node, unexport) + clean_storeconfigs(node, unexport) end # clean signed cert for +host+ diff --git a/lib/puppet/file_serving/content.rb b/lib/puppet/file_serving/content.rb index d8413b557..eda141d91 100644 --- a/lib/puppet/file_serving/content.rb +++ b/lib/puppet/file_serving/content.rb @@ -35,7 +35,7 @@ class Puppet::FileServing::Content < Puppet::FileServing::Base # This stat can raise an exception, too. raise(ArgumentError, "Cannot read the contents of links unless following links") if stat.ftype == "symlink" - @content = ::File.read(full_path) + @content = Puppet::Util.binread(full_path) end @content end diff --git a/lib/puppet/indirector/exec.rb b/lib/puppet/indirector/exec.rb index 8ea13ff95..e6325adaa 100644 --- a/lib/puppet/indirector/exec.rb +++ b/lib/puppet/indirector/exec.rb @@ -39,7 +39,7 @@ class Puppet::Indirector::Exec < Puppet::Indirector::Terminus end if output =~ /\A\s*\Z/ # all whitespace - Puppet.debug "Empty response for #{name} from exec #{self.name} terminus" + Puppet.debug "Empty response for #{name} from #{self.name} terminus" return nil else return output diff --git a/lib/puppet/indirector/facts/facter.rb b/lib/puppet/indirector/facts/facter.rb index 6312a95fb..e41ef02e7 100644 --- a/lib/puppet/indirector/facts/facter.rb +++ b/lib/puppet/indirector/facts/facter.rb @@ -7,6 +7,22 @@ class Puppet::Node::Facts::Facter < Puppet::Indirector::Code returns the local host's facts, regardless of what you attempt to find." + # Clear out all of the loaded facts. Reload facter but not puppet facts. + # NOTE: This is clumsy and shouldn't be required for later (1.5.x) versions + # of Facter. + def self.reload_facter + Facter.clear + + # Reload everything. + if Facter.respond_to? :loadfacts + Facter.loadfacts + elsif Facter.respond_to? :load + Facter.load + else + Puppet.warning "You should upgrade your version of Facter to at least 1.3.8" + end + end + def self.load_fact_plugins # Add any per-module fact directories to the factpath module_fact_dirs = Puppet[:modulepath].split(File::PATH_SEPARATOR).collect do |d| @@ -15,7 +31,7 @@ class Puppet::Node::Facts::Facter < Puppet::Indirector::Code end end.flatten dirs = module_fact_dirs + Puppet[:factpath].split(File::PATH_SEPARATOR) - x = dirs.each do |dir| + x = dirs.uniq.each do |dir| load_facts_in_dir(dir) end end @@ -27,7 +43,7 @@ class Puppet::Node::Facts::Facter < Puppet::Indirector::Code Dir.glob("*.rb").each do |file| fqfile = ::File.join(dir, file) begin - Puppet.info "Loading facts in #{::File.basename(file.sub(".rb",''))}" + Puppet.info "Loading facts in #{fqfile}" Timeout::timeout(self.timeout) do load file end @@ -57,17 +73,14 @@ class Puppet::Node::Facts::Facter < Puppet::Indirector::Code timeout end - def initialize(*args) - super - self.class.load_fact_plugins - end - def destroy(facts) raise Puppet::DevError, "You cannot destroy facts in the code store; it is only used for getting facts from Facter" end # Look a host's facts up in Facter. def find(request) + self.class.reload_facter + self.class.load_fact_plugins result = Puppet::Node::Facts.new(request.key, Facter.to_hash) result.add_local_facts diff --git a/lib/puppet/indirector/facts/inventory_active_record.rb b/lib/puppet/indirector/facts/inventory_active_record.rb index db4c63f00..93eee8a97 100644 --- a/lib/puppet/indirector/facts/inventory_active_record.rb +++ b/lib/puppet/indirector/facts/inventory_active_record.rb @@ -2,6 +2,7 @@ require 'puppet/rails' require 'puppet/rails/inventory_node' require 'puppet/rails/inventory_fact' require 'puppet/indirector/active_record' +require 'puppet/util/retryaction' class Puppet::Node::Facts::InventoryActiveRecord < Puppet::Indirector::ActiveRecord def find(request) @@ -13,19 +14,21 @@ class Puppet::Node::Facts::InventoryActiveRecord < Puppet::Indirector::ActiveRec end def save(request) - facts = request.instance - node = Puppet::Rails::InventoryNode.find_by_name(request.key) || Puppet::Rails::InventoryNode.create(:name => request.key, :timestamp => facts.timestamp) - node.timestamp = facts.timestamp + Puppet::Util::RetryAction.retry_action :retries => 4, :retry_exceptions => {ActiveRecord::StatementInvalid => 'MySQL Error. Retrying'} do + facts = request.instance + node = Puppet::Rails::InventoryNode.find_by_name(request.key) || Puppet::Rails::InventoryNode.create(:name => request.key, :timestamp => facts.timestamp) + node.timestamp = facts.timestamp - ActiveRecord::Base.transaction do - Puppet::Rails::InventoryFact.delete_all(:node_id => node.id) - # We don't want to save internal values as facts, because those are - # metadata that belong on the node - facts.values.each do |name,value| - next if name.to_s =~ /^_/ - node.facts.build(:name => name, :value => value) + ActiveRecord::Base.transaction do + Puppet::Rails::InventoryFact.delete_all(:node_id => node.id) + # We don't want to save internal values as facts, because those are + # metadata that belong on the node + facts.values.each do |name,value| + next if name.to_s =~ /^_/ + node.facts.build(:name => name, :value => value) + end + node.save end - node.save end end diff --git a/lib/puppet/indirector/indirection.rb b/lib/puppet/indirector/indirection.rb index 20b260b83..7296be2b9 100644 --- a/lib/puppet/indirector/indirection.rb +++ b/lib/puppet/indirector/indirection.rb @@ -1,12 +1,19 @@ require 'puppet/util/docs' require 'puppet/indirector/envelope' require 'puppet/indirector/request' +require 'puppet/util/instrumentation/instrumentable' # The class that connects functional classes with their different collection # back-ends. Each indirection has a set of associated terminus classes, # each of which is a subclass of Puppet::Indirector::Terminus. class Puppet::Indirector::Indirection include Puppet::Util::Docs + extend Puppet::Util::Instrumentation::Instrumentable + + probe :find, :label => Proc.new { |parent, key, *args| "find_#{parent.name}_#{parent.terminus_class}" }, :data => Proc.new { |parent, key, *args| { :key => key }} + probe :save, :label => Proc.new { |parent, key, *args| "save_#{parent.name}_#{parent.terminus_class}" }, :data => Proc.new { |parent, key, *args| { :key => key }} + probe :search, :label => Proc.new { |parent, key, *args| "search_#{parent.name}_#{parent.terminus_class}" }, :data => Proc.new { |parent, key, *args| { :key => key }} + probe :destroy, :label => Proc.new { |parent, key, *args| "destroy_#{parent.name}_#{parent.terminus_class}" }, :data => Proc.new { |parent, key, *args| { :key => key }} @@indirections = [] diff --git a/lib/puppet/indirector/instrumentation_data.rb b/lib/puppet/indirector/instrumentation_data.rb new file mode 100644 index 000000000..f1bea330d --- /dev/null +++ b/lib/puppet/indirector/instrumentation_data.rb @@ -0,0 +1,3 @@ +# A stub class, so our constants work. +class Puppet::Indirector::InstrumentationData +end diff --git a/lib/puppet/indirector/instrumentation_data/local.rb b/lib/puppet/indirector/instrumentation_data/local.rb new file mode 100644 index 000000000..6510d7f2d --- /dev/null +++ b/lib/puppet/indirector/instrumentation_data/local.rb @@ -0,0 +1,19 @@ +require 'puppet/indirector/instrumentation_data' + +class Puppet::Indirector::InstrumentationData::Local < Puppet::Indirector::Code + def find(request) + model.new(request.key) + end + + def search(request) + raise Puppet::DevError, "You cannot search for instrumentation data" + end + + def save(request) + raise Puppet::DevError, "You cannot save instrumentation data" + end + + def destroy(request) + raise Puppet::DevError, "You cannot remove instrumentation data" + end +end diff --git a/lib/puppet/indirector/instrumentation_data/rest.rb b/lib/puppet/indirector/instrumentation_data/rest.rb new file mode 100644 index 000000000..28b211781 --- /dev/null +++ b/lib/puppet/indirector/instrumentation_data/rest.rb @@ -0,0 +1,5 @@ +require 'puppet/indirector/rest' +require 'puppet/indirector/instrumentation_data' + +class Puppet::Indirector::InstrumentationData::Rest < Puppet::Indirector::REST +end diff --git a/lib/puppet/indirector/instrumentation_listener.rb b/lib/puppet/indirector/instrumentation_listener.rb new file mode 100644 index 000000000..6aaa71ce6 --- /dev/null +++ b/lib/puppet/indirector/instrumentation_listener.rb @@ -0,0 +1,3 @@ +# A stub class, so our constants work. +class Puppet::Indirector::InstrumentationListener +end diff --git a/lib/puppet/indirector/instrumentation_listener/local.rb b/lib/puppet/indirector/instrumentation_listener/local.rb new file mode 100644 index 000000000..72578014c --- /dev/null +++ b/lib/puppet/indirector/instrumentation_listener/local.rb @@ -0,0 +1,23 @@ +require 'puppet/indirector/instrumentation_listener' + +class Puppet::Indirector::InstrumentationListener::Local < Puppet::Indirector::Code + def find(request) + Puppet::Util::Instrumentation[request.key] + end + + def search(request) + Puppet::Util::Instrumentation.listeners + end + + def save(request) + res = request.instance + Puppet::Util::Instrumentation[res.name] = res + nil # don't leak the listener + end + + def destroy(request) + listener = Puppet::Util::Instrumentation[request.key] + raise "Listener #{request.key} hasn't been subscribed" unless listener + Puppet::Util::Instrumentation.unsubscribe(listener) + end +end diff --git a/lib/puppet/indirector/instrumentation_listener/rest.rb b/lib/puppet/indirector/instrumentation_listener/rest.rb new file mode 100644 index 000000000..0bc8122ea --- /dev/null +++ b/lib/puppet/indirector/instrumentation_listener/rest.rb @@ -0,0 +1,5 @@ +require 'puppet/indirector/instrumentation_listener' +require 'puppet/indirector/rest' + +class Puppet::Indirector::InstrumentationListener::Rest < Puppet::Indirector::REST +end diff --git a/lib/puppet/indirector/instrumentation_probe.rb b/lib/puppet/indirector/instrumentation_probe.rb new file mode 100644 index 000000000..3e514447a --- /dev/null +++ b/lib/puppet/indirector/instrumentation_probe.rb @@ -0,0 +1,3 @@ +# A stub class, so our constants work. +class Puppet::Indirector::InstrumentationProbe +end diff --git a/lib/puppet/indirector/instrumentation_probe/local.rb b/lib/puppet/indirector/instrumentation_probe/local.rb new file mode 100644 index 000000000..dd0a3f707 --- /dev/null +++ b/lib/puppet/indirector/instrumentation_probe/local.rb @@ -0,0 +1,24 @@ +require 'puppet/indirector/instrumentation_probe' +require 'puppet/indirector/code' +require 'puppet/util/instrumentation/indirection_probe' + +class Puppet::Indirector::InstrumentationProbe::Local < Puppet::Indirector::Code + def find(request) + end + + def search(request) + probes = [] + Puppet::Util::Instrumentation::Instrumentable.each_probe do |probe| + probes << Puppet::Util::Instrumentation::IndirectionProbe.new("#{probe.klass}.#{probe.method}") + end + probes + end + + def save(request) + Puppet::Util::Instrumentation::Instrumentable.enable_probes + end + + def destroy(request) + Puppet::Util::Instrumentation::Instrumentable.disable_probes + end +end diff --git a/lib/puppet/indirector/instrumentation_probe/rest.rb b/lib/puppet/indirector/instrumentation_probe/rest.rb new file mode 100644 index 000000000..57e6fcf3d --- /dev/null +++ b/lib/puppet/indirector/instrumentation_probe/rest.rb @@ -0,0 +1,5 @@ +require 'puppet/indirector/rest' +require 'puppet/indirector/instrumentation_probe' + +class Puppet::Indirector::InstrumentationProbe::Rest < Puppet::Indirector::REST +end diff --git a/lib/puppet/indirector/rest.rb b/lib/puppet/indirector/rest.rb index c8a45d76d..d76a2890f 100644 --- a/lib/puppet/indirector/rest.rb +++ b/lib/puppet/indirector/rest.rb @@ -95,7 +95,7 @@ class Puppet::Indirector::REST < Puppet::Indirector::Terminus rescue OpenSSL::SSL::SSLError => error if error.message.include? "certificate verify failed" raise Puppet::Error, "#{error.message}. This is often because the time is out of sync on the server or client" - elsif error.message.include? "hostname was not match" + elsif error.message =~ /hostname (was )?not match/ raise unless cert = peer_certs.find { |c| c.name !~ /^puppet ca/i } valid_certnames = [cert.name, *cert.subject_alt_names].uniq diff --git a/lib/puppet/module.rb b/lib/puppet/module.rb index 00468df96..7f86df83d 100644 --- a/lib/puppet/module.rb +++ b/lib/puppet/module.rb @@ -19,13 +19,6 @@ class Puppet::Module FILETYPES = [MANIFESTS, FILES, TEMPLATES, PLUGINS] - # Return an array of paths by splitting the +modulepath+ config - # parameter. Only consider paths that are absolute and existing - # directories - def self.modulepath(environment = nil) - Puppet::Node::Environment.new(environment).modulepath - end - # Find and return the +module+ that +path+ belongs to. If +path+ is # absolute, or if there is no module whose name is the first component # of +path+, return +nil+ @@ -48,15 +41,16 @@ class Puppet::Module return metadata.is_a?(Hash) && !metadata.keys.empty? end - def initialize(name, environment = nil) + def initialize(name, options = {}) @name = name + @path = options[:path] assert_validity - if environment.is_a?(Puppet::Node::Environment) - @environment = environment + if options[:environment].is_a?(Puppet::Node::Environment) + @environment = options[:environment] else - @environment = Puppet::Node::Environment.new(environment) + @environment = Puppet::Node::Environment.new(options[:environment]) end load_metadata if has_metadata? @@ -141,7 +135,7 @@ class Puppet::Module # Find this module in the modulepath. def path - environment.modulepath.collect { |path| File.join(path, name) }.find { |d| FileTest.directory?(d) } + @path ||= environment.modulepath.collect { |path| File.join(path, name) }.find { |d| FileTest.directory?(d) } end # Find all plugin directories. This is used by the Plugins fileserving mount. @@ -149,11 +143,6 @@ class Puppet::Module subpath("plugins") end - def requires(name, version = nil) - @requires ||= [] - @requires << [name, version] - end - def supports(name, version = nil) @supports ||= [] @supports << [name, version] @@ -204,4 +193,11 @@ class Puppet::Module def assert_validity raise InvalidName, "Invalid module name; module names must be alphanumeric (plus '-'), not '#{name}'" unless name =~ /^[-\w]+$/ end + + def ==(other) + self.name == other.name && + self.version == other.version && + self.path == other.path && + self.environment == other.environment + end end diff --git a/lib/puppet/module_tool.rb b/lib/puppet/module_tool.rb new file mode 100644 index 000000000..c11fd58b1 --- /dev/null +++ b/lib/puppet/module_tool.rb @@ -0,0 +1,97 @@ +# Load standard libraries +require 'pathname' +require 'fileutils' +require 'puppet/module_tool/utils' + +# Define tool +module Puppet + class Module + module Tool + + # Directory names that should not be checksummed. + ARTIFACTS = ['pkg', /^\./, /^~/, /^#/, 'coverage'] + FULL_MODULE_NAME_PATTERN = /\A([^-\/|.]+)[-|\/](.+)\z/ + REPOSITORY_URL = Puppet.settings[:module_repository] + + # Is this a directory that shouldn't be checksummed? + # + # TODO: Should this be part of Checksums? + # TODO: Rename this method to reflect it's purpose? + # TODO: Shouldn't this be used when building packages too? + def self.artifact?(path) + case File.basename(path) + when *ARTIFACTS + true + else + false + end + end + + # Return the +username+ and +modname+ for a given +full_module_name+, or raise an + # ArgumentError if the argument isn't parseable. + def self.username_and_modname_from(full_module_name) + if matcher = full_module_name.match(FULL_MODULE_NAME_PATTERN) + return matcher.captures + else + raise ArgumentError, "Not a valid full name: #{full_module_name}" + end + end + + # Read HTTP proxy configurationm from Puppet's config file, or the + # http_proxy environment variable. + def self.http_proxy_env + proxy_env = ENV["http_proxy"] || ENV["HTTP_PROXY"] || nil + begin + return URI.parse(proxy_env) if proxy_env + rescue URI::InvalidURIError + return nil + end + return nil + end + + def self.http_proxy_host + env = http_proxy_env + + if env and env.host then + return env.host + end + + if Puppet.settings[:http_proxy_host] == 'none' + return nil + end + + return Puppet.settings[:http_proxy_host] + end + + def self.http_proxy_port + env = http_proxy_env + + if env and env.port then + return env.port + end + + return Puppet.settings[:http_proxy_port] + end + + def self.find_module_root(path) + for dir in [path, Dir.pwd].compact + if File.exist?(File.join(dir, 'Modulefile')) + return dir + end + end + raise ArgumentError, "Could not find a valid module at #{path ? path.inspect : 'current directory'}" + end + end + end +end + +# Load remaining libraries +require 'puppet/module_tool/applications' +require 'puppet/module_tool/cache' +require 'puppet/module_tool/checksums' +require 'puppet/module_tool/contents_description' +require 'puppet/module_tool/dependency' +require 'puppet/module_tool/metadata' +require 'puppet/module_tool/modulefile' +require 'puppet/module_tool/repository' +require 'puppet/module_tool/skeleton' diff --git a/lib/puppet/module_tool/applications.rb b/lib/puppet/module_tool/applications.rb new file mode 100644 index 000000000..24bcbe279 --- /dev/null +++ b/lib/puppet/module_tool/applications.rb @@ -0,0 +1,13 @@ +module Puppet::Module::Tool + module Applications + require 'puppet/module_tool/applications/application' + require 'puppet/module_tool/applications/builder' + require 'puppet/module_tool/applications/checksummer' + require 'puppet/module_tool/applications/cleaner' + require 'puppet/module_tool/applications/generator' + require 'puppet/module_tool/applications/installer' + require 'puppet/module_tool/applications/searcher' + require 'puppet/module_tool/applications/unpacker' + require 'puppet/module_tool/applications/uninstaller' + end +end diff --git a/lib/puppet/module_tool/applications/application.rb b/lib/puppet/module_tool/applications/application.rb new file mode 100644 index 000000000..5f8c0c4bf --- /dev/null +++ b/lib/puppet/module_tool/applications/application.rb @@ -0,0 +1,83 @@ +require 'net/http' +require 'semver' + +module Puppet::Module::Tool + module Applications + class Application + include Utils::Interrogation + + def self.run(*args) + new(*args).run + end + + attr_accessor :options + + def initialize(options = {}) + @options = options + end + + def repository + @repository ||= Repository.new(@options[:module_repository]) + end + + def run + raise NotImplementedError, "Should be implemented in child classes." + end + + def discuss(response, success, failure) + case response + when Net::HTTPOK, Net::HTTPCreated + Puppet.notice success + else + errors = PSON.parse(response.body)['error'] rescue "HTTP #{response.code}, #{response.body}" + Puppet.warning "#{failure} (#{errors})" + end + end + + def metadata(require_modulefile = false) + unless @metadata + unless @path + raise ArgumentError, "Could not determine module path" + end + @metadata = Puppet::Module::Tool::Metadata.new + contents = ContentsDescription.new(@path) + contents.annotate(@metadata) + checksums = Checksums.new(@path) + checksums.annotate(@metadata) + modulefile_path = File.join(@path, 'Modulefile') + if File.file?(modulefile_path) + Puppet::Module::Tool::ModulefileReader.evaluate(@metadata, modulefile_path) + elsif require_modulefile + raise ArgumentError, "No Modulefile found." + end + end + @metadata + end + + def load_modulefile! + @metadata = nil + metadata(true) + end + + # Use to extract and validate a module name and version from a + # filename + # Note: Must have @filename set to use this + def parse_filename! + @release_name = File.basename(@filename,'.tar.gz') + match = /^(.*?)-(.*?)-(\d+\.\d+\.\d+.*?)$/.match(@release_name) + if match then + @username, @module_name, @version = match.captures + else + raise ArgumentError, "Could not parse filename to obtain the username, module name and version. (#{@release_name})" + end + @full_module_name = [@username, @module_name].join('-') + unless @username && @module_name + raise ArgumentError, "Username and Module name not provided" + end + unless SemVer.valid?(@version) + raise ArgumentError, "Invalid version format: #{@version} (Semantic Versions are acceptable: http://semver.org)" + end + end + end + end +end diff --git a/lib/puppet/module_tool/applications/builder.rb b/lib/puppet/module_tool/applications/builder.rb new file mode 100644 index 000000000..322020a8b --- /dev/null +++ b/lib/puppet/module_tool/applications/builder.rb @@ -0,0 +1,91 @@ +require 'fileutils' + +module Puppet::Module::Tool + module Applications + class Builder < Application + + def initialize(path, options = {}) + @path = File.expand_path(Puppet::Module::Tool.find_module_root(path)) + @pkg_path = File.join(@path, 'pkg') + super(options) + end + + def run + load_modulefile! + create_directory + copy_contents + add_metadata + Puppet.notice "Building #{@path} for release" + tar + gzip + relative = Pathname.new(File.join(@pkg_path, filename('tar.gz'))).relative_path_from(Pathname.new(Dir.pwd)) + + # Return the Pathname object representing the path to the release + # archive just created. This return value is used by the module_tool + # face build action, and displayed to on the console using the to_s + # method. + # + # Example return value: + # + # <Pathname:puppetlabs-apache/pkg/puppetlabs-apache-0.0.1.tar.gz> + # + relative + end + + private + + def filename(ext) + ext.sub!(/^\./, '') + "#{metadata.release_name}.#{ext}" + end + + def tar + tar_name = filename('tar') + Dir.chdir(@pkg_path) do + FileUtils.rm tar_name rescue nil + unless system "tar -cf #{tar_name} #{metadata.release_name}" + raise RuntimeError, "Could not create #{tar_name}" + end + end + end + + def gzip + Dir.chdir(@pkg_path) do + FileUtils.rm filename('tar.gz') rescue nil + unless system "gzip #{filename('tar')}" + raise RuntimeError, "Could not compress #{filename('tar')}" + end + end + end + + def create_directory + FileUtils.mkdir(@pkg_path) rescue nil + if File.directory?(build_path) + FileUtils.rm_rf(build_path) + end + FileUtils.mkdir(build_path) + end + + def copy_contents + Dir[File.join(@path, '*')].each do |path| + case File.basename(path) + when *Puppet::Module::Tool::ARTIFACTS + next + else + FileUtils.cp_r path, build_path + end + end + end + + def add_metadata + File.open(File.join(build_path, 'metadata.json'), 'w') do |f| + f.write(PSON.pretty_generate(metadata)) + end + end + + def build_path + @build_path ||= File.join(@pkg_path, metadata.release_name) + end + end + end +end diff --git a/lib/puppet/module_tool/applications/checksummer.rb b/lib/puppet/module_tool/applications/checksummer.rb new file mode 100644 index 000000000..2ea1ef587 --- /dev/null +++ b/lib/puppet/module_tool/applications/checksummer.rb @@ -0,0 +1,47 @@ +module Puppet::Module::Tool + module Applications + class Checksummer < Application + + def initialize(path, options = {}) + @path = Pathname.new(path) + super(options) + end + + def run + changes = [] + if metadata_file.exist? + sums = Checksums.new(@path) + (metadata['checksums'] || {}).each do |child_path, canonical_checksum| + path = @path + child_path + if canonical_checksum != sums.checksum(path) + changes << child_path + end + end + else + raise ArgumentError, "No metadata.json found." + end + + # Return an Array of strings representing file paths of files that have + # been modified since this module was installed. All paths are relative + # to the installed module directory. This return value is used by the + # module_tool face changes action, and displayed on the console. + # + # Example return value: + # + # [ "REVISION", "metadata.json", "manifests/init.pp"] + # + changes + end + + private + + def metadata + PSON.parse(metadata_file.read) + end + + def metadata_file + (@path + 'metadata.json') + end + end + end +end diff --git a/lib/puppet/module_tool/applications/cleaner.rb b/lib/puppet/module_tool/applications/cleaner.rb new file mode 100644 index 000000000..c42687fc7 --- /dev/null +++ b/lib/puppet/module_tool/applications/cleaner.rb @@ -0,0 +1,16 @@ +module Puppet::Module::Tool + module Applications + class Cleaner < Application + def run + Puppet::Module::Tool::Cache.clean + + # Return a status Hash containing the status of the clean command + # and a status message. This return value is used by the module_tool + # face clean action, and the status message, return_value[:msg], is + # displayed on the console. + # + { :status => "success", :msg => "Cleaned module cache." } + end + end + end +end diff --git a/lib/puppet/module_tool/applications/generator.rb b/lib/puppet/module_tool/applications/generator.rb new file mode 100644 index 000000000..699fe246d --- /dev/null +++ b/lib/puppet/module_tool/applications/generator.rb @@ -0,0 +1,141 @@ +require 'pathname' +require 'fileutils' +require 'erb' + +module Puppet::Module::Tool + module Applications + class Generator < Application + + def initialize(full_module_name, options = {}) + begin + @metadata = Metadata.new(:full_module_name => full_module_name) + rescue ArgumentError + raise "Could not generate directory #{full_module_name.inspect}, you must specify a dash-separated username and module name." + end + super(options) + end + + def skeleton + @skeleton ||= Skeleton.new + end + + def get_binding + binding + end + + def run + if destination.directory? + raise ArgumentError, "#{destination} already exists." + end + Puppet.notice "Generating module at #{Dir.pwd}/#{@metadata.dashed_name}" + files_created = [] + skeleton.path.find do |path| + if path == skeleton + destination.mkpath + else + node = Node.on(path, self) + if node + node.install! + files_created << node.target + else + Puppet.notice "Could not generate from #{path}" + end + end + end + + # Return an array of Pathname objects representing file paths of files + # and directories just generated. This return value is used by the + # module_tool face generate action, and displayed on the console. + # + # Example return value: + # + # [ + # #<Pathname:puppetlabs-apache>, + # #<Pathname:puppetlabs-apache/tests>, + # #<Pathname:puppetlabs-apache/tests/init.pp>, + # #<Pathname:puppetlabs-apache/spec>, + # #<Pathname:puppetlabs-apache/spec/spec_helper.rb>, + # #<Pathname:puppetlabs-apache/spec/spec.opts>, + # #<Pathname:puppetlabs-apache/README>, + # #<Pathname:puppetlabs-apache/Modulefile>, + # #<Pathname:puppetlabs-apache/metadata.json>, + # #<Pathname:puppetlabs-apache/manifests>, + # #<Pathname:puppetlabs-apache/manifests/init.pp" + # ] + # + files_created + end + + def destination + @destination ||= Pathname.new(@metadata.dashed_name) + end + + class Node + def self.types + @types ||= [] + end + def self.inherited(klass) + types << klass + end + def self.on(path, generator) + klass = types.detect { |t| t.matches?(path) } + if klass + klass.new(path, generator) + end + end + def initialize(source, generator) + @generator = generator + @source = source + end + def read + @source.read + end + def target + target = @generator.destination + @source.relative_path_from(@generator.skeleton.path) + components = target.to_s.split(File::SEPARATOR).map do |part| + part == 'NAME' ? @generator.metadata.name : part + end + Pathname.new(components.join(File::SEPARATOR)) + end + def install! + raise NotImplementedError, "Abstract" + end + end + + class DirectoryNode < Node + def self.matches?(path) + path.directory? + end + def install! + target.mkpath + end + end + + class ParsedFileNode < Node + def self.matches?(path) + path.file? && path.extname == '.erb' + end + def target + path = super + path.parent + path.basename('.erb') + end + def contents + template = ERB.new(read) + template.result(@generator.send(:get_binding)) + end + def install! + target.open('w') { |f| f.write contents } + end + end + + class FileNode < Node + def self.matches?(path) + path.file? + end + def install! + FileUtils.cp(@source, target) + end + end + end + end +end diff --git a/lib/puppet/module_tool/applications/installer.rb b/lib/puppet/module_tool/applications/installer.rb new file mode 100644 index 000000000..ad423bd83 --- /dev/null +++ b/lib/puppet/module_tool/applications/installer.rb @@ -0,0 +1,89 @@ +require 'open-uri' +require 'pathname' +require 'tmpdir' + +module Puppet::Module::Tool + module Applications + class Installer < Application + + def initialize(name, options = {}) + if File.exist?(name) + if File.directory?(name) + # TODO Unify this handling with that of Unpacker#check_clobber! + raise ArgumentError, "Module already installed: #{name}" + end + @source = :filesystem + @filename = File.expand_path(name) + parse_filename! + else + @source = :repository + begin + @username, @module_name = Puppet::Module::Tool::username_and_modname_from(name) + rescue ArgumentError + raise "Could not install module with invalid name: #{name}" + end + @version_requirement = options[:version] + end + super(options) + end + + def force? + options[:force] + end + + def run + case @source + when :repository + if match['file'] + begin + cache_path = repository.retrieve(match['file']) + rescue OpenURI::HTTPError => e + raise RuntimeError, "Could not install module: #{e.message}" + end + module_dir = Unpacker.run(cache_path, options) + else + raise RuntimeError, "Malformed response from module repository." + end + when :filesystem + repository = Repository.new('file:///') + uri = URI.parse("file://#{URI.escape(File.expand_path(@filename))}") + cache_path = repository.retrieve(uri) + module_dir = Unpacker.run(cache_path, options) + else + raise ArgumentError, "Could not determine installation source" + end + + # Return the Pathname object representing the path to the installed + # module. This return value is used by the module_tool face install + # action, and displayed to on the console. + # + # Example return value: + # + # "/etc/puppet/modules/apache" + # + module_dir + end + + private + + def match + return @match ||= begin + url = repository.uri + "/users/#{@username}/modules/#{@module_name}/releases/find.json" + if @version_requirement + url.query = "version=#{URI.escape(@version_requirement)}" + end + begin + raw_result = read_match(url) + rescue => e + raise ArgumentError, "Could not find a release for this module (#{e.message})" + end + @match = PSON.parse(raw_result) + end + end + + def read_match(url) + return url.read + end + end + end +end diff --git a/lib/puppet/module_tool/applications/searcher.rb b/lib/puppet/module_tool/applications/searcher.rb new file mode 100644 index 000000000..0a2267b3c --- /dev/null +++ b/lib/puppet/module_tool/applications/searcher.rb @@ -0,0 +1,40 @@ +module Puppet::Module::Tool + module Applications + class Searcher < Application + + def initialize(term, options = {}) + @term = term + super(options) + end + + def run + request = Net::HTTP::Get.new("/modules.json?q=#{URI.escape(@term)}") + response = repository.make_http_request(request) + case response + when Net::HTTPOK + matches = PSON.parse(response.body) + else + raise RuntimeError, "Could not execute search (HTTP #{response.code})" + matches = [] + end + + # Return a list of module metadata hashes that match the search query. + # This return value is used by the module_tool face install search, + # and displayed to on the console. + # + # Example return value: + # + # [ + # { + # "name" => "nginx", + # "project_url" => "http://github.com/puppetlabs/puppetlabs-nginx", + # "version" => "0.0.1", + # "full_name" => "puppetlabs/nginx" # full_name comes back from + # } # API all to the forge. + # ] + # + matches + end + end + end +end diff --git a/lib/puppet/module_tool/applications/uninstaller.rb b/lib/puppet/module_tool/applications/uninstaller.rb new file mode 100644 index 000000000..9cd4d8bd3 --- /dev/null +++ b/lib/puppet/module_tool/applications/uninstaller.rb @@ -0,0 +1,33 @@ +module Puppet::Module::Tool + module Applications + class Uninstaller < Application + + def initialize(name, options = {}) + @name = name + @target_directories = options[:target_directories] + @removed_dirs = [] + end + + def run + uninstall + Puppet.notice "#{@name} is not installed" if @removed_dirs.empty? + @removed_dirs + end + + private + + def uninstall + # TODO: #11803 Check for broken dependencies before uninstalling modules. + # + # Search each path in the target directories for the specified module + # and delete the directory. + @target_directories.each do |target| + if File.directory? target + module_path = File.join(target, @name) + @removed_dirs << FileUtils.rm_rf(module_path).first if File.directory?(module_path) + end + end + end + end + end +end diff --git a/lib/puppet/module_tool/applications/unpacker.rb b/lib/puppet/module_tool/applications/unpacker.rb new file mode 100644 index 000000000..6dd1fca55 --- /dev/null +++ b/lib/puppet/module_tool/applications/unpacker.rb @@ -0,0 +1,70 @@ +require 'pathname' +require 'tmpdir' + +module Puppet::Module::Tool + module Applications + class Unpacker < Application + + def initialize(filename, options = {}) + @filename = Pathname.new(filename) + parse_filename! + super(options) + @module_dir = Pathname.new(options[:install_dir]) + @module_name + end + + def run + extract_module_to_install_dir + tag_revision + + # Return the Pathname object representing the directory where the + # module release archive was unpacked the to, and the module release + # name. + @module_dir + end + + private + + def tag_revision + File.open("#{@module_dir}/REVISION", 'w') do |f| + f.puts "module: #{@username}/#{@module_name}" + f.puts "version: #{@version}" + f.puts "url: file://#{@filename.expand_path}" + f.puts "installed: #{Time.now}" + end + end + + def extract_module_to_install_dir + delete_existing_installation_or_abort! + + build_dir = Puppet::Module::Tool::Cache.base_path + "tmp-unpacker-#{Digest::SHA1.hexdigest(@filename.basename.to_s)}" + build_dir.mkpath + begin + Puppet.notice "Installing #{@filename.basename} to #{@module_dir.expand_path}" + unless system "tar xzf #{@filename} -C #{build_dir}" + raise RuntimeError, "Could not extract contents of module archive." + end + # grab the first directory + extracted = build_dir.children.detect { |c| c.directory? } + FileUtils.mv extracted, @module_dir + ensure + build_dir.rmtree + end + end + + def delete_existing_installation_or_abort! + return unless @module_dir.exist? + + if !options[:force] + Puppet.warning "Existing module '#{@module_dir.expand_path}' found" + response = prompt "Overwrite module installed at #{@module_dir.expand_path}? [y/N]" + unless response =~ /y/i + raise RuntimeError, "Aborted installation." + end + end + + Puppet.warning "Deleting #{@module_dir.expand_path}" + FileUtils.rm_rf @module_dir + end + end + end +end diff --git a/lib/puppet/module_tool/cache.rb b/lib/puppet/module_tool/cache.rb new file mode 100644 index 000000000..b25478096 --- /dev/null +++ b/lib/puppet/module_tool/cache.rb @@ -0,0 +1,56 @@ +require 'uri' + +module Puppet::Module::Tool + + # = Cache + # + # Provides methods for reading files from local cache, filesystem or network. + class Cache + + # Instantiate new cahe for the +repositry+ instance. + def initialize(repository, options = {}) + @repository = repository + @options = options + end + + # Return filename retrieved from +uri+ instance. Will download this file and + # cache it if needed. + # + # TODO: Add checksum support. + # TODO: Add error checking. + def retrieve(url) + (path + File.basename(url.to_s)).tap do |cached_file| + uri = url.is_a?(::URI) ? url : ::URI.parse(url) + unless cached_file.file? + if uri.scheme == 'file' + FileUtils.cp(URI.unescape(uri.path), cached_file) + else + # TODO: Handle HTTPS; probably should use repository.contact + data = read_retrieve(uri) + cached_file.open('wb') { |f| f.write data } + end + end + end + end + + # Return contents of file at the given URI's +uri+. + def read_retrieve(uri) + return uri.read + end + + # Return Pathname for repository's cache directory, create it if needed. + def path + return @path ||= (self.class.base_path + @repository.cache_key).tap{ |o| o.mkpath } + end + + # Return the base Pathname for all the caches. + def self.base_path + Pathname(Puppet.settings[:module_working_dir]) + 'cache' + end + + # Clean out all the caches. + def self.clean + base_path.rmtree if base_path.exist? + end + end +end diff --git a/lib/puppet/module_tool/checksums.rb b/lib/puppet/module_tool/checksums.rb new file mode 100644 index 000000000..a566090d6 --- /dev/null +++ b/lib/puppet/module_tool/checksums.rb @@ -0,0 +1,52 @@ +require 'digest/md5' + +module Puppet::Module::Tool + + # = Checksums + # + # This class proides methods for generating checksums for data and adding + # them to +Metadata+. + class Checksums + include Enumerable + + # Instantiate object with string +path+ to create checksums from. + def initialize(path) + @path = Pathname.new(path) + end + + # Return checksum for the +Pathname+. + def checksum(pathname) + return Digest::MD5.hexdigest(pathname.read) + end + + # Return checksums for object's +Pathname+, generate if it's needed. + # Result is a hash of path strings to checksum strings. + def data + unless @data + @data = {} + @path.find do |descendant| + if Puppet::Module::Tool.artifact?(descendant) + Find.prune + elsif descendant.file? + path = descendant.relative_path_from(@path) + @data[path.to_s] = checksum(descendant) + end + end + end + return @data + end + + # TODO: Why? + def each(&block) + data.each(&block) + end + + # Update +Metadata+'s checksums with this object's. + def annotate(metadata) + metadata.checksums.replace(data) + end + + # TODO: Move the Checksummer#run checksum checking to here? + + end +end diff --git a/lib/puppet/module_tool/contents_description.rb b/lib/puppet/module_tool/contents_description.rb new file mode 100644 index 000000000..f6a94e22d --- /dev/null +++ b/lib/puppet/module_tool/contents_description.rb @@ -0,0 +1,82 @@ +module Puppet::Module::Tool + + # = ContentsDescription + # + # This class populates +Metadata+'s Puppet type information. + class ContentsDescription + + # Instantiate object for string +module_path+. + def initialize(module_path) + @module_path = module_path + end + + # Update +Metadata+'s Puppet type information. + def annotate(metadata) + metadata.types.replace data.clone + end + + # Return types for this module. Result is an array of hashes, each of which + # describes a Puppet type. The type description hash structure is: + # * :name => Name of this Puppet type. + # * :doc => Documentation for this type. + # * :properties => Array of hashes representing the type's properties, each + # containing :name and :doc. + # * :parameters => Array of hashes representing the type's parameters, each + # containing :name and :doc. + # * :providers => Array of hashes representing the types providers, each + # containing :name and :doc. + # TODO Write a TypeDescription to encapsulate these structures and logic? + def data + unless @data + @data = [] + type_names = [] + for module_filename in Dir[File.join(@module_path, "lib/puppet/type/*.rb")] + require module_filename + type_name = File.basename(module_filename, ".rb") + type_names << type_name + + for provider_filename in Dir[File.join(@module_path, "lib/puppet/provider/#{type_name}/*.rb")] + require provider_filename + end + end + + type_names.each do |type_name| + if type = Puppet::Type.type(type_name.to_sym) + type_hash = {:name => type_name, :doc => type.doc} + type_hash[:properties] = attr_doc(type, :property) + type_hash[:parameters] = attr_doc(type, :param) + if type.providers.size > 0 + type_hash[:providers] = provider_doc(type) + end + @data << type_hash + else + Puppet.warning "Could not find/load type: #{type_name}" + end + end + end + @data + end + + # Return an array of hashes representing this +type+'s attrs of +kind+ + # (e.g. :param or :property), each containing :name and :doc. + def attr_doc(type, kind) + [].tap do |attrs| + type.allattrs.each do |name| + if type.attrtype(name) == kind && name != :provider + attrs.push(:name => name, :doc => type.attrclass(name).doc) + end + end + end + end + + # Return an array of hashes representing this +type+'s providers, each + # containing :name and :doc. + def provider_doc(type) + [].tap do |providers| + type.providers.sort_by{ |o| o.to_s }.each do |prov| + providers.push(:name => prov, :doc => type.provider(prov).doc) + end + end + end + end +end diff --git a/lib/puppet/module_tool/dependency.rb b/lib/puppet/module_tool/dependency.rb new file mode 100644 index 000000000..bb55f5945 --- /dev/null +++ b/lib/puppet/module_tool/dependency.rb @@ -0,0 +1,24 @@ +module Puppet::Module::Tool + + class Dependency + + # Instantiates a new module dependency with a +full_module_name+ (e.g. + # "myuser-mymodule"), and optional +version_requirement+ (e.g. "0.0.1") and + # optional repository (a URL string). + def initialize(full_module_name, version_requirement = nil, repository = nil) + @full_module_name = full_module_name + # TODO: add error checking, the next line raises ArgumentError when +full_module_name+ is invalid + @username, @name = Puppet::Module::Tool.username_and_modname_from(full_module_name) + @version_requirement = version_requirement + @repository = repository ? Repository.new(repository) : nil + end + + # Return PSON representation of this data. + def to_pson(*args) + result = { :name => @full_module_name } + result[:version_requirement] = @version_requirement if @version_requirement && ! @version_requirement.nil? + result[:repository] = @repository.to_s if @repository && ! @repository.nil? + result.to_pson(*args) + end + end +end diff --git a/lib/puppet/module_tool/metadata.rb b/lib/puppet/module_tool/metadata.rb new file mode 100644 index 000000000..a84100a10 --- /dev/null +++ b/lib/puppet/module_tool/metadata.rb @@ -0,0 +1,141 @@ +module Puppet::Module::Tool + + # = Metadata + # + # This class provides a data structure representing a module's metadata. + # It provides some basic parsing, but other data is injected into it using + # +annotate+ methods in other classes. + class Metadata + + # The full name of the module, which is a dash-separated combination of the + # +username+ and module +name+. + attr_reader :full_module_name + + # The name of the user that owns this module. + attr_reader :username + + # The name of this module. See also +full_module_name+. + attr_reader :name + + # The version of this module. + attr_reader :version + + # Instantiate from a hash, whose keys are setters in this class. + def initialize(settings={}) + settings.each do |key, value| + send("#{key}=", value) + end + end + + # Set the full name of this module, and from it, the +username+ and + # module +name+. + def full_module_name=(full_module_name) + @full_module_name = full_module_name + @username, @name = Puppet::Module::Tool::username_and_modname_from(full_module_name) + end + + # Return an array of the module's Dependency objects. + def dependencies + return @dependencies ||= [] + end + + def author + @author || @username + end + + def author=(author) + @author = author + end + + def source + @source || 'UNKNOWN' + end + + def source=(source) + @source = source + end + + def license + @license || 'Apache License, Version 2.0' + end + + def license=(license) + @license = license + end + + def summary + @summary || 'UNKNOWN' + end + + def summary=(summary) + @summary = summary + end + + def description + @description || 'UNKNOWN' + end + + def description=(description) + @description = description + end + + def project_page + @project_page || 'UNKNOWN' + end + + def project_page=(project_page) + @project_page = project_page + end + + # Return an array of the module's Puppet types, each one is a hash + # containing :name and :doc. + def types + return @types ||= [] + end + + # Return module's file checksums. + def checksums + return @checksums ||= {} + end + + # Return the dashed name of the module, which may either be the + # dash-separated combination of the +username+ and module +name+, or just + # the module +name+. + def dashed_name + return [@username, @name].compact.join('-') + end + + # Return the release name, which is the combination of the +dashed_name+ + # of the module and its +version+ number. + def release_name + return [dashed_name, @version].join('-') + end + + # Set the version of this module, ensure a string like '0.1.0' see the + # Semantic Versions here: http://semver.org + def version=(version) + if SemVer.valid?(version) + @version = version + else + raise ArgumentError, "Invalid version format: #{@version} (Semantic Versions are acceptable: http://semver.org)" + end + end + + # Return the PSON record representing this instance. + def to_pson(*args) + return { + :name => @full_module_name, + :version => @version, + :source => source, + :author => author, + :license => license, + :summary => summary, + :description => description, + :project_page => project_page, + :dependencies => dependencies, + :types => types, + :checksums => checksums + }.to_pson(*args) + end + end +end diff --git a/lib/puppet/module_tool/modulefile.rb b/lib/puppet/module_tool/modulefile.rb new file mode 100644 index 000000000..35c1e9973 --- /dev/null +++ b/lib/puppet/module_tool/modulefile.rb @@ -0,0 +1,75 @@ +module Puppet::Module::Tool + + # = Modulefile + # + # This class provides the DSL used for evaluating the module's 'Modulefile'. + # These methods are used to concisely define this module's attributes, which + # are later rendered as PSON into a 'metadata.json' file. + class ModulefileReader + + # Read the +filename+ and eval its Ruby code to set values in the Metadata + # +metadata+ instance. + def self.evaluate(metadata, filename) + returning(new(metadata)) do |builder| + if File.file?(filename) + builder.instance_eval(File.read(filename.to_s), filename.to_s, 1) + else + Puppet.warning "No Modulefile: #{filename}" + end + end + end + + # Instantiate with the Metadata +metadata+ instance. + def initialize(metadata) + @metadata = metadata + end + + # Set the +full_module_name+ (e.g. "myuser-mymodule"), which will also set the + # +username+ and module +name+. Required. + def name(name) + @metadata.full_module_name = name + end + + # Set the module +version+ (e.g., "0.0.1"). Required. + def version(version) + @metadata.version = version + end + + # Add a dependency with the full_module_name +name+ (e.g. "myuser-mymodule"), an + # optional +version_requirement+ (e.g. "0.0.1") and +repository+ (a URL + # string). Optional. Can be called multiple times to add many dependencies. + def dependency(name, version_requirement = nil, repository = nil) + @metadata.dependencies << Dependency.new(name, version_requirement, repository) + end + + # Set the source + def source(source) + @metadata.source = source + end + + # Set the author or default to +username+ + def author(author) + @metadata.author = author + end + + # Set the license + def license(license) + @metadata.license = license + end + + # Set the summary + def summary(summary) + @metadata.summary = summary + end + + # Set the description + def description(description) + @metadata.description = description + end + + # Set the project page + def project_page(project_page) + @metadata.project_page = project_page + end + end +end diff --git a/lib/puppet/module_tool/repository.rb b/lib/puppet/module_tool/repository.rb new file mode 100644 index 000000000..7f0ad849f --- /dev/null +++ b/lib/puppet/module_tool/repository.rb @@ -0,0 +1,79 @@ +require 'net/http' +require 'digest/sha1' +require 'uri' + +module Puppet::Module::Tool + + # = Repository + # + # This class is a file for accessing remote repositories with modules. + class Repository + include Utils::Interrogation + + attr_reader :uri, :cache + + # Instantiate a new repository instance rooted at the optional string + # +url+, else an instance of the default Puppet modules repository. + def initialize(url=Puppet[:module_repository]) + @uri = url.is_a?(::URI) ? url : ::URI.parse(url) + @cache = Cache.new(self) + end + + # Return a Net::HTTPResponse read for this +request+. + # + # Options: + # * :authenticate => Request authentication on the terminal. Defaults to false. + def make_http_request(request, options = {}) + if options[:authenticate] + authenticate(request) + end + if ! @uri.user.nil? && ! @uri.password.nil? + request.basic_auth(@uri.user, @uri.password) + end + return read_response(request) + end + + # Return a Net::HTTPResponse read from this HTTPRequest +request+. + def read_response(request) + begin + Net::HTTP::Proxy( + Puppet::Module::Tool::http_proxy_host, + Puppet::Module::Tool::http_proxy_port + ).start(@uri.host, @uri.port) do |http| + http.request(request) + end + rescue Errno::ECONNREFUSED, SocketError + raise RuntimeError, "Could not reach remote repository" + end + end + + # Set the HTTP Basic Authentication parameters for the Net::HTTPRequest + # +request+ by asking the user for input on the console. + def authenticate(request) + Puppet.notice "Authenticating for #{@uri}" + email = prompt('Email Address') + password = prompt('Password', true) + request.basic_auth(email, password) + end + + # Return the local file name containing the data downloaded from the + # repository at +release+ (e.g. "myuser-mymodule"). + def retrieve(release) + return cache.retrieve(@uri + release) + end + + # Return the URI string for this repository. + def to_s + return @uri.to_s + end + + # Return the cache key for this repository, this a hashed string based on + # the URI. + def cache_key + return @cache_key ||= [ + @uri.to_s.gsub(/[^[:alnum:]]+/, '_').sub(/_$/, ''), + Digest::SHA1.hexdigest(@uri.to_s) + ].join('-') + end + end +end diff --git a/lib/puppet/module_tool/skeleton.rb b/lib/puppet/module_tool/skeleton.rb new file mode 100644 index 000000000..4c41c7611 --- /dev/null +++ b/lib/puppet/module_tool/skeleton.rb @@ -0,0 +1,34 @@ +module Puppet::Module::Tool + + # = Skeleton + # + # This class provides methods for finding templates for the 'generate' action. + class Skeleton + + # TODO Review whether the 'freeze' feature should be fixed or deleted. + # def freeze! + # FileUtils.rm_fr custom_path rescue nil + # FileUtils.cp_r default_path, custom_path + # end + + # Return Pathname with 'generate' templates. + def path + paths.detect { |path| path.directory? } + end + + # Return Pathnames to look for 'generate' templates. + def paths + @paths ||= [ custom_path, default_path ] + end + + # Return Pathname of custom templates directory. + def custom_path + Pathname(Puppet.settings[:module_working_dir]) + 'skeleton' + end + + # Return Pathname of default template directory. + def default_path + Pathname(__FILE__).dirname + 'skeleton/templates/generator' + end + end +end diff --git a/lib/puppet/module_tool/skeleton/templates/generator/Modulefile.erb b/lib/puppet/module_tool/skeleton/templates/generator/Modulefile.erb new file mode 100644 index 000000000..845102d6e --- /dev/null +++ b/lib/puppet/module_tool/skeleton/templates/generator/Modulefile.erb @@ -0,0 +1,11 @@ +name '<%= metadata.full_module_name %>' +version '0.0.1' +source '<%= metadata.source %>' +author '<%= metadata.author %>' +license '<%= metadata.license %>' +summary '<%= metadata.summary %>' +description '<%= metadata.description %>' +project_page '<%= metadata.project_page %>' + +## Add dependencies, if any: +# dependency 'username/name', '>= 1.2.0' diff --git a/lib/puppet/module_tool/skeleton/templates/generator/README.erb b/lib/puppet/module_tool/skeleton/templates/generator/README.erb new file mode 100644 index 000000000..95a6cc472 --- /dev/null +++ b/lib/puppet/module_tool/skeleton/templates/generator/README.erb @@ -0,0 +1,16 @@ +<%= metadata.name %> + +This is the <%= metadata.name %> module. + +License +------- + + +Contact +------- + + +Support +------- + +Please log tickets and issues at our [Projects site](http://projects.example.com) diff --git a/lib/puppet/module_tool/skeleton/templates/generator/manifests/init.pp.erb b/lib/puppet/module_tool/skeleton/templates/generator/manifests/init.pp.erb new file mode 100644 index 000000000..04dd52583 --- /dev/null +++ b/lib/puppet/module_tool/skeleton/templates/generator/manifests/init.pp.erb @@ -0,0 +1,41 @@ +# == Class: <%= metadata.name %> +# +# Full description of class <%= metadata.name %> here. +# +# === Parameters +# +# Document parameters here. +# +# [*sample_parameter*] +# Explanation of what this parameter affects and what it defaults to. +# e.g. "Specify one or more upstream ntp servers as an array." +# +# === Variables +# +# Here you should define a list of variables that this module would require. +# +# [*sample_variable*] +# Explanation of how this variable affects the funtion of this class and if it +# has a default. e.g. "The parameter enc_ntp_servers must be set by the +# External Node Classifier as a comma separated list of hostnames." (Note, +# global variables should not be used in preference to class parameters as of +# Puppet 2.6.) +# +# === Examples +# +# class { <%= metadata.name %>: +# servers => [ 'pool.ntp.org', 'ntp.local.company.com' ] +# } +# +# === Authors +# +# Author Name <author@domain.com> +# +# === Copyright +# +# Copyright 2011 Your name here, unless otherwise noted. +# +class <%= metadata.name %> { + + +} diff --git a/lib/puppet/module_tool/skeleton/templates/generator/metadata.json b/lib/puppet/module_tool/skeleton/templates/generator/metadata.json new file mode 100644 index 000000000..8ce7797ff --- /dev/null +++ b/lib/puppet/module_tool/skeleton/templates/generator/metadata.json @@ -0,0 +1,12 @@ +/* ++-----------------------------------------------------------------------+ +| | +| ==> DO NOT EDIT THIS FILE! <== | +| | +| You should edit the `Modulefile` and run `puppet-module build` | +| to generate the `metadata.json` file for your releases. | +| | ++-----------------------------------------------------------------------+ +*/ + +{} diff --git a/lib/puppet/module_tool/skeleton/templates/generator/spec/spec_helper.rb b/lib/puppet/module_tool/skeleton/templates/generator/spec/spec_helper.rb new file mode 100644 index 000000000..5fda58875 --- /dev/null +++ b/lib/puppet/module_tool/skeleton/templates/generator/spec/spec_helper.rb @@ -0,0 +1,17 @@ +dir = File.expand_path(File.dirname(__FILE__)) +$LOAD_PATH.unshift File.join(dir, 'lib') + +require 'mocha' +require 'puppet' +require 'rspec' +require 'spec/autorun' + +Spec::Runner.configure do |config| + config.mock_with :mocha +end + +# We need this because the RAL uses 'should' as a method. This +# allows us the same behaviour but with a different method name. +class Object + alias :must :should +end diff --git a/lib/puppet/module_tool/skeleton/templates/generator/tests/init.pp.erb b/lib/puppet/module_tool/skeleton/templates/generator/tests/init.pp.erb new file mode 100644 index 000000000..ba4456396 --- /dev/null +++ b/lib/puppet/module_tool/skeleton/templates/generator/tests/init.pp.erb @@ -0,0 +1,11 @@ +# The baseline for module testing used by Puppet Labs is that each manifest +# should have a corresponding test manifest that declares that class or defined +# type. +# +# Tests are then run by using puppet apply --noop (to check for compilation errors +# and view a log of events) or by fully applying the test in a virtual environment +# (to compare the resulting system state to the desired state). +# +# Learn more about module testing here: http://docs.puppetlabs.com/guides/tests_smoke.html +# +include <%= metadata.name %> diff --git a/lib/puppet/module_tool/utils.rb b/lib/puppet/module_tool/utils.rb new file mode 100644 index 000000000..85f57c973 --- /dev/null +++ b/lib/puppet/module_tool/utils.rb @@ -0,0 +1,5 @@ +module Puppet::Module::Tool + module Utils + require 'puppet/module_tool/utils/interrogation' + end +end diff --git a/lib/puppet/module_tool/utils/interrogation.rb b/lib/puppet/module_tool/utils/interrogation.rb new file mode 100644 index 000000000..19450dedd --- /dev/null +++ b/lib/puppet/module_tool/utils/interrogation.rb @@ -0,0 +1,25 @@ +module Puppet::Module::Tool + module Utils + + # = Interrogation + # + # This module contains methods to emit questions to the console. + module Interrogation + def confirms?(question) + $stderr.print "#{question} [y/N]: " + $stdin.gets =~ /y/i + end + + def prompt(question, quiet = false) + $stderr.print "#{question}: " + system 'stty -echo' if quiet + $stdin.gets.strip + ensure + if quiet + system 'stty echo' + say "\n---------" + end + end + end + end +end diff --git a/lib/puppet/network/http/api/v1.rb b/lib/puppet/network/http/api/v1.rb index 388d54961..852568a56 100644 --- a/lib/puppet/network/http/api/v1.rb +++ b/lib/puppet/network/http/api/v1.rb @@ -74,7 +74,8 @@ module Puppet::Network::HTTP::API::V1 result = (indirection =~ /s$|_search$/) ? :plural : :singular - indirection.sub!(/s$|_search$|es$/, '') + indirection.sub!(/s$|_search$/, '') + indirection.sub!(/statuse$/, 'status') result end diff --git a/lib/puppet/node/environment.rb b/lib/puppet/node/environment.rb index 4fc314a6a..326809050 100644 --- a/lib/puppet/node/environment.rb +++ b/lib/puppet/node/environment.rb @@ -54,7 +54,6 @@ class Puppet::Node::Environment @root end - # This is only used for testing. def self.clear @seen.clear end @@ -88,7 +87,7 @@ class Puppet::Node::Environment end def module(name) - mod = Puppet::Module.new(name, self) + mod = Puppet::Module.new(name, :environment => self) return nil unless mod.exist? mod end @@ -107,13 +106,27 @@ class Puppet::Node::Environment module_names = modulepath.collect { |path| Dir.entries(path) }.flatten.uniq module_names.collect do |path| begin - Puppet::Module.new(path, self) + Puppet::Module.new(path, :environment => self) rescue Puppet::Module::Error => e nil end end.compact end + # Modules broken out by directory in the modulepath + def modules_by_path + modules_by_path = {} + modulepath.each do |path| + Dir.chdir(path) do + module_names = Dir.glob('*').select { |d| FileTest.directory? d } + modules_by_path[path] = module_names.map do |name| + Puppet::Module.new(name, :environment => self, :path => File.join(path, name)) + end + end + end + modules_by_path + end + def to_s name.to_s end diff --git a/lib/puppet/parser/ast/leaf.rb b/lib/puppet/parser/ast/leaf.rb index c8ebc9483..34252f7b8 100644 --- a/lib/puppet/parser/ast/leaf.rb +++ b/lib/puppet/parser/ast/leaf.rb @@ -161,7 +161,7 @@ class Puppet::Parser::AST raise Puppet::ParseError, "#{variable} is not an hash or array when accessing it with #{accesskey}" unless object.is_a?(Hash) or object.is_a?(Array) - object[array_index_or_key(object, accesskey)] + object[array_index_or_key(object, accesskey)] || :undef end # Assign value to this hashkey or array index diff --git a/lib/puppet/parser/functions/create_resources.rb b/lib/puppet/parser/functions/create_resources.rb index 646761957..3c91b4111 100644 --- a/lib/puppet/parser/functions/create_resources.rb +++ b/lib/puppet/parser/functions/create_resources.rb @@ -1,7 +1,7 @@ Puppet::Parser::Functions::newfunction(:create_resources, :doc => <<-'ENDHEREDOC') do |args| Converts a hash into a set of resources and adds them to the catalog. - This function takes two arguments: a resource type, and a hash describing + This function takes two mandatory arguments: a resource type, and a hash describing a set of resources. The hash should be in the form `{title => {parameters} }`: # A hash of user resources: @@ -14,13 +14,26 @@ Puppet::Parser::Functions::newfunction(:create_resources, :doc => <<-'ENDHEREDOC groups => ['developers', 'prosvc', 'release'], } } - create_resource(user, $myusers) + create_resources(user, $myusers) + + A third, optional parameter may be given, also as a hash: + + $defaults => { + 'ensure' => present, + 'provider' => 'ldap', + } + + create_resources(user, $myusers, $defaults) + + The values given on the third argument are added to the parameters of each resource + present in the set given on the second argument. If a parameter is present on both + the second and third arguments, the one on the second argument takes precedence. This function can be used to create defined resources and classes, as well as native resources. ENDHEREDOC - raise ArgumentError, ("create_resources(): wrong number of arguments (#{args.length}; must be 2)") if args.length != 2 - #raise ArgumentError, 'requires resource type and param hash' if args.size < 2 + raise ArgumentError, ("create_resources(): wrong number of arguments (#{args.length}; must be 2 or 3)") if args.length < 2 || args.length > 3 + # figure out what kind of resource we are type_of_resource = nil type_name = args[0].downcase @@ -36,8 +49,10 @@ Puppet::Parser::Functions::newfunction(:create_resources, :doc => <<-'ENDHEREDOC end end # iterate through the resources to create + defaults = args[2] || {} args[1].each do |title, params| raise ArgumentError, 'params should not contain title' if(params['title']) + params = defaults.merge(params) case type_of_resource # JJM The only difference between a type and a define is the call to instantiate_resource # for a defined type. @@ -54,7 +69,7 @@ Puppet::Parser::Functions::newfunction(:create_resources, :doc => <<-'ENDHEREDOC klass = find_hostclass(title) raise ArgumentError, "could not find hostclass #{title}" unless klass klass.ensure_in_catalog(self, params) - compiler.catalog.add_class([title]) + compiler.catalog.add_class(title) end end end diff --git a/lib/puppet/parser/type_loader.rb b/lib/puppet/parser/type_loader.rb index 68def068d..3cba89515 100644 --- a/lib/puppet/parser/type_loader.rb +++ b/lib/puppet/parser/type_loader.rb @@ -112,7 +112,7 @@ class Puppet::Parser::TypeLoader # behavior) only load files from the first module of a given name. E.g., # given first/foo and second/foo, only files from first/foo will be loaded. module_names.each do |name| - mod = Puppet::Module.new(name, environment) + mod = Puppet::Module.new(name, :environment => environment) Find.find(File.join(mod.path, "manifests")) do |path| if path =~ /\.pp$/ or path =~ /\.rb$/ import(path) diff --git a/lib/puppet/property.rb b/lib/puppet/property.rb index ee8f3b4c1..9d2d98b0e 100644 --- a/lib/puppet/property.rb +++ b/lib/puppet/property.rb @@ -166,22 +166,43 @@ class Puppet::Property < Puppet::Parameter raise "Puppet::Property#safe_insync? shouldn't be overridden; please override insync? instead" if sym == :safe_insync? end - # This method should be overridden by derived classes if necessary + # This method may be overridden by derived classes if necessary # to provide extra logic to determine whether the property is in - # sync. + # sync. In most cases, however, only `property_matches?` needs to be + # overridden to give the correct outcome - without reproducing all the array + # matching logic, etc, found here. def insync?(is) self.devfail "#{self.class.name}'s should is not array" unless @should.is_a?(Array) # an empty array is analogous to no should values return true if @should.empty? - # Look for a matching value - return (is == @should or is == @should.collect { |v| v.to_s }) if match_all? + # Look for a matching value, either for all the @should values, or any of + # them, depending on the configuration of this property. + if match_all? then + old = (is == @should or is == @should.collect { |v| v.to_s }) + new = Array(is).zip(@should).all? {|is, want| property_matches?(is, want) } - @should.each { |val| return true if is == val or is == val.to_s } + puts "old and new mismatch!" unless old == new + fail "old and new mismatch!" unless old == new - # otherwise, return false - false + # We need to pairwise compare the entries; this preserves the old + # behaviour while using the new pair comparison code. + return Array(is).zip(@should).all? {|is, want| property_matches?(is, want) } + else + return @should.any? {|want| property_matches?(is, want) } + end + end + + # Compare the current and desired value of a property in a property-specific + # way. Invoked by `insync?`; this should be overridden if your property + # has a different comparison type but does not actually differentiate the + # overall insync? logic. + def property_matches?(current, desired) + # This preserves the older Puppet behaviour of doing raw and string + # equality comparisons for all equality. I am not clear this is globally + # desirable, but at least it is not a breaking change. --daniel 2011-11-11 + current == desired or current == desired.to_s end # because the @should and @is vars might be in weird formats, @@ -194,13 +215,10 @@ class Puppet::Property < Puppet::Parameter # Send a log message. def log(msg) - - Puppet::Util::Log.create( - - :level => resource[:loglevel], + Puppet::Util::Log.create( + :level => resource[:loglevel], :message => msg, - - :source => self + :source => self ) end diff --git a/lib/puppet/provider.rb b/lib/puppet/provider.rb index 4456feb4e..295ae8357 100644 --- a/lib/puppet/provider.rb +++ b/lib/puppet/provider.rb @@ -204,15 +204,15 @@ class Puppet::Provider dochook(:defaults) do if @defaults.length > 0 - return " Default for " + @defaults.collect do |f, v| - "`#{f}` == `#{v}`" + return "Default for " + @defaults.collect do |f, v| + "`#{f}` == `#{[v].flatten.join(', ')}`" end.join(" and ") + "." end end dochook(:commands) do if @commands.length > 0 - return " Required binaries: " + @commands.collect do |n, c| + return "Required binaries: " + @commands.collect do |n, c| "`#{c}`" end.join(", ") + "." end @@ -220,7 +220,7 @@ class Puppet::Provider dochook(:features) do if features.length > 0 - return " Supported features: " + features.collect do |f| + return "Supported features: " + features.collect do |f| "`#{f}`" end.join(", ") + "." end diff --git a/lib/puppet/provider/augeas/augeas.rb b/lib/puppet/provider/augeas/augeas.rb index 7214a8b33..703a01bcc 100644 --- a/lib/puppet/provider/augeas/augeas.rb +++ b/lib/puppet/provider/augeas/augeas.rb @@ -17,10 +17,12 @@ require 'augeas' if Puppet.features.augeas? require 'strscan' require 'puppet/util' require 'puppet/util/diff' +require 'puppet/util/package' Puppet::Type.type(:augeas).provide(:augeas) do include Puppet::Util include Puppet::Util::Diff + include Puppet::Util::Package confine :true => Puppet.features.augeas? @@ -149,7 +151,7 @@ Puppet::Type.type(:augeas).provide(:augeas) do debug("Opening augeas with root #{root}, lens path #{load_path}, flags #{flags}") @aug = Augeas::open(root, load_path,flags) - debug("Augeas version #{get_augeas_version} is installed") if get_augeas_version >= "0.3.6" + debug("Augeas version #{get_augeas_version} is installed") if versioncmp(get_augeas_version, "0.3.6") >= 0 if resource[:incl] aug.set("/augeas/load/Xfm/lens", resource[:lens]) @@ -285,7 +287,7 @@ Puppet::Type.type(:augeas).provide(:augeas) do # If we have a verison of augeas which is at least 0.3.6 then we # can make the changes now, see if changes were made, and # actually do the save. - if return_value and get_augeas_version >= "0.3.6" + if return_value and versioncmp(get_augeas_version, "0.3.6") >= 0 debug("Will attempt to save and only run if files changed") set_augeas_save_mode(SAVE_NEWFILE) do_execute_changes @@ -325,7 +327,7 @@ Puppet::Type.type(:augeas).provide(:augeas) do begin open_augeas saved_files = @aug.match("/augeas/events/saved") - if saved_files + unless saved_files.empty? saved_files.each do |key| root = resource[:root].sub(/^\/$/, "") saved_file = @aug.get(key).to_s.sub(/^\/files/, root) @@ -337,7 +339,7 @@ Puppet::Type.type(:augeas).provide(:augeas) do end else debug("No saved files, re-executing augeas") - set_augeas_save_mode(SAVE_OVERWRITE) if get_augeas_version >= "0.3.6" + set_augeas_save_mode(SAVE_OVERWRITE) if versioncmp(get_augeas_version, "0.3.6") >= 0 do_execute_changes success = @aug.save fail("Save failed with return code #{success}") if success != true diff --git a/lib/puppet/provider/group/pw.rb b/lib/puppet/provider/group/pw.rb index a054d1ff1..b6c74f766 100644 --- a/lib/puppet/provider/group/pw.rb +++ b/lib/puppet/provider/group/pw.rb @@ -1,34 +1,48 @@ require 'puppet/provider/nameservice/pw' Puppet::Type.type(:group).provide :pw, :parent => Puppet::Provider::NameService::PW do - desc "Group management via `pw`. + desc "Group management via `pw` on FreeBSD." - Only works on FreeBSD. + commands :pw => "pw" + has_features :manages_members - " - - commands :pw => "/usr/sbin/pw" defaultfor :operatingsystem => :freebsd + options :members, :flag => "-M", :method => :mem + verify :gid, "GID must be an integer" do |value| value.is_a? Integer end def addcmd cmd = [command(:pw), "groupadd", @resource[:name]] + if gid = @resource.should(:gid) unless gid == :absent cmd << flag(:gid) << gid end end - # Apparently, contrary to the man page, groupadd does - # not accept -o. - #if @parent[:allowdupe] == :true - # cmd << "-o" - #end + if members = @resource.should(:members) + unless members == :absent + if members.is_a?(Array) + members = members.join(",") + end + cmd << "-M" << members + end + end + + cmd << "-o" if @resource.allowdupe? cmd end + + def modifycmd(param, value) + # members may be an array, need a comma separated list + if param == :members and value.is_a?(Array) + value = value.join(",") + end + super(param, value) + end end diff --git a/lib/puppet/provider/nameservice/directoryservice.rb b/lib/puppet/provider/nameservice/directoryservice.rb index 35ac8d76a..76c79f685 100644 --- a/lib/puppet/provider/nameservice/directoryservice.rb +++ b/lib/puppet/provider/nameservice/directoryservice.rb @@ -2,7 +2,7 @@ require 'puppet' require 'puppet/provider/nameservice' require 'facter/util/plist' require 'cgi' - +require 'fileutils' class Puppet::Provider::NameService class DirectoryService < Puppet::Provider::NameService @@ -21,6 +21,7 @@ class DirectoryService < Puppet::Provider::NameService commands :dscl => "/usr/bin/dscl" commands :dseditgroup => "/usr/sbin/dseditgroup" commands :sw_vers => "/usr/bin/sw_vers" + commands :plutil => '/usr/bin/plutil' confine :operatingsystem => :darwin defaultfor :operatingsystem => :darwin @@ -60,6 +61,8 @@ class DirectoryService < Puppet::Provider::NameService } @@password_hash_dir = "/var/db/shadow/hash" + @@users_plist_dir = '/var/db/dslocal/nodes/Default/users' + def self.instances # JJM Class method that provides an array of instance objects of this @@ -193,7 +196,7 @@ class DirectoryService < Puppet::Provider::NameService # stored in the user record. It is stored at a path that involves the # UUID of the user record for non-Mobile local acccounts. # Mobile Accounts are out of scope for this provider for now - attribute_hash[:password] = self.get_password(attribute_hash[:guid]) if @resource_type.validproperties.include?(:password) and Puppet.features.root? + attribute_hash[:password] = self.get_password(attribute_hash[:guid], attribute_hash[:name]) if @resource_type.validproperties.include?(:password) and Puppet.features.root? attribute_hash end @@ -268,46 +271,144 @@ class DirectoryService < Puppet::Provider::NameService end def self.set_password(resource_name, guid, password_hash) - password_hash_file = "#{@@password_hash_dir}/#{guid}" - begin - File.open(password_hash_file, 'w') { |f| f.write(password_hash)} - rescue Errno::EACCES => detail - fail("Could not write to password hash file: #{detail}") + # Use Puppet::Util::Package.versioncmp() to catch the scenario where a + # version '10.10' would be < '10.7' with simple string comparison. This + # if-statement only executes if the current version is less-than 10.7 + if (Puppet::Util::Package.versioncmp(get_macosx_version_major, '10.7') == -1) + password_hash_file = "#{@@password_hash_dir}/#{guid}" + begin + File.open(password_hash_file, 'w') { |f| f.write(password_hash)} + rescue Errno::EACCES => detail + fail("Could not write to password hash file: #{detail}") + end + + # NBK: For shadow hashes, the user AuthenticationAuthority must contain a value of + # ";ShadowHash;". The LKDC in 10.5 makes this more interesting though as it + # will dynamically generate ;Kerberosv5;;username@LKDC:SHA1 attributes if + # missing. Thus we make sure we only set ;ShadowHash; if it is missing, and + # we can do this with the merge command. This allows people to continue to + # use other custom AuthenticationAuthority attributes without stomping on them. + # + # There is a potential problem here in that we're only doing this when setting + # the password, and the attribute could get modified at other times while the + # hash doesn't change and so this doesn't get called at all... but + # without switching all the other attributes to merge instead of create I can't + # see a simple enough solution for this that doesn't modify the user record + # every single time. This should be a rather rare edge case. (famous last words) + + dscl_vector = self.get_exec_preamble("-merge", resource_name) + dscl_vector << "AuthenticationAuthority" << ";ShadowHash;" + begin + dscl_output = execute(dscl_vector) + rescue Puppet::ExecutionFailure => detail + fail("Could not set AuthenticationAuthority.") + end + else + # 10.7 uses salted SHA512 password hashes which are 128 characters plus + # an 8 character salt. Previous versions used a SHA1 hash padded with + # zeroes. If someone attempts to use a password hash that worked with + # a previous version of OX X, we will fail early and warn them. + if password_hash.length != 136 + fail("OS X 10.7 requires a Salted SHA512 hash password of 136 characters. \ + Please check your password and try again.") + end + + if File.exists?("#{@@users_plist_dir}/#{resource_name}.plist") + # If a plist already exists in /var/db/dslocal/nodes/Default/users, then + # we will need to extract the binary plist from the 'ShadowHashData' + # key, log the new password into the resultant plist's 'SALTED-SHA512' + # key, and then save the entire structure back. + users_plist = Plist::parse_xml(plutil( '-convert', 'xml1', '-o', '/dev/stdout', \ + "#{@@users_plist_dir}/#{resource_name}.plist")) + + # users_plist['ShadowHashData'][0].string is actually a binary plist + # that's nested INSIDE the user's plist (which itself is a binary + # plist). + password_hash_plist = users_plist['ShadowHashData'][0].string + converted_hash_plist = convert_binary_to_xml(password_hash_plist) + + # converted_hash_plist['SALTED-SHA512'].string expects a Base64 encoded + # string. The password_hash provided as a resource attribute is a + # hex value. We need to convert the provided hex value to a Base64 + # encoded string to nest it in the converted hash plist. + converted_hash_plist['SALTED-SHA512'].string = \ + password_hash.unpack('a2'*(password_hash.size/2)).collect { |i| i.hex.chr }.join + + # Finally, we can convert the nested plist back to binary, embed it + # into the user's plist, and convert the resultant plist back to + # a binary plist. + changed_plist = convert_xml_to_binary(converted_hash_plist) + users_plist['ShadowHashData'][0].string = changed_plist + Plist::Emit.save_plist(users_plist, "#{@@users_plist_dir}/#{resource_name}.plist") + plutil('-convert', 'binary1', "#{@@users_plist_dir}/#{resource_name}.plist") + end + end + end + + def self.get_password(guid, username) + # Use Puppet::Util::Package.versioncmp() to catch the scenario where a + # version '10.10' would be < '10.7' with simple string comparison. This + # if-statement only executes if the current version is less-than 10.7 + if (Puppet::Util::Package.versioncmp(get_macosx_version_major, '10.7') == -1) + password_hash = nil + password_hash_file = "#{@@password_hash_dir}/#{guid}" + if File.exists?(password_hash_file) and File.file?(password_hash_file) + fail("Could not read password hash file at #{password_hash_file}") if not File.readable?(password_hash_file) + f = File.new(password_hash_file) + password_hash = f.read + f.close + end + password_hash + else + if File.exists?("#{@@users_plist_dir}/#{username}.plist") + # If a plist exists in /var/db/dslocal/nodes/Default/users, we will + # extract the binary plist from the 'ShadowHashData' key, decode the + # salted-SHA512 password hash, and then return it. + users_plist = Plist::parse_xml(plutil('-convert', 'xml1', '-o', '/dev/stdout', "#{@@users_plist_dir}/#{username}.plist")) + if users_plist['ShadowHashData'] + # users_plist['ShadowHashData'][0].string is actually a binary plist + # that's nested INSIDE the user's plist (which itself is a binary + # plist). + password_hash_plist = users_plist['ShadowHashData'][0].string + converted_hash_plist = convert_binary_to_xml(password_hash_plist) + + # converted_hash_plist['SALTED-SHA512'].string is a Base64 encoded + # string. The password_hash provided as a resource attribute is a + # hex value. We need to convert the Base64 encoded string to a + # hex value and provide it back to Puppet. + password_hash = converted_hash_plist['SALTED-SHA512'].string.unpack("H*")[0] + password_hash + end + end end + end - # NBK: For shadow hashes, the user AuthenticationAuthority must contain a value of - # ";ShadowHash;". The LKDC in 10.5 makes this more interesting though as it - # will dynamically generate ;Kerberosv5;;username@LKDC:SHA1 attributes if - # missing. Thus we make sure we only set ;ShadowHash; if it is missing, and - # we can do this with the merge command. This allows people to continue to - # use other custom AuthenticationAuthority attributes without stomping on them. - # - # There is a potential problem here in that we're only doing this when setting - # the password, and the attribute could get modified at other times while the - # hash doesn't change and so this doesn't get called at all... but - # without switching all the other attributes to merge instead of create I can't - # see a simple enough solution for this that doesn't modify the user record - # every single time. This should be a rather rare edge case. (famous last words) - - dscl_vector = self.get_exec_preamble("-merge", resource_name) - dscl_vector << "AuthenticationAuthority" << ";ShadowHash;" - begin - dscl_output = execute(dscl_vector) - rescue Puppet::ExecutionFailure => detail - fail("Could not set AuthenticationAuthority.") + # This method will accept a hash that has been returned from Plist::parse_xml + # and convert it to a binary plist (string value). + def self.convert_xml_to_binary(plist_data) + Puppet.debug('Converting XML plist to binary') + Puppet.debug('Executing: \'plutil -convert binary1 -o - -\'') + IO.popen('plutil -convert binary1 -o - -', mode='r+') do |io| + io.write plist_data.to_plist + io.close_write + @converted_plist = io.read end + @converted_plist end - def self.get_password(guid) - password_hash = nil - password_hash_file = "#{@@password_hash_dir}/#{guid}" - if File.exists?(password_hash_file) and File.file?(password_hash_file) - fail("Could not read password hash file at #{password_hash_file}") if not File.readable?(password_hash_file) - f = File.new(password_hash_file) - password_hash = f.read - f.close + # This method will accept a binary plist (as a string) and convert it to a + # hash via Plist::parse_xml. + def self.convert_binary_to_xml(plist_data) + Puppet.debug('Converting binary plist to XML') + Puppet.debug('Executing: \'plutil -convert xml1 -o - -\'') + IO.popen('plutil -convert xml1 -o - -', mode='r+') do |io| + io.write plist_data + io.close_write + @converted_plist = io.read end - password_hash + Puppet.debug('Converting XML values to a hash.') + @plist_hash = Plist::parse_xml(@converted_plist) + @plist_hash end # Unlike most other *nixes, OS X doesn't provide built in functionality @@ -468,7 +569,14 @@ class DirectoryService < Puppet::Provider::NameService begin execute(cmd) rescue Puppet::ExecutionFailure => detail - fail("Could not remove #{member} from group: #{@resource.name}, #{detail}") + # TODO: We're falling back to removing the member using dscl due to rdar://8481241 + # This bug causes dseditgroup to fail to remove a member if that member doesn't exist + cmd = [:dscl, ".", "-delete", "/Groups/#{@resource.name}", "GroupMembership", member] + begin + execute(cmd) + rescue Puppet::ExecutionFailure => detail + fail("Could not remove #{member} from group: #{@resource.name}, #{detail}") + end end end end @@ -535,3 +643,4 @@ class DirectoryService < Puppet::Provider::NameService end end end + diff --git a/lib/puppet/provider/package/pip.rb b/lib/puppet/provider/package/pip.rb index ccc389564..5f6806e6f 100644 --- a/lib/puppet/provider/package/pip.rb +++ b/lib/puppet/provider/package/pip.rb @@ -102,7 +102,7 @@ Puppet::Type.type(:package).provide :pip, self.class.commands :pip => pathname pip *args else - raise e + raise e, 'Could not locate the pip command.' end end end diff --git a/lib/puppet/provider/package/yum.rb b/lib/puppet/provider/package/yum.rb index c66c46c7e..f072779b9 100755..100644 --- a/lib/puppet/provider/package/yum.rb +++ b/lib/puppet/provider/package/yum.rb @@ -56,7 +56,6 @@ Puppet::Type.type(:package).provide :yum, :parent => :rpm, :source => :rpm do wanted = @resource[:name] operation = :install - # XXX: We don't actually deal with epochs here. case should when true, false, Symbol # pass @@ -87,7 +86,7 @@ Puppet::Type.type(:package).provide :yum, :parent => :rpm, :source => :rpm do unless upd.nil? # FIXME: there could be more than one update for a package # because of multiarch - return "#{upd[:version]}-#{upd[:release]}" + return "#{upd[:epoch]}:#{upd[:version]}-#{upd[:release]}" else # Yum didn't find updates, pretend the current # version is the latest diff --git a/lib/puppet/provider/service/debian.rb b/lib/puppet/provider/service/debian.rb index 6030da1e9..e9ee7e92c 100755 --- a/lib/puppet/provider/service/debian.rb +++ b/lib/puppet/provider/service/debian.rb @@ -42,11 +42,25 @@ Puppet::Type.type(:service).provide :debian, :parent => :init do # See x-man-page://invoke-rc.d if [104, 106].include?($CHILD_STATUS.exitstatus) return :true + elsif [105].include?($CHILD_STATUS.exitstatus) + # 105 is unknown, which generally means the the iniscript does not support query + # The debian policy states that the initscript should support methods of query + # For those that do not, peform the checks manually + # http://www.debian.org/doc/debian-policy/ch-opersys.html + if get_start_link_count >= 4 + return :true + else + return :false + end else return :false end end + def get_start_link_count + Dir.glob("/etc/rc*.d/S*#{@resource[:name]}").length + end + def enable update_rc "-f", @resource[:name], "remove" update_rc @resource[:name], "defaults" diff --git a/lib/puppet/provider/service/launchd.rb b/lib/puppet/provider/service/launchd.rb index 73d4b3c07..79dcf87be 100644 --- a/lib/puppet/provider/service/launchd.rb +++ b/lib/puppet/provider/service/launchd.rb @@ -196,7 +196,7 @@ Puppet::Type.type(:service).provide :launchd, :parent => :base do did_enable_job = false cmds = [] cmds << :launchctl << :load - if self.enabled? == :false # launchctl won't load disabled jobs + if self.enabled? == :false || self.status == :stopped # launchctl won't load disabled jobs cmds << "-w" did_enable_job = true end diff --git a/lib/puppet/provider/service/smf.rb b/lib/puppet/provider/service/smf.rb index c8cc262e9..5567aa45b 100755 --- a/lib/puppet/provider/service/smf.rb +++ b/lib/puppet/provider/service/smf.rb @@ -58,7 +58,7 @@ Puppet::Type.type(:service).provide :smf, :parent => :base do when :maintenance [command(:adm), :clear, @resource[:name]] else - [command(:adm), :enable, @resource[:name]] + [command(:adm), :enable, "-s", @resource[:name]] end end @@ -98,7 +98,7 @@ Puppet::Type.type(:service).provide :smf, :parent => :base do end def stopcmd - [command(:adm), :disable, @resource[:name]] + [command(:adm), :disable, "-s", @resource[:name]] end end diff --git a/lib/puppet/provider/user/pw.rb b/lib/puppet/provider/user/pw.rb index a5988cad1..842397971 100644 --- a/lib/puppet/provider/user/pw.rb +++ b/lib/puppet/provider/user/pw.rb @@ -1,16 +1,18 @@ require 'puppet/provider/nameservice/pw' +require 'open3' Puppet::Type.type(:user).provide :pw, :parent => Puppet::Provider::NameService::PW do desc "User management via `pw` on FreeBSD." commands :pw => "pw" - has_features :manages_homedir, :allows_duplicates + has_features :manages_homedir, :allows_duplicates, :manages_passwords, :manages_expiry defaultfor :operatingsystem => :freebsd options :home, :flag => "-d", :method => :dir options :comment, :method => :gecos options :groups, :flag => "-G" + options :expiry, :method => :expire verify :gid, "GID must be an integer" do |value| value.is_a? Integer @@ -23,10 +25,14 @@ Puppet::Type.type(:user).provide :pw, :parent => Puppet::Provider::NameService:: def addcmd cmd = [command(:pw), "useradd", @resource[:name]] @resource.class.validproperties.each do |property| - next if property == :ensure + next if property == :ensure or property == :password # the value needs to be quoted, mostly because -c might # have spaces in it if value = @resource.should(property) and value != "" + if property == :expiry + # FreeBSD uses DD-MM-YYYY rather than YYYY-MM-DD + value = value.split("-").reverse.join("-") + end cmd << flag(property) << value end end @@ -37,5 +43,53 @@ Puppet::Type.type(:user).provide :pw, :parent => Puppet::Provider::NameService:: cmd end + + def modifycmd(param, value) + if param == :expiry + # FreeBSD uses DD-MM-YYYY rather than YYYY-MM-DD + value = value.split("-").reverse.join("-") + end + cmd = super(param, value) + cmd << "-m" if @resource.managehome? + cmd + end + + def create + super + + # Set the password after create if given + self.password = @resource[:password] if @resource[:password] + end + + # use pw to update password hash + def password=(cryptopw) + Puppet.debug "change password for user '#{@resource[:name]}' method called with hash '#{cryptopw}'" + stdin, stdout, stderr = Open3.popen3("pw user mod #{@resource[:name]} -H 0") + stdin.puts(cryptopw) + stdin.close + Puppet.debug "finished password for user '#{@resource[:name]}' method called with hash '#{cryptopw}'" + end + + # get password from /etc/master.passwd + def password + Puppet.debug "checking password for user '#{@resource[:name]}' method called" + current_passline = `getent passwd #{@resource[:name]}` + current_password = current_passline.chomp.split(':')[1] if current_passline + Puppet.debug "finished password for user '#{@resource[:name]}' method called : '#{current_password}'" + current_password + end + + # Get expiry from system and convert to Puppet-style date + def expiry + expiry = self.get(:expiry) + expiry = :absent if expiry == 0 + + if expiry != :absent + t = Time.at(expiry) + expiry = "%4d-%02d-%02d" % [t.year, t.month, t.mday] + end + + expiry + end end diff --git a/lib/puppet/provider/user/windows_adsi.rb b/lib/puppet/provider/user/windows_adsi.rb index 6b0a9bce7..045a84bdb 100644 --- a/lib/puppet/provider/user/windows_adsi.rb +++ b/lib/puppet/provider/user/windows_adsi.rb @@ -22,6 +22,7 @@ Puppet::Type.type(:user).provide :windows_adsi do def create @user = Puppet::Util::ADSI::User.create(@resource[:name]) + @user.password = @resource[:password] @user.commit [:comment, :home, :groups].each do |prop| diff --git a/lib/puppet/rails.rb b/lib/puppet/rails.rb index f74e63f20..c52b6871b 100644 --- a/lib/puppet/rails.rb +++ b/lib/puppet/rails.rb @@ -47,7 +47,7 @@ module Puppet::Rails case adapter when "sqlite3" args[:database] = Puppet[:dblocation] - when "mysql", "postgresql" + when "mysql", "mysql2", "postgresql" args[:host] = Puppet[:dbserver] unless Puppet[:dbserver].to_s.empty? args[:port] = Puppet[:dbport] unless Puppet[:dbport].to_s.empty? args[:username] = Puppet[:dbuser] unless Puppet[:dbuser].to_s.empty? diff --git a/lib/puppet/rails/database/schema.rb b/lib/puppet/rails/database/schema.rb index 7b75f4216..3eb2589f1 100644 --- a/lib/puppet/rails/database/schema.rb +++ b/lib/puppet/rails/database/schema.rb @@ -22,7 +22,7 @@ class Puppet::Rails::Schema # Thanks, mysql! MySQL requires a length on indexes in text fields. # So, we provide them for mysql and handle everything else specially. # Oracle doesn't index on CLOB fields, so we skip it - if Puppet[:dbadapter] == "mysql" + if ['mysql','mysql2'].include? Puppet[:dbadapter] execute "CREATE INDEX typentitle ON resources (restype,title(50));" elsif Puppet[:dbadapter] != "oracle_enhanced" add_index :resources, [:title, :restype] diff --git a/lib/puppet/reports/store.rb b/lib/puppet/reports/store.rb index 997206ec4..cd188fafd 100644 --- a/lib/puppet/reports/store.rb +++ b/lib/puppet/reports/store.rb @@ -1,4 +1,6 @@ require 'puppet' +require 'fileutils' +require 'tempfile' Puppet::Reports.register_report(:store) do desc "Store the yaml report on disk. Each host sends its report as a YAML dump @@ -29,10 +31,15 @@ Puppet::Reports.register_report(:store) do file = File.join(dir, name) + f = Tempfile.new(name, dir) begin - File.open(file, "w", 0640) do |f| + begin + f.chmod(0640) f.print to_yaml + ensure + f.close end + FileUtils.mv(f.path, file) rescue => detail puts detail.backtrace if Puppet[:trace] Puppet.warning "Could not write report for #{client} at #{file}: #{detail}" diff --git a/lib/puppet/resource/catalog.rb b/lib/puppet/resource/catalog.rb index 8dc0727a9..fc341094e 100644 --- a/lib/puppet/resource/catalog.rb +++ b/lib/puppet/resource/catalog.rb @@ -547,7 +547,11 @@ class Puppet::Resource::Catalog < Puppet::SimpleGraph ::File.open(Puppet[:resourcefile], "w") do |f| to_print = resources.map do |resource| next unless resource.managed? - "#{resource.type}[#{resource[resource.name_var]}]" + if resource.name_var + "#{resource.type}[#{resource[resource.name_var]}]" + else + "#{resource.ref.downcase}" + end end.compact f.puts to_print.join("\n") end diff --git a/lib/puppet/simple_graph.rb b/lib/puppet/simple_graph.rb index 671eef150..6d9365f7e 100644 --- a/lib/puppet/simple_graph.rb +++ b/lib/puppet/simple_graph.rb @@ -136,18 +136,7 @@ class Puppet::SimpleGraph s[:seen][top] = false this_scc << top end until top == vertex - # NOTE: if we don't reverse we get the components in the opposite - # order to what a human being would expect; reverse should be an - # O(1) operation, without even copying, because we know the length - # of the source, but I worry that an implementation will get this - # wrong. Still, the worst case is O(n) for n vertices as we can't - # possibly put a vertex into two SCCs. - # - # Also, my feeling is that most implementations are going to do - # better with a reverse operation than a string of 'unshift' - # insertions at the head of the array; if they were going to mess - # up the performance of one, it would be unshift. - s[:scc] << this_scc.reverse + s[:scc] << this_scc end recur.pop # done with this node, finally. end @@ -181,7 +170,17 @@ class Puppet::SimpleGraph end end - state[:scc].select { |c| c.length > 1 } + # To provide consistent results to the user, given that a hash is never + # assured to return the same order, and given our graph processing is + # based on hash tables, we need to sort the cycles internally, as well as + # the set of cycles. + # + # Given we are in a failure state here, any extra cost is more or less + # irrelevant compared to the cost of a fix - which is on a human + # time-scale. + state[:scc].select { |c| c.length > 1 }.map do |x| + x.sort_by {|a| a.to_s } + end.sort end # Perform a BFS on the sub graph representing the cycle, with a view to diff --git a/lib/puppet/ssl/host.rb b/lib/puppet/ssl/host.rb index d70fe32c7..a4baf5b7c 100644 --- a/lib/puppet/ssl/host.rb +++ b/lib/puppet/ssl/host.rb @@ -199,18 +199,26 @@ class Puppet::SSL::Host return nil unless Certificate.indirection.find("ca") unless ca? return nil unless @certificate = Certificate.indirection.find(name) - unless certificate_matches_key? - raise Puppet::Error, "Retrieved certificate does not match private key; please remove certificate from server and regenerate it with the current key" - end + validate_certificate_with_key end @certificate end - def certificate_matches_key? - return false unless key - return false unless certificate - - certificate.content.check_private_key(key.content) + def validate_certificate_with_key + raise Puppet::Error, "No certificate to validate." unless certificate + raise Puppet::Error, "No private key with which to validate certificate with fingerprint: #{certificate.fingerprint}" unless key + unless certificate.content.check_private_key(key.content) + raise Puppet::Error, <<ERROR_STRING +The certificate retrieved from the master does not match the agent's private key. +Certificate fingerprint: #{certificate.fingerprint} +To fix this, remove the certificate from both the master and the agent and then start a puppet run, which will automatically regenerate a certficate. +On the master: + puppet cert clean #{Puppet[:certname]} +On the agent: + rm -f #{Puppet[:hostcert]} + puppet agent -t +ERROR_STRING + end end # Generate all necessary parts of our ssl host. diff --git a/lib/puppet/transaction.rb b/lib/puppet/transaction.rb index e4bc0f299..20f7108a8 100644 --- a/lib/puppet/transaction.rb +++ b/lib/puppet/transaction.rb @@ -100,6 +100,7 @@ class Puppet::Transaction if resource.is_a?(Puppet::Type::Component) Puppet.warning "Somehow left a component in the relationship graph" else + resource.info "Starting to evaluate the resource" if Puppet[:evaltrace] and @catalog.host_config? seconds = thinmark { eval_resource(resource) } resource.info "Evaluated in %0.2f seconds" % seconds if Puppet[:evaltrace] and @catalog.host_config? end @@ -317,10 +318,6 @@ class Puppet::Transaction @blockers = {} @unguessable_deterministic_key = Hash.new { |h,k| h[k] = Digest::SHA1.hexdigest("NaCl, MgSO4 (salts) and then #{k.ref}") } @providerless_types = [] - vertices.each do |v| - blockers[v] = direct_dependencies_of(v).length - enqueue(v) if blockers[v] == 0 - end end def method_missing(*args,&block) real_graph.send(*args,&block) @@ -335,6 +332,13 @@ class Puppet::Transaction real_graph.add_edge(f,t,label) end + # Enqueue the initial set of resources, those with no dependencies. + def enqueue_roots + vertices.each do |v| + blockers[v] = direct_dependencies_of(v).length + enqueue(v) if blockers[v] == 0 + end + end # Decrement the blocker count for the resource by 1. If the number of # blockers is unknown, count them and THEN decrement by 1. def unblock(resource) @@ -364,6 +368,8 @@ class Puppet::Transaction def traverse(&block) real_graph.report_cycles_in_graph + enqueue_roots + deferred_resources = [] while (resource = next_resource) && !transaction.stop_processing? diff --git a/lib/puppet/transaction/report.rb b/lib/puppet/transaction/report.rb index 807163961..b66183117 100644 --- a/lib/puppet/transaction/report.rb +++ b/lib/puppet/transaction/report.rb @@ -115,7 +115,7 @@ class Puppet::Transaction::Report # Provide a raw hash summary of this report. def raw_summary - report = {} + report = { "version" => { "config" => configuration_version, "puppet" => Puppet.version } } @metrics.each do |name, metric| key = metric.name.to_s @@ -151,7 +151,7 @@ class Puppet::Transaction::Report def calculate_event_metrics metrics = Hash.new(0) - metrics["total"] = 0 + %w{total failure success}.each { |m| metrics[m] = 0 } resource_statuses.each do |name, status| metrics["total"] += status.events.length status.events.each do |event| @@ -163,9 +163,15 @@ class Puppet::Transaction::Report end def calculate_resource_metrics - metrics = Hash.new(0) + metrics = {} metrics["total"] = resource_statuses.length + # force every resource key in the report to be present + # even if no resources is in this given state + Puppet::Resource::Status::STATES.each do |state| + metrics[state.to_s] = 0 + end + resource_statuses.each do |name, status| Puppet::Resource::Status::STATES.each do |state| metrics[state.to_s] += 1 if status.send(state) diff --git a/lib/puppet/type.rb b/lib/puppet/type.rb index 94f26cfe5..f1c891254 100644 --- a/lib/puppet/type.rb +++ b/lib/puppet/type.rb @@ -107,11 +107,9 @@ class Type def self.ensurable? # If the class has all three of these methods defined, then it's # ensurable. - ens = [:exists?, :create, :destroy].inject { |set, method| - set &&= self.public_method_defined?(method) + [:exists?, :create, :destroy].all? { |method| + self.public_method_defined?(method) } - - ens end def self.apply_to_device @@ -1500,11 +1498,14 @@ class Type # We need to add documentation for each provider. def self.doc - @doc + " Available providers are:\n\n" + parenttype.providers.sort { |a,b| + # Since we're mixing @doc with text from other sources, we must normalize + # its indentation with scrub. But we don't need to manually scrub the + # provider's doc string, since markdown_definitionlist sanitizes its inputs. + scrub(@doc) + "Available providers are:\n\n" + parenttype.providers.sort { |a,b| a.to_s <=> b.to_s }.collect { |i| - "* **#{i}**: #{parenttype().provider(i).doc}" - }.join("\n") + markdown_definitionlist( i, scrub(parenttype().provider(i).doc) ) + }.join end defaultto { diff --git a/lib/puppet/type/exec.rb b/lib/puppet/type/exec.rb index ec497630d..8525e8689 100755 --- a/lib/puppet/type/exec.rb +++ b/lib/puppet/type/exec.rb @@ -170,7 +170,7 @@ module Puppet desc "The group to run the command as. This seems to work quite haphazardly on different platforms -- it is a platform issue not a Ruby or Puppet one, since the same variety exists when - running commnands as different users in the shell." + running commands as different users in the shell." # Validation is handled by the SUIDManager class. end diff --git a/lib/puppet/type/file.rb b/lib/puppet/type/file.rb index 20c774abf..a7608c9e8 100644 --- a/lib/puppet/type/file.rb +++ b/lib/puppet/type/file.rb @@ -9,11 +9,14 @@ require 'puppet/network/handler' require 'puppet/util/diff' require 'puppet/util/checksums' require 'puppet/util/backups' +require 'puppet/util/symbolic_file_mode' Puppet::Type.newtype(:file) do include Puppet::Util::MethodHelper include Puppet::Util::Checksums include Puppet::Util::Backups + include Puppet::Util::SymbolicFileMode + @doc = "Manages local files, including setting ownership and permissions, creation of both files and directories, and retrieving entire files from remote servers. As Puppet matures, it @@ -261,13 +264,18 @@ Puppet::Type.newtype(:file) do # Autorequire the nearest ancestor directory found in the catalog. autorequire(:file) do + req = [] path = Pathname.new(self[:path]) if !path.root? # Start at our parent, to avoid autorequiring ourself parents = path.parent.enum_for(:ascend) - found = parents.find { |p| catalog.resource(:file, p.to_s) } - found and found.to_s + if found = parents.find { |p| catalog.resource(:file, p.to_s) } + req << found.to_s + end end + # if the resource is a link, make sure the target is created first + req << self[:target] if self[:target] + req end # Autorequire the owner and group of the file. @@ -729,7 +737,7 @@ Puppet::Type.newtype(:file) do mode = self.should(:mode) # might be nil umask = mode ? 000 : 022 - mode_int = mode ? mode.to_i(8) : nil + mode_int = mode ? symbolic_mode_to_int(mode, 0644) : nil content_checksum = Puppet::Util.withumask(umask) { ::File.open(path, 'wb', mode_int ) { |f| write_content(f) } } diff --git a/lib/puppet/type/file/ctime.rb b/lib/puppet/type/file/ctime.rb index 90d95da64..5f94863b4 100644 --- a/lib/puppet/type/file/ctime.rb +++ b/lib/puppet/type/file/ctime.rb @@ -10,7 +10,7 @@ module Puppet current_value end - validate do + validate do |val| fail "ctime is read-only" end end diff --git a/lib/puppet/type/file/ensure.rb b/lib/puppet/type/file/ensure.rb index a846856c8..b7614f3fb 100755 --- a/lib/puppet/type/file/ensure.rb +++ b/lib/puppet/type/file/ensure.rb @@ -1,6 +1,10 @@ + module Puppet Puppet::Type.type(:file).ensurable do require 'etc' + require 'puppet/util/symbolic_file_mode' + include Puppet::Util::SymbolicFileMode + desc <<-EOT Whether to create files that don't currently exist. Possible values are *absent*, *present*, *file*, and *directory*. @@ -63,7 +67,7 @@ module Puppet end if mode Puppet::Util.withumask(000) do - Dir.mkdir(@resource[:path], mode.to_i(8)) + Dir.mkdir(@resource[:path], symbolic_mode_to_int(mode, 755, true)) end else Dir.mkdir(@resource[:path]) diff --git a/lib/puppet/type/file/mode.rb b/lib/puppet/type/file/mode.rb index b246652a0..8c7020ba4 100755 --- a/lib/puppet/type/file/mode.rb +++ b/lib/puppet/type/file/mode.rb @@ -3,6 +3,9 @@ # specifying the full mode. module Puppet Puppet::Type.type(:file).newproperty(:mode) do + require 'puppet/util/symbolic_file_mode' + include Puppet::Util::SymbolicFileMode + desc "Mode the file should be. Currently relatively limited: you must specify the exact mode the file should be. @@ -23,23 +26,32 @@ module Puppet mode 644, and all of the directories will have mode 755." validate do |value| - if value.is_a?(String) and value !~ /^[0-7]+$/ - raise Puppet::Error, "File modes can only be octal numbers, not #{should.inspect}" + unless value.nil? or valid_symbolic_mode?(value) + raise Puppet::Error, "The file mode specification is invalid: #{value.inspect}" end end - munge do |should| - if should.is_a?(String) - should.to_i(8).to_s(8) - else - should.to_s(8) + munge do |value| + return nil if value.nil? + + unless valid_symbolic_mode?(value) + raise Puppet::Error, "The file mode specification is invalid: #{value.inspect}" end + + normalize_symbolic_mode(value) + end + + def desired_mode_from_current(desired, current) + current = current.to_i(8) if current.is_a? String + is_a_directory = @resource.stat and @resource.stat.directory? + symbolic_mode_to_int(desired, current, is_a_directory) end # If we're a directory, we need to be executable for all cases # that are readable. This should probably be selectable, but eh. def dirmask(value) - if FileTest.directory?(resource[:path]) + orig = value + if FileTest.directory?(resource[:path]) and value =~ /^\d+$/ then value = value.to_i(8) value |= 0100 if value & 0400 != 0 value |= 010 if value & 040 != 0 @@ -61,6 +73,13 @@ module Puppet end end + def property_matches?(current, desired) + return false unless current + current_bits = normalize_symbolic_mode(current) + desired_bits = desired_mode_from_current(desired, current).to_s(8) + current_bits == desired_bits + end + # Ideally, dirmask'ing could be done at munge time, but we don't know if 'ensure' # will eventually be a directory or something else. And unfortunately, that logic # depends on the ensure, source, and target properties. So rather than duplicate @@ -74,12 +93,28 @@ module Puppet super end + # Finally, when we sync the mode out we need to transform it; since we + # don't have access to the calculated "desired" value here, or the + # "current" value, only the "should" value we need to retrieve again. + def sync + current = @resource.stat ? @resource.stat.mode : 0644 + set(desired_mode_from_current(@should[0], current).to_s(8)) + end + + def change_to_s(old_value, desired) + return super if desired =~ /^\d+$/ + + old_bits = normalize_symbolic_mode(old_value) + new_bits = normalize_symbolic_mode(desired_mode_from_current(desired, old_bits)) + super(old_bits, new_bits) + " (#{desired})" + end + def should_to_s(should_value) - should_value.rjust(4,"0") + should_value.rjust(4, "0") end def is_to_s(currentvalue) - currentvalue.rjust(4,"0") + currentvalue.rjust(4, "0") end end end diff --git a/lib/puppet/type/file/mtime.rb b/lib/puppet/type/file/mtime.rb index 5952b4b84..10867ddf4 100644 --- a/lib/puppet/type/file/mtime.rb +++ b/lib/puppet/type/file/mtime.rb @@ -10,7 +10,7 @@ module Puppet current_value end - validate do + validate do |val| fail "mtime is read-only" end end diff --git a/lib/puppet/type/file/source.rb b/lib/puppet/type/file/source.rb index 9375550a9..5d4fb9b85 100755 --- a/lib/puppet/type/file/source.rb +++ b/lib/puppet/type/file/source.rb @@ -132,6 +132,10 @@ module Puppet next if metadata_method == :checksum and metadata.ftype == "directory" next if metadata_method == :checksum and metadata.ftype == "link" and metadata.links == :manage + if Puppet.features.microsoft_windows? + next if [:owner, :group].include?(metadata_method) and !local? + end + if resource[param_name].nil? or resource[param_name] == :absent resource[param_name] = metadata.send(metadata_method) end diff --git a/lib/puppet/type/file/type.rb b/lib/puppet/type/file/type.rb index 864d3b1a4..38f301573 100755 --- a/lib/puppet/type/file/type.rb +++ b/lib/puppet/type/file/type.rb @@ -11,7 +11,7 @@ module Puppet current_value end - validate do + validate do |val| fail "type is read-only" end end diff --git a/lib/puppet/type/host.rb b/lib/puppet/type/host.rb index 8f6aa9ad3..f4ced3170 100755 --- a/lib/puppet/type/host.rb +++ b/lib/puppet/type/host.rb @@ -7,12 +7,26 @@ module Puppet newproperty(:ip) do desc "The host's IP address, IPv4 or IPv6." - validate do |value| - unless value =~ /^((([0-9a-fA-F]+:){7}[0-9a-fA-F]+)|(([0-9a-fA-F]+:)*[0-9a-fA-F]+)?::(([0-9a-fA-F]+:)*[0-9a-fA-F]+)?)|((25[0-5]|2[0-4][\d]|[0-1]?\d?\d)(\.(25[0-5]|2[0-4]\d|[0-1]?\d?\d)){3})$/ - raise Puppet::Error, "Invalid IP address" + + def valid_v4?(addr) + if /^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$/ =~ addr + return $~.captures.all? {|i| i = i.to_i; i >= 0 and i <= 255 } end + return false end + def valid_v6?(addr) + # http://forums.dartware.com/viewtopic.php?t=452 + # ...and, yes, it is this hard. Doing it programatically is harder. + return true if addr =~ /^\s*((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)?\s*$/ + + return false + end + + validate do |value| + return true if valid_v4?(value) or valid_v6?(value) + raise Puppet::Error, "Invalid IP address #{value.inspect}" + end end # for now we use OrderedList to indicate that the order does matter. diff --git a/lib/puppet/type/schedule.rb b/lib/puppet/type/schedule.rb index 2baf726c7..3193c74e3 100755 --- a/lib/puppet/type/schedule.rb +++ b/lib/puppet/type/schedule.rb @@ -81,7 +81,8 @@ module Puppet } This is mostly useful for restricting certain resources to being - applied in maintenance windows or during off-peak hours. + applied in maintenance windows or during off-peak hours. Multiple + ranges can be applied in array context. EOT # This is lame; properties all use arrays as values, but parameters don't. @@ -178,7 +179,7 @@ module Puppet #self.info limits.inspect #self.notice now - return now.between?(*limits) + return true if now.between?(*limits) end # Else, return false, since our current time isn't between diff --git a/lib/puppet/util/anonymous_filelock.rb b/lib/puppet/util/anonymous_filelock.rb new file mode 100644 index 000000000..ff09c5d12 --- /dev/null +++ b/lib/puppet/util/anonymous_filelock.rb @@ -0,0 +1,36 @@ + +class Puppet::Util::AnonymousFilelock + attr_reader :lockfile + + def initialize(lockfile) + @lockfile = lockfile + end + + def anonymous? + true + end + + def lock(msg = '') + return false if locked? + + File.open(@lockfile, 'w') { |fd| fd.print(msg) } + true + end + + def unlock + if locked? + File.unlink(@lockfile) + true + else + false + end + end + + def locked? + File.exists? @lockfile + end + + def message + return File.read(@lockfile) if locked? + end +end
\ No newline at end of file diff --git a/lib/puppet/util/docs.rb b/lib/puppet/util/docs.rb index 4344d67ab..0a7c9bce8 100644 --- a/lib/puppet/util/docs.rb +++ b/lib/puppet/util/docs.rb @@ -20,10 +20,10 @@ module Puppet::Util::Docs def doc extra = methods.find_all { |m| m.to_s =~ /^dochook_.+/ }.sort.collect { |m| self.send(m) - }.join(" ") + }.delete_if {|r| r.nil? }.join(" ") if @doc - @doc + extra + @doc + (extra.empty? ? '' : "\n\n" + extra) else extra end @@ -73,6 +73,22 @@ module Puppet::Util::Docs value.to_s + (" " * (length - value.to_s.length)) end + HEADER_LEVELS = [nil, "#", "##", "###", "####", "#####"] + + def markdown_header(name, level) + "#{HEADER_LEVELS[level]} #{name}\n\n" + end + + def markdown_definitionlist(term, definition) + lines = scrub(definition).split("\n") + str = "#{term}\n: #{lines.shift}\n" + lines.each do |line| + str << " " if line =~ /\S/ + str << "#{line}\n" + end + str << "\n" + end + # Handle the inline indentation in the docs. def scrub(text) # Stupid markdown diff --git a/lib/puppet/util/instrumentation.rb b/lib/puppet/util/instrumentation.rb new file mode 100644 index 000000000..bd0ed3ba5 --- /dev/null +++ b/lib/puppet/util/instrumentation.rb @@ -0,0 +1,173 @@ +require 'puppet' +require 'puppet/util/classgen' +require 'puppet/util/instance_loader' + +class Puppet::Util::Instrumentation + extend Puppet::Util::ClassGen + extend Puppet::Util::InstanceLoader + extend MonitorMixin + + # we're using a ruby lazy autoloader to prevent a loop when requiring listeners + # since this class sets up an indirection which is also used in Puppet::Indirector::Indirection + # which is used to setup indirections... + autoload :Listener, 'puppet/util/instrumentation/listener' + autoload :Data, 'puppet/util/instrumentation/data' + + # Set up autoloading and retrieving of instrumentation listeners. + instance_load :listener, 'puppet/util/instrumentation/listeners' + + class << self + attr_accessor :listeners, :listeners_of + end + + # instrumentation layer + + # Triggers an instrumentation + # + # Call this method around the instrumentation point + # Puppet::Util::Instrumentation.instrument(:my_long_computation) do + # ... a long computation + # end + # + # This will send an event to all the listeners of "my_long_computation". + # Note: this method uses ruby yield directive to call the instrumented code. + # It is usually way slower than calling start and stop directly around the instrumented code. + # For high traffic code path, it is thus advisable to not use this method. + def self.instrument(label, data = {}) + id = self.start(label, data) + yield + ensure + self.stop(label, id, data) + end + + # Triggers a "start" instrumentation event + # + # Important note: + # For proper use, the data hash instance used for start should also + # be used when calling stop. The idea is to use the current scope + # where start is called to retain a reference to 'data' so that it is possible + # to send it back to stop. + # This way listeners can match start and stop events more easily. + def self.start(label, data) + data[:started] = Time.now + publish(label, :start, data) + data[:id] = next_id + end + + # Triggers a "stop" instrumentation event + def self.stop(label, id, data) + data[:finished] = Time.now + publish(label, :stop, data) + end + + def self.publish(label, event, data) + each_listener(label) do |k,l| + l.notify(label, event, data) + end + end + + def self.listeners + @listeners.values + end + + def self.each_listener(label) + synchronize { + @listeners_of[label] ||= @listeners.select do |k,l| + l.listen_to?(label) + end + }.each do |l| + yield l + end + end + + # Adds a new listener + # + # Usage: + # Puppet::Util::Instrumentation.new_listener(:my_instrumentation, pattern) do + # + # def notify(label, data) + # ... do something for data... + # end + # end + # + # It is possible to use a "pattern". The listener will be notified only + # if the pattern match the label of the event. + # The pattern can be a symbol, a string or a regex. + # If no pattern is provided, then the listener will be called for every events + def self.new_listener(name, options = {}, &block) + Puppet.debug "new listener called #{name}" + name = symbolize(name) + listener = genclass(name, :hash => instance_hash(:listener), :block => block) + listener.send(:define_method, :name) do + name + end + subscribe(listener.new, options[:label_pattern], options[:event]) + end + + def self.subscribe(listener, label_pattern, event) + synchronize { + raise "Listener #{listener.name} is already subscribed" if @listeners.include?(listener.name) + Puppet.debug "registering instrumentation listener #{listener.name}" + @listeners[listener.name] = Listener.new(listener, label_pattern, event) + listener.subscribed if listener.respond_to?(:subscribed) + rehash + } + end + + def self.unsubscribe(listener) + synchronize { + Puppet.warning("#{listener.name} hasn't been registered but asked to be unregistered") unless @listeners.include?(listener.name) + Puppet.info "unregistering instrumentation listener #{listener.name}" + @listeners.delete(listener.name) + listener.unsubscribed if listener.respond_to?(:unsubscribed) + rehash + } + end + + def self.init + # let's init our probe indirection + require 'puppet/util/instrumentation/indirection_probe' + synchronize { + @listeners ||= {} + @listeners_of ||= {} + instance_loader(:listener).loadall + } + end + + def self.clear + synchronize { + @listeners = {} + @listeners_of = {} + @id = 0 + } + end + + def self.[](key) + synchronize { + key = symbolize(key) + @listeners[key] + } + end + + def self.[]=(key, value) + synchronize { + key = symbolize(key) + @listeners[key] = value + rehash + } + end + + private + + # should be called only under the guard + # self.synchronize + def self.rehash + @listeners_of = {} + end + + def self.next_id + synchronize { + @id = (@id || 0) + 1 + } + end +end diff --git a/lib/puppet/util/instrumentation/data.rb b/lib/puppet/util/instrumentation/data.rb new file mode 100644 index 000000000..9157f58fc --- /dev/null +++ b/lib/puppet/util/instrumentation/data.rb @@ -0,0 +1,34 @@ +require 'puppet/indirector' +require 'puppet/util/instrumentation' + +# This is just a transport class to be used through the instrumentation_data +# indirection. All the data resides in the real underlying listeners which this +# class delegates to. +class Puppet::Util::Instrumentation::Data + extend Puppet::Indirector + + indirects :instrumentation_data, :terminus_class => :local + + attr_reader :listener + + def initialize(listener_name) + @listener = Puppet::Util::Instrumentation[listener_name] + raise "Listener #{listener_name} wasn't registered" unless @listener + end + + def name + @listener.name + end + + def to_pson(*args) + result = { + 'document_type' => "Puppet::Util::Instrumentation::Data", + 'data' => { :name => name }.merge(@listener.respond_to?(:data) ? @listener.data : {}) + } + result.to_pson(*args) + end + + def self.from_pson(data) + data + end +end diff --git a/lib/puppet/util/instrumentation/indirection_probe.rb b/lib/puppet/util/instrumentation/indirection_probe.rb new file mode 100644 index 000000000..66e5f92ab --- /dev/null +++ b/lib/puppet/util/instrumentation/indirection_probe.rb @@ -0,0 +1,29 @@ +require 'puppet/indirector' +require 'puppet/util/instrumentation' + +# We need to use a class other than Probe for the indirector because +# the Indirection class might declare some probes, and this would be a huge unbreakable +# dependency cycle. +class Puppet::Util::Instrumentation::IndirectionProbe + extend Puppet::Indirector + + indirects :instrumentation_probe, :terminus_class => :local + + attr_reader :probe_name + + def initialize(probe_name) + @probe_name = probe_name + end + + def to_pson(*args) + result = { + :document_type => "Puppet::Util::Instrumentation::IndirectionProbe", + :data => { :name => probe_name } + } + result.to_pson(*args) + end + + def self.from_pson(data) + self.new(data["name"]) + end +end
\ No newline at end of file diff --git a/lib/puppet/util/instrumentation/instrumentable.rb b/lib/puppet/util/instrumentation/instrumentable.rb new file mode 100644 index 000000000..5789dcbe2 --- /dev/null +++ b/lib/puppet/util/instrumentation/instrumentable.rb @@ -0,0 +1,143 @@ +require 'monitor' +require 'puppet/util/instrumentation' + +# This is the central point of all declared probes. +# Every class needed to declare probes should include this module +# and declare the methods that are subject to instrumentation: +# +# class MyClass +# extend Puppet::Util::Instrumentation::Instrumentable +# +# probe :mymethod +# +# def mymethod +# ... this is code to be instrumented ... +# end +# end +module Puppet::Util::Instrumentation::Instrumentable + INSTRUMENTED_CLASSES = {}.extend(MonitorMixin) + + attr_reader :probes + + class Probe + attr_reader :klass, :method, :label, :data + + def initialize(method, klass, options = {}) + @method = method + @klass = klass + + @label = options[:label] || method + @data = options[:data] || {} + end + + def enable + raise "Probe already enabled" if enabled? + + # We're forced to perform this copy because in the class_eval'uated + # block below @method would be evaluated in the class context. It's better + # to close on locally-scoped variables than to resort to complex namespacing + # to get access to the probe instance variables. + method = @method; label = @label; data = @data + klass.class_eval { + alias_method("instrumented_#{method}", method) + define_method(method) do |*args| + id = nil + instrumentation_data = nil + begin + instrumentation_label = label.respond_to?(:call) ? label.call(self, args) : label + instrumentation_data = data.respond_to?(:call) ? data.call(self, args) : data + id = Puppet::Util::Instrumentation.start(instrumentation_label, instrumentation_data) + send("instrumented_#{method}".to_sym, *args) + ensure + Puppet::Util::Instrumentation.stop(instrumentation_label, id, instrumentation_data || {}) + end + end + } + @enabled = true + end + + def disable + raise "Probe is not enabled" unless enabled? + + # For the same reason as in #enable, we're forced to do a local + # copy + method = @method + klass.class_eval do + alias_method(method, "instrumented_#{method}") + remove_method("instrumented_#{method}".to_sym) + end + @enabled = false + end + + def enabled? + !!@enabled + end + end + + # Declares a new probe + # + # It is possible to pass several options that will be later on evaluated + # and sent to the instrumentation layer. + # + # label:: + # this can either be a static symbol/string or a block. If it's a block + # this one will be evaluated on every call of the instrumented method and + # should return a string or a symbol + # + # data:: + # this can be a hash or a block. If it's a block this one will be evaluated + # on every call of the instrumented method and should return a hash. + # + #Example: + # + # class MyClass + # extend Instrumentable + # + # probe :mymethod, :data => Proc.new { |args| { :data => args[1] } }, :label => Proc.new { |args| args[0] } + # + # def mymethod(name, options) + # end + # + # end + # + def probe(method, options = {}) + INSTRUMENTED_CLASSES.synchronize { + (@probes ||= []) << Probe.new(method, self, options) + INSTRUMENTED_CLASSES[self] = @probes + } + end + + def self.probes + @probes + end + + def self.probe_names + probe_names = [] + each_probe { |probe| probe_names << "#{probe.klass}.#{probe.method}" } + probe_names + end + + def self.enable_probes + each_probe { |probe| probe.enable } + end + + def self.disable_probes + each_probe { |probe| probe.disable } + end + + def self.clear_probes + INSTRUMENTED_CLASSES.synchronize { + INSTRUMENTED_CLASSES.clear + } + nil # do not leak our probes to the exterior world + end + + def self.each_probe + INSTRUMENTED_CLASSES.synchronize { + INSTRUMENTED_CLASSES.each_key do |klass| + klass.probes.each { |probe| yield probe } + end + } + nil # do not leak our probes to the exterior world + end +end
\ No newline at end of file diff --git a/lib/puppet/util/instrumentation/listener.rb b/lib/puppet/util/instrumentation/listener.rb new file mode 100644 index 000000000..42ec0c0e9 --- /dev/null +++ b/lib/puppet/util/instrumentation/listener.rb @@ -0,0 +1,60 @@ +require 'puppet/indirector' +require 'puppet/util/instrumentation' +require 'puppet/util/instrumentation/data' + +class Puppet::Util::Instrumentation::Listener + include Puppet::Util + include Puppet::Util::Warnings + extend Puppet::Indirector + + indirects :instrumentation_listener, :terminus_class => :local + + attr_reader :pattern, :listener + attr_accessor :enabled + + def initialize(listener, pattern = nil, enabled = false) + @pattern = pattern.is_a?(Symbol) ? pattern.to_s : pattern + raise "Listener isn't a correct listener (it doesn't provide the notify method)" unless listener.respond_to?(:notify) + @listener = listener + @enabled = enabled + end + + def notify(label, event, data) + listener.notify(label, event, data) + rescue => e + warnonce("Error during instrumentation notification: #{e}") + end + + def listen_to?(label) + enabled? and (!@pattern || @pattern === label.to_s) + end + + def enabled? + !!@enabled + end + + def name + @listener.name.to_s + end + + def data + { :data => @listener.data } + end + + def to_pson(*args) + result = { + :document_type => "Puppet::Util::Instrumentation::Listener", + :data => { + :name => name, + :pattern => pattern, + :enabled => enabled? + } + } + result.to_pson(*args) + end + + def self.from_pson(data) + result = Puppet::Util::Instrumentation[data["name"]] + self.new(result.listener, result.pattern, data["enabled"]) + end +end diff --git a/lib/puppet/util/instrumentation/listeners/log.rb b/lib/puppet/util/instrumentation/listeners/log.rb new file mode 100644 index 000000000..59e9daff9 --- /dev/null +++ b/lib/puppet/util/instrumentation/listeners/log.rb @@ -0,0 +1,29 @@ +require 'monitor' + +# This is an example instrumentation listener that stores the last +# 20 instrumented probe run time. +Puppet::Util::Instrumentation.new_listener(:log) do + + SIZE = 20 + + attr_accessor :last_logs + + def initialize + @last_logs = {}.extend(MonitorMixin) + end + + def notify(label, event, data) + return if event == :start + log_line = "#{label} took #{data[:finished] - data[:started]}" + @last_logs.synchronize { + (@last_logs[label] ||= []) << log_line + @last_logs[label].shift if @last_logs[label].length > SIZE + } + end + + def data + @last_logs.synchronize { + @last_logs.dup + } + end +end
\ No newline at end of file diff --git a/lib/puppet/util/instrumentation/listeners/performance.rb b/lib/puppet/util/instrumentation/listeners/performance.rb new file mode 100644 index 000000000..b3175a0a2 --- /dev/null +++ b/lib/puppet/util/instrumentation/listeners/performance.rb @@ -0,0 +1,30 @@ +require 'monitor' + +Puppet::Util::Instrumentation.new_listener(:performance) do + + attr_reader :samples + + def initialize + @samples = {}.extend(MonitorMixin) + end + + def notify(label, event, data) + return if event == :start + + duration = data[:finished] - data[:started] + samples.synchronize do + @samples[label] ||= { :count => 0, :max => 0, :min => nil, :sum => 0, :average => 0 } + @samples[label][:count] += 1 + @samples[label][:sum] += duration + @samples[label][:max] = [ @samples[label][:max], duration ].max + @samples[label][:min] = [ @samples[label][:min], duration ].reject { |val| val.nil? }.min + @samples[label][:average] = @samples[label][:sum] / @samples[label][:count] + end + end + + def data + samples.synchronize do + @samples.dup + end + end +end
\ No newline at end of file diff --git a/lib/puppet/util/instrumentation/listeners/process_name.rb b/lib/puppet/util/instrumentation/listeners/process_name.rb new file mode 100644 index 000000000..88a185c41 --- /dev/null +++ b/lib/puppet/util/instrumentation/listeners/process_name.rb @@ -0,0 +1,112 @@ +require 'monitor' + +# Unlike the other instrumentation plugins, this one doesn't give back +# data. Instead it changes the process name of the currently running process +# with the last labels and data. +Puppet::Util::Instrumentation.new_listener(:process_name) do + include Sync_m + + # start scrolling when process name is longer than + SCROLL_LENGTH = 50 + + attr_accessor :active, :reason + + def notify(label, event, data) + start(label) if event == :start + stop if event == :stop + end + + def start(activity) + push_activity(Thread.current, activity) + end + + def stop() + pop_activity(Thread.current) + end + + def subscribed + synchronize do + @oldname = $0 + @scroller ||= Thread.new do + loop do + scroll + sleep 1 + end + end + end + end + + def unsubscribed + synchronize do + $0 = @oldname if @oldname + Thread.kill(@scroller) + @scroller = nil + end + end + + def setproctitle + $0 = "#{base}: " + rotate(process_name,@x) + end + + def push_activity(thread, activity) + synchronize do + @reason ||= {} + @reason[thread] ||= [] + @reason[thread].push(activity) + setproctitle + end + end + + def pop_activity(thread) + synchronize do + @reason[thread].pop + if @reason[thread].empty? + @reason.delete(thread) + end + setproctitle + end + end + + def process_name + out = (@reason || {}).inject([]) do |out, reason| + out << "#{thread_id(reason[0])} #{reason[1].join(',')}" + end + out.join(' | ') + end + + # Getting the ruby thread id might not be portable to other ruby + # interpreters than MRI, because Thread#inspect might not return the same + # information on a different runtime. + def thread_id(thread) + thread.inspect.gsub(/^#<.*:0x([a-f0-9]+) .*>$/, '\1') + end + + def rotate(string, steps) + steps ||= 0 + if string.length > 0 && steps > 0 + steps = steps % string.length + return string[steps..-1].concat " -- #{string[0..(steps-1)]}" + end + string + end + + def base + basename = case Puppet.run_mode.name + when :master + "master" + when :agent + "agent" + else + "puppet" + end + end + + def scroll + @x ||= 1 + return if process_name.length < SCROLL_LENGTH + synchronize do + setproctitle + @x += 1 + end + end +end
\ No newline at end of file diff --git a/lib/puppet/util/monkey_patches.rb b/lib/puppet/util/monkey_patches.rb index 81bcdf12f..5186a4e5e 100644 --- a/lib/puppet/util/monkey_patches.rb +++ b/lib/puppet/util/monkey_patches.rb @@ -130,3 +130,11 @@ class IO lines end end + +# Ruby 1.8.5 doesn't have tap +module Kernel + def tap + yield(self) + self + end unless method_defined?(:tap) +end diff --git a/lib/puppet/util/pidlock.rb b/lib/puppet/util/pidlock.rb index fcf0cf296..9ed86352d 100644 --- a/lib/puppet/util/pidlock.rb +++ b/lib/puppet/util/pidlock.rb @@ -1,11 +1,7 @@ require 'fileutils' +require 'puppet/util/anonymous_filelock' -class Puppet::Util::Pidlock - attr_reader :lockfile - - def initialize(lockfile) - @lockfile = lockfile - end +class Puppet::Util::Pidlock < Puppet::Util::AnonymousFilelock def locked? clear_if_stale @@ -17,29 +13,18 @@ class Puppet::Util::Pidlock end def anonymous? - return false unless File.exists?(@lockfile) - File.read(@lockfile) == "" + false end - def lock(opts = {}) - opts = {:anonymous => false}.merge(opts) + def lock + return mine? if locked? - if locked? - mine? - else - if opts[:anonymous] - File.open(@lockfile, 'w') { |fd| true } - else - File.open(@lockfile, "w") { |fd| fd.write(Process.pid) } - end - true - end + File.open(@lockfile, "w") { |fd| fd.write(Process.pid) } + true end def unlock(opts = {}) - opts = {:anonymous => false}.merge(opts) - - if mine? or (opts[:anonymous] and anonymous?) + if mine? File.unlink(@lockfile) true else @@ -47,7 +32,6 @@ class Puppet::Util::Pidlock end end - private def lock_pid if File.exists? @lockfile File.read(@lockfile).to_i @@ -56,6 +40,7 @@ class Puppet::Util::Pidlock end end + private def clear_if_stale return if lock_pid.nil? diff --git a/lib/puppet/util/queue/stomp.rb b/lib/puppet/util/queue/stomp.rb index cabc56627..4a7081bc7 100644 --- a/lib/puppet/util/queue/stomp.rb +++ b/lib/puppet/util/queue/stomp.rb @@ -2,12 +2,14 @@ require 'puppet/util/queue' require 'stomp' require 'uri' -# Implements the Ruby Stomp client as a queue type within the Puppet::Indirector::Queue::Client -# registry, for use with the <tt>:queue</tt> indirection terminus type. +# Implements the Ruby Stomp client as a queue type within the +# Puppet::Indirector::Queue::Client registry, for use with the <tt>:queue</tt> +# indirection terminus type. # -# Looks to <tt>Puppet[:queue_source]</tt> for the sole argument to the underlying Stomp::Client constructor; -# consequently, for this client to work, <tt>Puppet[:queue_source]</tt> must use the Stomp::Client URL-like -# syntax for identifying the Stomp message broker: <em>login:pass@host.port</em> +# Looks to <tt>Puppet[:queue_source]</tt> for the sole argument to the +# underlying Stomp::Client constructor; consequently, for this client to work, +# <tt>Puppet[:queue_source]</tt> must use the Stomp::Client URL-like syntax +# for identifying the Stomp message broker: <em>login:pass@host.port</em> class Puppet::Util::Queue::Stomp attr_accessor :stomp_client @@ -26,10 +28,21 @@ class Puppet::Util::Queue::Stomp rescue => detail raise ArgumentError, "Could not create Stomp client instance with queue source #{Puppet[:queue_source]}: got internal Stomp client error #{detail}" end + + # Identify the supported method for sending messages. + @method = + case + when stomp_client.respond_to?(:publish) + :publish + when stomp_client.respond_to?(:send) + :send + else + raise ArgumentError, "STOMP client does not respond to either publish or send" + end end def publish_message(target, msg) - stomp_client.publish(stompify_target(target), msg, :persistent => true) + stomp_client.__send__(@method, stompify_target(target), msg, :persistent => true) end def subscribe(target) diff --git a/lib/puppet/util/rdoc/parser.rb b/lib/puppet/util/rdoc/parser.rb index a8996ee9a..05e3aab4d 100644 --- a/lib/puppet/util/rdoc/parser.rb +++ b/lib/puppet/util/rdoc/parser.rb @@ -99,7 +99,7 @@ class Parser modpath = $1 name = $2 Puppet.debug "rdoc: module #{name} into #{modpath} ?" - Puppet::Module.modulepath.each do |mp| + Puppet::Node::Environment.new.modulepath.each do |mp| if File.identical?(modpath,mp) Puppet.debug "rdoc: found module #{name}" return name @@ -110,7 +110,7 @@ class Parser # there can be paths we don't want to scan under modules # imagine a ruby or manifest that would be distributed as part as a module # but we don't want those to be hosted under <site> - Puppet::Module.modulepath.each do |mp| + Puppet::Node::Environment.new.modulepath.each do |mp| # check that fullpath is a descendant of mp dirname = fullpath previous = dirname diff --git a/lib/puppet/util/reference.rb b/lib/puppet/util/reference.rb index ae5f2d44c..600325109 100644 --- a/lib/puppet/util/reference.rb +++ b/lib/puppet/util/reference.rb @@ -64,8 +64,6 @@ class Puppet::Util::Reference loaded_instances(:reference).sort { |a,b| a.to_s <=> b.to_s } end - HEADER_LEVELS = [nil, "#", "##", "###", "####", "#####"] - attr_accessor :page, :depth, :header, :title, :dynamic attr_writer :doc @@ -81,20 +79,6 @@ class Puppet::Util::Reference self.dynamic end - def markdown_header(name, level) - "#{HEADER_LEVELS[level]} #{name}\n\n" - end - - def markdown_definitionlist(term, definition) - lines = definition.split("\n") - str = "#{term}\n: #{lines.shift}\n" - lines.each do |line| - str << " " if line =~ /\S/ - str << "#{line}\n" - end - str << "\n" - end - def initialize(name, options = {}, &block) @name = name options.each do |option, value| diff --git a/lib/puppet/util/retryaction.rb b/lib/puppet/util/retryaction.rb new file mode 100644 index 000000000..ba318ec1a --- /dev/null +++ b/lib/puppet/util/retryaction.rb @@ -0,0 +1,48 @@ +module Puppet::Util::RetryAction + class RetryException < Exception; end + class RetryException::NoBlockGiven < RetryException; end + class RetryException::NoRetriesGiven < RetryException;end + class RetryException::RetriesExceeded < RetryException; end + + def self.retry_action( parameters = { :retry_exceptions => nil, :retries => nil } ) + # Retry actions for a specified amount of time. This method will allow the final + # retry to complete even if that extends beyond the timeout period. + unless block_given? + raise RetryException::NoBlockGiven + end + + raise RetryException::NoRetriesGiven if parameters[:retries].nil? + parameters[:retry_exceptions] ||= Hash.new + + start = Time.now + failures = 0 + + begin + yield + rescue Exception => e + # If we were giving exceptions to catch, + # catch the excptions we care about and retry. + # All others fail hard + + raise RetryException::RetriesExceeded if parameters[:retries] == 0 + + if (not parameters[:retry_exceptions].keys.empty?) and parameters[:retry_exceptions].keys.include?(e.class) + Puppet.info("Caught exception #{e.class}:#{e}") + Puppet.info(parameters[:retry_exceptions][e.class]) + elsif (not parameters[:retry_exceptions].keys.empty?) + # If the exceptions is not in the list of retry_exceptions re-raise. + raise e + end + + failures += 1 + parameters[:retries] -= 1 + + # Increase the amount of time that we sleep after every + # failed retry attempt. + sleep (((2 ** failures) -1) * 0.1) + + retry + + end + end +end diff --git a/lib/puppet/util/suidmanager.rb b/lib/puppet/util/suidmanager.rb index d733883b4..82524d031 100644 --- a/lib/puppet/util/suidmanager.rb +++ b/lib/puppet/util/suidmanager.rb @@ -5,7 +5,8 @@ module Puppet::Util::SUIDManager include Puppet::Util::Warnings extend Forwardable - # Note groups= is handled specially due to a bug in OS X 10.6 + # Note groups= is handled specially due to a bug in OS X 10.6, 10.7, + # and probably upcoming releases... to_delegate_to_process = [ :euid=, :euid, :egid=, :egid, :uid=, :uid, :gid=, :gid, :groups ] to_delegate_to_process.each do |method| @@ -28,10 +29,25 @@ module Puppet::Util::SUIDManager module_function :osx_maj_ver def groups=(grouplist) - if osx_maj_ver == '10.6' - return true - else + begin return Process.groups = grouplist + rescue Errno::EINVAL => e + #We catch Errno::EINVAL as some operating systems (OS X in particular) can + # cause troubles when using Process#groups= to change *this* user / process + # list of supplementary groups membership. This is done via Ruby's function + # "static VALUE proc_setgroups(VALUE obj, VALUE ary)" which is effectively + # a wrapper for "int setgroups(size_t size, const gid_t *list)" (part of SVr4 + # and 4.3BSD but not in POSIX.1-2001) that fails and sets errno to EINVAL. + # + # This does not appear to be a problem with Ruby but rather an issue on the + # operating system side. Therefore we catch the exception and look whether + # we run under OS X or not -- if so, then we acknowledge the problem and + # re-throw the exception otherwise. + if osx_maj_ver and not osx_maj_ver.empty? + return true + else + raise e + end end end module_function :groups= diff --git a/lib/puppet/util/symbolic_file_mode.rb b/lib/puppet/util/symbolic_file_mode.rb new file mode 100644 index 000000000..de07b061a --- /dev/null +++ b/lib/puppet/util/symbolic_file_mode.rb @@ -0,0 +1,140 @@ +require 'puppet/util' + +module Puppet::Util::SymbolicFileMode + SetUIDBit = ReadBit = 4 + SetGIDBit = WriteBit = 2 + StickyBit = ExecBit = 1 + SymbolicMode = { 'x' => ExecBit, 'w' => WriteBit, 'r' => ReadBit } + SymbolicSpecialToBit = { + 't' => { 'u' => StickyBit, 'g' => StickyBit, 'o' => StickyBit }, + 's' => { 'u' => SetUIDBit, 'g' => SetGIDBit, 'o' => StickyBit } + } + + def valid_symbolic_mode?(value) + value = normalize_symbolic_mode(value) + return true if value =~ /^0?[0-7]{1,4}$/ + return true if value =~ /^([ugoa]*[-=+][-=+rstwxXugo]*)(,[ugoa]*[-=+][-=+rstwxXugo]*)*$/ + return false + end + + def normalize_symbolic_mode(value) + return nil if value.nil? + + # We need to treat integers as octal numbers. + if value.is_a? Numeric then + return value.to_s(8) + elsif value =~ /^0?[0-7]{1,4}$/ then + return value.to_i(8).to_s(8) + else + return value + end + end + + def symbolic_mode_to_int(modification, to_mode = 0, is_a_directory = false) + if modification.nil? or modification == '' then + raise Puppet::Error, "An empty mode string is illegal" + end + if modification =~ /^[0-7]+$/ then return modification.to_i(8) end + if modification =~ /^\d+$/ then + raise Puppet::Error, "Numeric modes must be in octal, not decimal!" + end + + fail "non-numeric current mode (#{to_mode.inspect})" unless to_mode.is_a?(Numeric) + + original_mode = { + 's' => (to_mode & 07000) >> 9, + 'u' => (to_mode & 00700) >> 6, + 'g' => (to_mode & 00070) >> 3, + 'o' => (to_mode & 00007) >> 0, + # Are there any execute bits set in the original mode? + 'any x?' => (to_mode & 00111) != 0 + } + final_mode = { + 's' => original_mode['s'], + 'u' => original_mode['u'], + 'g' => original_mode['g'], + 'o' => original_mode['o'], + } + + modification.split(/\s*,\s*/).each do |part| + begin + _, to, dsl = /^([ugoa]*)([-+=].*)$/.match(part).to_a + if dsl.nil? then raise Puppet::Error, 'Missing action' end + to = "a" unless to and to.length > 0 + + # We want a snapshot of the mode before we start messing with it to + # make actions like 'a-g' atomic. Various parts of the DSL refer to + # the original mode, the final mode, or the current snapshot of the + # mode, for added fun. + snapshot_mode = {} + final_mode.each {|k,v| snapshot_mode[k] = v } + + to.gsub('a', 'ugo').split('').uniq.each do |who| + value = snapshot_mode[who] + + action = '!' + actions = { + '!' => lambda {|_,_| raise Puppet::Error, 'Missing operation (-, =, or +)' }, + '=' => lambda {|m,v| m | v }, + '+' => lambda {|m,v| m | v }, + '-' => lambda {|m,v| m & ~v }, + } + + dsl.split('').each do |op| + case op + when /[-+=]/ then + action = op + # Clear all bits, if this is assignment + value = 0 if op == '=' + + when /[ugo]/ then + value = actions[action].call(value, snapshot_mode[op]) + + when /[rwx]/ then + value = actions[action].call(value, SymbolicMode[op]) + + when 'X' then + # Only meaningful in combination with "set" actions. + if action != '+' then + raise Puppet::Error, "X only works with the '+' operator" + end + + # As per the BSD manual page, set if this is a directory, or if + # any execute bit is set on the original (unmodified) mode. + # Ignored otherwise; it is "add if", not "add or clear". + if is_a_directory or original_mode['any x?'] then + value = actions[action].call(value, ExecBit) + end + + when /[st]/ then + bit = SymbolicSpecialToBit[op][who] or fail "internal error" + final_mode['s'] = actions[action].call(final_mode['s'], bit) + + else + raise Puppet::Error, 'Unknown operation' + end + end + + # Now, assign back the value. + final_mode[who] = value + end + + rescue Puppet::Error => e + if part.inspect != modification.inspect then + rest = " at #{part.inspect}" + else + rest = '' + end + + raise Puppet::Error, "#{e}#{rest} in symbolic mode #{modification.inspect}" + end + end + + result = + final_mode['s'] << 9 | + final_mode['u'] << 6 | + final_mode['g'] << 3 | + final_mode['o'] << 0 + return result + end +end diff --git a/lib/puppet/util/zaml.rb b/lib/puppet/util/zaml.rb index bbb2af2d2..e07a2d7b9 100644 --- a/lib/puppet/util/zaml.rb +++ b/lib/puppet/util/zaml.rb @@ -1,4 +1,15 @@ +# encoding: UTF-8 # +# The above encoding line is a magic comment to set the default source encoding +# of this file for the Ruby interpreter. It must be on the first or second +# line of the file if an interpreter is in use. In Ruby 1.9 and later, the +# source encoding determines the encoding of String and Regexp objects created +# from this source file. This explicit encoding is important becuase otherwise +# Ruby will pick an encoding based on LANG or LC_CTYPE environment variables. +# These may be different from site to site so it's important for us to +# establish a consistent behavior. For more information on M17n please see: +# http://links.puppetlabs.com/understanding_m17n + # ZAML -- A partial replacement for YAML, writen with speed and code clarity # in mind. ZAML fixes one YAML bug (loading Exceptions) and provides # a replacement for YAML.dump unimaginatively called ZAML.dump, @@ -218,10 +229,16 @@ end class String ZAML_ESCAPES = %w{\x00 \x01 \x02 \x03 \x04 \x05 \x06 \a \x08 \t \n \v \f \r \x0e \x0f \x10 \x11 \x12 \x13 \x14 \x15 \x16 \x17 \x18 \x19 \x1a \e \x1c \x1d \x1e \x1f } def escaped_for_zaml - gsub( /\x5C/, "\\\\\\" ). # Demi-kludge for Maglev/rubinius; the regexp should be /\\/ but parsetree chokes on that. - gsub( /"/, "\\\"" ). - gsub( /([\x00-\x1F])/ ) { |x| ZAML_ESCAPES[ x.unpack("C")[0] ] }. - gsub( /([\x80-\xFF])/ ) { |x| "\\x#{x.unpack("C")[0].to_s(16)}" } + # JJM (Note the trailing dots to construct a multi-line method chain.) This + # code is meant to escape all bytes which are not ASCII-8BIT printable + # characters. Multi-byte unicode characters are handled just fine because + # each byte of the character results in an escaped string emitted to the + # YAML stream. When the YAML is de-serialized back into a String the bytes + # will be reconstructed properly into the unicode character. + self.to_ascii8bit.gsub( /\x5C/n, "\\\\\\" ). # Demi-kludge for Maglev/rubinius; the regexp should be /\\/ but parsetree chokes on that. + gsub( /"/n, "\\\"" ). + gsub( /([\x00-\x1F])/n ) { |x| ZAML_ESCAPES[ x.unpack("C")[0] ] }. + gsub( /([\x80-\xFF])/n ) { |x| "\\x#{x.unpack("C")[0].to_s(16)}" } end def to_zaml(z) z.first_time_only(self) { @@ -238,7 +255,10 @@ class String (self =~ /[\s:]$/) or (self =~ /^[>|][-+\d]*\s/i) or (self[-1..-1] =~ /\s/) or - (self =~ /[\x00-\x08\x0B\x0C\x0E-\x1F\x80-\xFF]/) or + # This regular expression assumes the string is a byte sequence. + # It does not concern itself with characters so we convert the string + # to ASCII-8BIT for Ruby 1.9 to match up encodings. + (self.to_ascii8bit=~ /[\x00-\x08\x0B\x0C\x0E-\x1F\x80-\xFF]/n) or (self =~ /[,\[\]\{\}\r\t]|:\s|\s#/) or (self =~ /\A([-:?!#&*'"]|<<|%.+:.)/) ) @@ -251,6 +271,20 @@ class String end } end + + # Return a guranteed ASCII-8BIT encoding for Ruby 1.9 This is a helper + # method for other methods that perform regular expressions against byte + # sequences deliberately rather than dealing with characters. + # The method may or may not return a new instance. + def to_ascii8bit + if self.respond_to?(:encoding) and self.encoding.name != "ASCII-8BIT" then + str = self.dup + str.force_encoding("ASCII-8BIT") + return str + else + return self + end + end end class Hash |