Initial commit

.gitignore

@@ -0,0 +1 @@
+pkg

README.md

@@ -0,0 +1,94 @@
+Manage a AIO Installation of The Marionette Collective
+======================================================
+
+This is an oppinionated module to manage an already installed Puppet AIO based mcollective
+
+  * Configures the main `server.cfg` and `client.cfg` and service
+  * Provides a mcollective plugin packager that produce AIO specific modules of mco plugins
+  * Installs a number of default puppet related plugins
+  * Creates directories the AIO packages failed to create
+
+Configuring Server and Client
+-----------------------------
+
+Server and Client settings are managed using Hiera data and these keys are set up for deep
+hash merging using look up strategies.
+
+Some settings like the connector and security provider tend to be common to the Client and
+Server, these are configured using Hiera:
+
+```yaml
+mcollective::common_config:
+  securityprovider: puppet
+```
+
+This will result in both the main `server.cfg` and `client.cfg` having these settings.
+
+Likewise client specific settings can be set with `mcollective::client_config` and
+`mcollective::server_config`.
+
+To enable a specific node to be a MCollective client you have to set `mcollective::client`
+to `true` via hiera
+
+Installing Plugins
+------------------
+
+By default plugins like the puppet, service, package agents and packager will be installed.
+These are typically shipped as their own classes or modules and you can add to the list by
+setting the following in hiera:
+
+```yaml
+mcollective::plugin_classes:
+  - your::agent
+```
+
+Or you can just install them however you prefer.
+
+Plugin Packaging
+----------------
+
+MCollective has a framework for packaging plugins, while this framework is a bit buggy
+and old it's still useful for the common cases, this module provides a plugin to the
+packager that lets you build AIO specific modules to install your agents.
+
+It will be installed on all nodes that are set to install the client, to use it do something
+like this:
+
+```
+$ cd your_agent
+$ mco plugin package --format aiomodulepackage --vendor yourco
+```
+
+When you're done you'll have a new Puppet module called someting like `yourco-mcollective_agent_example`
+that has a dependency on this module.
+
+The resulting module is heavily customizable using Hiera, see the `README` in your module for details.
+It will have a data file `data/plugin.yaml` full of file lists and such, you can mix in configuration,
+dependencies and more by creating a file `.plugin.yaml` in your repo that might look like this:
+
+```yaml
+mcollective_agent_example::gem_dependencies:
+  eventmachine:
+    "1.0.7"
+
+mcollective_agent_example::class_dependencies:
+  - another::class
+
+mcollective_agent_example::common_config:
+  "some.setting": "config value"
+```
+
+This will be merged into the final module data and configure the resulting Puppet module based on
+Puppet 4 data in modules.
+
+Status
+------
+
+It's early days for this module and right now it only works with RedHat - or probably all Unix thanks
+to the AIO standards.  Supporting other OSes is quite easy by adding files in `os/OsFamily.yaml`,
+contributions appreciated.
+
+I have a goal to make using files like `~/.mcollective` and the very complex nature of managing
+those completely redundant.  So this module does nothing at present to help you manage those.  They
+are INI files though so you can easily use `puppetlabs-inifile`, which this module use to manage the
+main config files too.

data/common.yaml

@@ -0,0 +1,45 @@
+---
+mcollective::common_config:
+  identity: "%{trusted.certname}"
+  #securityprovider: puppet
+  #connector: nats
+
+mcollective::plugin_classes:
+  - mcollective::packager
+  - mcollective_agent_puppet
+  - mcollective_agent_service
+    #  - mcollective_connector_nats
+    #  - mcollective_security_puppet
+
+mcollective::server: true
+mcollective::client: false
+mcollective::service_name: "mcollective"
+mcollective::service_ensure: "running"
+mcollective::service_enable: true
+mcollective::plugin_owner: "root"
+mcollective::plugin_group: "root"
+mcollective::plugin_mode: "0664"
+mcollective::libdir: "/opt/puppetlabs/mcollective/plugins/mcollective"
+mcollective::plugintypes:
+  - "agent"
+  - "aggregate"
+  - "application"
+  - "audit"
+  - "connector"
+  - "data"
+  - "discovery"
+  - "registration"
+  - "security"
+  - "util"
+  - "validator"
+  - "pluginpackager"
+
+lookup_options:
+  mcollective::plugin_classes:
+    merge: unique
+  mcollective::client_config:
+    merge:
+      strategy: deep
+  mcollective::server_config:
+    merge:
+      strategy: deep

hiera.yaml

@@ -0,0 +1,9 @@
+---
+version: 4
+datadir: "data"
+hierarchy:
+  - name: "OS family"
+    backend: yaml
+    path: "os/%{facts.os.family}"
+  - name: "common"
+    backend: "yaml"

lib/mcollective/pluginpackager/aiomodulepackage_packager.rb

@@ -0,0 +1,172 @@
+require "yaml"
+
+module MCollective
+  module PluginPackager
+    class AiomodulepackagePackager
+      def initialize(plugin, pluginpath = nil, signature = nil, verbose = false, keep_artifacts = nil, module_template = nil)
+        @plugin = plugin
+        @verbose = verbose
+        @keep_artifacts = keep_artifacts
+        @module_template = module_template || File.join(File.dirname(__FILE__), 'templates', 'aiomodule')
+      end
+
+      def create_packages
+        assert_new_enough_puppet
+
+        begin
+          puts("Building AIO module %s" % module_name)
+
+          @tmpdir = Dir.mktmpdir('mcollective_packager')
+
+          make_module_dirs
+          copy_module_files
+          render_templates
+          run_build
+          move_package
+
+          puts("Completed building module for %s" % module_name)
+        ensure
+          if @keep_artifacts
+            puts("Keeping build artifacts")
+            puts("Build artifacts saved in %s" %  @tmpdir)
+          else
+            cleanup_tmpdirs
+          end
+        end
+      end
+
+      def module_name
+        "mcollective_%s_%s" % [@plugin.plugintype.downcase, @plugin.metadata[:name].downcase]
+      end
+
+      def module_file_name
+        "%s-%s-%s.tar.gz" % [@plugin.vendor, module_name, @plugin.metadata[:version]]
+      end
+
+      def dirlist(type)
+        @plugin.packagedata[type][:files].map do |file|
+          file.gsub(/^\.\//, "") if File.directory?(file)
+        end.compact
+      rescue
+        []
+      end
+
+      def filelist(type)
+        @plugin.packagedata[type][:files].map do |file|
+          file.gsub(/^\.\//, "") unless File.directory?(file)
+        end.compact
+      rescue
+        []
+      end
+
+      def hierakey(var)
+        "%s::%s" % [module_name, var]
+      end
+
+      def module_override_data
+        YAML.load_file(".plugin.yaml")
+      rescue
+        {}
+      end
+
+      def plugin_hiera_data
+        {
+          hierakey(:config_name) => @plugin.metadata[:name].downcase,
+          hierakey(:common_files) => filelist(:common),
+          hierakey(:common_directories) => dirlist(:common),
+          hierakey(:server_files) => filelist(:agent),
+          hierakey(:server_directories) => dirlist(:agent),
+          hierakey(:client_files) => filelist(:client),
+          hierakey(:client_directories) => dirlist(:client),
+        }.merge(module_override_data)
+      end
+
+      def make_module_dirs
+        ["data", "manifests", "files/mcollective"].each do |dir|
+          FileUtils.mkdir_p(File.join(@tmpdir, dir))
+        end
+      end
+
+      def copy_module_files
+        @plugin.packagedata.each do |klass, data|
+          data[:files].each do |file|
+            dest_dir = File.expand_path(File.join(@tmpdir, "files", "mcollective", File.dirname(file)))
+            FileUtils.mkdir_p(dest_dir) unless File.directory?(dest_dir)
+            FileUtils.cp(file, dest_dir) if File.file?(file)
+          end
+        end
+      end
+
+      def render_templates
+        templates = Dir.chdir(@module_template) do |path|
+          Dir.glob("**/*.erb")
+        end
+
+        templates.each do |template|
+          infile = File.join(@module_template, template)
+          outfile = File.join(@tmpdir, template.gsub(/\.erb$/, ""))
+          render_template(infile, outfile)
+        end
+      end
+
+      def render_template(infile, outfile)
+        begin
+          erb = ERB.new(File.read(infile), nil, '-')
+          File.open(outfile, 'w') do |f|
+            f.puts erb.result(binding)
+          end
+        rescue
+          STDERR.puts("Could not render template %s to %s" % [infile, outfile])
+          raise
+        end
+      end
+
+      def assert_new_enough_puppet
+        unless File.executable?("/opt/puppetlabs/bin/puppet")
+          raise("Cannot build package. '/opt/puppetlabs/bin/puppet' is not present on the system.")
+        end
+
+        s = Shell.new("/opt/puppetlabs/bin/puppet --version")
+        s.runcommand
+        actual_version = s.stdout.chomp
+        required_version = '4.5.1'
+
+        if Util.versioncmp(actual_version, required_version) < 0
+          raise("Cannot build package. puppet #{required_version} or greater required.  We have #{actual_version}.")
+        end
+      end
+
+      def run_build
+        begin
+          PluginPackager.execute_verbosely(@verbose) do
+            Dir.chdir(@tmpdir) do
+              PluginPackager.safe_system('puppet module build')
+            end
+          end
+        rescue
+          STDERR.puts("Build process has failed")
+          raise
+        end
+      end
+
+      def move_package
+        begin
+          package_file = File.join(@tmpdir, "pkg", module_file_name)
+          FileUtils.cp(package_file, ".")
+        rescue
+          STDERR.puts("Could not copy package to working directory")
+          raise
+        end
+      end
+
+      def cleanup_tmpdirs
+        begin
+          FileUtils.rm_r(@tmpdir) if File.directory?(@tmpdir)
+        rescue
+          STDERR.puts("Could not remove temporary build directory %s" % [@tmpdir])
+          raise
+        end
+      end
+    end
+  end
+end

lib/mcollective/pluginpackager/templates/aiomodule/README.md.erb

@@ -0,0 +1 @@
+# <%= module_name %>

lib/mcollective/pluginpackager/templates/aiomodule/data/defaults.yaml.erb

@@ -0,0 +1,16 @@
+lookup_options:
+  <%= module_name %>::gem_dependencies:
+    merge:
+      strategy: deep
+  <%= module_name %>::package_dependencies:
+    merge:
+      strategy: deep
+  <%= module_name %>::config:
+    merge:
+      strategy: deep
+  <%= module_name %>::client_config:
+    merge:
+      strategy: deep
+  <%= module_name %>::server_config:
+    merge:
+      strategy: deep

lib/mcollective/pluginpackager/templates/aiomodule/data/plugin.yaml.erb

@@ -0,0 +1 @@
+<%= YAML.dump(plugin_hiera_data) %>

lib/mcollective/pluginpackager/templates/aiomodule/hiera.yaml.erb

@@ -0,0 +1,8 @@
+---
+version: 4
+datadir: "data"
+hierarchy:
+  - name: "plugin"
+    backend: "yaml"
+  - name: "defaults"
+    backend: "yaml"

lib/mcollective/pluginpackager/templates/aiomodule/manifests/init.pp.erb

@@ -0,0 +1,42 @@
+class <%= module_name %> (
+  String $config_name,
+  Array[String] $client_files = [],
+  Array[String] $client_directories = [],
+  Array[String] $server_files = [],
+  Array[String] $server_directories = [],
+  Array[String] $common_files = [],
+  Array[String] $common_directories = [],
+  Boolean $manage_gem_dependencies = true,
+  Hash $gem_dependencies = {},
+  Boolean $manage_package_dependencies = true,
+  Hash $package_dependencies = {},
+  Boolean $manage_class_dependencies = true,
+  Array[String] $class_dependencies = [],
+  Hash $config = {},
+  Hash $client_config = {},
+  Hash $server_config = {},
+  Boolean $client = $mcollective::client,
+  Boolean $server = $mcollective::server,
+  Enum["present", "absent"] $ensure = "present"
+) {
+  mcollective::module_plugin{$name:
+    config_name                 => $config_name,
+    client_files                => $client_files,
+    server_files                => $server_files,
+    common_files                => $common_files,
+    client_directories          => $client_directories,
+    server_directories          => $server_directories,
+    common_directories          => $common_directories,
+    gem_dependencies            => $gem_dependencies,
+    manage_gem_dependencies     => $manage_gem_dependencies,
+    package_dependencies        => $package_dependencies,
+    manage_package_dependencies => $manage_package_dependencies,
+    class_dependencies          => $class_dependencies,
+    config                      => $config,
+    client_config               => $client_config,
+    server_config               => $server_config,
+    client                      => $client,
+    server                      => $server,
+    ensure                      => $ensure
+  }
+}