diff --git a/src/clj/puppetlabs/services/legacy_routes/legacy_routes_service.clj b/src/clj/puppetlabs/services/legacy_routes/legacy_routes_service.clj index 270cffaf5..5a5dd3043 100644 --- a/src/clj/puppetlabs/services/legacy_routes/legacy_routes_service.clj +++ b/src/clj/puppetlabs/services/legacy_routes/legacy_routes_service.clj @@ -44,8 +44,6 @@ (constantly nil) false nil - nil - nil (get-in config [:puppetserver :certname]))) master-route-handler (comidi/routes->handler master-routes) master-mount (master-core/get-master-mount diff --git a/src/clj/puppetlabs/services/master/file_serving.clj b/src/clj/puppetlabs/services/master/file_serving.clj deleted file mode 100644 index b7c3c1aa3..000000000 --- a/src/clj/puppetlabs/services/master/file_serving.clj +++ /dev/null @@ -1,372 +0,0 @@ -(ns puppetlabs.services.master.file-serving - (:require [bidi.bidi :as bidi] - [clojure.java.io :as io] - [clojure.string :as str] - [digest :as digest] - [me.raynes.fs :as fs] - [puppetlabs.i18n.core :as i18n] - [puppetlabs.puppetserver.common :as common] - [puppetlabs.ring-middleware.utils :as middleware-utils] - [ring.util.response :as rr]) - (:import com.sun.security.auth.module.UnixSystem - [java.nio.file Files FileSystems FileVisitOption FileVisitResult LinkOption Paths SimpleFileVisitor])) - -(def follow-links - (into-array LinkOption [])) - -(defn as-path - [string & more] - (Paths/get string (into-array String more))) - -(defn path-exists? - [path] - (Files/exists path follow-links)) - -(defn get-checksum - [file algorithm] - (case algorithm - "md5" (str "{md5}" (digest/md5 (io/file file))) - "sha256" (str "{sha256}" (digest/sha-256 (io/file file))) - :default (throw (Exception. "Unsupported digest")))) - -(defn read-attributes - [path checksum-type ignore-source-permissions] - (let [attributes (Files/readAttributes path "unix:*" follow-links)] - {:owner (if ignore-source-permissions - (.getUid (UnixSystem.)) - (get attributes "uid")) - :group (if ignore-source-permissions - (.getGid (UnixSystem.)) - (get attributes "gid")) - :mode (if ignore-source-permissions - 0644 ;; Yes, Puppet even sends 644 for directories in this case - (bit-and (get attributes "mode") 07777)) - :type (if (get attributes "isDirectory") - "directory" - "file") - :checksum (if (get attributes "isDirectory") - {:type "ctime" - :value (str "{ctime}" (get attributes "ctime"))} - {:type checksum-type - :value (get-checksum path checksum-type)})})) - -(defn get-file-metadata - [checksum-type ignore-source-permissions [relative-path root]] - (merge (read-attributes (.resolve root relative-path) checksum-type ignore-source-permissions) - {:path (.toString root) - :relative_path (.toString relative-path) - :links "follow" - :destination nil})) - -(defn relativize - "A version of java's Path#relativize that matches puppet's behavior" - [root path] - (let [relative-path (.relativize root path)] - (if (= (.toString relative-path) "") - (as-path ".") - relative-path))) - -(defn make-visitor - [tree root ignores] - (let [filesystem (FileSystems/getDefault) - matchers (map #(.getPathMatcher filesystem (str "glob:" %)) ignores) - ignore? (fn [path] - (let [file-name (.getFileName path)] - (some #(.matches % file-name) matchers)))] - (proxy [SimpleFileVisitor] [] - (preVisitDirectory [path attributes] - (if (ignore? path) - FileVisitResult/SKIP_SUBTREE - (do - (swap! tree assoc (relativize root path) root) - FileVisitResult/CONTINUE))) - (visitFile [path attributes] - (when-not (ignore? path) - (swap! tree assoc (relativize root path) root)) - FileVisitResult/CONTINUE)))) - -(defn walk-directory - [path ignores] - (let [results (atom {}) - absolute-path (.toAbsolutePath path) - visitor (make-visitor results absolute-path ignores)] - (Files/walkFileTree absolute-path #{FileVisitOption/FOLLOW_LINKS} Integer/MAX_VALUE visitor) - @results)) - -(defn is-bolt-project? - [dir] - (or (fs/exists? (str dir "/bolt-project.yaml")) - (fs/exists? (str dir "/bolt.yaml")) - (fs/exists? (str dir "/Boltdir")))) - -(defn get-project-root - "Lookup a project by name and return the path where its project files are - located as a string." - [projects-dir versioned-project] - (let [boltdir-path (str projects-dir "/" versioned-project "/Boltdir")] - (if (fs/exists? boltdir-path) - boltdir-path - (str projects-dir "/" versioned-project)))) - -(defn list-dirs-in-paths - "Given a list of directories, return a list of all the subdirectories - concatenated, preserving the order of the original list. Directories that - don't exist will be ignored." - [paths] - (->> paths - (filter fs/exists?) - (mapcat (comp sort fs/list-dir)) - (filter fs/directory?))) - -(defn dirs-in-project-modulepath - "List all directories in a bolt project's modulepath. Returns a sequence of - File objects." - [modulepath project-root] - (->> modulepath - (map #(str project-root "/" %)) - list-dirs-in-paths)) - -(defn find-project-module - "Given the path of a project, the name of a module, and the modulepath defined - in the project's config (can be nil) search the project's module path and - return the path to that module as a File, or nil." - [bolt-builtin-content-dir project-root module modulepath] - (->> bolt-builtin-content-dir - list-dirs-in-paths - (concat (dirs-in-project-modulepath modulepath project-root)) - (filter #(= module (fs/base-name %))) - first)) - -(defn mount->path-component - "Map the fileserving \"mount\" to the subdirectory of modules it will serve - files out of." - [mount] - (case mount - "modules" "files" - ;; default to itself - mount)) - -(defn read-bolt-project-config - [project-dir] - (let [config-path (str project-dir "/bolt-project.yaml")] - (when (fs/file? config-path) - (common/parse-yaml (slurp config-path))))) - -(defn parse-modulepath - "The modulepath for a bolt project can either be an array of paths or a string - with a path separator." - [modulepath] - (if (string? modulepath) - (str/split modulepath #":") - modulepath)) - -(defn get-project-modulepath - "Given a map representing the bolt project configuration, return the - modulepath. If the modulepath is not defined or the configuration map passed - is nil then return the default modulepath." - [project-config] - ;; Note that the :modulepath config key's useful when the project - ;; organizes its modules inside relative directories. It doesn't - ;; make sense for absolute paths since those may not exist - ;; on the Puppetserver host. - (-> (get project-config :modulepath "modules") - (parse-modulepath) - (concat [".modules"]))) - -(defn find-project-file - "Find a file in a project using the parameters from a `file_content` request. - Returns the path as as string. If the module name is the same as the project - name then files are served from the project directly." - [bolt-builtin-content-dir bolt-projects-dir versioned-project mount module path] - (when (is-bolt-project? (str bolt-projects-dir "/" versioned-project)) - (let [project-root (get-project-root bolt-projects-dir versioned-project) - project-config (read-bolt-project-config project-root) - project-name (get project-config :name) - modulepath (get-project-modulepath project-config) - module-root (if (= project-name module) - project-root - (find-project-module bolt-builtin-content-dir project-root module modulepath)) - file-path (str module-root "/" (mount->path-component mount) "/" path)] - (when (fs/exists? file-path) - file-path)))) - -(defn mount-dirs-in-modulepath - "Collect all the paths represented by a specific mount that are found in the - modulepath. If `project-as-module?` is truthy then include any mount - directory found at the top level of the project." - [bolt-builtin-content-dir mount modulepath project-root project-as-module?] - (let [sub-dir (case mount - "plugins" "lib" - "pluginfacts" "facts.d") - modules-in-modulepath (concat - (dirs-in-project-modulepath modulepath project-root) - (list-dirs-in-paths bolt-builtin-content-dir)) - module-dirs (if project-as-module? - (cons (fs/file project-root) modules-in-modulepath) - modules-in-modulepath)] - (->> module-dirs - (map #(.resolve (.toPath %) sub-dir)) - (filter path-exists?)))) - -(defn project-configured-as-module? - "Given a project config, return whether it is configured to use the project - itself as a module. Users opt in to this by including a project name setting - in the configuration file. " - [project-config] - (boolean (get project-config :name))) - -(defn plugin-file-if-exists - "Given a relative path as received by the plugins mount, and the path of a lib - dir in a module, return the path of the file if it exists in the lib dir." - [relative-path lib-root] - (let [file-path (.resolve lib-root relative-path)] - (when (path-exists? file-path) - (.toString file-path)))) - -(defn find-project-plugin-file - "Given a relative path as received by the plugins mount, search all lib dirs - and return the path to the file if it's found in any of them." - [bolt-builtin-content-dir bolt-projects-dir versioned-project mount relative-path] - (let [project-root (get-project-root bolt-projects-dir versioned-project) - project-config (read-bolt-project-config project-root) - project-as-module? (project-configured-as-module? project-config) - modulepath (get-project-modulepath project-config) - mount-dirs (mount-dirs-in-modulepath bolt-builtin-content-dir mount modulepath project-root project-as-module?)] - (some (partial plugin-file-if-exists relative-path) mount-dirs))) - -(defn get-plugins-metadata - "Return the metadata for pluginsync. This scans the lib directories of all - modules and returns a list of files smashed together." - [bolt-builtin-content-dir bolt-projects-dir versioned-project mount checksum-type ignores ignore-source-permissions] - (when (is-bolt-project? (str bolt-projects-dir "/" versioned-project)) - (let [project-root (get-project-root bolt-projects-dir versioned-project) - project-config (read-bolt-project-config project-root) - project-as-module? (project-configured-as-module? project-config) - modulepath (get-project-modulepath project-config) - files (->> (mount-dirs-in-modulepath bolt-builtin-content-dir mount modulepath project-root project-as-module?) - (map #(walk-directory % ignores)) - reverse - (apply merge))] - (map (partial get-file-metadata checksum-type ignore-source-permissions) files)))) - -(def project-routes - "Bidi routing table for project file_content endpoint. This is done separately - because we have to do additional routing after branching on the query - parameter, which is not natively supported in bidi." - - ["" {[[#"modules|scripts|tasks" :mount-point] "/" :module "/" [#".+" :file-path]] :basic - [[#"plugins|pluginfacts" :mount-point] #"/?" [#".*" :file-path]] :pluginsync}]) - -(defn make-file-content-response - "Given a path to a file, generate an appropriate ring response map. Returns a - 404 response if passed `nil`." - [file requested-file-path] - (if file - (-> file - rr/file-response - (rr/content-type "application/octet-stream")) - {:status 404 - :headers {"Content-Type" "text/plain"} - :body (i18n/tru "Could not find file_content for path: {0}" requested-file-path)})) - -(defn handle-project-file-content - "Handle a file_content request for a bolt project." - [bolt-builtin-content-dir bolt-projects-dir request] - (let [versioned-project (get-in request [:params "versioned_project"]) - path (get-in request [:params :rest]) - match (bidi/match-route project-routes path)] - (when match - (let [mount-point (get-in match [:route-params :mount-point]) - mount-type (get-in match [:handler]) - module (get-in match [:route-params :module]) - file-path (get-in match [:route-params :file-path])] - (case mount-type - :basic (make-file-content-response - (find-project-file bolt-builtin-content-dir bolt-projects-dir versioned-project mount-point module file-path) - file-path) - :pluginsync (make-file-content-response - (find-project-plugin-file bolt-builtin-content-dir bolt-projects-dir versioned-project mount-point file-path) - file-path) - {:status 400 - :headers {"Content-Type" "text/plain"} - :body (i18n/tru "Unsupported mount: {0}" mount-point)}))))) - -(defn file-content-handler - "Handle file_content requests and dispatch them to the correct handler for - environments or projects." - [bolt-builtin-content-dir bolt-projects-dir ruby-request-handler request] - (let [versioned-project (get-in request [:params "versioned_project"]) - environment (get-in request [:params "environment"])] - (cond - (and versioned-project environment) - {:status 400 - :headers {"Content-Type" "text/plain"} - :body (i18n/tru "A file_content request cannot specify both `environment` and `versioned_project` query parameters.")} - (and (nil? versioned-project) (nil? environment)) - {:status 400 - :headers {"Content-Type" "text/plain"} - :body (i18n/tru "A file_content request must include an `environment` or `versioned_project` query parameter.")} - versioned-project - (handle-project-file-content bolt-builtin-content-dir bolt-projects-dir request) - - :else - (ruby-request-handler request)))) - -(defn metadatas-params-errors - "Reject requests with parameter values that we haven't implemented yet. - Returns a list of errors." - [params] - (->> - (for [[value default] {"recurse" "true" - "links" "follow" - "source_permissions" "ignore" - "recurselimit" "infinite"}] - (when (and (get params value) - (not (= value default))) - (str "The only supported value of `" value "` at this time is `" default "`"))) - (filter (comp not nil?)))) - -(defn handle-project-file-metadatas - "Handle a file_metadatas request for a bolt project." - [bolt-builtin-content-dir bolt-projects-dir request] - (let [versioned-project (get-in request [:params "versioned_project"]) - path (get-in request [:params :rest]) - match (bidi/match-route project-routes path)] - (when match - (let [mount-point (get-in match [:route-params :mount-point]) - mount-type (get-in match [:handler]) - ignore (get-in request [:params "ignore"] []) - checksum-type (get-in request [:params "checksum_type"] "sha256")] - (case mount-type - :pluginsync (let [errors (metadatas-params-errors (get-in request [:params]))] - (if (empty? errors) - (middleware-utils/json-response - 200 - (get-plugins-metadata bolt-builtin-content-dir bolt-projects-dir versioned-project mount-point checksum-type ignore true)) - {:status 400 - :headers {"Content-Type" "text/plain"} - :body (str/join "\n" (cons "Not all parameter values are supported in this implementation: " errors))})) - {:status 400 - :headers {"Content-Type" "text/plain"} - :body (i18n/tru "Unsupported mount: {0}" mount-point)}))))) - -(defn file-metadatas-handler - "Handle file_metadatas requests and dispatch them to the correct handler for - environments or projects." - [bolt-builtin-content-dir bolt-projects-dir ruby-request-handler request] - (let [versioned-project (get-in request [:params "versioned_project"]) - environment (get-in request [:params "environment"])] - (cond - (and versioned-project environment) - {:status 400 - :headers {"Content-Type" "text/plain"} - :body (i18n/tru "A file_metadatas request cannot specify both `environment` and `versioned_project` query parameters.")} - (and (nil? versioned-project) (nil? environment)) - {:status 400 - :headers {"Content-Type" "text/plain"} - :body (i18n/tru "A file_metadatas request must include an `environment` or `versioned_project` query parameter.")} - versioned-project - (handle-project-file-metadatas bolt-builtin-content-dir bolt-projects-dir request) - - :else - (ruby-request-handler request)))) diff --git a/src/clj/puppetlabs/services/master/master_core.clj b/src/clj/puppetlabs/services/master/master_core.clj index 5dfea3ae6..981d59cda 100644 --- a/src/clj/puppetlabs/services/master/master_core.clj +++ b/src/clj/puppetlabs/services/master/master_core.clj @@ -18,7 +18,6 @@ [puppetlabs.puppetserver.ringutils :as ringutils] [puppetlabs.ring-middleware.core :as middleware] [puppetlabs.ring-middleware.utils :as middleware-utils] - [puppetlabs.services.master.file-serving :as file-serving] [puppetlabs.services.protocols.jruby-puppet :as jruby-protocol] [puppetlabs.trapperkeeper.services.status.status-core :as status-core] [ring.middleware.params :as ring] @@ -963,55 +962,26 @@ (catch Throwable e (log/info e (i18n/trs "Failed to report action"))))))))) -(defn parse-project-compile-data - "Parse data required to compile a catalog inside a project. Data required includes - * Root path to project - * modulepath - * hiera config - * project_name" - [request-options - versioned-project - bolt-projects-dir] - (let [project-root (file-serving/get-project-root bolt-projects-dir versioned-project) - project-config (file-serving/read-bolt-project-config project-root)] - (assoc request-options "project_root" project-root - "modulepath" (map #(str project-root "/" %) (file-serving/get-project-modulepath project-config)) - "hiera_config" (str project-root "/" (get project-config :hiera-config "hiera.yaml")) - "project_name" (:name project-config)))) - (schema/defn ^:always-validate compile-fn :- IFn [jruby-service :- (schema/protocol jruby-protocol/JRubyPuppetService) current-code-id-fn :- IFn - boltlib-path :- (schema/maybe [schema/Str]) - bolt-projects-dir :- (schema/maybe schema/Str)] + boltlib-path :- (schema/maybe [schema/Str])] (fn [request] (let [request-options (-> request :body slurp (validated-body CompileRequest)) - versioned-project (get request-options "versioned_project") environment (get request-options "environment")] - ;; Check to ensure environment/versioned_project are mutually exlusive and - ;; at least one of them is set. - (cond - (and versioned-project environment) - {:status 400 - :headers {"Content-Type" "text/plain"} - :body (i18n/tru "A compile request cannot specify both `environment` and `versioned_project` parameters.")} - - (and (nil? versioned-project) (nil? environment)) + ;; Check to ensure environment is set. + + (if (nil? environment) {:status 400 :headers {"Content-Type" "text/plain"} :body (i18n/tru "A compile request must include an `environment` or `versioned_project` parameter.")} - :else - (let [compile-options (if versioned-project - ;; we need to parse some data from the project config for project compiles - (parse-project-compile-data request-options versioned-project bolt-projects-dir) - ;; environment compiles only need to set the code ID - (assoc request-options + (let [compile-options (assoc request-options "code_id" - (current-code-id-fn environment)))] + (current-code-id-fn environment))] {:status 200 :headers {"Content-Type" "application/json"} :body (json/encode @@ -1035,9 +1005,8 @@ [jruby-service :- (schema/protocol jruby-protocol/JRubyPuppetService) wrap-with-jruby-queue-limit :- IFn current-code-id-fn :- IFn - boltlib-path :- (schema/maybe [schema/Str]) - bolt-projects-dir :- (schema/maybe schema/Str)] - (-> (compile-fn jruby-service current-code-id-fn boltlib-path bolt-projects-dir) + boltlib-path :- (schema/maybe [schema/Str])] + (-> (compile-fn jruby-service current-code-id-fn boltlib-path) (jruby-request/wrap-with-jruby-instance jruby-service) wrap-with-jruby-queue-limit jruby-request/wrap-with-error-handling)) @@ -1061,17 +1030,14 @@ v3-ruby-routes :- bidi-schema/RoutePair "v3 route tree for the ruby side of the master service." [request-handler :- IFn - bolt-builtin-content-dir :- (schema/maybe [schema/Str]) - bolt-projects-dir :- (schema/maybe schema/Str) certname :- schema/Str] (comidi/routes (comidi/GET ["/node/" [#".*" :rest]] request (request-handler request)) (comidi/GET ["/file_content/" [#".*" :rest]] request - ;; Not strictly ruby routes anymore because of this - (file-serving/file-content-handler bolt-builtin-content-dir bolt-projects-dir request-handler (ring/params-request request))) + (request-handler request)) (comidi/GET ["/file_metadatas/" [#".*" :rest]] request - (file-serving/file-metadatas-handler bolt-builtin-content-dir bolt-projects-dir request-handler (ring/params-request request))) + (request-handler request)) (comidi/GET ["/file_metadata/" [#".*" :rest]] request (request-handler request)) (comidi/GET ["/file_bucket_file/" [#".*" :rest]] request @@ -1101,8 +1067,7 @@ current-code-id-fn :- IFn cache-enabled :- schema/Bool wrap-with-jruby-queue-limit :- IFn - boltlib-path :- (schema/maybe [schema/Str]) - bolt-projects-dir :- (schema/maybe schema/Str)] + boltlib-path :- (schema/maybe [schema/Str])] (let [class-handler (create-cacheable-info-handler-with-middleware (fn [jruby env] (some-> jruby-service @@ -1137,8 +1102,7 @@ jruby-service wrap-with-jruby-queue-limit current-code-id-fn - boltlib-path - bolt-projects-dir)] + boltlib-path)] (comidi/routes (comidi/POST "/compile" request (compile-handler' request)) @@ -1190,19 +1154,16 @@ environment-class-cache-enabled :- schema/Bool wrap-with-jruby-queue-limit :- IFn boltlib-path :- (schema/maybe [schema/Str]) - bolt-builtin-content-dir :- (schema/maybe [schema/Str]) - bolt-projects-dir :- (schema/maybe schema/Str) certname :- schema/Str] (comidi/context "/v3" - (v3-ruby-routes ruby-request-handler bolt-builtin-content-dir bolt-projects-dir certname) + (v3-ruby-routes ruby-request-handler certname) (comidi/wrap-routes (v3-clojure-routes jruby-service get-code-content-fn current-code-id-fn environment-class-cache-enabled wrap-with-jruby-queue-limit - boltlib-path - bolt-projects-dir) + boltlib-path) clojure-request-wrapper))) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -1305,8 +1266,6 @@ current-code-id-fn :- IFn environment-class-cache-enabled :- schema/Bool boltlib-path :- (schema/maybe [schema/Str]) - bolt-builtin-content-dir :- (schema/maybe [schema/Str]) - bolt-projects-dir :- (schema/maybe schema/Str) certname :- schema/Str] (comidi/routes (v3-routes ruby-request-handler @@ -1317,8 +1276,6 @@ environment-class-cache-enabled wrap-with-jruby-queue-limit boltlib-path - bolt-builtin-content-dir - bolt-projects-dir certname) (v4-routes clojure-request-wrapper jruby-service @@ -1381,8 +1338,6 @@ wrap-with-jruby-queue-limit :- IFn environment-class-cache-enabled :- schema/Bool boltlib-path :- (schema/maybe [schema/Str]) - bolt-builtin-content-dir :- (schema/maybe [schema/Str]) - bolt-projects-dir :- (schema/maybe schema/Str) certname :- schema/Str] (let [ruby-request-handler (wrap-middleware handle-request wrap-with-authorization-check @@ -1400,8 +1355,6 @@ current-code-id environment-class-cache-enabled boltlib-path - bolt-builtin-content-dir - bolt-projects-dir certname))) (def MasterStatusV1 diff --git a/src/clj/puppetlabs/services/master/master_service.clj b/src/clj/puppetlabs/services/master/master_service.clj index 1c332f100..e97563fc3 100644 --- a/src/clj/puppetlabs/services/master/master_service.clj +++ b/src/clj/puppetlabs/services/master/master_service.clj @@ -115,8 +115,6 @@ certname (get-in config [:puppetserver :certname]) localcacert (get-in config [:puppetserver :localcacert]) puppet-version (get-in config [:puppetserver :puppet-version]) - bolt-builtin-content-dir (get-in config [:bolt :builtin-content-dir] []) - bolt-projects-dir (get-in config [:bolt :projects-dir]) max-queued-requests (get-in config [:jruby-puppet :max-queued-requests] 0) max-retry-delay (get-in config [:jruby-puppet :max-retry-delay] 1800) settings (ca/config->master-settings config) @@ -147,8 +145,6 @@ wrap-with-jruby-queue-limit environment-class-cache-enabled boltlib-path - bolt-builtin-content-dir - bolt-projects-dir certname)) routes (comidi/context path ring-app) route-metadata (comidi/route-metadata routes) diff --git a/src/ruby/puppetserver-lib/puppet/server/master.rb b/src/ruby/puppetserver-lib/puppet/server/master.rb index 9414b7ae9..d9a18188a 100644 --- a/src/ruby/puppetserver-lib/puppet/server/master.rb +++ b/src/ruby/puppetserver-lib/puppet/server/master.rb @@ -248,7 +248,7 @@ def check_cadir_for_deprecation_warning # Each array element is examined, if it is expected to be a map # we call back to the convert_java_args_to_ruby method, if it # is expected to be an array, we recurse otherwise we do not modify - # the value. + # the value. def resolve_java_objects_from_list(list) list.map do |value| if value.kind_of?(Java::ClojureLang::IPersistentMap) diff --git a/test/integration/puppetlabs/services/master/master_service_test.clj b/test/integration/puppetlabs/services/master/master_service_test.clj index 257e9c1c9..b45f50ce1 100644 --- a/test/integration/puppetlabs/services/master/master_service_test.clj +++ b/test/integration/puppetlabs/services/master/master_service_test.clj @@ -770,155 +770,3 @@ Integer/parseInt)] (is (= 503 status-code)) (is (<= 0 retry-after 1800)))))))) - -(deftest ^:integration project-file-content - (bootstrap-testutils/with-puppetserver-running - app - {:bolt {:builtin-content-dir ["./dev-resources/puppetlabs/services/master/master_core_test/builtin_bolt_content"] - :projects-dir "./dev-resources/puppetlabs/services/master/master_core_test/bolt_projects"} - :jruby-puppet {:gem-path gem-path - :max-active-instances 1 - :server-code-dir test-resources-code-dir - :server-conf-dir master-service-test-runtime-dir}} - - (testing "can retrieve file_content from the modules mount from a project" - (let [response (http-get "/puppet/v3/file_content/modules/utilities/etc/greeting?versioned_project=local_23")] - (is (= 200 (:status response))) - (is (= "Good morning\n" (:body response))) - (is (= "13" (get-in response [:headers "content-length"]))) - (is (= "application/octet-stream" (get-in response [:headers "content-type"]))))) - - ;; this also tests that we can retrieve file content from the .modules mount of the project - ;; since that is where this task is located - (testing "can retrieve file_content from the tasks mount from a project" - (let [response (http-get "/puppet/v3/file_content/tasks/utilities/blah?versioned_project=local_23")] - (is (= 200 (:status response))) - (is (= "bye" (:body response))) - (is (= "3" (get-in response [:headers "content-length"]))) - (is (= "application/octet-stream" (get-in response [:headers "content-type"]))))) - - (testing "can retrieve file_content from the top level of a project" - (let [response (http-get "/puppet/v3/file_content/tasks/local/init.sh?versioned_project=local_23")] - (is (= 200 (:status response))) - (is (= ". $PT__installdir/helpers/files/marco.sh\nmarco\n" (:body response))) - (is (= "63" (get-in response [:headers "content-length"]))) - (is (= "application/octet-stream" (get-in response [:headers "content-type"]))))) - - (testing "can retrieve file_content from a project with an embedded structure" - (let [response (http-get "/puppet/v3/file_content/modules/test/packages?versioned_project=embedded_e19e09")] - (is (= 200 (:status response))) - (is (= "vim" (:body response))) - (is (= "3" (get-in response [:headers "content-length"]))) - (is (= "application/octet-stream" (get-in response [:headers "content-type"]))))) - - (testing "can retrieve file_content from a custom modulepath" - (let [response (http-get "/puppet/v3/file_content/modules/helpers/marco.sh?versioned_project=local_afafaf")] - (is (= 200 (:status response))) - (is (= "marco () {\n echo \"polo\"\n}\n" (:body response))) - (is (= "27" (get-in response [:headers "content-length"]))) - (is (= "application/octet-stream" (get-in response [:headers "content-type"]))))) - - (testing "can retrieve built-in file_content" - (let [response (http-get "/puppet/v3/file_content/tasks/bic_module_one/init.sh?versioned_project=local_23")] - (is (= 200 (:status response))) - (is (= ". $PT__installdir/helpers/files/marco.sh\nmarco\n" (:body response))) - (is (= "63" (get-in response [:headers "content-length"]))) - (is (= "application/octet-stream" (get-in response [:headers "content-type"]))))) - - (testing "can retrieve overriden built-in file_content" - (let [response (http-get "/puppet/v3/file_content/tasks/bic_module_one/init.sh?versioned_project=override_builtin_content")] - (is (= 200 (:status response))) - (is (= ". $PT__installdir/helpers/files/marco.sh\noverride_marco\n" (:body response))) - (is (= "73" (get-in response [:headers "content-length"]))) - (is (= "application/octet-stream" (get-in response [:headers "content-type"]))))) - - (testing "cannot retrieve file_content from the default modulepath when a custom modulepath is set" - (let [response (http-get "/puppet/v3/file_content/tasks/utilities/blah?versioned_project=local_afafaf")] - (is (= 404 (:status response))))) - - (testing "mount point not found" - (let [response (http-get "/puppet/v3/file_content/glorb/test/packages?versioned_project=local_23")] - (is (= 404 (:status response))))) - - (testing "project not found" - (let [response (http-get "/puppet/v3/file_content/modules/test/packages?versioned_project=nothere")] - (is (= 404 (:status response))))) - - (testing "module not found" - (let [response (http-get "/puppet/v3/file_content/modules/nomodule/packages?versioned_project=embedded_e19e09")] - (is (= 404 (:status response))))) - - (testing "missing path?" - (let [response (http-get "/puppet/v3/file_content/modules/test/?versioned_project=embedded_e19e09")] - (is (= 404 (:status response))))) - - (testing "can retrieve plugin metadata" - (let [response (http-get "/puppet/v3/file_metadatas/plugins?versioned_project=local_23") - [file-entry] (filter #(= "puppet/monkey_patch.rb" (get % "relative_path")) (json/decode (:body response)))] - ;; Only check some of the entries that won't vary based on the test environment - (is (= nil (get file-entry "destination"))) - (is (= "file" (get file-entry "type"))) - (is (= "sha256" (get-in file-entry ["checksum" "type"]))) - (is (= "{sha256}76b2e03b82880885385595045033c4e3122e373c7023e037461a650ec85829ad" (get-in file-entry ["checksum" "value"]))) - ;; Does it choose from the right module? - (is (str/ends-with? (get file-entry "path") "modules/helpers/lib")))) - - (testing "can retrieve builtin plugin metadata" - (let [response (http-get "/puppet/v3/file_metadatas/plugins?versioned_project=local_23") - [file-entry] (filter #(= "puppet/builtin_monkey_patch.rb" (get % "relative_path")) (json/decode (:body response)))] - ;; Only check some of the entries that won't vary based on the test environment - (is (= nil (get file-entry "destination"))) - (is (= "file" (get file-entry "type"))) - (is (= "sha256" (get-in file-entry ["checksum" "type"]))) - (is (= "{sha256}76b2e03b82880885385595045033c4e3122e373c7023e037461a650ec85829ad" (get-in file-entry ["checksum" "value"]))) - ;; Does it choose from the right module? - (is (str/ends-with? (get file-entry "path") "bic_module_one/lib")))) - - (testing "can retrieve plugin files" - (let [response (http-get "/puppet/v3/file_content/plugins/puppet/monkey_patch.rb?versioned_project=local_23")] - (is (= 200 (:status response))) - (is (= "class NilClass\n def empty?\n true\n end\nend\n" (:body response))) - (is (= "59" (get-in response [:headers "content-length"]))) - (is (= "application/octet-stream" (get-in response [:headers "content-type"]))))) - - (testing "can retrieve builtin plugin files" - (let [response (http-get "/puppet/v3/file_content/plugins/puppet/builtin_monkey_patch.rb?versioned_project=local_23")] - (is (= 200 (:status response))) - (is (= "class NilClass\n def empty?\n true\n end\nend\n" (:body response))) - (is (= "59" (get-in response [:headers "content-length"]))) - (is (= "application/octet-stream" (get-in response [:headers "content-type"]))))) - - (testing "can retrieve overridden builtin plugin files" - (let [response (http-get "/puppet/v3/file_content/plugins/puppet/builtin_monkey_patch.rb?versioned_project=override_builtin_content")] - (is (= 200 (:status response))) - (is (= "overridden_class NilClass\n def empty?\n true\n end\nend\n" (:body response))) - (is (= "70" (get-in response [:headers "content-length"]))) - (is (= "application/octet-stream" (get-in response [:headers "content-type"]))))) - - (testing "can retrieve plugin files from the top level project lib dir" - (let [response (http-get "/puppet/v3/file_content/plugins/puppet/comment.rb?versioned_project=local_23")] - (is (= 200 (:status response))) - (is (= "# This is the project\n" (:body response))) - (is (= "22" (get-in response [:headers "content-length"]))) - (is (= "application/octet-stream" (get-in response [:headers "content-type"]))))) - - (testing "can retrieve pluginfacts metadata" - (let [response (http-get "/puppet/v3/file_metadatas/pluginfacts?versioned_project=local_23") - [file-entry] (filter #(= "something" (get % "relative_path")) (json/decode (:body response)))] - (is (= nil (get file-entry "destination"))) - (is (= "file" (get file-entry "type"))) - (is (= "sha256" (get-in file-entry ["checksum" "type"]))) - (is (= "{sha256}4bc453b53cb3d914b45f4b250294236adba2c0e09ff6f03793949e7e39fd4cc1" (get-in file-entry ["checksum" "value"]))))) - - (testing "can retrieve pluginfacts files" - (let [response (http-get "/puppet/v3/file_content/pluginfacts/unhelpful?versioned_project=local_23")] - (is (= 200 (:status response))) - (is (= "factually unhelpful\n" (:body response))) - (is (= "20" (get-in response [:headers "content-length"]))) - (is (= "application/octet-stream" (get-in response [:headers "content-type"]))))) - - (testing "doesn't support nonstandard options" - (let [response (http-get "/puppet/v3/file_metadatas/plugins?versioned_project=local_23&links=manage")] - (is (= 400 (:status response))) - (is (= "Not all parameter values are supported in this implementation: \nThe only supported value of `links` at this time is `follow`" - (:body response))))))) diff --git a/test/unit/puppetlabs/services/master/master_core_test.clj b/test/unit/puppetlabs/services/master/master_core_test.clj index 9386820c0..7b33895bc 100644 --- a/test/unit/puppetlabs/services/master/master_core_test.clj +++ b/test/unit/puppetlabs/services/master/master_core_test.clj @@ -50,8 +50,6 @@ (constantly nil) true nil - ["./dev-resources/puppetlabs/services/master/master_core_test/builtin_bolt_content"] - "./dev-resources/puppetlabs/services/master/master_core_test/bolt_projects" "test-certname") (comidi/routes->handler) (wrap-middleware identity puppet-version))) @@ -447,36 +445,11 @@ :trusted_facts {:values {}} :variables {:values {}}}))))] (is (= 200 (:status response))))) - (testing "compile endpoint for projects" + (testing "compile endpoint fails with no environment" (let [response (app (-> {:request-method :post :uri "/v3/compile" :content-type "application/json"} (ring-mock/body (json/encode {:certname "foo" - :versioned_project "fake_project" - :code_ast "{\"__pcore_something\": \"Foo\"}" - :facts {:values {}} - :trusted_facts {:values {}} - :variables {:values {}} - :options {:compile_for_plan true}}))))] - (is (= 200 (:status response))))) - (testing "compile endpoint fails with no environment or versioned_project" - (let [response (app (-> {:request-method :post - :uri "/v3/compile" - :content-type "application/json"} - (ring-mock/body (json/encode {:certname "foo" - :code_ast "{\"__pcore_something\": \"Foo\"}" - :facts {:values {}} - :trusted_facts {:values {}} - :variables {:values {}} - :options {:compile_for_plan true}}))))] - (is (= 400 (:status response))))) - (testing "compile endpoint fails with both environment and versioned_project" - (let [response (app (-> {:request-method :post - :uri "/v3/compile" - :content-type "application/json"} - (ring-mock/body (json/encode {:certname "foo" - :environment "production" - :versioned_project "fake_project" :code_ast "{\"__pcore_something\": \"Foo\"}" :facts {:values {}} :trusted_facts {:values {}}