From 6ed0ff1cb6f9519cfad0b32c3cd2f130bf43ceb2 Mon Sep 17 00:00:00 2001 From: Eugen Stan Date: Tue, 31 Oct 2023 23:49:12 +0200 Subject: [PATCH 1/9] [#240] Improve cli ux and add cli example / docs * BREAKING? Enhanced Store protocol with close alias for disconnect * BREAKING? connect now returns the store (this) * create (migration) fn now resolves absolute file path for migration * Enhanced docs for Store protocol * CLI can load config from file * CLI can load config from env * CLI can load config from cli args * CLI merges configs in this order: file, env, args * Added some tests for CLI parsing * Added status command, to display migration status: - connection info - so we know which server we are connecting to - migrations directory - so we know where we get migrations from - the latest applied migration - the list of not-applied migrations - any migrations applied and not present ?! --- .gitignore | 2 +- CHANGES.md | 5 + README.md | 15 +- examples/postgres/README.md | 77 ++++ examples/postgres/deps.edn | 10 + examples/postgres/migratus.sh | 4 + .../src/migratus/examples/postgres.clj | 12 + src/migratus/cli.clj | 356 ++++++++++++++---- src/migratus/core.clj | 4 +- src/migratus/database.clj | 30 +- src/migratus/migrations.clj | 16 +- src/migratus/protocols.clj | 9 +- test/migratus/mock.clj | 3 +- test/migratus/test/cli_test.clj | 39 ++ 14 files changed, 477 insertions(+), 105 deletions(-) create mode 100644 examples/postgres/README.md create mode 100644 examples/postgres/deps.edn create mode 100755 examples/postgres/migratus.sh create mode 100644 examples/postgres/src/migratus/examples/postgres.clj create mode 100644 test/migratus/test/cli_test.clj diff --git a/.gitignore b/.gitignore index 63146cf..385e2a4 100644 --- a/.gitignore +++ b/.gitignore @@ -16,5 +16,5 @@ migratus.iml .lsp/.cache .portal .cpcache -.calva/output-window/ +.calva .clj-kondo/ \ No newline at end of file diff --git a/CHANGES.md b/CHANGES.md index e156d49..473107b 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,8 @@ +### Unreleased +* Improved CLI interface +* Improved CLI documentation +* Added example project + ### 1.5.6 * [Fix spec problem for next.jdbc.sql/insert! and next.jdbc.sql/delete! ](https://github.com/yogthos/migratus/pull/262) diff --git a/README.md b/README.md index e57ddca..d35737d 100644 --- a/README.md +++ b/README.md @@ -130,6 +130,11 @@ The environment variable associated with the `database.table` key will replace ` ### Setup +Migratus can be used as a library in your project or as a CLI tool. +There is also an option (from a third party) to run a migratus as native binary - for PostgreSQL only. + +#### Using Migratus as a library in your project + - Add Migratus as a dependency to your `project.clj` ```clojure :dependencies [[migratus ]] @@ -214,6 +219,14 @@ It is possible to pass a `java.sql.Connection` or `javax.sql.DataSource` in plac (def config {:db {:datasource (hk/make-datasource datasource-options)}}) ``` +#### Using migratus as a command line (cli) tool + +Migratus exposes a CLI interface via `migratus.cli` namespace. +It uses [tools.cli](https://github.com/clojure/tools.cli) for argument parsing. + + + + #### Running as native image (Postgres only) [PGMig](https://github.com/leafclick/pgmig) is a standalone tool built with migratus that's compiled as a standalone GraalVM native image executable. @@ -391,7 +404,7 @@ This is intended for use with http://2ndquadrant.com/en/resources/pglogical/ and | `migratus.core/rollback` | Run `down` for the last migration that was run. | | `migratus.core/rollback-until-just-after` | Run `down` all migrations after `migration-id`. This only considers completed migrations, and will not migrate up. | | `migratus.core/up` | Run `up` for the specified migration ids. Will skip any migration that is already up. | - | `migratus.core/down` | Run `down` for the specified migration ids. Will skip any migration that is already down. + | `migratus.core/down` | Run `down` for the specified migration ids. Will skip any migration that is already down. | `migratus.core/reset` | Reset the database by down-ing all migrations successfully applied, then up-ing all migratinos. | `migratus.core/pending-list` | Returns a list of pending migrations. | | `migratus.core/migrate-until-just-before` | Run `up` for for any pending migrations which precede the given migration id (good for testing migrations). | diff --git a/examples/postgres/README.md b/examples/postgres/README.md new file mode 100644 index 0000000..0507943 --- /dev/null +++ b/examples/postgres/README.md @@ -0,0 +1,77 @@ +# Example using migratus with Postgres + +This is an example project that uses migratus to apply migrations to a PostgreSQL database. + +TODO: Add instructions on how to use migratus via code. + +## Using migratus via cli + +### Setup your database + +If you have an existing database, skip this step. +This guide uses docker for PostgreSQL setup, but you can setup PostgreSQL any way you like. + +Bellow is a short guide on how to manage a PostgreSQL db as a container for the purpose of the guide. + +For more information on PostgreSQL see [postgres image](https://hub.docker.com/_/postgres/) + +```shell +# Run a postgresql instance as a container named migratus-pg. +# We ask PostgreSQL to create a database named migratus_example_db +docker run --name migratus-pg --detach -p 5432:5432 \ + -e POSTGRES_PASSWORD=migrate-me \ + -e POSTGRES_DB=migratus_example_db \ + -v migratus_data:/var/lib/postgresql/data \ + postgres:latest + +# If all is well, we should see the container running +docker ps + +> CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES +> c37a91d27631 postgres:latest "docker-entrypoint.s…" 23 seconds ago Up 23 seconds 0.0.0.0:5432->5432/tcp, :::5432->5432/tcp migratus-pg + +# View the data volume for postgres +docker volume ls + +# And we can view the logs (in another terminal perhaps ?!) +docker logs -f migratus-pg + +# You can stop and start the container +docker container stop migratus-pg +docker container start migratus-pg + +# We can remove the container once you are done, or if you want to reset everything +docker container rm --force --volumes migratus-pg +``` + +### Setup migratus cli + +We use migratus cli via `deps.edn` aliases. +See the `deps.edn` file in this project for details. + +The file should look like this +```clojure +{:paths ["resources"] + :deps {io.github.yogthos/migratus {:mvn/version "RELEASE"} + org.postgresql/postgresql {:mvn/version "42.6.0"}} + :aliases + {:migratus {:jvm-opts ["-Dclojure.main.report=stderr"] + :main-opts ["-m" "migratus.cli"]}}} +``` + +If you have such a configuration, we can use `clojure` or `clj` tool to drive the CLI. +Since Migratus is a clojure library, we need to run it via clojure like this `clojure -M:migratus --help` + +There is also a bash script `migratus.sh` that does the same: `./migratus.sh --help` + + +Commands with migratus + +```shell + +# We export the configuration as env variable, but we can use cli or file as well +export MIGRATUS_CONFIG='{:store :database :db {:jdbcUrl "jdbc:postgresql://localhost:5432/migratus_example_db?user=postgres&password=migrate-me"}}' + +clojure -M:migratus status + +``` \ No newline at end of file diff --git a/examples/postgres/deps.edn b/examples/postgres/deps.edn new file mode 100644 index 0000000..037908b --- /dev/null +++ b/examples/postgres/deps.edn @@ -0,0 +1,10 @@ +{;; migration sql files will be on "resources" + :paths ["src" "resources"] + ;; we need migratus and postgresql jdbc driver on the classpath + :deps {io.github.yogthos/migratus {:local/root "../.."} + org.postgresql/postgresql {:mvn/version "42.6.0"}} + :aliases + {:migratus {:jvm-opts [;; print clojure errors to standard out instead of cli + "-Dclojure.main.report=stderr"] + ;; Run migratus.cli -main fn + :main-opts ["-m" "migratus.cli"]}}} diff --git a/examples/postgres/migratus.sh b/examples/postgres/migratus.sh new file mode 100755 index 0000000..fdd028b --- /dev/null +++ b/examples/postgres/migratus.sh @@ -0,0 +1,4 @@ +#!/bin/sh + +# Run migratus passing all args to it +clojure -J-Dclojure.main.report=stderr -M:migratus "$@" \ No newline at end of file diff --git a/examples/postgres/src/migratus/examples/postgres.clj b/examples/postgres/src/migratus/examples/postgres.clj new file mode 100644 index 0000000..b8bb7a6 --- /dev/null +++ b/examples/postgres/src/migratus/examples/postgres.clj @@ -0,0 +1,12 @@ +(ns migratus.examples.postgres + (:require [migratus.cli :as cli])) + + + +(comment + + (apply cli/-main ["list"]) + + + + ) \ No newline at end of file diff --git a/src/migratus/cli.clj b/src/migratus/cli.clj index f77ed16..1efe235 100644 --- a/src/migratus/cli.clj +++ b/src/migratus/cli.clj @@ -1,12 +1,16 @@ (ns migratus.cli - (:require [clojure.data.json :as json] - [clojure.core :as core] + (:require [clojure.core :as core] + [clojure.data.json :as json] + [clojure.edn :as edn] [clojure.java.io :as io] [clojure.string :as str] [clojure.tools.cli :refer [parse-opts]] [clojure.tools.logging :as log] - [migratus.core :as migratus]) - (:import [java.time ZoneId ZoneOffset] + [migratus.core :as migratus] + [migratus.migrations :as mig] + [migratus.protocols :as proto]) + (:import [java.io IOException] + [java.time ZoneId ZoneOffset] [java.time.format DateTimeFormatter] [java.util.logging ConsoleHandler @@ -15,35 +19,170 @@ Logger SimpleFormatter])) -;; needed fro Clojure 1.10 compatibility -(defn parse-long [s] +(defn my-parse-long + "parse-long version for Clojure 1.10 compatibility" + [s] (Long/valueOf s)) +(defn my-parse-boolean + [s] + (Boolean/parseBoolean s)) + +(def app-config + "Application options to be used for output/logging. + To avoid passing them around everywhere." + (atom {:verbosity 0 + :output-format "plain"})) + +(defn deep-merge + "From https://clojuredocs.org/clojure.core/merge ." + [a & maps] + (if (map? a) + (apply merge-with deep-merge a maps) + (apply merge-with deep-merge maps))) + +(defn println-err + [& more] + (binding [*out* *err*] + (apply println more))) + +(defn env->config! + "Try to load migratus configuration options from environment. + We produce a configuration that can be merged with other configs. + Missing values are ok as configuring via env is optional. + + Looks and processes the following env vars: + + - MIGRATUS_CONFIG - read string as edn and return config. + Do not process any other migratus env var. + + - MIGRATUS_STORE - apply clojure.core/keyword fn to value + - MIGRATUS_MIGRATION_DIR - expect string, use as is + - MIGRATUS_DB_SPEC - read string as edn + - MIGRATUS_TABLE_NAME - string + - MIGRATUS_INIT_IN_TRANSACTION - parse boolean + + Return: A map representing the migratus configuration." + ([] + (env->config! (System/getenv))) + ([env] + (let [config (get env "MIGRATUS_CONFIG")] + (if config + (edn/read-string config) + ;; we don't have MIGRATUS_CONFIG - check the other vars + (let [store (get env "MIGRATUS_STORE") + migration-dir (get env "MIGRATUS_MIGRATION_DIR") + table (get env "MIGRATUS_TABLE_NAME") + init-in-transaction (get env "MIGRATUS_INIT_IN_TRANSACTION") + db (get env "MIGRATUS_DB_SPEC")] + (cond-> {} + store (assoc :store (keyword store)) + migration-dir (assoc :migration-dir migration-dir) + table (assoc :migration-table-name table) + init-in-transaction (assoc :init-in-transaction + (my-parse-boolean init-in-transaction)) + db (assoc :db (edn/read-string db)))))))) + +(defn cli-args->config + "Parse any migratus configuration options from cli args. + + Return a migratus configuration map with any values. + + We expect the args we receive to be values + processed by tools.cli parse-opts fn." + [config-edn-str] + (let [config (edn/read-string config-edn-str)] + (if (map? config) + config + {}))) + +(defn file->config! + "Read config-file as a edn. + If config-file is nil, return nil. + On IO exception print warning and return nil." + [^String config-file] + (when config-file + (let [config-path (.getAbsolutePath (io/file config-file))] + (try + (edn/read-string (slurp config-path)) + (catch IOException e + (println-err + "WARN: Error reading config" (.getMessage e) + "\nYou can use --config path_to_file to specify a path to config file")))))) + +(defn load-config! + "Load configuration and merge options. + + Options are loaded in this order. + Sbsequent values are deepmerged and replace previous ones. + + - configuration file - if it exists and we can parse it as edn + - environment variables + - command line arguments passed to the application + + Return a migratus configuration map." + [config-file config-data] + (let [config (file->config! config-file) + env (env->config!) + args (cli-args->config config-data)] + (deep-merge config env args))) + +#_(defn valid-config? + "Validate a migratus configuration for required options. + If valid, return true. + If invalid return map with reasons why validation failed." + [config] + (if (map? config) + (let [store (:store config)] + (if store + true + {:errors ["Missing :store key in configuration"]})) + {:errors ["Config is nil or not a map"]})) + +#_(defn- do-invalid-config + "We got invalid configuration. + Print error and exit" + [valid? cfg] + (println-err "Invalid configuration:" (:errors valid?) + "\nMigratus can load configuration from: file, env vars, cli args." + "\nSee documentation and/or use --help") + (println-err "Configuration is" cfg)) + (defn validate-format [s] (boolean (some (set (list s)) #{"plain" "edn" "json"}))) (def global-cli-options - [[nil "--config NAME" "Configuration file name" :default "migratus.edn"] - ["-v" nil "Verbosity level; may be specified multiple times to increase value" + [[nil "--config-file NAME" "Configuration file name"] + ["-v" nil + "Verbosity level; may be specified multiple times to increase value" :id :verbosity :default 0 :update-fn inc] + [nil "--output-format FORMAT" + "Option to print in plain text (default), edn or json" + :default "plain" + :validate [#(validate-format %) + "Unsupported format. Valid options: plain (default), edn, json."]] + [nil "--config CONFIG" "Configuration as edn"] ["-h" "--help"]]) (def migrate-cli-options - [[nil "--until-just-before MIGRATION-ID" "Run all migrations preceding migration-id. This is useful when testing that a migration behaves as expected on fixture data. This only considers uncompleted migrations, and will not migrate down."] + [[nil "--until-just-before MIGRATION-ID" + "Run all migrations preceding migration-id. + This is useful when testing that a migration behaves as expected on fixture data. + This only considers uncompleted migrations, and will not migrate down."] ["-h" "--help"]]) (def rollback-cli-options - [[nil "--until-just-after MIGRATION-ID" "Migrate down all migrations after migration-id. This only considers completed migrations, and will not migrate up."] + [[nil "--until-just-after MIGRATION-ID" + "Migrate down all migrations after migration-id. + This only considers completed migrations, and will not migrate up."] ["-h" "--help"]]) (def list-cli-options [[nil "--available" "List all migrations, applied and non applied"] [nil "--pending" "List pending migrations"] [nil "--applied" "List applied migrations"] - [nil "--format FORMAT" "Option to print in plain text (default), edn or json" :default "plain" - :validate [#(validate-format %) "Unsupported format. Valid options: plain (default), edn, json."]] ["-h" "--help"]]) (defn usage [options-summary] @@ -51,6 +190,7 @@ "" "Actions:" " init" + " status" " create" " migrate" " reset" @@ -59,39 +199,41 @@ " down" " list" "" - "options:" + "global options:" options-summary] (str/join \newline))) (defn error-msg [errors] - (binding [*out* *err*] - (println "The following errors occurred while parsing your command:\n\n" - (str/join \newline errors)))) + (println-err + "The following errors occurred while parsing your command:\n\n" + (str/join \newline errors))) (defn no-match-message "No matching clause message info" [arguments summary] - (binding [*out* *err*] - (println "Migratus API does not support this action(s) : " arguments "\n\n" - (str/join (usage summary))))) + (println-err + "Migratus API does not support this action(s) : " arguments "\n\n" + (str/join (usage summary)))) -(defn run-migrate [cfg args] - (let [{:keys [options arguments errors summary]} (parse-opts args migrate-cli-options :in-order true) +(defn run-migrate! [cfg args] + (let [cmd-opts (parse-opts args migrate-cli-options :in-order true) + {:keys [options arguments errors summary]} cmd-opts rest-args (rest arguments)] (cond errors (error-msg errors) (:until-just-before options) (do (log/debug "configuration is: \n" cfg "\n" - "arguments:" rest-args) + "arguments:" rest-args) (migratus/migrate-until-just-before cfg rest-args)) (empty? args) (do (log/debug "calling (migrate cfg)" cfg) (migratus/migrate cfg)) :else (no-match-message args summary)))) -(defn run-rollback [cfg args] - (let [{:keys [options arguments errors summary]} (parse-opts args rollback-cli-options :in-order true) +(defn run-rollback! [cfg args] + (let [cmd-opts (parse-opts args rollback-cli-options :in-order true) + {:keys [options arguments errors summary]} cmd-opts rest-args (rest arguments)] (cond @@ -99,7 +241,7 @@ (:until-just-after options) (do (log/debug "configuration is: \n" cfg "\n" - "args:" rest-args) + "args:" rest-args) (migratus/rollback-until-just-after cfg rest-args)) (empty? args) @@ -150,32 +292,32 @@ "pending" applied) fmt-str "%1$-15s | %2$-22s | %3$-20s"] - (println (core/format fmt-str, id, name, applied?)))) + (prn (core/format fmt-str, id, name, applied?)))) (defn format-pending-mig-data [m] (let [{:keys [id name]} m fmt-str "%1$-15s| %2$-22s%3$s"] - (println (core/format fmt-str, id, name, )))) + (prn (core/format fmt-str, id, name)))) (defn mig-print-fmt [data & format-opts] (let [pending? (:pending format-opts)] (if pending? - (do (println (table-line 43)) - (println (core/format "%-15s%-24s", - "MIGRATION-ID" "| NAME")) - (println (table-line 41)) + (do (prn (table-line 43)) + (prn (core/format "%-15s%-24s", + "MIGRATION-ID" "| NAME")) + (prn (table-line 41)) (doseq [d data] (format-pending-mig-data d))) - (do (println (table-line 67)) - (println (core/format "%-16s%-25s%-22s", - "MIGRATION-ID" "| NAME" "| APPLIED")) - (println (table-line 67)) + (do (prn (table-line 67)) + (prn (core/format "%-16s%-25s%-22s", + "MIGRATION-ID" "| NAME" "| APPLIED")) + (prn (table-line 67)) (doseq [d data] (format-mig-data d)))))) (defn cli-print-migs! [data f & format-opts] (case f "plain" (mig-print-fmt data format-opts) - "edn" (println data) - "json" (println (json/write-str data)) + "edn" (prn data) + "json" (prn (json/write-str data)) nil)) (defn list-pending-migrations [migs format] @@ -226,12 +368,12 @@ 1 java.util.logging.Level/FINE ;; :debug java.util.logging.Level/FINEST)) ;; :trace -(defn set-logger-format +(defn set-logger-format! "Configure JUL logger to use a custom log formatter. * formatter - instance of java.util.logging.Formatter" ([verbosity] - (set-logger-format verbosity (simple-formatter format-log-record))) + (set-logger-format! verbosity (simple-formatter format-log-record))) ([verbosity ^Formatter formatter] (let [main-logger (doto (Logger/getLogger "") (.setUseParentHandlers false) @@ -244,54 +386,108 @@ (.removeHandler main-logger h)) (.addHandler main-logger handler)))) -(defn load-config! - "Returns the content of config file as a clojure map datastructure" - [^String config] - (let [config-path (.getAbsolutePath (io/file config))] - (try - (read-string (slurp config-path)) - (catch java.io.FileNotFoundException e - (binding [*out* *err*] - (println "Missing config file" (.getMessage e) - "\nYou can use --config path_to_file to specify a path to config file")))))) - -(defn up [cfg args] +(defn run-up! [cfg args] (if (empty? args) - (binding [*out* *err*] - (println "To run action up you must provide a migration-id as a parameter: - up ")) + (println-err + "To run action up you must provide a migration-id as a parameter: + up ") (->> args - (map #(parse-long %)) + (map #(my-parse-long %)) (apply migratus/up cfg)))) -(defn down [cfg args] +(defn run-down! [cfg args] (if (empty? args) - (binding [*out* *err*] - (println "To run action down you must provide a migration-id as a parameter: - down ")) + (println-err + "To run action down you must provide a migration-id as a parameter: + down ") (->> args - (map #(parse-long %)) + (map #(my-parse-long %)) (apply migratus/down cfg)))) +(defn run-status + "Display migratus status. + - display last local migration + - display database connection string with credentials REDACTED) + - display last applied migration to database" + [cfg rest-args] + (prn "Not yet implemented")) + +(defn create-migration + "Create a new migration with the current date." + [config & [name type]] + (when-not name + (throw (ex-info "Required name for migration" {}))) + (mig/create config name (or type :sql))) + +(defn run-create-migration! + "Run migratus create command" + [config arguments] + (try + (let [name (first arguments) + file (create-migration config name)] + (println "Created migration" file)) + (catch Exception e + (println-err (ex-data e))))) + +(defn- run-init! + [cfg] + (migratus/init cfg)) + +(defn do-print-usage + ([summary] + (do-print-usage summary nil)) + ([summary errors] + (println (usage summary)) + (when errors + (println-err errors)))) + +(defn- run-reset! [cfg] + (migratus/reset cfg)) + +(defn do-store-actions + [config action action-args] + ;; make store and connect + (with-open [store (doto (proto/make-store config) + (proto/connect))] + (case action + "init" (run-init! store) + "list" (run-list store action-args) + "status" (run-status store action-args) + "up" (run-up! store action-args) + "down" (run-down! store action-args) + "migrate" (run-migrate! store action-args) + "reset" (run-reset! store) + "rollback" (run-rollback! store action-args) + (throw (IllegalArgumentException. (str "Unknown action " action)))))) + (defn -main [& args] - (let [{:keys [options arguments _errors summary]} (parse-opts args global-cli-options :in-order true) - config (:config options) - verbosity (:verbosity options) - cfg (load-config! config) - action (first arguments) - rest-args (rest arguments)] - (set-logger-format verbosity) - (cond - (:help options) (usage summary) - (nil? (:config options)) (error-msg "No config provided \n --config [file-name]>") - :else (case action - "init" (migratus/init cfg) - "create" (migratus/create cfg (second arguments)) - "migrate" (run-migrate cfg rest-args) - "rollback" (run-rollback cfg rest-args) - "reset" (migratus/reset cfg) - "up" (up cfg rest-args) - "down" (down cfg rest-args) - "list" (run-list cfg rest-args) - (no-match-message arguments summary))))) + (try + (let [parsed-opts (parse-opts args global-cli-options) + {:keys [options arguments errors summary]} parsed-opts + {:keys [config-file config verbosity output-format]} options + _ (when (<= 2 verbosity) + (prn "Parsed options:" parsed-opts)) + action (first arguments) + action (when action + (str/lower-case action)) + rest-args (rest arguments) + loaded-config (load-config! config-file config) + no-store-action? (contains? #{"create"} action)] + (swap! app-config assoc + :verbosity verbosity + :output-format output-format) + ;; (prn @app-config) + (set-logger-format! verbosity) + (cond + (:help options) (do-print-usage summary) + (nil? action) (do + (do-print-usage summary) + (println "No action supplied")) + (some? errors) (do-print-usage summary errors) + ;; do not require store + no-store-action? (case action + "create" (run-create-migration! loaded-config rest-args)) + :else (do-store-actions loaded-config action rest-args))) + (catch Exception e + (println-err "Error: " (ex-message e))))) diff --git a/src/migratus/core.clj b/src/migratus/core.clj index b9e8f5e..b9a1e6f 100644 --- a/src/migratus/core.clj +++ b/src/migratus/core.clj @@ -70,7 +70,7 @@ (let [completed? (set (proto/completed-ids store))] (filter (comp completed? proto/id) (mig/list-migrations config)))) -(defn gather-migrations +(defn gather-migrations "Returns a list of all migrations from migration dir and db with enriched data: - date and time when was applied; @@ -194,7 +194,7 @@ (proto/init (proto/make-store config))) (defn create - "Create a new migration with the current date" + "Create a new migration with the current date." [config & [name type]] (mig/create config name (or type :sql))) diff --git a/src/migratus/database.clj b/src/migratus/database.clj index 5e8ccd1..114c545 100644 --- a/src/migratus/database.clj +++ b/src/migratus/database.clj @@ -290,7 +290,8 @@ proto/Store (config [this] config) (init [this] - (let [conn (connect* (assoc (:db config) :transaction? (:init-in-transaction? config)))] + (let [conn (connect* (assoc (:db config) + :transaction? (:init-in-transaction? config)))] (try (init-db! conn (utils/get-migration-dir config) @@ -305,24 +306,25 @@ (completed [this] (completed* @connection (migration-table-name config))) (migrate-up [this migration] - (log/info "Connection is " @connection - "Config is" (update config :db utils/censor-password)) - (if (proto/tx? migration :up) - (jdbc/with-transaction [t-con (connection-or-spec @connection)] - (migrate-up* t-con config migration)) - (migrate-up* (:db config) config migration))) + (log/info "Connection is " @connection + "Config is" (update config :db utils/censor-password)) + (if (proto/tx? migration :up) + (jdbc/with-transaction [t-con (connection-or-spec @connection)] + (migrate-up* t-con config migration)) + (migrate-up* (:db config) config migration))) (migrate-down [this migration] - (log/info "Connection is " @connection - "Config is" (update config :db utils/censor-password)) - (if (proto/tx? migration :down) - (jdbc/with-transaction [t-con (connection-or-spec @connection)] - (migrate-down* t-con config migration)) - (migrate-down* (:db config) config migration))) + (log/info "Connection is " @connection + "Config is" (update config :db utils/censor-password)) + (if (proto/tx? migration :down) + (jdbc/with-transaction [t-con (connection-or-spec @connection)] + (migrate-down* t-con config migration)) + (migrate-down* (:db config) config migration))) (connect [this] (reset! connection (connect* (:db config))) (init-schema! @connection (migration-table-name config) - (sql-mig/wrap-modify-sql-fn (:modify-sql-fn config)))) + (sql-mig/wrap-modify-sql-fn (:modify-sql-fn config))) + this) (disconnect [this] (disconnect* @connection config) (reset! connection nil))) diff --git a/src/migratus/migrations.clj b/src/migratus/migrations.clj index a0900ab..c98ec13 100644 --- a/src/migratus/migrations.clj +++ b/src/migratus/migrations.clj @@ -158,17 +158,25 @@ (props/load-properties config))] (make-migration config id mig)))) -(defn create [config name migration-type] +(defn create + "Create a migration file given a configuration, a name and migration type. + Resolves the absolute file name. + Returns the migration file name as string. + + Migrations are created in migration-dir. + If migration-dir does not exist, it will be created. " + [config name migration-type] (let [migration-dir (find-or-create-migration-dir (utils/get-parent-migration-dir config) (utils/get-migration-dir config)) migration-name (->kebab-case (str (timestamp) name))] (doall (for [mig-file (proto/migration-files* migration-type migration-name)] - (let [file (io/file migration-dir mig-file)] + (let [file (io/file migration-dir mig-file) + file (.getAbsoluteFile file)] (.createNewFile file) - (.getName (io/file migration-dir mig-file))))))) - + (.getName file)))))) + (defn destroy [config name] (let [migration-dir (utils/find-migration-dir (utils/get-migration-dir config)) diff --git a/src/migratus/protocols.clj b/src/migratus/protocols.clj index 0a802f3..d0d4368 100644 --- a/src/migratus/protocols.clj +++ b/src/migratus/protocols.clj @@ -36,9 +36,14 @@ (migrate-down [this migration] "Run and record a down migration") (connect [this] - "Opens resources necessary to run migrations against the store.") + "Opens resources necessary to run migrations against the store. + Returns the store on sucess so we can participate in with-open. + Throws exception on failure.") (disconnect [this] - "Frees resources necessary to run migrations against the store.")) + "Frees resources necessary to run migrations against the store.") + (close [this] + "Frees resources necessary to run migrations against the store. + Allow store to work with with-open.")) (defmulti make-store :store) diff --git a/test/migratus/mock.clj b/test/migratus/mock.clj index c731c93..532d365 100644 --- a/test/migratus/mock.clj +++ b/test/migratus/mock.clj @@ -40,7 +40,8 @@ (proto/down migration config) (swap! completed-ids disj (proto/id migration))) (connect [this]) - (disconnect [this])) + (disconnect [this]) + (close [this])) (defn make-migration [{:keys [id name ups downs]}] (MockMigration. nil id name ups downs)) diff --git a/test/migratus/test/cli_test.clj b/test/migratus/test/cli_test.clj new file mode 100644 index 0000000..9fd0802 --- /dev/null +++ b/test/migratus/test/cli_test.clj @@ -0,0 +1,39 @@ +(ns migratus.test.cli-test + (:require [clojure.test :refer :all] + [migratus.cli :as cli]) + (:import (java.util HashMap))) + +(deftest load-env-config-test + (testing "Test load config from env - load only MIGRATUS_CONFIG" + (let [env (doto (HashMap.) + (.put "MIGRATUS_CONFIG" "{:store :database :migration-dir \"my-migrations\"}") + (.put "MIGRATUS_DB_SPEC" + "{:jdbcUrl \"config-whould-be-ignored\"}")) + config (cli/env->config! env)] + (is (= {:store :database + :migration-dir "my-migrations"} + config)))) + + (testing "Test load config from env - empty when no env vars present" + (let [env (HashMap.) + config (cli/env->config! env)] + (is (= {} config)))) + + (testing "Test load migratus config from env" + (let [env (doto (HashMap.) + (.put "MIGRATUS_STORE" "database") + (.put "MIGRATUS_MIGRATION_DIR" "resources/migrations") + (.put "MIGRATUS_DB_SPEC" + "{:jdbcUrl \"jdbc:h2:local-db\"}")) + config (cli/env->config! env)] + (is (= {:store :database + :migration-dir "resources/migrations" + :db {:jdbcUrl "jdbc:h2:local-db"}} + config))))) + +(comment + + (run-test load-env-config-test) + + ) + From 34a95f9f0c4f270fecbd6bba8bbd3dd17957110a Mon Sep 17 00:00:00 2001 From: Eugen Stan Date: Fri, 3 Nov 2023 21:56:51 +0200 Subject: [PATCH 2/9] [#240] Basic config validation for cli --- src/migratus/cli.clj | 124 ++++++++++++++++++++++++++++++++++--------- 1 file changed, 98 insertions(+), 26 deletions(-) diff --git a/src/migratus/cli.clj b/src/migratus/cli.clj index 1efe235..a5cb090 100644 --- a/src/migratus/cli.clj +++ b/src/migratus/cli.clj @@ -127,26 +127,91 @@ args (cli-args->config config-data)] (deep-merge config env args))) -#_(defn valid-config? +(defn config-check-store + [store] + (if store + true + "Missing :store key in configuration")) +(defn config-check-db-spec + [db] + (if (map? db) + () + "Value for :db key should be a map")) + +(defn- validate-db-config + [db] + (cond-> [] + (nil? db) (concat ["Missing :db option for :database store"]) + ;; + (not (map? db)) + (concat ["Value of :db should be a map"]))) + +(comment + + (validate-db-config nil) + ;; => ("Missing :db option for :database store" "Value of :db should be a map") + + + (validate-db-config "") + ;; => ["Value of :db should be a map"] + + (validate-db-config {}) + ;; => [] + ) + + +(defn valid-config? "Validate a migratus configuration for required options. If valid, return true. - If invalid return map with reasons why validation failed." + If invalid return map with reasons why validation failed. + + We expect most people will use the database store so we have extra checks." [config] (if (map? config) - (let [store (:store config)] - (if store - true - {:errors ["Missing :store key in configuration"]})) + (let [valid true + store (:store config) + db (:db config) + errors (cond-> [] + ;; some store checks + (nil? store) + (concat ["Missing :store option"]) + ;; + (not (keyword? store)) + (concat ["Value of :store should be a keyword"]) + ;; + (= :database store) + (concat (validate-db-config db)))] + (if (pos? (count errors)) + {:errors errors} + valid)) {:errors ["Config is nil or not a map"]})) -#_(defn- do-invalid-config - "We got invalid configuration. +(comment + + (valid-config? nil) + ;; => {:errors ["Config is nil or not a map"]} + + (valid-config? {}) + ;; => {:errors ("Missing :store option" "Value of :store should be a keyword")} + + (valid-config? {:store :database}) + ;; => {:errors ("Missing :db option for :database store" "Value of :db should be a map")} + + (valid-config? {:store :database + :db {}}) + ;; => true + + + ) + +(defn- do-invalid-config-and-die + "We got invalid configuration. Print error and exit" - [valid? cfg] - (println-err "Invalid configuration:" (:errors valid?) - "\nMigratus can load configuration from: file, env vars, cli args." - "\nSee documentation and/or use --help") - (println-err "Configuration is" cfg)) + [valid? cfg] + (println-err "Invalid configuration:" (:errors valid?) + "\nMigratus can load configuration from: file, env vars, cli args." + "\nSee documentation and/or use --help") + (println-err "Configuration is" cfg)) (defn validate-format [s] (boolean (some (set (list s)) #{"plain" "edn" "json"}))) @@ -447,18 +512,22 @@ (defn do-store-actions [config action action-args] ;; make store and connect - (with-open [store (doto (proto/make-store config) - (proto/connect))] - (case action - "init" (run-init! store) - "list" (run-list store action-args) - "status" (run-status store action-args) - "up" (run-up! store action-args) - "down" (run-down! store action-args) - "migrate" (run-migrate! store action-args) - "reset" (run-reset! store) - "rollback" (run-rollback! store action-args) - (throw (IllegalArgumentException. (str "Unknown action " action)))))) + (let [valid? (valid-config? config)] + (if (map? valid?) + (do-invalid-config-and-die valid? config) + ;; normall processing + (with-open [store (doto (proto/make-store config) + (proto/connect))] + (case action + "init" (run-init! store) + "list" (run-list store action-args) + "status" (run-status store action-args) + "up" (run-up! store action-args) + "down" (run-down! store action-args) + "migrate" (run-migrate! store action-args) + "reset" (run-reset! store) + "rollback" (run-rollback! store action-args) + (throw (IllegalArgumentException. (str "Unknown action " action)))))))) (defn -main [& args] (try @@ -479,12 +548,15 @@ ;; (prn @app-config) (set-logger-format! verbosity) (cond + ;; display help if user asks (:help options) (do-print-usage summary) + ;; if no action is supplied, throw error (nil? action) (do (do-print-usage summary) (println "No action supplied")) + ;; in case of errors during args processing - show usage (some? errors) (do-print-usage summary errors) - ;; do not require store + ;; actions that do not require store no-store-action? (case action "create" (run-create-migration! loaded-config rest-args)) :else (do-store-actions loaded-config action rest-args))) From 3cba8ea8dbe101586bc7eeaa30cff212772e2db2 Mon Sep 17 00:00:00 2001 From: Eugen Stan Date: Sat, 4 Nov 2023 15:55:52 +0200 Subject: [PATCH 3/9] [#240] find-migration-dir works with absolute path + tests --- src/migratus/cli.clj | 57 ++++++++++++++-------- src/migratus/database.clj | 12 +++-- src/migratus/migrations.clj | 95 +++++++++++++++++++++++++++++------- src/migratus/properties.clj | 11 +++-- src/migratus/utils.clj | 18 ++++--- test/migratus/test/utils.clj | 43 +++++++++++++++- 6 files changed, 182 insertions(+), 54 deletions(-) diff --git a/src/migratus/cli.clj b/src/migratus/cli.clj index a5cb090..acf8e69 100644 --- a/src/migratus/cli.clj +++ b/src/migratus/cli.clj @@ -1,4 +1,5 @@ (ns migratus.cli + {:authors ["Eugen Stan config config-data)] (deep-merge config env args))) -(defn config-check-store - [store] - (if store - true - "Missing :store key in configuration")) -(defn config-check-db-spec - [db] - (if (map? db) - () - "Value for :db key should be a map")) - (defn- validate-db-config [db] (cond-> [] @@ -322,12 +312,16 @@ local-datetime (.atZone instant zone-id)] local-datetime))) +(defn- applied-date->str + [applied] + (when (some? applied) + (-> + (util-date-to-local-datetime applied) + (.format DateTimeFormatter/ISO_LOCAL_DATE_TIME)))) + (defn parse-migration-applied-date [m] (let [{:keys [id name applied]} m - local-date (when (some? applied) - (-> - (util-date-to-local-datetime applied) - (.format DateTimeFormatter/ISO_LOCAL_DATE_TIME)))] + local-date (applied-date->str applied)] {:id id :name name :applied local-date})) (defn parsed-migrations-data [cfg] @@ -347,9 +341,9 @@ [n] (apply str (repeat n "-"))) -(defn table-line [n] - (let [str (str "%-" n "s")] - (core/format str, (col-width n)))) +(defn table-line + [n] + (core/format (str "%-" n "s"), (col-width n))) (defn format-mig-data [m] (let [{:keys [id name applied]} m @@ -469,14 +463,39 @@ (map #(my-parse-long %)) (apply migratus/down cfg)))) +(defn reverse-sort-migrations + "Reverse sort Migration sequences by id" + ([migs] + (sort-by :id #(compare %2 %1) migs))) (defn run-status "Display migratus status. - display last local migration - display database connection string with credentials REDACTED) - display last applied migration to database" - [cfg rest-args] + [store rest-args] + (let [cfg (proto/config store) + migs (mig/list-migrations cfg) + migs (reverse-sort-migrations migs)]) (prn "Not yet implemented")) +(comment + + (def migs (mig/list-migrations {:migration-dir "test/migrations"})) + + (-> (first migs) + :id) + (sort-by :id #(compare %2 %1) migs) + ;; => (#migratus.migration.sql.SqlMigration{:id 20120827170200, :name "multiple-statements", :up "-- this is the first statement\n\nCREATE TABLE\nquux\n(id bigint,\n name varchar(255));\n\n--;;\n-- comment for the second statement\n\nCREATE TABLE quux2(id bigint, name varchar(255));\n", :down "DROP TABLE quux2;\n--;;\nDROP TABLE quux;\n"} #migratus.migration.sql.SqlMigration{:id 20111202110600, :name "create-foo-table", :up "CREATE TABLE IF NOT EXISTS foo(id bigint);\n", :down "DROP TABLE IF EXISTS foo;\n"} #migratus.migration.sql.SqlMigration{:id 20111202113000, :name "create-bar-table", :up "CREATE TABLE IF NOT EXISTS bar(id BIGINT);\n", :down "DROP TABLE IF EXISTS bar;\n"}) + + (def store (doto (proto/make-store config))) + + (proto/init store) + + (proto/completed-ids store) + + ) + + (defn create-migration "Create a new migration with the current date." [config & [name type]] diff --git a/src/migratus/database.clj b/src/migratus/database.clj index 114c545..a0fc760 100644 --- a/src/migratus/database.clj +++ b/src/migratus/database.clj @@ -245,7 +245,8 @@ (str "ALTER TABLE " table-name " ADD COLUMN applied timestamp")])]))) -(defn init-schema! [db table-name modify-sql-fn] +(defn init-schema! + [db table-name modify-sql-fn] ;; Note: the table-exists? *has* to be done in its own top-level ;; transaction. It can't be run in the same transaction as other code, because ;; if the table doesn't exist, then the error it raises internally in @@ -258,7 +259,8 @@ (or (migration-table-up-to-date? db table-name) (update-migration-table! db modify-sql-fn table-name))) -(defn run-init-script! [init-script-name init-script conn modify-sql-fn transaction?] +(defn run-init-script! + [init-script-name init-script conn modify-sql-fn transaction?] (try (log/info "running initialization script '" init-script-name "'") (log/trace "\n" init-script "\n") @@ -271,12 +273,14 @@ (log/error t "failed to initialize the database with:\n" init-script "\n") (throw t)))) -(defn inject-properties [init-script properties] +(defn inject-properties + [init-script properties] (if properties (props/inject-properties properties init-script) init-script)) -(defn init-db! [db migration-dir init-script-name modify-sql-fn transaction? properties] +(defn init-db! + [db migration-dir init-script-name modify-sql-fn transaction? properties] (if-let [init-script (some-> (find-init-script migration-dir init-script-name) slurp (inject-properties properties))] diff --git a/src/migratus/migrations.clj b/src/migratus/migrations.clj index c98ec13..17c8ba3 100644 --- a/src/migratus/migrations.clj +++ b/src/migratus/migrations.clj @@ -1,4 +1,5 @@ (ns migratus.migrations + "Namespace to handle migrations stored on filesystem, as files." (:require [clojure.java.io :as io] [clojure.string :as str] @@ -15,7 +16,14 @@ java.text.SimpleDateFormat java.util.regex.Pattern)) -(defn ->kebab-case [s] +(defn ->kebab-case + "Convert a string to kebab case. + + - convert CamelCase to camel-case + - replace multiple white spaces with a single dash + - replace underscores with dash + - converts to lower case" + [s] (-> (reduce (fn [s c] (if (and @@ -29,12 +37,22 @@ (.replaceAll "_" "-") (.toLowerCase))) -(defn- timestamp [] +(comment + + (->kebab-case "hello javaMigrations2") + ;; => "hello-java-migrations2" + ) + +(defn- timestamp + "Return the current date and time as a string timestamp at UTC." + [] (let [fmt (doto (SimpleDateFormat. "yyyyMMddHHmmss ") (.setTimeZone (TimeZone/getTimeZone "UTC")))] (.format fmt (Date.)))) -(defn parse-migration-id [id] +(defn parse-migration-id + "Parse migration id as a java.lang.Long." + [id] (try (Long/parseLong id) (catch Exception e @@ -74,19 +92,49 @@ (props/inject-properties properties content) content))) -(defn find-migration-files [migration-dir exclude-scripts properties] +(defn find-migration-files + "Looks for all migration files in 'migration-dir' path. + Excludes from results the migrations that match globs in 'exclude-scripts' + + Parses the file names according to migratus rules. + Returns a sequence of maps. + Each map represents a single migration file. + + A migration map has a single key - migration id as string. + The value is a map with migration name as key and + a another map as value representing the migration. + + Example of structure: + { + { + {:sql {:up } }}} + " + ;; ieugen: I wonder why we have to realize the migrations in memory + ;; The store could be enhanced to support a fetch-migration call to fetch the contents. + ;; Most of the time we are dealing with metdata. + ;; We need the migration body only when we apply it. + [migration-dir exclude-scripts properties] (log/debug "Looking for migrations in" migration-dir) - (->> (for [f (filter (fn [^File f] (.isFile f)) - (file-seq migration-dir)) - :let [file-name (.getName ^File f)] - :when (not (utils/script-excluded? file-name migration-dir exclude-scripts))] - (if-let [mig (parse-name file-name)] - (migration-map mig (slurp f) properties) - (warn-on-invalid-migration file-name))) - (remove nil?))) - - -(defn find-migration-resources [dir jar exclude-scripts properties] + (let [migration-dir (io/as-file migration-dir)] + (->> (for [f (filter (fn [^File f] (.isFile f)) + (file-seq migration-dir)) + :let [file-name (.getName ^File f)] + :when (not (utils/script-excluded? file-name migration-dir exclude-scripts))] + (if-let [mig (parse-name file-name)] + (migration-map mig (slurp f) properties) + (warn-on-invalid-migration file-name))) + (remove nil?)))) + +(comment + + (take 2 (find-migration-files "test/migrations" nil nil)) + + ) + +(defn find-migration-resources + "Looks for migration files in classpath and java jar archives. + Returns a sequence of migrations similar to find-migration-files fn." + [dir jar exclude-scripts properties] (log/debug "Looking for migrations in" dir jar) (->> (for [entry (enumeration-seq (.entries ^JarFile jar)) :when (.matches (.getName ^JarEntry entry) @@ -103,7 +151,11 @@ (warn-on-invalid-migration file-name))) (remove nil?))) -(defn read-migrations [dir exclude-scripts properties] +(defn read-migrations + "Looks for migrations files accessible and return a sequence. + Reads the migration contents as string in memory. + See find-migration-files for a descriptin of the format." + [dir exclude-scripts properties] (when-let [migration-dir (utils/find-migration-dir dir)] (if (instance? File migration-dir) (find-migration-files migration-dir exclude-scripts properties) @@ -121,6 +173,8 @@ (into {} (map fm) dirs))) (defn find-or-create-migration-dir + "Checks the migration directory exists. + Creates it and the parent directories if it does not exist." ([dir] (find-or-create-migration-dir utils/default-migration-parent dir)) ([parent-dir dir] (if-let [migration-dir (utils/find-migration-dir dir)] @@ -151,7 +205,9 @@ id (pr-str (keys mig)))))) (throw (Exception. (str "Invalid migration id: " id))))) -(defn list-migrations [config] +(defn list-migrations + "Find all migrations and return a sequence of Migration instances" + [config] (doall (for [[id mig] (find-migrations (utils/get-migration-dir config) (utils/get-exclude-scripts config) @@ -177,7 +233,10 @@ (.createNewFile file) (.getName file)))))) -(defn destroy [config name] +(defn destroy + "Delete both files associated with a migration (up and down). + Migration is identified by name." + [config name] (let [migration-dir (utils/find-migration-dir (utils/get-migration-dir config)) migration-name (->kebab-case name) diff --git a/src/migratus/properties.clj b/src/migratus/properties.clj index a9766ba..cbd5a06 100644 --- a/src/migratus/properties.clj +++ b/src/migratus/properties.clj @@ -15,12 +15,13 @@ {} (System/getenv))) -(defn inject-properties [properties text] +(defn inject-properties + [properties text] (let [text-with-props (reduce - (fn [text [k v]] - (.replace text k (str v))) - text - properties)] + (fn [text [k v]] + (.replace text k (str v))) + text + properties)] (doseq [x (re-seq #"\$\{[a-zA-Z0-9\-_\.]+}" text-with-props)] (log/warn "no property found for key:" x)) text-with-props)) diff --git a/src/migratus/utils.clj b/src/migratus/utils.clj index b15fe10..19812d3 100644 --- a/src/migratus/utils.clj +++ b/src/migratus/utils.clj @@ -86,12 +86,18 @@ (if (= "jar" (.getProtocol url)) (jar-file url) (File. (URLDecoder/decode (.getFile url) "UTF-8"))) - (let [migration-dir (io/file parent-dir dir)] - (if (.exists migration-dir) - migration-dir - (let [no-implicit-parent-dir (io/file dir)] - (when (.exists no-implicit-parent-dir) - no-implicit-parent-dir))))))) + ;; we don't have URL resource, try file + (let [fdir (io/file dir)] + (if (.exists fdir) + fdir + (if (.isAbsolute fdir) + ;; if path is absolute and does no exist, throw error + (throw (IllegalStateException. + (str "Could not find migrations dir " dir))) + ;; if relative path, try with parent dir logic + (let [migration-dir (io/file parent-dir dir)] + (when (.exists migration-dir) + migration-dir)))))))) (defn deep-merge "Merge keys at all nested levels of the maps." diff --git a/test/migratus/test/utils.clj b/test/migratus/test/utils.clj index 634d33e..1534e7d 100644 --- a/test/migratus/test/utils.clj +++ b/test/migratus/test/utils.clj @@ -1,6 +1,7 @@ (ns migratus.test.utils (:require [clojure.test :refer :all] - [migratus.utils :refer :all])) + [migratus.utils :refer :all] + [clojure.java.io :as io])) (deftest test-censor-password (is (= nil (censor-password nil))) @@ -10,7 +11,7 @@ (censor-password {:password "1234" :user "user"}))) (is (= "uri-censored" (censor-password - "jdbc:postgresql://fake.example.org/my_dev?user=my_user&password=thisIsNot123ARealPass"))) + "jdbc:postgresql://fake.example.org/my_dev?user=my_user&password=thisIsNot123ARealPass"))) (is (= {:connection-uri "uri-censored"} (censor-password {:connection-uri "jdbc:postgresql://fake.example.org/my_dev?user=my_user&password=thisIsNot123ARealPass"}))) (is (= {:connection-uri "uri-censored" :password "1" :user "user"} @@ -25,3 +26,41 @@ (testing "handles '+' in paths" (is (= "/tmp/default+uberjar/foo.jar" (jar-name "/tmp/default+uberjar/foo.jar"))))) + + +(deftest testfind-migration-dir + + (testing "returns nil for path that does not exist" + (let [dir (find-migration-dir "migration-dir-does-not-exist")] + (is (nil? dir)))) + + (testing "finds migration dir with relative path" + (let [dir (find-migration-dir "migrations") + expected (-> (io/as-file "test/migrations") + (.getAbsoluteFile) + (.toString))] + (println "Migration dir" dir) + (is (some? dir)) + (is (= (.toString dir) expected)))) + + (testing "finds migration dir with absolute path and missing dir throws" + (is (thrown? + IllegalStateException + (find-migration-dir "/non-existing-absolute-path"))))) + +(testing "finds migration dir with absolute path and missing dir" + (let [dir (find-migration-dir "test/migrations") + expected (-> (io/as-file "test/migrations") + (.getAbsoluteFile) + (.toString))] + (println "Migration dir" dir) + (is (some? dir)) + (is (= (.toString dir) expected)))) + + +(comment + + (run-test testfind-migration-dir) + + + ) From 3c2e4aecfd48b2f739c974940651dfff0fd1504c Mon Sep 17 00:00:00 2001 From: Eugen Stan Date: Sun, 5 Nov 2023 02:06:41 +0200 Subject: [PATCH 4/9] Censor jdbcUrl for next-jdbc --- src/migratus/utils.clj | 9 ++++++--- test/migratus/test/utils.clj | 5 ++++- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/migratus/utils.clj b/src/migratus/utils.clj index 19812d3..f118ce7 100644 --- a/src/migratus/utils.clj +++ b/src/migratus/utils.clj @@ -126,7 +126,7 @@ "uri-censored")) (defmethod censor-password :default - [{:keys [password connection-uri] :as db-spec}] + [{:keys [password connection-uri jdbcUrl] :as db-spec}] (let [password-map (if (empty? password) nil @@ -138,5 +138,8 @@ (if (empty? connection-uri) nil ;; Censor entire uri instead of trying to parse out and replace only a possible password parameter - {:connection-uri "uri-censored"})] - (merge db-spec password-map uri-map))) + {:connection-uri "uri-censored"}) + jdbcUrl-map (if (empty? jdbcUrl) + nil + {:jdbcUrl "uri-censored"})] + (merge db-spec password-map uri-map jdbcUrl-map))) diff --git a/test/migratus/test/utils.clj b/test/migratus/test/utils.clj index 1534e7d..f9f653a 100644 --- a/test/migratus/test/utils.clj +++ b/test/migratus/test/utils.clj @@ -16,7 +16,10 @@ (censor-password {:connection-uri "jdbc:postgresql://fake.example.org/my_dev?user=my_user&password=thisIsNot123ARealPass"}))) (is (= {:connection-uri "uri-censored" :password "1" :user "user"} (censor-password {:password "1234" :user "user" - :connection-uri "jdbc:postgresql://fake.example.org/my_dev?user=my_user&password=thisIsNot123ARealPass"})))) + :connection-uri "jdbc:postgresql://fake.example.org/my_dev?user=my_user&password=thisIsNot123ARealPass"}))) + (is (= {:jdbcUrl "uri-censored" :password "1" :user "user"} + (censor-password {:password "1234" :user "user" + :jdbcUrl "jdbc:postgresql://fake.example.org/my_dev?user=my_user&password=thisIsNot123ARealPass"})))) (deftest test-jar-name (is (nil? (jar-name nil))) From 4ec5711ad3d1210e6558f06f53f4c375df3a3f07 Mon Sep 17 00:00:00 2001 From: Eugen Stan Date: Mon, 6 Nov 2023 14:33:21 +0200 Subject: [PATCH 5/9] [#240] migratus status - alpha * Implemnted status command that works mostly like list --- deps.edn | 4 +++ examples/postgres/deps.edn | 2 +- src/migratus/cli.clj | 70 +++++++++++++++++++++++++++++--------- 3 files changed, 58 insertions(+), 18 deletions(-) diff --git a/deps.edn b/deps.edn index 60bce74..43246b0 100644 --- a/deps.edn +++ b/deps.edn @@ -21,6 +21,10 @@ hikari-cp/hikari-cp {:mvn/version "2.13.0"} org.clojure/tools.trace {:mvn/version "0.7.11"} org.postgresql/postgresql {:mvn/version "42.2.5"}}} + :migratus {:jvm-opts [;; print clojure errors to standard out instead of file + "-Dclojure.main.report=stderr"] + :extra-deps {org.postgresql/postgresql {:mvn/version "42.2.5"}} + :main-opts ["-m" "migratus.cli"]} :test-runner {:extra-paths ["test"] :extra-deps {lambdaisland/kaocha {:mvn/version "1.66.1034"} diff --git a/examples/postgres/deps.edn b/examples/postgres/deps.edn index 037908b..9dad124 100644 --- a/examples/postgres/deps.edn +++ b/examples/postgres/deps.edn @@ -4,7 +4,7 @@ :deps {io.github.yogthos/migratus {:local/root "../.."} org.postgresql/postgresql {:mvn/version "42.6.0"}} :aliases - {:migratus {:jvm-opts [;; print clojure errors to standard out instead of cli + {:migratus {:jvm-opts [;; print clojure errors to standard out instead of fli "-Dclojure.main.report=stderr"] ;; Run migratus.cli -main fn :main-opts ["-m" "migratus.cli"]}}} diff --git a/src/migratus/cli.clj b/src/migratus/cli.clj index acf8e69..22400c0 100644 --- a/src/migratus/cli.clj +++ b/src/migratus/cli.clj @@ -221,6 +221,11 @@ [nil "--config CONFIG" "Configuration as edn"] ["-h" "--help"]]) +(def status-cli-options + [[nil "--show-config" + "Display full configuration - may include secrets and credentials"] + ["-h" "--help"]]) + (def migrate-cli-options [[nil "--until-just-before MIGRATION-ID" "Run all migrations preceding migration-id. @@ -312,12 +317,14 @@ local-datetime (.atZone instant zone-id)] local-datetime))) +(def date-time-fmt + (DateTimeFormatter/ofPattern "uuuu-MM-dd HH:mm:ss")) (defn- applied-date->str [applied] (when (some? applied) (-> (util-date-to-local-datetime applied) - (.format DateTimeFormatter/ISO_LOCAL_DATE_TIME)))) + (.format date-time-fmt)))) (defn parse-migration-applied-date [m] (let [{:keys [id name applied]} m @@ -463,37 +470,66 @@ (map #(my-parse-long %)) (apply migratus/down cfg)))) -(defn reverse-sort-migrations - "Reverse sort Migration sequences by id" - ([migs] - (sort-by :id #(compare %2 %1) migs))) +(defn format-status-line + "Format transaction status line" + [id local applied] + (let [missing-applied? (nil? applied) + missing-local? (nil? local) + name (get local :name) + applied-date (get applied :applied) + date-str (or (applied-date->str applied-date) "") + description (get applied :description) + name-or-desc (or name description) + notes (cond-> [] + missing-applied? (conj :migration-not-applied) + missing-local? (conj :missing-local-migration) + (and (not missing-applied?) + (nil? description)) (conj :missing-db-description) + (and (not missing-applied?) + (nil? applied-date)) (conj :missing-applied-date))] + (format "%d %20s %-30s %s" id date-str name-or-desc notes))) + (defn run-status "Display migratus status. - display last local migration - display database connection string with credentials REDACTED) - display last applied migration to database" [store rest-args] - (let [cfg (proto/config store) - migs (mig/list-migrations cfg) - migs (reverse-sort-migrations migs)]) - (prn "Not yet implemented")) + (let [{:keys [options arguments errors summary]} + (parse-opts rest-args status-cli-options) + {:keys [show-config]} options + cfg (proto/config store)] + ;; TODO: do cli validation + (log/debug "TODO: Implement validatin for status command!") + (when show-config + (println "Migratus configuration is" cfg \newline)) + (let [migrations (mig/list-migrations cfg) + completed (proto/completed store) + migratons-by-id (group-by :id migrations) + complete-by-id (group-by :id completed) + all-ids-once (into #{} (concat (keys migratons-by-id) + (keys complete-by-id))) + ordered-ids (sort all-ids-once)] + (doseq [id ordered-ids] + (let [local (first (get migratons-by-id id)) + applied (first (get complete-by-id id))] + (println (format-status-line id local applied))))))) (comment - (def migs (mig/list-migrations {:migration-dir "test/migrations"})) + (def migs (mig/list-migrations config)) (-> (first migs) :id) - (sort-by :id #(compare %2 %1) migs) - ;; => (#migratus.migration.sql.SqlMigration{:id 20120827170200, :name "multiple-statements", :up "-- this is the first statement\n\nCREATE TABLE\nquux\n(id bigint,\n name varchar(255));\n\n--;;\n-- comment for the second statement\n\nCREATE TABLE quux2(id bigint, name varchar(255));\n", :down "DROP TABLE quux2;\n--;;\nDROP TABLE quux;\n"} #migratus.migration.sql.SqlMigration{:id 20111202110600, :name "create-foo-table", :up "CREATE TABLE IF NOT EXISTS foo(id bigint);\n", :down "DROP TABLE IF EXISTS foo;\n"} #migratus.migration.sql.SqlMigration{:id 20111202113000, :name "create-bar-table", :up "CREATE TABLE IF NOT EXISTS bar(id BIGINT);\n", :down "DROP TABLE IF EXISTS bar;\n"}) - (def store (doto (proto/make-store config))) + (group-by :id (sort-by :id #(compare %2 %1) (take 10 migs))) - (proto/init store) + (def store (proto/make-store config)) - (proto/completed-ids store) + (proto/init store) + (proto/connect store) - ) + (group-by :id (proto/completed store))) (defn create-migration @@ -550,7 +586,7 @@ (defn -main [& args] (try - (let [parsed-opts (parse-opts args global-cli-options) + (let [parsed-opts (parse-opts args global-cli-options :in-order true) {:keys [options arguments errors summary]} parsed-opts {:keys [config-file config verbosity output-format]} options _ (when (<= 2 verbosity) From 956a65a0efe9d75fc562478ecccfd2f85b457cfd Mon Sep 17 00:00:00 2001 From: Eugen Stan Date: Mon, 6 Nov 2023 15:00:06 +0200 Subject: [PATCH 6/9] [#240] Restored list command functionality --- src/migratus/cli.clj | 66 ++++++++++++++++++++++++++++--------------- src/migratus/core.clj | 3 +- 2 files changed, 46 insertions(+), 23 deletions(-) diff --git a/src/migratus/cli.clj b/src/migratus/cli.clj index 22400c0..c8e40e9 100644 --- a/src/migratus/cli.clj +++ b/src/migratus/cli.clj @@ -29,6 +29,11 @@ [s] (Boolean/parseBoolean s)) +(defn validate-format + [s] + (boolean (some (set (list s)) + #{"plain" "edn" "json"}))) + (def app-config "Application options to be used for output/logging. To avoid passing them around everywhere." @@ -243,6 +248,11 @@ [[nil "--available" "List all migrations, applied and non applied"] [nil "--pending" "List pending migrations"] [nil "--applied" "List applied migrations"] + [nil "--format FORMAT" + "Option to print in plain text (default), edn or json" + :default "plain" + :validate [#(validate-format %) + "Unsupported format. Valid options: plain (default), edn, json."]] ["-h" "--help"]]) (defn usage [options-summary] @@ -331,17 +341,21 @@ local-date (applied-date->str applied)] {:id id :name name :applied local-date})) -(defn parsed-migrations-data [cfg] - (let [all-migrations (migratus/all-migrations cfg)] +(defn parsed-migrations-data + [store] + (let [config (proto/config store) + all-migrations (migratus/all-migrations config)] (map parse-migration-applied-date all-migrations))) -(defn pending-migrations [cfg] +(defn pending-migrations + [store] (let [keep-pending-migs (fn [mig] (nil? (:applied mig)))] - (filter keep-pending-migs (parsed-migrations-data cfg)))) + (filter keep-pending-migs (parsed-migrations-data store)))) -(defn applied-migrations [cfg] +(defn applied-migrations + [store] (let [keep-applied-migs (fn [mig] (not= nil (:applied mig)))] - (filter keep-applied-migs (parsed-migrations-data cfg)))) + (filter keep-applied-migs (parsed-migrations-data store)))) (defn col-width "Set column width for CLI table" @@ -358,25 +372,25 @@ "pending" applied) fmt-str "%1$-15s | %2$-22s | %3$-20s"] - (prn (core/format fmt-str, id, name, applied?)))) + (println (core/format fmt-str, id, name, applied?)))) (defn format-pending-mig-data [m] (let [{:keys [id name]} m fmt-str "%1$-15s| %2$-22s%3$s"] - (prn (core/format fmt-str, id, name)))) + (println (core/format fmt-str, id, name)))) (defn mig-print-fmt [data & format-opts] (let [pending? (:pending format-opts)] (if pending? - (do (prn (table-line 43)) - (prn (core/format "%-15s%-24s", + (do (println (table-line 43)) + (println (core/format "%-15s%-24s", "MIGRATION-ID" "| NAME")) - (prn (table-line 41)) + (println (table-line 41)) (doseq [d data] (format-pending-mig-data d))) - (do (prn (table-line 67)) - (prn (core/format "%-16s%-25s%-22s", + (do (println (table-line 67)) + (println (core/format "%-16s%-25s%-22s", "MIGRATION-ID" "| NAME" "| APPLIED")) - (prn (table-line 67)) + (println (table-line 67)) (doseq [d data] (format-mig-data d)))))) (defn cli-print-migs! [data f & format-opts] @@ -389,19 +403,20 @@ (defn list-pending-migrations [migs format] (cli-print-migs! migs format {:pending true})) -(defn run-list [cfg args] - (let [{:keys [options errors summary]} (parse-opts args list-cli-options :in-order true) +(defn run-list [store args] + (let [{:keys [options errors summary]} + (parse-opts args list-cli-options) {:keys [available pending applied]} options - {f :format} options] + f (get options :format)] (cond errors (error-msg errors) - applied (let [applied-migs (applied-migrations cfg)] + applied (let [applied-migs (applied-migrations store)] (cli-print-migs! applied-migs f)) - pending (let [pending-migs (pending-migrations cfg)] + pending (let [pending-migs (pending-migrations store)] (list-pending-migrations pending-migs f)) - available (let [available-migs (parsed-migrations-data cfg)] + available (let [available-migs (parsed-migrations-data store)] (cli-print-migs! available-migs f)) - (or (empty? args) f) (let [pending-migs (pending-migrations cfg)] + (or (empty? args) f) (let [pending-migs (pending-migrations store)] (list-pending-migrations pending-migs f)) :else (no-match-message args summary)))) @@ -517,6 +532,11 @@ (comment + (def config {:store :database + :migration-dir "test/migrations" + :db {:jdbcUrl + "jdbc:postgresql://localhost:5432/migratus_example_db?user=postgres&password=migrate-me"}}) + (def migs (mig/list-migrations config)) (-> (first migs) @@ -528,8 +548,10 @@ (proto/init store) (proto/connect store) + (applied-migrations store) - (group-by :id (proto/completed store))) + (group-by :id (proto/completed store)) + ) (defn create-migration diff --git a/src/migratus/core.clj b/src/migratus/core.clj index b9a1e6f..0eb27c1 100644 --- a/src/migratus/core.clj +++ b/src/migratus/core.clj @@ -83,7 +83,8 @@ unify-mig-values (fn [[_ v]] (apply merge v))] (map unify-mig-values grouped-migrations-by-id))) -(defn all-migrations [config] +(defn all-migrations + [config] (with-store [store (proto/make-store config)] (->> store From f8c1c05a0a059eb3376d777a918865591ae32f3d Mon Sep 17 00:00:00 2001 From: Eugen Stan Date: Mon, 6 Nov 2023 15:42:10 +0200 Subject: [PATCH 7/9] Process MIGRATUS_CONFIG and the other env vars, merge them --- src/migratus/cli.clj | 32 +++++++++++++++----------------- test/migratus/test/cli_test.clj | 5 +++-- 2 files changed, 18 insertions(+), 19 deletions(-) diff --git a/src/migratus/cli.clj b/src/migratus/cli.clj index c8e40e9..dc6e81f 100644 --- a/src/migratus/cli.clj +++ b/src/migratus/cli.clj @@ -60,7 +60,6 @@ Looks and processes the following env vars: - MIGRATUS_CONFIG - read string as edn and return config. - Do not process any other migratus env var. - MIGRATUS_STORE - apply clojure.core/keyword fn to value - MIGRATUS_MIGRATION_DIR - expect string, use as is @@ -72,22 +71,21 @@ ([] (env->config! (System/getenv))) ([env] - (let [config (get env "MIGRATUS_CONFIG")] - (if config - (edn/read-string config) - ;; we don't have MIGRATUS_CONFIG - check the other vars - (let [store (get env "MIGRATUS_STORE") - migration-dir (get env "MIGRATUS_MIGRATION_DIR") - table (get env "MIGRATUS_TABLE_NAME") - init-in-transaction (get env "MIGRATUS_INIT_IN_TRANSACTION") - db (get env "MIGRATUS_DB_SPEC")] - (cond-> {} - store (assoc :store (keyword store)) - migration-dir (assoc :migration-dir migration-dir) - table (assoc :migration-table-name table) - init-in-transaction (assoc :init-in-transaction - (my-parse-boolean init-in-transaction)) - db (assoc :db (edn/read-string db)))))))) + (let [config (get env "MIGRATUS_CONFIG") + ;; parse config from env + config (if config (edn/read-string config) {}) + store (get env "MIGRATUS_STORE") + migration-dir (get env "MIGRATUS_MIGRATION_DIR") + table (get env "MIGRATUS_TABLE_NAME") + init-in-transaction (get env "MIGRATUS_INIT_IN_TRANSACTION") + db (get env "MIGRATUS_DB_SPEC")] + (cond-> config + store (assoc :store (keyword store)) + migration-dir (assoc :migration-dir migration-dir) + table (assoc :migration-table-name table) + init-in-transaction (assoc :init-in-transaction + (my-parse-boolean init-in-transaction)) + db (assoc :db (edn/read-string db)))))) (defn cli-args->config "Parse any migratus configuration options from cli args. diff --git a/test/migratus/test/cli_test.clj b/test/migratus/test/cli_test.clj index 9fd0802..b840382 100644 --- a/test/migratus/test/cli_test.clj +++ b/test/migratus/test/cli_test.clj @@ -8,10 +8,11 @@ (let [env (doto (HashMap.) (.put "MIGRATUS_CONFIG" "{:store :database :migration-dir \"my-migrations\"}") (.put "MIGRATUS_DB_SPEC" - "{:jdbcUrl \"config-whould-be-ignored\"}")) + "{:jdbcUrl \"config-could-be-ignored\"}")) config (cli/env->config! env)] (is (= {:store :database - :migration-dir "my-migrations"} + :migration-dir "my-migrations" + :db {:jdbcUrl "config-could-be-ignored"}} config)))) (testing "Test load config from env - empty when no env vars present" From 298e9279739e6f783308d067ff6ede5b9a51805f Mon Sep 17 00:00:00 2001 From: Eugen Stan Date: Mon, 6 Nov 2023 15:52:33 +0200 Subject: [PATCH 8/9] [#240] Added example on how to call migrations with bb --- examples/postgres/README.md | 27 ++++++++++++++++++++++++++- examples/postgres/bb.edn | 8 ++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) create mode 100644 examples/postgres/bb.edn diff --git a/examples/postgres/README.md b/examples/postgres/README.md index 0507943..298cb06 100644 --- a/examples/postgres/README.md +++ b/examples/postgres/README.md @@ -64,6 +64,28 @@ Since Migratus is a clojure library, we need to run it via clojure like this `cl There is also a bash script `migratus.sh` that does the same: `./migratus.sh --help` +#### Calling migratus via babashka task + +We can use [babashka](https://babashka.org/) tasks to drive migratus. +This is usefull for creating developer and ops tooling. + +Developers can easily create migrations from cli. +Operations people can query the migration status on the server. + +Bellow is a sample `bb.edn` file that calls the migratus alias in `deps.edn`, that we created earlier. + +bb.edn +```clojure +;; example on how to call migrations via babashka +{:tasks {:requires ([clojure.string :as str]) + migrate + {:doc "Run migration tasks. Specify MIGRATUS_DB_SPEC to choose db." + :task (clojure {:dir "." + :extra-env {"MIGRATUS_STORE" "database" + "MIGRATUS_MIGRATION_DIR" "resources/migrations"}} + (str/join " " (list* "-M:migratus" *command-line-args*)))}}} + +``` Commands with migratus @@ -72,6 +94,9 @@ Commands with migratus # We export the configuration as env variable, but we can use cli or file as well export MIGRATUS_CONFIG='{:store :database :db {:jdbcUrl "jdbc:postgresql://localhost:5432/migratus_example_db?user=postgres&password=migrate-me"}}' + clojure -M:migratus status +# via bb +bb migratus status -``` \ No newline at end of file +``` diff --git a/examples/postgres/bb.edn b/examples/postgres/bb.edn new file mode 100644 index 0000000..f97ced4 --- /dev/null +++ b/examples/postgres/bb.edn @@ -0,0 +1,8 @@ +;; example on how to call migrations via babashka +{:tasks {:requires ([clojure.string :as str]) + migrate + {:doc "Run migration tasks. Specify MIGRATUS_DB_SPEC to choose db." + :task (clojure {:dir "." + :extra-env {"MIGRATUS_STORE" "database" + "MIGRATUS_MIGRATION_DIR" "resources/migrations"}} + (str/join " " (list* "-M:migratus" *command-line-args*)))}}} From 45230466a9142cc847ec1eb9b192bfb47f2f1ad3 Mon Sep 17 00:00:00 2001 From: Eugen Stan Date: Mon, 13 Nov 2023 23:37:42 +0200 Subject: [PATCH 9/9] [#240] Fix broken cli cals - pass config instead of store --- examples/postgres/README.md | 1 + src/migratus/cli.clj | 23 ++++++++++++----------- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/examples/postgres/README.md b/examples/postgres/README.md index 298cb06..0891f49 100644 --- a/examples/postgres/README.md +++ b/examples/postgres/README.md @@ -91,6 +91,7 @@ Commands with migratus ```shell +cd examples/postgres # We export the configuration as env variable, but we can use cli or file as well export MIGRATUS_CONFIG='{:store :database :db {:jdbcUrl "jdbc:postgresql://localhost:5432/migratus_example_db?user=postgres&password=migrate-me"}}' diff --git a/src/migratus/cli.clj b/src/migratus/cli.clj index dc6e81f..c19a10d 100644 --- a/src/migratus/cli.clj +++ b/src/migratus/cli.clj @@ -283,11 +283,11 @@ "Migratus API does not support this action(s) : " arguments "\n\n" (str/join (usage summary)))) -(defn run-migrate! [cfg args] +(defn run-migrate! [store args] (let [cmd-opts (parse-opts args migrate-cli-options :in-order true) {:keys [options arguments errors summary]} cmd-opts - rest-args (rest arguments)] - + rest-args (rest arguments) + cfg (proto/config store)] (cond errors (error-msg errors) (:until-just-before options) @@ -299,10 +299,11 @@ (migratus/migrate cfg)) :else (no-match-message args summary)))) -(defn run-rollback! [cfg args] +(defn run-rollback! [store args] (let [cmd-opts (parse-opts args rollback-cli-options :in-order true) {:keys [options arguments errors summary]} cmd-opts - rest-args (rest arguments)] + rest-args (rest arguments) + cfg (proto/config store)] (cond errors (error-msg errors) @@ -465,23 +466,23 @@ (.removeHandler main-logger h)) (.addHandler main-logger handler)))) -(defn run-up! [cfg args] +(defn run-up! [store args] (if (empty? args) (println-err "To run action up you must provide a migration-id as a parameter: up ") (->> args (map #(my-parse-long %)) - (apply migratus/up cfg)))) + (apply migratus/up (proto/config store))))) -(defn run-down! [cfg args] +(defn run-down! [store args] (if (empty? args) (println-err "To run action down you must provide a migration-id as a parameter: down ") (->> args (map #(my-parse-long %)) - (apply migratus/down cfg)))) + (apply migratus/down (proto/config store))))) (defn format-status-line "Format transaction status line" @@ -570,8 +571,8 @@ (println-err (ex-data e))))) (defn- run-init! - [cfg] - (migratus/init cfg)) + [store] + (migratus/init (proto/config store))) (defn do-print-usage ([summary]