From 57cfb055eb7e11cc670bc0b6579aad8e93c95331 Mon Sep 17 00:00:00 2001 From: Collin Brittle Date: Fri, 10 Jul 2020 05:34:38 +0000 Subject: [PATCH 1/2] Makes bundle cache key creation more repeatable In CircleCI, different parallel runs could order the files found for the bundle cache key differently, resulting in cache misses, even though the files found and the MD5s were the same. Now, the checksums are sorted. The file that becomes part of the cache key is also added to the run's artifacts, so future debugging won't have to do so much guesswork. --- src/commands/bundle.yml | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/src/commands/bundle.yml b/src/commands/bundle.yml index b7c32f2..c34e05f 100644 --- a/src/commands/bundle.yml +++ b/src/commands/bundle.yml @@ -9,12 +9,31 @@ parameters: type: string default: '1' steps: - # Generate md5s for any Gemfile*, and *.gemspec files to generate a unique - # cache key representing the contents of all files. - run: name: Generate a cache key for the bundle - command: | - echo $(find . -type f \( -name "Gemfile*" -o -name "*.gemspec" \) -exec md5sum {} \;) >> "BUNDLE_CACHE_KEY" + # Let's break this command down step by step. + # `command : >` - YAML magic to concatenate the lines, stripping indentation equal to the next line. + # `find .` - To start, we're going to find things decending from the currect directory. + # `-path "*vendor/bundle" -prune -o` - Prevent find from searching the vendor/bundle directory. + # This helps with idempotentcy, as gems will be installed into vendor/bundle, and they have Gemfiles. + # `-path "*.git" -prune -o` - Prevent find from searching the .git directory. + # `-type f \( -name "Gemfile*" -o -name "*.gemspec" \)` - Match any file beginning with "Gemfile" or + # ending in ".gemspec". No, the '.' is not expanded like with regex. + # `-exec md5sum {} \+` - Execute md5sum on the files found. BUT! The '+' means call md5sum only once, + # with all the filenames one after another. + # `| sort -o "BUNDLE_CACHE_KEY"` send the output of the find command into sort, and write the output to a + # file named 'BUNDLE_CACHE_KEY'. The files will be sorted by their md5 rather than their name. + command: > + find . + -path "*vendor/bundle" -prune -o + -path "*.git" -prune -o + -type f \( -name "Gemfile*" -o -name "*.gemspec" \) + -exec md5sum {} \+ + | sort -o "BUNDLE_CACHE_KEY" + + - store_artifacts: + path: "BUNDLE_CACHE_KEY" + destination: "BUNDLE_CACHE_KEY" - restore_cache: name: Restore bundle from cache From 5864d9646c58fedc751f94403ec5f942f4a76e61 Mon Sep 17 00:00:00 2001 From: Collin Brittle Date: Fri, 10 Jul 2020 16:12:48 +0000 Subject: [PATCH 2/2] Changes the bundle cache key construction The cache key now is of the form `v-ruby-bundler-bundle` to allow bundle checksum misses to hit a better, if less specific, cache. In general, I expect this order to be least-often-changed to most-often-changed. Also shows the BUNDLE_CACHE_KEY contents in a step rather than saving the file as an artifact. --- src/commands/bundle.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/commands/bundle.yml b/src/commands/bundle.yml index c34e05f..32f3c3c 100644 --- a/src/commands/bundle.yml +++ b/src/commands/bundle.yml @@ -21,7 +21,7 @@ steps: # ending in ".gemspec". No, the '.' is not expanded like with regex. # `-exec md5sum {} \+` - Execute md5sum on the files found. BUT! The '+' means call md5sum only once, # with all the filenames one after another. - # `| sort -o "BUNDLE_CACHE_KEY"` send the output of the find command into sort, and write the output to a + # `| sort -o "BUNDLE_CACHE_KEY"` - Send the output of the find command into sort, and write the output to a # file named 'BUNDLE_CACHE_KEY'. The files will be sorted by their md5 rather than their name. command: > find . @@ -31,17 +31,17 @@ steps: -exec md5sum {} \+ | sort -o "BUNDLE_CACHE_KEY" - - store_artifacts: - path: "BUNDLE_CACHE_KEY" - destination: "BUNDLE_CACHE_KEY" + - run: + name: Show contents of BUNDLE_CACHE_KEY + command: cat "BUNDLE_CACHE_KEY" - restore_cache: name: Restore bundle from cache keys: - - v<< parameters.cache_version >>-bundle-{{ checksum "BUNDLE_CACHE_KEY" }}-<< parameters.ruby_version >>-<< parameters.bundler_version >> - - v<< parameters.cache_version >>-bundle-{{ checksum "BUNDLE_CACHE_KEY" }}-<< parameters.ruby_version >> - - v<< parameters.cache_version >>-bundle-{{ checksum "BUNDLE_CACHE_KEY" }} - - v<< parameters.cache_version >>-bundle + - v<< parameters.cache_version >>-ruby<< parameters.ruby_version >>-bundler<< parameters.bundler_version >>-bundle{{ checksum "BUNDLE_CACHE_KEY" }} + - v<< parameters.cache_version >>-ruby<< parameters.ruby_version >>-bundler<< parameters.bundler_version >> + - v<< parameters.cache_version >>-ruby<< parameters.ruby_version >> + - v<< parameters.cache_version >> - run: name: Update bundler @@ -55,7 +55,7 @@ steps: - save_cache: name: Save bundle cache - key: v<< parameters.cache_version >>-bundle-{{ checksum "BUNDLE_CACHE_KEY" }}-<< parameters.ruby_version >>-<< parameters.bundler_version >> + key: v<< parameters.cache_version >>-ruby<< parameters.ruby_version >>-bundler<< parameters.bundler_version >>-bundle{{ checksum "BUNDLE_CACHE_KEY" }} paths: - vendor/bundle