Optimize feed generation

lib/create-new-post.ts

@@ -74,6 +74,7 @@ const createQuestions: (
     choices: authors.map((ch) => ({ title: ch, value: ch })),
     max: 1, // Change this to support multiple authors
     hint: "- Space to select. Return to submit",
+    format: (value) => value[0], // We only support a single author
   },
   {
     type: (prev) => (prev.length === 0 ? "text" : null), // hide this questions if author is picked from the previous question

lib/edit-post.ts

@@ -8,7 +8,7 @@ import pick = require("lodash.pick");
 
 function updatePostOnDisk(post: Post): void {
   fs.writeFileSync(
-    path.join(POSTS_DIR, `${post.publishedAt}-${post.slug}.mdx`),
+    path.join(POSTS_DIR, post.fileName),
     makeFrontmatterFromPost(post),
     { encoding: "utf8" }
   );
@@ -32,7 +32,7 @@ const createQuestions: (posts: Post[]) => prompts.PromptObject[] = (posts) => [
 
 (async function main() {
   try {
-    const posts = await getAllPosts();
+    const posts = await getAllPosts({ withContent: true });
     const answers = await prompts(createQuestions(posts));
     const pickedTitle = answers.title[0]; // Since the prompt is of type multiselect but limited to max 1 picked value, it's always a single item array
 

lib/postsCache.ts

@@ -0,0 +1,22 @@
+import Cache from "file-system-cache";
+import path from "path";
+import * as lzstring from "lz-string";
+import { getAllPosts } from "../src/posts";
+
+export const cache = new Cache({
+  basePath: path.join(process.cwd(), ".cache"),
+});
+
+export const postsCache = {
+  arePostsChanged: async () => {
+    const prevToken = await cache.get("postsToken", "");
+    return (await postsCache.getPostsToken()) !== prevToken;
+  },
+  updatePostsCache: async () => {
+    cache.set("postsToken", await postsCache.getPostsToken());
+  },
+  getPostsToken: async () => {
+    const posts = await getAllPosts({ withContent: true });
+    return lzstring.compress(JSON.stringify(posts));
+  },
+};

lib/utils.ts

@@ -1,4 +1,5 @@
 import matter from "gray-matter";
+import omit from "lodash.omit";
 import { Post } from "../src/types";
 
 export const formatDate = (date: Intl.DateTimeFormatPart[]): string => {
@@ -10,7 +11,10 @@ export const formatDate = (date: Intl.DateTimeFormatPart[]): string => {
 };
 
 export const makeFrontmatterFromPost = (post: Post): string => {
-  return matter.stringify("Here goes the content of your post", post);
+  return matter.stringify(
+    post.content || "Here goes the content of your post",
+    omit(post, "content", "fileName")
+  );
 };
 
 export const makeSetFromArray = <T>(array: T[]) => Array.from(new Set(array));

package.json

@@ -24,12 +24,14 @@
     "@xstate/react": "^2.0.0",
     "dayjs": "^1.10.7",
     "feed": "^4.2.2",
+    "file-system-cache": "^1.0.5",
     "framer-motion": "^4",
     "fuzzy-search": "^3.2.1",
     "gray-matter": "^4.0.3",
     "jest": "^27.2.5",
     "lodash.omit": "^4.5.0",
     "lodash.pick": "^4.4.0",
+    "lz-string": "^1.4.4",
     "mdx-embed": "^0.0.22",
     "next": "11.1.2",
     "next-mdx-remote": "^3.0.4",
@@ -52,6 +54,7 @@
     "@types/jest": "^27.0.2",
     "@types/lodash.omit": "^4.5.6",
     "@types/lodash.pick": "^4.4.6",
+    "@types/lz-string": "^1.3.34",
     "@types/prompts": "^2.0.14",
     "@types/react": "17.0.24",
     "@types/react-dom": "^17.0.9",

pages/index.tsx

@@ -15,6 +15,7 @@ import Link from "next/link";
 import { Seo } from "../src/Seo";
 import { formatDate } from "../src/utils";
 import { generateFeed } from "../src/feed";
+import { postsCache } from "../lib/postsCache";
 
 const Home: NextPage<{ posts: Post[] }> = ({ posts }) => {
   return (
@@ -70,16 +71,21 @@ const Home: NextPage<{ posts: Post[] }> = ({ posts }) => {
 };
 
 export const getStaticProps = async () => {
-  // TODO: Generate the feed only when there is a change in the posts
   const posts = await getAllPosts({ withContent: true });
 
-  if (process.env.NODE_ENV === "production") {
+  if (
+    (await postsCache.arePostsChanged()) &&
+    process.env.NODE_ENV === "production"
+  ) {
+    console.log("generating feed");
     const feed = await generateFeed(posts);
 
     fs.mkdirSync("public/feeds/", { recursive: true });
     fs.writeFileSync("public/feeds/rss.xml", feed.rss2());
     fs.writeFileSync("public/feeds/feed.json", feed.json1());
     fs.writeFileSync("public/feeds/atom.xml", feed.atom1());
+
+    await postsCache.updatePostsCache();
   }
 
   return { props: { posts } };

yarn.lock

@@ -1678,6 +1678,11 @@
   resolved "https://registry.yarnpkg.com/@types/lodash/-/lodash-4.14.178.tgz#341f6d2247db528d4a13ddbb374bcdc80406f4f8"
   integrity sha512-0d5Wd09ItQWH1qFbEyQ7oTQ3GZrMfth5JkbN3EvTKLXcHLRDSXeLnlvlOn0wvxVIwK5o2M8JzP/OWz7T3NRsbw==
 
+"@types/lz-string@^1.3.34":
+  version "1.3.34"
+  resolved "https://registry.yarnpkg.com/@types/lz-string/-/lz-string-1.3.34.tgz#69bfadde419314b4a374bf2c8e58659c035ed0a5"
+  integrity sha512-j6G1e8DULJx3ONf6NdR5JiR2ZY3K3PaaqiEuKYkLQO0Czfi1AzrtjfnfCROyWGeDd5IVMKCwsgSmMip9OWijow==
+
 "@types/mdast@^3.0.0":
   version "3.0.10"
   resolved "https://registry.yarnpkg.com/@types/mdast/-/mdast-3.0.10.tgz#4724244a82a4598884cbbe9bcfd73dff927ee8af"
@@ -2228,6 +2233,11 @@ binary-extensions@^2.0.0:
   resolved "https://registry.yarnpkg.com/binary-extensions/-/binary-extensions-2.2.0.tgz#75f502eeaf9ffde42fc98829645be4ea76bd9e2d"
   integrity sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA==
 
+bluebird@^3.3.5:
+  version "3.7.2"
+  resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.7.2.tgz#9f229c15be272454ffa973ace0dbee79a1b0c36f"
+  integrity sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==
+
 bn.js@^4.0.0, bn.js@^4.1.0, bn.js@^4.11.9:
   version "4.12.0"
   resolved "https://registry.yarnpkg.com/bn.js/-/bn.js-4.12.0.tgz#775b3f278efbb9718eec7361f483fb36fbbfea88"
@@ -3487,6 +3497,15 @@ file-entry-cache@^6.0.1:
   dependencies:
     flat-cache "^3.0.4"
 
+file-system-cache@^1.0.5:
+  version "1.0.5"
+  resolved "https://registry.yarnpkg.com/file-system-cache/-/file-system-cache-1.0.5.tgz#84259b36a2bbb8d3d6eb1021d3132ffe64cfff4f"
+  integrity sha1-hCWbNqK7uNPW6xAh0xMv/mTP/08=
+  dependencies:
+    bluebird "^3.3.5"
+    fs-extra "^0.30.0"
+    ramda "^0.21.0"
+
 fill-range@^7.0.1:
   version "7.0.1"
   resolved "https://registry.yarnpkg.com/fill-range/-/fill-range-7.0.1.tgz#1919a6a7c75fe38b2c7c77e5198535da9acdda40"
@@ -3606,6 +3625,17 @@ [email protected]:
   dependencies:
     tslib "^2.1.0"
 
+fs-extra@^0.30.0:
+  version "0.30.0"
+  resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-0.30.0.tgz#f233ffcc08d4da7d432daa449776989db1df93f0"
+  integrity sha1-8jP/zAjU2n1DLapEl3aYnbHfk/A=
+  dependencies:
+    graceful-fs "^4.1.2"
+    jsonfile "^2.1.0"
+    klaw "^1.0.0"
+    path-is-absolute "^1.0.0"
+    rimraf "^2.2.8"
+
 fs-extra@^7.0.1:
   version "7.0.1"
   resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-7.0.1.tgz#4f189c44aa123b895f722804f55ea23eadc348e9"
@@ -3754,7 +3784,7 @@ globby@^11.0.3:
     merge2 "^1.4.1"
     slash "^3.0.0"
 
-graceful-fs@^4.1.11, graceful-fs@^4.1.2, graceful-fs@^4.1.6, graceful-fs@^4.2.4:
+graceful-fs@^4.1.11, graceful-fs@^4.1.2, graceful-fs@^4.1.6, graceful-fs@^4.1.9, graceful-fs@^4.2.4:
   version "4.2.9"
   resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.9.tgz#041b05df45755e587a24942279b9d113146e1c96"
   integrity sha512-NtNxqUcXgpW2iMrfqSfR73Glt39K+BLwWsPs94yR63v45T0Wbej7eRmL5cWfwEgqXnmjQp3zaJTshdRW/qC2ZQ==
@@ -4962,6 +4992,13 @@ json5@^1.0.1:
   dependencies:
     minimist "^1.2.0"
 
+jsonfile@^2.1.0:
+  version "2.4.0"
+  resolved "https://registry.yarnpkg.com/jsonfile/-/jsonfile-2.4.0.tgz#3736a2b428b87bbda0cc83b53fa3d633a35c2ae8"
+  integrity sha1-NzaitCi4e72gzIO1P6PWM6NcKug=
+  optionalDependencies:
+    graceful-fs "^4.1.6"
+
 jsonfile@^4.0.0:
   version "4.0.0"
   resolved "https://registry.yarnpkg.com/jsonfile/-/jsonfile-4.0.0.tgz#8771aae0799b64076b76640fca058f9c10e33ecb"
@@ -4989,6 +5026,13 @@ klaw-sync@^6.0.0:
   dependencies:
     graceful-fs "^4.1.11"
 
+klaw@^1.0.0:
+  version "1.3.1"
+  resolved "https://registry.yarnpkg.com/klaw/-/klaw-1.3.1.tgz#4088433b46b3b1ba259d78785d8e96f73ba02439"
+  integrity sha1-QIhDO0azsbolnXh4XY6W9zugJDk=
+  optionalDependencies:
+    graceful-fs "^4.1.9"
+
 kleur@^3.0.3:
   version "3.0.3"
   resolved "https://registry.yarnpkg.com/kleur/-/kleur-3.0.3.tgz#a79c9ecc86ee1ce3fa6206d1216c501f147fc07e"
@@ -5157,6 +5201,11 @@ lru-cache@^6.0.0:
   dependencies:
     yallist "^4.0.0"
 
+lz-string@^1.4.4:
+  version "1.4.4"
+  resolved "https://registry.yarnpkg.com/lz-string/-/lz-string-1.4.4.tgz#c0d8eaf36059f705796e1e344811cf4c498d3a26"
+  integrity sha1-wNjq82BZ9wV5bh40SBHPTEmNOiY=
+
 magic-string@^0.25.3:
   version "0.25.7"
   resolved "https://registry.yarnpkg.com/magic-string/-/magic-string-0.25.7.tgz#3f497d6fd34c669c6798dcb821f2ef31f5445051"
@@ -6043,6 +6092,11 @@ [email protected]:
   dependencies:
     inherits "~2.0.3"
 
+ramda@^0.21.0:
+  version "0.21.0"
+  resolved "https://registry.yarnpkg.com/ramda/-/ramda-0.21.0.tgz#a001abedb3ff61077d4ff1d577d44de77e8d0a35"
+  integrity sha1-oAGr7bP/YQd9T/HVd9RN536NCjU=
+
 randombytes@^2.0.0, randombytes@^2.0.1, randombytes@^2.0.5:
   version "2.1.0"
   resolved "https://registry.yarnpkg.com/randombytes/-/randombytes-2.1.0.tgz#df6f84372f0270dc65cdf6291349ab7a473d4f2a"
@@ -6470,7 +6524,7 @@ reusify@^1.0.4:
   resolved "https://registry.yarnpkg.com/reusify/-/reusify-1.0.4.tgz#90da382b1e126efc02146e90845a88db12925d76"
   integrity sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==
 
-rimraf@^2.6.3:
+rimraf@^2.2.8, rimraf@^2.6.3:
   version "2.7.1"
   resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-2.7.1.tgz#35797f13a7fdadc566142c29d4f07ccad483e3ec"
   integrity sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w==