This closes #30

.env.example

@@ -1,5 +1,5 @@
 MONGO_URI=mongodb://mongo:27017/overflow
-REDIS_URI=redis
+REDIS_URI=redis://redis:6379
 CRAWL_QUEUE=feeds-to-crawl
 FEEDS_FILE=./assets/engineering_blogs.opml
 SENTRY_DSN=https://[email protected]/ID

package.json

@@ -9,8 +9,8 @@
     "lint": "eslint src/",
     "start:worker": "node ./src/worker.js",
     "start:server": "node ./src/server.js",
-    "start:dev_worker": "nodemon ./src/worker.js --exec \"node -r dotenv/config -r babel-register\"",
-    "start:dev_server": "nodemon ./src/server.js --exec \"node -r dotenv/config -r babel-register\"",
+    "start:dev_worker": "nodemon ./src/worker.js",
+    "start:dev_server": "nodemon ./src/server.js",
     "test:strict": "jest",
     "test": "npm run lint && npm run test:strict"
   },
@@ -44,10 +44,9 @@
     }
   },
   "dependencies": {
-    "bee-queue": "^1.2.2",
+    "bull": "^3.4.2",
     "cors": "^2.8.4",
     "cron": "^1.3.0",
-    "dotenv": "^5.0.1",
     "ejs": "^2.5.7",
     "express": "^4.16.2",
     "express-graphql": "^0.6.12",
@@ -61,18 +60,9 @@
     "xml2js": "^0.4.19"
   },
   "devDependencies": {
-    "babel-cli": "^6.26.0",
-    "babel-eslint": "^8.2.2",
-    "babel-jest": "^22.4.3",
-    "babel-plugin-transform-async-to-generator": "^6.24.1",
-    "babel-plugin-transform-object-rest-spread": "^6.26.0",
-    "babel-plugin-transform-runtime": "^6.23.0",
-    "babel-preset-env": "^1.6.1",
-    "babel-register": "^6.26.0",
     "coveralls": "^3.0.0",
     "eslint": "^4.19.1",
     "jest": "^22.4.3",
-    "nodemon": "^1.17.2",
-    "npm-run-all": "^4.1.2"
+    "nodemon": "^1.17.2"
   }
 }

src/lib/job.js

@@ -1,20 +1,21 @@
-function crawlFeed(URL, callback) {
-  const FeedParser = require('feedparser');
-  const request = require('request');
+const FeedParser = require('feedparser');
+const request = require('request');
+const Queue = require('bull');
+const Feed = require('../models/feed');
 
+function crawlFeed(URL, callback) {
   const feedparser = new FeedParser();
   let posts = [];
 
   request(URL)
   .on('error', function (error) {
-    console.log('\x1b[31m[ERROR]\x1b[0m', error.message);
-    callback(error);
+    return callback(error);
   })
   .on('response', function () {
     this.pipe(feedparser);
 
     feedparser.on('error', function (error) {
-      console.log('\x1b[31m[ERROR]\x1b[0m', error.message);
+      return callback(error);
     });
 
     feedparser.on('readable', function () {
@@ -29,30 +30,23 @@ function crawlFeed(URL, callback) {
   });
 }
 
-function fetchLatestPosts() {
-  const Feed = require('../models/feed');
-
+function fetchLatestPosts(callback) {
   Feed.find({}, function (error, feeds) {
     if (error) {
-      console.log('\x1b[31m[ERROR]\x1b[0m', error.message);
-    } else {
-      const Queue = require('bee-queue');
-      const queue = new Queue(process.env.CRAWL_QUEUE, {
-        redis: {
-          host: process.env.REDIS_URI
-        }
-      });
-
-      feeds.forEach(function (feed) {
-        queue.createJob(feed).save();
-      });
-
-      console.log('\x1b[34m[INFO]\x1b[0m', `${feeds.length} feeds to crawl`);
+      return callback(error);
     }
+
+    const queue = new Queue(process.env.CRAWL_QUEUE, process.env.REDIS_URI);
+
+    feeds.forEach(function (feed) {
+      queue.add(feed);
+    });
+
+    callback(null);
   });
 }
 
 module.exports = {
   crawlFeed,
   fetchLatestPosts
-};
+};