diff --git a/app.js b/app.js index 8e14976..9d31d44 100644 --- a/app.js +++ b/app.js @@ -33,10 +33,10 @@ app.use((req, res, next) => { // set pug globals for 'universe' and 'node_env' app.use(function(req, res, next){ - if(req.app.get('env') === 'production'){ - req.universe = req.hostname.replace('.r-universe.dev', ''); - } else if(process.env.UNIVERSE){ + if(process.env.UNIVERSE){ req.universe = process.env.UNIVERSE; + } else if(req.app.get('env') === 'production'){ + req.universe = req.hostname.replace('.r-universe.dev', ''); } res.locals.universe = req.universe || 'ropensci'; res.locals.node_env = req.app.get('env'); diff --git a/package.json b/package.json index c752f17..cf6797d 100644 --- a/package.json +++ b/package.json @@ -10,6 +10,7 @@ "debug": "^4.3.4", "express": "^4.19.2", "http-errors": "*", + "mongodb": "^6.7.0", "morgan": "^1.10.0", "pug": "^3.0.2" } diff --git a/routes/pkginfo.js b/routes/pkginfo.js index 3687ce6..e78aeb4 100644 --- a/routes/pkginfo.js +++ b/routes/pkginfo.js @@ -1,26 +1,15 @@ -var express = require('express'); -var router = express.Router(); +const express = require('express'); +const router = express.Router(); +const db = require("../src/db.js"); function avatar_url(login, size = 120){ + if(login == 'bioc') login = 'bioconductor'; if(login.startsWith('gitlab-')) login = 'gitlab'; if(login.startsWith('bitbucket-')) login = 'atlassian'; login = login.replace('[bot]', ''); return `https://r-universe.dev/avatars/${login}.png?size=${size}`; } -function get_url(url){ - return fetch(url).then((res) => { - if (res.ok) { - return res; - } - throw new Error(`HTTP ${res.status} for: ${url}`); - }); -} - -function get_json(url){ - return get_url(url).then((res) => res.json()); -} - function normalize_authors(str){ // nested parentheses due to parenthesis inside the comment return str.replace(/\s*\([^()]*\)/g, '').replace(/\s*\([\s\S]+?\)/g,""); @@ -162,6 +151,12 @@ function problem_summary(src){ } } +//this also works if x was already a json ISO string +function date_to_string(x){ + var date = new Date(x); + return date.toDateString().substring(4); +} + function pretty_time_diff(ts){ var date = new Date(ts*1000); var now = new Date(); @@ -180,12 +175,13 @@ function pretty_time_diff(ts){ } router.get('/:package', function(req, res, next) { - return get_json(`https://cran.dev/${req.params.package}/json`).then(function(pkgdata){ + return db.get_package_info(req.params.package, req.universe).then(function(pkgdata){ pkgdata.format_count = format_count; pkgdata.universe = pkgdata._user; pkgdata.avatar_url = avatar_url; pkgdata.title = `${pkgdata.Package}: ${pkgdata.Title}`; pkgdata.Author = normalize_authors( pkgdata.Author); + pkgdata._created = date_to_string(pkgdata._created); pkgdata._grouped = group_binaries(pkgdata); pkgdata._bugtracker = guess_tracker_url(pkgdata); pkgdata._sysdeps = filter_sysdeps(pkgdata); diff --git a/routes/universe.js b/routes/universe.js index 69292ec..0272fe3 100644 --- a/routes/universe.js +++ b/routes/universe.js @@ -1,31 +1,6 @@ -var express = require('express'); -var router = express.Router(); - -function get_url(url){ - return fetch(url).then((res) => { - if (res.ok) { - return res; - } - throw new Error(`HTTP ${res.status} for: ${url}`); - }); -} - -function get_json(url){ - return get_url(url).then((res) => res.json()); -} - -function get_text(url){ - return get_url(url).then((res) => res.text()); -} - -function get_ndjson(url){ - return get_text(url).then(txt => txt.split('\n').filter(x => x.length).map(JSON.parse)); -} - -function get_universe_data(universe, fields, all = true){ - var apiurl = `https://${universe}.r-universe.dev/api/packages?fields=${fields.join()}&limit=2500${all ? '&all=true' : ''}`; - return get_json(apiurl) -} +const express = require('express'); +const router = express.Router(); +const db = require("../src/db.js"); function sort_by_package(x,y){ return x.Package.toLowerCase() < y.Package.toLowerCase() ? -1 : 1 @@ -101,7 +76,7 @@ router.get('/builds', function(req, res, next) { var fields = ['Package', 'Version', 'OS_type', '_user', '_owner', '_commit.time', '_commit.id', '_maintainer', '_upstream', '_registered', '_created', '_linuxdevel', '_winbinary', '_macbinary', '_wasmbinary', '_pkgdocs', '_status', '_buildurl', '_failure']; - get_universe_data(res.locals.universe, fields).then(function(pkgdata){ + db.get_universe_packages(res.locals.universe, fields).then(function(pkgdata){ res.render('builds', { format_yymmdd: format_yymmdd, all_ok: all_ok, @@ -115,7 +90,7 @@ router.get('/builds', function(req, res, next) { router.get("/packages", function(req, res, next){ var fields = ['Package', 'Version', 'Title', 'Description', '_user', '_commit.time', '_stars', '_rundeps', '_usedby', '_score', '_topics', '_pkglogo', '_sysdeps']; - get_universe_data(res.locals.universe, fields).then(function(pkgdata){ + db.get_universe_packages(res.locals.universe, fields).then(function(pkgdata){ res.render('packages', { format_count: format_count, format_time_since: format_time_since, @@ -127,7 +102,7 @@ router.get("/packages", function(req, res, next){ router.get("/badges", function(req, res, next){ var universe = res.locals.universe; var fields = ['Package', '_user', '_registered']; - get_universe_data(res.locals.universe, fields).then(function(pkgdata){ + db.get_universe_packages(res.locals.universe, fields).then(function(pkgdata){ pkgdata = pkgdata.filter(x => x._registered); pkgdata.unshift({Package: ':total', _user: universe}); pkgdata.unshift({Package: ':registry', _user: universe}); @@ -146,7 +121,7 @@ router.get("/badges", function(req, res, next){ router.get("/apis", function(req, res, next){ var fields = ['_datasets']; - get_universe_data(res.locals.universe, fields, false).then(function(pkgdata){ + db.get_universe_packages(res.locals.universe, fields, false).then(function(pkgdata){ res.render('apis', { pkgdata: pkgdata.sort(sort_by_package) }); @@ -158,16 +133,20 @@ router.get("/contributors", function(req, res, next){ }); router.get("/articles", function(req, res, next){ - get_ndjson(`https://${res.locals.universe}.r-universe.dev/stats/vignettes?all=true`).then(function(articles){ + db.get_universe_vignettes(res.locals.universe).then(function(articles){ + articles = articles.map(function(x){ + x.host = (x.user !== res.locals.universe) ? `https://${x.user}.r-universe.dev` : ""; + return x; + }).sort((x,y) => x.vignette.modified > y.vignette.modified ? -1 : 1); res.render('articles', { format_time_since: format_time_since, - articles: articles.sort((x,y) => x.vignette.modified > y.vignette.modified ? -1 : 1) + articles: articles }); }); }); router.get("/articles/:package/:vignette", function(req, res, next){ - return get_json(`https://cran.dev/${req.params.package}/json`).then(function(pkgdata){ + return db.get_package_info(req.params.package, req.universe).then(function(pkgdata){ var article = pkgdata._vignettes && pkgdata._vignettes.find(x => x.filename == req.params.vignette); if(article){ //do not open pdf files in iframe @@ -190,7 +169,6 @@ router.get("/search", function(req, res, next){ res.render("search"); }); - router.get('/favicon.ico', function(req, res, next) { res.status(404).send("No favicon yet") }); diff --git a/src/db.js b/src/db.js new file mode 100644 index 0000000..6dfe08e --- /dev/null +++ b/src/db.js @@ -0,0 +1,195 @@ +/* Database */ +const mongodb = require('mongodb'); +const HOST = process.env.CRANLIKE_MONGODB_SERVER || '127.0.0.1'; +const PORT = process.env.CRANLIKE_MONGODB_PORT || 27017; +const USER = process.env.CRANLIKE_MONGODB_USERNAME || 'root'; +const PASS = process.env.CRANLIKE_MONGODB_PASSWORD; +const AUTH = PASS ? (USER + ':' + PASS + "@") : ""; +const URL = 'mongodb://' + AUTH + HOST + ':' + PORT; +const production = process.env.NODE_ENV == 'production'; +var mongo_find; +var mongo_aggregate; + +if(production){ + console.log("Connecting to database....") + const connection = mongodb.MongoClient.connect(URL); + connection.then(function(client) { + console.log("Connected to MongoDB!") + const db = client.db('cranlike'); + const col = db.collection('packages'); + mongo_find = function(q){ + if(!col) + throw new Error("No mongodb connection available."); + return col.find(q); + } + mongo_aggregate = function(q){ + if(!col) + throw new Error("No mongodb connection available."); + return col.aggregate(q); + } + }).catch(function(error){ + console.log("Failed to connect to mongodb!\n" + error) + throw error; + }); +} + +function group_package_data(docs){ + var src = docs.find(x => x['_type'] == 'src'); + var failure = docs.find(x => x['_type'] == 'failure'); + if(!src){ + //no src found, package probably only has a 'failure' submission + if(failure) { + src = Object.assign({}, failure); //shallow copy to delete src.Version + delete src.Version; + } else { + return null; + } + } + if(failure){ + src._failure = { + version: failure.Version, + commit: failure._commit, + buildurl: failure._buildurl, + date: failure._created + } + } + src._binaries = docs.filter(x => x.Built).map(function(x){ + return { + r: x.Built.R, + os: x['_type'], + version: x.Version, + date: x._created, + distro: x['_type'] == 'linux' && x._distro || undefined, + arch: x.Built.Platform && x.Built.Platform.split("-")[0] || undefined, + commit: x._commit.id, + fileid: x['_fileid'], + status: x['_status'], + buildurl: x['_buildurl'] + } + }); + return src; +} + +function get_url(url){ + return fetch(url).then((res) => { + if (res.ok) { + return res; + } + throw new Error(`HTTP ${res.status} for: ${url}`); + }); +} + +function get_json(url){ + return get_url(url).then((res) => res.json()); +} + +function get_text(url){ + return get_url(url).then((res) => res.text()); +} + +function get_ndjson(url){ + return get_text(url).then(txt => txt.split('\n').filter(x => x.length).map(JSON.parse)); +} + +function days_ago(n){ + var now = new Date(); + return now.getTime()/1000 - (n*60*60*24); +} + +function build_projection(fields){ + var projection = {Package:1, _type:1, _user:1, _indexed: 1, _id:0}; + fields.forEach(function (f) { + projection[f] = 1; + }); + return projection; +} + +function mongo_package_info(package, universe){ + return mongo_find({_user: universe, Package: package}).toArray().then(function(docs){ + if(!docs.length) + throw new Error(`Package ${package} not found in ${universe}`); + return group_package_data(docs); + }); +} + +function mongo_universe_packages(user, fields, all){ + var query = all ? {'_universes': user} : {'_user': user}; + if(user == ":any" || user == 'cran'){ + query['_commit.time'] = {'$gt': days_ago(7)}; + } + var projection = build_projection(fields); + var cursor = mongo_aggregate([ + {$match: query}, + {$project: projection}, + {$group : { + _id : {'Package': '$Package', 'user':'$_user'}, + indexed: { $addToSet: "$_indexed" }, + timestamp: { $max : "$_commit.time" }, + files: { '$push': '$$ROOT' } + }}, + {$match: {'$or' : [{indexed: true}, {'_id.user': user}]}}, + {$sort : {timestamp : -1}}, + {$limit : 2500} + ]); + return cursor.toArray().then(function(pkglist){ + return pkglist.map(x => group_package_data(x.files)); + }); +} + +function mongo_universe_vignettes(user){ + var limit = 200; + var cursor = mongo_aggregate([ + {$match: {_universes: user, _type: 'src', '_vignettes' : {$exists: true}}}, + {$sort : {'_commit.time' : -1}}, + {$limit : limit}, + {$project: { + _id: 0, + user: '$_user', + package: '$Package', + version: '$Version', + maintainer: '$Maintainer', + universe: '$_user', + pkglogo: '$_pkglogo', + upstream: '$_upstream', + login: '$_maintainer.login', + published: '$_commit.time', + vignette: '$_vignettes' + }}, + {$unwind: '$vignette'} + ]); + return cursor.toArray(); +} + +function get_package_info(package, universe){ + if(production){ + return mongo_package_info(package, universe); + } else { + console.warn(`Fetching ${package} info from API...`) + return get_json(`https://cran.dev/${package}/json`); + } +} + +function get_universe_vignettes(universe){ + if(production){ + return mongo_universe_vignettes(universe) + } else { + console.warn(`Fetching ${universe} vignettes from API...`) + return get_ndjson(`https://${universe}.r-universe.dev/stats/vignettes?all=true`) + } +} + +function get_universe_packages(universe, fields, all = true){ + if(production){ + return mongo_universe_packages(universe, fields, all) + } else { + console.warn(`Fetching ${universe} packages from API...`) + var apiurl = `https://${universe}.r-universe.dev/api/packages?fields=${fields.join()}&limit=2500${all ? '&all=true' : ''}`; + return get_json(apiurl) + } +} + +module.exports = { + get_package_info: get_package_info, + get_universe_packages: get_universe_packages, + get_universe_vignettes: get_universe_vignettes +}; diff --git a/views/articles.pug b/views/articles.pug index af02d7b..6a17807 100644 --- a/views/articles.pug +++ b/views/articles.pug @@ -3,7 +3,8 @@ extends layout block content .list-group for x in articles - a.list-group-item.list-group-item-action(href=`/articles/${x.package}/${x.vignette.filename}`) + if x.user == universe + a.list-group-item.list-group-item-action(href=`${x.host}/articles/${x.package}/${x.vignette.filename}`) .d-flex.w-100.justify-content-between h2.h5.mb-1 #{x.vignette.title} small.text-body-secondary Updated #{format_time_since(x.vignette.modified)} diff --git a/views/pkginfo.pug b/views/pkginfo.pug index 511fb74..2677982 100644 --- a/views/pkginfo.pug +++ b/views/pkginfo.pug @@ -251,9 +251,7 @@ block content code.mx-1.detail-article-source #{article.source} | using code.mx-1.detail-article-engine #{article.engine} - | on - span.mx-1.detail-article-build #{_created.substring(0, 19).replace("T", " ")} - | . + span.detail-article-build on #{_created}. p.text-end small.text-muted.article-modified.text-nowrap Last update: #{(article.modified || "??").substring(0, 10)} br