Update 'global' pages (#6)

routes/global.js

@@ -14,35 +14,105 @@ function format_yymmdd(x){
   return `${year}-${month}-${day}`;
 }
 
+function cleanup_sysdep_desc(str){
+  if(!str) return "";
+  var str = str.charAt(0).toUpperCase() + str.slice(1);
+  return str.replace(/\(.*\)$/, '').replace('SASL -', 'SASL').replace(/[-,]+ .*(shared|runtime|binary|library|legacy|precision|quantum).*$/i, '');
+}
+
+function check_to_color(check){
+  switch (check) {
+    case 'ERROR':
+      return 'text-danger';
+    case 'WARNING':
+      return 'text-warning';
+    case 'NOTE':
+      return 'text-success';
+    case 'OK':
+      return 'text-success';
+    default:
+      return 'text-dark';
+  }
+}
+
+function os_icon(type){
+  switch (type) {
+    case 'win':
+      return 'fa-windows';
+    case 'linux':
+      return 'fa-linux';
+    case 'mac':
+      return 'fa-apple';
+    default:
+      return 'fa-question';
+  }
+}
+
 router.get("/_global/search", function(req, res, next){
-  res.render("search");
+  res.render("global/search");
 });
 
 router.get("/_global/activity", function(req, res, next){
-  res.render("activity");
+  res.render("global/activity");
+});
+
+router.get("/_global/builds", function(req, res, next){
+  db.get_builds().then(function(packages){
+    packages.forEach(function(x){
+      var checks = x.runs.filter(run => run.check && run.built.Platform !== 'x86_64-apple-darwin20');
+      x.check_icon_html = function(version, type){
+        var bin = checks.find(run => run.built.R.startsWith(version) && run.type == type);
+        if(bin){
+          return `<i class="fa-fw fab ${os_icon(bin.type)} ${check_to_color(bin.check)}"></i>`
+        } else if(type != 'linux' && x.user == 'cran'){
+          return '<i class="fa-fw fa-solid fa-minus"></i>';
+        } else {
+          return '<i class="fa-fw fa-solid fa-xmark text-danger"></i>';
+        }
+      };
+      x.date = format_yymmdd(x.timestamp * 1000);
+      x.src = x.runs.find(run => run.type == 'src') || {};
+      x.failure = x.runs.find(run => run.type == 'failure');
+    });
+    res.render('global/builds', {packages: packages});
+  }).catch(next);
 });
 
 router.get("/_global/organizations", function(req, res, next){
   db.get_organizations().then(function(orgs){
-    res.render('organizations', {orgs: orgs});
+    orgs.forEach(function(x){
+      x.avatar = x.uuid ? `https://avatars.githubusercontent.com/u/${x.uuid}` : `https://r-universe.dev/avatars/${x.universe}.png`;
+    });
+    res.render('global/organizations', {orgs: orgs});
   }).catch(next);
 });
 
 router.get("/_global/repositories", function(req, res, next){
   db.get_repositories().then(function(repos){
-    res.render('repositories', {repos: repos, format_yymmdd: format_yymmdd});
+    res.render('global/repositories', {repos: repos, format_yymmdd: format_yymmdd});
   });
 });
 
 router.get("/_global/scores", function(req, res, next){
   db.get_scores().then(function(packages){
-    res.render('scores', {packages: packages});
+    res.render('global/scores', {packages: packages});
+  });
+});
+
+router.get("/_global/sysdeps", function(req, res, next){
+  db.get_sysdeps().then(function(sysdeps){
+    sysdeps = sysdeps.filter(x => x.library && x.library !== 'c++');
+    sysdeps.forEach(function(x){
+      x.description = cleanup_sysdep_desc(x.description);
+      x.usedby = x.usedby.sort((a,b) => a.package.toLowerCase() < b.package.toLowerCase() ? -1 : 1);
+    });
+    res.render('global/sysdeps', {sysdeps: sysdeps});
   });
 });
 
 router.get("/_global/galaxy", function(req, res, next){
   db.get_organizations().then(function(orgs){
-    res.render('galaxy', {rnd: rnd, orgs: orgs.slice(0, 250)});
+    res.render('global/galaxy', {rnd: rnd, orgs: orgs.slice(0, 250)});
   }).catch(next);
 });
 

src/db.js

@@ -184,6 +184,128 @@ function mongo_universe_vignettes(user){
   return cursor.toArray();
 }
 
+function mongo_all_universes(organizations_only){
+  var query = {_type: 'src', _registered: true};
+  if(organizations_only){
+    query['_userbio.type'] = 'organization';
+    query['_user'] = {$ne: 'cran'};
+  }
+  var cursor = mongo_aggregate([
+    {$match: query},
+    {$sort:{ _id: -1}},
+    {$group: {
+      _id : '$_user',
+      updated: { $max: '$_commit.time'},
+      packages: { $sum: 1 },
+      indexed: { $sum: { $toInt: '$_indexed' }},
+      name: { $first: '$_userbio.name'},
+      type: { $first: '$_userbio.type'},
+      uuid: { $first: '$_userbio.uuid'},
+      bio: { $first: '$_userbio.description'},
+      emails: { $addToSet: '$_maintainer.email'}
+    }},
+    {$project: {_id: 0, universe: '$_id', packages: 1, updated: 1, type: 1, uuid: 1,
+      indexed:1, name: 1, type: 1, bio: 1, maintainers: { $size: '$emails' },
+    }},
+    {$sort:{ indexed: -1}}
+  ]);
+  return cursor.toArray();
+}
+
+function mongo_all_scores(){
+  function array_size(key){
+    return {$cond: [{ $isArray: key }, {$size: key}, 0 ]};
+  }
+  var query = {_type: 'src', _indexed: true};
+  var projection = {
+    _id: 0,
+    package: '$Package',
+    universe: "$_user",
+    score: '$_score',
+    stars: "$_stars",
+    downloads: "$_downloads.count",
+    scripts: "$_searchresults",
+    dependents: '$_usedby',
+    commits: {$sum: '$_updates.n'},
+    contributors: array_size({$objectToArray: '$_contributions'}),
+    datasets: array_size('$_datasets'),
+    vignettes: array_size('$_vignettes'),
+    releases: array_size('$_releases')
+  }
+  var cursor = mongo_find(query).sort({_score: -1}).project(projection);
+  return cursor.toArray();
+}
+
+function mongo_all_sysdeps(distro){
+  var query = {_type: 'src', '_sysdeps': {$exists: true}};
+  if(distro){
+    query['_distro'] = distro;
+  }
+  var cursor = mongo_aggregate([
+    {$match: query},
+    {$unwind: '$_sysdeps'},
+    {$group: {
+      _id : '$_sysdeps.name',
+      packages: { $addToSet: '$_sysdeps.package'},
+      headers: { $addToSet: '$_sysdeps.headers'},
+      version: { $first: '$_sysdeps.version'},
+      homepage: { $addToSet: '$_sysdeps.homepage'},
+      description: { $addToSet: '$_sysdeps.description'},
+      distro : { $addToSet: '$_distro'},
+      usedby : { $addToSet: {owner: '$_owner', package:'$Package'}}
+    }},
+    {$project: {_id: 0, library: '$_id', packages: 1, headers: 1, version: 1, usedby: 1,
+      homepage: { '$first' : '$homepage'}, description: { '$first' : '$description'}, distro:{ '$first' : '$distro'}}},
+    {$sort:{ library: 1}}
+  ]);
+  return cursor.toArray();
+}
+
+function days_ago(n){
+  var now = new Date();
+  return now.getTime()/1000 - (n*60*60*24);
+}
+
+function mongo_recent_builds(days = 7){
+  var query = {'_commit.time' : {'$gt': days_ago(days)}};
+  var cursor = mongo_aggregate([
+    {$match: query},
+    {$group : {
+      _id : { user: '$_user', package: '$Package', commit: '$_commit.id'},
+      version: { $first : "$Version" },
+      maintainer: { $first : "$_maintainer.name" },
+      maintainerlogin: { $first : "$_maintainer.login" },
+      timestamp: { $first : "$_commit.time" },
+      upstream: { $first : "$_upstream" },
+      registered: { $first: "$_registered" },
+      os_restriction: { $addToSet: '$OS_type'},
+      macbinary: { $addToSet : '$_macbinary' },
+      winbinary: { $addToSet : '$_winbinary' },
+      runs : { $addToSet:
+        { type: "$_type", built: '$Built', date:'$_published', url: '$_buildurl', status: '$_status', distro: '$_distro', check: '$_check'}
+      }
+    }},
+    {$sort : {"timestamp" : -1}},
+    {$project: {
+      _id: 0,
+      user: '$_id.user',
+      package: '$_id.package',
+      commit: '$_id.commit',
+      maintainer: 1,
+      maintainerlogin: 1,
+      version: 1,
+      timestamp: 1,
+      registered: 1,
+      runs: 1,
+      upstream: 1,
+      macbinary: { $first: "$macbinary" },
+      winbinary: { $first: "$winbinary" },
+      os_restriction:{ $first: "$os_restriction" }
+    }}
+  ]);
+  return cursor.toArray();
+}
+
 function get_package_info(package, universe){
   if(production){
     return mongo_package_info(package, universe);
@@ -236,15 +358,35 @@ function get_scores(){
 
 function get_organizations(){
   if(production){
-    return mongo_all_universes()
+    return mongo_all_universes(true);
   } else {
     console.warn(`Fetching universes data from API...`);
     return get_ndjson(`https://r-universe.dev/api/universes?type=organization&skipcran=1&stream=1`);
   }
 }
 
+function get_sysdeps(){
+  if(production){
+    return mongo_all_sysdeps()
+  } else {
+    console.warn(`Fetching sysdeps data from API...`);
+    return get_ndjson(`https://r-universe.dev/stats/sysdeps?all=1`);
+  }
+}
+
+function get_builds(){
+  if(production){
+    return mongo_recent_builds()
+  } else {
+    console.warn(`Fetching builds data from API...`);
+    return get_ndjson(`https://r-universe.dev/stats/builds?limit=1000`);
+  }
+}
+
 module.exports = {
   get_scores: get_scores,
+  get_builds : get_builds,
+  get_sysdeps : get_sysdeps,
   get_repositories: get_repositories,
   get_organizations: get_organizations,
   get_package_info: get_package_info,

static/layout.css

@@ -149,4 +149,31 @@ canvas {
   p.overflow-x-invisible::-webkit-scrollbar {
     display: none;
   }
-}
+}
+
+.logo-img{
+  height: 70px
+}
+
+#global-sidebar {
+  width: 60px;
+  padding: 15px 5px 15px 5px;
+}
+
+#global-sidebar:hover{
+  width: 240px;
+  margin-right: -180px;
+  z-index: 9;
+}
+
+#global-sidebar .menu-expanded {
+  display: none;
+}
+
+#global-sidebar:hover .menu-expanded {
+  display: unset;
+}
+
+#global-sidebar:hover .menu-collapsed {
+  display: none;
+}