[WIP] PoC for a Node.JS backend using nodegit

.gitignore

@@ -1,10 +1,13 @@
 *.pyc
 /bin/
 env/
+.env/
 .idea
 *.swp
 tests/repos/build
 *.egg-info
 build/
 dist/
+.vscode/
 .DS_Store
+/repositories

basic.py

@@ -0,0 +1,9 @@
+from klaus import Klaus
+
+repos = [
+    "./repositories/huggingface/moon-landing.git",
+    "./repositories/transformers.git",
+]
+
+app = Klaus(repos, "klaus.py", use_smarthttp=False)
+app.run(debug=True)

klaus/static/klaus.css

@@ -159,7 +159,7 @@ footer a:hover { text-decoration: none; }
 a.commit { color: black !important; }
 
 .tree ul { font-family: monospace; border-top: 1px solid #e0e0e0; }
-.tree li { background-color: #f9f9f9; border: 1px solid #e0e0e0; border-top: 0; }
+.tree li { background-color: white; border: 1px solid #e0e0e0; border-top: 0; }
 .tree li a {
   padding: 5px 7px 6px 7px;
   display: block;

node/.gitignore

@@ -0,0 +1,2 @@
+node_modules/
+dist/

node/app/Context.ts

@@ -0,0 +1,147 @@
+import * as express from 'express';
+import * as Git from 'nodegit';
+import { c } from '../lib/Log';
+import { Repo } from './Repo';
+
+
+const DEFAULT_BRANCH = `master`;
+
+interface BreadcrumbPath {
+	dir:   string;
+	href?: string;
+}
+
+export class NotFoundError extends Error {}
+
+
+export class Context {
+	/**
+	 * Repo id (repo or user/repo)
+	 */
+	repoName:  string;
+	/**
+	 * Branch/commit-sha/tag
+	 */
+	rev:       string;
+	/**
+	 * path is undefined for repo's root
+	 * (only makes sense for tree)
+	 */
+	path?:     string;
+
+	/// After `.initialize()`
+	repo:      Git.Repository;
+	commit:    Git.Commit;
+	treeEntry: Git.TreeEntry;
+	/// Other ad hoc data, for convenient access from templates
+	data: Record<string, any> = {};
+
+	constructor(req: express.Request) {
+		/**
+		 * Note: we only support branch names and tag names
+		 * not containing a `/`.
+		 */
+		this.repoName = req.params.namespace
+			? `${req.params.namespace}/${req.params.repo}`
+			: req.params.repo
+		;
+		this.rev = req.params.rev ?? DEFAULT_BRANCH;
+		this.path = req.params[0];
+	}
+
+	async initialize(): Promise<void> {
+		try {
+			this.repo = await Git.Repository.openBare(`${Repo.ROOT_REPOS}/${this.repoName}.git`);
+		} catch {
+			throw new NotFoundError(`No such repository ${this.repoName}`);
+		}
+
+		try {
+			this.commit = await this.repo.getCommit(this.rev);
+		} catch {
+			try {
+				this.commit = await this.repo.getBranchCommit(this.rev);
+			} catch {
+				throw new NotFoundError(`Invalid rev id`);
+			}
+		};
+	}
+
+	/**
+	 * For the breadcrumbs
+	 */
+	get subpaths(): BreadcrumbPath[] | undefined {
+		if (! this.path) {
+			return undefined;
+		}
+		const parts = this.path.split('/');
+		return parts.map((dir, i) => {
+			const href = (i === parts.length - 1)
+				? undefined
+				: parts.slice(0, i+1).join('/')
+			;
+			return { dir, href };
+		});
+	}
+
+	/**
+	 * For links in templates (e.g. branch_selector)
+	 */
+	get view(): "tree" | "blob" | string {
+		throw new Error(`Implemented by concrete subclass`);
+	}
+}
+
+export class TreeContext extends Context {
+	tree: Git.Tree;
+
+	async initialize() {
+		await super.initialize();
+
+		if (this.path === undefined) {
+			/// "Root" tree
+			this.tree = await this.commit.getTree();
+		} else {
+			const commitTree = await this.commit.getTree();
+			const treeEntry = await commitTree.getEntry(this.path);
+			if (! treeEntry.isTree()) {
+				throw new NotFoundError(`No such tree ${this.path} in repository ${this.repoName}/${this.rev}`);
+			}
+			this.tree = await treeEntry.getTree();
+		}
+	}
+
+	get view() {
+		return `tree`;
+	}
+}
+
+export class BlobContext extends Context {
+	blob: Git.Blob;
+
+	async initialize() {
+		await super.initialize();
+
+		if (this.path === undefined) {
+			throw new NotFoundError(`Invalid blob, path is undefined`);
+		} else {
+			const commitTree = await this.commit.getTree();
+			const treeEntry = await commitTree.getEntry(this.path);
+			if (! treeEntry.isBlob()) {
+				throw new NotFoundError(`No such blob ${this.path} in repository ${this.repoName}/${this.rev}`);
+			}
+			this.blob = await treeEntry.getBlob();
+		}
+	}
+
+	get view() {
+		return `blob`;
+	}
+
+	get isBinary(): boolean {
+		return this.blob.isBinary() !== 0;
+	}
+	get isTooLarge(): boolean {
+		return this.blob.rawsize() > 10**9;
+	}
+}

node/app/Repo.ts

@@ -0,0 +1,50 @@
+import * as Git from 'nodegit';
+import * as path from 'path';
+import { c } from '../lib/Log';
+import __rootDir from '../lib/RootDirFinder';
+import { Utils } from '../lib/Utils';
+
+
+/**
+ * Helpers for our repos.
+ */
+export namespace Repo {
+	export const ROOT_REPOS = __rootDir+`/repositories`;
+
+	export function name(r: Git.Repository) {
+		return Utils.trimSuffix(path.relative(ROOT_REPOS, r.path()), '.git');
+	}
+
+	export async function refs(r: Git.Repository): Promise<{
+		tags: string[];
+		branches: string[];
+	}> {
+		const tags = await Git.Tag.list(r);
+		const branches = (await r.getReferences())
+			.filter(x => x.isBranch())
+			.map(x => x.shorthand())
+		;
+		return { tags, branches };
+	}
+
+	export async function numOfCommits(
+		repo: Git.Repository,
+		before: Git.Commit
+	): Promise<number> {
+		const revWalk = repo.createRevWalk();
+		revWalk.push(before.id());
+		let i = 1;
+		/// ^^ Include `before` in the count.
+		while (true) {
+			try {
+				await revWalk.next();
+				i++;
+			} catch(err) {
+				if (err.errno === Git.Error.CODE.ITEROVER) {
+					return i;
+				}
+				throw err;
+			}
+		}
+	}
+}

node/app/routes.ts

@@ -0,0 +1,116 @@
+import * as express from 'express';
+import * as Git from 'nodegit';
+import * as hljs from 'highlight.js';
+import { extname } from 'path';
+import { c } from '../lib/Log';
+import { Repo } from './Repo';
+import { Utils } from '../lib/Utils';
+import {
+	TreeContext,
+	BlobContext,
+	NotFoundError,
+} from './Context';
+
+
+
+
+
+export const getLastCommit = async (
+	repo: Git.Repository,
+	path: string,
+	before: Git.Commit,
+): Promise<Git.Commit | undefined> => {
+	const revWalk = repo.createRevWalk();
+	revWalk.push(before.id());
+	revWalk.sorting(Git.Revwalk.SORT.TIME);
+	const last = (await revWalk.fileHistoryWalk(path, 1_000))[0];
+	if (last && last.commit instanceof Git.Commit) {
+		return last.commit;
+	}
+	return undefined;
+};
+
+
+export const indexTree: express.RequestHandler = async function(req, res) {
+	const context = new TreeContext(req);
+	try {
+		await context.initialize();
+	} catch(err) {
+		if (err instanceof NotFoundError) {
+			return res.status(404).send(`Not Found: ${err}`);
+		}
+	}
+
+	const entries = context.tree.entries();
+
+	const commitsLast = await Promise.all(entries.map(async x => {
+		const l = await getLastCommit(context.repo, x.path(), context.commit);
+		(<any>x).lastCommit = l;
+		return l;
+	}));
+	const commitLast: Git.Commit | undefined = Utils.filterUndef(commitsLast)
+		.sort((a, b) => b.time() - a.time())[0]
+	;
+
+	if (context.path === undefined) {
+		const n = await Repo.numOfCommits(context.repo, context.commit);
+		context.data.historyLink = `History: ${n} commits`;
+	}
+
+	res.render('index_tree', {
+		refs: await Repo.refs(context.repo),
+		context,
+		commitLast,
+		dirs:  entries.filter(x => x.isTree()),
+		files: entries.filter(x => x.isBlob()),
+		layout: 'base',
+	});
+}
+
+export const indexBlob: express.RequestHandler = async function(req, res) {
+	const context = new BlobContext(req);
+	try {
+		await context.initialize();
+	} catch(err) {
+		if (err instanceof NotFoundError) {
+			return res.status(404).send(`Not Found: ${err}`);
+		}
+	}
+
+	const commitLast = await getLastCommit(context.repo, context.path!, context.commit);
+
+	if (!context.isBinary && !context.isTooLarge) {
+		const ext = Utils.trimPrefix(extname(context.path!), ".");
+		try {
+			const res = hljs.highlight(ext, context.blob.toString(), true);
+			context.data.code = res.value;
+		} catch {
+			/// Fallback to automatic detection.
+			const res = hljs.highlightAuto(context.blob.toString());
+			context.data.code = res.value;
+		}
+		c.debug(context.data.code);
+	}
+
+	res.render('index_blob', {
+		refs: await Repo.refs(context.repo),
+		context,
+		commitLast,
+		layout: 'base',
+	});
+};
+
+export const rawBlob: express.RequestHandler = async function(req, res) {
+	const context = new BlobContext(req);
+	try {
+		await context.initialize();
+	} catch(err) {
+		if (err instanceof NotFoundError) {
+			return res.status(404).send(`Not Found: ${err}`);
+		}
+	}
+	/// we don't really need to set a content-type.
+	const type = "text/plain";
+	res.set('content-type', type);
+	res.send(context.blob.content());
+};