Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[WIP] PoC for a Node.JS backend using nodegit #259

Draft
wants to merge 36 commits into
base: master
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from 14 commits
Commits
Show all changes
36 commits
Select commit Hold shift + click to select a range
6198242
Gitignore a few paths
julien-c Oct 8, 2020
83f7c72
(remove) simplest possible klaus.py server
julien-c Oct 8, 2020
b288c9f
[node] Lightweight scaffolding for an express app in Typescript
julien-c Oct 8, 2020
fb801c2
[node] PoC for tiny subset of features
julien-c Oct 8, 2020
2cac8fe
Name="klaus.py" for clarity
julien-c Oct 12, 2020
631de67
Handle nested repos
julien-c Oct 12, 2020
09237d8
cs(Add quotes around html attrs)
julien-c Oct 19, 2020
769bdd3
Update basic.py
julien-c Oct 19, 2020
cac1208
More complete PoC
julien-c Oct 20, 2020
70e96ed
blob view
julien-c Oct 20, 2020
3abb01f
Blame blob
julien-c Oct 20, 2020
12531a6
commit view
julien-c Oct 20, 2020
874ab4a
Better hljs integration + Fixup
julien-c Oct 20, 2020
8b376c4
Also support non-bare repos (very useful for debugging)
julien-c Oct 21, 2020
d271d9c
Link from commit to parent(s)
julien-c Oct 21, 2020
d1f8a85
Fixup h/t @pierrci
julien-c Oct 23, 2020
69323fa
upgrade deps
julien-c Oct 23, 2020
67ace25
Add readme
julien-c Oct 23, 2020
6eed736
Add helper button to sync all repos from origin
julien-c Oct 24, 2020
b327351
implement a "Check on GitHub" link that opens the "same" page on GitHub
julien-c Oct 24, 2020
94228dc
Support all DEFAULT_BRANCH natively
julien-c Oct 24, 2020
d39a77c
view_commit: display more info for dev purposes
julien-c Oct 24, 2020
f438804
Add missing tree/:rev route
julien-c Oct 24, 2020
c87919d
Tweaks
julien-c Oct 24, 2020
7702649
Tweak
julien-c Oct 24, 2020
a1f2e6f
even more tweaks
julien-c Oct 24, 2020
f3e85ad
cs
julien-c Oct 24, 2020
e3bbf49
commit history
julien-c Oct 24, 2020
a321038
Always keep the most generic routes at the end.
julien-c Oct 24, 2020
2688f86
Tweak to speed things up
julien-c Oct 24, 2020
b126751
mention tsc -w
julien-c Oct 26, 2020
26e700d
Example of how to setup source-map-support for TS
julien-c Oct 28, 2020
99b17e2
[/fetch_all] more robust error handling
julien-c Oct 28, 2020
9b5bbe4
Revert "Tweak to speed things up"
julien-c Oct 28, 2020
530007e
Improve perf of history(path)
julien-c Oct 28, 2020
ebcfcd4
Add perf note
julien-c Oct 28, 2020
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -1,10 +1,13 @@
*.pyc
/bin/
env/
.env/
.idea
*.swp
tests/repos/build
*.egg-info
build/
dist/
.vscode/
.DS_Store
/repositories
9 changes: 9 additions & 0 deletions basic.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
from klaus import Klaus

repos = [
"./repositories/huggingface/moon-landing.git",
"./repositories/transformers.git",
]

app = Klaus(repos, "klaus.py", use_smarthttp=False)
app.run(debug=True)
2 changes: 1 addition & 1 deletion klaus/static/klaus.css
Original file line number Diff line number Diff line change
Expand Up @@ -159,7 +159,7 @@ footer a:hover { text-decoration: none; }
a.commit { color: black !important; }

.tree ul { font-family: monospace; border-top: 1px solid #e0e0e0; }
.tree li { background-color: #f9f9f9; border: 1px solid #e0e0e0; border-top: 0; }
.tree li { background-color: white; border: 1px solid #e0e0e0; border-top: 0; }
.tree li a {
padding: 5px 7px 6px 7px;
display: block;
Expand Down
2 changes: 2 additions & 0 deletions node/.gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
node_modules/
dist/
183 changes: 183 additions & 0 deletions node/app/Context.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,183 @@
import * as express from 'express';
import * as Git from 'nodegit';
import * as hljs from 'highlight.js';
import { extname } from 'path';
import { c } from '../lib/Log';
import { Repo } from './Repo';
import { Utils } from '../lib/Utils';


const DEFAULT_BRANCH = `master`;
julien-c marked this conversation as resolved.
Show resolved Hide resolved

interface BreadcrumbPath {
dir: string;
href?: string;
}

export class NotFoundError extends Error {}


const repoNameFromRequest = (req: express.Request) => {
return req.params.namespace
? `${req.params.namespace}/${req.params.repo}`
: req.params.repo
;
}


export class Context {
/**
* Repo id (repo or user/repo)
*/
repoName: string;
/**
* Branch/commit-sha/tag
*/
rev: string;
/**
* path is undefined for repo's root
* (only makes sense for tree)
*/
path?: string;

/// After `.initialize()`
repo: Git.Repository;
commit: Git.Commit;
/// Other ad hoc data, for convenient access from templates
data: Record<string, any> = {};

constructor(req: express.Request) {
/**
* Note: we only support branch names and tag names
* not containing a `/`.
*/
this.repoName = repoNameFromRequest(req);
this.rev = req.params.rev ?? DEFAULT_BRANCH;
this.path = req.params[0];
}

async initialize(): Promise<void> {
if (await Utils.fileExists(`${Repo.ROOT_REPOS}/${this.repoName}.git`)) {
/// bare repo
this.repo = await Git.Repository.openBare(`${Repo.ROOT_REPOS}/${this.repoName}.git`);
} else if (await Utils.fileExists(`${Repo.ROOT_REPOS}/${this.repoName}/.git`)) {
/// non-bare repo
this.repo = await Git.Repository.open(`${Repo.ROOT_REPOS}/${this.repoName}/.git`);
} else {
throw new NotFoundError(`No such repository ${this.repoName}`);
}

try {
this.commit = await this.repo.getCommit(this.rev);
} catch {
try {
this.commit = await this.repo.getBranchCommit(this.rev);
} catch {
throw new NotFoundError(`Invalid rev id`);
}
};
}

/**
* For the breadcrumbs
*/
get subpaths(): BreadcrumbPath[] | undefined {
if (! this.path) {
return undefined;
}
const parts = this.path.split('/');
return parts.map((dir, i) => {
const href = (i === parts.length - 1)
? undefined
: parts.slice(0, i+1).join('/')
;
return { dir, href };
});
}

/**
* For links in templates (e.g. branch_selector)
*/
get view(): "tree" | "blob" | string {
throw new Error(`Implemented by concrete subclass`);
}
}


export class CommitContext extends Context {}


export class TreeContext extends Context {
tree: Git.Tree;

async initialize() {
await super.initialize();

if (this.path === undefined) {
/// "Root" tree
this.tree = await this.commit.getTree();
} else {
const commitTree = await this.commit.getTree();
const treeEntry = await commitTree.getEntry(this.path);
if (! treeEntry.isTree()) {
throw new NotFoundError(`No such tree ${this.path} in repository ${this.repoName}/${this.rev}`);
}
this.tree = await treeEntry.getTree();
}
}

get view() {
return `tree`;
}
}


export class BlobContext extends Context {
blob: Git.Blob;

async initialize() {
await super.initialize();

if (this.path === undefined) {
throw new NotFoundError(`Invalid blob, path is undefined`);
} else {
const commitTree = await this.commit.getTree();
const treeEntry = await commitTree.getEntry(this.path);
if (! treeEntry.isBlob()) {
throw new NotFoundError(`No such blob ${this.path} in repository ${this.repoName}/${this.rev}`);
}
this.blob = await treeEntry.getBlob();
julien-c marked this conversation as resolved.
Show resolved Hide resolved
}
}

get view() {
return `blob`;
}

get isBinary(): boolean {
return this.blob.isBinary() !== 0;
}
get isTooLarge(): boolean {
return this.blob.rawsize() > 10**9;
}

/**
* Highlight content with hljs,
* and store in this.data.
*/
renderText() {
const ext = Utils.trimPrefix(extname(this.path!), ".");
const str = this.blob.toString();
if (hljs.getLanguage(ext)) {
const res = hljs.highlight(ext, str, true);
this.data.code = res.value;
} else {
/// Fallback to automatic detection.
const res = hljs.highlightAuto(str);
this.data.code = res.value;
}
const n_lines = str.split('\n').length;
this.data.line_gutter = Utils.range(1, n_lines+1).join("\n");
}
}

56 changes: 56 additions & 0 deletions node/app/Repo.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
import * as Git from 'nodegit';
import * as path from 'path';
import { c } from '../lib/Log';
import __rootDir from '../lib/RootDirFinder';
import { Utils } from '../lib/Utils';


/**
* Helpers for our repos.
*/
export namespace Repo {
export const ROOT_REPOS = __rootDir+`/repositories`;

export function name(repo: Git.Repository) {
const rel = path.relative(ROOT_REPOS, repo.path());
if (rel.endsWith(`/.git`)) {
/// non-bare repo
return Utils.trimSuffix(rel, `/.git`);
} else {
/// bare repo
return Utils.trimSuffix(rel, '.git');
}
}

export async function refs(repo: Git.Repository): Promise<{
tags: string[];
branches: string[];
}> {
const tags = await Git.Tag.list(repo);
const branches = (await repo.getReferences())
.filter(x => x.isBranch())
.map(x => x.shorthand())
;
return { tags, branches };
}

export async function numOfCommits(
repo: Git.Repository,
before: Git.Commit
): Promise<number> {
const revWalk = repo.createRevWalk();
revWalk.push(before.id());
let i = 0;
/// ^^ Include `before` in the count.
while (true) {
try {
i++;
} catch(err) {
if (err.errno === Git.Error.CODE.ITEROVER) {
return i;
}
throw err;
}
}
}
}
Loading