port to Cloudflare Worker

README.md

@@ -41,6 +41,13 @@ In `tools/` run `npm install`.
 bash ./scripts/test-process-version.sh package-name package-version
 ```
 
+### Compile for Cloudflare Workers
+
+```
+cd ./worker
+GOOS=js GOARCH=wasm tinygo build -o wasm.wasm ./main.go
+```
+
 ## License
 
 Each library hosted on cdnjs is released under its own license. This cdnjs repository is published under [MIT license](LICENSE).

algolia/index.go

@@ -55,7 +55,8 @@ func getAlternativeNames(name string) []string {
 	return names
 }
 
-var githubURL = regexp.MustCompile(`github\.com[/|:]([\w\.-]+)\/([\w\.-]+)\/?`)
+// var githubURL = regexp.MustCompile(`github\.com[/|:]([\w\.-]+)\/([\w\.-]+)\/?`)
+var githubURL = regexp.MustCompile(`github\.com/.*`)
 
 func getGitHubMeta(repo *packages.Repository) (*GitHubMeta, error) {
 	if repo == nil {

functions/algolia-pump/main.go → functions/algolia-pump/algolia_pump.go

@@ -17,10 +17,10 @@ import (
 
 	"github.com/cdnjs/tools/algolia"
 	"github.com/cdnjs/tools/audit"
-	"github.com/cdnjs/tools/gcp"
 	"github.com/cdnjs/tools/kv"
 	"github.com/cdnjs/tools/metrics"
 	"github.com/cdnjs/tools/packages"
+	"github.com/cdnjs/tools/r2"
 	"github.com/cdnjs/tools/sentry"
 )
 
@@ -45,17 +45,14 @@ func getExistingVersionsFromAggregatedMetadata(p *packages.Package) ([]string, e
 	return versions, nil
 }
 
-func Invoke(ctx context.Context, e gcp.GCSEvent) error {
+func Run(ctx context.Context, bucket string, file string, pkgName string, currVersion string, config string) error {
 	sentry.Init()
 	defer sentry.PanicHandler()
 
-	log.Printf("File: %v\n", e.Name)
-	log.Printf("Metadata: %v\n", e.Metadata)
+	log.Printf("File: %v\n", file)
+	log.Printf("Config: %v\n", config)
 
-	pkgName := e.Metadata["package"].(string)
-	currVersion := e.Metadata["version"].(string)
-
-	configStr, err := b64.StdEncoding.DecodeString(e.Metadata["config"].(string))
+	configStr, err := b64.StdEncoding.DecodeString(config)
 	if err != nil {
 		return fmt.Errorf("could not decode config: %v", err)
 	}
@@ -76,7 +73,7 @@ func Invoke(ctx context.Context, e gcp.GCSEvent) error {
 	if err != nil {
 		return fmt.Errorf("failed to retrieve existing versions: %s", err)
 	}
-	archive, err := gcp.ReadObject(ctx, e.Bucket, e.Name)
+	archive, err := r2.ReadObject(ctx, bucket, file)
 	if err != nil {
 		return fmt.Errorf("could not read object: %v", err)
 	}
@@ -102,7 +99,7 @@ func Invoke(ctx context.Context, e gcp.GCSEvent) error {
 		}
 		return nil
 	}
-	if err := gcp.Inflate(bytes.NewReader(archive), onFile); err != nil {
+	if err := r2.Inflate(bytes.NewReader(archive), onFile); err != nil {
 		return fmt.Errorf("could not inflate archive: %s", err)
 	}
 

gcp/gcp.go

@@ -1,25 +1,25 @@
 package gcp
 
-import (
-	"os"
-	"path"
+// import (
+// 	"os"
+// 	"path"
 
-	"github.com/cdnjs/tools/util"
+// 	"github.com/cdnjs/tools/util"
 
-	"golang.org/x/net/context"
-	"google.golang.org/api/option"
+// 	"golang.org/x/net/context"
+// 	"google.golang.org/api/option"
 
-	"cloud.google.com/go/storage"
-)
+// 	"cloud.google.com/go/storage"
+// )
 
-func getCredentialsFile() string {
-	home, err := os.UserHomeDir()
-	util.Check(err)
+// func getCredentialsFile() string {
+// 	home, err := os.UserHomeDir()
+// 	util.Check(err)
 
-	return path.Join(home, "google_storage_cdnjs_assets.json")
-}
+// 	return path.Join(home, "google_storage_cdnjs_assets.json")
+// }
 
-// GetStorageClient gets the GCP Storage Client.
-func GetStorageClient(ctx context.Context) (*storage.Client, error) {
-	return storage.NewClient(ctx, option.WithCredentialsFile(getCredentialsFile()))
-}
+// // GetStorageClient gets the GCP Storage Client.
+// func GetStorageClient(ctx context.Context) (*storage.Client, error) {
+// 	return storage.NewClient(ctx, option.WithCredentialsFile(getCredentialsFile()))
+// }

gcp/gcs.go

@@ -2,41 +2,38 @@ package gcp
 
 import (
 	"archive/tar"
-	"bufio"
-	"bytes"
 	"compress/gzip"
 	"context"
 	"io"
 	"time"
 
 	"github.com/pkg/errors"
-
-	"cloud.google.com/go/storage"
 )
 
 func ReadObject(ctx context.Context, bucket string, name string) ([]byte, error) {
-	client, err := storage.NewClient(ctx)
-	if err != nil {
-		return nil, errors.Wrap(err, "could not create client")
-	}
+	return nil, nil
+	// client, err := storage.NewClient(ctx)
+	// if err != nil {
+	// 	return nil, errors.Wrap(err, "could not create client")
+	// }
 
-	bkt := client.Bucket(bucket)
-	obj := bkt.Object(name)
+	// bkt := client.Bucket(bucket)
+	// obj := bkt.Object(name)
 
-	r, err := obj.NewReader(ctx)
-	if err != nil {
-		return nil, errors.Wrap(err, "could not create client")
-	}
-	defer r.Close()
+	// r, err := obj.NewReader(ctx)
+	// if err != nil {
+	// 	return nil, errors.Wrap(err, "could not create client")
+	// }
+	// defer r.Close()
 
-	var buff bytes.Buffer
-	w := bufio.NewWriter(&buff)
+	// var buff bytes.Buffer
+	// w := bufio.NewWriter(&buff)
 
-	if _, err := io.Copy(w, r); err != nil {
-		return nil, errors.Wrap(err, "could not read object")
-	}
+	// if _, err := io.Copy(w, r); err != nil {
+	// 	return nil, errors.Wrap(err, "could not read object")
+	// }
 
-	return buff.Bytes(), nil
+	// return buff.Bytes(), nil
 }
 
 // GCSEvent is the payload of a GCS event.

gcp/incoming.go

@@ -2,61 +2,55 @@ package gcp
 
 import (
 	"bytes"
-	"context"
-	"encoding/json"
-	"fmt"
-	"io"
 	"os"
 
 	"github.com/cdnjs/tools/packages"
 	"github.com/cdnjs/tools/version"
-
-	"cloud.google.com/go/storage"
-	"github.com/pkg/errors"
+	// "cloud.google.com/go/storage"
 )
 
 var (
 	GCS_BUCKET = os.Getenv("GCS_BUCKET")
 )
 
 func AddIncomingFile(fileName string, buff bytes.Buffer, pckg *packages.Package, v version.Version) error {
-	// Create GCS connection
-	ctx := context.Background()
-	client, err := storage.NewClient(ctx)
-	if err != nil {
-		return fmt.Errorf("HTTP response error: %v", err)
-	}
-
-	bucket := client.Bucket(GCS_BUCKET)
-	obj := bucket.Object(fileName)
-	w := obj.NewWriter(ctx)
-	w.ACL = []storage.ACLRule{
-		{Entity: storage.AllUsers, Role: storage.RoleReader},
-	}
-
-	if _, err := io.Copy(w, bytes.NewReader(buff.Bytes())); err != nil {
-		return fmt.Errorf("Failed to copy to bucket: %v", err)
-	}
-	if err := w.Close(); err != nil {
-		return fmt.Errorf("Failed to close: %v", err)
-	}
-
-	configBytes, err := json.Marshal(pckg)
-	if err != nil {
-		return fmt.Errorf("failed to marshal filemap: %v", err)
-	}
-
-	// update the metadata once the object is written
-	_, err = obj.Update(ctx, storage.ObjectAttrsToUpdate{
-		Metadata: map[string]string{
-			"version": v.Version,
-			"package": *pckg.Name,
-			"config":  string(configBytes),
-		},
-	})
-	if err != nil {
-		return errors.Wrap(err, "could not update metadata")
-	}
+	// // Create GCS connection
+	// ctx := context.Background()
+	// client, err := storage.NewClient(ctx)
+	// if err != nil {
+	// 	return fmt.Errorf("HTTP response error: %v", err)
+	// }
+
+	// bucket := client.Bucket(GCS_BUCKET)
+	// obj := bucket.Object(fileName)
+	// w := obj.NewWriter(ctx)
+	// w.ACL = []storage.ACLRule{
+	// 	{Entity: storage.AllUsers, Role: storage.RoleReader},
+	// }
+
+	// if _, err := io.Copy(w, bytes.NewReader(buff.Bytes())); err != nil {
+	// 	return fmt.Errorf("Failed to copy to bucket: %v", err)
+	// }
+	// if err := w.Close(); err != nil {
+	// 	return fmt.Errorf("Failed to close: %v", err)
+	// }
+
+	// configBytes, err := json.Marshal(pckg)
+	// if err != nil {
+	// 	return fmt.Errorf("failed to marshal filemap: %v", err)
+	// }
+
+	// // update the metadata once the object is written
+	// _, err = obj.Update(ctx, storage.ObjectAttrsToUpdate{
+	// 	Metadata: map[string]string{
+	// 		"version": v.Version,
+	// 		"package": *pckg.Name,
+	// 		"config":  string(configBytes),
+	// 	},
+	// })
+	// if err != nil {
+	// 	return errors.Wrap(err, "could not update metadata")
+	// }
 
 	return nil
 }

git/github.go

@@ -37,7 +37,8 @@ func getRepo(gitURL string) string {
 	// "[email protected]:chris-pearce/backpack.css.git"
 	// "git+https://github.com/18F/web-design-standards.git"
 	// "https://github.com/epeli/underscore.string"
-	re := regexp.MustCompile(`.*github\.com[:|/](.*?)(?:\.git)?$`)
+	// re := regexp.MustCompile(`.*github\.com[:|/](.*?)(?:\.git)?$`)
+	re := regexp.MustCompile(`.*github\.com/.*`)
 	return re.ReplaceAllString(gitURL, "$1")
 }
 

r2/r2.go

@@ -0,0 +1,50 @@
+package r2
+
+import (
+	"archive/tar"
+	"compress/gzip"
+	"context"
+	"io"
+
+	"github.com/pkg/errors"
+)
+
+func ReadObject(ctx context.Context, bucket string, name string) ([]byte, error) {
+	panic("TODO")
+}
+
+func Inflate(gzipStream io.Reader, onFile func(string, io.Reader) error) error {
+	uncompressedStream, err := gzip.NewReader(gzipStream)
+	if err != nil {
+		return errors.Wrap(err, "ExtractTarGz: NewReader failed")
+	}
+
+	tarReader := tar.NewReader(uncompressedStream)
+
+	for {
+		header, err := tarReader.Next()
+
+		if err == io.EOF {
+			break
+		}
+
+		if err != nil {
+			return errors.Wrap(err, "ExtractTarGz: Next() failed")
+		}
+
+		switch header.Typeflag {
+		case tar.TypeDir:
+			// do nothing
+		case tar.TypeReg:
+			if err := onFile(header.Name, tarReader); err != nil {
+				return errors.Wrap(err, "failed to handle file")
+			}
+		default:
+			return errors.Errorf(
+				"ExtractTarGz: uknown type: %x in %s",
+				header.Typeflag,
+				header.Name)
+		}
+	}
+	return nil
+}

sandbox/sandbox.go

@@ -16,11 +16,6 @@ import (
 	"github.com/pkg/errors"
 )
 
-var (
-	DOCKER_IMAGE      = os.Getenv("DOCKER_IMAGE")
-	CONTAINER_NAME_RE = regexp.MustCompile(`[^a-zA-Z0-9-_]+`)
-)
-
 func Setup() (string, string, error) {
 	tmpDir := os.TempDir()
 	inDir, err := ioutil.TempDir(tmpDir, "in")
@@ -36,6 +31,7 @@ func Setup() (string, string, error) {
 }
 
 func Init(ctx context.Context) error {
+	DOCKER_IMAGE = os.Getenv("DOCKER_IMAGE")
 	if DOCKER_IMAGE == "" {
 		return errors.New("DOCKER_IMAGE needs to be present")
 	}
@@ -72,8 +68,10 @@ func Run(ctx context.Context, containerName, in, out string) (string, error) {
 
 	// Sanitize the container's name because some package use special character
 	// in their versions and Docker doesn't accept that.
+	CONTAINER_NAME_RE = regexp.MustCompile(`[^a-zA-Z0-9-_]+`)
 	containerName = CONTAINER_NAME_RE.ReplaceAllString(containerName, "-")
 
+	DOCKER_IMAGE = os.Getenv("DOCKER_IMAGE")
 	resp, err := cli.ContainerCreate(ctx,
 		&container.Config{
 			Image: DOCKER_IMAGE,