Merge pull request #278 from cdnjs/sven/remove-r2-file-ext

r2-pump: remove FILE_EXTENSION
cdnjs · Feb 5, 2024 · 7f30c03 · 7f30c03
2 parents 3d32735 + 7ec0838
commit 7f30c03
Show file tree

Hide file tree

Showing 2 changed files with 11 additions and 21 deletions.
diff --git a/audit/audit.go b/audit/audit.go
@@ -187,14 +187,14 @@ func WroteAlgolia(ctx context.Context, pkgName string, currVersion string, lastV
 	return nil
 }
 
-func WroteR2(ctx context.Context, pkgName string, version string, keys []string, ext string) error {
+func WroteR2(ctx context.Context, pkgName string, version string, keys []string) error {
 	content := bytes.NewBufferString("")
 	fmt.Fprint(content, "Files:\n")
 	for _, key := range keys {
 		fmt.Fprintf(content, "- %s\n", key)
 	}
 
-	if err := create(ctx, pkgName, version, "r2-publish/"+ext, content); err != nil {
+	if err := create(ctx, pkgName, version, "r2-publish", content); err != nil {
 		return errors.Wrap(err, "could not create audit log file")
 	}
 	return nil

diff --git a/functions/r2-pump-http/main.go b/functions/r2-pump-http/main.go
@@ -11,7 +11,6 @@ import (
 	"log"
 	"net/http"
 	"os"
-	"path/filepath"
 	"strconv"
 	"time"
 
@@ -34,12 +33,6 @@ var (
 	R2_KEY_SECRET = os.Getenv("R2_KEY_SECRET")
 	R2_ENDPOINT   = os.Getenv("R2_ENDPOINT")
 
-	// In an attempt to spread the load across multiple gcp functions, we split
-	// the upload per file extension (either gz, br or woff2). There should
-	// be 50/50 for gz and br. Unknown for woff2.
-	// Example: FILE_EXTENSION=gz
-	FILE_EXTENSION = os.Getenv("FILE_EXTENSION")
-
 	UPLOAD_CONCURENCY = os.Getenv("UPLOAD_CONCURENCY")
 )
 
@@ -97,7 +90,6 @@ func InvokeInner(d InvokePayload, ctx context.Context) error {
 	}
 
 	onFile := func(name string, r io.Reader) error {
-		ext := filepath.Ext(name)
 		// remove leading slash
 		name = name[1:]
 		key := fmt.Sprintf("%s/%s/%s", pkgName, version, name)
@@ -107,19 +99,17 @@ func InvokeInner(d InvokePayload, ctx context.Context) error {
 			return errors.Wrap(err, "could not read file")
 		}
 
-		if ext == "."+FILE_EXTENSION {
-			keys = append(keys, key)
+		keys = append(keys, key)
 
-			meta := newMetadata(len(content))
+		meta := newMetadata(len(content))
 
-			s3Object := s3.PutObjectInput{
-				Body:     bytes.NewReader(content),
-				Bucket:   bucket,
-				Key:      aws.String(key),
-				Metadata: meta,
-			}
-			uploadQueue <- s3Object
+		s3Object := s3.PutObjectInput{
+			Body:     bytes.NewReader(content),
+			Bucket:   bucket,
+			Key:      aws.String(key),
+			Metadata: meta,
 		}
+		uploadQueue <- s3Object
 		return nil
 	}
 	if err := gcp.Inflate(bytes.NewReader(archive), onFile); err != nil {
@@ -136,7 +126,7 @@ func InvokeInner(d InvokePayload, ctx context.Context) error {
 		return fmt.Errorf("failed to parse config: %s", err)
 	}
 
-	if err := audit.WroteR2(ctx, pkgName, version, keys, FILE_EXTENSION); err != nil {
+	if err := audit.WroteR2(ctx, pkgName, version, keys); err != nil {
 		log.Printf("failed to audit: %s\n", err)
 	}
 	if err := metrics.NewUpdatePublishedR2(); err != nil {