Download preview workspace over webrtc

.github/workflows/preview.yaml

@@ -3,8 +3,8 @@ name: Speculative Run
 on:
   workflow_dispatch:
     inputs:
-      workspace_transfer_url:
-        description: "URL from which to download the workspace"
+      webrtc-session:
+        description: "Session (base64 encoded) for the workspace download over webrtc"
         required: true
         type: string
 
@@ -16,11 +16,16 @@ jobs:
       TF_HTTP_PASSWORD: ${{ github.token }}
       TF_IN_AUTOMATION: "true"
     steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
       - name: Download Workspace
         run: |
-          curl ${{ inputs.workspace_transfer_url }} --fail --silent | tar -xzf -
-      - uses: hashicorp/setup-terraform@v3
-        with:
-          terraform_version: "1.6.6"
-      - run: terraform init
-      - run: terraform plan
+          go run ./cmd/webrtc-downloader -session ${{ inputs.webrtc-session }}
+      - run: tar -ztvf workspace.tar.gz
+      # - uses: hashicorp/setup-terraform@v3
+      #   with:
+      #     terraform_version: "1.6.6"
+      # - run: terraform init
+      # - run: terraform plan

.terraformignore

@@ -1,4 +1,5 @@
 /.github/
 /cmd/
+/pkg/
 /go.*
 *.tar.gz

cmd/tf-preview-gh/main.go

@@ -17,77 +17,10 @@ import (
 	"time"
 
 	"github.com/cenkalti/backoff"
+	"github.com/ffddorf/tf-preview-github/pkg/quicpunch"
 	"github.com/google/go-github/v57/github"
-	"github.com/hashicorp/go-slug"
-	"golang.ngrok.com/ngrok"
-	"golang.ngrok.com/ngrok/config"
 )
 
-type LocalContent struct {
-	dir string
-}
-
-func (c *LocalContent) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
-	rw.Header().Set("Content-Type", "application/octet-stream")
-
-	_, err := slug.Pack(c.dir, rw, true)
-	if err != nil {
-		fmt.Printf("failed to pack contents: %+v\n", err)
-		return
-	}
-
-	fmt.Println("workspace was downloaded")
-}
-
-func startServer(ctx context.Context) (string, error) {
-	listenerCtx, cancelListener := context.WithCancel(context.Background())
-
-	connected := make(chan struct{})
-	go func() {
-		select {
-		case <-connected:
-		case <-time.After(10 * time.Second):
-			cancelListener()
-		}
-	}()
-
-	listener, err := ngrok.Listen(listenerCtx, config.HTTPEndpoint(), ngrok.WithAuthtokenFromEnv())
-	if err != nil {
-		return "", err
-	}
-	close(connected)
-
-	cwd, err := os.Getwd()
-	if err != nil {
-		listener.Close()
-		return "", err
-	}
-	handler := &LocalContent{
-		dir: cwd,
-	}
-
-	server := &http.Server{
-		Handler: handler,
-	}
-
-	go func() {
-		<-ctx.Done()
-		shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-		defer cancel()
-		if err := server.Shutdown(shutdownCtx); err != nil {
-			fmt.Printf("failed to shutdown server: %+v\n", err)
-		}
-	}()
-
-	go func() {
-		if err := server.Serve(listener); err != http.ErrServerClosed {
-			fmt.Printf("server failed: %+v\n", err)
-		}
-	}()
-
-	return listener.URL(), nil
-}
-
 type countingReader struct {
 	io.Reader
 	readBytes int
@@ -152,6 +85,7 @@ func streamLogs(logsURL *url.URL, skip int64) (int64, error) {
 var (
 	owner            string
 	repo             string
+	workflowRef      string
 	workflowFilename string
 )
 
@@ -162,16 +96,28 @@ func main() {
 	flag.StringVar(&owner, "github-owner", "ffddorf", "Repository owner")
 	flag.StringVar(&repo, "github-repo", "", "Repository name")
 	flag.StringVar(&workflowFilename, "workflow-file", "preview.yaml", "Name of the workflow file to run for previews")
+	flag.StringVar(&workflowRef, "workflow-ref", "main", "Ref to run the workflow from")
 	flag.Parse()
 
 	if repo == "" {
 		panic("Missing flag: -github-repo")
 	}
 
-	serverURL, err := startServer(ctx)
+	cwd, err := os.Getwd()
+	if err != nil {
+		panic(err)
+	}
+
+	sess, serve, err := quicpunch.ServeWorkspace(ctx, cwd)
 	if err != nil {
 		panic(err)
 	}
+	go func() {
+		err := serve(ctx)
+		if err != nil {
+			fmt.Printf("workspace server failed: %v", err)
+		}
+	}()
 
 	// steal token from GH CLI
 	cmd := exec.CommandContext(ctx, "gh", "auth", "token")
@@ -189,9 +135,9 @@ func main() {
 	_, err = gh.Actions.CreateWorkflowDispatchEventByFileName(ctx,
 		owner, repo, workflowFilename,
 		github.CreateWorkflowDispatchEventRequest{
-			Ref: "main",
+			Ref: workflowRef,
 			Inputs: map[string]interface{}{
-				"workspace_transfer_url": serverURL,
+				"webrtc-session": sess,
 			},
 		},
 	)
@@ -208,6 +154,7 @@ func main() {
 			ctx, owner, repo, workflowFilename,
 			&github.ListWorkflowRunsOptions{
 				Created: fmt.Sprintf(">=%s", startedAt.Format("2006-01-02T15:04")),
+				Branch:  workflowRef,
 			},
 		)
 		if err != nil {

cmd/webrtc-downloader/main.go

@@ -0,0 +1,31 @@
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"os"
+	"os/signal"
+	"syscall"
+
+	"github.com/ffddorf/tf-preview-github/pkg/quicpunch"
+)
+
+func main() {
+	var session string
+	flag.StringVar(&session, "session", "", "WebRTC session (base64)")
+	flag.Parse()
+
+	ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGTERM, syscall.SIGHUP)
+	defer cancel()
+
+	f, err := os.Create("workspace.tar.gz")
+	if err != nil {
+		panic(err)
+	}
+	defer f.Close()
+	if err := quicpunch.FetchWorkspace(ctx, session, f); err != nil {
+		panic(err)
+	}
+	fmt.Println("Done downloading")
+}

go.mod

@@ -5,27 +5,36 @@ go 1.21.5
 require (
 	github.com/cenkalti/backoff v2.2.1+incompatible
 	github.com/google/go-github/v57 v57.0.0
-	github.com/hashicorp/go-slug v0.13.3
-	github.com/stretchr/testify v1.8.4
-	golang.ngrok.com/ngrok v1.7.0
+	github.com/hashicorp/go-slug v0.14.0
+	github.com/pion/ice/v2 v2.3.13
+	github.com/pion/stun v0.6.1
+	github.com/quic-go/quic-go v0.41.0
 )
 
 require (
 	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/go-stack/stack v1.8.1 // indirect
+	github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect
 	github.com/google/go-querystring v1.1.0 // indirect
-	github.com/inconshreveable/log15 v3.0.0-testing.5+incompatible // indirect
-	github.com/inconshreveable/log15/v3 v3.0.0-testing.5 // indirect
-	github.com/jpillora/backoff v1.0.0 // indirect
-	github.com/mattn/go-colorable v0.1.13 // indirect
-	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/google/pprof v0.0.0-20240207164012-fb44976bdcd5 // indirect
+	github.com/google/uuid v1.6.0 // indirect
+	github.com/onsi/ginkgo/v2 v2.15.0 // indirect
+	github.com/pion/dtls/v2 v2.2.10 // indirect
+	github.com/pion/logging v0.2.2 // indirect
+	github.com/pion/mdns v0.0.12 // indirect
+	github.com/pion/randutil v0.1.0 // indirect
+	github.com/pion/transport/v2 v2.2.4 // indirect
+	github.com/pion/turn/v2 v2.1.5 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	go.uber.org/multierr v1.11.0 // indirect
-	golang.ngrok.com/muxado/v2 v2.0.0 // indirect
-	golang.org/x/net v0.19.0 // indirect
-	golang.org/x/sync v0.6.0 // indirect
-	golang.org/x/sys v0.16.0 // indirect
-	golang.org/x/term v0.16.0 // indirect
+	github.com/quic-go/qpack v0.4.0 // indirect
+	github.com/stretchr/testify v1.8.4 // indirect
+	go.uber.org/mock v0.4.0 // indirect
+	golang.org/x/crypto v0.19.0 // indirect
+	golang.org/x/exp v0.0.0-20240205201215-2c58cdc269a3 // indirect
+	golang.org/x/mod v0.15.0 // indirect
+	golang.org/x/net v0.21.0 // indirect
+	golang.org/x/sys v0.17.0 // indirect
+	golang.org/x/text v0.14.0 // indirect
+	golang.org/x/tools v0.17.0 // indirect
 	google.golang.org/protobuf v1.32.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )