feat: enhanced extension engine (#629)

internal/protocol/http/fetcher.go

@@ -5,12 +5,6 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"github.com/GopeedLab/gopeed/internal/controller"
-	"github.com/GopeedLab/gopeed/internal/fetcher"
-	"github.com/GopeedLab/gopeed/pkg/base"
-	fhttp "github.com/GopeedLab/gopeed/pkg/protocol/http"
-	"github.com/xiaoqidun/setft"
-	"golang.org/x/sync/errgroup"
 	"io"
 	"mime"
 	"net/http"
@@ -21,6 +15,13 @@ import (
 	"strconv"
 	"strings"
 	"time"
+
+	"github.com/GopeedLab/gopeed/internal/controller"
+	"github.com/GopeedLab/gopeed/internal/fetcher"
+	"github.com/GopeedLab/gopeed/pkg/base"
+	fhttp "github.com/GopeedLab/gopeed/pkg/protocol/http"
+	"github.com/xiaoqidun/setft"
+	"golang.org/x/sync/errgroup"
 )
 
 type RequestError struct {
@@ -376,7 +377,8 @@ func (f *Fetcher) buildRequest(ctx context.Context, req *base.Request) (httpReq
 		method string
 		body   io.Reader
 	)
-	headers := make(map[string][]string)
+
+	headers := http.Header{}
 	if req.Extra == nil {
 		method = http.MethodGet
 	} else {
@@ -388,16 +390,15 @@ func (f *Fetcher) buildRequest(ctx context.Context, req *base.Request) (httpReq
 		}
 		if len(extra.Header) > 0 {
 			for k, v := range extra.Header {
-				headers[k] = []string{v}
+				headers.Set(k, v)
 			}
 		}
 		if extra.Body != "" {
 			body = bytes.NewBufferString(extra.Body)
 		}
 	}
-	if v := headers[base.HttpHeaderUserAgent]; len(v) == 0 || v[0] == "" {
-		// load user agent from config
-		headers[base.HttpHeaderUserAgent] = []string{f.config.UserAgent}
+	if _, ok := headers[base.HttpHeaderUserAgent]; !ok {
+		headers.Set(base.HttpHeaderUserAgent, f.config.UserAgent)
 	}
 
 	if ctx != nil {

pkg/download/engine/engine_test.go

@@ -6,26 +6,29 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"github.com/GopeedLab/gopeed/internal/test"
-	"github.com/GopeedLab/gopeed/pkg/base"
-	gojaerror "github.com/GopeedLab/gopeed/pkg/download/engine/inject/error"
-	"github.com/GopeedLab/gopeed/pkg/download/engine/inject/file"
-	gojautil "github.com/GopeedLab/gopeed/pkg/download/engine/util"
-	"github.com/dop251/goja"
 	"io"
 	"net"
 	"net/http"
 	"strconv"
 	"strings"
 	"testing"
 	"time"
+
+	"github.com/GopeedLab/gopeed/internal/test"
+	"github.com/GopeedLab/gopeed/pkg/base"
+	gojaerror "github.com/GopeedLab/gopeed/pkg/download/engine/inject/error"
+	"github.com/GopeedLab/gopeed/pkg/download/engine/inject/file"
+	gojautil "github.com/GopeedLab/gopeed/pkg/download/engine/util"
+	"github.com/dop251/goja"
 )
 
 func TestPolyfill(t *testing.T) {
 	doTestPolyfill(t, "MessageError")
 	doTestPolyfill(t, "XMLHttpRequest")
 	doTestPolyfill(t, "Blob")
 	doTestPolyfill(t, "FormData")
+	doTestPolyfill(t, "TextDecoder")
+	doTestPolyfill(t, "TextEncoder")
 	doTestPolyfill(t, "fetch")
 	doTestPolyfill(t, "__gopeed_create_vm")
 }
@@ -69,6 +72,44 @@ async function testOctetStream(file){
 	return await resp.text();
 }
 
+async function testRedirect() {
+    const url = host + '/redirect?num=3'
+    return await new Promise((resolve, reject) => {
+        fetch(url, {
+            method: 'HEAD',
+            redirect: 'error',
+        }).then(()=>reject())
+
+        fetch(url, {
+            method: 'HEAD',
+            redirect: 'follow',
+        }).then((res) =>res.headers.has('location') && reject()).catch(() => reject())
+
+        fetch(url, {
+            method: 'HEAD',
+            redirect: 'manual',
+        }).then((res) => {
+			const location = res.headers.get('location');
+			location ? resolve(location) : reject()
+        }).catch(() => reject())
+    })
+}
+
+async function testResponseUrl() {
+    return new Promise((resolve, reject) => {
+		const xhr = new XMLHttpRequest();
+		xhr.open('GET', host+'/redirect?num=3');
+		xhr.onload = function(){
+            if (xhr.responseURL.includes('/redirect?num=0')){
+                resolve();
+            }else{
+                reject();
+            }
+		};
+		xhr.send();
+	});
+}
+
 async function testFormData(file){
 	const formData = new FormData();
 	formData.append('name', 'test');
@@ -196,6 +237,20 @@ function testTimeout(){
 		}
 	}()
 
+	t.Run("testRedirect", func(t *testing.T) {
+		_, err := callTestFun(engine, "testRedirect")
+		if err != nil {
+			t.Fatal(err)
+		}
+	})
+
+	t.Run("testResponseUrl", func(t *testing.T) {
+		_, err = callTestFun(engine, "testResponseUrl")
+		if err != nil {
+			t.Fatal(err)
+		}
+	})
+
 	func() {
 		jsFile, goFile, md5 := buildFile(t, engine.Runtime)
 		result, err = callTestFun(engine, "testFormData", jsFile)
@@ -405,6 +460,16 @@ func startServer() net.Listener {
 		w.WriteHeader(http.StatusOK)
 		w.Write([]byte("ok"))
 	})
+	mux.HandleFunc("/redirect", func(w http.ResponseWriter, r *http.Request) {
+		num := r.URL.Query().Get("num")
+		n, _ := strconv.Atoi(num)
+		if n > 0 {
+			http.Redirect(w, r, fmt.Sprintf("/redirect?num=%d", n-1), http.StatusFound)
+		} else {
+			w.WriteHeader(http.StatusOK)
+			w.Write([]byte("ok"))
+		}
+	})
 	server.Handler = mux
 	go server.Serve(listener)
 	return listener

pkg/download/engine/inject/xhr/module.go

@@ -3,17 +3,18 @@ package xhr
 import (
 	"bytes"
 	"errors"
-	"github.com/GopeedLab/gopeed/pkg/download/engine/inject/file"
-	"github.com/GopeedLab/gopeed/pkg/download/engine/inject/formdata"
-	"github.com/GopeedLab/gopeed/pkg/download/engine/util"
-	"github.com/dop251/goja"
 	"io"
 	"mime/multipart"
 	"net"
 	"net/http"
 	"net/url"
 	"strings"
 	"time"
+
+	"github.com/GopeedLab/gopeed/pkg/download/engine/inject/file"
+	"github.com/GopeedLab/gopeed/pkg/download/engine/inject/formdata"
+	"github.com/GopeedLab/gopeed/pkg/download/engine/util"
+	"github.com/dop251/goja"
 )
 
 const (
@@ -25,6 +26,12 @@ const (
 	eventTimeout          = "timeout"
 )
 
+const (
+	redirectError  = "error"
+	redirectFollow = "follow"
+	redirectManual = "manual"
+)
+
 type ProgressEvent struct {
 	Type             string `json:"type"`
 	LengthComputable bool   `json:"lengthComputable"`
@@ -131,6 +138,14 @@ type XMLHttpRequest struct {
 	StatusText   string                `json:"statusText"`
 	Response     string                `json:"response"`
 	ResponseText string                `json:"responseText"`
+	// https://developer.mozilla.org/zh-CN/docs/Web/API/XMLHttpRequest/responseURL
+	// https://xhr.spec.whatwg.org/#the-responseurl-attribute
+	ResponseUrl string `json:"responseURL"`
+	// extend fetch redirect
+	// https://developer.mozilla.org/en-US/docs/Web/API/RequestInit#redirect
+	// https://fetch.spec.whatwg.org/#concept-request-redirect-mode
+	Redirect string `json:"redirect"`
+
 	*EventProp
 	Onreadystatechange func(event *ProgressEvent) `json:"onreadystatechange"`
 }
@@ -159,28 +174,28 @@ func (xhr *XMLHttpRequest) Send(data goja.Value) {
 	if d == nil || xhr.method == "GET" || xhr.method == "HEAD" {
 		req, err = http.NewRequest(xhr.method, xhr.url, nil)
 	} else {
-		switch d.(type) {
+		switch v := d.(type) {
 		case string:
-			req, err = http.NewRequest(xhr.method, xhr.url, bytes.NewBufferString(d.(string)))
+			req, err = http.NewRequest(xhr.method, xhr.url, bytes.NewBufferString(v))
 			contentType = "text/plain;charset=UTF-8"
-			contentLength = int64(len(d.(string)))
+			contentLength = int64(len(v))
 			isStringBody = true
 		case *file.File:
-			req, err = http.NewRequest(xhr.method, xhr.url, d.(*file.File).Reader)
+			req, err = http.NewRequest(xhr.method, xhr.url, v.Reader)
 			contentType = "application/octet-stream"
-			contentLength = d.(*file.File).Size
+			contentLength = v.Size
 		case *formdata.FormData:
 			pr, pw := io.Pipe()
 			mw := NewMultipart(pw)
-			for _, e := range d.(*formdata.FormData).Entries() {
+			for _, e := range v.Entries() {
 				arr := e.([]any)
 				k := arr[0].(string)
 				v := arr[1]
-				switch v.(type) {
+				switch v := v.(type) {
 				case string:
-					mw.WriteField(k, v.(string))
+					mw.WriteField(k, v)
 				case *file.File:
-					mw.WriteFile(k, v.(*file.File))
+					mw.WriteFile(k, v)
 				}
 			}
 			go func() {
@@ -211,6 +226,18 @@ func (xhr *XMLHttpRequest) Send(data goja.Value) {
 	client := &http.Client{
 		Transport: transport,
 		Timeout:   time.Duration(xhr.Timeout) * time.Millisecond,
+		CheckRedirect: func(req *http.Request, via []*http.Request) error {
+			if xhr.Redirect == redirectManual {
+				return http.ErrUseLastResponse
+			}
+			if xhr.Redirect == redirectError {
+				return errors.New("redirect failed")
+			}
+			if len(via) > 20 {
+				return errors.New("too many redirects")
+			}
+			return nil
+		},
 	}
 	resp, err := client.Do(req)
 	if err != nil {
@@ -232,6 +259,11 @@ func (xhr *XMLHttpRequest) Send(data goja.Value) {
 	if !xhr.aborted {
 		xhr.Upload.callOnload()
 	}
+
+	responseUrl := resp.Request.URL
+	responseUrl.Fragment = ""
+	xhr.ResponseUrl = responseUrl.String()
+
 	xhr.responseHeaders = resp.Header
 	xhr.Status = resp.StatusCode
 	xhr.StatusText = resp.Status
@@ -387,29 +419,27 @@ func (w *multipartWrapper) Size() int64 {
 	w.statWriter.Close()
 	size := int64(w.statBuffer.Len())
 	for _, v := range w.fields {
-		switch v.(type) {
+		switch v := v.(type) {
 		case *file.File:
-			f := v.(*file.File)
-			size += f.Size
+			size += v.Size
 		}
 	}
 	return size
 }
 
 func (w *multipartWrapper) Send() error {
 	for k, v := range w.fields {
-		switch v.(type) {
+		switch v := v.(type) {
 		case string:
-			if err := w.writer.WriteField(k, v.(string)); err != nil {
+			if err := w.writer.WriteField(k, v); err != nil {
 				return err
 			}
 		case *file.File:
-			f := v.(*file.File)
-			fw, err := w.writer.CreateFormFile(k, f.Name)
+			fw, err := w.writer.CreateFormFile(k, v.Name)
 			if err != nil {
 				return err
 			}
-			if _, err = io.Copy(fw, f); err != nil {
+			if _, err = io.Copy(fw, v); err != nil {
 				return err
 			}
 		}