From da382f58431c576b2a8daec6353e645fecf68267 Mon Sep 17 00:00:00 2001 From: Imran Ismail Date: Wed, 25 Nov 2020 07:33:47 +0800 Subject: [PATCH] chore: reintroduce martianurl and namespace bff implementation under bffurl --- bffurl/host.go | 64 --------- bffurl/host_test.go | 41 ------ bffurl/url_filter.go | 4 +- bffurl/url_matcher.go | 3 +- bffurl/url_modifier.go | 2 +- bffurl/url_regex_filter.go | 93 ------------- bffurl/url_regex_filter_test.go | 240 -------------------------------- bffurl/url_regex_matcher.go | 49 ------- bffurl/url_verifier.go | 17 ++- proxy/proxy.go | 1 + 10 files changed, 16 insertions(+), 498 deletions(-) delete mode 100644 bffurl/host.go delete mode 100644 bffurl/host_test.go delete mode 100644 bffurl/url_regex_filter.go delete mode 100644 bffurl/url_regex_filter_test.go delete mode 100644 bffurl/url_regex_matcher.go diff --git a/bffurl/host.go b/bffurl/host.go deleted file mode 100644 index 0007a9d..0000000 --- a/bffurl/host.go +++ /dev/null @@ -1,64 +0,0 @@ -// Copyright 2016 Google Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package bffurl - -// MatchHost matches two URL hosts with support for wildcards. -func MatchHost(host, match string) bool { - // Short circuit if host is empty. - if host == "" { - return false - } - - // Exact match, no need to loop. - if host == match { - return true - } - - // Walk backward over the host. - hi := len(host) - 1 - for mi := len(match) - 1; mi >= 0; mi-- { - // Found wildcard, skip to next period. - if match[mi] == '*' { - for hi > 0 && host[hi] != '.' { - hi-- - } - - // Wildcard was the leftmost part and we have walked the entire host, - // success. - if mi == 0 && hi == 0 { - return true - } - - continue - } - - if host[hi] != match[mi] { - return false - } - - // We have walked the entire host, if we have not walked the entire matcher - // (mi != 0) that means the matcher has remaining characters to match and - // thus the host cannot match. - if hi == 0 { - return mi == 0 - } - - hi-- - } - - // We have walked the entire length of the matcher, but haven't finished - // walking the host thus they cannot match. - return false -} diff --git a/bffurl/host_test.go b/bffurl/host_test.go deleted file mode 100644 index f395d3e..0000000 --- a/bffurl/host_test.go +++ /dev/null @@ -1,41 +0,0 @@ -// copyright 2016 google inc. all rights reserved. -// -// licensed under the apache license, version 2.0 (the "license"); -// you may not use this file except in compliance with the license. -// you may obtain a copy of the license at -// -// http://www.apache.org/licenses/license-2.0 -// -// unless required by applicable law or agreed to in writing, software -// distributed under the license is distributed on an "as is" basis, -// without warranties or conditions of any kind, either express or implied. -// see the license for the specific language governing permissions and -// limitations under the license. - -package bffurl - -import "testing" - -func TestMatchHost(t *testing.T) { - tt := []struct { - host, match string - want bool - }{ - {"example.com", "example.com", true}, - {"example.com", "example.org", false}, - {"ample.com", "example.com", false}, - {"example.com", "ample.com", false}, - {"example.com", "example.*", true}, - {"www.example.com", "*.example.com", true}, - {"one.two.example.com", "*.example.com", false}, - {"one.two.example.com", "*.*.example.com", true}, - {"", "", false}, - {"", "foo", false}, - } - - for i, tc := range tt { - if got := MatchHost(tc.host, tc.match); got != tc.want { - t.Errorf("%d. MatchHost(%s, %s): got %t, want %t", i, tc.host, tc.match, got, tc.want) - } - } -} diff --git a/bffurl/url_filter.go b/bffurl/url_filter.go index f145250..1021718 100644 --- a/bffurl/url_filter.go +++ b/bffurl/url_filter.go @@ -24,10 +24,10 @@ import ( "github.com/google/martian/v3/parse" ) -var noop = martian.Noop("url.Filter") +var noop = martian.Noop("bffurl.Filter") func init() { - parse.Register("url.Filter", filterFromJSON) + parse.Register("bffurl.Filter", filterFromJSON) } // Filter runs modifiers iff the request URL matches all of the segments in url. diff --git a/bffurl/url_matcher.go b/bffurl/url_matcher.go index b8abfd0..09f66ef 100644 --- a/bffurl/url_matcher.go +++ b/bffurl/url_matcher.go @@ -19,6 +19,7 @@ import ( "net/url" "github.com/google/martian/v3/log" + "github.com/google/martian/v3/martianurl" ) // Matcher is a conditional evaluator of request urls to be used in @@ -65,7 +66,7 @@ func (m *Matcher) matches(r *http.Request) bool { switch { case m.url.Scheme != "" && m.url.Scheme != r.URL.Scheme: return false - case m.url.Host != "" && !MatchHost(r.URL.Host, m.url.Host): + case m.url.Host != "" && !martianurl.MatchHost(r.URL.Host, m.url.Host): return false case m.url.Path != "" && !m.pattern.Match(r): return false diff --git a/bffurl/url_modifier.go b/bffurl/url_modifier.go index 781c523..d68ff16 100644 --- a/bffurl/url_modifier.go +++ b/bffurl/url_modifier.go @@ -42,7 +42,7 @@ type modifierJSON struct { } func init() { - parse.Register("url.Modifier", modifierFromJSON) + parse.Register("bffurl.Modifier", modifierFromJSON) } // ModifyRequest sets the fields of req.URL to m.Url if they are not the zero value. diff --git a/bffurl/url_regex_filter.go b/bffurl/url_regex_filter.go deleted file mode 100644 index 1f3c50e..0000000 --- a/bffurl/url_regex_filter.go +++ /dev/null @@ -1,93 +0,0 @@ -// Copyright 2015 Google Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package bffurl - -import ( - "encoding/json" - "regexp" - - "github.com/google/martian/v3/filter" - "github.com/google/martian/v3/parse" -) - -func init() { - parse.Register("url.RegexFilter", regexFilterFromJSON) -} - -// URLRegexFilter runs Modifier if the request URL matches the regex, and runs ElseModifier if not. -// This is not to be confused with url.Filter that does string matching on URL segments. -type URLRegexFilter struct { - *filter.Filter -} - -type regexFilterJSON struct { - Regex string `json:"regex"` - Modifier json.RawMessage `json:"modifier"` - ElseModifier json.RawMessage `json:"else"` - Scope []parse.ModifierType `json:"scope"` -} - -// NewRegexFilter constructs a filter that matches on regular expressions. -func NewRegexFilter(r *regexp.Regexp) *URLRegexFilter { - filter := filter.New() - matcher := NewRegexMatcher(r) - filter.SetRequestCondition(matcher) - filter.SetResponseCondition(matcher) - return &URLRegexFilter{filter} -} - -// regexFilterFromJSON takes a JSON message as a byte slice and returns a -// parse.Result that contains a URLRegexFilter and a scope. The regex syntax is RE2 -// as described at https://golang.org/s/re2syntax. -// -// Example JSON configuration message: -// { -// "scope": ["request", "response"], -// "regex": ".*www.example.com.*" -// } -func regexFilterFromJSON(b []byte) (*parse.Result, error) { - msg := ®exFilterJSON{} - if err := json.Unmarshal(b, msg); err != nil { - return nil, err - } - - matcher, err := regexp.Compile(msg.Regex) - if err != nil { - return nil, err - } - - filter := NewRegexFilter(matcher) - - m, err := parse.FromJSON(msg.Modifier) - if err != nil { - return nil, err - } - - filter.RequestWhenTrue(m.RequestModifier()) - filter.ResponseWhenTrue(m.ResponseModifier()) - - if len(msg.ElseModifier) > 0 { - em, err := parse.FromJSON(msg.ElseModifier) - if err != nil { - return nil, err - } - if em != nil { - filter.RequestWhenFalse(em.RequestModifier()) - filter.ResponseWhenFalse(em.ResponseModifier()) - } - } - - return parse.NewResult(filter, msg.Scope) -} diff --git a/bffurl/url_regex_filter_test.go b/bffurl/url_regex_filter_test.go deleted file mode 100644 index 218d8a1..0000000 --- a/bffurl/url_regex_filter_test.go +++ /dev/null @@ -1,240 +0,0 @@ -// Copyright 2015 Google Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package bffurl - -import ( - "net/http" - "regexp" - "testing" - - "github.com/google/martian/v3" - _ "github.com/google/martian/v3/header" - "github.com/google/martian/v3/parse" - "github.com/google/martian/v3/proxyutil" -) - -func TestRegexFilterModifyRequest(t *testing.T) { - tt := []struct { - want bool - match string - regstr string - }{ - { - match: "https://www.example.com", - regstr: "https://.*", - want: true, - }, - { - match: "http://www.example.com/subpath", - regstr: ".*www.example.com.*", - want: true, - }, - { - match: "https://www.example.com/subpath", - regstr: ".*www.example.com.*", - want: true, - }, - { - match: "http://www.example.com/test", - regstr: ".*/test", - want: true, - }, - { - match: "http://www.example.com?test=true", - regstr: ".*test=true.*", - want: true, - }, - { - match: "http://www.example.com#test", - regstr: ".*test.*", - want: true, - }, - { - match: "https://martian.local/test?test=true#test", - regstr: "https://martian.local/test\\?test=true#test", - want: true, - }, - { - match: "http://www.youtube.com/get_tags?tagone=yes", - regstr: ".*www.youtube.com/get_tags\\?.*", - want: true, - }, - { - match: "https://www.example.com", - regstr: "http://.*", - want: false, - }, - { - match: "http://www.martian.external", - regstr: ".*www.martian.local.*", - want: false, - }, - { - match: "http://www.example.com/testing", - regstr: ".*/test$", - want: false, - }, - { - match: "http://www.example.com?test=false", - regstr: ".*test=true.*", - want: false, - }, - { - match: "http://www.example.com#test", - regstr: ".*#testing.*", - want: false, - }, - { - match: "https://martian.local/test?test=true#test", - // "\\\\" was the old way of adding a backslash in SAVR - regstr: "https://martian.local/test\\\\?test=true#test", - want: false, - }, - { - match: "http://www.youtube.com/get_tags/nope", - regstr: ".*www.youtube.com/get_ad_tags\\?.*", - want: false, - }, - } - - for i, tc := range tt { - req, err := http.NewRequest("GET", tc.match, nil) - if err != nil { - t.Fatalf("%d. NewRequest(): got %v, want no error", i, err) - } - regex, err := regexp.Compile(tc.regstr) - if err != nil { - t.Fatalf("%d. regexp.Compile(): got %v, want no error", i, err) - } - - var modRun bool - mod := NewRegexFilter(regex) - mod.SetRequestModifier(martian.RequestModifierFunc( - func(*http.Request) error { - modRun = true - return nil - })) - - if err := mod.ModifyRequest(req); err != nil { - t.Fatalf("%d. ModifyRequest(): got %q, want no error", i, err) - } - - if modRun != tc.want { - t.Errorf("%d. modRun: got %t, want %t", i, modRun, tc.want) - } - } -} - -// The matching functionality is already tested above, so this just tests response setting. -func TestRegexFilterModifyResponse(t *testing.T) { - tt := []struct { - want bool - match string - regstr string - }{ - { - match: "https://www.example.com", - regstr: ".*www.example.com.*", - want: true, - }, - { - match: "http://www.martian.external", - regstr: ".*www.martian.local.*", - want: false, - }, - } - - for i, tc := range tt { - req, err := http.NewRequest("GET", tc.match, nil) - if err != nil { - t.Fatalf("%d. NewRequest(): got %v, want no error", i, err) - } - res := proxyutil.NewResponse(200, nil, req) - regex, err := regexp.Compile(tc.regstr) - if err != nil { - t.Fatalf("%d. regexp.Compile(): got %v, want no error", i, err) - } - - var modRun bool - mod := NewRegexFilter(regex) - mod.SetResponseModifier(martian.ResponseModifierFunc( - func(*http.Response) error { - modRun = true - return nil - })) - - if err := mod.ModifyResponse(res); err != nil { - t.Fatalf("%d. ModifyResponse(): got %q, want no error", i, err) - } - - if modRun != tc.want { - t.Errorf("%d. modRun: got %t, want %t", i, modRun, tc.want) - } - } -} - -func TestRegexFilterFromJSON(t *testing.T) { - rawMsg := ` - { - "url.RegexFilter": { - "scope": ["request", "response"], - "regex": ".*martian.test.*", - "modifier": { - "header.Modifier": { - "name": "Martian-Test", - "value": "true", - "scope": ["request", "response"] - } - } - } - }` - - r, err := parse.FromJSON([]byte(rawMsg)) - if err != nil { - t.Fatalf("parse.FromJSON(): got %v, want no error", err) - } - - reqmod := r.RequestModifier() - if reqmod == nil { - t.Errorf("reqmod: got nil, want not nil") - } - - req, err := http.NewRequest("GET", "https://martian.test", nil) - if err != nil { - t.Fatalf("http.NewRequest(): got %v, want no error", err) - } - - if err := reqmod.ModifyRequest(req); err != nil { - t.Fatalf("reqmod.ModifyRequest(): got %v, want no error", err) - } - - if got, want := req.Header.Get("Martian-Test"), "true"; got != want { - t.Errorf("req.Header.Get(%q): got %q, want %q", "Martian-Test", got, want) - } - - resmod := r.ResponseModifier() - if resmod == nil { - t.Fatalf("resmod: got nil, want not nil") - } - - res := proxyutil.NewResponse(200, nil, req) - if err := resmod.ModifyResponse(res); err != nil { - t.Fatalf("resmod.ModifyResponse(): got %v, want no error", err) - } - - if got, want := res.Header.Get("Martian-Test"), "true"; got != want { - t.Errorf("res.Header.Get(%q): got %q, want %q", "Martian-Test", got, want) - } -} diff --git a/bffurl/url_regex_matcher.go b/bffurl/url_regex_matcher.go deleted file mode 100644 index 9ab18e8..0000000 --- a/bffurl/url_regex_matcher.go +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright 2017 Google Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package bffurl - -import ( - "net/http" - "net/url" - "regexp" -) - -// RegexMatcher is a conditional evaluator of request urls to be used in -// filters that take conditionals. -type RegexMatcher struct { - r *regexp.Regexp -} - -// NewRegexMatcher builds a new url matcher from a compiled Regexp. -func NewRegexMatcher(r *regexp.Regexp) *RegexMatcher { - return &RegexMatcher{ - r: r, - } -} - -// MatchRequest retuns true if the request URL matches r. -func (m *RegexMatcher) MatchRequest(req *http.Request) bool { - return m.matches(req.URL) -} - -// MatchResponse retuns true if the response URL matches r. -func (m *RegexMatcher) MatchResponse(res *http.Response) bool { - return m.matches(res.Request.URL) -} - -// matches checks if a url matches r. -func (m *RegexMatcher) matches(u *url.URL) bool { - return m.r.MatchString(u.String()) -} diff --git a/bffurl/url_verifier.go b/bffurl/url_verifier.go index 1c74812..4f8000e 100644 --- a/bffurl/url_verifier.go +++ b/bffurl/url_verifier.go @@ -22,6 +22,7 @@ import ( "strings" "github.com/google/martian/v3" + "github.com/google/martian/v3/martianurl" "github.com/google/martian/v3/parse" "github.com/google/martian/v3/verify" ) @@ -32,13 +33,14 @@ const ( ) func init() { - parse.Register("url.Verifier", verifierFromJSON) + parse.Register("bffurl.Verifier", verifierFromJSON) } // Verifier verifies the structure of URLs. type Verifier struct { - url *url.URL - err *martian.MultiError + url *url.URL + err *martian.MultiError + pattern *Pattern } type verifierJSON struct { @@ -52,8 +54,9 @@ type verifierJSON struct { // NewVerifier returns a new URL verifier. func NewVerifier(url *url.URL) verify.RequestVerifier { return &Verifier{ - url: url, - err: martian.NewMultiError(), + url: url, + err: martian.NewMultiError(), + pattern: NewPattern(url.Path), } } @@ -74,11 +77,11 @@ func (v *Verifier) ModifyRequest(req *http.Request) error { f := fmt.Sprintf(errPartFormat, "Scheme", u.Scheme, v.url.Scheme) failures = append(failures, f) } - if v.url.Host != "" && !MatchHost(u.Host, v.url.Host) { + if v.url.Host != "" && !martianurl.MatchHost(u.Host, v.url.Host) { f := fmt.Sprintf(errPartFormat, "Host", u.Host, v.url.Host) failures = append(failures, f) } - if v.url.Path != "" && v.url.Path != u.Path { + if v.url.Path != "" && !v.pattern.Match(req) { f := fmt.Sprintf(errPartFormat, "Path", u.Path, v.url.Path) failures = append(failures, f) } diff --git a/proxy/proxy.go b/proxy/proxy.go index 69315d9..f7af4cf 100644 --- a/proxy/proxy.go +++ b/proxy/proxy.go @@ -26,6 +26,7 @@ import ( _ "github.com/google/martian/v3/cookie" _ "github.com/google/martian/v3/failure" _ "github.com/google/martian/v3/header" + _ "github.com/google/martian/v3/martianurl" _ "github.com/google/martian/v3/method" _ "github.com/google/martian/v3/pingback" _ "github.com/google/martian/v3/port"