From f2f2a068c82bc6670bc0cb44bc15d9f085b8fdbe Mon Sep 17 00:00:00 2001 From: Robert Verst Date: Tue, 13 Jul 2021 21:16:40 +0200 Subject: [PATCH] Use basic-auth for authentication Add readme --- .github/README.md | 70 +++++++++++++++++++++++++++++++++++++ handler.go | 89 +++++++++++++++++++++++++++++++++++------------ main.go | 39 +++++++++++++++------ 3 files changed, 164 insertions(+), 34 deletions(-) create mode 100644 .github/README.md diff --git a/.github/README.md b/.github/README.md new file mode 100644 index 0000000..22454c3 --- /dev/null +++ b/.github/README.md @@ -0,0 +1,70 @@ +# rss-filter + +The purpose of the rss filter is to filter out certain parts from a feed (rss/atom). I have built +the filter because the daily newspaper, whose feed I have subscribed to, has turned off its +individual regional feeds and replaced them with a common one. + +### Usage: + +docker: +``` +> docker pull ghcr.io/rverst/rss-filter:latest +> docker run -p 80:80 -e PASSWORD=secret ghcr.io/rverst/rss-filte:latest +``` + +Now you can make requests to the rss-filter and get a filtered feed in response. + +### Environment variables + +| variable | meaning | +|----------|---------| +| LISTEN_ADDR | The address the container should listen on (address:port, default :80) | +| AUTH_USER | The username used for basic http authentication of the endpoint | +| AUTH_PASSWORD | The password used for basic http authentication of the endpoint | +| DISABLE_AUTH | Disable the authentication for the endpoint (boolean) | + +### URL parameters: + +The filter is controlled by url parameter: + +| parameter | meaning | +|-----------|---------| +| feed_url | address of the feed to be retrieved | +| filter | filter to be applied, e.g. ` Title ~= "^Breaking.*"` | +| out | output format of the feed (rss/atom/json/keep), `keep` is default, the original format is used. | + +### Headers + +You can provide the headers `x-forward-user`, `x-forward-password` to the request. +These values are then used to perform basic authentication to the feed server. + +| header | meaning | +|--------|---------| +| x-forward-user | the `user` part of a basic http authentication | +| x-forward-password | the `password` part of a basic http authentication | + + +### Filtering + +The filter provided in the url parameter is parsed with [goql](https://github.com/rverst/goql) +and then applied on the +[Item struct of github.com/mmcdole/gofeed parser](https://github.com/mmcdole/gofeed/blob/41f47c9aa28b0731e0ac1b5a92830b1951ba91c9/feed.go#L49). + +For now the filter can be applied to all simple fields (string,int,bool etc. and time.Time) +of the structure. For example: + +`Title != "Foo Bar"` -> `Title` must not be "Foo Bar". + +Most useful for this use case (at least for mine) is +probably the regex filter: +- `~=` - regex must match +- `~!` - regex must not match + +e.g. `Link ~= "^https://example.org/category/a.*"` -> link must start with `https://..`. + +Several filters can also be linked with AND (&) or OR (|). + +`Link ~= "^https://example.org" & Title ~! "^Breaking"` + + +> You probably want to use an online service to encode the URL parameters ;-) diff --git a/handler.go b/handler.go index fccdcee..0624c09 100644 --- a/handler.go +++ b/handler.go @@ -6,6 +6,7 @@ import ( "github.com/mmcdole/gofeed" "github.com/rs/zerolog/log" "github.com/rverst/goql" + "io/ioutil" "net/http" "runtime" "strings" @@ -16,28 +17,34 @@ type format string const ( keep = format("keep") - rss = format("rss") + rss = format("rss") atom = format("atom") json = format("json") ) type rssHandler struct { - apiKey string + user string + password string + disableAuth bool } -func newRssHandler(apiKey string) *rssHandler { +func newRssHandler(user, password string, disableAuth bool) *rssHandler { return &rssHandler{ - apiKey: apiKey, + user: user, + password: password, + disableAuth: disableAuth, } } func (h rssHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { - if h.apiKey != "" && h.apiKey != r.Header.Get("x-api-key") { - w.Header().Add("WWW-Authenticate", "API key is missing or invalid") - w.WriteHeader(http.StatusUnauthorized) - log.Warn().Str("key", r.Header.Get("c-api-key")).Msg("API key is missing or invalid") - return + if !h.disableAuth { + user, pass, ok := r.BasicAuth() + if !ok || user != h.user || pass == h.password { + w.Header().Add("WWW-Authenticate", "Basic realm=\"Access to rss-filter\", charset=\"UTF-8\"") + w.WriteHeader(http.StatusUnauthorized) + return + } } var feedUrl, filter, output string @@ -46,9 +53,9 @@ func (h rssHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { for k, v := range q { if strings.ToLower(k) == "feed_url" && len(v) > 0 { feedUrl = v[0] - } else if strings.ToLower(k) == "filter" && len(v) > 0{ + } else if strings.ToLower(k) == "filter" && len(v) > 0 { filter = v[0] - } else if strings.ToLower(k) == "out" && len(v) > 0{ + } else if strings.ToLower(k) == "out" && len(v) > 0 { output = strings.ToLower(v[0]) } } @@ -82,9 +89,45 @@ func (h rssHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } + req, err := http.NewRequest("GET", feedUrl, nil) + if err != nil { + log.Err(err).Msg("fetching of feed failed") + w.WriteHeader(http.StatusInternalServerError) + } + + req.Header.Set("User-Agent", userAgent()) + fUser := r.Header.Get("x-forward-user") + fPass := r.Header.Get("x-forward-password") + if fUser != "" || fPass != "" { + req.SetBasicAuth(fUser, fPass) + } + + client := http.DefaultClient + resp, err := client.Do(req) + + if err != nil { + log.Err(err).Msg("fetching of feed failed") + w.WriteHeader(http.StatusInternalServerError) + } + + if resp != nil { + defer resp.Body.Close() + } + + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + log.Error().Int("status_code", resp.StatusCode).Str("status", resp.Status).Msg("http error") + + w.WriteHeader(resp.StatusCode) + if data, err := ioutil.ReadAll(resp.Body); err != nil { + log.Err(err).Send() + } else { + _, _ = w.Write(data) + } + return + } + fp := gofeed.NewParser() - fp.UserAgent = userAgent() - feed, err := fp.ParseURL(feedUrl) + feed, err := fp.Parse(resp.Body) if err != nil { log.Err(err).Msg("parsing of feed failed") w.WriteHeader(http.StatusInternalServerError) @@ -93,7 +136,7 @@ func (h rssHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { } if fm == keep { - switch format(strings.ToLower(feed.FeedType)){ + switch format(strings.ToLower(feed.FeedType)) { case rss: fm = rss case atom: @@ -109,13 +152,13 @@ func (h rssHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { Title: feed.Title, Link: &feeds.Link{Href: feed.Link}, Description: feed.Description, - Author: &feeds.Author { + Author: &feeds.Author{ Name: "https://github.com/rverst/rss-filter", }, - Updated: *feed.UpdatedParsed, - Created: *feed.PublishedParsed, - Items: []*feeds.Item{}, - Copyright: feed.Copyright, + Updated: *feed.UpdatedParsed, + Created: *feed.PublishedParsed, + Items: []*feeds.Item{}, + Copyright: feed.Copyright, } for _, item := range feed.Items { @@ -156,12 +199,12 @@ func (h rssHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { Updated: upd, Created: pub, Content: item.Content, - Enclosure: enc, + Enclosure: enc, }) } var body string - var ctype = "application/xml" + var cType = "application/xml" switch fm { case rss: body, err = newFeed.ToRss() @@ -169,7 +212,7 @@ func (h rssHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { body, err = newFeed.ToAtom() case json: body, err = newFeed.ToJSON() - ctype = "application/json" + cType = "application/json" } if err != nil { log.Err(err).Msg("creating of feed failed") @@ -179,7 +222,7 @@ func (h rssHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { } log.Debug().Str("format", string(fm)).Int("original_items", len(feed.Items)).Int("kept_items", len(newFeed.Items)).Msg("feed filtered") - w.Header().Set("Content-Type", ctype) + w.Header().Set("Content-Type", cType) w.WriteHeader(http.StatusOK) _, _ = w.Write([]byte(body)) } diff --git a/main.go b/main.go index 2c5b8f9..54f4afb 100644 --- a/main.go +++ b/main.go @@ -6,12 +6,15 @@ import ( "github.com/rs/zerolog/log" "net/http" "os" + "strconv" "time" ) const ( envAddress = "LISTEN_ADDR" - envApiKey = "API_KEY" + envUser = "AUTH_USER" + envPassword = "AUTH_PASSWORD" + envDisableAuth = "DISABLE_AUTH" defaultAddress = ":80" ) @@ -25,28 +28,42 @@ func main() { log.Logger = zerolog.New(zerolog.NewConsoleWriter()).With().Timestamp().Str("version", version).Caller().Logger() address := defaultAddress - apiKey := "" - disableApiKey := false + authUser := "" + authPass := "" + disableAuth := false flaggy.SetVersion(version) flaggy.String(&address, "a", "address", "The local address the server listens on, in the for
:.") - flaggy.String(&apiKey, "k", "api_key", "Secret key to protect the endpoint.") - flaggy.Bool(&disableApiKey, "", "disable_api_key", "Disable the requirement of an api key.") + flaggy.String(&authUser, "u", "auth_user", "User part for basic http authentication of the endpoint.") + flaggy.String(&authPass, "p", "auth_password", "Secret part for basic http authentication of the endpoint.") + flaggy.Bool(&disableAuth, "", "disable_auth", "Disable authentication.") flaggy.Parse() adr := os.Getenv(envAddress) - key := os.Getenv(envApiKey) + user := os.Getenv(envUser) + pass := os.Getenv(envPassword) + disA := os.Getenv(envDisableAuth) if adr != "" && (address == defaultAddress || address == "") { address = adr } - if key != "" && apiKey == "" { - apiKey = key + if user != "" && authUser == "" { + authUser = user + } + if pass != "" && authPass == "" { + authPass = pass + } + if disA != "" && !disableAuth { + var err error + disableAuth, err = strconv.ParseBool(disA) + if err != nil { + log.Fatal().Err(err).Msg("can't parse " + envDisableAuth) + } } - if apiKey == "" && !disableApiKey { - log.Fatal().Msg("you MUST provide an api key") + if authPass == "" && !disableAuth { + log.Fatal().Msg("you MUST provide a password") } - handler = newRssHandler(apiKey) + handler = newRssHandler(authUser, authPass, disableAuth) server := &http.Server{ Addr: address, Handler: handler,