IEX data plugin

Makefile

@@ -26,6 +26,7 @@ plugins:
 	$(MAKE) -C contrib/polygon
 	$(MAKE) -C contrib/bitmexfeeder
 	$(MAKE) -C contrib/binancefeeder
+	$(MAKE) -C contrib/iex
 
 unittest: install
 	go fmt ./...

contrib/iex/Makefile

@@ -0,0 +1,3 @@
+GOPATH0 := $(firstword $(subst :, ,$(GOPATH)))
+all:
+	go build -o $(GOPATH0)/bin/iex.so -buildmode=plugin .

contrib/iex/README.md

@@ -0,0 +1,41 @@
+# IEX Data Fetcher
+
+This module builds a MarketStore background worker which fetches historical
+price data of US stocks from [IEX's API](https://iextrading.com/).  It runs
+as a goroutine behind the MarketStore process and keeps writing to the disk.
+The module uses the HTTP interface to query the latest bars and stay up-to-date.
+Note that only 1Min -> 1D bars are supported at this time.
+
+## Configuration
+iex.so comes with the server by default, so you can simply configure it
+in the MarketStore configuration file.
+
+### Options
+Name | Type | Default | Description
+--- | --- | --- | ---
+daily | boolean | false | Pull daily (1D) bars
+intraday | string | false | Pull intraday (1Min bars)
+symbols | slice of strings | none | The symbols to retrieve chart bars for
+
+### Example
+Add the following to your config file:
+```
+bgworkers:
+  - module: iex.so
+    config:
+        daily: true
+        intraday: true
+        symbols:
+          - AAPL
+          - SPY
+```
+
+### Backfilling
+IEX's `/chart` API doesn't support querying intraday bar history further back
+than the current market day. In order to properly backfill intraday bars before
+running the plugin live, a backfill script has been included for the specific
+purpose of backfilling the history of intraday (1Min) bars. Daily bars will
+automatically be backfilled by the plugin for the trailing 5 years upon the recipt
+of a 1D bar for a given symbol. Also note that intraday bars will be backfilled for
+the given market day in the event of starting the system up after market open, or
+unexpected intraday downtime.

contrib/iex/api/api.go

@@ -0,0 +1,174 @@
+package api
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"strconv"
+	"strings"
+	"time"
+)
+
+const (
+	base      = "https://api.iextrading.com/1.0"
+	BatchSize = 100
+)
+
+var (
+	NY, _ = time.LoadLocation("America/New_York")
+)
+
+type GetBarsResponse map[string]ChartResponse
+
+type ChartResponse struct {
+	Chart []Chart `json:"chart"`
+}
+
+type Chart struct {
+	Date                 string  `json:"date"`
+	Minute               string  `json:"minute"`
+	Label                string  `json:"label"`
+	High                 float32 `json:"high"`
+	Low                  float32 `json:"low"`
+	Average              float64 `json:"average"`
+	Volume               int32   `json:"volume"`
+	Notional             float64 `json:"notional"`
+	NumberOfTrades       int     `json:"numberOfTrades"`
+	MarketHigh           float64 `json:"marketHigh"`
+	MarketLow            float64 `json:"marketLow"`
+	MarketAverage        float64 `json:"marketAverage"`
+	MarketVolume         int     `json:"marketVolume"`
+	MarketNotional       float64 `json:"marketNotional"`
+	MarketNumberOfTrades int     `json:"marketNumberOfTrades"`
+	Open                 float32 `json:"open"`
+	Close                float32 `json:"close"`
+	MarketOpen           float64 `json:"marketOpen,omitempty"`
+	MarketClose          float64 `json:"marketClose,omitempty"`
+	ChangeOverTime       int     `json:"changeOverTime"`
+	MarketChangeOverTime int     `json:"marketChangeOverTime"`
+}
+
+func (c *Chart) GetTimestamp() (ts time.Time, err error) {
+	if c.Minute == "" {
+		// daily bar
+		ts, err = time.ParseInLocation("2006-01-02", c.Date, NY)
+	} else {
+		// intraday bar
+		tStr := fmt.Sprintf("%v %v", c.Date, c.Minute)
+		ts, err = time.ParseInLocation("20060102 15:04", tStr, NY)
+	}
+	return
+}
+
+func SupportedRange(r string) bool {
+	switch r {
+	case "5y":
+	case "2y":
+	case "1y":
+	case "ytd":
+	case "6m":
+	case "3m":
+	case "1m":
+	case "1d":
+	case "date":
+	case "dynamic":
+	default:
+		return false
+	}
+	return true
+}
+
+func GetBars(symbols []string, barRange string, limit *int, retries int) (*GetBarsResponse, error) {
+	u, err := url.Parse(fmt.Sprintf("%s/stock/market/batch", base))
+	if err != nil {
+		return nil, err
+	}
+
+	if len(symbols) == 0 {
+		return &GetBarsResponse{}, nil
+	}
+
+	q := u.Query()
+
+	q.Set("symbols", strings.Join(symbols, ","))
+	q.Set("types", "chart")
+
+	if SupportedRange(barRange) {
+		q.Set("range", barRange)
+	} else {
+		return nil, fmt.Errorf("%v is not a supported bar range", barRange)
+	}
+
+	if limit != nil && *limit > 0 {
+		q.Set("chartLast", strconv.FormatInt(int64(*limit), 10))
+	}
+
+	u.RawQuery = q.Encode()
+
+	res, err := http.Get(u.String())
+	if err != nil {
+		return nil, err
+	}
+
+	defer res.Body.Close()
+
+	if res.StatusCode == http.StatusTooManyRequests {
+		if retries > 0 {
+			<-time.After(time.Second)
+			return GetBars(symbols, barRange, limit, retries-1)
+		}
+
+		return nil, fmt.Errorf("retry count exceeded")
+	}
+
+	var resp *GetBarsResponse
+
+	body, err := ioutil.ReadAll(res.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	if err = json.Unmarshal(body, &resp); err != nil {
+		return nil, err
+	}
+
+	return resp, nil
+}
+
+type ListSymbolsResponse []struct {
+	Symbol    string `json:"symbol"`
+	Name      string `json:"name"`
+	Date      string `json:"date"`
+	IsEnabled bool   `json:"isEnabled"`
+	Type      string `json:"type"`
+	IexID     string `json:"iexId"`
+}
+
+func ListSymbols() (*ListSymbolsResponse, error) {
+	url := fmt.Sprintf("%s/ref-data/symbols", base)
+
+	res, err := http.Get(url)
+
+	if err != nil {
+		return nil, err
+	}
+
+	if res.StatusCode > http.StatusMultipleChoices {
+		return nil, fmt.Errorf("status code %v", res.StatusCode)
+	}
+
+	var resp *ListSymbolsResponse
+
+	body, err := ioutil.ReadAll(res.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	if err = json.Unmarshal(body, resp); err != nil {
+		return nil, err
+	}
+
+	return resp, nil
+}