Skip to content

Commit

Permalink
Merge pull request #83 from Financial-Times/UPPSF-5389-sync-annotatio…
Browse files Browse the repository at this point in the history
…ns-pac-and-publish

Synchronise calls between PAC and Publishing
  • Loading branch information
angelraynovft authored Oct 3, 2024
2 parents 7eefaf3 + b24e767 commit 2126d4d
Show file tree
Hide file tree
Showing 6 changed files with 200 additions and 23 deletions.
7 changes: 1 addition & 6 deletions .circleci/config.yml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
version: 2.1
orbs:
ft-golang-ci: financial-times/golang-ci@1
ft-golang-ci: financial-times/golang-ci@2
jobs:
dredd:
working_directory: /go/src/github.com/Financial-Times/draft-annotations-api
Expand Down Expand Up @@ -41,8 +41,3 @@ workflows:
name: build-docker-image
requires:
- build-and-test-project
snyk-scanning:
jobs:
- ft-golang-ci/scan:
name: scan-dependencies
context: cm-team-snyk
9 changes: 5 additions & 4 deletions go.mod
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
module github.com/Financial-Times/draft-annotations-api

go 1.21
go 1.23

require (
github.com/Financial-Times/api-endpoint v1.0.1
Expand All @@ -11,7 +11,7 @@ require (
github.com/Financial-Times/transactionid-utils-go v0.2.0
github.com/Pallinder/go-randomdata v0.0.0-20170410161340-8c3362a5e678
github.com/google/uuid v1.3.0
github.com/husobee/vestigo v1.0.2
github.com/husobee/vestigo v1.1.1
github.com/jawher/mow.cli v0.0.0-20170712113824-a6088643acff
github.com/pkg/errors v0.8.1
github.com/rcrowley/go-metrics v0.0.0-20161128210544-1f30fe9094a5
Expand All @@ -26,9 +26,10 @@ require (
github.com/kr/pretty v0.1.0 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/stretchr/objx v0.5.0 // indirect
golang.org/x/crypto v0.0.0-20170825220121-81e90905daef // indirect
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd // indirect
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 // indirect
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 // indirect
golang.org/x/sys v0.0.0-20211013075003-97ac67df715c // indirect
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 // indirect
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
Expand Down
15 changes: 11 additions & 4 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,8 @@ github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+
github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek=
github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
github.com/husobee/vestigo v1.0.2 h1:K4Awra33kZsLUQeTwrtdkj/Yf6pIy7b6qMtJH3s5SA4=
github.com/husobee/vestigo v1.0.2/go.mod h1:JigD7C8lzUfpo1uzqYgefpyZLswrtJbAQxMw7ds7YCE=
github.com/husobee/vestigo v1.1.1 h1:bsReVP78YhmHUn/nQ4AxIEfObmWMSLGLGXP1OwgFa9s=
github.com/husobee/vestigo v1.1.1/go.mod h1:JigD7C8lzUfpo1uzqYgefpyZLswrtJbAQxMw7ds7YCE=
github.com/jawher/mow.cli v0.0.0-20170712113824-a6088643acff h1:x5pzpfFtFQYcypjIah0Tj8lpo/eEmqZNHeME2u2/EOo=
github.com/jawher/mow.cli v0.0.0-20170712113824-a6088643acff/go.mod h1:5hQj2V8g+qYmLUVWqu4Wuja1pI57M83EChYLVZ0sMKk=
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
Expand Down Expand Up @@ -54,15 +54,22 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
golang.org/x/crypto v0.0.0-20170825220121-81e90905daef h1:R8ubLIilYRXIXpgjOg2l/ECVs3HzVKIjJEhxSsQ91u4=
golang.org/x/crypto v0.0.0-20170825220121-81e90905daef/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd h1:nTDtHvHSdCn1m6ITfMRqtOd/9+7a3s8RBNOZ3eYZzJA=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 h1:7I4JAnoQBe7ZtJcBaYHi5UtiO8tQHbUSXxL+pnGRANg=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 h1:qWPm9rbaAMKs8Bq/9LRpbMqxWRVUAQwMI9fVrssnTfw=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20211013075003-97ac67df715c h1:taxlMj0D/1sOAuv/CbSD+MMDof2vbyPTqz5FNYKpXt8=
golang.org/x/sys v0.0.0-20211013075003-97ac67df715c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
gopkg.in/airbrake/gobrake.v2 v2.0.9 h1:7z2uVWwn7oVeeugY1DtlPAy5H+KYgB1KeKTnqjNatLo=
gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
Expand Down
10 changes: 10 additions & 0 deletions helm/draft-annotations-api/templates/deployment.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,16 @@ spec:
configMapKeyRef:
name: global-config
key: internal-concordances-endpoint
- name: DRAFT_ANNOTATIONS_PUBLISH_ENDPOINT
valueFrom:
configMapKeyRef:
name: global-config
key: draft-annotations-publish-endpoint
- name: PUBLISH_BASIC_AUTH
valueFrom:
secretKeyRef:
name: doppler-global-secrets
key: PUBLISH_BASIC_AUTH
- name: DELIVERY_BASIC_AUTH
valueFrom:
secretKeyRef:
Expand Down
53 changes: 44 additions & 9 deletions main.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (
"github.com/Financial-Times/draft-annotations-api/concept"
"github.com/Financial-Times/draft-annotations-api/handler"
"github.com/Financial-Times/draft-annotations-api/health"
"github.com/Financial-Times/draft-annotations-api/synchronise"
"github.com/Financial-Times/go-ft-http/fthttp"
"github.com/Financial-Times/http-handlers-go/httphandlers"
status "github.com/Financial-Times/service-status-go/httphandlers"
Expand All @@ -20,7 +21,10 @@ import (
log "github.com/sirupsen/logrus"
)

const appDescription = "PAC Draft Annotations API"
const (
platform = "PAC"
appDescription = "PAC Draft Annotations API"
)

func main() {
app := cli.App("draft-annotations-api", appDescription)
Expand Down Expand Up @@ -92,6 +96,20 @@ func main() {
EnvVar: "LOG_LEVEL",
})

draftAnnotationsPublishEndpoint := app.String(cli.StringOpt{
Name: "draft-annotations-publish-endpoint",
Value: "http://localhost:8081",
Desc: "Endpoint to sync requests between pac and publish cluster",
EnvVar: "DRAFT_ANNOTATIONS_PUBLISH_ENDPOINT",
})

publishBasicAuth := app.String(cli.StringOpt{
Name: "publish-basic-auth",
Value: "username:password",
Desc: "Basic auth for access to the publish UPP clusters",
EnvVar: "PUBLISH_BASIC_AUTH",
})

log.SetFormatter(&log.JSONFormatter{})
log.Infof("[Startup] %v is starting", *appSystemCode)

Expand All @@ -111,22 +129,27 @@ func main() {
log.WithError(err).Fatal("Please provide a valid timeout duration")
}

client := fthttp.NewClientWithDefaultTimeout("PAC", *appSystemCode)
client := fthttp.NewClientWithDefaultTimeout(platform, *appSystemCode)

basicAuthCredentials := strings.Split(*deliveryBasicAuth, ":")
if len(basicAuthCredentials) != 2 {
log.Fatal("error while resolving basic auth")
}

publishBasicAuthCredentials := strings.Split(*publishBasicAuth, ":")
if len(publishBasicAuthCredentials) != 2 {
log.Fatal("error while resolving publish basic auth")
}

rw := annotations.NewRW(client, *annotationsRWEndpoint)
annotationsAPI := annotations.NewUPPAnnotationsAPI(client, *annotationsAPIEndpoint, basicAuthCredentials[0], basicAuthCredentials[1])
c14n := annotations.NewCanonicalizer(annotations.NewCanonicalAnnotationSorter)
conceptRead := concept.NewReadAPI(client, *internalConcordancesEndpoint, basicAuthCredentials[0], basicAuthCredentials[1], *internalConcordancesBatchSize)
augmenter := annotations.NewAugmenter(conceptRead)
annotationsHandler := handler.New(rw, annotationsAPI, c14n, augmenter, time.Millisecond*httpTimeout)
healthService := health.NewHealthService(*appSystemCode, *appName, appDescription, rw, annotationsAPI, conceptRead)

serveEndpoints(*port, apiYml, annotationsHandler, healthService)
syncAPI := synchronise.NewAPI(client, publishBasicAuthCredentials[0], publishBasicAuthCredentials[1], *draftAnnotationsPublishEndpoint)
serveEndpoints(*port, apiYml, annotationsHandler, healthService, syncAPI)
}

err := app.Run(os.Args)
Expand All @@ -136,14 +159,26 @@ func main() {
}
}

func serveEndpoints(port string, apiYml *string, handler *handler.Handler, healthService *health.HealthService) {
func serveEndpoints(port string, apiYml *string, handler *handler.Handler, healthService *health.HealthService, sapi *synchronise.API) {
r := vestigo.NewRouter()

r.Delete("/drafts/content/:uuid/annotations/:cuuid", handler.DeleteAnnotation)
requestMiddleware := func(f http.HandlerFunc) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
err := sapi.SyncWithPublishingCluster(r)
if err != nil {
log.WithError(err).Info("error while sending request to publishing cluster")
}
// before the request
f(w, r)
// after the request
}
}

r.Delete("/drafts/content/:uuid/annotations/:cuuid", handler.DeleteAnnotation, requestMiddleware)
r.Get("/drafts/content/:uuid/annotations", handler.ReadAnnotations)
r.Put("/drafts/content/:uuid/annotations", handler.WriteAnnotations)
r.Post("/drafts/content/:uuid/annotations", handler.AddAnnotation)
r.Patch("/drafts/content/:uuid/annotations/:cuuid", handler.ReplaceAnnotation)
r.Put("/drafts/content/:uuid/annotations", handler.WriteAnnotations, requestMiddleware)
r.Post("/drafts/content/:uuid/annotations", handler.AddAnnotation, requestMiddleware)
r.Patch("/drafts/content/:uuid/annotations/:cuuid", handler.ReplaceAnnotation, requestMiddleware)

var monitoringRouter http.Handler = r
monitoringRouter = httphandlers.TransactionAwareRequestLoggingHandler(log.StandardLogger(), monitoringRouter)
Expand Down
129 changes: 129 additions & 0 deletions synchronise/synchronise.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,129 @@
// Package synchronise: This package is responsible for synchronising the draft annotations between the PAC and the publishing cluster.
// And it's a temporary solution part of the PAC decommissioning process.
package synchronise

import (
"bytes"
"encoding/json"
"fmt"
"io"
"net/http"
"strings"

tidutils "github.com/Financial-Times/transactionid-utils-go"
log "github.com/sirupsen/logrus"
)

const (
publishOrigin = "draft-annotations-publishing"
pacOrigin = "draft-annotations-pac"
forwardedHeader = "X-Forwarded-By"
originSystemIDHeader = "X-Origin-System-Id"
PACOriginSystemID = "http://cmdb.ft.com/systems/pac"
FTPinkPublication = "88fdde6c-2aa4-4f78-af02-9f680097cfd6"
)

type API struct {
client *http.Client
username string
password string
endpoint string
}

func NewAPI(client *http.Client, username string, password string, endpoint string) *API {
return &API{
client: client,
username: username,
password: password,
endpoint: endpoint,
}
}

// SyncWithPublishingCluster forwards the request to the publishing cluster.
func (api *API) SyncWithPublishingCluster(req *http.Request) error {
tID := tidutils.GetTransactionIDFromRequest(req)

// Check if the request is already forwarded by publishing cluster to avoid infinite loop
if req.Header.Get(forwardedHeader) == publishOrigin {
return nil
}

// Copy the request
bodyBytes, err := io.ReadAll(req.Body)
if err != nil {
return err
}

// Restore the io.ReadCloser after reading from it
req.Body = io.NopCloser(bytes.NewBuffer(bodyBytes))

bodyBytes, err = addPublicationToBody(bodyBytes)
if err != nil {
return err
}

// Create a new request
path := strings.Replace(req.URL.Path, "/drafts", "/draft-annotations", 1)
newReq, err := http.NewRequest(req.Method, fmt.Sprintf("%s%s", api.endpoint, path), bytes.NewBuffer(bodyBytes))
if err != nil {
return err
}

// Copy the headers
for name, values := range req.Header {
for _, value := range values {
newReq.Header.Add(name, value)
}
}

// Add the X-Forwarded-By header
newReq.Header.Add(forwardedHeader, pacOrigin)

// Add the X-Origin-System-Id header as it is mandatory in the publishing cluster
if newReq.Header.Get(originSystemIDHeader) == "" {
newReq.Header.Add(originSystemIDHeader, PACOriginSystemID)
}

// Set basic auth
newReq.SetBasicAuth(api.username, api.password)

log.WithFields(map[string]interface{}{
"method": newReq.Method,
"url": newReq.URL.String(),
"forwardedHeader": newReq.Header.Get(forwardedHeader),
"originHeader": newReq.Header.Get(originSystemIDHeader),
"host": newReq.URL.Host,
"transaction_id": tID,
}).Info("Sending request to publishing cluster")

// Send the request
resp, err := api.client.Do(newReq)
if err != nil {
return err
}
defer resp.Body.Close()

return nil
}

func addPublicationToBody(bodyBytes []byte) ([]byte, error) {
// Decode the body into a map
var bodyMap map[string]interface{}
err := json.Unmarshal(bodyBytes, &bodyMap)
if err != nil {
return nil, err
}

// Check if the map contains a key for "publication"
if _, ok := bodyMap["publication"]; !ok {
// If not, add it
bodyMap["publication"] = []string{FTPinkPublication}

// Re-encode the body into JSON
bodyBytes, err = json.Marshal(bodyMap)
if err != nil {
return nil, err
}
}
return bodyBytes, nil
}

0 comments on commit 2126d4d

Please sign in to comment.