Skip to content
This repository has been archived by the owner on May 24, 2024. It is now read-only.

Commit

Permalink
Enhance caching and logging (#12)
Browse files Browse the repository at this point in the history 
* add project to logs

Signed-off-by: Ayman <[email protected]>

* add project to logs

Signed-off-by: Ayman <[email protected]>

* handle deleted comments

Signed-off-by: Ayman <[email protected]>

* clean up

Signed-off-by: Ayman <[email protected]>

* use latest shared

Signed-off-by: Ayman <[email protected]>

* use latest shared

Signed-off-by: Ayman <[email protected]>

* update shared version

Signed-off-by: Ayman <[email protected]>

Signed-off-by: Ayman <[email protected]>
Co-authored-by: Ayman <[email protected]>
  • Loading branch information
@khalifapro @enkhalifapro
khalifapro and enkhalifapro authored Aug 22, 2022
1 parent 5534373 commit c5450a0
Show file tree
Hide file tree
Showing 3 changed files with 134 additions and 18 deletions.
142 changes: 131 additions & 11 deletions cmd/jira/jira.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -397,6 +397,23 @@ func (j *DSJira) GetModelData(ctx *shared.Ctx, docs []interface{}) (map[string][
}
}
// Comments start
uComments := make(map[string]jira.IssueComment)
commentsCacheID := fmt.Sprintf("%s-%s-comments", JiraIssue, issueID)
commentsFileData, er := j.cacheProvider.GetFileByKey(fmt.Sprintf("%s/%s", j.endpoint, JiraIssue), commentsCacheID)
if er != nil {
err = er
j.log.WithFields(logrus.Fields{"operation": "GetModelData"}).Errorf("GetFileByKey get cached comments error: %v", err)
return data, err
}
oldComments := IssueComments{}
if commentsFileData != nil {
er = json.Unmarshal(commentsFileData, &oldComments)
if er != nil {
err = er
j.log.WithFields(logrus.Fields{"operation": "GetModelData"}).Errorf("unmarshall old cached comments error: %v", err)
return data, err
}
}
comments, okComments := doc["issue_comments"].([]map[string]interface{})
if okComments {
for _, comment := range comments {
Expand Down Expand Up @@ -510,9 +527,95 @@ func (j *DSJira) GetModelData(ctx *shared.Ctx, docs []interface{}) (map[string][
ary = append(ary, issueComment)
}
data[key] = ary
found := false
for _, oldc := range oldComments.Comments {
if oldc.ID == issueCommentID {
found = true
break
}
}
if !found {
key := "comment_added"
ary, ok := data[key]
if !ok {
ary = []interface{}{issueComment}
} else {
ary = append(ary, issueComment)
}
data[key] = ary
}
uComments[issueCommentID] = issueComment
}
}
}
for _, comm := range oldComments.Comments {
deleted := true
edited := false
for newCommID, commentVal := range uComments {
if newCommID == comm.ID {
deleted = false
if commentVal.Body != comm.Body {
edited = true
}
break
}
}
if deleted {
rvComm := jira.DeleteIssueComment{
ID: comm.ID,
IssueID: issueID,
}
key := "comment_deleted"
ary, ok := data[key]
if !ok {
ary = []interface{}{rvComm}
} else {
ary = append(ary, rvComm)
}
data[key] = ary
}
if edited {
editedComment := jira.IssueComment{
ID: comm.ID,
IssueID: issueID,
Comment: insights.Comment{
Body: uComments[comm.ID].Body,
CommentURL: uComments[comm.ID].CommentURL,
CommentID: uComments[comm.ID].CommentID,
Contributor: uComments[comm.ID].Contributor,
SyncTimestamp: time.Now(),
SourceTimestamp: uComments[comm.ID].SourceTimestamp,
},
}
key := "comment_edited"
ary, ok := data[key]
if !ok {
ary = []interface{}{editedComment}
} else {
ary = append(ary, editedComment)
}
data[key] = ary
}
}
if len(comments) > 0 {
var updatedComments IssueComments
for _, comm := range uComments {
updatedComments.Comments = append(updatedComments.Comments, IssueComment{
ID: comm.ID,
Body: comm.Body,
})
}
b, er := json.Marshal(updatedComments)
if er != nil {
err = er
j.log.WithFields(logrus.Fields{"operation": "GetModelData"}).Errorf("error marshal updated issue comments cache. comments data: %+v, error: %v", updatedComments, err)
return data, err
}
if err = j.cacheProvider.UpdateFileByKey(fmt.Sprintf("%s/%s", j.endpoint, JiraIssue), commentsCacheID, b); err != nil {
j.log.WithFields(logrus.Fields{"operation": "GetModelData"}).Errorf("UpdateFileByKey error update issue comments cache. path: %s, cache id: %s, comments data: %v, error: %v", fmt.Sprintf("%s/%s/", j.endpoint, JiraIssue), commentsCacheID, b, err)
return data, err
}
}
sourceTimestamp := createdOn
if updatedOn.After(createdOn) {
sourceTimestamp = updatedOn
Expand Down Expand Up @@ -1458,9 +1561,9 @@ func (j *DSJira) ProcessIssue(ctx *shared.Ctx, allIssues, allDocs *[]interface{}
url := urlRoot
if encodeInQuery {
// ?startAt=0&maxResults=100&jql=updated+%3E+0+order+by+updated+asc
url += fmt.Sprintf(`?startAt=%d&maxResults=%d&jql=`, startAt, maxResults) + neturl.QueryEscape(jql)
url += fmt.Sprintf(`?maxResults=%d&jql=`, maxResults) + neturl.QueryEscape(jql)
} else {
payloadBytes = []byte(fmt.Sprintf(`{"startAt":%d,"maxResults":%d,"jql":"%s"}`, startAt, maxResults, jql))
payloadBytes = []byte(fmt.Sprintf(`{"maxResults":%d,"jql":"%s"}`, maxResults, jql))
}
var res interface{}
res, _, _, _, e = shared.Request(
Expand Down Expand Up @@ -1977,28 +2080,34 @@ func main() {
if err != nil {
return
}
err = jira.WriteLog(&ctx, timestamp, logger.InProgress, "jira connector started")
if err != nil {
jira.log.WithFields(logrus.Fields{"operation": "main"}).Errorf("WriteLog Error : %+v", err)
return
}
for _, p := range projects {
err = jira.WriteLog(&ctx, timestamp, logger.InProgress, "")
if err != nil {
jira.log.WithFields(logrus.Fields{"operation": "main"}).Errorf("WriteLog Error : %+v", err)
return
}
ctx.Project = p.ID
ctx.ProjectFilter = true
jira.endpoint = strings.ReplaceAll(strings.TrimPrefix(strings.TrimPrefix(jira.URL, "https://"), "http://"), "/", "-") + "/" + p.Key
jira.log = jira.log.WithFields(logrus.Fields{"project": p.Key})
err = jira.Sync(&ctx)
if err != nil {
jira.log.WithFields(logrus.Fields{"operation": "main"}).Errorf("Error Sync jira: %+v", err)
// Update status to failed in log cluster
er := jira.WriteLog(&ctx, timestamp, logger.Failed, err.Error())
if er != nil {
err = er
jira.log.WithFields(logrus.Fields{"operation": "main"}).Errorf("WriteLog Error : %+v", err)
shared.FatalOnError(er)
}
}
// Update status to done in log cluster
err = jira.WriteLog(&ctx, timestamp, logger.Done, "")
shared.FatalOnError(err)
}

// Update status to done in log cluster
err = jira.WriteLog(&ctx, timestamp, logger.Done, "")
if err != nil {
jira.log.WithFields(logrus.Fields{"operation": "main"}).Errorf("WriteLog Error : %+v", err)
}
shared.FatalOnError(err)
}

// createStructuredLogger...
Expand Down Expand Up @@ -2049,3 +2158,14 @@ type project struct {
ID string `json:"id"`
Key string `json:"key"`
}

// IssueComments ...
type IssueComments struct {
Comments []IssueComment
}

// IssueComment ...
type IssueComment struct {
ID string `json:"id"`
Body string `json:"body"`
}
2 changes: 1 addition & 1 deletion go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ module github.com/LF-Engineering/insights-connector-jira
go 1.17

require (
github.com/LF-Engineering/insights-datasource-shared v1.5.12
github.com/LF-Engineering/insights-datasource-shared v1.5.13-0.20220819120005-3a892d388ffb
github.com/LF-Engineering/lfx-event-schema v0.1.29-0.20220814063134-d194fa6ec2ef
github.com/aws/aws-lambda-go v1.28.0
github.com/aws/aws-sdk-go v1.43.4
Expand Down
8 changes: 2 additions & 6 deletions go.sum
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,7 @@
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/LF-Engineering/insights-datasource-shared v1.5.12 h1:26zqp9OO15HS1KENSaHoXpizAtcSmHVAJvQ/yryGsxY=
github.com/LF-Engineering/insights-datasource-shared v1.5.12/go.mod h1:ZQCRAJNyizhBemQDSjqm14G2MPkOJbRzCPfjwFhYKy4=
github.com/LF-Engineering/insights-datasource-shared v1.4.5-0.20220511063206-6754a12066b9 h1:CcKhPF0SgXbOFasPhoD/1of1ohvgLc4aASJj2KTdsrI=
github.com/LF-Engineering/insights-datasource-shared v1.4.5-0.20220511063206-6754a12066b9/go.mod h1:ZQCRAJNyizhBemQDSjqm14G2MPkOJbRzCPfjwFhYKy4=
github.com/LF-Engineering/insights-datasource-shared v1.5.13-0.20220819120005-3a892d388ffb h1:dgtYhCimKfw0SYrbNT0CSpcgo2KdcilqBr5Q1AtSHgs=
github.com/LF-Engineering/insights-datasource-shared v1.5.13-0.20220819120005-3a892d388ffb/go.mod h1:ZQCRAJNyizhBemQDSjqm14G2MPkOJbRzCPfjwFhYKy4=
github.com/LF-Engineering/lfx-event-schema v0.1.14/go.mod h1:CfFIZ4mwzo88umf5+KxDQEzqlVkPG7Vx8eLK2oDfWIs=
github.com/LF-Engineering/lfx-event-schema v0.1.20-0.20220510142557-956ba192fade h1:YTBk0uQnUVrfEFcyHaR2hzVXDh6ebmeliV0f/O/jj8c=
github.com/LF-Engineering/lfx-event-schema v0.1.20-0.20220510142557-956ba192fade/go.mod h1:CfFIZ4mwzo88umf5+KxDQEzqlVkPG7Vx8eLK2oDfWIs=
github.com/LF-Engineering/lfx-event-schema v0.1.29-0.20220814063134-d194fa6ec2ef h1:CcdZsFrNs3gL9QOTDMFy+XOFyXpuhXOTO1+3y5eP6Q8=
github.com/LF-Engineering/lfx-event-schema v0.1.29-0.20220814063134-d194fa6ec2ef/go.mod h1:CfFIZ4mwzo88umf5+KxDQEzqlVkPG7Vx8eLK2oDfWIs=
github.com/alecthomas/jsonschema v0.0.0-20210920000243-787cd8204a0d/go.mod h1:/n6+1/DWPltRLWL/VKyUxg6tzsl5kHUCcraimt4vr60=
Expand Down

0 comments on commit c5450a0

Please sign in to comment.