diff --git a/cmd/jira/jira.go b/cmd/jira/jira.go index 748712e..f660913 100644 --- a/cmd/jira/jira.go +++ b/cmd/jira/jira.go @@ -397,6 +397,23 @@ func (j *DSJira) GetModelData(ctx *shared.Ctx, docs []interface{}) (map[string][ } } // Comments start + uComments := make(map[string]jira.IssueComment) + commentsCacheID := fmt.Sprintf("%s-%s-comments", JiraIssue, issueID) + commentsFileData, er := j.cacheProvider.GetFileByKey(fmt.Sprintf("%s/%s", j.endpoint, JiraIssue), commentsCacheID) + if er != nil { + err = er + j.log.WithFields(logrus.Fields{"operation": "GetModelData"}).Errorf("GetFileByKey get cached comments error: %v", err) + return data, err + } + oldComments := IssueComments{} + if commentsFileData != nil { + er = json.Unmarshal(commentsFileData, &oldComments) + if er != nil { + err = er + j.log.WithFields(logrus.Fields{"operation": "GetModelData"}).Errorf("unmarshall old cached comments error: %v", err) + return data, err + } + } comments, okComments := doc["issue_comments"].([]map[string]interface{}) if okComments { for _, comment := range comments { @@ -510,9 +527,95 @@ func (j *DSJira) GetModelData(ctx *shared.Ctx, docs []interface{}) (map[string][ ary = append(ary, issueComment) } data[key] = ary + found := false + for _, oldc := range oldComments.Comments { + if oldc.ID == issueCommentID { + found = true + break + } + } + if !found { + key := "comment_added" + ary, ok := data[key] + if !ok { + ary = []interface{}{issueComment} + } else { + ary = append(ary, issueComment) + } + data[key] = ary + } + uComments[issueCommentID] = issueComment } } } + for _, comm := range oldComments.Comments { + deleted := true + edited := false + for newCommID, commentVal := range uComments { + if newCommID == comm.ID { + deleted = false + if commentVal.Body != comm.Body { + edited = true + } + break + } + } + if deleted { + rvComm := jira.DeleteIssueComment{ + ID: comm.ID, + IssueID: issueID, + } + key := "comment_deleted" + ary, ok := data[key] + if !ok { + ary = []interface{}{rvComm} + } else { + ary = append(ary, rvComm) + } + data[key] = ary + } + if edited { + editedComment := jira.IssueComment{ + ID: comm.ID, + IssueID: issueID, + Comment: insights.Comment{ + Body: uComments[comm.ID].Body, + CommentURL: uComments[comm.ID].CommentURL, + CommentID: uComments[comm.ID].CommentID, + Contributor: uComments[comm.ID].Contributor, + SyncTimestamp: time.Now(), + SourceTimestamp: uComments[comm.ID].SourceTimestamp, + }, + } + key := "comment_edited" + ary, ok := data[key] + if !ok { + ary = []interface{}{editedComment} + } else { + ary = append(ary, editedComment) + } + data[key] = ary + } + } + if len(comments) > 0 { + var updatedComments IssueComments + for _, comm := range uComments { + updatedComments.Comments = append(updatedComments.Comments, IssueComment{ + ID: comm.ID, + Body: comm.Body, + }) + } + b, er := json.Marshal(updatedComments) + if er != nil { + err = er + j.log.WithFields(logrus.Fields{"operation": "GetModelData"}).Errorf("error marshal updated issue comments cache. comments data: %+v, error: %v", updatedComments, err) + return data, err + } + if err = j.cacheProvider.UpdateFileByKey(fmt.Sprintf("%s/%s", j.endpoint, JiraIssue), commentsCacheID, b); err != nil { + j.log.WithFields(logrus.Fields{"operation": "GetModelData"}).Errorf("UpdateFileByKey error update issue comments cache. path: %s, cache id: %s, comments data: %v, error: %v", fmt.Sprintf("%s/%s/", j.endpoint, JiraIssue), commentsCacheID, b, err) + return data, err + } + } sourceTimestamp := createdOn if updatedOn.After(createdOn) { sourceTimestamp = updatedOn @@ -1458,9 +1561,9 @@ func (j *DSJira) ProcessIssue(ctx *shared.Ctx, allIssues, allDocs *[]interface{} url := urlRoot if encodeInQuery { // ?startAt=0&maxResults=100&jql=updated+%3E+0+order+by+updated+asc - url += fmt.Sprintf(`?startAt=%d&maxResults=%d&jql=`, startAt, maxResults) + neturl.QueryEscape(jql) + url += fmt.Sprintf(`?maxResults=%d&jql=`, maxResults) + neturl.QueryEscape(jql) } else { - payloadBytes = []byte(fmt.Sprintf(`{"startAt":%d,"maxResults":%d,"jql":"%s"}`, startAt, maxResults, jql)) + payloadBytes = []byte(fmt.Sprintf(`{"maxResults":%d,"jql":"%s"}`, maxResults, jql)) } var res interface{} res, _, _, _, e = shared.Request( @@ -1977,28 +2080,34 @@ func main() { if err != nil { return } + err = jira.WriteLog(&ctx, timestamp, logger.InProgress, "jira connector started") + if err != nil { + jira.log.WithFields(logrus.Fields{"operation": "main"}).Errorf("WriteLog Error : %+v", err) + return + } for _, p := range projects { - err = jira.WriteLog(&ctx, timestamp, logger.InProgress, "") - if err != nil { - jira.log.WithFields(logrus.Fields{"operation": "main"}).Errorf("WriteLog Error : %+v", err) - return - } ctx.Project = p.ID ctx.ProjectFilter = true jira.endpoint = strings.ReplaceAll(strings.TrimPrefix(strings.TrimPrefix(jira.URL, "https://"), "http://"), "/", "-") + "/" + p.Key + jira.log = jira.log.WithFields(logrus.Fields{"project": p.Key}) err = jira.Sync(&ctx) if err != nil { jira.log.WithFields(logrus.Fields{"operation": "main"}).Errorf("Error Sync jira: %+v", err) // Update status to failed in log cluster er := jira.WriteLog(&ctx, timestamp, logger.Failed, err.Error()) if er != nil { - err = er + jira.log.WithFields(logrus.Fields{"operation": "main"}).Errorf("WriteLog Error : %+v", err) + shared.FatalOnError(er) } } - // Update status to done in log cluster - err = jira.WriteLog(&ctx, timestamp, logger.Done, "") + shared.FatalOnError(err) } - + // Update status to done in log cluster + err = jira.WriteLog(&ctx, timestamp, logger.Done, "") + if err != nil { + jira.log.WithFields(logrus.Fields{"operation": "main"}).Errorf("WriteLog Error : %+v", err) + } + shared.FatalOnError(err) } // createStructuredLogger... @@ -2049,3 +2158,14 @@ type project struct { ID string `json:"id"` Key string `json:"key"` } + +// IssueComments ... +type IssueComments struct { + Comments []IssueComment +} + +// IssueComment ... +type IssueComment struct { + ID string `json:"id"` + Body string `json:"body"` +} diff --git a/go.mod b/go.mod index 85a907b..5296bcf 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/LF-Engineering/insights-connector-jira go 1.17 require ( - github.com/LF-Engineering/insights-datasource-shared v1.5.12 + github.com/LF-Engineering/insights-datasource-shared v1.5.13-0.20220819120005-3a892d388ffb github.com/LF-Engineering/lfx-event-schema v0.1.29-0.20220814063134-d194fa6ec2ef github.com/aws/aws-lambda-go v1.28.0 github.com/aws/aws-sdk-go v1.43.4 diff --git a/go.sum b/go.sum index 7d10675..02f89dd 100644 --- a/go.sum +++ b/go.sum @@ -1,11 +1,7 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/LF-Engineering/insights-datasource-shared v1.5.12 h1:26zqp9OO15HS1KENSaHoXpizAtcSmHVAJvQ/yryGsxY= -github.com/LF-Engineering/insights-datasource-shared v1.5.12/go.mod h1:ZQCRAJNyizhBemQDSjqm14G2MPkOJbRzCPfjwFhYKy4= -github.com/LF-Engineering/insights-datasource-shared v1.4.5-0.20220511063206-6754a12066b9 h1:CcKhPF0SgXbOFasPhoD/1of1ohvgLc4aASJj2KTdsrI= -github.com/LF-Engineering/insights-datasource-shared v1.4.5-0.20220511063206-6754a12066b9/go.mod h1:ZQCRAJNyizhBemQDSjqm14G2MPkOJbRzCPfjwFhYKy4= +github.com/LF-Engineering/insights-datasource-shared v1.5.13-0.20220819120005-3a892d388ffb h1:dgtYhCimKfw0SYrbNT0CSpcgo2KdcilqBr5Q1AtSHgs= +github.com/LF-Engineering/insights-datasource-shared v1.5.13-0.20220819120005-3a892d388ffb/go.mod h1:ZQCRAJNyizhBemQDSjqm14G2MPkOJbRzCPfjwFhYKy4= github.com/LF-Engineering/lfx-event-schema v0.1.14/go.mod h1:CfFIZ4mwzo88umf5+KxDQEzqlVkPG7Vx8eLK2oDfWIs= -github.com/LF-Engineering/lfx-event-schema v0.1.20-0.20220510142557-956ba192fade h1:YTBk0uQnUVrfEFcyHaR2hzVXDh6ebmeliV0f/O/jj8c= -github.com/LF-Engineering/lfx-event-schema v0.1.20-0.20220510142557-956ba192fade/go.mod h1:CfFIZ4mwzo88umf5+KxDQEzqlVkPG7Vx8eLK2oDfWIs= github.com/LF-Engineering/lfx-event-schema v0.1.29-0.20220814063134-d194fa6ec2ef h1:CcdZsFrNs3gL9QOTDMFy+XOFyXpuhXOTO1+3y5eP6Q8= github.com/LF-Engineering/lfx-event-schema v0.1.29-0.20220814063134-d194fa6ec2ef/go.mod h1:CfFIZ4mwzo88umf5+KxDQEzqlVkPG7Vx8eLK2oDfWIs= github.com/alecthomas/jsonschema v0.0.0-20210920000243-787cd8204a0d/go.mod h1:/n6+1/DWPltRLWL/VKyUxg6tzsl5kHUCcraimt4vr60=