From b9bc132bb50211eabf93f46f4a14cc0ad66a6869 Mon Sep 17 00:00:00 2001 From: Bujjibabukatta Date: Sat, 13 Jun 2026 21:10:52 +0530 Subject: [PATCH] fix(jira): collect board issues via saved filter JQL --- backend/plugins/jira/models/board.go | 1 + .../20260613_add_board_subquery.go | 46 ++++ .../jira/models/migrationscripts/register.go | 1 + .../tasks/board_filter_begin_collector.go | 42 +++- .../jira/tasks/board_filter_end_collector.go | 15 +- backend/plugins/jira/tasks/issue_collector.go | 200 +++++++++++++++--- .../jira/tasks/issue_collector_test.go | 91 ++++++++ 7 files changed, 355 insertions(+), 41 deletions(-) create mode 100644 backend/plugins/jira/models/migrationscripts/20260613_add_board_subquery.go diff --git a/backend/plugins/jira/models/board.go b/backend/plugins/jira/models/board.go index 267c1cdc0ec..f76af2a5247 100644 --- a/backend/plugins/jira/models/board.go +++ b/backend/plugins/jira/models/board.go @@ -34,6 +34,7 @@ type JiraBoard struct { Self string `json:"self" mapstructure:"self" gorm:"type:varchar(255)"` Type string `json:"type" mapstructure:"type" gorm:"type:varchar(100)"` Jql string `json:"jql" mapstructure:"jql"` + SubQuery string `json:"subQuery" mapstructure:"subQuery"` } func (b JiraBoard) ScopeId() string { diff --git a/backend/plugins/jira/models/migrationscripts/20260613_add_board_subquery.go b/backend/plugins/jira/models/migrationscripts/20260613_add_board_subquery.go new file mode 100644 index 00000000000..8a52dcf93d6 --- /dev/null +++ b/backend/plugins/jira/models/migrationscripts/20260613_add_board_subquery.go @@ -0,0 +1,46 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/helpers/migrationhelper" +) + +type JiraBoard20260613 struct { + SubQuery string +} + +func (JiraBoard20260613) TableName() string { + return "_tool_jira_boards" +} + +type addBoardSubQuery20260613 struct{} + +func (script *addBoardSubQuery20260613) Up(basicRes context.BasicRes) errors.Error { + return migrationhelper.AutoMigrateTables(basicRes, &JiraBoard20260613{}) +} + +func (*addBoardSubQuery20260613) Version() uint64 { + return 20260613120000 +} + +func (*addBoardSubQuery20260613) Name() string { + return "add sub_query to _tool_jira_boards" +} diff --git a/backend/plugins/jira/models/migrationscripts/register.go b/backend/plugins/jira/models/migrationscripts/register.go index 9c334a9ef88..c7c871a58ee 100644 --- a/backend/plugins/jira/models/migrationscripts/register.go +++ b/backend/plugins/jira/models/migrationscripts/register.go @@ -55,5 +55,6 @@ func All() []plugin.MigrationScript { new(flushJiraIssues), new(updateScopeConfig), new(addFixVersions20250619), + new(addBoardSubQuery20260613), } } diff --git a/backend/plugins/jira/tasks/board_filter_begin_collector.go b/backend/plugins/jira/tasks/board_filter_begin_collector.go index 6c513f1f83c..9dbd9afa7bc 100644 --- a/backend/plugins/jira/tasks/board_filter_begin_collector.go +++ b/backend/plugins/jira/tasks/board_filter_begin_collector.go @@ -41,12 +41,15 @@ func CollectBoardFilterBegin(taskCtx plugin.SubTaskContext) errors.Error { logger := taskCtx.GetLogger() db := taskCtx.GetDal() logger.Info("collect board in collectBoardFilterBegin: %d", data.Options.BoardId) - // get board filter id - filterId, err := getBoardFilterId(data) + // get board filter configuration + boardConfiguration, err := getBoardConfiguration(data) if err != nil { - return errors.Default.Wrap(err, fmt.Sprintf("error getting board filter id for connection_id:%d board_id:%d", data.Options.ConnectionId, data.Options.BoardId)) + return errors.Default.Wrap(err, fmt.Sprintf("error getting board filter configuration for connection_id:%d board_id:%d", data.Options.ConnectionId, data.Options.BoardId)) } + filterId := boardConfiguration.Filter.ID + subQuery := boardConfiguration.SubQuery logger.Info("collect board filter:%s", filterId) + logger.Info("collect board subQuery:%s", subQuery) // get board filter jql filterInfo, err := getBoardFilterJql(data, filterId) @@ -65,13 +68,14 @@ func CollectBoardFilterBegin(taskCtx plugin.SubTaskContext) errors.Error { // full sync syncPolicy := taskCtx.TaskContext().SyncPolicy() if syncPolicy != nil && syncPolicy.FullSync { - if record.Jql != jql { + if record.Jql != jql || record.SubQuery != subQuery { record.Jql = jql + record.SubQuery = subQuery err = db.Update(&record, dal.Where("connection_id = ? AND board_id = ? ", data.Options.ConnectionId, data.Options.BoardId)) if err != nil { return errors.Default.Wrap(err, fmt.Sprintf("error updating record in _tool_jira_boards table for connection_id:%d board_id:%d", data.Options.ConnectionId, data.Options.BoardId)) } - logger.Info("full sync mode, update jql to %s", record.Jql) + logger.Info("full sync mode, update jql to %s, subQuery to %s", record.Jql, record.SubQuery) } return nil } @@ -79,11 +83,12 @@ func CollectBoardFilterBegin(taskCtx plugin.SubTaskContext) errors.Error { // first run if record.Jql == "" { record.Jql = jql + record.SubQuery = subQuery err = db.Update(&record, dal.Where("connection_id = ? AND board_id = ? ", data.Options.ConnectionId, data.Options.BoardId)) if err != nil { return errors.Default.Wrap(err, fmt.Sprintf("error updating record in _tool_jira_boards table for connection_id:%d board_id:%d", data.Options.ConnectionId, data.Options.BoardId)) } - logger.Info("first run, update jql to %s", record.Jql) + logger.Info("first run, update jql to %s, subQuery to %s", record.Jql, record.SubQuery) return nil } // change @@ -95,6 +100,7 @@ func CollectBoardFilterBegin(taskCtx plugin.SubTaskContext) errors.Error { // set full sync taskCtx.TaskContext().SetSyncPolicy(&coreModels.SyncPolicy{TriggerSyncPolicy: coreModels.TriggerSyncPolicy{FullSync: true}}) record.Jql = jql + record.SubQuery = subQuery err = db.Update(&record, dal.Where("connection_id = ? AND board_id = ? ", data.Options.ConnectionId, data.Options.BoardId)) if err != nil { return errors.Default.Wrap(err, fmt.Sprintf("error updating record in _tool_jira_boards table for connection_id:%d board_id:%d", data.Options.ConnectionId, data.Options.BoardId)) @@ -103,23 +109,38 @@ func CollectBoardFilterBegin(taskCtx plugin.SubTaskContext) errors.Error { return errors.Default.New(fmt.Sprintf("connection_id:%d board_id:%d filter jql has changed, please use fullSync mode. And the previous jql is %s, now jql is %s", data.Options.ConnectionId, data.Options.BoardId, record.Jql, jql)) } } + if record.SubQuery != subQuery { + record.SubQuery = subQuery + err = db.Update(&record, dal.Where("connection_id = ? AND board_id = ? ", data.Options.ConnectionId, data.Options.BoardId)) + if err != nil { + return errors.Default.Wrap(err, fmt.Sprintf("error updating record in _tool_jira_boards table for connection_id:%d board_id:%d", data.Options.ConnectionId, data.Options.BoardId)) + } + logger.Info("update board subQuery to %s", record.SubQuery) + } // no change return nil } func getBoardFilterId(data *JiraTaskData) (string, error) { + boardConfiguration, err := getBoardConfiguration(data) + if err != nil { + return "", err + } + return boardConfiguration.Filter.ID, nil +} + +func getBoardConfiguration(data *JiraTaskData) (*BoardConfiguration, error) { url := fmt.Sprintf("agile/1.0/board/%d/configuration", data.Options.BoardId) boardConfiguration, err := data.ApiClient.Get(url, nil, nil) if err != nil { - return "", err + return nil, err } bc := &BoardConfiguration{} err = helper.UnmarshalResponse(boardConfiguration, bc) if err != nil { - return "", err + return nil, err } - filterId := bc.Filter.ID - return filterId, nil + return bc, nil } func getBoardFilterJql(data *JiraTaskData, filterId string) (*FilterInfo, error) { @@ -152,6 +173,7 @@ type BoardConfiguration struct { ID string `json:"id"` Self string `json:"self"` } `json:"filter"` + SubQuery string `json:"subQuery"` ColumnConfig struct { Columns []struct { Name string `json:"name"` diff --git a/backend/plugins/jira/tasks/board_filter_end_collector.go b/backend/plugins/jira/tasks/board_filter_end_collector.go index 65d8eca14fd..cea47ffe45a 100644 --- a/backend/plugins/jira/tasks/board_filter_end_collector.go +++ b/backend/plugins/jira/tasks/board_filter_end_collector.go @@ -40,11 +40,12 @@ func CollectBoardFilterEnd(taskCtx plugin.SubTaskContext) errors.Error { db := taskCtx.GetDal() logger.Info("collect board in collectBoardFilterEnd: %d", data.Options.BoardId) - // get board filter id - filterId, err := getBoardFilterId(data) + // get board filter configuration + boardConfiguration, err := getBoardConfiguration(data) if err != nil { - return errors.Default.Wrap(err, fmt.Sprintf("error getting board filter id for connection_id:%d board_id:%d", data.Options.ConnectionId, data.Options.BoardId)) + return errors.Default.Wrap(err, fmt.Sprintf("error getting board filter configuration for connection_id:%d board_id:%d", data.Options.ConnectionId, data.Options.BoardId)) } + filterId := boardConfiguration.Filter.ID logger.Info("collect board filter:%s", filterId) // get board filter jql @@ -70,6 +71,14 @@ func CollectBoardFilterEnd(taskCtx plugin.SubTaskContext) errors.Error { return errors.Default.New(fmt.Sprintf("connection_id:%d board_id:%d filter jql has changed, please use fullSync mode. And the previous jql is %s, now jql is %s", data.Options.ConnectionId, data.Options.BoardId, record.Jql, jql)) } } + if record.SubQuery != boardConfiguration.SubQuery { + record.SubQuery = boardConfiguration.SubQuery + err = db.Update(&record, dal.Where("connection_id = ? AND board_id = ? ", data.Options.ConnectionId, data.Options.BoardId)) + if err != nil { + return errors.Default.Wrap(err, fmt.Sprintf("error updating record in _tool_jira_boards table for connection_id:%d board_id:%d", data.Options.ConnectionId, data.Options.BoardId)) + } + logger.Info("update board subQuery to %s", record.SubQuery) + } return nil } diff --git a/backend/plugins/jira/tasks/issue_collector.go b/backend/plugins/jira/tasks/issue_collector.go index 9a361cbbcca..3843e14d710 100644 --- a/backend/plugins/jira/tasks/issue_collector.go +++ b/backend/plugins/jira/tasks/issue_collector.go @@ -77,12 +77,28 @@ func CollectIssues(taskCtx plugin.SubTaskContext) errors.Error { } else { logger.Info("got user's timezone: %v", loc.String()) } - jql := "ORDER BY created ASC" - if apiCollector.GetSince() != nil { - jql = buildJQL(*apiCollector.GetSince(), loc) + boardJQL, err := getBoardJQL(taskCtx.GetDal(), data) + if err != nil { + return err + } + jql := buildBoardIssueJQL(boardJQL, apiCollector.GetSince(), loc) + + if strings.EqualFold(string(data.JiraServerInfo.DeploymentType), string(models.DeploymentServer)) { + logger.Info("Using api/2/search for Jira issue collection") + err = setupIssueApiV2Collector(apiCollector, data, jql) + } else { + logger.Info("Using api/3/search/jql for Jira issue collection") + err = setupIssueApiV3Collector(apiCollector, data, jql) + } + if err != nil { + return err } - err = apiCollector.InitCollector(api.ApiCollectorArgs{ + return apiCollector.Execute() +} + +func setupIssueApiV2Collector(apiCollector *api.StatefulApiCollector, data *JiraTaskData, jql string) errors.Error { + return apiCollector.InitCollector(api.ApiCollectorArgs{ ApiClient: data.ApiClient, PageSize: data.Options.PageSize, /* @@ -94,7 +110,7 @@ func CollectIssues(taskCtx plugin.SubTaskContext) errors.Error { avoid duplicate logic for every tasks, and when we have a better idea like improving performance, we can do it in one place */ - UrlTemplate: "agile/1.0/board/{{ .Params.BoardId }}/issue", + UrlTemplate: "api/2/search", /* (Optional) Return query string for request, or you can plug them into UrlTemplate directly */ @@ -104,6 +120,7 @@ func CollectIssues(taskCtx plugin.SubTaskContext) errors.Error { query.Set("startAt", fmt.Sprintf("%v", reqData.Pager.Skip)) query.Set("maxResults", fmt.Sprintf("%v", reqData.Pager.Size)) query.Set("expand", "changelog") + query.Set("fields", "*all") return query, nil }, /* @@ -123,44 +140,171 @@ func CollectIssues(taskCtx plugin.SubTaskContext) errors.Error { For api endpoint that returns number of total pages, ApiCollector can collect pages in parallel with ease, or other techniques are required if this information was missing. */ - GetTotalPages: GetTotalPagesFromResponse, - Concurrency: 10, - ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) { - var data struct { - Issues []json.RawMessage `json:"issues"` - } - blob, err := io.ReadAll(res.Body) - if err != nil { - return nil, errors.Convert(err) - } - err = json.Unmarshal(blob, &data) - if err != nil { - return nil, errors.Convert(err) + GetTotalPages: GetTotalPagesFromResponse, + Concurrency: 10, + ResponseParser: parseIssuesFromResponse, + }) +} + +func setupIssueApiV3Collector(apiCollector *api.StatefulApiCollector, data *JiraTaskData, jql string) errors.Error { + return apiCollector.InitCollector(api.ApiCollectorArgs{ + ApiClient: data.ApiClient, + PageSize: data.Options.PageSize, + UrlTemplate: "api/3/search/jql", + GetNextPageCustomData: getNextPageCustomDataForV3, + Query: func(reqData *api.RequestData) (url.Values, errors.Error) { + query := url.Values{} + query.Set("jql", jql) + query.Set("maxResults", fmt.Sprintf("%v", reqData.Pager.Size)) + query.Set("expand", "changelog") + query.Set("fields", "*all") + if reqData.CustomData != nil { + query.Set("nextPageToken", reqData.CustomData.(string)) } - return data.Issues, nil + return query, nil }, + ResponseParser: parseIssuesFromResponse, }) +} + +func parseIssuesFromResponse(res *http.Response) ([]json.RawMessage, errors.Error) { + var data struct { + Issues []json.RawMessage `json:"issues"` + } + blob, err := io.ReadAll(res.Body) if err != nil { - return err + return nil, errors.Convert(err) + } + err = json.Unmarshal(blob, &data) + if err != nil { + return nil, errors.Convert(err) } + return data.Issues, nil +} - return apiCollector.Execute() +func getBoardJQL(db dal.Dal, data *JiraTaskData) (string, errors.Error) { + var record models.JiraBoard + err := db.First(&record, dal.Where("connection_id = ? AND board_id = ? ", data.Options.ConnectionId, data.Options.BoardId)) + if err != nil { + return "", errors.Default.Wrap(err, fmt.Sprintf("error finding record in _tool_jira_boards table for connection_id:%d board_id:%d", data.Options.ConnectionId, data.Options.BoardId)) + } + if strings.TrimSpace(record.Jql) == "" { + return "", errors.Default.New(fmt.Sprintf("connection_id:%d board_id:%d filter jql is empty, please run collectBoardFilterBegin first", data.Options.ConnectionId, data.Options.BoardId)) + } + return record.Jql, nil } // buildJQL build jql based on timeAfter and incremental mode func buildJQL(since time.Time, location *time.Location) string { - jql := "ORDER BY created ASC" - if !since.IsZero() { - if location != nil { - since = since.In(location) - } else { - since = since.In(time.UTC).Add(-24 * time.Hour) + updatedAfter := buildUpdatedAfterJQL(since, location) + if updatedAfter == "" { + return jiraIssueOrderBy + } + return fmt.Sprintf("%s %s", updatedAfter, jiraIssueOrderBy) +} + +const jiraIssueOrderBy = "ORDER BY created ASC" + +func buildBoardIssueJQL(boardJQL string, since *time.Time, location *time.Location) string { + var clauses []string + boardJQL = stripJQLOrderBy(boardJQL) + if boardJQL != "" { + clauses = append(clauses, fmt.Sprintf("(%s)", boardJQL)) + } + if since != nil { + updatedAfter := buildUpdatedAfterJQL(*since, location) + if updatedAfter != "" { + clauses = append(clauses, updatedAfter) + } + } + if len(clauses) == 0 { + return jiraIssueOrderBy + } + return fmt.Sprintf("%s %s", strings.Join(clauses, " AND "), jiraIssueOrderBy) +} + +func buildUpdatedAfterJQL(since time.Time, location *time.Location) string { + if since.IsZero() { + return "" + } + if location != nil { + since = since.In(location) + } else { + since = since.In(time.UTC).Add(-24 * time.Hour) + } + return fmt.Sprintf("updated >= '%s'", since.Format("2006/01/02 15:04")) +} + +func stripJQLOrderBy(jql string) string { + jql = strings.TrimSpace(jql) + inSingleQuote := false + inDoubleQuote := false + escaped := false + for i := 0; i < len(jql); i++ { + c := jql[i] + if escaped { + escaped = false + continue + } + if (inSingleQuote || inDoubleQuote) && c == '\\' { + escaped = true + continue + } + if !inDoubleQuote && c == '\'' { + inSingleQuote = !inSingleQuote + continue + } + if !inSingleQuote && c == '"' { + inDoubleQuote = !inDoubleQuote + continue + } + if inSingleQuote || inDoubleQuote { + continue + } + if isJQLOrderByAt(jql, i) { + return strings.TrimSpace(jql[:i]) } - jql = fmt.Sprintf("updated >= '%s' %s", since.Format("2006/01/02 15:04"), jql) } return jql } +func isJQLOrderByAt(jql string, index int) bool { + order := "order" + if !hasPrefixFold(jql[index:], order) { + return false + } + before := index - 1 + if before >= 0 && !isJQLTokenBoundary(jql[before]) { + return false + } + afterOrder := index + len(order) + if afterOrder >= len(jql) || !isJQLWhitespace(jql[afterOrder]) { + return false + } + byIndex := afterOrder + for byIndex < len(jql) && isJQLWhitespace(jql[byIndex]) { + byIndex++ + } + by := "by" + if !hasPrefixFold(jql[byIndex:], by) { + return false + } + afterBy := byIndex + len(by) + return afterBy == len(jql) || isJQLTokenBoundary(jql[afterBy]) +} + +func hasPrefixFold(s string, prefix string) bool { + return len(s) >= len(prefix) && strings.EqualFold(s[:len(prefix)], prefix) +} + +func isJQLWhitespace(c byte) bool { + return c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\f' +} + +func isJQLTokenBoundary(c byte) bool { + return isJQLWhitespace(c) || c == '(' || c == ')' +} + // getTimeZone get user's timezone from jira API func getTimeZone(taskCtx plugin.SubTaskContext) (*time.Location, errors.Error) { data := taskCtx.GetData().(*JiraTaskData) diff --git a/backend/plugins/jira/tasks/issue_collector_test.go b/backend/plugins/jira/tasks/issue_collector_test.go index 99bf5a53367..f2761e4971a 100644 --- a/backend/plugins/jira/tasks/issue_collector_test.go +++ b/backend/plugins/jira/tasks/issue_collector_test.go @@ -61,3 +61,94 @@ func Test_buildJQL(t *testing.T) { }) } } + +func Test_buildBoardIssueJQL(t *testing.T) { + base := time.Date(2021, 2, 3, 4, 5, 6, 7, time.UTC) + add48 := base.Add(48 * time.Hour) + loc, _ := time.LoadLocation("Asia/Shanghai") + tests := []struct { + name string + boardJQL string + since *time.Time + location *time.Location + want string + }{ + { + name: "full sync uses board filter", + boardJQL: "project = DLK AND status != Done", + want: "(project = DLK AND status != Done) ORDER BY created ASC", + }, + { + name: "saved filter order is replaced", + boardJQL: "project = DLK ORDER BY Rank ASC", + want: "(project = DLK) ORDER BY created ASC", + }, + { + name: "incremental sync combines board filter and updated date", + boardJQL: "project = DLK ORDER BY Rank ASC", + since: &add48, + location: loc, + want: "(project = DLK) AND updated >= '2021/02/05 12:05' ORDER BY created ASC", + }, + { + name: "quoted order by is kept", + boardJQL: `summary ~ "order by" ORDER BY Rank ASC`, + want: `(summary ~ "order by") ORDER BY created ASC`, + }, + { + name: "empty board filter keeps stable ordering", + boardJQL: "", + want: "ORDER BY created ASC", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := buildBoardIssueJQL(tt.boardJQL, tt.since, tt.location); got != tt.want { + t.Errorf("buildBoardIssueJQL() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_stripJQLOrderBy(t *testing.T) { + tests := []struct { + name string + jql string + want string + }{ + { + name: "strips simple order by", + jql: "project = DLK ORDER BY Rank ASC", + want: "project = DLK", + }, + { + name: "strips mixed-case order by", + jql: "project = DLK order by created DESC", + want: "project = DLK", + }, + { + name: "keeps order by inside double quotes", + jql: `summary ~ "order by" ORDER BY Rank ASC`, + want: `summary ~ "order by"`, + }, + { + name: "keeps order by inside single quotes", + jql: "description ~ 'order by' ORDER BY created DESC", + want: "description ~ 'order by'", + }, + { + name: "does not strip orderby token", + jql: "project = DLK AND component = ORDERBY", + want: "project = DLK AND component = ORDERBY", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := stripJQLOrderBy(tt.jql); got != tt.want { + t.Errorf("stripJQLOrderBy() = %v, want %v", got, tt.want) + } + }) + } +}