diff --git a/src/pkg/reg/adapter/gitlab/adapter.go b/src/pkg/reg/adapter/gitlab/adapter.go index bef16cb575c..f5545448422 100644 --- a/src/pkg/reg/adapter/gitlab/adapter.go +++ b/src/pkg/reg/adapter/gitlab/adapter.go @@ -15,6 +15,7 @@ package gitlab import ( + "net/url" "strings" "github.com/goharbor/harbor/src/lib/log" @@ -127,6 +128,7 @@ func (a *adapter) FetchArtifacts(filters []*model.Filter) ([]*model.Resource, er } else { pathPatterns = append(pathPatterns, nameFilter) } + log.Debugf("Patterns: %v", pathPatterns) for _, project := range projects { if !project.RegistryEnabled { @@ -143,8 +145,10 @@ func (a *adapter) FetchArtifacts(filters []*model.Filter) ([]*model.Resource, er } for _, repository := range repositories { if !existPatterns(repository.Path, pathPatterns) { + log.Debugf("Skipping repository path=%s and id=%d", repository.Path, repository.ID) continue } + log.Debugf("Search tags repository path=%s and id=%d", repository.Path, repository.ID) vTags, err := a.clientGitlabAPI.getTags(project.ID, repository.ID) if err != nil { return nil, err @@ -183,21 +187,12 @@ func (a *adapter) FetchArtifacts(filters []*model.Filter) ([]*model.Resource, er func (a *adapter) getProjectsByPattern(pattern string) ([]*Project, error) { var projects []*Project - projectset := make(map[string]bool) var err error if len(pattern) > 0 { names, ok := util.IsSpecificPath(pattern) if ok { for _, name := range names { - substrings := strings.Split(name, "/") - if len(substrings) < 2 { - continue - } - if _, ok := projectset[substrings[1]]; ok { - continue - } - projectset[substrings[1]] = true - var projectsByName, err = a.clientGitlabAPI.getProjectsByName(substrings[1]) + var projectsByName, err = a.clientGitlabAPI.getProjectsByName(url.QueryEscape(name)) if err != nil { return nil, err } @@ -207,20 +202,20 @@ func (a *adapter) getProjectsByPattern(pattern string) ([]*Project, error) { projects = append(projects, projectsByName...) } } else { - substrings := strings.Split(pattern, "/") - if len(substrings) < 2 { - return projects, nil - } - projectName := substrings[1] - if projectName == "*" { - return projects, nil + projectName := "" + for i, substring := range strings.Split(pattern, "/") { + if strings.Contains(substring, "*") { + if i != 0 { + break + } + } else { + projectName += substring + "/" + } } - projectName = strings.Trim(projectName, "*") - - if strings.Contains(projectName, "*") { + if projectName == "" { return projects, nil } - projects, err = a.clientGitlabAPI.getProjectsByName(projectName) + projects, err = a.clientGitlabAPI.getProjectsByName(url.QueryEscape(projectName)) if err != nil { return nil, err } diff --git a/src/pkg/reg/adapter/gitlab/adapter_test.go b/src/pkg/reg/adapter/gitlab/adapter_test.go index adaab523495..ccda60b5375 100644 --- a/src/pkg/reg/adapter/gitlab/adapter_test.go +++ b/src/pkg/reg/adapter/gitlab/adapter_test.go @@ -38,16 +38,10 @@ func getServer(t *testing.T) *httptest.Server { w.Header().Set("X-Next-Page", "") switch search { - case "dev-docker": + case "library/dev-docker", "library", "library/", "dev-docker/", "dev-docker": mustWriteHTTPResponse(t, w, "testdata/projects/dev-docker.json") break - case "dev-": - mustWriteHTTPResponse(t, w, "testdata/projects/dev-docker.json") - break - case "-docker": - mustWriteHTTPResponse(t, w, "testdata/projects/-docker.json") - break - case "": + case "", "library/dockers": mustWriteHTTPResponse(t, w, "testdata/projects/all.json") break default: diff --git a/src/pkg/reg/adapter/gitlab/client.go b/src/pkg/reg/adapter/gitlab/client.go index 45d53c975df..af268da62d8 100644 --- a/src/pkg/reg/adapter/gitlab/client.go +++ b/src/pkg/reg/adapter/gitlab/client.go @@ -25,6 +25,7 @@ import ( common_http "github.com/goharbor/harbor/src/common/http" liberrors "github.com/goharbor/harbor/src/lib/errors" + "github.com/goharbor/harbor/src/lib/log" "github.com/goharbor/harbor/src/pkg/reg/model" "github.com/goharbor/harbor/src/pkg/reg/util" ) @@ -82,7 +83,7 @@ func (c *Client) getProjects() ([]*Project, error) { func (c *Client) getProjectsByName(name string) ([]*Project, error) { var projects []*Project - urlAPI := fmt.Sprintf("%s/api/v4/projects?search=%s&membership=1&per_page=50", c.url, name) + urlAPI := fmt.Sprintf("%s/api/v4/projects?search=%s&membership=true&search_namespaces=true&per_page=50", c.url, name) if err := c.GetAndIteratePagination(urlAPI, &projects); err != nil { return nil, err } @@ -94,6 +95,7 @@ func (c *Client) getRepositories(projectID int64) ([]*Repository, error) { if err := c.GetAndIteratePagination(urlAPI, &repositories); err != nil { return nil, err } + log.Debugf("Count repositories %d in project %d", len(repositories), projectID) return repositories, nil } @@ -103,6 +105,7 @@ func (c *Client) getTags(projectID int64, repositoryID int64) ([]*Tag, error) { if err := c.GetAndIteratePagination(urlAPI, &tags); err != nil { return nil, err } + log.Debugf("Count tags %d in repository %d, and project %d", len(tags), repositoryID, projectID) return tags, nil } @@ -113,7 +116,6 @@ func (c *Client) GetAndIteratePagination(endpoint string, v interface{}) error { if err != nil { return err } - rv := reflect.ValueOf(v) if rv.Kind() != reflect.Ptr { return errors.New("v should be a pointer to a slice") @@ -122,7 +124,7 @@ func (c *Client) GetAndIteratePagination(endpoint string, v interface{}) error { if elemType.Kind() != reflect.Slice { return errors.New("v should be a pointer to a slice") } - + log.Debugf("Gitlab request %s", urlAPI) resources := reflect.Indirect(reflect.New(elemType)) for len(endpoint) > 0 { req, err := c.newRequest(http.MethodGet, endpoint, nil) diff --git a/src/pkg/reg/adapter/gitlab/testdata/projects/-docker.json b/src/pkg/reg/adapter/gitlab/testdata/projects/-docker.json deleted file mode 100644 index 3bd096f8bfd..00000000000 --- a/src/pkg/reg/adapter/gitlab/testdata/projects/-docker.json +++ /dev/null @@ -1,97 +0,0 @@ -[ - { - "id": 3, - "description": "", - "name": "dev-docker", - "name_with_namespace": "Library / dev-docker", - "path": "dev-docker", - "path_with_namespace": "library/dev-docker", - "created_at": "2019-01-17T09:47:07.504Z", - "default_branch": "master", - "tag_list": [], - - "avatar_url": null, - "star_count": 0, - "forks_count": 0, - "last_activity_at": "2019-06-09T15:18:10.045Z", - "empty_repo": false, - "archived": false, - "visibility": "private", - "resolve_outdated_diff_discussions": false, - "container_registry_enabled": true, - "issues_enabled": true, - "merge_requests_enabled": true, - "wiki_enabled": true, - "jobs_enabled": true, - "snippets_enabled": true, - "shared_runners_enabled": true, - "lfs_enabled": true, - "creator_id": 1, - "forked_from_project": {}, - "import_status": "finished", - "open_issues_count": 0, - "ci_default_git_depth": null, - "public_jobs": true, - "ci_config_path": null, - "shared_with_groups": [], - "only_allow_merge_if_pipeline_succeeds": false, - "request_access_enabled": false, - "only_allow_merge_if_all_discussions_are_resolved": false, - "printing_merge_request_link_enabled": true, - "merge_method": "merge", - "external_authorization_classification_label": "", - "permissions": { - "project_access": null, - "group_access": null - }, - "mirror": false - }, - { - "id": 4, - "description": "", - "name": "dev-docker-test", - "name_with_namespace": "Library / dev-docker-test", - "path": "dev-docker-test", - "path_with_namespace": "library/dev-docker-test", - "created_at": "2019-01-17T09:47:07.504Z", - "default_branch": "master", - "tag_list": [], - - "avatar_url": null, - "star_count": 0, - "forks_count": 0, - "last_activity_at": "2019-06-09T15:18:10.045Z", - "empty_repo": false, - "archived": false, - "visibility": "private", - "resolve_outdated_diff_discussions": false, - "container_registry_enabled": true, - "issues_enabled": true, - "merge_requests_enabled": true, - "wiki_enabled": true, - "jobs_enabled": true, - "snippets_enabled": true, - "shared_runners_enabled": true, - "lfs_enabled": true, - "creator_id": 1, - "forked_from_project": {}, - "import_status": "finished", - "open_issues_count": 0, - "ci_default_git_depth": null, - "public_jobs": true, - "ci_config_path": null, - "shared_with_groups": [], - "only_allow_merge_if_pipeline_succeeds": false, - "request_access_enabled": false, - "only_allow_merge_if_all_discussions_are_resolved": false, - "printing_merge_request_link_enabled": true, - "merge_method": "merge", - "external_authorization_classification_label": "", - "permissions": { - "project_access": null, - "group_access": null - }, - "mirror": false - } - -] \ No newline at end of file diff --git a/src/pkg/reg/adapter/gitlab/testdata/projects/dev-.json b/src/pkg/reg/adapter/gitlab/testdata/projects/dev-.json deleted file mode 100644 index 7c1b6237c16..00000000000 --- a/src/pkg/reg/adapter/gitlab/testdata/projects/dev-.json +++ /dev/null @@ -1,96 +0,0 @@ -[ - { - "id": 3, - "description": "", - "name": "dev-docker", - "name_with_namespace": "Library / dev-docker", - "path": "dev-docker", - "path_with_namespace": "library/dev-docker", - "created_at": "2019-01-17T09:47:07.504Z", - "default_branch": "master", - "tag_list": [], - - "avatar_url": null, - "star_count": 0, - "forks_count": 0, - "last_activity_at": "2019-06-09T15:18:10.045Z", - "empty_repo": false, - "archived": false, - "visibility": "private", - "resolve_outdated_diff_discussions": false, - "container_registry_enabled": true, - "issues_enabled": true, - "merge_requests_enabled": true, - "wiki_enabled": true, - "jobs_enabled": true, - "snippets_enabled": true, - "shared_runners_enabled": true, - "lfs_enabled": true, - "creator_id": 1, - "forked_from_project": {}, - "import_status": "finished", - "open_issues_count": 0, - "ci_default_git_depth": null, - "public_jobs": true, - "ci_config_path": null, - "shared_with_groups": [], - "only_allow_merge_if_pipeline_succeeds": false, - "request_access_enabled": false, - "only_allow_merge_if_all_discussions_are_resolved": false, - "printing_merge_request_link_enabled": true, - "merge_method": "merge", - "external_authorization_classification_label": "", - "permissions": { - "project_access": null, - "group_access": null - }, - "mirror": false - }, - { - "id": 4, - "description": "", - "name": "dev-docker-test", - "name_with_namespace": "Library / dev-docker-test", - "path": "dev-docker-test", - "path_with_namespace": "library/dev-docker-test", - "created_at": "2019-01-17T09:47:07.504Z", - "default_branch": "master", - "tag_list": [], - - "avatar_url": null, - "star_count": 0, - "forks_count": 0, - "last_activity_at": "2019-06-09T15:18:10.045Z", - "empty_repo": false, - "archived": false, - "visibility": "private", - "resolve_outdated_diff_discussions": false, - "container_registry_enabled": true, - "issues_enabled": true, - "merge_requests_enabled": true, - "wiki_enabled": true, - "jobs_enabled": true, - "snippets_enabled": true, - "shared_runners_enabled": true, - "lfs_enabled": true, - "creator_id": 1, - "forked_from_project": {}, - "import_status": "finished", - "open_issues_count": 0, - "ci_default_git_depth": null, - "public_jobs": true, - "ci_config_path": null, - "shared_with_groups": [], - "only_allow_merge_if_pipeline_succeeds": false, - "request_access_enabled": false, - "only_allow_merge_if_all_discussions_are_resolved": false, - "printing_merge_request_link_enabled": true, - "merge_method": "merge", - "external_authorization_classification_label": "", - "permissions": { - "project_access": null, - "group_access": null - }, - "mirror": false - } -] \ No newline at end of file