Add support of remote task on remote Pipeline

We now support remote tasks on remote Pipeline, allowing to share
a remote Pipeline across multiple repositories.

User can override tasks from the remote pipeline by adding a task with
the same name.

Signed-off-by: Chmouel Boudjnah <[email protected]>

docs/content/docs/guide/resolver.md

@@ -174,18 +174,51 @@ If the object fetched cannot be parsed as a Tekton `Task` it will error out.
 
 ## Remote Pipeline annotations
 
-Remote Pipeline can be referenced by annotation, this allows you to share your Pipeline definition across.
+Remote Pipeline can be referenced by annotation, allowing you to share a Pipeline across multiple repositories.
 
-Only one Pipeline is allowed in annotation.
+Only one Pipeline is allowed on the `PipelineRun` annotation.
 
-An annotation to a remote pipeline looks like this :
+An annotation to a remote pipeline looks like this, using a remote URL:
 
 ```yaml
 pipelinesascode.tekton.dev/pipeline: "https://git.provider/raw/pipeline.yaml
 ```
 
-It supports remote URL and files inside the same Git repository.
+or from a relative path inside the repository:
 
-{{< hint info >}}
-[Tekton Hub](https://hub.tekton.dev) doesn't currently have support for `Pipeline`.
-{{< /hint >}}
+```yaml
+pipelinesascode.tekton.dev/pipeline: "./tasks/pipeline.yaml
+```
+
+Fetching `Pipelines` from the [Tekton Hub](https://hub.tekton.dev) is not currently supported.
+
+### Overriding tasks from a remote pipeline from a PipelineRun
+
+Remote task annotations on the remote pipeline are supported but no other annotations are supported.
+
+If a user wants to override one of the tasks in the remote pipeline, they can do so by adding a task with the same name
+n their `PipelineRun` annotations.
+
+For example if the user PipelineRun contains those annotations:
+
+```yaml
+kind: PipelineRun
+metadata:
+  annotations:
+    pipelinesascode.tekton.dev/pipeline: "https://git.provider/raw/pipeline.yaml
+    pipelinesascode.tekton.dev/task-1: "./my-git-clone-task.yaml
+```
+
+and the Pipeline referenced by the `pipelinesascode.tekton.dev/pipeline` annotation
+in "<https://git.provider/raw/pipeline.yaml>"  contains those annotations:
+
+```yaml
+kind: Pipeline
+metadata:
+  annotations:
+    pipelinesascode.tekton.dev/task-1: "./my-task.yaml
+    pipelinesascode.tekton.dev/task-2: "git-clone"
+```
+
+In this case if the `my-git-clone-task.yaml` file contains a task named `git-clone` it will be used instead
+of the one from the remote pipeline.

pkg/action/patch_test.go

@@ -22,7 +22,7 @@ func TestPatchPipelineRun(t *testing.T) {
 	observer, _ := zapobserver.New(zap.InfoLevel)
 	logger := zap.New(observer).Sugar()
 
-	testPR := tektontest.MakePR("namespace", "force-me", []pipelinev1.ChildStatusReference{
+	testPR := tektontest.MakePRStatus("namespace", "force-me", []pipelinev1.ChildStatusReference{
 		tektontest.MakeChildStatusReference("first"),
 		tektontest.MakeChildStatusReference("last"),
 		tektontest.MakeChildStatusReference("middle"),

pkg/pipelineascode/pipelineascode_test.go

@@ -517,7 +517,7 @@ func TestRun(t *testing.T) {
 				},
 				Repositories: tt.repositories,
 				PipelineRuns: []*pipelinev1.PipelineRun{
-					tektontest.MakePR("namespace", "force-me", []pipelinev1.ChildStatusReference{
+					tektontest.MakePRStatus("namespace", "force-me", []pipelinev1.ChildStatusReference{
 						tektontest.MakeChildStatusReference("first"),
 						tektontest.MakeChildStatusReference("last"),
 						tektontest.MakeChildStatusReference("middle"),

pkg/reconciler/controller_test.go

@@ -33,7 +33,7 @@ func TestCheckStateAndEnqueue(t *testing.T) {
 	})
 
 	// Create a new PipelineRun object with the "started" state label.
-	testPR := tektontest.MakePR("namespace", "force-me", []pipelinev1.ChildStatusReference{
+	testPR := tektontest.MakePRStatus("namespace", "force-me", []pipelinev1.ChildStatusReference{
 		tektontest.MakeChildStatusReference("first"),
 		tektontest.MakeChildStatusReference("last"),
 		tektontest.MakeChildStatusReference("middle"),

pkg/resolve/resolve.go

@@ -25,6 +25,19 @@ type TektonTypes struct {
 	Tasks        []*tektonv1.Task
 }
 
+type NamedItem interface {
+	GetName() string
+}
+
+func alreadySeen[T NamedItem](items []T, item T) bool {
+	for _, value := range items {
+		if value.GetName() == item.GetName() {
+			return true
+		}
+	}
+	return false
+}
+
 var yamlDocSeparatorRe = regexp.MustCompile(`(?m)^---\s*$`)
 
 func ReadTektonTypes(ctx context.Context, log *zap.SugaredLogger, data string) (TektonTypes, error) {
@@ -136,6 +149,70 @@ type Opts struct {
 	ProviderToken string
 }
 
+// getRemoteTasks will get remote tasks from annotations on pipelinerun, remote
+// pipeline and remote tasks from remote pipeline unless already requested in
+// PipelineRun
+// This means the PipelineRun task will always take precedence and this allows
+// for example overriding the tasks from the global template with a task locally.
+func getRemoteTasks(ctx context.Context, rt *matcher.RemoteTasks, types *TektonTypes) error {
+	for _, pipelinerun := range types.PipelineRuns {
+		if len(pipelinerun.GetObjectMeta().GetAnnotations()) == 0 {
+			continue
+		}
+
+		remoteTasks, err := rt.GetTaskFromAnnotations(ctx, pipelinerun.GetObjectMeta().GetAnnotations())
+		if err != nil {
+			return fmt.Errorf("error getting remote task from pipelinerun annotations: %w", err)
+		}
+
+		// Merge remote tasks with local tasks
+		for _, remoteTask := range remoteTasks {
+			if alreadySeen(types.Tasks, remoteTask) {
+				rt.Logger.Infof("skipping remote task %s from remote pipelinerun %s as already seen", remoteTask.GetName(), pipelinerun.GetName())
+				continue
+			}
+			types.Tasks = append(types.Tasks, remoteTask)
+		}
+
+		remotePipelines, err := rt.GetPipelineFromAnnotations(ctx, pipelinerun.GetObjectMeta().GetAnnotations())
+		if err != nil {
+			return fmt.Errorf("error getting remote pipeline from pipelinerun annotation: %w", err)
+		}
+		// make sure it's not already there for example if it was in .tekton/
+		// it would have been already grabbed since we grab everything from
+		// there, and if the user specify it in the annotations we don't want
+		// to add it twice and resolve the remote annotations on there twice (or more).
+		// this avoid conflicts as well
+		for _, remotePipeline := range remotePipelines {
+			if alreadySeen(types.Pipelines, remotePipeline) {
+				rt.Logger.Debugf("skipping remote pipeline %s as already seend", remotePipeline.GetName())
+				continue
+			}
+			types.Pipelines = append(types.Pipelines, remotePipeline)
+		}
+	}
+
+	// grab the tasks from the remote pipeline
+	for _, pipeline := range types.Pipelines {
+		if pipeline.GetObjectMeta().GetAnnotations() == nil {
+			continue
+		}
+		remoteTasks, err := rt.GetTaskFromAnnotations(ctx, pipeline.GetObjectMeta().GetAnnotations())
+		if err != nil {
+			return fmt.Errorf("error getting remote tasks from remote pipeline %s: %w", pipeline.GetName(), err)
+		}
+
+		for _, remoteTask := range remoteTasks {
+			if alreadySeen(types.Tasks, remoteTask) {
+				rt.Logger.Infof("skipping remote task %s from remote pipeline %s as already defined in pipelinerun", remoteTask.GetName(), pipeline.GetName())
+				continue
+			}
+			types.Tasks = append(types.Tasks, remoteTask)
+		}
+	}
+	return nil
+}
+
 // Resolve gets a large string which is a yaml multi documents containing
 // Pipeline/PipelineRuns/Tasks and resolve them inline as a single PipelineRun
 // generateName can be set as True to set the name as a generateName + "-" for
@@ -149,27 +226,17 @@ func Resolve(ctx context.Context, cs *params.Run, logger *zap.SugaredLogger, pro
 		return []*tektonv1.PipelineRun{}, err
 	}
 
-	// First resolve Annotations Tasks
-	for _, pipelinerun := range types.PipelineRuns {
-		if ropt.RemoteTasks && pipelinerun.GetObjectMeta().GetAnnotations() != nil {
-			rt := matcher.RemoteTasks{
-				Run:               cs,
-				Event:             event,
-				ProviderInterface: providerintf,
-				Logger:            logger,
-			}
-			remoteTasks, err := rt.GetTaskFromAnnotations(ctx, pipelinerun.GetObjectMeta().GetAnnotations())
-			if err != nil {
-				return []*tektonv1.PipelineRun{}, err
-			}
-			// Merge remote tasks with local tasks
-			types.Tasks = append(types.Tasks, remoteTasks...)
-
-			remotePipelines, err := rt.GetPipelineFromAnnotations(ctx, pipelinerun.GetObjectMeta().GetAnnotations())
-			if err != nil {
-				return []*tektonv1.PipelineRun{}, err
-			}
-			types.Pipelines = append(types.Pipelines, remotePipelines...)
+	// Resolve remote annotations on remote task or remote pipeline or tasks
+	// inside remote pipeline
+	if ropt.RemoteTasks {
+		rt := &matcher.RemoteTasks{
+			Run:               cs,
+			Event:             event,
+			ProviderInterface: providerintf,
+			Logger:            logger,
+		}
+		if err := getRemoteTasks(ctx, rt, &types); err != nil {
+			return []*tektonv1.PipelineRun{}, err
 		}
 	}