Merge pull request #1452 from cloud-pi-native/develop

Develop

apps/client/src/views/admin/AdminProject.vue

@@ -2,7 +2,7 @@
 import { onBeforeMount, ref } from 'vue'
 // @ts-ignore '@gouvminint/vue-dsfr' missing types
 import { getRandomId } from '@gouvminint/vue-dsfr'
-import type { Environment, Log, PluginsUpdateBody, ProjectService, ProjectV2, Repo } from '@cpn-console/shared'
+import type { CleanedCluster, Environment, Log, PluginsUpdateBody, ProjectService, ProjectV2, Repo, Zone } from '@cpn-console/shared'
 import fr from 'javascript-time-ago/locale/fr'
 import TimeAgo from 'javascript-time-ago'
 import { useSnackbarStore } from '@/stores/snackbar.js'
@@ -13,10 +13,14 @@ import { useStageStore } from '@/stores/stage.js'
 import { bts } from '@/utils/func.js'
 import { useLogStore } from '@/stores/log.js'
 import router from '@/router/index.js'
+import { useClusterStore } from '@/stores/cluster.js'
+import { useZoneStore } from '@/stores/zone.js'
 
 const props = defineProps<{ projectId: ProjectV2['id'] }>()
 
 const projectStore = useProjectStore()
+const zoneStore = useZoneStore()
+const clusterStore = useClusterStore()
 const userStore = useUserStore()
 const snackbarStore = useSnackbarStore()
 const quotaStore = useQuotaStore()
@@ -28,7 +32,7 @@ const repositoriesCtKey = ref(getRandomId('repository'))
 const isArchivingProject = ref(false)
 const projectToArchive = ref('')
 
-const headerEnvs = ['Nom', 'Type d\'environnement', 'Quota', 'Date']
+const headerEnvs = ['Nom', 'Type', 'Quota', 'Localisation', 'Date']
 const headerRepos = ['Nom', 'Type', 'Privé ?', 'url', 'Date']
 const membersId = 'membersTable'
 const repositoriesId = 'repositoriesTable'
@@ -37,7 +41,7 @@ const servicesId = 'servicesTable'
 const logsId = 'logsView'
 
 const project = computed(() => projectStore.projectsById[props.projectId])
-const environments = ref<Environment[]>()
+const environments = ref<(Environment & { cluster?: CleanedCluster & { zone?: Zone } })[]>()
 const repositories = ref<Repo[]>()
 // Add locale-specific relative date/time formatting rules.
 TimeAgo.addLocale(fr)
@@ -87,8 +91,20 @@ async function getProjectDetails() {
       project.value.refresh(),
       reloadProjectServices(),
       showLogs(0),
+      clusterStore.getClusters(),
+      zoneStore.getAllZones(),
     ])
-    environments.value = projectDetails.environments
+    environments.value = projectDetails.environments?.map((environment) => {
+      const cluster = clusterStore.clusters.find(cluster => cluster.id === environment.clusterId) as CleanedCluster
+      const zone = zoneStore.zones.find(zone => zone.id === cluster.zoneId) as Zone
+      return {
+        ...environment,
+        cluster: {
+          ...cluster,
+          zone,
+        },
+      }
+    })
     repositories.value = projectDetails.repositories
   } catch (error) {
     console.trace(error)
@@ -278,6 +294,10 @@ async function getProjectLogs({ offset, limit }: { offset: number, limit: number
               to: `#${servicesId}`,
               text: '#Services',
             },
+            {
+              to: `#${logsId}`,
+              text: '#Journaux',
+            },
           ]"
         />
         <hr>
@@ -317,6 +337,9 @@ async function getProjectLogs({ offset, limit }: { offset: number, limit: number
                   @update:model-value="(event: string) => updateEnvironmentQuota({ environmentId: env.id, quotaId: event })"
                 />
               </td>
+              <td>
+                {{ env.cluster?.label ?? 'Cluster inconnu' }} - {{ env.cluster?.zone?.label ?? 'Zone inconnue' }}
+              </td>
               <td
                 :title="(new Date(env.createdAt)).toLocaleString()"
               >

plugins/argocd/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@cpn-console/argocd-plugin",
   "type": "module",
-  "version": "2.1.0",
+  "version": "2.1.1",
   "private": false,
   "description": "",
   "main": "dist/index.js",

plugins/argocd/src/functions.ts

@@ -61,93 +61,97 @@ export const upsertProject: StepCall<Project> = async (payload) => {
 
     const applications = uniqueResource(applicationsList.body.items)
     const appProjects = uniqueResource(appProjectsList.body.items)
-    for (const environment of project.environments) {
-      const cluster = getCluster(project, environment)
-      const infraProject = await gitlabApi.getOrCreateInfraProject(cluster.zone.slug)
-      const appProjectName = generateAppProjectName(project.organization.name, project.name, environment.name)
-      const appNamespace = kubeApi.namespaces[environment.name].nsObject.metadata.name
-      const destination: ArgoDestination = {
-        namespace: appNamespace,
-        name: cluster.label,
-      }
-      const roGroup = (await keycloakApi.getEnvGroup(environment.name)).subgroups.RO
-      const rwGroup = (await keycloakApi.getEnvGroup(environment.name)).subgroups.RW
-
-      await ensureInfraEnvValues(
-        project,
-        environment,
-        appNamespace,
-        roGroup,
-        rwGroup,
-        appProjectName,
-        infraRepositories,
-        infraProject.id,
-        gitlabApi,
-        vaultApi,
-      )
-
-      if (!cluster.user.keyData && !cluster.user.token) {
-        console.log(`Direct argocd API calls are disabled for cluster ${cluster.label}`)
-        continue
-      }
 
-      // @ts-ignore
-      const appProject = findAppProject(appProjects, environment.name)
-      if (appProject) {
-        const minimalAppProject = getMinimalAppProjectPatch(
-          destination,
-          appProjectName,
-          sourceRepos,
-          roGroup,
-          rwGroup,
+    await Promise.all([
+      ...project.environments.map(async (environment) => {
+        const cluster = getCluster(project, environment)
+        const infraProject = await gitlabApi.getOrCreateInfraProject(cluster.zone.slug)
+        const appProjectName = generateAppProjectName(project.organization.name, project.name, environment.name)
+        const appNamespace = kubeApi.namespaces[environment.name].nsObject.metadata.name
+        const destination: ArgoDestination = {
+          namespace: appNamespace,
+          name: cluster.label,
+        }
+        const roGroup = (await keycloakApi.getEnvGroup(environment.name)).subgroups.RO
+        const rwGroup = (await keycloakApi.getEnvGroup(environment.name)).subgroups.RW
+
+        await ensureInfraEnvValues(
           project,
           environment,
-        )
-        await customK8sApi.patchNamespacedCustomObject('argoproj.io', 'v1alpha1', getConfig().namespace, 'appprojects', appProjectName, minimalAppProject, undefined, undefined, undefined, patchOptions)
-      } else {
-        const appProjectObject = getAppProjectObject({
-          name: appProjectName,
-          sourceRepos,
-          destination,
+          appNamespace,
           roGroup,
           rwGroup,
-          environment,
-          project,
-        })
-        await customK8sApi.createNamespacedCustomObject('argoproj.io', 'v1alpha1', getConfig().namespace, 'appprojects', appProjectObject)
-      }
+          appProjectName,
+          infraRepositories,
+          infraProject.id,
+          gitlabApi,
+          vaultApi,
+        )
 
-      // manage every infra repositories
-      for (const repository of infraRepositories) {
-        const application = findApplication(applications, repository.internalRepoName, environment.name)
-        const applicationName = generateApplicationName(project.organization.name, project.name, environment.name, repository.internalRepoName)
-        const repoURL = await gitlabApi.getRepoUrl(repository.internalRepoName)
+        if (!cluster.user.keyData && !cluster.user.token) {
+          console.log(`Direct argocd API calls are disabled for cluster ${cluster.label}`)
+          return undefined
+        }
 
-        if (application) {
-          const minimalPatch = getMinimalApplicationObject({
-            name: applicationName,
+        // @ts-ignore
+        const appProject = findAppProject(appProjects, environment.name)
+        if (appProject) {
+          const minimalAppProject = getMinimalAppProjectPatch(
             destination,
-            repoURL,
             appProjectName,
+            sourceRepos,
+            roGroup,
+            rwGroup,
             project,
-            repository,
             environment,
-          })
-          await customK8sApi.patchNamespacedCustomObject('argoproj.io', 'v1alpha1', getConfig().namespace, 'applications', application.metadata.name, minimalPatch, undefined, undefined, undefined, patchOptions)
+          )
+          await customK8sApi.patchNamespacedCustomObject('argoproj.io', 'v1alpha1', getConfig().namespace, 'appprojects', appProjectName, minimalAppProject, undefined, undefined, undefined, patchOptions)
         } else {
-          const applicationObject = getApplicationObject({
-            name: applicationName,
+          const appProjectObject = getAppProjectObject({
+            name: appProjectName,
+            sourceRepos,
             destination,
-            repoURL,
-            appProjectName,
-            project,
-            repository,
+            roGroup,
+            rwGroup,
             environment,
+            project,
           })
-          await customK8sApi.createNamespacedCustomObject('argoproj.io', 'v1alpha1', getConfig().namespace, 'applications', applicationObject)
+          await customK8sApi.createNamespacedCustomObject('argoproj.io', 'v1alpha1', getConfig().namespace, 'appprojects', appProjectObject)
         }
-      }
-    }
+
+        // manage every infra repositories
+        return infraRepositories.map(async (repository) => {
+          const application = findApplication(applications, repository.internalRepoName, environment.name)
+          const applicationName = generateApplicationName(project.organization.name, project.name, environment.name, repository.internalRepoName)
+          const repoURL = await gitlabApi.getRepoUrl(repository.internalRepoName)
+
+          if (application) {
+            const minimalPatch = getMinimalApplicationObject({
+              name: applicationName,
+              destination,
+              repoURL,
+              appProjectName,
+              project,
+              repository,
+              environment,
+            })
+            await customK8sApi.patchNamespacedCustomObject('argoproj.io', 'v1alpha1', getConfig().namespace, 'applications', application.metadata.name, minimalPatch, undefined, undefined, undefined, patchOptions)
+          } else {
+            const applicationObject = getApplicationObject({
+              name: applicationName,
+              destination,
+              repoURL,
+              appProjectName,
+              project,
+              repository,
+              environment,
+            })
+            await customK8sApi.createNamespacedCustomObject('argoproj.io', 'v1alpha1', getConfig().namespace, 'applications', applicationObject)
+          }
+        })
+      }),
+    ])
+
     await removeInfraEnvValues(project, gitlabApi)
 
     // then destroy what should not exist
@@ -262,7 +266,7 @@ async function getArgoRepoSource(repoName: string, env: string, gitlabApi: Gitla
   const targetRevision = 'HEAD'
   const repoId = await gitlabApi.getProjectId(repoName)
   const repoURL = await gitlabApi.getRepoUrl(repoName)
-  const files = await gitlabApi.listFiles(repoId, '/', 'HEAD')
+  const files = await gitlabApi.listFiles(repoId, { path: '/', ref: 'HEAD', recursive: false })
   const valueFiles = [] // Empty means not a Helm repository
   let path = '.'
   const result = files.find(f => f.name === 'values.yaml')
@@ -292,7 +296,7 @@ function getCluster(p: Project, e: Environment): ClusterObject {
 async function removeInfraEnvValues(project: Project, gitlabApi: GitlabProjectApi) {
   for (const z of getDistinctZones(project)) {
     const infraProject = await gitlabApi.getOrCreateInfraProject(z)
-    const existingFiles = await gitlabApi.listFiles(infraProject.id, `${project.name}/`)
+    const existingFiles = await gitlabApi.listFiles(infraProject.id, { path: `${project.name}/`, recursive: true })
     const neededFiles = project.environments.map(env => getValueFilePath(project, getCluster(project, env), env))
     const filesToDelete: string[] = []
     for (const existingFile of existingFiles) {
@@ -331,7 +335,7 @@ export const deleteProject: StepCall<Project> = async (payload) => {
 
     for (const z of getDistinctZones(project)) {
       const infraProject = await gitlabApi.getOrCreateInfraProject(z)
-      const projectValueFiles = await gitlabApi.listFiles(infraProject.id, project.name)
+      const projectValueFiles = await gitlabApi.listFiles(infraProject.id, { path: project.name, recursive: true })
       const filesToDelete = projectValueFiles.filter(f => f.type === 'blob').map(f => f.path)
       await gitlabApi.commitDelete(infraProject.id, filesToDelete)
     }

plugins/gitlab/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@cpn-console/gitlab-plugin",
   "type": "module",
-  "version": "2.4.1",
+  "version": "2.4.2",
   "private": false,
   "description": "",
   "main": "dist/index.js",

plugins/gitlab/src/class.ts

@@ -1,7 +1,7 @@
 import { createHash } from 'node:crypto'
 import { PluginApi, type Project, type RepoCreds, type UniqueRepo } from '@cpn-console/hooks'
 import type { AccessTokenScopes, CommitAction, GroupSchema, GroupStatisticsSchema, MemberSchema, ProjectVariableSchema, VariableSchema } from '@gitbeaker/rest'
-import type { CondensedProjectSchema, Gitlab } from '@gitbeaker/core'
+import type { AllRepositoryTreesOptions, CondensedProjectSchema, Gitlab, PaginationRequestOptions, RepositoryFileExpandedSchema, RepositoryTreeSchema } from '@gitbeaker/core'
 import { AccessLevel } from '@gitbeaker/core'
 import type { VaultProjectApi } from '@cpn-console/vault-plugin/types/class.js'
 import { objectEntries } from '@cpn-console/shared'
@@ -237,17 +237,18 @@ export class GitlabProjectApi extends PluginApi {
 
     const branches = await this.api.Branches.all(repoId)
     if (branches.some(b => b.name === branch)) {
-      const filesTree = await this.listFiles(repoId, '/', branch)
-      if (filesTree.find(f => f.path === filePath)) {
-        const actualFile = await this.api.RepositoryFiles.show(repoId, filePath, branch)
+      let actualFile: RepositoryFileExpandedSchema | undefined
+      try {
+        actualFile = await this.api.RepositoryFiles.show(repoId, filePath, branch)
+      } catch (_) {}
+      if (actualFile) {
         const newContentDigest = createHash('sha256').update(fileContent).digest('hex')
-        if (!actualFile || actualFile.content_sha256 !== newContentDigest) {
-          // Update needed
-          action = 'update'
-        } else {
+        if (actualFile.content_sha256 === newContentDigest) {
           // Already up-to-date
           return false
         }
+        // Update needed
+        action = 'update'
       }
     }
 
@@ -287,9 +288,23 @@ export class GitlabProjectApi extends PluginApi {
     return false
   }
 
-  public async listFiles(repoId: number, path: string = '/', branch: string = 'main') {
+  public async listFiles(repoId: number, options: AllRepositoryTreesOptions & PaginationRequestOptions<'keyset'> = {}) {
+    options.path = options?.path ?? '/'
+    options.ref = options?.ref ?? 'main'
+    options.recursive = options?.recursive ?? false
     try {
-      const files = await this.api.Repositories.allRepositoryTrees(repoId, { path, ref: branch, recursive: true, perPage: 1000 })
+      const files: RepositoryTreeSchema[] = await this.api.Repositories.allRepositoryTrees(repoId, options)
+      // if (depth >= 0) {
+      //   for (const file of files) {
+      //     if (file.type !== 'tree') {
+      //       return []
+      //     }
+      //     const childrenFiles = await this.listFiles(repoId, { depth: depth - 1, ...options, path: file.path })
+      //     console.trace({ file, childrenFiles })
+
+      //     files.push(...childrenFiles)
+      //   }
+      // }
       return files
     } catch (error) {
       const { cause } = error as GitbeakerRequestError