Merge pull request #264 from signaux-faibles/feat/supprime_une_liste_…

…de_predictions

Feat/supprime une liste de predictions

build-container/Dockerfile

@@ -2,11 +2,16 @@ FROM alpine:3.18.5
 RUN apk add --no-cache python3 python3-dev musl-dev libxml2 libxml2-dev libxslt libxslt-dev gcc
 RUN python3 -m ensurepip
 RUN pip3 install docx-mailmerge==0.5.0
+
 COPY ./docxify3.py /app/docxify3.py
 COPY ./template.docx /app/template.docx
 COPY ./datapi /app/datapi
 COPY ./migrations/ /app/migrations
 RUN chmod 555 /app/datapi
 WORKDIR /app
 EXPOSE 3000
+
+# emplacement des fichiers à intégrer pendant la phase d'intégration
+VOLUME /app/data
+
 CMD ["/app/datapi"]

config.toml.example

@@ -36,7 +36,6 @@ sourceEntreprise = "/foo/bar/dbmongo-data-export-entreprises.json.gz"
 sourceEtablissement = "/foo/bar/dbmongo-data-export-etablissements.json.gz"
 algoIgnore = ["algo_sans_urssaf"]
 batchIgnore = ["1910_8", "1906_7", "1901_interim"]
-refreshScript = "/foo/bar/so_fresh_so_clean.sql"
 
 # Wekan
 wekanURL = "wekanURL"

go.mod

@@ -25,6 +25,7 @@ require (
 	github.com/stretchr/testify v1.8.4
 	github.com/tealeg/xlsx v1.0.5
 	github.com/xuri/excelize/v2 v2.8.0
+	github.com/xwb1989/sqlparser v0.0.0-20180606152119-120387863bf2
 	go.mongodb.org/mongo-driver v1.14.0
 )
 

go.sum

@@ -244,6 +244,8 @@ github.com/xuri/excelize/v2 v2.8.0 h1:Vd4Qy809fupgp1v7X+nCS/MioeQmYVVzi495UCTqB7
 github.com/xuri/excelize/v2 v2.8.0/go.mod h1:6iA2edBTKxKbZAa7X5bDhcCg51xdOn1Ar5sfoXRGrQg=
 github.com/xuri/nfp v0.0.0-20230819163627-dc951e3ffe1a h1:Mw2VNrNNNjDtw68VsEj2+st+oCSn4Uz7vZw6TbhcV1o=
 github.com/xuri/nfp v0.0.0-20230819163627-dc951e3ffe1a/go.mod h1:WwHg+CVyzlv/TX9xqBFXEZAuxOPxn2k1GNHwG41IIUQ=
+github.com/xwb1989/sqlparser v0.0.0-20180606152119-120387863bf2 h1:zzrxE1FKn5ryBNl9eKOeqQ58Y/Qpo3Q9QNxKHX5uzzQ=
+github.com/xwb1989/sqlparser v0.0.0-20180606152119-120387863bf2/go.mod h1:hzfGeIUDq/j97IG+FhNqkowIyEcD88LrW6fyU3K3WqY=
 github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a h1:fZHgsYlfvtyqToslyjUt3VOPF4J7aK/3MPcK7xp3PDk=
 github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a/go.mod h1:ul22v+Nro/R083muKhosV54bj5niojjWZvU8xrevuH4=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=

main.go

@@ -2,9 +2,10 @@ package main
 
 import (
 	"context"
-	"datapi/pkg/ops/campaignops"
 	"log"
 
+	"datapi/pkg/ops/campaignops"
+
 	"github.com/gin-gonic/gin"
 	"github.com/spf13/viper"
 
@@ -13,7 +14,7 @@ import (
 	"datapi/pkg/kanban"
 	"datapi/pkg/ops/imports"
 	"datapi/pkg/ops/misc"
-	"datapi/pkg/ops/refresh"
+	"datapi/pkg/ops/scripts"
 	"datapi/pkg/stats"
 	"datapi/pkg/utils"
 )
@@ -51,7 +52,7 @@ func initAndStartAPI(datapi *core.Datapi, statsAPI *stats.API) {
 	needRoleStats := core.CheckAllRolesMiddleware("stats")
 	core.AddEndpoint(router, "/ops/utils", misc.ConfigureEndpoint, core.AdminAuthMiddleware)
 	core.AddEndpoint(router, "/ops/imports", imports.ConfigureEndpoint, core.AdminAuthMiddleware)
-	core.AddEndpoint(router, "/ops/refresh", refresh.ConfigureEndpoint, core.AdminAuthMiddleware)
+	core.AddEndpoint(router, "/ops/scripts", scripts.ConfigureEndpoint, core.AdminAuthMiddleware)
 	core.AddEndpoint(router, "/ops/campaign", campaignops.ConfigureEndpoint(datapi.KanbanService), core.AdminAuthMiddleware)
 	core.AddEndpoint(router, "/campaign", campaign.ConfigureEndpoint(datapi.KanbanService), core.AuthMiddleware(), datapi.LogMiddleware)
 	core.AddEndpoint(router, "/stats", statsAPI.ConfigureEndpoint, core.AuthMiddleware(), datapi.LogMiddleware, needRoleStats)

ops_import_integration_test.go

@@ -3,19 +3,81 @@
 package main
 
 import (
+	"context"
+	"net/http"
 	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"datapi/pkg/db"
+	"datapi/pkg/ops/imports"
+	"datapi/pkg/ops/scripts"
+	"datapi/pkg/test"
+	"datapi/pkg/utils"
 )
 
-// on pourra commiter ce test quand on aura des fichiers entreprise et etablissement anonymisés
-func TestMepHandler(t *testing.T) {
-	//ass := assert.New(t)
-	//rPath := "/utils/mep"
-	//err := test.Viperize(map[string]string{
-	//	"sourceEntreprise":    "/Users/raphaelsquelbut/Downloads/entreprise_sample.json.gz",
-	//	"sourceEtablissement": "/Users/raphaelsquelbut/Downloads/etablissement_sample.json.gz",
-	//})
-	//ass.NotNil(err)
-	//
-	//response := test.HTTPGet(t, rPath)
-	//ass.Equalf(http.StatusOK, response.StatusCode, "body de la réponse : %s", test.GetBodyQuietly(response))
+func TestApi_OpsImports_UrssafAggregate(t *testing.T) {
+	ass := assert.New(t)
+	path := "/ops/imports/urssaf/aggregate"
+	response := test.HTTPGet(t, path)
+
+	ass.Equalf(http.StatusOK, response.StatusCode, "body de la réponse : %s", test.GetBodyQuietly(response))
+}
+
+func Test_UrssafAgregate(t *testing.T) {
+	//utils.ConfigureLogLevel("warn")
+	ass := assert.New(t)
+	ctx := context.Background()
+
+	current := scripts.StartRefreshScript(ctx, db.Get(), imports.ExecuteAggregationURSSAF)
+
+	actual := scripts.Run{}
+	var err error
+	for actual.Status != scripts.Failed {
+		actual, err = scripts.Fetch(current.UUID)
+		require.NoError(t, err)
+		t.Log(actual)
+	}
+
+	ass.Equal(current.UUID, actual.UUID)
+	ass.Equal(scripts.Failed, actual.Status) // -> l'import des données URSSAF n'a pas eu lieu, l'erreur est normale
+}
+
+func TestApi_OpsImports_PredictionsRefresh(t *testing.T) {
+	ass := assert.New(t)
+	path := "/ops/imports/liste/refresh"
+	response := test.HTTPGet(t, path)
+
+	ass.Equalf(http.StatusOK, response.StatusCode, "body de la réponse : %s", test.GetBodyQuietly(response))
+}
+
+func Test_PredictionsRefresh(t *testing.T) {
+	utils.ConfigureLogLevel("warn")
+	ass := assert.New(t)
+	ctx := context.Background()
+
+	current := scripts.StartRefreshScript(ctx, db.Get(), imports.ExecuteRefreshVTables)
+
+	// Création d'un canal pour gérer le timeout
+	timeout := time.After(10 * time.Second) // Timeout de 5 secondes
+	actual := scripts.Run{}
+	var err error
+	// Boucle while avec timeout
+	for actual.Status != scripts.Finished {
+		time.Sleep(1 * time.Second)
+		select {
+		case <-timeout:
+			t.Error("time out !!!")
+			t.Fail()
+			return
+		default:
+			actual, err = scripts.Fetch(current.UUID)
+			require.NoError(t, err)
+		}
+	}
+
+	ass.Equal(current.UUID, actual.UUID)
+	ass.Equal(scripts.Finished, actual.Status)
 }