diff --git a/dao/courses.go b/dao/courses.go index 542fd92b4..47f4069b3 100644 --- a/dao/courses.go +++ b/dao/courses.go @@ -35,6 +35,7 @@ type CoursesDao interface { GetAvailableSemesters(c context.Context) []model.Semester GetCourseByShortLink(link string) (model.Course, error) GetCourseAdmins(courseID uint) ([]model.User, error) + // ExecAllCourses executes f on all courses from database without batching ExecAllCourses(f func([]Course)) UpdateCourse(ctx context.Context, course model.Course) error @@ -289,23 +290,17 @@ type Course struct { } // ExecAllCourses executes f on all courses. +// +// loads every course into memory func (d coursesDao) ExecAllCourses(f func([]Course)) { var res []Course - batchNum := 0 - batchSize := 100 - var numCourses int64 - DB.Model(&model.Course{}).Count(&numCourses) - for batchSize*batchNum < int(numCourses) { - err := DB.Raw(`SELECT id, name, slug, year, teaching_term, visibility + err := DB.Raw(`SELECT id, name, slug, year, teaching_term, visibility FROM courses - WHERE deleted_at IS NULL - LIMIT ? OFFSET ?`, batchSize, batchNum*batchSize).Scan(&res).Error - if err != nil { - fmt.Println(err) - } - f(res) - batchNum++ + WHERE deleted_at IS NULL ORDER BY id`).Scan(&res).Error + if err != nil { + fmt.Println(err) } + f(res) } func (d coursesDao) UpdateCourse(ctx context.Context, course model.Course) error { diff --git a/tools/meiliExporter.go b/tools/meiliExporter.go index a1db0a6c8..14253f4b8 100644 --- a/tools/meiliExporter.go +++ b/tools/meiliExporter.go @@ -140,7 +140,7 @@ func (m *MeiliExporter) Export() { Visibility: course.Visibility, } } - _, err := coursesIndex.AddDocuments(&meilicourses, "ID") + _, err := coursesIndex.AddDocumentsInBatches(&meilicourses, 500, "ID") if err != nil { logger.Error("issue adding courses to meili", "err", err) }