feature: complete image service

cmd/media_service/main.go

@@ -9,13 +9,14 @@ import (
 func main() {
 	//Load secrets
 	_ = godotenv.Load()
-	secrets, err := utils.LoadSecrets(utils.LoadEnv())
+	env := utils.LoadEnv()
+	secrets, err := utils.LoadSecrets(env)
 	if err != nil {
 		panic(err)
 	}
-	infra, err := media.Setup(media.NewConfig(secrets))
+	infra, err := media.Setup(media.NewConfig(env, secrets))
 	if err != nil {
 		panic(err)
 	}
-	media.NewServer(infra).Start()
+	media.NewHttpServer(infra)
 }

lib/media/storage.go

@@ -0,0 +1,12 @@
+package media
+
+type Storage interface {
+	// Store
+	// Stores the image in the storage and returns the URL
+	Store(path string, data []byte) error
+
+	Delete(path string) error
+
+	GetHost() string
+	GetUrl(path string) string
+}

lib/media/tmp_image.go

@@ -10,14 +10,14 @@ type TmpImage struct {
 	ExpectedUsage Usage
 	Uploader      uuid.UUID
 	UploadedAt    time.Time
-	URL           string
+	Path          string
 }
 
 type ConfirmedImage struct {
 	Id          uuid.UUID
 	Usage       Usage
 	Uploader    uuid.UUID
 	UploadedAt  time.Time
-	URL         string
+	Path        string
 	ConfirmedAt time.Time
 }

lib/utils/infisical.go

@@ -4,9 +4,11 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
+	"github.com/joho/godotenv"
 	"io"
 	"net/http"
 	"os"
+	"path/filepath"
 )
 
 type idLoginBody struct {
@@ -67,6 +69,7 @@ func GetSecrets(token string, environment string) (Secrets, error) {
 }
 
 func LoadSecrets(env string) (map[string]string, error) {
+	loadEnvFile()
 	println("Loading secrets with env: ", env)
 	id, ok := os.LookupEnv("CLIENT_ID")
 	if !ok {
@@ -92,6 +95,7 @@ func LoadSecrets(env string) (map[string]string, error) {
 }
 
 func LoadEnv() string {
+	loadEnvFile()
 	env, ok := os.LookupEnv("ENVIRONMENT")
 	if !ok {
 		return "dev"
@@ -107,3 +111,11 @@ func LoadEnv() string {
 		return "dev"
 	}
 }
+
+func loadEnvFile() {
+	envFilePath, err := filepath.Abs("../../.env")
+	if err != nil {
+		panic(err)
+	}
+	_ = godotenv.Load(envFilePath)
+}

migrations/2406170339_create_image_table.up.sql

@@ -1,16 +1,17 @@
 CREATE TABLE TmpImage(
     imgId uuid PRIMARY KEY,
-    url varchar(100) NOT NULL ,
+    path varchar(100) NOT NULL ,
     expected_usage int8 NOT NULL ,
     uploader uuid references user_identity(user_id) NOT NULL,
     uploaded_at timestamp NOT NULL
 );
 
 CREATE TABLE ConfirmedImage(
     imgId uuid PRIMARY KEY,
-    url varchar(100) NOT NULL ,
+    path varchar(100) NOT NULL ,
     usage int8 NOT NULL ,
     uploader uuid references user_identity(user_id) NOT NULL,
     uploaded_at timestamp NOT NULL,
     confirmed_at timestamp NOT NULL
-)
+)
+

services/api/controllers/group/get_group.go

@@ -60,7 +60,7 @@ func GetGroupInfo(ctx context.Context, groupId uuid.UUID) (*monify.GetGroupInfoR
 	response := &monify.GetGroupInfoResponse{
 		GroupId: groupId.String(),
 	}
-	err := db.QueryRowContext(ctx, "SELECT name, description, avatar_url FROM 'group' WHERE group_id = $1", groupId).Scan(&response.Name, &response.Description, &response.AvatarUrl)
+	err := db.QueryRowContext(ctx, `SELECT name, description, avatar_url FROM "group" WHERE group_id = $1`, groupId).Scan(&response.Name, &response.Description, &response.AvatarUrl)
 	if err != nil {
 		if err == sql.ErrNoRows {
 			return nil, status.Error(codes.NotFound, "Group not found")

services/api/controllers/group_bill/create.go

@@ -74,7 +74,7 @@ func (s Service) CreateGroupBill(ctx context.Context, req *monify.CreateGroupBil
 		BillId:           billId,
 		Title:            req.Title,
 	}); err != nil {
-		logger.Error("", zap.Error(err))
+
 		return nil, status.Error(codes.Internal, "Internal")
 	}
 

services/api/controllers/group_bill/service.go

@@ -28,11 +28,13 @@ func processGroupBillModifyEvent(ctx context.Context, db *sql.Tx, history group_
 	kfWriter := ctx.Value(lib.KafkaWriterContextKey{}).(infra.KafkaWriters)
 	serialized, err := json.Marshal(history)
 	if err != nil {
+		logger.Error("", zap.Error(err))
 		return err
 	}
 	if err = kfWriter.GroupBill.WriteMessages(ctx,
 		kafka.Message{Value: serialized},
 	); err != nil {
+		logger.Error("", zap.Error(err))
 		return err
 	}
 	return nil

services/media/README.md

@@ -0,0 +1,9 @@
+# MediaService
+
+This service has two main functions
+
+1. User can upload temporary images, temp images will be cleaned up after some duration.
+2. Other services confirm the usage of the temporary images and make them temporary.
+
+Upload images with port 8080 using multipart form file with key "image"  
+Confirm usages with port 8081 using gprc.

services/media/api_test.go

@@ -1,7 +1,111 @@
 package media
 
-import "testing"
+import (
+	"bytes"
+	"encoding/json"
+	"github.com/stretchr/testify/assert"
+	"io"
+	"mime/multipart"
+	"monify/lib/utils"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"sync"
+	"testing"
+)
 
-func TestW(t *testing.T) {
+type TestServerState struct {
+	server  Server
+	infra   Infra
+	started bool
+	mutex   sync.Mutex
+}
+
+var state TestServerState
+
+func SetupTestServer() {
+	state.mutex.Lock()
+	if state.started {
+		state.mutex.Unlock()
+		return
+	}
+
+	secrets, err := utils.LoadSecrets(utils.LoadEnv())
+	if err != nil {
+		panic(err)
+	}
+	infra, err := Setup(NewConfig("dev", secrets))
+	if err != nil {
+		panic(err)
+	}
+	state.infra = infra
+	state.server = NewServer(infra)
+	state.started = true
+	state.mutex.Unlock()
+}
+
+func getTestFilePath() string {
+	abs, err := filepath.Abs("test.png")
+	if err != nil {
+		panic(err)
+	}
+	return abs
+}
+
+func TestS3Storage(t *testing.T) {
+	SetupTestServer()
+	fp := getTestFilePath()
+	file, err := os.ReadFile(fp)
+	if err != nil {
+		panic(err)
+	}
+
+	err = state.infra.objStorage.Delete("test.png")
+	assert.NoError(t, err)
+	err = state.infra.objStorage.Store("test.png", file)
+	assert.NoError(t, err)
+
+	response, err := http.Get(state.infra.objStorage.GetUrl("test.png"))
+	assert.NoError(t, err)
+	assert.Equal(t, http.StatusOK, response.StatusCode)
+
+	err = state.infra.objStorage.Delete("test.png")
+	assert.NoError(t, err)
+
+	response, err = http.Get(state.infra.objStorage.GetUrl("test.png"))
+	assert.NoError(t, err)
+	assert.Equal(t, http.StatusForbidden, response.StatusCode)
+}
+
+func TestUpload(t *testing.T) {
+	SetupTestServer()
+	fp := getTestFilePath()
+	file, err := os.Open(fp)
+	assert.NoError(t, err)
+	defer file.Close()
+
+	var requestBody bytes.Buffer
+	writer := multipart.NewWriter(&requestBody)
+	// Add the file to the form
+	part, err := writer.CreateFormFile("image", filepath.Base("test.png"))
+	assert.NoError(t, err)
+	_, err = io.Copy(part, file)
+	assert.NoError(t, err)
+	err = writer.Close()
+	assert.NoError(t, err)
+	req, err := http.NewRequest("POST", "/image", &requestBody)
+	assert.NoError(t, err)
+	req.Header.Set("Content-Type", writer.FormDataContentType())
+
+	// Create a response recorder
+	response := httptest.NewRecorder()
+	state.server.mux.ServeHTTP(response, req)
+	assert.Equal(t, http.StatusOK, response.Code)
 
+	resBody := UploadImageResponse{}
+	err = json.Unmarshal(response.Body.Bytes(), &resBody)
+	assert.NoError(t, err)
+	println(resBody.Url)
+	println(resBody.ImageId)
 }

services/media/config.go

@@ -1,6 +1,7 @@
 package media
 
 type Config struct {
+	Environment string
 	PostgresURI string
 	S3Host      string
 	S3Bucket    string
@@ -10,8 +11,9 @@ type Config struct {
 	JwtSecret   string
 }
 
-func NewConfig(secrets map[string]string) Config {
+func NewConfig(env string, secrets map[string]string) Config {
 	return Config{
+		Environment: env,
 		JwtSecret:   secrets["JWT_SECRET"],
 		PostgresURI: secrets["POSTGRES_URI"],
 		S3Host:      secrets["S3_HOST"],

services/media/handler.go

@@ -31,8 +31,8 @@ func uploadImage(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	// Parse our multipart form, 10 << 20 specifies a maximum upload of 10 MB files.
-	parseErr := r.ParseMultipartForm(10 << 20)
+	// Parse our multipart form, a maximum upload of 5 MB files.
+	parseErr := r.ParseMultipartForm(5 << 20)
 	if parseErr != nil {
 		http.Error(w, "Could not parse multipart form", http.StatusBadRequest)
 		return
@@ -47,10 +47,12 @@ func uploadImage(w http.ResponseWriter, r *http.Request) {
 	}
 
 	// Get the file from the form data
-	file, handler, err := r.FormFile("image")
+	file, fileHeader, err := r.FormFile("image")
 	if err != nil {
 		fmt.Println("Error Retrieving the File")
 		fmt.Println(err)
+		http.Error(w, "Could not retrieve the file", http.StatusBadRequest)
+		return
 	}
 	defer file.Close()
 
@@ -59,23 +61,26 @@ func uploadImage(w http.ResponseWriter, r *http.Request) {
 	if err != nil {
 		fmt.Println("Error Reading the File")
 		fmt.Println(err)
+		http.Error(w, "Could not read the file", http.StatusBadRequest)
+		return
 	}
 
 	if !strings.Contains(http.DetectContentType(imageData), "image") {
 		http.Error(w, "該檔案不是圖片", http.StatusBadRequest)
+		return
 	}
 
-	imageStorage := r.Context().Value(lib.ImageStorageContextKey{}).(media.ImageStorage)
-	config := r.Context().Value(lib.ConfigContextKey{}).(Config)
+	imageStorage := r.Context().Value(lib.ImageStorageContextKey{}).(media.Storage)
 	imgId := uuid.New()
-	url, err := imageStorage.Store(extractFileNameSuffix(handler.Filename), imageData, imgId.String())
+	path := generatePath(fileHeader.Filename, imgId)
+	err = imageStorage.Store(path, imageData)
 	if err != nil {
 		http.Error(w, err.Error(), http.StatusInternalServerError)
 		return
 	}
 
 	tmpImage := media.TmpImage{
-		URL:           url,
+		Path:          path,
 		ExpectedUsage: usage,
 		Uploader:      userId,
 		Id:            imgId,
@@ -89,7 +94,7 @@ func uploadImage(w http.ResponseWriter, r *http.Request) {
 
 	//Success Response
 	res := UploadImageResponse{
-		Url:     config.S3Host + url,
+		Url:     imageStorage.GetUrl(path),
 		ImageId: imgId.String(),
 	}
 	w.Header().Set("Content-Type", "application/json")
@@ -106,13 +111,20 @@ func uploadImage(w http.ResponseWriter, r *http.Request) {
 	return
 }
 
+func generatePath(originFileName string, imgId uuid.UUID) string {
+	now := time.Now()
+	fn := imgId.String() + "/" + extractFileNameSuffix(originFileName)
+	dir := fmt.Sprintf("%d/%d/%d", now.Year(), now.Month(), now.Day())
+	return dir + "/" + fn
+}
+
 func extractFileNameSuffix(fileName string) string {
 	split := strings.Split(fileName, ".")
 	return split[len(split)-1]
 }
 
 func StoreTmpImg(ctx context.Context, img media.TmpImage) error {
 	db := ctx.Value(lib.DatabaseContextKey{}).(*sql.DB)
-	_, err := db.ExecContext(ctx, "INSERT INTO tmpimage (imgid, url, uploader, expected_usage, uploaded_at) VALUES ($1, $2, $3, $4, $5)", img.Id, img.URL, img.Uploader, img.ExpectedUsage, img.UploadedAt)
+	_, err := db.ExecContext(ctx, "INSERT INTO tmpimage (imgid, url, uploader, expected_usage, uploaded_at) VALUES ($1, $2, $3, $4, $5)", img.Id, img.Path, img.Uploader, img.ExpectedUsage, img.UploadedAt)
 	return err
 }