Merge branch 'develop'

Include several fixes in the latest release

Dockerfile

@@ -0,0 +1,8 @@
+FROM golang:buster as build
+WORKDIR /app
+ADD ./ /app
+RUN ./build.sh
+
+FROM scratch
+COPY --from=build /app/prosody-filer /prosody-filer
+ENTRYPOINT ["/prosody-filer"]

README.md

@@ -8,19 +8,7 @@ A simple file server for handling XMPP http_upload requests. This server is mean
 
 ## Why should I use this server?
 
-* Prosody developers recommend using http_upload_external instead of http_upload (Matthew Wild on the question if http_upload is memory leaking):
-    > "BTW, I am not aware of any memory leaks in the HTTP upload code. However it is known to be very inefficient.
-    > That's why it has a very low upload limit, and **we encourage people to use mod_http_upload_external instead**.
-    > We set out to write a good XMPP server, not  HTTP server (of which many good ones already exist), so our HTTP server is optimised for small bits of data, like BOSH and websocket.
-    > Handling large uploads and downloads was not a goal (and implementing a great HTTP server is not a high priority for the project compared to other things).
-    > **Our HTTP code buffers the entire upload into memory.
-    > More, it does it in an inefficient way that can use up to 4x the actual size of the data (if the data is large).
-    > So uploading a 10MB file can in theory use 40MB RAM.**
-    > But it's not a leak, the RAM is later cleared and reused. [...]
-    > The GC will free the memory at some point, but the OS may still report that Prosody is using that memory due to the way the libc allocator works.
-    > Most long lived processes behave this way (only increasing RAM, rarely decreasing)."
-* This server works without any script interpreters or additional dependencies. It is delivered as a binary.
-* Go is very good at serving HTTP requests and "made for this task".
+Originally this software was written to circumvent memory limitations / issues with the Prosody-internal http_upload implementation at the time. These limitations do not exist, anymore. Still this software can be used with Ejabberd and Prosody as an alternative to the internal http_upload servers. 
 
 
 ## Download
@@ -37,7 +25,7 @@ To compile the server, you need a full Golang development environment. This can
 Then checkout this repo:
 
 ```sh
-go get github.com/ThomasLeister/prosody-filer
+go install github.com/ThomasLeister/prosody-filer
 ```
 
 and switch to the new directory:
@@ -140,6 +128,17 @@ In addition to that, make sure that the nginx user or group can read the files u
 via prosody-filer if you want to have them served by nginx directly.
 
 
+### Docker usage 
+
+To build container:
+
+```docker build . -t prosody-filer:latest```
+
+To run container use:
+
+```docker run -it --rm -v $PWD/config.example.toml:/config.toml prosody-filer -config /config.toml```
+
+
 ### Systemd service file
 
 Create a new Systemd service file: ```/etc/systemd/system/prosody-filer.service```
@@ -309,7 +308,7 @@ server {
 
 Prosody Filer has no immediate knowlegde over all the stored files and the time they were uploaded, since no database exists for that. Also Prosody is not capable to do auto deletion if *mod_http_upload_external* is used. Therefore the suggested way of purging the uploads directory is to execute a purge command via a cron job:
 
-    @daily    find /home/prosody-filer/upload/ -type d -mtime +28 | xargs rm -rf
+    @daily    find /home/prosody-filer/upload/ -mindepth 1 -type d -mtime +28 -print0 | xargs -0 -- rm -rf
 
 This will delete uploads older than 28 days.
 

go.mod

@@ -0,0 +1,5 @@
+module github.com/ThomasLeister/prosody-filer
+
+go 1.16
+
+require github.com/BurntSushi/toml v1.3.2

go.sum

@@ -0,0 +1,2 @@
+github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8=
+github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=

prosody-filer.go

@@ -15,9 +15,11 @@ import (
 	"io/ioutil"
 	"log"
 	"mime"
+	"net"
 	"net/http"
 	"net/url"
 	"os"
+	"path"
 	"path/filepath"
 	"strconv"
 	"strings"
@@ -30,6 +32,7 @@ import (
  */
 type Config struct {
 	Listenport   string
+	UnixSocket   bool
 	Secret       string
 	Storedir     string
 	UploadSubDir string
@@ -38,12 +41,22 @@ type Config struct {
 var conf Config
 var versionString string = "0.0.0"
 
+var ALLOWED_METHODS string = strings.Join(
+    []string{
+        http.MethodOptions,
+        http.MethodHead,
+        http.MethodGet,
+        http.MethodPut,
+    },
+    ", ",
+)
+
 /*
  * Sets CORS headers
  */
 func addCORSheaders(w http.ResponseWriter) {
 	w.Header().Set("Access-Control-Allow-Origin", "*")
-	w.Header().Set("Access-Control-Allow-Methods", "OPTIONS, HEAD, GET, PUT")
+	w.Header().Set("Access-Control-Allow-Methods", ALLOWED_METHODS)
 	w.Header().Set("Access-Control-Allow-Headers", "Authorization, Content-Type")
 	w.Header().Set("Access-Control-Allow-Credentials", "true")
 	w.Header().Set("Access-Control-Max-Age", "7200")
@@ -57,22 +70,31 @@ func handleRequest(w http.ResponseWriter, r *http.Request) {
 	log.Println("Incoming request:", r.Method, r.URL.String())
 
 	// Parse URL and args
-	u, err := url.Parse(r.URL.String())
-	if err != nil {
-		log.Println("Failed to parse URL:", err)
-	}
+	p := r.URL.Path
 
-	a, err := url.ParseQuery(u.RawQuery)
+	a, err := url.ParseQuery(r.URL.RawQuery)
 	if err != nil {
 		log.Println("Failed to parse URL query params:", err)
+		http.Error(w, "500 Internal Server Error", 500)
+		return
 	}
 
-	fileStorePath := strings.TrimPrefix(u.Path, "/"+conf.UploadSubDir)
+	subdir := path.Join("/", conf.UploadSubDir)
+	fileStorePath := strings.TrimPrefix(p, subdir)
+	if fileStorePath == "" || fileStorePath == "/" {
+		log.Println("Empty request URL")
+		http.Error(w, "403 Forbidden", 403)
+		return
+	} else if fileStorePath[0] == '/' {
+		fileStorePath = fileStorePath[1:]
+	}
+
+	absFilename := filepath.Join(conf.Storedir, fileStorePath)
 
 	// Add CORS headers
 	addCORSheaders(w)
 
-	if r.Method == "PUT" {
+	if r.Method == http.MethodPut {
 		// Check if MAC is attached to URL
 		if a["v"] == nil {
 			log.Println("Error: No HMAC attached to URL.")
@@ -96,9 +118,14 @@ func handleRequest(w http.ResponseWriter, r *http.Request) {
 		 */
 		if hmac.Equal([]byte(macString), []byte(a["v"][0])) {
 			// Make sure the path exists
-			os.MkdirAll(filepath.Dir(conf.Storedir+fileStorePath), os.ModePerm)
+			err := os.MkdirAll(filepath.Dir(absFilename), os.ModePerm)
+			if err != nil {
+				log.Println("Could not make directories:", err)
+				http.Error(w, "500 Internal Server Error", 500)
+				return
+			}
 
-			file, err := os.OpenFile(conf.Storedir+fileStorePath, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0755)
+			file, err := os.OpenFile(absFilename, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0644)
 			defer file.Close()
 			if err != nil {
 				log.Println("Creating new file failed:", err)
@@ -115,17 +142,22 @@ func handleRequest(w http.ResponseWriter, r *http.Request) {
 
 			log.Println("Successfully written", n, "bytes to file", fileStorePath)
 			w.WriteHeader(http.StatusCreated)
+			return
 		} else {
 			log.Println("Invalid MAC.")
 			http.Error(w, "403 Forbidden", 403)
 			return
 		}
-	} else if r.Method == "HEAD" {
-		fileinfo, err := os.Stat(conf.Storedir + fileStorePath)
+	} else if r.Method == http.MethodHead || r.Method == http.MethodGet {
+		fileinfo, err := os.Stat(absFilename)
 		if err != nil {
 			log.Println("Getting file information failed:", err)
 			http.Error(w, "404 Not Found", 404)
 			return
+		} else if fileinfo.IsDir() {
+			log.Println("Directory listing forbidden!")
+			http.Error(w, "403 Forbidden", 403)
+			return
 		}
 
 		/*
@@ -134,20 +166,21 @@ func handleRequest(w http.ResponseWriter, r *http.Request) {
 		 * relying on file extensions.
 		 */
 		contentType := mime.TypeByExtension(filepath.Ext(fileStorePath))
-		w.Header().Set("Content-Length", strconv.FormatInt(fileinfo.Size(), 10))
-		w.Header().Set("Content-Type", contentType)
-	} else if r.Method == "GET" {
-		contentType := mime.TypeByExtension(filepath.Ext(fileStorePath))
-		if f, err := os.Stat(conf.Storedir + fileStorePath); err != nil || f.IsDir() {
-			log.Println("Directory listing forbidden!")
-			http.Error(w, "403 Forbidden", 403)
-			return
-		}
 		if contentType == "" {
 			contentType = "application/octet-stream"
 		}
-		http.ServeFile(w, r, conf.Storedir+fileStorePath)
 		w.Header().Set("Content-Type", contentType)
+
+		if r.Method == http.MethodHead {
+			w.Header().Set("Content-Length", strconv.FormatInt(fileinfo.Size(), 10))
+		} else {
+			http.ServeFile(w, r, absFilename)
+		}
+
+		return
+	} else if r.Method == http.MethodOptions {
+		w.Header().Set("Allow", ALLOWED_METHODS)
+		return
 	} else {
 		log.Println("Invalid method", r.Method, "for access to ", conf.UploadSubDir)
 		http.Error(w, "405 Method Not Allowed", 405)
@@ -176,25 +209,47 @@ func readConfig(configfilename string, conf *Config) error {
  * Main function
  */
 func main() {
+	var configFile string
+	var proto string
+
 	/*
 	 * Read startup arguments
 	 */
-	var argConfigFile = flag.String("config", "./config.toml", "Path to configuration file \"config.toml\".")
+	flag.StringVar(&configFile, "config", "./config.toml", "Path to configuration file \"config.toml\".")
 	flag.Parse()
 
+	if !flag.Parsed() {
+		log.Fatalln("Could not parse flags")
+	}
+
+
 	/*
 	 * Read config file
 	 */
-	err := readConfig(*argConfigFile, &conf)
+	err := readConfig(configFile, &conf)
 	if err != nil {
-		log.Println("There was an error while reading the configuration file:", err)
+		log.Fatalln("There was an error while reading the configuration file:", err)
+	}
+
+	if conf.UnixSocket {
+		proto = "unix"
+	} else {
+		proto = "tcp"
 	}
 
 	/*
 	 * Start HTTP server
 	 */
 	log.Println("Starting Prosody-Filer", versionString, "...")
-	http.HandleFunc("/"+conf.UploadSubDir, handleRequest)
+	listener, err := net.Listen(proto, conf.Listenport)
+	if err != nil {
+		log.Fatalln("Could not open listening socket:", err)
+	}
+
+	subpath := path.Join("/", conf.UploadSubDir)
+	subpath = strings.TrimRight(subpath, "/")
+	subpath += "/"
+	http.HandleFunc(subpath, handleRequest)
 	log.Printf("Server started on port %s. Waiting for requests.\n", conf.Listenport)
-	http.ListenAndServe(conf.Listenport, nil)
+	http.Serve(listener, nil)
 }