Merge pull request #4 from ashwanthkumar/symlink-for-filenames

Symlink for filenames

main.go

@@ -14,6 +14,8 @@ var marathonURI string
 var mesosSlavePort int
 var appCheckInterval time.Duration
 var taskMaxHeartBeatInterval time.Duration
+
+var workDir string
 var rsyslogConfigurationDir string
 var rsyslogRestartCommand string
 
@@ -46,7 +48,8 @@ func main() {
 
 	loggers := make(map[string]Logger)
 	loggers["rsyslog"] = &Rsyslog{
-		ConfigLocation: rsyslogConfigurationDir,
+		WorkDir: workDir,
+		SyslogConfigLocation: rsyslogConfigurationDir,
 		RestartCommand: rsyslogRestartCommand,
 	}
 	logManager = LogManager{
@@ -65,6 +68,7 @@ func init() {
 	flag.IntVar(&mesosSlavePort, "slave-port", 5051, "Mesos slave port")
 	flag.DurationVar(&appCheckInterval, "app-check-interval", 30*time.Second, "Frequency at which we check for new tasks")
 	flag.DurationVar(&taskMaxHeartBeatInterval, "task-max-heart-beat-interval", 30*time.Minute, "Max heartbeat interval after which the task is considered dead and logger is removed")
+	flag.StringVar(&workDir, "work-dir", "/tmp/", "Location on the Filesystem where we create symlinks to app location base dir. This is needed to ensure the file paths don't become too long and crashes rsyslog")
 	flag.StringVar(&rsyslogConfigurationDir, "rsyslog-configuration-dir", "/etc/rsyslog.d", "Location on the Filesystem where the rsyslog configurations needs to be written")
 	flag.StringVar(&rsyslogRestartCommand, "rsyslog-restart-cmd", "service rsyslog restart", "Restart command for rsyslog backend")
 }

rsyslog-logger.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"fmt"
 	"io/ioutil"
+	"os"
 	"os/exec"
 	"text/template"
 )
@@ -21,19 +22,17 @@ const RsyslogTemplate = `
 # File - {{ .FileName }}
 ######################################
 
-module(load="imfile")
-
 input(type="imfile"
-			File="{{ .CWD }}/{{ .FileName }}"
-			Tag="{{ .CleanAppName }}	{{.TaskID}}"
-			statefile="{{ .TaskID }}"
-      Severity="info")
+	File="{{ .WorkDir }}/{{ .FileName }}"
+	Tag="{{ .CleanAppName }}	{{.TaskID}}"
+	Severity="info")
 `
 
 // Rsyslog backend implementation
 type Rsyslog struct {
-	ConfigLocation string
-	RestartCommand string
+	WorkDir              string
+	SyslogConfigLocation string
+	RestartCommand       string
 }
 
 // AddTask - Adds a task definition file to FS
@@ -44,18 +43,29 @@ func (r *Rsyslog) AddTask(taskInfo TaskInfo) error {
 		fmt.Printf("[ERROR] %v\n", err)
 		return err
 	}
-	// TODO - Support multiple file configurations
-	configFileLocation := fmt.Sprintf("%s/%s-%s.conf", r.ConfigLocation, CommonPrefixToConfigFiles, taskInfo.TaskID)
+
+	// We create symlink of the tasks' sandbox dir so that,
+	// we can reduce the length of the file path that we provide as
+	// part of configs in rsyslog's imfile directive.
+	// When the full file path > 200, rsyslog crashes with "buffer overflow" (evil smile)
+	err = os.Symlink(taskInfo.CWD, r.symlink(taskInfo.TaskID))
+	if err != nil {
+		fmt.Printf("[ERROR] %v\n", err)
+		return err
+	}
+
+	configFileLocation := fmt.Sprintf("%s/%s-%s.conf", r.SyslogConfigLocation, CommonPrefixToConfigFiles, taskInfo.TaskID)
 	err = ioutil.WriteFile(configFileLocation, []byte(template), 0644)
 	if err != nil {
 		fmt.Printf("[ERROR] %v\n", err)
 		return err
 	}
+
 	err = exec.Command("/bin/sh", "-c", r.RestartCommand).Run()
 	return err
 }
 
-// RemoveTask - Remove a task definition from the FS
+// TODO - RemoveTask removes a task definition from the FS
 func (r *Rsyslog) RemoveTask(taskId string) error {
 	fmt.Printf("[Rsyslog] Remove task info for %v\n", taskId)
 	return nil
@@ -72,6 +82,11 @@ func (r *Rsyslog) render(taskInfo TaskInfo) (string, error) {
 	if err != nil {
 		return "", err
 	}
+	taskInfo.WorkDir = r.symlink(taskInfo.TaskID)
 	err = tmpl.Execute(&configInBytes, &taskInfo)
 	return configInBytes.String(), err
 }
+
+func (r *Rsyslog) symlink(taskId string) string {
+	return fmt.Sprintf("%s/%s", r.WorkDir, taskId)
+}

rsyslog-logger_test.go

@@ -0,0 +1,44 @@
+package main
+
+import (
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestRenderRsyslogTemplate(t *testing.T) {
+
+	hostname, err := os.Hostname()
+	var rsyslog = Rsyslog{
+		WorkDir: "/foo/bar",
+	}
+	label := map[string]string{
+		"logs.enabled": "true",
+	}
+	taskInfo := TaskInfo{
+		App:      "/test.aayush.http",
+		Hostname: hostname,
+		Labels:   label,
+		TaskID:   "abcdefghij",
+		CWD:      "/foo/bar",
+		FileName: "test_file_name.txt",
+	}
+
+	expected := `
+######################################
+# Created via marathon-logger,
+# PLEASE DON'T EDIT THIS FILE MANUALLY
+# Name - /test.aayush.http
+# File - test_file_name.txt
+######################################
+
+input(type="imfile"
+	File="/foo/bar/abcdefghij/test_file_name.txt"
+	Tag="test.aayush.http	abcdefghij"
+	Severity="info")
+`
+	template, err := rsyslog.render(taskInfo)
+	assert.NoError(t, err)
+	assert.Equal(t, expected, template)
+}

task-info_test.go

@@ -1,7 +1,6 @@
 package main
 
 import (
-	"os"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -16,40 +15,3 @@ func TestCleanAppName(t *testing.T) {
 	testCaseThree.App = "/test.aayush.http/"
 	assert.Equal(t, testCaseThree.CleanAppName(), "test.aayush.http.", "They should be equal")
 }
-
-func TestRenderRsyslogTemplate(t *testing.T) {
-
-	hostname, err := os.Hostname()
-	var rsyslog Rsyslog
-	label := map[string]string{
-		"logs.enabled": "true",
-	}
-	taskInfo := TaskInfo{
-		App:      "/test.aayush.http",
-		Hostname: hostname,
-		Labels:   label,
-		TaskID:   "abcdefghij",
-		CWD:      "/foo/bar",
-		FileName: "test_file_name.txt",
-	}
-
-	expected := `
-######################################
-# Created via marathon-logger,
-# PLEASE DON'T EDIT THIS FILE MANUALLY
-# Name - /test.aayush.http
-# File - test_file_name.txt
-######################################
-
-module(load="imfile")
-
-input(type="imfile"
-			File="/foo/bar/test_file_name.txt"
-			Tag="test.aayush.http	abcdefghij"
-			statefile="abcdefghij"
-      Severity="info")
-`
-	template, err := rsyslog.render(taskInfo)
-	assert.NoError(t, err)
-	assert.Equal(t, expected, template)
-}

task-manager.go

@@ -21,6 +21,7 @@ type TaskInfo struct {
 	Hostname string
 	CWD      string // Current working directory of the task in the slave
 	FileName string // Actual file name to that we need monitor for logs
+	WorkDir  string // WorkDir location of marathon-logger where we setup Symlink
 }
 
 // CleanAppName cleans the app-name string for `/` characters