First commit

.gitignore

@@ -0,0 +1,51 @@
+# built application files
+*.apk
+*.ap_
+
+# files for the dex VM
+*.dex
+
+# Java class files
+*.class
+
+# generated files
+bin/
+gen/
+out/
+build/
+
+# Local configuration file (sdk path, etc)
+local.properties
+
+# Eclipse project files
+.classpath
+.project
+.settings
+
+# Windows thumbnail db
+.DS_Store
+
+# Windows exe files
+*.exe
+
+# IDEA/Android Studio project files, because
+# the project can be imported from settings.gradle
+.idea
+*.iml
+
+# Old-style IDEA project files
+*.ipr
+*.iws
+
+# Local IDEA workspace
+.idea/workspace.xml
+
+# Gradle cache
+.gradle
+
+# Sandbox stuff
+_sandbox
+
+*.pid
+.svn
+package/

AUTHORS

@@ -0,0 +1,13 @@
+# This is the official list of poseidon authors for copyright purposes.
+
+# Names should be added to this file as one ofdd
+#     Organization's name
+#     Individual's name <submission email address>
+#     Individual's name <submission email address> <email2> <emailN>
+
+# Please keep the list sorted.
+
+du xudong <[email protected]>
+guo jun <[email protected]> <[email protected]>
+li wei <[email protected]> <[email protected]>
+wei zili <[email protected]> <[email protected]>

build.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+if [ ! -f build.sh ]; then
+    echo 'build.sh must be run within its container folder' 1>&2
+    exit 1
+fi
+
+mkdir -p dist
+
+cd builder
+sh build.sh
+if [ $? -ne 0 ]; then exit -1; fi;
+cd ..
+
+cd service
+sh build.sh
+if [ $? -ne 0 ]; then exit -1; fi;
+cd ..
+

builder/build.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+dir=$(dirname $0)
+
+if [ ! -f build.sh ]; then
+    echo 'build.sh must be run within its container folder' 1>&2
+    exit 1
+fi
+
+(cd $dir/index; sh ./build.sh)
+(cd $dir/docformat; sh ./build.sh)

builder/docformat/Makefile

@@ -0,0 +1,16 @@
+CURDIR:=$(shell pwd)
+APP:=docformat
+all:
+	gofmt -w src/main src/job src/common
+	export GOPATH=$(CURDIR) && go build -o bin/$(APP) main
+
+clean:
+	rm -f bin/$(APP)
+
+test:
+	export GOPATH=$(CURDIR) && cd src/job && go test -v
+
+benchmark:
+	export GOPATH=$(CURDIR) && cd src/job && go test -test.bench=".*"
+
+.PHONY:all clean test benchmark

builder/docformat/build.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+if [ $# -ne 1 ];then
+    BUSI="docformat"
+else
+    BUSI=$1
+fi
+
+
+CURDIR=$(cd $(dirname $0); pwd -P)
+cd ${CURDIR}
+make -j || exit 1
+
+mkdir -p package
+
+VERSION="0.1"
+
+TARGET_DIR=.pack_tmp/${BUSI}-${VERSION}
+rm -rf ${TARGET_DIR}
+mkdir -p ${TARGET_DIR}/{bin,etc,data}
+
+# bin
+cp ${CURDIR}/bin/docformat ${TARGET_DIR}/bin
+cp -r ${CURDIR}/script/common/* ${TARGET_DIR}/bin/
+# cp -r ${CURDIR}/script/${BUSI}/* ${TARGET_DIR}/bin/
+sed "s/BUSI_DEFAULT/${BUSI}/g" -i ${TARGET_DIR}/bin/install.sh
+
+# etc
+# cp -r ${CURDIR}/etc/common/* ${TARGET_DIR}/etc/
+\cp -rf ${CURDIR}/etc/${BUSI}/* ${TARGET_DIR}/etc/
+(cd ${TARGET_DIR}/etc/ && mv docformat.json ${BUSI}.json)
+
+# make pkg
+(cd .pack_tmp && tar zcf ../package/${BUSI}-${VERSION}.tar.gz ${BUSI}-${VERSION} --exclude=.svn --exclude=.git)
+
+rm -rf ${TARGET_DIR}
+echo "success ${BUSI}-${VERSION}.tar.gz"
+cp package/${BUSI}-${VERSION}.tar.gz ../../dist

builder/docformat/etc/docformat/docformat.json

@@ -0,0 +1,62 @@
+{
+	"src_provider": {
+		"src_type":"dm",
+		"dm": {
+			"monitor_interval_sec":15,
+			"pop_last_file":false,
+			"time_out_ms":1000,
+			"data_dir":"./logs",
+			"monitor_paths": [
+                "/home/poseidon/data"
+			]
+		}
+	},
+	"main":
+	{
+		"processor_num":1,
+		"collector_num":5,
+		"msg_chan_size":2,
+		"item_chan_size":2,
+
+		"max_procs":30,
+
+		"tick_sec":20,
+		"mem_limit":5000
+	},
+
+	"processor": {
+		"LogtoHdfsProcessor":true
+	},
+
+	"collector": {
+		"logto_hdfs":"LogtoHdfsCollector"
+	},
+	"Hdfs": {
+		"fixed_idc":"idc",
+		"use_fixed_idc":true,
+
+		"write_dirs": [
+			"./data/dest_1",
+			"./data/dest_2"
+		],
+		"hadoop_remote_dir":"/home/poseidon/src/test",
+		"hadoop_remote_time_dirs": [
+			"Y-m-d"
+		],
+		"hadoop_remote_file_prefix":"",
+		"hadoop_remote_file_suffix":".gz",
+		"flush_minute":20,
+		"max_merge_file_size":50000000,
+		"retry_times":3,
+		"hadoop_cmd":"./bin/local-hadoop.sh",
+		"gather_minute":15,
+
+		"tick_interval":1,
+
+        "poseidon_mode":true,
+        "doc_lines":128,
+	    "readbuf_size_byte":40960000,
+	    "docid_domain":"127.0.0.1:39360",
+	    "docid_business":"test"
+	}
+}

builder/docformat/script/common/clean.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+current=$(cd $(dirname $0) && pwd -P)
+current=$(dirname $current)
+
+cd $current && sh bin/install.sh stop
+
+rm -fv /home/poseidon/data/*
+rm -fv $current/logs/last*

builder/docformat/script/common/demo.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+
+# 用于将文件按照指定的文件名放入文件夹
+# 文件名约定
+# 必须包含日期例如：2016-10-10-08[-20]
+# 必须以-分割，至少精确到小时，也可以精确到分钟
+# 新文件必须按照字典序最大
+
+if [ $# -lt 1 ];
+then
+    echo $0 logfile
+    echo usage: $0 ./weibo.txt
+    exit
+fi
+
+pwd=$(pwd)
+
+day=`date -d "24 hours ago" +"%Y-%m-%d"`
+hour=`date -d "24 hours ago" +"%H"`
+timestamp=`date -d "24 hours ago" +"%Y%m%d%H%M%S"`
+
+# 这里的timestamp仅仅是为了使最新的文件按照字典序最大
+
+cp -rf $1 /home/poseidon/data/log${timestamp}_$day-$hour.txt
+
+echo file has copyed to /home/poseidon/data
+echo wait for file in /home/poseidon/src/test/$day
+echo detail log see logs/docformat.log
+
+# echo and then you should go to run:
+# echo /bin/bash  $pwd/bin/mock_start.sh $day

builder/docformat/script/common/dummy-hadoop.sh

@@ -0,0 +1,8 @@
+#/bin/bash
+exec 1>>logs/dummy-hadoop.log 2>&1
+echo "$(date "+%Y/%m/%d %H:%M:%S") this a fake hadoop client for demo"
+echo "params is: $*"
+echo "now sleep for a while"
+sleep 5
+echo "now exit"
+exit 0

builder/docformat/script/common/install.sh

@@ -0,0 +1,46 @@
+#!/bin/bash
+
+if [[ -z "$1" || ("$1" != "start" && "$1" != "stop") ]]; then
+    echo "Usage: $0 <start|stop>"
+    echo "       $0 start"
+    exit
+fi
+
+BUSI="BUSI_DEFAULT"
+
+start_cmd="bin/docformat -c etc/${BUSI}.json"
+pid=$(ps axu | grep "${start_cmd}$" | awk '{print $2}')
+if [[ -z "$pid" && "$1" == "stop" ]]; then
+    echo "service has NOT been started"
+    exit
+elif [[ -n "$pid" && "$1" == "start" ]]; then
+    echo "service has been started"
+    ps axu | grep "${start_cmd}$"
+    exit
+fi
+
+current=$(cd $(dirname $0) && pwd -P)
+current=$(dirname $current)
+echo $current
+
+if [[ "$1" == "stop" || (-n "$pid" && "$1" == "restart") ]]; then
+    kill $pid
+    echo "service has start to stop, see ${current}/logs/docformat.log"
+    sleep 0.1
+    ps axu | grep "${start_cmd}$"
+    if [[ "$1" == "stop" ]]; then
+        exit
+    fi
+fi
+
+rm -rf $current/logs
+mkdir $current/logs
+
+log_postfix=$(date "+%Y%m%d-%H%M%S")
+if [[ "$1" == "start" ]]; then
+    cd $current
+    nohup $start_cmd &> logs/docformat.out.${log_postfix} &
+    echo "service started"
+    sleep 0.1
+    ps axu | grep "${start_cmd}$"
+fi

builder/docformat/script/common/local-hadoop.sh

@@ -0,0 +1,33 @@
+#/bin/bash
+# exec 1>>logs/local-hadoop.log 2>&1
+echo "$(date "+%Y/%m/%d %H:%M:%S") this a local hadoop client for demo"
+echo "params is: $*"
+# local-hadoop.sh fs -mkdir -p /home/poseidon/src/test
+if [ $1 = "fs" ]; then
+    echo "this is fs"
+    if [ $2 = "-mkdir" ]; then
+        echo "this is mkdir"
+        if [ $3 == "-p" ]; then 
+            echo "mkdir -p $4"
+            mkdir -p $4
+            exit $?
+        else
+            echo "mkdir $3"
+            mkdir $3
+            exit $?
+        fi
+    elif [ $2 = "-put" -o $2 = "-copyFromLocal" ]; then
+        echo "this is put"
+        if [ $3 == "-f" ]; then 
+            echo "cp -f $4 $5"
+            cp -f $4 $5
+            exit $?
+        else
+            echo "cp $3 $4"
+            cp $3 $4
+            exit $?
+        fi
+    fi
+fi
+echo "not processed"
+exit 1