Fixing issues #45 #46 #47

api/ct.go

@@ -93,7 +93,7 @@ func (s *Server) StoreLogEntries(str api.CtApi_StoreLogEntriesServer) error {
 
 				// write a chunk
 				finishTime := time.Since(startTime)
-				if _, err := s.BenchmarkFile.Write([]byte(strconv.FormatInt(finishTime.Microseconds(), 10) + ",")); err != nil {
+				if _, err := s.MapEntryExecTime.Write([]byte(strconv.FormatInt(finishTime.Microseconds(), 10) + ",")); err != nil {
 					panic(err)
 				}
 			}
@@ -111,18 +111,6 @@ func (s *Server) StoreLogEntries(str api.CtApi_StoreLogEntriesServer) error {
 				}
 			}()
 		}
-		//todo change the response to the client
-		//errs := s.Store.MapBatchWithCacheAndDB()
-		//if len(errs) != 0 {
-		//	fmt.Println(errs)
-		//}
-		//err = s.Store.StoreBatchPostHook()
-		//if err != nil {
-		//	fmt.Println(err)
-		//}
-
-		log.Info().Msgf("%v", s.Store.Counter)
-		s.Store.ResetCounter()
 	}
 	wg.Wait()
 

api/measurement.go

@@ -43,10 +43,6 @@ func (s *Server) StopMeasurement(ctx context.Context, muid *api.MeasurementId) (
 		return nil, status.Error(codes.Internal, err.Error())
 	}
 
-	if err := s.Store.RunPostHooks(); err != nil {
-		return nil, status.Error(codes.Internal, err.Error())
-	}
-
 	log.Debug().Str("muid", muid.Id).Msgf("stopped measurement")
 
 	return &api.Empty{}, nil
@@ -78,9 +74,6 @@ func (s *Server) StopStage(ctx context.Context, muid *api.MeasurementId) (*api.E
 		return nil, status.Error(codes.Internal, err.Error())
 	}
 
-	if err := s.Store.RunPostHooks(); err != nil {
-		return nil, status.Error(codes.Internal, err.Error())
-	}
 	log.Debug().Str("muid", muid.Id).Msgf("stopped stage")
 
 	return &api.Empty{}, nil

api/server.go

@@ -39,10 +39,10 @@ func timeFromUnix(ts int64) time.Time {
 }
 
 type Server struct {
-	Conf          Config
-	Store         *store.Store
-	Log           app.ErrLogger
-	BenchmarkFile *os.File
+	Conf             Config
+	Store            *store.Store
+	Log              app.ErrLogger
+	MapEntryExecTime *os.File
 }
 
 func (s *Server) Run(lis net.Listener) error {

app/cache/main.go

@@ -39,14 +39,14 @@ func main() {
 
 	opts := store.Opts{
 		AllowedInterval: 36 * time.Hour,
-		BatchSize:       50000,
+		BatchSize:       10000,
 		CacheOpts: store.CacheOpts{
 			LogSize:   1000,
 			TLDSize:   2000,
-			PSuffSize: 5000,
-			ApexSize:  20000,
-			FQDNSize:  20000,
-			CertSize:  20000,
+			PSuffSize: 10000,
+			ApexSize:  63000,
+			FQDNSize:  1474000,
+			CertSize:  673000,
 		},
 	}
 
@@ -82,25 +82,33 @@ func main() {
 		logger = app.NewZeroLogger(tags)
 	}
 
-	// open output file
-	fo, err := os.Create("output/output.txt")
+	fmp, err := os.Create("output/mapentry_exec_time.txt")
 	if err != nil {
 		panic(err)
 	}
 	// close fo on exit and check for its returned error
 	defer func() {
-		if err := fo.Close(); err != nil {
+		if err := fmp.Close(); err != nil {
 			panic(err)
 		}
 	}()
 
 	serv := api.Server{
-		Conf:          conf.Api,
-		Store:         s,
-		Log:           logger,
-		BenchmarkFile: fo,
+		Conf:             conf.Api,
+		Store:            s,
+		Log:              logger,
+		MapEntryExecTime: fmp,
 	}
 
+	defer func() {
+		if err := serv.Store.DBTime.Close(); err != nil {
+			panic(err)
+		}
+		if err := serv.Store.HitCount.Close(); err != nil {
+			panic(err)
+		}
+	}()
+
 	addr := fmt.Sprintf(":%d", conf.Api.Api.Port)
 	lis, err := net.Listen("tcp", addr)
 	if err != nil {

app/ct/main.go

@@ -201,7 +201,7 @@ func main() {
 				StartIndex:  0,
 				//EndIndex:    end,
 				//EndIndex: index.Start + 100,
-				EndIndex: 10000,
+				EndIndex: 50000000,
 			}
 
 			count, err = ct.Scan(ctx, &l, entryFn, opts)

collectors/entrada/entrada.go

@@ -3,9 +3,10 @@ package entrada
 import (
 	"context"
 	"database/sql"
-	imp "github.com/bippio/go-impala"
 	"io"
 	"time"
+
+	imp "github.com/bippio/go-impala"
 )
 
 var (

docker-compose.yml

@@ -12,6 +12,8 @@ services:
     ports:
       - "${GRPC_PORT}:${GRPC_PORT}"
       - "8881:8881"
+    networks:
+      - gollector
 #   entrada:
 #     container_name: gollector-entrada
 #     build:
@@ -41,6 +43,8 @@ services:
     command: [ "--config", "/config/ct.yml", "-logtostderr" ]
     volumes:
       - ./config:/config:ro # configuration files
+    networks:
+      - gollector
 #   splunk:
 #     container_name: gollector-splunk
 #     build:
@@ -61,3 +65,11 @@ services:
       - ${DB_VOLUME?err}:/var/lib/postgresql/data # persistent storage of database
     ports:
       - "${POSTGRES_HOST?err}:${POSTGRES_PORT?err}:5432"
+    networks:
+      - gollector
+
+# docker network create --subnet "10.13.20.0/24" gollector
+networks:
+  gollector:
+    external:
+      name: gollector