[MAR-3119] (#10)

* can insert logs into the database * update log code * fixed * fixed * adding logger manifests * prep code for deploying * add db * fix * rename * rename * changes made to these files * added ui from master * db updates * logger context updates * format code * changed the logger alpha memory * fixed logger tests to include a closed connection * increase memory for logger prod * fixed the readme to reflect logger application * removed using system * removed logger.LogInformation * removed package reference and using system * added gitignore .DS_Store * format of logger tests is fixed * format of program is fixed * .DS_Store banished! * adding db-service-dm-logger * Delete .gitignore * fixed the naming conventions for unit test * Update subscribers/logs/logger/AppService.cs Co-Authored-By: tobiasbrown <[email protected]> * Update subscribers/logs/logger.tests/logger.tests.csproj Co-Authored-By: tobiasbrown <[email protected]> * fixed slack tests * fixed part 2 slack * fixed slack pt3 * Update subscribers/logs/logger/AppService.cs Co-Authored-By: tobiasbrown <[email protected]> * Update subscribers/logs/logger/AppService.cs Co-Authored-By: tobiasbrown <[email protected]> * Update subscribers/logs/logger/AppService.cs Co-Authored-By: tobiasbrown <[email protected]> * indentation fixed * Update subscribers/logs/logger/MessageProcessor.cs Co-Authored-By: tobiasbrown <[email protected]> * Update subscribers/logs/logger/MessageProcessor.cs Co-Authored-By: tobiasbrown <[email protected]>
govau · Aug 6, 2019 · 54f3611 · 54f3611
1 parent e1f1e2c
commit 54f3611
Show file tree

Hide file tree

Showing 27 changed files with 862 additions and 4 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -91,6 +91,8 @@ jobs:
       - checkout
       - dot_net_build: 
           path: "subscribers/slack/worker"
+      - dot_net_build: 
+          path: "subscribers/logs/logger"
       - persist_to_workspace:
           root: .
           paths:
@@ -117,6 +119,8 @@ jobs:
           at: .
       - dot_net_test:
           path: "subscribers/slack/worker.tests"
+      - dot_net_test:
+          path: "subscribers/logs/logger.tests"
       - persist_to_workspace:
           root: .
           paths:
@@ -153,20 +157,36 @@ jobs:
       - cf_deploy:
           cf_app_name: dm-rc-slack-worker
           cf_manifest: slack-worker.rc.yml
+      - cf_deploy:
+          cf_app_name: dm-rc-logger
+          cf_manifest: logger.rc.yml    
 
   deploy_to_production_clone:
     <<: *circleci_deploy
     steps:
       - cf_deploy:
           cf_app_name: dm-pc-slack-worker
           cf_manifest: slack-worker.pc.yml
+      - cf_deploy:
+          cf_app_name: dm-pc-logger
+          cf_manifest: logger.pc.yml
 
   deploy_to_staging:
     <<: *circleci_deploy
     steps:
       - cf_deploy:
           cf_app_name: dm-dev-slack-worker
           cf_manifest: slack-worker.dev.yml
+      - cf_deploy:
+          cf_app_name: dm-dev-logger
+          cf_manifest: logger.dev.yml
+
+  deploy_to_alpha:
+    <<: *circleci_deploy
+    steps:
+      - cf_deploy:
+          cf_app_name: dm-alpha-logger
+          cf_manifest: logger.alpha.yml
 
   deploy_to_production:
     <<: *circleci_deploy
@@ -177,7 +197,12 @@ jobs:
           cf_password: $CF_PASSWORD_PROD
           cf_app_name: dm-slack-worker
           cf_manifest: slack-worker.prod.yml
-
+      - cf_deploy:
+          cf_url: https://api.system.b.cld.gov.au
+          cf_user: $CF_USER_PROD
+          cf_password: $CF_PASSWORD_PROD
+          cf_app_name: dm-logger
+          cf_manifest: logger.prod.yml
 
 workflows:
   version: 2
@@ -211,6 +236,14 @@ workflows:
           filters:
             tags:
               only: /[0-9]+(\.[0-9]+)*/
+      - deploy_to_alpha:
+          requires:
+            - run_dot_net_tests
+          filters:
+            tags:
+              ignore: /.*/
+            branches:
+              only: alpha
       - deploy_to_rc:
           requires:
             - run_dot_net_tests
@@ -242,4 +275,4 @@ workflows:
             tags:
               only: /[0-9]+(\.[0-9]+)*/
             branches:
-              ignore: /.*/
+              ignore: /.*/
diff --git a/.gitignore b/.gitignore
@@ -2,3 +2,4 @@
 **/obj/**/*.*
 **/node_modules/**/*
 **/build/**/*
+**/.DS_Store
diff --git a/.manifests/logger.alpha.yml b/.manifests/logger.alpha.yml
@@ -0,0 +1,11 @@
+applications:
+- name: dm-alpha-logger
+  buildpack: dotnet_core_buildpack
+  path: ../subscribers/logs/logger
+  health-check-type: process
+  instances: 1
+  memory: 50M
+  disk_quota: 100M
+  services:
+  - ups-dm-alpha-logger
+  - db-service-dm-alpha-logger
diff --git a/.manifests/logger.dev.yml b/.manifests/logger.dev.yml
@@ -0,0 +1,10 @@
+applications:
+- name: dm-dev-logger
+  buildpack: dotnet_core_buildpack
+  path: ../subscribers/logs/logger
+  health-check-type: process
+  instances: 1
+  memory: 50M
+  disk_quota: 100M
+  services:
+  - ups-dm-dev-logger
diff --git a/.manifests/logger.pc.yml b/.manifests/logger.pc.yml
@@ -0,0 +1,10 @@
+applications:
+- name: dm-pc-logger
+  buildpack: dotnet_core_buildpack
+  path: ../subscribers/logs/logger
+  health-check-type: process
+  instances: 1
+  memory: 50M
+  disk_quota: 100M
+  services:
+  - ups-dm-pc-logger
diff --git a/.manifests/logger.prod.yml b/.manifests/logger.prod.yml
@@ -0,0 +1,11 @@
+applications:
+- name: dm-logger
+  buildpack: dotnet_core_buildpack
+  path: ../subscribers/logs/logger
+  health-check-type: process
+  instances: 1
+  memory: 128M
+  disk_quota: 100M
+  services:
+  - ups-dm-logger
+  - db-service-dm-logger
diff --git a/.manifests/logger.rc.yml b/.manifests/logger.rc.yml
@@ -0,0 +1,10 @@
+applications:
+- name: dm-rc-logger
+  buildpack: dotnet_core_buildpack
+  path: ../subscribers/logs/logger
+  health-check-type: process
+  instances: 1
+  memory: 50M
+  disk_quota: 100M
+  services:
+  - ups-dm-rc-logger
diff --git a/.vscode/launch.json b/.vscode/launch.json
@@ -5,7 +5,7 @@
     "version": "0.2.0",
     "configurations": [
         {
-            "name": ".NET Core Launch (console)",
+            "name": "Launch slack worker",
             "type": "coreclr",
             "request": "launch",
             "preLaunchTask": "build",
@@ -22,6 +22,25 @@
                 "AWS_SQS_LONG_POLL_TIME_IN_SECONDS": "1"
             }
         },
+        {
+            "name": "Launch logger worker",
+            "type": "coreclr",
+            "request": "launch",
+            "preLaunchTask": "build",
+            "program": "${workspaceFolder}/subscribers/logs/logger/bin/Debug/netcoreapp2.2/logger.worker.dll",
+            "args": [],
+            "cwd": "${workspaceFolder}/subscribers/logs/logger",
+            "console": "internalConsole",
+            "stopAtEntry": false,
+            "internalConsoleOptions": "openOnSessionStart",
+            "env": {
+                "AWS_SQS_QUEUE_URL": "http://localhost:4576/queue/dta-marketplace-local-logger",
+                "AWS_SQS_SERVICE_URL": "http://localhost:4576",
+                "WORK_INTERVAL_IN_SECONDS": "5",
+                "AWS_SQS_LONG_POLL_TIME_IN_SECONDS": "1",
+                "AWS_SQS_REGION": "ap-southeast-2"
+            }
+        },
         {
             "name": ".NET Core Attach",
             "type": "coreclr",

diff --git a/subscribers/logs/logger.tests/LoggerTests.cs b/subscribers/logs/logger.tests/LoggerTests.cs
@@ -0,0 +1,92 @@
+using System;
+using Xunit;
+using System.Threading.Tasks;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+using Moq;
+using Newtonsoft.Json;
+using Microsoft.Data.Sqlite;
+using Dta.Marketplace.Subscribers.Logger.Worker;
+using Amazon.SQS.Model;
+
+using Microsoft.EntityFrameworkCore;
+
+namespace Dta.Marketplace.Subscribers.Logger.Worker.Logger.Tests {
+
+    public class LoggerTests {
+
+        [Fact]
+        public void Can_Log_Information_Message() {
+
+            // Arrange
+            var logger = new Mock<ILoggerAdapter<AppService>>();
+            var dbSet = new Mock<DbSet<LogEntry>>();
+            var connection = new SqliteConnection("DataSource=:memory:");
+            connection.Open();
+            try {
+                //In-memory database only exists while the connection is open
+                var options = new DbContextOptionsBuilder<LoggerContext>()
+                            .UseSqlite(connection)
+                            .Options;
+
+                //create schema in the database 
+                using (var context = new LoggerContext(options)) {
+                    context.Database.EnsureCreated();
+                }
+
+                var json = JsonConvert.SerializeObject(new {
+                    foo = "bar"
+                });
+
+                // Act
+                using (var context = new LoggerContext(options)) {
+                    var messageProcessor = new MessageProcessor(logger.Object, context);
+                    messageProcessor.Process(new Message {
+                        Body = json
+                    });
+                }
+
+                // Assert
+                using (var context = new LoggerContext(options)) {
+                    logger.Verify(l => l.LogInformation(json), Times.Once);
+                }
+            } finally {
+                connection.Close();
+            }
+        }
+
+
+        [Fact]
+        public void Can_Log_Exception() {
+            // Arrange
+            var logger = new Mock<ILoggerAdapter<AppService>>();
+            var dbSet = new Mock<DbSet<LogEntry>>();
+            var connection = new SqliteConnection("DataSource=:memory:");
+            connection.Open();
+            try {
+                var options = new DbContextOptionsBuilder<LoggerContext>()
+                            .UseSqlite(connection)
+                            .Options;
+
+                using (var context = new LoggerContext(options)) {
+                    context.Database.EnsureCreated();
+                }
+                var json = JsonConvert.SerializeObject(new {
+                    foo = ""
+                });
+
+                //Act
+                using (var context = new LoggerContext(options)) {
+                    var messageProcessor = new MessageProcessor(logger.Object, context);
+                    messageProcessor.Process(null);
+                }
+                //Assert
+                using (var context = new LoggerContext(options)) {
+                    logger.Verify(l => l.LogError(It.IsAny<string>(), It.IsAny<Exception>()), Times.Once);
+                }
+            } finally {
+                connection.Close();
+            }
+        }
+    }
+}
diff --git a/subscribers/logs/logger.tests/logger.tests.csproj b/subscribers/logs/logger.tests/logger.tests.csproj
@@ -0,0 +1,23 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <ItemGroup>
+    <ProjectReference Include="../logger/worker.csproj" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <PackageReference Include="AWSSDK.SQS" Version="3.3.100.40" />
+    <PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" Version="2.2.6" />
+    <PackageReference Include="Microsoft.Extensions.Logging" Version="2.2.0" />
+    <PackageReference Include="Microsoft.Extensions.Options" Version="2.2.0" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.1.1" />
+    <PackageReference Include="Moq" Version="4.12.0" />
+    <PackageReference Include="xunit" Version="2.4.1" />
+    <PackageReference Include="xunit.runner.visualstudio" Version="2.4.1" />
+  </ItemGroup>
+
+  <PropertyGroup>
+    <TargetFramework>netcoreapp2.2</TargetFramework>
+    <AssemblyName>Logger.Worker.logger.Tests</AssemblyName>
+  </PropertyGroup>
+
+</Project>
diff --git a/subscribers/logs/logger/AppConfig.cs b/subscribers/logs/logger/AppConfig.cs
@@ -0,0 +1,15 @@
+using System;
+
+namespace Dta.Marketplace.Subscribers.Logger.Worker {
+    public class AppConfig {
+        public string AwsSqsRegion { get; set; } = "ap-southeast-2";
+        public string AwsSqsQueueUrl { get; set; }
+        public string AwsSqsServiceUrl { get; set; }
+        public string AwsSqsAccessKeyId { get; set; }
+        public string AwsSqsSecretAccessKey { get; set; }
+        public int AwsSqsLongPollTimeInSeconds { get; set; } = 20;
+        public int WorkIntervalInSeconds { get; set; } = 60;
+        public string SentryDsn { get; set; }
+        public string ConnectionString { get; set; }
+    }
+}