Monorepo (Financial-Times#60)

* WIP Initialise monorepo

* Using namespaced packages, testing, linking

* Fitting master branch

* modify setup repository

* fix unit testing in each directory

* try to pass CI

* fix CI

* restructured packages

* modify CI

* fix CI

* fix makefile

* rename package prefix

* move test-helpers in package and use them

* add monorepo setup instructions in README

* fix init-db task

* pass verifying

* configure publishing

* tweak circleci's config

* remove unused scripts

* Update packages/tc-api-publish-adaptors/package.json

Co-Authored-By: Rhys Evans <[email protected]>

* remove n-cluster

* reflect review

* reflect review

* Revert test-helpers to be placed at root directory

* fix module resolution

* fix CI install step

* Tweak CI

* Remove additional setup instruction

* Attempt to install monorepo by script ourself

* Adopt windows specific trick

* Put comment for fake task and its purpose

* temporary use forked athloi package

* Revert eslintrc.js

* fit master branch

* tidy up

* revert commented out test

* add eslint related package

* fix linting related things

.circleci/config.yml

@@ -30,6 +30,10 @@ references:
       paths:
         - ./node_modules/
 
+  only_version_tags: &only_version_tags
+    tags:
+      only: /^v[0-9]+\.[0-9]+\.[0-9]+(-(alpha|beta)\.\d+)?$/
+
 jobs:
   install:
     <<: *default_container_config
@@ -41,7 +45,9 @@ jobs:
       - run:
           name: Install dependencies
           command: |
-            npm install --no-save
+            npm install -no-save
+            make monorepo
+
       - run:
           name: Get test env vars
           command: make env
@@ -89,6 +95,17 @@ jobs:
           name: Verify
           command: make verify -j 3
 
+  publish:
+    <<: *default_container_config
+    steps:
+      - *attach_workspace
+      - run:
+          name: Setup npm credetials
+          command: echo "//registry.npmjs.org/:_authToken=$NPM_AUTH_TOKEN" > ~/.npmrc
+      # - run:
+      #     name: Publish to npm registry
+      #     command: make npm-publish
+
 workflows:
   version: 2
   build-and-deploy:
@@ -102,3 +119,12 @@ workflows:
           context: rel-eng-creds
           requires:
             - install
+      - publish:
+          context: rel-eng-creds
+          requires:
+            - install
+            - test
+          filters:
+            <<: *only_version_tags
+            branches:
+              ignore: /.*/

.eslintignore

@@ -12,3 +12,4 @@ dist
 coverage
 
 secret-squirrel.js
+packages/**/node_modules

.gitignore

@@ -12,4 +12,6 @@ yarn-error.log
 /neo4j
 .eslintcache
 junit.xml
+packages/*/node_modules
+packages/**/package-lock.json
 

Makefile

@@ -26,7 +26,20 @@ env:
 
 verify:
 
-install:
+# monorepo task should be ran before root installation
+# because root of package.json refers some pacakges/tc-* package internally.
+#
+# And install-treecreeper task is just fake task in order to run `instal%` task of rel-engage.
+# see https://github.com/Financial-Times/rel-engage/blob/master/index.mk#L100
+install: monorepo install-treecreeper
+
+monorepo:
+	npm install ysugimoto/athloi
+	npx athloi exec -- npm install --no-package-lock
+
+npm-publish:
+	npx athloi exec -- npm version --no-git-tag-version $(CIRCLE_TAG)
+	npx athloi publish
 
 .PHONY: test
 
@@ -67,7 +80,6 @@ test-api-docs:
 test-pkg-api-publish:
 	TREECREEPER_SCHEMA_DIRECTORY=example-schema DEBUG=true TIMEOUT=500000 jest --config="./jest.config.js" "packages/api-publish/__tests__/.*.spec.js" --testEnvironment=node --watch; \
 
-
 run:
 	TREECREEPER_SCHEMA_DIRECTORY=example-schema nodemon --inspect api/server/app.js
 
@@ -78,7 +90,7 @@ run-db:
 	docker-compose up
 
 init-db:
-	TREECREEPER_SCHEMA_DIRECTORY=example-schema packages/api-db-manager/index.js
+	TREECREEPER_SCHEMA_DIRECTORY=example-schema packages/tc-api-db-manager/index.js
 
 # load-testing
 load-test-run:

api/__tests__/init-db.spec.js

@@ -1,4 +1,4 @@
-const schema = require('../../packages/schema-sdk');
+const schema = require('@financial-times/tc-schema-sdk');
 const { initConstraints } = require('../server/init-db');
 const { driver } = require('../server/lib/db-connection');
 

api/__tests__/lib/locked-fields.spec.js

@@ -1,4 +1,4 @@
-const schema = require('../../../packages/schema-sdk');
+const schema = require('../../../packages/tc-schema-sdk');
 
 const {
 	mergeLockedFields,

api/__tests__/schema-polling-startup.spec.js

@@ -1,5 +1,5 @@
 const fetch = require('node-fetch');
-const { getSchemaFilename } = require('../../packages/schema-file-name');
+const { getSchemaFilename } = require('../../packages/tc-schema-file-name');
 
 jest.useFakeTimers();
 

api/__tests__/schema-polling-updates.spec.js

@@ -1,9 +1,9 @@
 const fetch = require('node-fetch');
-const schemaPublisher = require('../../packages/schema-publisher');
+const schemaPublisher = require('../../packages/tc-schema-publisher');
 const request = require('./helpers/supertest').getNamespacedSupertest(
 	'schema-polling',
 );
-const { getSchemaFilename } = require('../../packages/schema-file-name');
+const { getSchemaFilename } = require('../../packages/tc-schema-file-name');
 
 jest.useFakeTimers();
 
@@ -187,13 +187,13 @@ describe('schema polling updates', () => {
 	// Not testing this as directly as I'd like as it's tricky
 	it('reinitialises database contraints', async () => {
 		const onChange = jest.fn();
-		jest.doMock('../../packages/schema-sdk', () => ({
+		jest.doMock('../../packages/tc-schema-sdk', () => ({
 			init: () => null,
 			onChange,
 			ready: () => Promise.resolve(),
 		}));
 		const { initConstraints } = require('../server/init-db');
 		expect(onChange).toHaveBeenCalledWith(initConstraints);
-		jest.dontMock('../../packages/schema-sdk');
+		jest.dontMock('../../packages/tc-schema-sdk');
 	});
 });

api/server/health/constraints.js

@@ -1,5 +1,5 @@
 const { stripIndents } = require('common-tags');
-const { getTypes } = require('../../../packages/schema-sdk');
+const { getTypes } = require('@financial-times/tc-schema-sdk');
 const { logger } = require('../lib/request-context');
 const healthcheck = require('./healthcheck');
 const { executeQuery } = require('../lib/db-connection');

api/server/init-db.js

@@ -1,7 +1,7 @@
 #!/usr/bin/env node
 
 const logger = require('@financial-times/n-logger').default;
-const schema = require('../../packages/schema-sdk');
+const schema = require('@financial-times/tc-schema-sdk');
 const { schemaReady } = require('./lib/init-schema');
 const { executeQueryWithSharedSession } = require('./lib/db-connection');
 

api/server/lib/init-schema.js

@@ -1,4 +1,4 @@
-const schema = require('../../../packages/schema-sdk');
+const schema = require('@financial-times/tc-schema-sdk');
 
 schema.init({ updateMode: 'poll' });
 

api/server/lib/log-to-kinesis.js

@@ -1,5 +1,6 @@
 const groupBy = require('lodash.groupby');
-const { getType } = require('../../../packages/schema-sdk');
+// eslint-disable-next-line import/no-extraneous-dependencies
+const { getType } = require('@financial-times/tc-schema-sdk');
 const { getContext } = require('./request-context');
 const EventLogWriter = require('./event-log-writer');
 const Kinesis = require('./kinesis-client');

api/server/lib/schema-helpers.js

@@ -1,4 +1,4 @@
-const schema = require('../../../packages/schema-sdk');
+const schema = require('@financial-times/tc-schema-sdk');
 
 const invertDirection = direction =>
 	direction === 'incoming' ? 'outgoing' : 'incoming';

api/server/routes/graphql/api.js

@@ -1,12 +1,12 @@
 const bodyParser = require('body-parser');
 const timeout = require('connect-timeout');
+const { onChange } = require('@financial-times/tc-schema-sdk');
+const { sendSchemaToS3 } = require('@financial-times/tc-schema-publisher');
 const { logger, setContext } = require('../../lib/request-context');
 const security = require('../../middleware/security');
 const maintenance = require('../../middleware/maintenance');
 const clientId = require('../../middleware/client-id');
 const { TIMEOUT } = require('../../constants');
-const { onChange } = require('../../../../packages/schema-sdk');
-const { sendSchemaToS3 } = require('../../../../packages/schema-publisher');
 
 let schemaVersionIsConsistent = true;
 let graphqlAPI;

api/server/routes/graphql/lib/get-augmented-schema.js

@@ -2,10 +2,7 @@ const logger = require('@financial-times/n-logger').default;
 const { makeAugmentedSchema } = require('neo4j-graphql-js');
 const { applyMiddleware } = require('graphql-middleware');
 const { parse } = require('graphql');
-const {
-	getGraphqlDefs,
-	getTypes,
-} = require('../../../../../packages/schema-sdk');
+const { getGraphqlDefs, getTypes } = require('@financial-times/tc-schema-sdk');
 const { middleware: requestTracer } = require('./request-tracer');
 
 const getDocs = async (obj, args, context, info) => {

api/server/routes/graphql/lib/request-tracer.js

@@ -1,4 +1,4 @@
-const { getType } = require('../../../../../packages/schema-sdk');
+const { getType } = require('@financial-times/tc-schema-sdk');
 const { logger } = require('../../../lib/request-context');
 
 class Tracer {

api/server/routes/rest/lib/diff-helpers.js

@@ -1,5 +1,5 @@
 const neo4jTemporalTypes = require('neo4j-driver/lib/v1/temporal-types');
-const { getType } = require('../../../../../packages/schema-sdk');
+const { getType } = require('@financial-times/tc-schema-sdk');
 
 const isNullValue = val => val === null || val === '';
 

api/server/routes/rest/lib/error-handling.js

@@ -1,6 +1,6 @@
 const { stripIndents } = require('common-tags');
 const httpErrors = require('http-errors');
-const schema = require('../../../../../packages/schema-sdk');
+const schema = require('@financial-times/tc-schema-sdk');
 const { executeQuery } = require('./neo4j-model');
 
 const ERROR_RX = Object.freeze({

api/server/routes/rest/lib/neo4j-model.js

@@ -1,4 +1,4 @@
-const { getType } = require('../../../../../packages/schema-sdk');
+const { getType } = require('@financial-times/tc-schema-sdk');
 const { convertNeo4jTypes } = require('./neo4j-type-conversion');
 const { executeQuery } = require('../../../lib/db-connection');
 

api/server/routes/rest/lib/neo4j-type-conversion.js

@@ -4,7 +4,7 @@ const {
 	isTime,
 } = require('neo4j-driver/lib/v1/temporal-types');
 const neo4jTemporalTypes = require('neo4j-driver/lib/v1/temporal-types');
-const { getType } = require('../../../../../packages/schema-sdk');
+const { getType } = require('@financial-times/tc-schema-sdk');
 const { diffProperties } = require('./diff-helpers');
 
 const entriesToObject = (map, [key, val]) => Object.assign(map, { [key]: val });

api/server/routes/rest/lib/relationship-write-helpers.js

@@ -1,5 +1,5 @@
 const { stripIndents } = require('common-tags');
-const { getType } = require('../../../../../packages/schema-sdk');
+const { getType } = require('@financial-times/tc-schema-sdk');
 const { metaPropertiesForCreate } = require('./metadata-helpers');
 const { findInversePropertyNames } = require('../../../lib/schema-helpers');
 

api/server/routes/rest/lib/validation.js

@@ -2,7 +2,7 @@ const httpErrors = require('http-errors');
 const {
 	validators,
 	TreecreeperUserError,
-} = require('../../../../../packages/schema-sdk');
+} = require('@financial-times/tc-schema-sdk'); // eslint-disable-line import/no-extraneous-dependencies
 
 const validation = Object.entries(validators).reduce(
 	(methods, [key, validator]) => {

api/server/routes/rest/lib/write-helpers.js

@@ -1,7 +1,7 @@
 const { stripIndents } = require('common-tags');
 const httpError = require('http-errors');
 const _isEmpty = require('lodash.isempty');
-const { getType } = require('../../../../../packages/schema-sdk');
+const { getType } = require('@financial-times/tc-schema-sdk');
 const { executeQuery } = require('./neo4j-model');
 const { getAddedRelationships } = require('./diff-helpers');
 const { constructNeo4jProperties } = require('./neo4j-type-conversion');

api/server/routes/rest/v2/merge.js

@@ -1,5 +1,5 @@
 const httpErrors = require('http-errors');
-const { getType } = require('../../../../../packages/schema-sdk');
+const { getType } = require('@financial-times/tc-schema-sdk');
 const { validateTypeName } = require('../lib/validation');
 const { executeQuery } = require('../lib/neo4j-model');
 const { setContext } = require('../../../lib/request-context');

api/server/routes/rest/v2/node/patch.js

@@ -1,5 +1,5 @@
 const { stripIndents } = require('common-tags');
-const { getType } = require('../../../../../../packages/schema-sdk');
+const { getType } = require('@financial-times/tc-schema-sdk');
 const { validateParams, validatePayload } = require('../../lib/validation');
 const {
 	dbErrorHandlers,

demo/api.js

@@ -2,7 +2,7 @@ const express = require('express');
 const rewrite = require('express-urlrewrite'); // eslint-disable-line  import/no-extraneous-dependencies
 const expressPlayground = require('graphql-playground-middleware-express')
 	.default;
-const { getApp } = require('../packages/api-express');
+const { getApp } = require('../packages/tc-api-express');
 
 const PORT = process.env.PORT || 8888;
 const app = express();

example-schema/__tests__/schema-valid.spec.js

@@ -1 +1 @@
-require('../../packages/schema-validator');
+require('../../packages/tc-schema-validator');