New image proxy implementation and image sanitisation code (#513)

* added new image proxy implementation. Image sanitisation is still WIP

* changes to make auth work. refactoring to support rewrite rules better

* disabled image proxy logger

* fixed header name

* extract type

package.json

@@ -89,7 +89,7 @@
     "graphology-layout-forceatlas2": "^0.10.1",
     "graphology-metrics": "^2.3.1",
     "helmet": "^3.21.1",
-    "http-proxy-middleware": "^2.0.1",
+    "http-proxy-middleware": "^3.0.3",
     "impresso-jscommons": "https://github.com/impresso/impresso-jscommons/tarball/v1.5.0",
     "json-bigint": "github:sidorares/json-bigint#8208aa1352eba252b60756672ab7c17f300d6ccd",
     "json2csv": "^4.3.3",
@@ -109,7 +109,7 @@
     "neo4j-driver": "^1.7.2",
     "node-eta": "^0.9.0",
     "node-fetch": "2.7.0",
-    "node-http-proxy-json": "^0.1.6",
+    "node-http-proxy-json": "^0.1.9",
     "passport-github": "^1.1.0",
     "pg": "^8.11.3",
     "pg-hstore": "^2.3.2",

src/app.hooks.ts

@@ -4,6 +4,7 @@ import { ApplicationHookFunction, ApplicationHookOptions, HookContext } from '@f
 import { ValidationError } from 'ajv'
 import { GeneralError, BadGateway, BadRequest, Unprocessable } from '@feathersjs/errors'
 import { ImpressoApplication } from './types'
+import { logger } from './logger'
 const { authenticate } = require('@feathersjs/authentication').hooks
 
 const debug = Debug('impresso/app.hooks')
@@ -56,7 +57,7 @@ const errorHandler = (ctx: HookContext<ImpressoApplication>) => {
     const error = ctx.error
 
     if (!excludedStatusCodes.includes(error.code) || !error.code) {
-      console.error(
+      logger.error(
         `ERROR ${error.code || error.type || 'N/A'} ${error.name} at ${ctx.path}:${ctx.method}: `,
         error.stack
       )

src/app.ts

@@ -21,7 +21,8 @@ import sequelize from './sequelize'
 import services from './services'
 import rateLimiter from './services/internal/rateLimiter/redis'
 import media from './services/media'
-import proxy from './services/proxy'
+// import imageProxyv1 from './services/proxy'
+import { init as imageProxy } from './middleware/imageProxy'
 import schemas from './services/schemas'
 import { AppServices, ImpressoApplication } from './types'
 import { customJsonMiddleware } from './util/express'
@@ -61,7 +62,7 @@ app.configure(celery)
 
 // configure express services
 app.configure(media)
-app.configure(proxy)
+app.configure(imageProxy)
 app.configure(schemas)
 
 // Enable Swagger and API validator if needed

src/hooks/resolvers.ts

@@ -1,4 +1,5 @@
 import lodash from 'lodash'
+import { logger } from '../logger'
 import Collection from '../models/collections.model'
 import { HookContext } from '@feathersjs/feathers'
 import { Service as SearchFacetService } from '../services/search-facets/search-facets.class'
@@ -61,8 +62,8 @@ export const resolveTextReuseClusters = () => async (context: HookContext<Impres
       return lodash.keyBy(data, 'textReuseCluster.id')
     })
     .catch((err: Error) => {
-      console.error('hook resolveTextReuseClusters ERROR')
-      console.error(err)
+      logger.error('hook resolveTextReuseClusters ERROR')
+      logger.error(err)
     })
   debug('resolveTextReuseClusters index keys:', Object.keys(index))
 
@@ -106,8 +107,8 @@ export const resolveCollections = () => async (context: HookContext<ImpressoAppl
       )
     )
     .catch((err: Error) => {
-      console.error('hook resolveCollections ERROR')
-      console.error(err)
+      logger.error('hook resolveCollections ERROR')
+      logger.error(err)
       return {}
     })) as lodash.Dictionary<any>
 

src/hooks/validators.js

@@ -1,4 +1,5 @@
 /* eslint-disable max-len */
+import { logger } from '../logger'
 const debug = require('debug')('impresso/hooks/validators')
 const { get, set } = require('lodash')
 const { BadRequest } = require('@feathersjs/errors')
@@ -41,7 +42,7 @@ const validateWithSchemaUri =
       try {
         candidate = JSON.parse(candidate)
       } catch (e) {
-        console.error(e)
+        logger.error(e)
         throw new BadRequest('Bad JSON received, check your input data.', {
           [labelPath]: 'should be a valid JSON parameter',
         })

src/index.ts

@@ -0,0 +1,31 @@
+import app from './app'
+import { logger } from './logger'
+
+process.on('unhandledRejection', (error: Error) => {
+  logger.error('unhandledRejection', error)
+})
+process.on('uncaughtException', (err: Error) => {
+  logger.error('uncaughtException', err)
+})
+
+const start = async () => {
+  const [host = 'localhost', port = 8080] = [app.get('host'), app.get('port')]
+
+  const server = await app
+    .listen(port, host, () => {
+      logger.info(`Application server listening on http://${host}:${port}`)
+    })
+    .catch(err => {
+      logger.error('Server startup error:', err)
+      process.exit(1)
+    })
+
+  server.on('error', err => {
+    logger.error('Server error', err)
+  })
+  server.on('close', () => {
+    logger.error('Server closed')
+  })
+}
+
+start()

src/logic/textReuse/solr.js

@@ -1,3 +1,4 @@
+import { logger } from '../../logger'
 const assert = require('assert')
 const { get, omitBy, isUndefined } = require('lodash')
 const { SolrMappings } = require('../../data/constants')
@@ -222,7 +223,7 @@ function getPaginationInfoFromPassagesSolrResponse(solrResponse) {
         total: get(solrResponse, 'response.numFound'),
       }
     } catch (e) {
-      console.warn(e)
+      logger.warning(e)
       return {
         limit: 10,
         offset: 0,

src/middleware/errorHandling.ts

@@ -120,6 +120,7 @@ export default (app: ImpressoApplication & ExpressApplication) => {
           } satisfies Problem)
         },
       },
+      logger: false,
     })
   )
 }