diff --git a/package-lock.json b/package-lock.json index 78e7853..4b79897 100644 --- a/package-lock.json +++ b/package-lock.json @@ -83,19 +83,6 @@ "resolved": "https://registry.npmjs.org/algorithmia/-/algorithmia-0.3.10.tgz", "integrity": "sha1-9xwgqf9WRmCk0c9zlHmV123jRQw=" }, - "ansi-styles": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", - "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", - "requires": { - "color-convert": "^1.9.0" - } - }, - "array-uniq": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/array-uniq/-/array-uniq-1.0.3.tgz", - "integrity": "sha1-r2rId6Jcx/dOBYiUdThY39sk/bY=" - }, "asn1": { "version": "0.2.4", "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.4.tgz", @@ -150,34 +137,11 @@ "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz", "integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw=" }, - "chalk": { - "version": "2.4.2", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", - "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", - "requires": { - "ansi-styles": "^3.2.1", - "escape-string-regexp": "^1.0.5", - "supports-color": "^5.3.0" - } - }, "co": { "version": "4.6.0", "resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz", "integrity": "sha1-bqa989hTrlTMuOR7+gvz+QMfsYQ=" }, - "color-convert": { - "version": "1.9.3", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", - "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", - "requires": { - "color-name": "1.1.3" - } - }, - "color-name": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", - "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=" - }, "combined-stream": { "version": "1.0.7", "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.7.tgz", @@ -220,37 +184,6 @@ "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", "integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk=" }, - "dom-serializer": { - "version": "0.1.1", - "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-0.1.1.tgz", - "integrity": "sha512-l0IU0pPzLWSHBcieZbpOKgkIn3ts3vAh7ZuFyXNwJxJXk/c4Gwj9xaTJwIDVQCXawWD0qb3IzMGH5rglQaO0XA==", - "requires": { - "domelementtype": "^1.3.0", - "entities": "^1.1.1" - } - }, - "domelementtype": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-1.3.1.tgz", - "integrity": "sha512-BSKB+TSpMpFI/HOxCNr1O8aMOTZ8hT3pM3GQ0w/mWRmkhEDSFJkkyzz4XQsBV44BChwGkrDfMyjVD0eA2aFV3w==" - }, - "domhandler": { - "version": "2.4.2", - "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-2.4.2.tgz", - "integrity": "sha512-JiK04h0Ht5u/80fdLMCEmV4zkNh2BcoMFBmZ/91WtYZ8qVXSKjiw7fXMgFPnHcSZgOo3XdinHvmnDUeMf5R4wA==", - "requires": { - "domelementtype": "1" - } - }, - "domutils": { - "version": "1.7.0", - "resolved": "https://registry.npmjs.org/domutils/-/domutils-1.7.0.tgz", - "integrity": "sha512-Lgd2XcJ/NjEw+7tFvfKxOzCYKZsdct5lczQ2ZaQY8Djz7pfAD3Gbp8ySJWtreII/vDlMVmxwa6pHmdxIYgttDg==", - "requires": { - "dom-serializer": "0", - "domelementtype": "1" - } - }, "dotenv": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-2.0.0.tgz", @@ -265,16 +198,6 @@ "safer-buffer": "^2.1.0" } }, - "entities": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/entities/-/entities-1.1.2.tgz", - "integrity": "sha512-f2LZMYl1Fzu7YSBKg+RoROelpOaNrcGmE9AZubeDfrCEia483oW4MI4VyFd5VNHIgQ/7qm1I0wUHK1eJnn2y2w==" - }, - "escape-string-regexp": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", - "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=" - }, "extend": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", @@ -337,24 +260,6 @@ "har-schema": "^2.0.0" } }, - "has-flag": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", - "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=" - }, - "htmlparser2": { - "version": "3.10.1", - "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-3.10.1.tgz", - "integrity": "sha512-IgieNijUMbkDovyoKObU1DUhm1iwNYE/fuifEoEHfd1oZKZDaONBSkal7Y01shxsM49R4XaMdGez3WnF9UfiCQ==", - "requires": { - "domelementtype": "^1.3.1", - "domhandler": "^2.3.0", - "domutils": "^1.5.1", - "entities": "^1.1.1", - "inherits": "^2.0.1", - "readable-stream": "^3.1.1" - } - }, "http-signature": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.2.0.tgz", @@ -393,11 +298,6 @@ } } }, - "inherits": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", - "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=" - }, "is-extendable": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-1.0.1.tgz", @@ -460,41 +360,21 @@ "verror": "1.10.0" } }, + "lexrank.js": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/lexrank.js/-/lexrank.js-1.3.3.tgz", + "integrity": "sha512-Dz8ak7ftRZhtB7/tIdrE7A9OXJ+dFlWpQc2AMOOQKLBetIB3juQwYAl7nTBRNIonE45ATmPZN/NlK7NQHx1XPQ==" + }, "lodash": { "version": "4.17.11", "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.11.tgz", "integrity": "sha512-cQKh8igo5QUhZ7lg38DYWAxMvjSAKG0A8wGSVimP07SIUEK2UO+arSRKbRZWtelMtN5V0Hkwh5ryOto/SshYIg==" }, - "lodash.clonedeep": { - "version": "4.5.0", - "resolved": "https://registry.npmjs.org/lodash.clonedeep/-/lodash.clonedeep-4.5.0.tgz", - "integrity": "sha1-4j8/nE+Pvd6HJSnBBxhXoIblzO8=" - }, - "lodash.escaperegexp": { - "version": "4.1.2", - "resolved": "https://registry.npmjs.org/lodash.escaperegexp/-/lodash.escaperegexp-4.1.2.tgz", - "integrity": "sha1-ZHYsSGGAglGKw99Mz11YhtriA0c=" - }, "lodash.get": { "version": "4.4.2", "resolved": "https://registry.npmjs.org/lodash.get/-/lodash.get-4.4.2.tgz", "integrity": "sha1-LRd/ZS+jHpObRDjVNBSZ36OCXpk=" }, - "lodash.isplainobject": { - "version": "4.0.6", - "resolved": "https://registry.npmjs.org/lodash.isplainobject/-/lodash.isplainobject-4.0.6.tgz", - "integrity": "sha1-fFJqUtibRcRcxpC4gWO+BJf1UMs=" - }, - "lodash.isstring": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/lodash.isstring/-/lodash.isstring-4.0.1.tgz", - "integrity": "sha1-1SfftUVuynzJu5XV2ur4i6VKVFE=" - }, - "lodash.mergewith": { - "version": "4.6.1", - "resolved": "https://registry.npmjs.org/lodash.mergewith/-/lodash.mergewith-4.6.1.tgz", - "integrity": "sha512-eWw5r+PYICtEBgrBE5hhlT6aAa75f411bgDz/ZL2KZqYV03USvucsxcHUIlGTDTECs1eunpI7HOV7U+WLDvNdQ==" - }, "mime-db": { "version": "1.38.0", "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.38.0.tgz", @@ -518,11 +398,6 @@ "resolved": "https://registry.npmjs.org/nan/-/nan-2.13.1.tgz", "integrity": "sha512-I6YB/YEuDeUZMmhscXKxGgZlFnhsn5y0hgOZBadkzfTRrZBtJDZeg6eQf7PYMIEclwmorTKK8GztsyOUSVBREA==" }, - "number-is-nan": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/number-is-nan/-/number-is-nan-1.0.1.tgz", - "integrity": "sha1-CXtgK1NCKlIsGvuHkDGDNpQaAR0=" - }, "oauth-sign": { "version": "0.8.2", "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.8.2.tgz", @@ -549,26 +424,6 @@ "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz", "integrity": "sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns=" }, - "postcss": { - "version": "7.0.14", - "resolved": "https://registry.npmjs.org/postcss/-/postcss-7.0.14.tgz", - "integrity": "sha512-NsbD6XUUMZvBxtQAJuWDJeeC4QFsmWsfozWxCJPWf3M55K9iu2iMDaKqyoOdTJ1R4usBXuxlVFAIo8rZPQD4Bg==", - "requires": { - "chalk": "^2.4.2", - "source-map": "^0.6.1", - "supports-color": "^6.1.0" - }, - "dependencies": { - "supports-color": { - "version": "6.1.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-6.1.0.tgz", - "integrity": "sha512-qe1jfm1Mg7Nq/NSh6XE24gPXROEVsWHxC1LIx//XNlD9iw7YZQGjZNjYN7xGaEG6iKdA8EtNFW6R0gjnVXp+wQ==", - "requires": { - "has-flag": "^3.0.0" - } - } - } - }, "punycode": { "version": "1.4.1", "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz", @@ -579,16 +434,6 @@ "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.2.tgz", "integrity": "sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA==" }, - "readable-stream": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.2.0.tgz", - "integrity": "sha512-RV20kLjdmpZuTF1INEb9IA3L68Nmi+Ri7ppZqo78wj//Pn62fCoJyV9zalccNzDD/OuJpMG4f+pfMl8+L6QdGw==", - "requires": { - "inherits": "^2.0.3", - "string_decoder": "^1.1.1", - "util-deprecate": "^1.0.1" - } - }, "readline-sync": { "version": "1.4.9", "resolved": "https://registry.npmjs.org/readline-sync/-/readline-sync-1.4.9.tgz", @@ -631,50 +476,11 @@ "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" }, - "sanitize-html": { - "version": "1.20.0", - "resolved": "https://registry.npmjs.org/sanitize-html/-/sanitize-html-1.20.0.tgz", - "integrity": "sha512-BpxXkBoAG+uKCHjoXFmox6kCSYpnulABoGcZ/R3QyY9ndXbIM5S94eOr1IqnzTG8TnbmXaxWoDDzKC5eJv7fEQ==", - "requires": { - "chalk": "^2.4.1", - "htmlparser2": "^3.10.0", - "lodash.clonedeep": "^4.5.0", - "lodash.escaperegexp": "^4.1.2", - "lodash.isplainobject": "^4.0.6", - "lodash.isstring": "^4.0.1", - "lodash.mergewith": "^4.6.1", - "postcss": "^7.0.5", - "srcset": "^1.0.0", - "xtend": "^4.0.1" - } - }, - "sbd": { - "version": "1.0.14", - "resolved": "https://registry.npmjs.org/sbd/-/sbd-1.0.14.tgz", - "integrity": "sha512-jrpm6EnGL4IFjzmvlQAyjaec0fnmUd8TNBnCdoq8jw1IeJXDg8129C8I6U1rNBSU42hb523Euj6ik0DTBjx6pw==", - "requires": { - "sanitize-html": "^1.18.4" - } - }, "semver": { "version": "5.6.0", "resolved": "https://registry.npmjs.org/semver/-/semver-5.6.0.tgz", "integrity": "sha512-RS9R6R35NYgQn++fkDWaOmqGoj4Ek9gGs+DPxNUZKuwE183xjJroKvyo1IzVFeXvUrvmALy6FWD5xrdJT25gMg==" }, - "source-map": { - "version": "0.6.1", - "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", - "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==" - }, - "srcset": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/srcset/-/srcset-1.0.0.tgz", - "integrity": "sha1-pWad4StC87HV6D7QPHEEb8SPQe8=", - "requires": { - "array-uniq": "^1.0.2", - "number-is-nan": "^1.0.0" - } - }, "sshpk": { "version": "1.16.1", "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.16.1.tgz", @@ -691,22 +497,6 @@ "tweetnacl": "~0.14.0" } }, - "string_decoder": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.2.0.tgz", - "integrity": "sha512-6YqyX6ZWEYguAxgZzHGL7SsCeGx3V2TtOTqZz1xSTSWnqsbWwbptafNyvf/ACquZUXV3DANr5BDIwNYe1mN42w==", - "requires": { - "safe-buffer": "~5.1.0" - } - }, - "supports-color": { - "version": "5.5.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", - "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", - "requires": { - "has-flag": "^3.0.0" - } - }, "tough-cookie": { "version": "2.3.4", "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.3.4.tgz", @@ -736,11 +526,6 @@ "is-typedarray": "^1.0.0" } }, - "util-deprecate": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", - "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=" - }, "uuid": { "version": "3.3.2", "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz", @@ -800,11 +585,6 @@ "yaeti": "^0.0.6" } }, - "xtend": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.1.tgz", - "integrity": "sha1-pcbVMr5lbiPbgg77lDofBJmNY68=" - }, "yaeti": { "version": "0.0.6", "resolved": "https://registry.npmjs.org/yaeti/-/yaeti-0.0.6.tgz", diff --git a/package.json b/package.json index 66aa1dc..2a7d529 100644 --- a/package.json +++ b/package.json @@ -18,8 +18,8 @@ "homepage": "https://github.com/filipedeschamps/video-maker#readme", "dependencies": { "algorithmia": "^0.3.10", + "lexrank.js": "^1.3.3", "readline-sync": "^1.4.9", - "sbd": "^1.0.14", "watson-developer-cloud": "^3.18.2" } } diff --git a/robots/text.js b/robots/text.js index 4b15de0..5a7ad8c 100644 --- a/robots/text.js +++ b/robots/text.js @@ -1,6 +1,6 @@ const algorithmia = require('algorithmia') const algorithmiaApiKey = require('../credentials/algorithmia.json').apiKey -const sentenceBoundaryDetection = require('sbd') +const summary = require('lexrank.js') const watsonApiKey = require('../credentials/watson-nlu.json').apikey const NaturalLanguageUnderstandingV1 = require('watson-developer-cloud/natural-language-understanding/v1.js') @@ -14,7 +14,7 @@ var nlu = new NaturalLanguageUnderstandingV1({ async function robot(content) { await fetchContentFromWikipedia(content) sanitizeContent(content) - breakContentIntoSentences(content) + await breakContentIntoLexicalRankedSentences(content) limitMaximumSentences(content) await fetchKeywordsOfAllSentences(content) @@ -24,7 +24,7 @@ async function robot(content) { const wikipediaResponse = await wikipediaAlgorithm.pipe(content.searchTerm) const wikipediaContent = wikipediaResponse.get() - content.sourceContentOriginal = wikipediaContent.content + content.sourceContentOriginal = wikipediaContent.summary } function sanitizeContent(content) { @@ -52,15 +52,27 @@ async function robot(content) { return text.replace(/\((?:\([^()]*\)|[^()])*\)/gm, '').replace(/ /g,' ') } - function breakContentIntoSentences(content) { - content.sentences = [] + async function breakContentIntoLexicalRankedSentences(content) { + return new Promise((resolve, reject) => { + content.sentences = [] - const sentences = sentenceBoundaryDetection.sentences(content.sourceContentSanitized) - sentences.forEach((sentence) => { - content.sentences.push({ - text: sentence, - keywords: [], - images: [] + summary.lexrank(content.sourceContentSanitized, (error, result) => { + if (error) { + throw error + return reject(error) + } + + sentences = result[0].sort(function(a,b){return b.weight.average - a.weight.average}) + + sentences.forEach((sentence) => { + content.sentences.push({ + text: sentence.text, + keywords: [], + images: [] + }) + }) + + resolve(sentences) }) }) }