diff --git a/README.md b/README.md index 79315868..d502d0ae 100644 --- a/README.md +++ b/README.md @@ -407,7 +407,7 @@ Emitted when a direct message is sent to the user. Unfortunately, Twitter has no stream.on('direct_message', function (directMsg) { //... }) -``` + ## event: 'user_event' @@ -577,8 +577,25 @@ THE SOFTWARE. ## Changelog +### 2.3.1 (Alembic/Twothink, Inc. Internal Version) + * Patch path handling when using full URLs instead of a path @netik + * Fold-in @tykarol's 2.3.0 changes + * Possible breaking change: Stop collapsing arrays in reqOpts.body if a POST request is in use (i.e. JSON). This fixes JSON object creation which is subtly broken in prior releases. + * Clean up and expand JSONPAYLOD_PATHS to have enterprise API locations + +### 2.3.0 + * Allow uploading media files from file streams #1 @joeypatino + * Added an Direct message example for Twit API readme docs #2 @mtpjr88 + * Add headers to REST error object #3 @Harrison-M + * Fix banner update URL (update_profile_background_image is deprecated) #4 @it-fm + * Use correct-content type when GET-ing a media/ path #5 @AndrewBarba + * Add support for subtitles create and delete #6 @dyep49 + * Add new json payload path #7 @ashafa + * Video/upload chunk size #8 @pthieu + * Change MAX_VIDEO_CHUNK_BYTES to 5MB + ### 2.2.11 -* Fix media_category used for media uploads (thanks @BooDoo) + * Fix media_category used for media uploads (thanks @BooDoo) ### 2.2.10 * Update maximum Tweet characters to 280 (thanks @maziyarpanahi) diff --git a/examples.md b/examples.md new file mode 100644 index 00000000..d21d67e0 --- /dev/null +++ b/examples.md @@ -0,0 +1,25 @@ +Example for a DM response when a user follows +```javascript + +// Setting up a user stream +var stream = T.stream('user'); + +function followed(eventMsg) { + console.log('Follow event!'); + var name = eventMsg.source.name; + var screenName = eventMsg.source.screen_name; + + // Anytime Someone follows me + stream.on('follow', followed); + + // the post request for direct messages > need to add a function to handle errors + + setTimeout(function() { // wait 60 sec before sending direct message. + console.log("Direct Message sent"); + T.post("direct_messages/new", { + screen_name: screenName, + text: 'Thanks for following' + ' ' + screenName + '! ' + ' What you want to be sent to a new follower ' + }); + }, 1000*10); // will respond via direct message 10 seconds after a user follows. +}; +``` \ No newline at end of file diff --git a/lib/file_uploader.js b/lib/file_uploader.js index ffaf1b94..3d462ea7 100644 --- a/lib/file_uploader.js +++ b/lib/file_uploader.js @@ -2,9 +2,13 @@ var assert = require('assert'); var fs = require('fs'); var mime = require('mime'); var util = require('util'); +var request = require('http'); +var fileType = require('file-type'); var MAX_FILE_SIZE_BYTES = 15 * 1024 * 1024; var MAX_FILE_CHUNK_BYTES = 5 * 1024 * 1024; +var MAX_VIDEO_SIZE_BYTES = 512 * 1024 * 1024; +var MAX_VIDEO_CHUNK_BYTES = 5 * 1024 * 1024; /** * FileUploader class used to upload a file to twitter via the /media/upload (chunked) API. @@ -14,18 +18,28 @@ var MAX_FILE_CHUNK_BYTES = 5 * 1024 * 1024; * console.log(err, bodyObj); * }) * - * @param {Object} params Object of the form { file_path: String }. + * var request = require('request') + * var stream = request('http://www.domain.com/file_to_upload.ext') + * var fu = new FileUploader({ file_stream: stream }, twit); + * fu.upload(function (err, bodyObj, resp) { + * console.log(err, bodyObj); + * }) + * + * @param {Object} params Object of the form { file_path: String } or { file_strea: ReadableStream } * @param {Twit(object)} twit Twit instance. */ var FileUploader = function (params, twit) { assert(params) - assert(params.file_path, 'Must specify `file_path` to upload a file. Got: ' + params.file_path + '.') + assert(params.file_path || params.file_stream, 'Must specify `file_path` or `file_stream` to upload a file.') var self = this; self._file_path = params.file_path; + self._file_stream = params.file_stream; + self._remoteUpload = self._file_stream ? true : false; self._twit = twit; self._isUploading = false; self._isFileStreamEnded = false; self._isSharedMedia = !!params.shared; + if (self._remoteUpload) {self._file_stream.pause();} } /** @@ -42,9 +56,10 @@ FileUploader.prototype.upload = function (cb) { cb(err); return; } else { + var MAX_CHUNK_BYTES = bodyObj.MAX_CHUNK_BYTES || MAX_FILE_CHUNK_BYTES; var mediaTmpId = bodyObj.media_id_string; var chunkNumber = 0; - var mediaFile = fs.createReadStream(self._file_path, { highWatermark: MAX_FILE_CHUNK_BYTES }); + var mediaFile = self._file_stream || fs.createReadStream(self._file_path, { highWaterMark: MAX_CHUNK_BYTES }); mediaFile.on('data', function (chunk) { // Pause our file stream from emitting `data` events until the upload of this chunk completes. @@ -76,6 +91,7 @@ FileUploader.prototype.upload = function (cb) { self._finalizeMedia(mediaTmpId, cb); } }); + mediaFile.resume(); } }) } @@ -117,6 +133,38 @@ FileUploader.prototype._appendMedia = function(media_id_string, chunk_part, segm }, cb); } +FileUploader.prototype._getFileInfoForUpload = function(cb) { + var self = this; + + if (self._remoteUpload === true) { + + request.get(self._file_stream.uri.href, function(res){ + res.once('data', function(chunk){ + + var len = res.headers['content-length'] + if (!len) { + return cb(new Error('Unable to determine file size')) + } + len = +len + if (len !== len) { + return cb(new Error('Invalid Content-Length received')) + } + + var mediaFileSizeBytes = +res.headers['content-length'] + var mediaType = fileType(chunk)["mime"]; + + cb(null, mediaType, mediaFileSizeBytes); + res.destroy(); + }); + }); + } + else { + var mediaType = mime.lookup(self._file_path); + var mediaFileSizeBytes = fs.statSync(self._file_path).size; + cb(null, mediaType, mediaFileSizeBytes); + } +} + /** * Send INIT command for our underlying media object. * @@ -124,31 +172,41 @@ FileUploader.prototype._appendMedia = function(media_id_string, chunk_part, segm */ FileUploader.prototype._initMedia = function (cb) { var self = this; - var mediaType = mime.lookup(self._file_path); - var mediaFileSizeBytes = fs.statSync(self._file_path).size; - var shared = self._isSharedMedia; - var media_category = 'tweet_image'; - - if (mediaType.toLowerCase().indexOf('gif') > -1) { - media_category = 'tweet_gif'; - } else if (mediaType.toLowerCase().indexOf('video') > -1) { - media_category = 'tweet_video'; - } - // Check the file size - it should not go over 15MB for video. - // See https://dev.twitter.com/rest/reference/post/media/upload-chunked - if (mediaFileSizeBytes < MAX_FILE_SIZE_BYTES) { - self._twit.post('media/upload', { - 'command': 'INIT', - 'media_type': mediaType, - 'total_bytes': mediaFileSizeBytes, - 'shared': shared, - 'media_category': media_category - }, cb); - } else { - var errMsg = util.format('This file is too large. Max size is %dB. Got: %dB.', MAX_FILE_SIZE_BYTES, mediaFileSizeBytes); - cb(new Error(errMsg)); - } + self._getFileInfoForUpload(function(err, mediaType, mediaFileSizeBytes){ + var shared = self._isSharedMedia; + var media_category = 'tweet_image'; + var maxFileBytes = MAX_FILE_SIZE_BYTES; + var maxChunkBytes = MAX_FILE_CHUNK_BYTES; + + if (mediaType.toLowerCase().indexOf('gif') > -1) { + media_category = 'tweet_gif'; + } else if (mediaType.toLowerCase().indexOf('video') > -1) { + media_category = 'tweet_video'; + maxFileBytes = MAX_VIDEO_SIZE_BYTES; + maxChunkBytes = MAX_VIDEO_CHUNK_BYTES; + } else if (mediaType.toLowerCase().indexOf('subrip') > -1) { + media_category = 'SUBTITLES' + } + + // Check the file size - it should not go over 15MB for video. + // See https://dev.twitter.com/rest/reference/post/media/upload-chunked + if (mediaFileSizeBytes < maxFileBytes) { + self._twit.post('media/upload', { + 'command': 'INIT', + 'media_type': mediaType, + 'total_bytes': mediaFileSizeBytes, + 'shared': shared, + 'media_category': media_category + }, function (err, bodyObj, resp) { + bodyObj.MAX_CHUNK_BYTES = maxChunkBytes; + cb(err, bodyObj, resp); + }); + } else { + var errMsg = util.format('This file is too large. Max size is %dB. Got: %dB.', maxFileBytes, mediaFileSizeBytes); + cb(new Error(errMsg)); + } + }); } module.exports = FileUploader diff --git a/lib/twitter.js b/lib/twitter.js index 9540857e..6588fc88 100644 --- a/lib/twitter.js +++ b/lib/twitter.js @@ -26,14 +26,26 @@ var required_for_user_auth = required_for_app_auth.concat([ var FORMDATA_PATHS = [ 'media/upload', 'account/update_profile_image', - 'account/update_profile_background_image', + 'account/update_profile_banner', ]; +// if the path starts with anything on this list, then the +// payload will be sent as a json payload. var JSONPAYLOAD_PATHS = [ - 'media/metadata/create', - 'direct_messages/events/new', - 'direct_messages/welcome_messages/new', - 'direct_messages/welcome_messages/rules/new', + 'collections/entries/curate', + 'direct_messages/events/new', + 'direct_messages/events/new', + 'direct_messages/welcome_messages/new', + 'direct_messages/welcome_messages/rules/new', + 'insights/engagement', + 'insights/engagement/28hr', + 'insights/engagement/historical', + 'insights/engagement/totals', + 'media/metadata/create', + 'media/subtitles/create', + 'media/subtitles/delete', + 'search/30day', + 'search/fullarchive' ]; // @@ -97,9 +109,10 @@ Twitter.prototype.request = function (method, path, params, callback) { } var twitOptions = (params && params.twit_options) || {}; - + process.nextTick(function () { // ensure all HTTP i/o occurs after the user has a chance to bind their event handlers + self._doRestApiRequest(reqOpts, twitOptions, method, function (err, parsedBody, resp) { self._updateClockOffsetFromResponse(resp); var peerCertificate = resp && resp.socket && resp.socket.getPeerCertificate(); @@ -195,8 +208,18 @@ Twitter.prototype._buildReqOpts = function (method, path, params, isStreaming, c } // clone `params` object so we can modify it without modifying the user's reference var paramsClone = JSON.parse(JSON.stringify(params)) - // convert any arrays in `paramsClone` to comma-seperated strings - var finalParams = this.normalizeParams(paramsClone) + + // jna: only permit this to happen if we are doing a GET request. There is no + // reason to do this in a POST. it breaks things like the enterprise API. + + // convert any arrays in `paramsClone` to comma-separated strings + var finalParams; + if (method == 'GET') { + finalParams = this.normalizeParams(paramsClone) + } else { + finalParams = paramsClone; + } + delete finalParams.twit_options // the options object passed to `request` used to perform the HTTP request @@ -219,7 +242,31 @@ Twitter.prototype._buildReqOpts = function (method, path, params, isStreaming, c // finalize the `path` value by building it using user-supplied params // when json parameters should not be in the payload - if (JSONPAYLOAD_PATHS.indexOf(path) === -1) { + + // jna: The public version of this library does not deal with + // full paths. The matcher breaks substantially when the path + // starts with http(s)://. Fix this by extracting the path + // component and testing that instead of the full "path", which is + // actually a "url". + let testPath; + + try { + // if we can parse the string as a url, we must extract the path + const parsedUrl = new URL(path); + testPath = parsedUrl.pathname; + if (testPath[0] == '/') { + testPath = testPath.substring(1, testPath.length); + } + } catch(e) { + // we only expect TypeError here, which reads as "Invalid URL." + if (! e instanceof TypeError) { + throw e; // unexpected, so throw it again. Not our problem. + } + // just pass it through untouched. + testPath = path; + } + + if (JSONPAYLOAD_PATHS.indexOf(testPath) === -1) { try { path = helpers.moveParamsIntoPath(finalParams, path) } catch (e) { @@ -230,11 +277,20 @@ Twitter.prototype._buildReqOpts = function (method, path, params, isStreaming, c if (path.match(/^https?:\/\//i)) { // This is a full url request - reqOpts.url = path + reqOpts.url = path; + + // if the endpoint requires JSON, set up options correctly. + if (JSONPAYLOAD_PATHS.indexOf(testPath) !== -1) { + reqOpts.headers['Content-type'] = 'application/json'; + reqOpts.json = true; + reqOpts.body = finalParams; + + // avoid appending query string for body params + finalParams = {}; + } } else if (isStreaming) { // This is a Streaming API request. - var stream_endpoint_map = { user: endpoints.USER_STREAM, site: endpoints.SITE_STREAM @@ -251,13 +307,13 @@ Twitter.prototype._buildReqOpts = function (method, path, params, isStreaming, c reqOpts.url = endpoints.REST_ROOT + path + '.json'; } - if (FORMDATA_PATHS.indexOf(path) !== -1) { + if (FORMDATA_PATHS.indexOf(path) !== -1 && method === 'POST') { reqOpts.headers['Content-type'] = 'multipart/form-data'; reqOpts.form = finalParams; // set finalParams to empty object so we don't append a query string // of the params finalParams = {}; - } else if (JSONPAYLOAD_PATHS.indexOf(path) !== -1) { + } else if (JSONPAYLOAD_PATHS.indexOf(testPath) !== -1) { reqOpts.headers['Content-type'] = 'application/json'; reqOpts.json = true; reqOpts.body = finalParams; @@ -343,6 +399,7 @@ Twitter.prototype._doRestApiRequest = function (reqOpts, twitOptions, method, ca // place the errors in the HTTP response body into the Error object and pass control to caller var err = helpers.makeTwitError('Twitter API Error') err.statusCode = response ? response.statusCode: null; + err.headers = response ? response.headers : null; helpers.attachBodyInfoToError(err, body); callback(err, body, response); return diff --git a/package.json b/package.json index a8bcdcaa..2f36d9e4 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "twit", "description": "Twitter API client for node (REST & Streaming)", - "version": "2.2.11", + "version": "2.3.1", "author": "Tolga Tezel", "keywords": [ "twitter", @@ -13,6 +13,7 @@ ], "dependencies": { "bluebird": "^3.1.5", + "file-type": "^3.9.0", "mime": "^1.3.4", "request": "^2.68.0" }, diff --git a/tests/rest.js b/tests/rest.js index 63fcbdbf..5d95d5a9 100644 --- a/tests/rest.js +++ b/tests/rest.js @@ -638,6 +638,7 @@ describe('REST API', function () { assert(err.statusCode === 401) assert(err.code > 0) assert(err.message.match(/token/)) + assert(typeof err.headers === 'object') assert(err.twitterReply) assert(err.allErrors) assert(res) @@ -654,6 +655,7 @@ describe('REST API', function () { .catch(err => { assert(err instanceof Error) assert(err.statusCode === 401) + assert(typeof err.headers === 'object') assert(err.code > 0) assert(err.message.match(/token/)) assert(err.twitterReply)