Skip to content

Commit

Permalink
Merge pull request #1843 from filipedeschamps/use-remove-markdown-lib…
Browse files Browse the repository at this point in the history 
…rary

Volta a usar a biblioteca `remove-markdown`
  • Loading branch information
@aprendendofelipe
aprendendofelipe authored Jan 23, 2025
2 parents 5d1f45a + ecd4ebd commit fa73f50
Show file tree
Hide file tree
Showing 6 changed files with 135 additions and 94 deletions.
121 changes: 34 additions & 87 deletions models/remove-markdown.js
View file
Original file line number Diff line number Diff line change
@@ -1,91 +1,15 @@
// Original code from: https://github.com/stiang/remove-markdown
// With a fix by @aprendendofelipe https://github.com/stiang/remove-markdown/pull/57
import removeMarkdown from 'remove-markdown';

import logger from 'infra/logger';

export default function removeMarkdown(md, options = {}) {
export default function customRemoveMarkdown(md, options = {}) {
options.oneLine = Object.hasOwn(options, 'oneLine') ? options.oneLine : true;
options.listUnicodeChar = Object.hasOwn(options, 'listUnicodeChar') ? options.listUnicodeChar : false;
options.stripListLeaders = Object.hasOwn(options, 'stripListLeaders') ? options.stripListLeaders : true;
options.gfm = Object.hasOwn(options, 'gfm') ? options.gfm : true;
options.useImgAltText = Object.hasOwn(options, 'useImgAltText') ? options.useImgAltText : true;
options.abbr = Object.hasOwn(options, 'abbr') ? options.abbr : false;
options.replaceLinksWithURL = Object.hasOwn(options, 'replaceLinksWithURL') ? options.replaceLinksWithURL : false;
options.htmlTagsToSkip = Object.hasOwn(options, 'htmlTagsToSkip') ? options.htmlTagsToSkip : [];

var output = md || '';
options.throwError = Object.hasOwn(options, 'logError') ? options.throwError : true;

// Remove horizontal rules (stripListHeaders conflict with this rule, which is why it has been moved to the top)
output = output.replace(/^(-\s*?|\*\s*?|_\s*?){3,}\s*/gm, '');
let output = md || '';

try {
if (options.stripListLeaders) {
if (options.listUnicodeChar)
output = output.replace(/^([\s\t]*)([*\-+]|\d+\.)\s+/gm, options.listUnicodeChar + ' $1');
else output = output.replace(/^([\s\t]*)([*\-+]|\d+\.)\s+/gm, '$1');
}
if (options.gfm) {
output = output
// Header
.replace(/\n={2,}/g, '\n')
// Fenced codeblocks
.replace(/~{3}.*\n/g, '')
// Strikethrough
.replace(/~~/g, '')
// Fenced codeblocks
.replace(/`{3}.*\n/g, '');
}
if (options.abbr) {
// Remove abbreviations
output = output.replace(/\*\[.*\]:.*\n/, '');
}
output = output
// Remove HTML tags
.replace(/<[^>]*>/g, '');

var htmlReplaceRegex = new RegExp('<[^>]*>', 'g');
if (options.htmlTagsToSkip.length > 0) {
// Using negative lookahead. Eg. (?!sup|sub) will not match 'sup' and 'sub' tags.
var joinedHtmlTagsToSkip = '(?!' + options.htmlTagsToSkip.join('|') + ')';

// Adding the lookahead literal with the default regex for html. Eg./<(?!sup|sub)[^>]*>/ig
htmlReplaceRegex = new RegExp('<' + joinedHtmlTagsToSkip + '[^>]*>', 'ig');
}

output = output
// Remove HTML tags
.replace(htmlReplaceRegex, '')
// Remove setext-style headers
.replace(/^[=-]{2,}\s*$/g, '')
// Remove footnotes?
.replace(/\[\^.+?\](: .*?$)?/g, '')
.replace(/\s{0,2}\[.*?\]: .*?$/g, '')
// Remove images
.replace(/!\[(.*?)\][[(].*?[\])]/g, options.useImgAltText ? '$1' : '')
// Remove inline links
.replace(/\[([^\]]*?)\][[(].*?[\])]/g, options.replaceLinksWithURL ? '$2' : '$1')
// Remove blockquotes
.replace(/^(\n)?\s{0,3}>\s?/gm, '$1')
// .replace(/(^|\n)\s{0,3}>\s?/g, '\n\n')
// Remove reference-style links?
.replace(/^\s{1,2}\[(.*?)\]: (\S+)( ".*?")?\s*$/g, '')
// Remove atx-style headers
.replace(/^(\n)?\s{0,}#{1,6}\s*( (.+))? +#+$|^(\n)?\s{0,}#{1,6}\s*( (.+))?$/gm, '$1$3$4$6')
// Remove * emphasis
.replace(/([*]+)(\S)(.*?\S)??\1/g, '$2$3')
// Remove _ emphasis. Unlike *, _ emphasis gets rendered only if
// 1. Either there is a whitespace character before opening _ and after closing _.
// 2. Or _ is at the start/end of the string.
.replace(/(^|\W)([_]+)(\S)(.*?\S)??\2($|\W)/g, '$1$3$4$5')
// Remove code blocks
.replace(/(`{3,})(.*?)\1/gm, '$2')
// Remove inline code
.replace(/`(.+?)`/g, '$1')
// // Replace two or more newlines with exactly two? Not entirely sure this belongs here...
// .replace(/\n{2,}/g, '\n\n')
// // Remove newlines in a paragraph
// .replace(/(\S+)\n\s*(\S+)/g, '$1 $2')
// Replace strike through
.replace(/~(.*?)~/g, '$1');
output = removeMarkdown(md, options);

if (options.oneLine) {
output = output.replace(/\s+/g, ' ');
Expand All @@ -99,14 +23,37 @@ export default function removeMarkdown(md, options = {}) {
}

if (options.trim) {
output = output.replace(
/^[\s\p{C}\u034f\u17b4\u17b5\u2800\u115f\u1160\u3164\uffa0]+|[\s\p{C}\u034f\u17b4\u17b5\u2800\u115f\u1160\u3164\uffa0]+$|\u0000/gsu,
'',
);
output = trimStart(output);
output = trimEnd(output);
}
} catch (e) {
logger.error(e);
if (options.throwError) {
logger.error(e);
}

return md;
}
return output;
}

const whitespaceAndControlCharRegex = /[\s\p{C}\u034f\u17b4\u17b5\u2800\u115f\u1160\u3164\uffa0]/u;

export function trimStart(str) {
let i = 0;

while (i < str.length && whitespaceAndControlCharRegex.test(str[i])) {
i++;
}

return str.slice(i);
}

export function trimEnd(str) {
let i = str.length - 1;

while (i >= 0 && whitespaceAndControlCharRegex.test(str[i])) {
i--;
}

return str.slice(0, i + 1);
}
15 changes: 8 additions & 7 deletions models/validator.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ import Joi from 'joi';

import { ValidationError } from 'errors';
import webserver from 'infra/webserver';
import removeMarkdown from 'models/remove-markdown';
import removeMarkdown, { trimEnd, trimStart } from 'models/remove-markdown';
import availableFeatures from 'models/user-features';

const MAX_INTEGER = 2147483647;
Expand Down Expand Up @@ -159,8 +159,9 @@ const schemas = {
description: function () {
return Joi.object({
description: Joi.string()
.replace(/[\s\p{C}\u034f\u17b4\u17b5\u2800\u115f\u1160\u3164\uffa0]+$|\u0000/gsu, '')
.max(5000)
.replace(/\u0000/gu, '')
.custom(trimEnd)
.allow('')
.when('$required.description', { is: 'required', then: Joi.required(), otherwise: Joi.optional() }),
});
Expand Down Expand Up @@ -230,12 +231,11 @@ const schemas = {
title: function () {
return Joi.object({
title: Joi.string()
.replace(
/^[\s\p{C}\u034f\u17b4\u17b5\u2800\u115f\u1160\u3164\uffa0]+|[\s\p{C}\u034f\u17b4\u17b5\u2800\u115f\u1160\u3164\uffa0]+$|\u0000/gu,
'',
)
.min(1)
.max(255)
.replace(/\u0000/gu, '')
.custom(trimStart)
.custom(trimEnd)
.when('$required.title', { is: 'required', then: Joi.required(), otherwise: Joi.optional().allow(null) }),
});
},
Expand All @@ -244,9 +244,10 @@ const schemas = {
return Joi.object({
body: Joi.string()
.pattern(/^[\s\p{C}\u034f\u17b4\u17b5\u2800\u115f\u1160\u3164\uffa0].*$/su, { invert: true })
.replace(/[\s\p{C}\u034f\u17b4\u17b5\u2800\u115f\u1160\u3164\uffa0]+$|\u0000/gsu, '')
.min(1)
.max(20000)
.replace(/\u0000/gu, '')
.custom(trimEnd)
.custom(withoutMarkdown, 'check if is empty without markdown')
.when('$required.body', { is: 'required', then: Joi.required(), otherwise: Joi.optional() })
.messages({
Expand Down
7 changes: 7 additions & 0 deletions package-lock.json
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@
"recharts": "2.12.7",
"rehype-external-links": "3.0.0",
"rehype-slug": "6.0.0",
"remove-markdown": "0.6.0",
"resend": "4.0.1",
"satori": "0.10.13",
"slug": "9.1.0",
Expand Down
56 changes: 56 additions & 0 deletions tests/integration/api/v1/contents/post.test.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1867,6 +1867,62 @@ describe('POST /api/v1/contents', () => {
expect(uuidVersion(responseBody.request_id)).toBe(4);
});

test('Content with "title" containing 100.000 invalid characters', async () => {
const contentsRequestBuilder = new RequestBuilder('/api/v1/contents');
await contentsRequestBuilder.buildUser();

const { response, responseBody } = await contentsRequestBuilder.post({
title: '!' + '\uffa0'.repeat(100_000) + '\uffa0!',
body: 'With invalid title',
slug: 'nodejs',
});

expect.soft(response.status).toBe(400);

expect(responseBody).toStrictEqual({
status_code: 400,
name: 'ValidationError',
message: '"title" deve conter no máximo 255 caracteres.',
action: 'Ajuste os dados enviados e tente novamente.',
error_location_code: 'MODEL:VALIDATOR:FINAL_SCHEMA',
key: 'title',
type: 'string.max',
error_id: responseBody.error_id,
request_id: responseBody.request_id,
});

expect(uuidVersion(responseBody.error_id)).toBe(4);
expect(uuidVersion(responseBody.request_id)).toBe(4);
});

test('Content with "body" containing 100.000 invalid characters', async () => {
const contentsRequestBuilder = new RequestBuilder('/api/v1/contents');
await contentsRequestBuilder.buildUser();

const { response, responseBody } = await contentsRequestBuilder.post({
title: 'With invalid body',
body: '!' + '\u034f'.repeat(100_000) + '\u034f!',
slug: 'nodejs',
});

expect.soft(response.status).toBe(400);

expect(responseBody).toStrictEqual({
status_code: 400,
name: 'ValidationError',
message: '"body" deve conter no máximo 20000 caracteres.',
action: 'Ajuste os dados enviados e tente novamente.',
error_location_code: 'MODEL:VALIDATOR:FINAL_SCHEMA',
key: 'body',
type: 'string.max',
error_id: responseBody.error_id,
request_id: responseBody.request_id,
});

expect(uuidVersion(responseBody.error_id)).toBe(4);
expect(uuidVersion(responseBody.request_id)).toBe(4);
});

describe('Notifications', () => {
test('Create "root" content', async () => {
await orchestrator.deleteAllEmails();
Expand Down
29 changes: 29 additions & 0 deletions tests/integration/api/v1/users/[username]/patch.test.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -912,6 +912,35 @@ describe('PATCH /api/v1/users/[username]', () => {
expect(responseBody.error_location_code).toBe('MODEL:VALIDATOR:FINAL_SCHEMA');
});

test('Patching itself with a "description" containing 100.000 invalid characters', async () => {
const defaultUser = await orchestrator.createUser();
await orchestrator.activateUser(defaultUser);
const defaultUserSession = await orchestrator.createSession(defaultUser);

const response = await fetch(`${orchestrator.webserverUrl}/api/v1/users/${defaultUser.username}`, {
method: 'PATCH',
headers: {
'Content-Type': 'application/json',
cookie: `session_id=${defaultUserSession.token}`,
},

body: JSON.stringify({
description: '!' + '\u17b4'.repeat(100_000) + '\u17b4!',
}),
});

const responseBody = await response.json();
expect.soft(response.status).toBe(400);
expect.soft(responseBody.status_code).toBe(400);
expect(responseBody.name).toBe('ValidationError');
expect(responseBody.message).toBe('"description" deve conter no máximo 5000 caracteres.');
expect(responseBody.action).toBe('Ajuste os dados enviados e tente novamente.');
expect(responseBody.type).toBe('string.max');
expect(uuidVersion(responseBody.error_id)).toBe(4);
expect(uuidVersion(responseBody.request_id)).toBe(4);
expect(responseBody.error_location_code).toBe('MODEL:VALIDATOR:FINAL_SCHEMA');
});

test('Patching itself with a "description" containing value null', async () => {
const defaultUser = await orchestrator.createUser();
await orchestrator.activateUser(defaultUser);
Expand Down

0 comments on commit fa73f50

Please sign in to comment.