From 71e052ab4da3536bf8048f9be3c28f37ffd8f117 Mon Sep 17 00:00:00 2001
From: "Ibrahima G. Coulibaly" <ibracool99@gmail.com>
Date: Sun, 2 Mar 2025 02:58:50 +0000
Subject: [PATCH] feat: remove duplicate lines

---
 .codebuddy/.gitignore                         |   1 +
 .codebuddy/summary.md                         |  39 +++
 .idea/workspace.xml                           | 206 ++++++++------
 scripts/create-tool.mjs                       |   4 +-
 src/components/Hero.tsx                       |  12 +-
 src/pages/tools/string/index.ts               |   2 +
 .../string/remove-duplicate-lines/index.tsx   | 260 ++++++++++++++++++
 .../string/remove-duplicate-lines/meta.ts     |  13 +
 .../remove-duplicate-lines.service.test.ts    | 200 ++++++++++++++
 .../string/remove-duplicate-lines/service.ts  |  88 ++++++
 src/tools/defineTool.tsx                      |   4 +-
 11 files changed, 731 insertions(+), 98 deletions(-)
 create mode 100644 .codebuddy/.gitignore
 create mode 100644 .codebuddy/summary.md
 create mode 100644 src/pages/tools/string/remove-duplicate-lines/index.tsx
 create mode 100644 src/pages/tools/string/remove-duplicate-lines/meta.ts
 create mode 100644 src/pages/tools/string/remove-duplicate-lines/remove-duplicate-lines.service.test.ts
 create mode 100644 src/pages/tools/string/remove-duplicate-lines/service.ts

diff --git a/.codebuddy/.gitignore b/.codebuddy/.gitignore
new file mode 100644
index 0000000..9f4c740
--- /dev/null
+++ b/.codebuddy/.gitignore
@@ -0,0 +1 @@
+db/
\ No newline at end of file
diff --git a/.codebuddy/summary.md b/.codebuddy/summary.md
new file mode 100644
index 0000000..d8d7f32
--- /dev/null
+++ b/.codebuddy/summary.md
@@ -0,0 +1,39 @@
+# Project Summary
+
+## Overview of Technologies Used
+This project is primarily built using the following technologies:
+- **Languages**: TypeScript, JavaScript, HTML, CSS
+- **Frameworks**:
+  - React (for building user interfaces)
+  - Playwright (for end-to-end testing)
+- **Main Libraries**:
+  - Tailwind CSS (for styling)
+  - MUI (Material-UI for components)
+  - pnpm (for package management)
+
+## Purpose of the Project
+The project appears to be a web application that provides various tools for image, JSON, list, number, and string manipulations. It is designed to offer users functionalities such as converting image formats, generating random numbers, and manipulating strings. The structure indicates a focus on modular components, making it easy to extend or modify specific tools without affecting the entire application.
+
+## Build and Configuration Files
+The following files are relevant for the configuration and building of the project:
+- `Dockerfile`: `/Dockerfile`
+- `package.json`: `/package.json`
+- `pnpm-lock.yaml`: `/pnpm-lock.yaml`
+- `playwright.config.ts`: `/playwright.config.ts`
+- `postcss.config.mjs`: `/postcss.config.mjs`
+- `tailwind.config.mjs`: `/tailwind.config.mjs`
+- `tsconfig.json`: `/tsconfig.json`
+- `vite.config.ts`: `/vite.config.ts`
+- `commitlint.config.js`: `/commitlint.config.js`
+
+## Source Files Directory
+The source files can be found in the following directory:
+- `/src`
+
+## Documentation Files Location
+Documentation files are located in the root directory:
+- `README.md`: `/README.md`
+- `LICENSE`: `/LICENSE`
+- `CODEOWNERS`: `/CODEOWNERS`
+
+This summary encapsulates the key aspects of the project, including its technological stack, purpose, file structure, and documentation locations.
\ No newline at end of file
diff --git a/.idea/workspace.xml b/.idea/workspace.xml
index 409761d..3111c4b 100644
--- a/.idea/workspace.xml
+++ b/.idea/workspace.xml
@@ -4,11 +4,18 @@
     <option name="autoReloadType" value="SELECTIVE" />
   </component>
   <component name="ChangeListManager">
-    <list default="true" id="b30e2810-c4c1-4aad-b134-794e52cc1c7d" name="Changes" comment="chore: img">
-      <change beforePath="$PROJECT_DIR$/.gitignore" beforeDir="false" afterPath="$PROJECT_DIR$/.gitignore" afterDir="false" />
+    <list default="true" id="b30e2810-c4c1-4aad-b134-794e52cc1c7d" name="Changes" comment="docs: readme">
+      <change afterPath="$PROJECT_DIR$/.codebuddy/.gitignore" afterDir="false" />
+      <change afterPath="$PROJECT_DIR$/.codebuddy/summary.md" afterDir="false" />
+      <change afterPath="$PROJECT_DIR$/src/pages/tools/string/remove-duplicate-lines/index.tsx" afterDir="false" />
+      <change afterPath="$PROJECT_DIR$/src/pages/tools/string/remove-duplicate-lines/meta.ts" afterDir="false" />
+      <change afterPath="$PROJECT_DIR$/src/pages/tools/string/remove-duplicate-lines/remove-duplicate-lines.service.test.ts" afterDir="false" />
+      <change afterPath="$PROJECT_DIR$/src/pages/tools/string/remove-duplicate-lines/service.ts" afterDir="false" />
       <change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/README.md" beforeDir="false" afterPath="$PROJECT_DIR$/README.md" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/src/pages/tools/image/png/index.ts" beforeDir="false" afterPath="$PROJECT_DIR$/src/pages/tools/image/png/index.ts" afterDir="false" />
+      <change beforePath="$PROJECT_DIR$/scripts/create-tool.mjs" beforeDir="false" afterPath="$PROJECT_DIR$/scripts/create-tool.mjs" afterDir="false" />
+      <change beforePath="$PROJECT_DIR$/src/components/Hero.tsx" beforeDir="false" afterPath="$PROJECT_DIR$/src/components/Hero.tsx" afterDir="false" />
+      <change beforePath="$PROJECT_DIR$/src/pages/tools/string/index.ts" beforeDir="false" afterPath="$PROJECT_DIR$/src/pages/tools/string/index.ts" afterDir="false" />
+      <change beforePath="$PROJECT_DIR$/src/tools/defineTool.tsx" beforeDir="false" afterPath="$PROJECT_DIR$/src/tools/defineTool.tsx" afterDir="false" />
     </list>
     <option name="SHOW_DIALOG" value="false" />
     <option name="HIGHLIGHT_CONFLICTS" value="true" />
@@ -60,48 +67,51 @@
     <option name="hideEmptyMiddlePackages" value="true" />
     <option name="showLibraryContents" value="true" />
   </component>
-  <component name="PropertiesComponent">{
-  &quot;keyToString&quot;: {
-    &quot;ASKED_ADD_EXTERNAL_FILES&quot;: &quot;true&quot;,
-    &quot;ASKED_SHARE_PROJECT_CONFIGURATION_FILES&quot;: &quot;true&quot;,
-    &quot;Docker.Dockerfile build.executor&quot;: &quot;Run&quot;,
-    &quot;Docker.Dockerfile.executor&quot;: &quot;Run&quot;,
-    &quot;Playwright.JoinText Component.executor&quot;: &quot;Run&quot;,
-    &quot;Playwright.JoinText Component.should merge text pieces with specified join character.executor&quot;: &quot;Run&quot;,
-    &quot;RunOnceActivity.OpenProjectViewOnStart&quot;: &quot;true&quot;,
-    &quot;RunOnceActivity.ShowReadmeOnStart&quot;: &quot;true&quot;,
-    &quot;RunOnceActivity.git.unshallow&quot;: &quot;true&quot;,
-    &quot;Vitest.compute function (1).executor&quot;: &quot;Run&quot;,
-    &quot;Vitest.compute function.executor&quot;: &quot;Run&quot;,
-    &quot;Vitest.mergeText.executor&quot;: &quot;Run&quot;,
-    &quot;Vitest.mergeText.should merge lines and preserve blank lines when deleteBlankLines is false.executor&quot;: &quot;Run&quot;,
-    &quot;Vitest.mergeText.should merge lines, preserve blank lines and trailing spaces when both deleteBlankLines and deleteTrailingSpaces are false.executor&quot;: &quot;Run&quot;,
-    &quot;git-widget-placeholder&quot;: &quot;main&quot;,
-    &quot;ignore.virus.scanning.warn.message&quot;: &quot;true&quot;,
-    &quot;kotlin-language-version-configured&quot;: &quot;true&quot;,
-    &quot;last_opened_file_path&quot;: &quot;C:/Users/Ibrahima/IdeaProjects/omni-tools/public/assets&quot;,
-    &quot;node.js.detected.package.eslint&quot;: &quot;true&quot;,
-    &quot;node.js.detected.package.tslint&quot;: &quot;true&quot;,
-    &quot;node.js.selected.package.eslint&quot;: &quot;(autodetect)&quot;,
-    &quot;node.js.selected.package.tslint&quot;: &quot;(autodetect)&quot;,
-    &quot;nodejs_package_manager_path&quot;: &quot;npm&quot;,
-    &quot;npm.build.executor&quot;: &quot;Run&quot;,
-    &quot;npm.dev.executor&quot;: &quot;Run&quot;,
-    &quot;npm.lint.executor&quot;: &quot;Run&quot;,
-    &quot;npm.prebuild.executor&quot;: &quot;Run&quot;,
-    &quot;npm.script:create:tool.executor&quot;: &quot;Run&quot;,
-    &quot;npm.test.executor&quot;: &quot;Run&quot;,
-    &quot;npm.test:e2e.executor&quot;: &quot;Run&quot;,
-    &quot;npm.test:e2e:run.executor&quot;: &quot;Run&quot;,
-    &quot;prettierjs.PrettierConfiguration.Package&quot;: &quot;C:\\Users\\Ibrahima\\IdeaProjects\\omni-tools\\node_modules\\prettier&quot;,
-    &quot;project.structure.last.edited&quot;: &quot;Problems&quot;,
-    &quot;project.structure.proportion&quot;: &quot;0.0&quot;,
-    &quot;project.structure.side.proportion&quot;: &quot;0.2&quot;,
-    &quot;settings.editor.selected.configurable&quot;: &quot;settings.typescriptcompiler&quot;,
-    &quot;ts.external.directory.path&quot;: &quot;C:\\Users\\Ibrahima\\IdeaProjects\\omni-tools\\node_modules\\typescript\\lib&quot;,
-    &quot;vue.rearranger.settings.migration&quot;: &quot;true&quot;
+  <component name="PropertiesComponent"><![CDATA[{
+  "keyToString": {
+    "ASKED_ADD_EXTERNAL_FILES": "true",
+    "ASKED_SHARE_PROJECT_CONFIGURATION_FILES": "true",
+    "Docker.Dockerfile build.executor": "Run",
+    "Docker.Dockerfile.executor": "Run",
+    "Playwright.JoinText Component.executor": "Run",
+    "Playwright.JoinText Component.should merge text pieces with specified join character.executor": "Run",
+    "RunOnceActivity.OpenProjectViewOnStart": "true",
+    "RunOnceActivity.ShowReadmeOnStart": "true",
+    "RunOnceActivity.git.unshallow": "true",
+    "Vitest.compute function (1).executor": "Run",
+    "Vitest.compute function.executor": "Run",
+    "Vitest.mergeText.executor": "Run",
+    "Vitest.mergeText.should merge lines and preserve blank lines when deleteBlankLines is false.executor": "Run",
+    "Vitest.mergeText.should merge lines, preserve blank lines and trailing spaces when both deleteBlankLines and deleteTrailingSpaces are false.executor": "Run",
+    "Vitest.removeDuplicateLines function.executor": "Run",
+    "Vitest.removeDuplicateLines function.newlines option.executor": "Run",
+    "Vitest.removeDuplicateLines function.newlines option.should filter newlines when newlines is set to filter.executor": "Run",
+    "git-widget-placeholder": "main",
+    "ignore.virus.scanning.warn.message": "true",
+    "kotlin-language-version-configured": "true",
+    "last_opened_file_path": "C:/Users/Ibrahima/IdeaProjects/omni-tools/public/assets",
+    "node.js.detected.package.eslint": "true",
+    "node.js.detected.package.tslint": "true",
+    "node.js.selected.package.eslint": "(autodetect)",
+    "node.js.selected.package.tslint": "(autodetect)",
+    "nodejs_package_manager_path": "npm",
+    "npm.build.executor": "Run",
+    "npm.dev.executor": "Run",
+    "npm.lint.executor": "Run",
+    "npm.prebuild.executor": "Run",
+    "npm.script:create:tool.executor": "Run",
+    "npm.test.executor": "Run",
+    "npm.test:e2e.executor": "Run",
+    "npm.test:e2e:run.executor": "Run",
+    "prettierjs.PrettierConfiguration.Package": "C:\\Users\\Ibrahima\\IdeaProjects\\omni-tools\\node_modules\\prettier",
+    "project.structure.last.edited": "Problems",
+    "project.structure.proportion": "0.0",
+    "project.structure.side.proportion": "0.2",
+    "settings.editor.selected.configurable": "settings.typescriptcompiler",
+    "ts.external.directory.path": "C:\\Users\\Ibrahima\\IdeaProjects\\omni-tools\\node_modules\\typescript\\lib",
+    "vue.rearranger.settings.migration": "true"
   }
-}</component>
+}]]></component>
   <component name="ReactDesignerToolWindowState">
     <option name="myId2Visible">
       <map>
@@ -127,13 +137,47 @@
       <recent name="C:\Users\HP\IdeaProjects\omni-tools\src\components\options" />
     </key>
   </component>
-  <component name="RunManager" selected="npm.build">
-    <configuration name="Dockerfile" type="docker-deploy" factoryName="dockerfile" temporary="true" server-name="Docker">
-      <deployment type="dockerfile">
-        <settings>
-          <option name="sourceFilePath" value="Dockerfile" />
-        </settings>
-      </deployment>
+  <component name="RunManager" selected="Vitest.removeDuplicateLines function">
+    <configuration name="removeDuplicateLines function" type="JavaScriptTestRunnerVitest" temporary="true" nameIsGenerated="true">
+      <node-interpreter value="project" />
+      <vitest-package value="$PROJECT_DIR$/node_modules/vitest" />
+      <working-dir value="$PROJECT_DIR$" />
+      <vitest-options value="--run" />
+      <envs />
+      <scope-kind value="SUITE" />
+      <test-file value="$PROJECT_DIR$/src/pages/tools/string/remove-duplicate-lines/remove-duplicate-lines.service.test.ts" />
+      <test-names>
+        <test-name value="removeDuplicateLines function" />
+      </test-names>
+      <method v="2" />
+    </configuration>
+    <configuration name="removeDuplicateLines function.newlines option" type="JavaScriptTestRunnerVitest" temporary="true" nameIsGenerated="true">
+      <node-interpreter value="project" />
+      <vitest-package value="$PROJECT_DIR$/node_modules/vitest" />
+      <working-dir value="$PROJECT_DIR$" />
+      <vitest-options value="--run" />
+      <envs />
+      <scope-kind value="SUITE" />
+      <test-file value="$PROJECT_DIR$/src/pages/tools/string/remove-duplicate-lines/remove-duplicate-lines.service.test.ts" />
+      <test-names>
+        <test-name value="removeDuplicateLines function" />
+        <test-name value="newlines option" />
+      </test-names>
+      <method v="2" />
+    </configuration>
+    <configuration name="removeDuplicateLines function.newlines option.should filter newlines when newlines is set to filter" type="JavaScriptTestRunnerVitest" temporary="true" nameIsGenerated="true">
+      <node-interpreter value="project" />
+      <vitest-package value="$PROJECT_DIR$/node_modules/vitest" />
+      <working-dir value="$PROJECT_DIR$" />
+      <vitest-options value="--run" />
+      <envs />
+      <scope-kind value="TEST" />
+      <test-file value="$PROJECT_DIR$/src/pages/tools/string/remove-duplicate-lines/remove-duplicate-lines.service.test.ts" />
+      <test-names>
+        <test-name value="removeDuplicateLines function" />
+        <test-name value="newlines option" />
+        <test-name value="should filter newlines when newlines is set to filter" />
+      </test-names>
       <method v="2" />
     </configuration>
     <configuration default="true" type="docker-deploy" factoryName="dockerfile" temporary="true">
@@ -162,40 +206,20 @@
       <envs />
       <method v="2" />
     </configuration>
-    <configuration name="lint" type="js.build_tools.npm" temporary="true" nameIsGenerated="true">
-      <package-json value="$PROJECT_DIR$/package.json" />
-      <command value="run" />
-      <scripts>
-        <script value="lint" />
-      </scripts>
-      <node-interpreter value="project" />
-      <envs />
-      <method v="2" />
-    </configuration>
-    <configuration name="test" type="js.build_tools.npm" temporary="true" nameIsGenerated="true">
-      <package-json value="$PROJECT_DIR$/package.json" />
-      <command value="run" />
-      <scripts>
-        <script value="test" />
-      </scripts>
-      <node-interpreter value="project" />
-      <envs />
-      <method v="2" />
-    </configuration>
     <list>
-      <item itemvalue="Docker.Dockerfile" />
       <item itemvalue="npm.build" />
-      <item itemvalue="npm.test" />
       <item itemvalue="npm.dev" />
-      <item itemvalue="npm.lint" />
+      <item itemvalue="Vitest.removeDuplicateLines function" />
+      <item itemvalue="Vitest.removeDuplicateLines function.newlines option" />
+      <item itemvalue="Vitest.removeDuplicateLines function.newlines option.should filter newlines when newlines is set to filter" />
     </list>
     <recent_temporary>
       <list>
-        <item itemvalue="npm.build" />
+        <item itemvalue="Vitest.removeDuplicateLines function" />
+        <item itemvalue="Vitest.removeDuplicateLines function.newlines option" />
+        <item itemvalue="Vitest.removeDuplicateLines function.newlines option.should filter newlines when newlines is set to filter" />
         <item itemvalue="npm.dev" />
-        <item itemvalue="Docker.Dockerfile" />
-        <item itemvalue="npm.test" />
-        <item itemvalue="npm.lint" />
+        <item itemvalue="npm.build" />
       </list>
     </recent_temporary>
   </component>
@@ -268,14 +292,8 @@
       <workItem from="1740670449847" duration="4776000" />
       <workItem from="1740702343843" duration="657000" />
       <workItem from="1740788381920" duration="465000" />
-    </task>
-    <task id="LOCAL-00088" summary="feat: sort list">
-      <option name="closed" value="true" />
-      <created>1720545582958</created>
-      <option name="number" value="00088" />
-      <option name="presentableId" value="LOCAL-00088" />
-      <option name="project" value="LOCAL" />
-      <updated>1720545582958</updated>
+      <workItem from="1740788856134" duration="659000" />
+      <workItem from="1740880919391" duration="3339000" />
     </task>
     <task id="LOCAL-00089" summary="feat: find most popular ui">
       <option name="closed" value="true" />
@@ -661,7 +679,15 @@
       <option name="project" value="LOCAL" />
       <updated>1740680778110</updated>
     </task>
-    <option name="localTasksCounter" value="137" />
+    <task id="LOCAL-00137" summary="docs: readme">
+      <option name="closed" value="true" />
+      <created>1740788899030</created>
+      <option name="number" value="00137" />
+      <option name="presentableId" value="LOCAL-00137" />
+      <option name="project" value="LOCAL" />
+      <updated>1740788899030</updated>
+    </task>
+    <option name="localTasksCounter" value="138" />
     <servers />
   </component>
   <component name="TypeScriptGeneratedFilesManager">
@@ -733,7 +759,6 @@
     <MESSAGE value="style: background svg" />
     <MESSAGE value="docs: img" />
     <MESSAGE value="fix: bg" />
-    <MESSAGE value="docs: readme" />
     <MESSAGE value="chore: handle enter press on search" />
     <MESSAGE value="chore: show tooloptions in example" />
     <MESSAGE value="refact: examples" />
@@ -745,7 +770,8 @@
     <MESSAGE value="chore: prettify json in home" />
     <MESSAGE value="feat: jakarta font" />
     <MESSAGE value="chore: img" />
-    <option name="LAST_COMMIT_MESSAGE" value="chore: img" />
+    <MESSAGE value="docs: readme" />
+    <option name="LAST_COMMIT_MESSAGE" value="docs: readme" />
   </component>
   <component name="XSLT-Support.FileAssociations.UIState">
     <expand />
diff --git a/scripts/create-tool.mjs b/scripts/create-tool.mjs
index 2338683..03628db 100644
--- a/scripts/create-tool.mjs
+++ b/scripts/create-tool.mjs
@@ -78,8 +78,8 @@ import { Box } from '@mui/material';
 import React from 'react';
 import * as Yup from 'yup';
 
-const initialValues = {};
-type InitialValuesType = typeof initialValues;
+type InitialValuesType = {};
+const initialValues: InitialValuesType = {};
 const validationSchema = Yup.object({
   // splitSeparator: Yup.string().required('The separator is required')
 });
diff --git a/src/components/Hero.tsx b/src/components/Hero.tsx
index 0b4c0ce..59fbd13 100644
--- a/src/components/Hero.tsx
+++ b/src/components/Hero.tsx
@@ -7,6 +7,7 @@ import { DefinedTool } from '@tools/defineTool';
 import { filterTools, tools } from '@tools/index';
 import { useNavigate } from 'react-router-dom';
 import _ from 'lodash';
+import { Icon } from '@iconify/react';
 
 const exampleTools: { label: string; url: string }[] = [
   {
@@ -97,10 +98,13 @@ export default function Hero() {
             {...props}
             onClick={() => navigate('/' + option.path)}
           >
-            <Box>
-              <Typography fontWeight={'bold'}>{option.name}</Typography>
-              <Typography fontSize={12}>{option.shortDescription}</Typography>
-            </Box>
+            <Stack direction={'row'} spacing={2} alignItems={'center'}>
+              <Icon fontSize={20} icon={option.icon} />
+              <Box>
+                <Typography fontWeight={'bold'}>{option.name}</Typography>
+                <Typography fontSize={12}>{option.shortDescription}</Typography>
+              </Box>
+            </Stack>
           </Box>
         )}
         onKeyDown={(event) => {
diff --git a/src/pages/tools/string/index.ts b/src/pages/tools/string/index.ts
index 39f8d1f..b9e5b0f 100644
--- a/src/pages/tools/string/index.ts
+++ b/src/pages/tools/string/index.ts
@@ -1,3 +1,4 @@
+import { tool as stringRemoveDuplicateLines } from './remove-duplicate-lines/meta';
 import { tool as stringReverse } from './reverse/meta';
 import { tool as stringRandomizeCase } from './randomize-case/meta';
 import { tool as stringUppercase } from './uppercase/meta';
@@ -11,6 +12,7 @@ import { tool as stringJoin } from './join/meta';
 export const stringTools = [
   stringSplit,
   stringJoin,
+  stringRemoveDuplicateLines,
   stringToMorse
   // stringReverse,
   // stringRandomizeCase,
diff --git a/src/pages/tools/string/remove-duplicate-lines/index.tsx b/src/pages/tools/string/remove-duplicate-lines/index.tsx
new file mode 100644
index 0000000..06a6caf
--- /dev/null
+++ b/src/pages/tools/string/remove-duplicate-lines/index.tsx
@@ -0,0 +1,260 @@
+import { Box } from '@mui/material';
+import React, { useRef, useState } from 'react';
+import ToolTextInput from '@components/input/ToolTextInput';
+import ToolTextResult from '@components/result/ToolTextResult';
+import ToolOptions, { GetGroupsType } from '@components/options/ToolOptions';
+import SimpleRadio from '@components/options/SimpleRadio';
+import CheckboxWithDesc from '@components/options/CheckboxWithDesc';
+import ToolInputAndResult from '@components/ToolInputAndResult';
+import ToolExamples, {
+  CardExampleType
+} from '@components/examples/ToolExamples';
+import { ToolComponentProps } from '@tools/defineTool';
+import { FormikProps } from 'formik';
+import removeDuplicateLines, {
+  DuplicateRemovalMode,
+  DuplicateRemoverOptions,
+  NewlineOption
+} from './service';
+
+// Initial values for our form
+const initialValues: DuplicateRemoverOptions = {
+  mode: 'all',
+  newlines: 'filter',
+  sortLines: false,
+  trimTextLines: false
+};
+
+// Operation mode options
+const operationModes = [
+  {
+    title: 'Remove All Duplicate Lines',
+    description:
+      'If this option is selected, then all repeated lines across entire text are removed, starting from the second occurrence.',
+    value: 'all' as DuplicateRemovalMode
+  },
+  {
+    title: 'Remove Consecutive Duplicate Lines',
+    description:
+      'If this option is selected, then only consecutive repeated lines are removed.',
+    value: 'consecutive' as DuplicateRemovalMode
+  },
+  {
+    title: 'Leave Absolutely Unique Text Lines',
+    description:
+      'If this option is selected, then all lines that appear more than once are removed.',
+    value: 'unique' as DuplicateRemovalMode
+  }
+];
+
+// Newlines options
+const newlineOptions = [
+  {
+    title: 'Preserve All Newlines',
+    description: 'Leave all empty lines in the output.',
+    value: 'preserve' as NewlineOption
+  },
+  {
+    title: 'Filter All Newlines',
+    description: 'Process newlines as regular lines.',
+    value: 'filter' as NewlineOption
+  },
+  {
+    title: 'Delete All Newlines',
+    description: 'Before filtering uniques, remove all newlines.',
+    value: 'delete' as NewlineOption
+  }
+];
+
+// Example cards for demonstration
+const exampleCards: CardExampleType<typeof initialValues>[] = [
+  {
+    title: 'Remove Duplicate Items from List',
+    description:
+      'Removes duplicate items from a shopping list, keeping only the first occurrence of each item.',
+    sampleText: `Apples
+Bananas
+Milk
+Eggs
+Bread
+Milk
+Cheese
+Apples
+Yogurt`,
+    sampleResult: `Apples
+Bananas
+Milk
+Eggs
+Bread
+Cheese
+Yogurt`,
+    sampleOptions: {
+      ...initialValues,
+      mode: 'all',
+      newlines: 'filter',
+      sortLines: false,
+      trimTextLines: false
+    }
+  },
+  {
+    title: 'Clean Consecutive Duplicates',
+    description:
+      'Removes consecutive duplicates from log entries, which often happen when a system repeatedly logs the same error.',
+    sampleText: `[INFO] Application started
+[ERROR] Connection failed
+[ERROR] Connection failed
+[ERROR] Connection failed
+[INFO] Retrying connection
+[ERROR] Authentication error
+[ERROR] Authentication error
+[INFO] Connection established`,
+    sampleResult: `[INFO] Application started
+[ERROR] Connection failed
+[INFO] Retrying connection
+[ERROR] Authentication error
+[INFO] Connection established`,
+    sampleOptions: {
+      ...initialValues,
+      mode: 'consecutive',
+      newlines: 'filter',
+      sortLines: false,
+      trimTextLines: false
+    }
+  },
+  {
+    title: 'Extract Unique Entries Only',
+    description:
+      'Filters a list to keep only entries that appear exactly once, removing any duplicated items entirely.',
+    sampleText: `Red
+Blue
+Green
+Blue
+Yellow
+Purple
+Red
+Orange`,
+    sampleResult: `Green
+Yellow
+Purple
+Orange`,
+    sampleOptions: {
+      ...initialValues,
+      mode: 'unique',
+      newlines: 'filter',
+      sortLines: false,
+      trimTextLines: false
+    }
+  },
+  {
+    title: 'Sort and Clean Data',
+    description:
+      'Removes duplicate items from a list, trims whitespace, and sorts the results alphabetically.',
+    sampleText: `  Apple
+Banana
+ Cherry
+Apple
+   Banana
+Dragonfruit
+ Elderberry `,
+    sampleResult: `Apple
+Banana
+Cherry
+Dragonfruit
+Elderberry`,
+    sampleOptions: {
+      ...initialValues,
+      mode: 'all',
+      newlines: 'filter',
+      sortLines: true,
+      trimTextLines: true
+    }
+  }
+];
+
+export default function RemoveDuplicateLines({ title }: ToolComponentProps) {
+  const [input, setInput] = useState<string>('');
+  const [result, setResult] = useState<string>('');
+  const formRef = useRef<FormikProps<typeof initialValues>>(null);
+
+  const computeExternal = (
+    optionsValues: typeof initialValues,
+    inputText: string
+  ) => {
+    setResult(removeDuplicateLines(inputText, optionsValues));
+  };
+
+  const getGroups: GetGroupsType<typeof initialValues> = ({
+    values,
+    updateField
+  }) => [
+    {
+      title: 'Operation Mode',
+      component: operationModes.map(({ title, description, value }) => (
+        <SimpleRadio
+          key={value}
+          checked={value === values.mode}
+          title={title}
+          description={description}
+          onClick={() => updateField('mode', value)}
+        />
+      ))
+    },
+    {
+      title: 'Newlines, Tabs and Spaces',
+      component: [
+        ...newlineOptions.map(({ title, description, value }) => (
+          <SimpleRadio
+            key={value}
+            checked={value === values.newlines}
+            title={title}
+            description={description}
+            onClick={() => updateField('newlines', value)}
+          />
+        )),
+        <CheckboxWithDesc
+          key="trimTextLines"
+          checked={values.trimTextLines}
+          title="Trim Text Lines"
+          description="Before filtering uniques, remove tabs and spaces from the beginning and end of all lines."
+          onChange={(checked) => updateField('trimTextLines', checked)}
+        />
+      ]
+    },
+    {
+      title: 'Sort Lines',
+      component: [
+        <CheckboxWithDesc
+          key="sortLines"
+          checked={values.sortLines}
+          title="Sort the Output Lines"
+          description="After removing the duplicates, sort the unique lines."
+          onChange={(checked) => updateField('sortLines', checked)}
+        />
+      ]
+    }
+  ];
+
+  return (
+    <Box>
+      <ToolInputAndResult
+        input={<ToolTextInput value={input} onChange={setInput} />}
+        result={
+          <ToolTextResult title={'Text without duplicates'} value={result} />
+        }
+      />
+      <ToolOptions
+        compute={computeExternal}
+        getGroups={getGroups}
+        initialValues={initialValues}
+        input={input}
+      />
+      <ToolExamples
+        title={title}
+        exampleCards={exampleCards}
+        getGroups={getGroups}
+        formRef={formRef}
+        setInput={setInput}
+      />
+    </Box>
+  );
+}
diff --git a/src/pages/tools/string/remove-duplicate-lines/meta.ts b/src/pages/tools/string/remove-duplicate-lines/meta.ts
new file mode 100644
index 0000000..a1a6a8b
--- /dev/null
+++ b/src/pages/tools/string/remove-duplicate-lines/meta.ts
@@ -0,0 +1,13 @@
+import { defineTool } from '@tools/defineTool';
+import { lazy } from 'react';
+
+export const tool = defineTool('string', {
+  name: 'Remove duplicate lines',
+  path: 'remove-duplicate-lines',
+  icon: 'pepicons-print:duplicate-off',
+  description:
+    "Load your text in the input form on the left and you'll instantly get text with no duplicate lines in the output area. Powerful, free, and fast. Load text lines – get unique text lines",
+  shortDescription: 'Quickly delete all repeated lines from text',
+  keywords: ['remove', 'duplicate', 'lines'],
+  component: lazy(() => import('./index'))
+});
diff --git a/src/pages/tools/string/remove-duplicate-lines/remove-duplicate-lines.service.test.ts b/src/pages/tools/string/remove-duplicate-lines/remove-duplicate-lines.service.test.ts
new file mode 100644
index 0000000..9d8769a
--- /dev/null
+++ b/src/pages/tools/string/remove-duplicate-lines/remove-duplicate-lines.service.test.ts
@@ -0,0 +1,200 @@
+import { describe, expect, it } from 'vitest';
+import removeDuplicateLines, { DuplicateRemoverOptions } from './service';
+
+describe('removeDuplicateLines function', () => {
+  // Test for 'all' duplicate removal mode
+  describe('mode: all', () => {
+    it('should remove all duplicates keeping first occurrence', () => {
+      const input = 'line1\nline2\nline1\nline3\nline2';
+      const options: DuplicateRemoverOptions = {
+        mode: 'all',
+        newlines: 'filter',
+        sortLines: false,
+        trimTextLines: false
+      };
+      const result = removeDuplicateLines(input, options);
+      expect(result).toBe('line1\nline2\nline3');
+    });
+
+    it('should handle case-sensitive duplicates correctly', () => {
+      const input = 'Line1\nline1\nLine2\nline2';
+      const options: DuplicateRemoverOptions = {
+        mode: 'all',
+        newlines: 'filter',
+        sortLines: false,
+        trimTextLines: false
+      };
+      const result = removeDuplicateLines(input, options);
+      expect(result).toBe('Line1\nline1\nLine2\nline2');
+    });
+  });
+
+  // Test for 'consecutive' duplicate removal mode
+  describe('mode: consecutive', () => {
+    it('should remove only consecutive duplicates', () => {
+      const input = 'line1\nline1\nline2\nline3\nline3\nline1';
+      const options: DuplicateRemoverOptions = {
+        mode: 'consecutive',
+        newlines: 'filter',
+        sortLines: false,
+        trimTextLines: false
+      };
+      const result = removeDuplicateLines(input, options);
+      expect(result).toBe('line1\nline2\nline3\nline1');
+    });
+  });
+
+  // Test for 'unique' duplicate removal mode
+  describe('mode: unique', () => {
+    it('should keep only lines that appear exactly once', () => {
+      const input = 'line1\nline2\nline1\nline3\nline4\nline4';
+      const options: DuplicateRemoverOptions = {
+        mode: 'unique',
+        newlines: 'filter',
+        sortLines: false,
+        trimTextLines: false
+      };
+      const result = removeDuplicateLines(input, options);
+      expect(result).toBe('line2\nline3');
+    });
+  });
+
+  // Test for newlines handling
+  describe('newlines option', () => {
+    it('should filter newlines when newlines is set to filter', () => {
+      const input = 'line1\n\nline2\n\n\nline3';
+      const options: DuplicateRemoverOptions = {
+        mode: 'all',
+        newlines: 'filter',
+        sortLines: false,
+        trimTextLines: false
+      };
+      const result = removeDuplicateLines(input, options);
+      expect(result).toBe('line1\n\nline2\nline3');
+    });
+
+    it('should delete newlines when newlines is set to delete', () => {
+      const input = 'line1\n\nline2\n\n\nline3';
+      const options: DuplicateRemoverOptions = {
+        mode: 'all',
+        newlines: 'delete',
+        sortLines: false,
+        trimTextLines: false
+      };
+      const result = removeDuplicateLines(input, options);
+      expect(result).toBe('line1\nline2\nline3');
+    });
+
+    it('should preserve newlines when newlines is set to preserve', () => {
+      const input = 'line1\n\nline2\n\nline2\nline3';
+      const options: DuplicateRemoverOptions = {
+        mode: 'all',
+        newlines: 'preserve',
+        sortLines: false,
+        trimTextLines: false
+      };
+      const result = removeDuplicateLines(input, options);
+      // This test needs careful consideration of the expected behavior
+      expect(result).not.toContain('line2\nline2');
+      expect(result).toContain('line1');
+      expect(result).toContain('line2');
+      expect(result).toContain('line3');
+    });
+  });
+
+  // Test for sorting
+  describe('sortLines option', () => {
+    it('should sort lines when sortLines is true', () => {
+      const input = 'line3\nline1\nline2';
+      const options: DuplicateRemoverOptions = {
+        mode: 'all',
+        newlines: 'filter',
+        sortLines: true,
+        trimTextLines: false
+      };
+      const result = removeDuplicateLines(input, options);
+      expect(result).toBe('line1\nline2\nline3');
+    });
+  });
+
+  // Test for trimming
+  describe('trimTextLines option', () => {
+    it('should trim lines when trimTextLines is true', () => {
+      const input = '  line1  \n  line2  \nline3';
+      const options: DuplicateRemoverOptions = {
+        mode: 'all',
+        newlines: 'filter',
+        sortLines: false,
+        trimTextLines: true
+      };
+      const result = removeDuplicateLines(input, options);
+      expect(result).toBe('line1\nline2\nline3');
+    });
+
+    it('should consider trimmed lines as duplicates', () => {
+      const input = '  line1  \nline1\n  line2\nline2  ';
+      const options: DuplicateRemoverOptions = {
+        mode: 'all',
+        newlines: 'filter',
+        sortLines: false,
+        trimTextLines: true
+      };
+      const result = removeDuplicateLines(input, options);
+      expect(result).toBe('line1\nline2');
+    });
+  });
+
+  // Combined scenarios
+  describe('combined options', () => {
+    it('should handle all options together correctly', () => {
+      const input = '  line3  \nline1\n\nline3\nline2\nline1';
+      const options: DuplicateRemoverOptions = {
+        mode: 'all',
+        newlines: 'delete',
+        sortLines: true,
+        trimTextLines: true
+      };
+      const result = removeDuplicateLines(input, options);
+      expect(result).toBe('line1\nline2\nline3');
+    });
+  });
+
+  // Edge cases
+  describe('edge cases', () => {
+    it('should handle empty input', () => {
+      const input = '';
+      const options: DuplicateRemoverOptions = {
+        mode: 'all',
+        newlines: 'filter',
+        sortLines: false,
+        trimTextLines: false
+      };
+      const result = removeDuplicateLines(input, options);
+      expect(result).toBe('');
+    });
+
+    it('should handle input with only newlines', () => {
+      const input = '\n\n\n';
+      const options: DuplicateRemoverOptions = {
+        mode: 'all',
+        newlines: 'filter',
+        sortLines: false,
+        trimTextLines: false
+      };
+      const result = removeDuplicateLines(input, options);
+      expect(result).toBe('');
+    });
+
+    it('should handle input with only whitespace', () => {
+      const input = '   \n  \n    ';
+      const options: DuplicateRemoverOptions = {
+        mode: 'all',
+        newlines: 'filter',
+        sortLines: false,
+        trimTextLines: true
+      };
+      const result = removeDuplicateLines(input, options);
+      expect(result).toBe('');
+    });
+  });
+});
diff --git a/src/pages/tools/string/remove-duplicate-lines/service.ts b/src/pages/tools/string/remove-duplicate-lines/service.ts
new file mode 100644
index 0000000..3338cc8
--- /dev/null
+++ b/src/pages/tools/string/remove-duplicate-lines/service.ts
@@ -0,0 +1,88 @@
+export type NewlineOption = 'preserve' | 'filter' | 'delete';
+export type DuplicateRemovalMode = 'all' | 'consecutive' | 'unique';
+
+export interface DuplicateRemoverOptions {
+  mode: DuplicateRemovalMode;
+  newlines: NewlineOption;
+  sortLines: boolean;
+  trimTextLines: boolean;
+}
+
+/**
+ * Removes duplicate lines from text based on specified options
+ * @param text The input text to process
+ * @param options Configuration options for text processing
+ * @returns Processed text with duplicates removed according to options
+ */
+export default function removeDuplicateLines(
+  text: string,
+  options: DuplicateRemoverOptions
+): string {
+  // Split the text into individual lines
+  let lines = text.split('\n');
+
+  // Process newlines based on option
+  if (options.newlines === 'delete') {
+    // Remove all empty lines
+    lines = lines.filter((line) => line.trim() !== '');
+  }
+
+  // Trim lines if option is selected
+  if (options.trimTextLines) {
+    lines = lines.map((line) => line.trim());
+  }
+
+  // Remove duplicates based on mode
+  let processedLines: string[] = [];
+
+  if (options.mode === 'all') {
+    // Remove all duplicates, keeping only first occurrence
+    const seen = new Set<string>();
+    processedLines = lines.filter((line) => {
+      if (seen.has(line)) {
+        return false;
+      }
+      seen.add(line);
+      return true;
+    });
+  } else if (options.mode === 'consecutive') {
+    // Remove only consecutive duplicates
+    processedLines = lines.filter((line, index, arr) => {
+      return index === 0 || line !== arr[index - 1];
+    });
+  } else if (options.mode === 'unique') {
+    // Leave only absolutely unique lines
+    const lineCount = new Map<string, number>();
+    lines.forEach((line) => {
+      lineCount.set(line, (lineCount.get(line) || 0) + 1);
+    });
+
+    processedLines = lines.filter((line) => lineCount.get(line) === 1);
+  }
+
+  // Sort lines if option is selected
+  if (options.sortLines) {
+    processedLines.sort();
+  }
+
+  // Process newlines for output
+  if (options.newlines === 'filter') {
+    // Process newlines as regular lines (already done by default)
+  } else if (options.newlines === 'preserve') {
+    // Make sure empty lines are preserved in the output
+    processedLines = text.split('\n').map((line) => {
+      if (line.trim() === '') return line;
+      return processedLines.includes(line) ? line : '';
+    });
+  }
+
+  return processedLines.join('\n');
+}
+
+// Example usage:
+// const result = removeDuplicateLines(inputText, {
+//   mode: 'all',
+//   newlines: 'filter',
+//   sortLines: false,
+//   trimTextLines: true
+// });
diff --git a/src/tools/defineTool.tsx b/src/tools/defineTool.tsx
index 6541176..0e4dcbd 100644
--- a/src/tools/defineTool.tsx
+++ b/src/tools/defineTool.tsx
@@ -6,7 +6,7 @@ interface ToolOptions {
   path: string;
   component: LazyExoticComponent<JSXElementConstructor<ToolComponentProps>>;
   keywords: string[];
-  icon?: IconifyIcon | string;
+  icon: IconifyIcon | string;
   name: string;
   description: string;
   shortDescription: string;
@@ -26,7 +26,7 @@ export interface DefinedTool {
   name: string;
   description: string;
   shortDescription: string;
-  icon?: IconifyIcon | string;
+  icon: IconifyIcon | string;
   keywords: string[];
   component: () => JSX.Element;
 }