feat: remove duplicate lines

.codebuddy/.gitignore

@@ -0,0 +1 @@
+db/

.codebuddy/summary.md

@@ -0,0 +1,39 @@
+# Project Summary
+
+## Overview of Technologies Used
+This project is primarily built using the following technologies:
+- **Languages**: TypeScript, JavaScript, HTML, CSS
+- **Frameworks**:
+  - React (for building user interfaces)
+  - Playwright (for end-to-end testing)
+- **Main Libraries**:
+  - Tailwind CSS (for styling)
+  - MUI (Material-UI for components)
+  - pnpm (for package management)
+
+## Purpose of the Project
+The project appears to be a web application that provides various tools for image, JSON, list, number, and string manipulations. It is designed to offer users functionalities such as converting image formats, generating random numbers, and manipulating strings. The structure indicates a focus on modular components, making it easy to extend or modify specific tools without affecting the entire application.
+
+## Build and Configuration Files
+The following files are relevant for the configuration and building of the project:
+- `Dockerfile`: `/Dockerfile`
+- `package.json`: `/package.json`
+- `pnpm-lock.yaml`: `/pnpm-lock.yaml`
+- `playwright.config.ts`: `/playwright.config.ts`
+- `postcss.config.mjs`: `/postcss.config.mjs`
+- `tailwind.config.mjs`: `/tailwind.config.mjs`
+- `tsconfig.json`: `/tsconfig.json`
+- `vite.config.ts`: `/vite.config.ts`
+- `commitlint.config.js`: `/commitlint.config.js`
+
+## Source Files Directory
+The source files can be found in the following directory:
+- `/src`
+
+## Documentation Files Location
+Documentation files are located in the root directory:
+- `README.md`: `/README.md`
+- `LICENSE`: `/LICENSE`
+- `CODEOWNERS`: `/CODEOWNERS`
+
+This summary encapsulates the key aspects of the project, including its technological stack, purpose, file structure, and documentation locations.

.idea/workspace.xml

@@ -4,11 +4,18 @@
     <option name="autoReloadType" value="SELECTIVE" />
   </component>
   <component name="ChangeListManager">
-    <list default="true" id="b30e2810-c4c1-4aad-b134-794e52cc1c7d" name="Changes" comment="chore: img">
+    <list default="true" id="b30e2810-c4c1-4aad-b134-794e52cc1c7d" name="Changes" comment="docs: readme">
-      <change beforePath="$PROJECT_DIR$/.gitignore" beforeDir="false" afterPath="$PROJECT_DIR$/.gitignore" afterDir="false" />
+      <change afterPath="$PROJECT_DIR$/.codebuddy/.gitignore" afterDir="false" />
+      <change afterPath="$PROJECT_DIR$/.codebuddy/summary.md" afterDir="false" />
+      <change afterPath="$PROJECT_DIR$/src/pages/tools/string/remove-duplicate-lines/index.tsx" afterDir="false" />
+      <change afterPath="$PROJECT_DIR$/src/pages/tools/string/remove-duplicate-lines/meta.ts" afterDir="false" />
+      <change afterPath="$PROJECT_DIR$/src/pages/tools/string/remove-duplicate-lines/remove-duplicate-lines.service.test.ts" afterDir="false" />
+      <change afterPath="$PROJECT_DIR$/src/pages/tools/string/remove-duplicate-lines/service.ts" afterDir="false" />
       <change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/README.md" beforeDir="false" afterPath="$PROJECT_DIR$/README.md" afterDir="false" />
+      <change beforePath="$PROJECT_DIR$/scripts/create-tool.mjs" beforeDir="false" afterPath="$PROJECT_DIR$/scripts/create-tool.mjs" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/src/pages/tools/image/png/index.ts" beforeDir="false" afterPath="$PROJECT_DIR$/src/pages/tools/image/png/index.ts" afterDir="false" />
+      <change beforePath="$PROJECT_DIR$/src/components/Hero.tsx" beforeDir="false" afterPath="$PROJECT_DIR$/src/components/Hero.tsx" afterDir="false" />
+      <change beforePath="$PROJECT_DIR$/src/pages/tools/string/index.ts" beforeDir="false" afterPath="$PROJECT_DIR$/src/pages/tools/string/index.ts" afterDir="false" />
+      <change beforePath="$PROJECT_DIR$/src/tools/defineTool.tsx" beforeDir="false" afterPath="$PROJECT_DIR$/src/tools/defineTool.tsx" afterDir="false" />
     </list>
     <option name="SHOW_DIALOG" value="false" />
     <option name="HIGHLIGHT_CONFLICTS" value="true" />
@@ -60,48 +67,51 @@
     <option name="hideEmptyMiddlePackages" value="true" />
     <option name="showLibraryContents" value="true" />
   </component>
-  <component name="PropertiesComponent">{
+  <component name="PropertiesComponent"><![CDATA[{
-  &quot;keyToString&quot;: {
+  "keyToString": {
-    &quot;ASKED_ADD_EXTERNAL_FILES&quot;: &quot;true&quot;,
+    "ASKED_ADD_EXTERNAL_FILES": "true",
-    &quot;ASKED_SHARE_PROJECT_CONFIGURATION_FILES&quot;: &quot;true&quot;,
+    "ASKED_SHARE_PROJECT_CONFIGURATION_FILES": "true",
-    &quot;Docker.Dockerfile build.executor&quot;: &quot;Run&quot;,
+    "Docker.Dockerfile build.executor": "Run",
-    &quot;Docker.Dockerfile.executor&quot;: &quot;Run&quot;,
+    "Docker.Dockerfile.executor": "Run",
-    &quot;Playwright.JoinText Component.executor&quot;: &quot;Run&quot;,
+    "Playwright.JoinText Component.executor": "Run",
-    &quot;Playwright.JoinText Component.should merge text pieces with specified join character.executor&quot;: &quot;Run&quot;,
+    "Playwright.JoinText Component.should merge text pieces with specified join character.executor": "Run",
-    &quot;RunOnceActivity.OpenProjectViewOnStart&quot;: &quot;true&quot;,
+    "RunOnceActivity.OpenProjectViewOnStart": "true",
-    &quot;RunOnceActivity.ShowReadmeOnStart&quot;: &quot;true&quot;,
+    "RunOnceActivity.ShowReadmeOnStart": "true",
-    &quot;RunOnceActivity.git.unshallow&quot;: &quot;true&quot;,
+    "RunOnceActivity.git.unshallow": "true",
-    &quot;Vitest.compute function (1).executor&quot;: &quot;Run&quot;,
+    "Vitest.compute function (1).executor": "Run",
-    &quot;Vitest.compute function.executor&quot;: &quot;Run&quot;,
+    "Vitest.compute function.executor": "Run",
-    &quot;Vitest.mergeText.executor&quot;: &quot;Run&quot;,
+    "Vitest.mergeText.executor": "Run",
-    &quot;Vitest.mergeText.should merge lines and preserve blank lines when deleteBlankLines is false.executor&quot;: &quot;Run&quot;,
+    "Vitest.mergeText.should merge lines and preserve blank lines when deleteBlankLines is false.executor": "Run",
-    &quot;Vitest.mergeText.should merge lines, preserve blank lines and trailing spaces when both deleteBlankLines and deleteTrailingSpaces are false.executor&quot;: &quot;Run&quot;,
+    "Vitest.mergeText.should merge lines, preserve blank lines and trailing spaces when both deleteBlankLines and deleteTrailingSpaces are false.executor": "Run",
-    &quot;git-widget-placeholder&quot;: &quot;main&quot;,
+    "Vitest.removeDuplicateLines function.executor": "Run",
-    &quot;ignore.virus.scanning.warn.message&quot;: &quot;true&quot;,
+    "Vitest.removeDuplicateLines function.newlines option.executor": "Run",
-    &quot;kotlin-language-version-configured&quot;: &quot;true&quot;,
+    "Vitest.removeDuplicateLines function.newlines option.should filter newlines when newlines is set to filter.executor": "Run",
-    &quot;last_opened_file_path&quot;: &quot;C:/Users/Ibrahima/IdeaProjects/omni-tools/public/assets&quot;,
+    "git-widget-placeholder": "main",
-    &quot;node.js.detected.package.eslint&quot;: &quot;true&quot;,
+    "ignore.virus.scanning.warn.message": "true",
-    &quot;node.js.detected.package.tslint&quot;: &quot;true&quot;,
+    "kotlin-language-version-configured": "true",
-    &quot;node.js.selected.package.eslint&quot;: &quot;(autodetect)&quot;,
+    "last_opened_file_path": "C:/Users/Ibrahima/IdeaProjects/omni-tools/public/assets",
-    &quot;node.js.selected.package.tslint&quot;: &quot;(autodetect)&quot;,
+    "node.js.detected.package.eslint": "true",
-    &quot;nodejs_package_manager_path&quot;: &quot;npm&quot;,
+    "node.js.detected.package.tslint": "true",
-    &quot;npm.build.executor&quot;: &quot;Run&quot;,
+    "node.js.selected.package.eslint": "(autodetect)",
-    &quot;npm.dev.executor&quot;: &quot;Run&quot;,
+    "node.js.selected.package.tslint": "(autodetect)",
-    &quot;npm.lint.executor&quot;: &quot;Run&quot;,
+    "nodejs_package_manager_path": "npm",
-    &quot;npm.prebuild.executor&quot;: &quot;Run&quot;,
+    "npm.build.executor": "Run",
-    &quot;npm.script:create:tool.executor&quot;: &quot;Run&quot;,
+    "npm.dev.executor": "Run",
-    &quot;npm.test.executor&quot;: &quot;Run&quot;,
+    "npm.lint.executor": "Run",
-    &quot;npm.test:e2e.executor&quot;: &quot;Run&quot;,
+    "npm.prebuild.executor": "Run",
-    &quot;npm.test:e2e:run.executor&quot;: &quot;Run&quot;,
+    "npm.script:create:tool.executor": "Run",
-    &quot;prettierjs.PrettierConfiguration.Package&quot;: &quot;C:\\Users\\Ibrahima\\IdeaProjects\\omni-tools\\node_modules\\prettier&quot;,
+    "npm.test.executor": "Run",
-    &quot;project.structure.last.edited&quot;: &quot;Problems&quot;,
+    "npm.test:e2e.executor": "Run",
-    &quot;project.structure.proportion&quot;: &quot;0.0&quot;,
+    "npm.test:e2e:run.executor": "Run",
-    &quot;project.structure.side.proportion&quot;: &quot;0.2&quot;,
+    "prettierjs.PrettierConfiguration.Package": "C:\\Users\\Ibrahima\\IdeaProjects\\omni-tools\\node_modules\\prettier",
-    &quot;settings.editor.selected.configurable&quot;: &quot;settings.typescriptcompiler&quot;,
+    "project.structure.last.edited": "Problems",
-    &quot;ts.external.directory.path&quot;: &quot;C:\\Users\\Ibrahima\\IdeaProjects\\omni-tools\\node_modules\\typescript\\lib&quot;,
+    "project.structure.proportion": "0.0",
-    &quot;vue.rearranger.settings.migration&quot;: &quot;true&quot;
+    "project.structure.side.proportion": "0.2",
+    "settings.editor.selected.configurable": "settings.typescriptcompiler",
+    "ts.external.directory.path": "C:\\Users\\Ibrahima\\IdeaProjects\\omni-tools\\node_modules\\typescript\\lib",
+    "vue.rearranger.settings.migration": "true"
   }
-}</component>
+}]]></component>
   <component name="ReactDesignerToolWindowState">
     <option name="myId2Visible">
       <map>
@@ -127,13 +137,47 @@
       <recent name="C:\Users\HP\IdeaProjects\omni-tools\src\components\options" />
     </key>
   </component>
-  <component name="RunManager" selected="npm.build">
+  <component name="RunManager" selected="Vitest.removeDuplicateLines function">
-    <configuration name="Dockerfile" type="docker-deploy" factoryName="dockerfile" temporary="true" server-name="Docker">
+    <configuration name="removeDuplicateLines function" type="JavaScriptTestRunnerVitest" temporary="true" nameIsGenerated="true">
-      <deployment type="dockerfile">
+      <node-interpreter value="project" />
-        <settings>
+      <vitest-package value="$PROJECT_DIR$/node_modules/vitest" />
-          <option name="sourceFilePath" value="Dockerfile" />
+      <working-dir value="$PROJECT_DIR$" />
-        </settings>
+      <vitest-options value="--run" />
-      </deployment>
+      <envs />
+      <scope-kind value="SUITE" />
+      <test-file value="$PROJECT_DIR$/src/pages/tools/string/remove-duplicate-lines/remove-duplicate-lines.service.test.ts" />
+      <test-names>
+        <test-name value="removeDuplicateLines function" />
+      </test-names>
+      <method v="2" />
+    </configuration>
+    <configuration name="removeDuplicateLines function.newlines option" type="JavaScriptTestRunnerVitest" temporary="true" nameIsGenerated="true">
+      <node-interpreter value="project" />
+      <vitest-package value="$PROJECT_DIR$/node_modules/vitest" />
+      <working-dir value="$PROJECT_DIR$" />
+      <vitest-options value="--run" />
+      <envs />
+      <scope-kind value="SUITE" />
+      <test-file value="$PROJECT_DIR$/src/pages/tools/string/remove-duplicate-lines/remove-duplicate-lines.service.test.ts" />
+      <test-names>
+        <test-name value="removeDuplicateLines function" />
+        <test-name value="newlines option" />
+      </test-names>
+      <method v="2" />
+    </configuration>
+    <configuration name="removeDuplicateLines function.newlines option.should filter newlines when newlines is set to filter" type="JavaScriptTestRunnerVitest" temporary="true" nameIsGenerated="true">
+      <node-interpreter value="project" />
+      <vitest-package value="$PROJECT_DIR$/node_modules/vitest" />
+      <working-dir value="$PROJECT_DIR$" />
+      <vitest-options value="--run" />
+      <envs />
+      <scope-kind value="TEST" />
+      <test-file value="$PROJECT_DIR$/src/pages/tools/string/remove-duplicate-lines/remove-duplicate-lines.service.test.ts" />
+      <test-names>
+        <test-name value="removeDuplicateLines function" />
+        <test-name value="newlines option" />
+        <test-name value="should filter newlines when newlines is set to filter" />
+      </test-names>
       <method v="2" />
     </configuration>
     <configuration default="true" type="docker-deploy" factoryName="dockerfile" temporary="true">
@@ -162,40 +206,20 @@
       <envs />
       <method v="2" />
     </configuration>
-    <configuration name="lint" type="js.build_tools.npm" temporary="true" nameIsGenerated="true">
-      <package-json value="$PROJECT_DIR$/package.json" />
-      <command value="run" />
-      <scripts>
-        <script value="lint" />
-      </scripts>
-      <node-interpreter value="project" />
-      <envs />
-      <method v="2" />
-    </configuration>
-    <configuration name="test" type="js.build_tools.npm" temporary="true" nameIsGenerated="true">
-      <package-json value="$PROJECT_DIR$/package.json" />
-      <command value="run" />
-      <scripts>
-        <script value="test" />
-      </scripts>
-      <node-interpreter value="project" />
-      <envs />
-      <method v="2" />
-    </configuration>
     <list>
-      <item itemvalue="Docker.Dockerfile" />
       <item itemvalue="npm.build" />
-      <item itemvalue="npm.test" />
       <item itemvalue="npm.dev" />
-      <item itemvalue="npm.lint" />
+      <item itemvalue="Vitest.removeDuplicateLines function" />
+      <item itemvalue="Vitest.removeDuplicateLines function.newlines option" />
+      <item itemvalue="Vitest.removeDuplicateLines function.newlines option.should filter newlines when newlines is set to filter" />
     </list>
     <recent_temporary>
       <list>
-        <item itemvalue="npm.build" />
+        <item itemvalue="Vitest.removeDuplicateLines function" />
+        <item itemvalue="Vitest.removeDuplicateLines function.newlines option" />
+        <item itemvalue="Vitest.removeDuplicateLines function.newlines option.should filter newlines when newlines is set to filter" />
         <item itemvalue="npm.dev" />
-        <item itemvalue="Docker.Dockerfile" />
+        <item itemvalue="npm.build" />
-        <item itemvalue="npm.test" />
-        <item itemvalue="npm.lint" />
       </list>
     </recent_temporary>
   </component>
@@ -268,14 +292,8 @@
       <workItem from="1740670449847" duration="4776000" />
       <workItem from="1740702343843" duration="657000" />
       <workItem from="1740788381920" duration="465000" />
-    </task>
+      <workItem from="1740788856134" duration="659000" />
-    <task id="LOCAL-00088" summary="feat: sort list">
+      <workItem from="1740880919391" duration="3339000" />
-      <option name="closed" value="true" />
-      <created>1720545582958</created>
-      <option name="number" value="00088" />
-      <option name="presentableId" value="LOCAL-00088" />
-      <option name="project" value="LOCAL" />
-      <updated>1720545582958</updated>
     </task>
     <task id="LOCAL-00089" summary="feat: find most popular ui">
       <option name="closed" value="true" />
@@ -661,7 +679,15 @@
       <option name="project" value="LOCAL" />
       <updated>1740680778110</updated>
     </task>
-    <option name="localTasksCounter" value="137" />
+    <task id="LOCAL-00137" summary="docs: readme">
+      <option name="closed" value="true" />
+      <created>1740788899030</created>
+      <option name="number" value="00137" />
+      <option name="presentableId" value="LOCAL-00137" />
+      <option name="project" value="LOCAL" />
+      <updated>1740788899030</updated>
+    </task>
+    <option name="localTasksCounter" value="138" />
     <servers />
   </component>
   <component name="TypeScriptGeneratedFilesManager">
@@ -733,7 +759,6 @@
     <MESSAGE value="style: background svg" />
     <MESSAGE value="docs: img" />
     <MESSAGE value="fix: bg" />
-    <MESSAGE value="docs: readme" />
     <MESSAGE value="chore: handle enter press on search" />
     <MESSAGE value="chore: show tooloptions in example" />
     <MESSAGE value="refact: examples" />
@@ -745,7 +770,8 @@
     <MESSAGE value="chore: prettify json in home" />
     <MESSAGE value="feat: jakarta font" />
     <MESSAGE value="chore: img" />
-    <option name="LAST_COMMIT_MESSAGE" value="chore: img" />
+    <MESSAGE value="docs: readme" />
+    <option name="LAST_COMMIT_MESSAGE" value="docs: readme" />
   </component>
   <component name="XSLT-Support.FileAssociations.UIState">
     <expand />

scripts/create-tool.mjs

@@ -78,8 +78,8 @@ import { Box } from '@mui/material';
 import React from 'react';
 import * as Yup from 'yup';
 
-const initialValues = {};
+type InitialValuesType = {};
-type InitialValuesType = typeof initialValues;
+const initialValues: InitialValuesType = {};
 const validationSchema = Yup.object({
   // splitSeparator: Yup.string().required('The separator is required')
 });

src/components/Hero.tsx

@@ -7,6 +7,7 @@ import { DefinedTool } from '@tools/defineTool';
 import { filterTools, tools } from '@tools/index';
 import { useNavigate } from 'react-router-dom';
 import _ from 'lodash';
+import { Icon } from '@iconify/react';
 
 const exampleTools: { label: string; url: string }[] = [
   {
@@ -97,10 +98,13 @@ export default function Hero() {
             {...props}
             onClick={() => navigate('/' + option.path)}
           >
+            <Stack direction={'row'} spacing={2} alignItems={'center'}>
+              <Icon fontSize={20} icon={option.icon} />
               <Box>
                 <Typography fontWeight={'bold'}>{option.name}</Typography>
                 <Typography fontSize={12}>{option.shortDescription}</Typography>
               </Box>
+            </Stack>
           </Box>
         )}
         onKeyDown={(event) => {

src/pages/tools/string/index.ts

@@ -1,3 +1,4 @@
+import { tool as stringRemoveDuplicateLines } from './remove-duplicate-lines/meta';
 import { tool as stringReverse } from './reverse/meta';
 import { tool as stringRandomizeCase } from './randomize-case/meta';
 import { tool as stringUppercase } from './uppercase/meta';
@@ -11,6 +12,7 @@ import { tool as stringJoin } from './join/meta';
 export const stringTools = [
   stringSplit,
   stringJoin,
+  stringRemoveDuplicateLines,
   stringToMorse
   // stringReverse,
   // stringRandomizeCase,

src/pages/tools/string/remove-duplicate-lines/index.tsx

@@ -0,0 +1,260 @@
+import { Box } from '@mui/material';
+import React, { useRef, useState } from 'react';
+import ToolTextInput from '@components/input/ToolTextInput';
+import ToolTextResult from '@components/result/ToolTextResult';
+import ToolOptions, { GetGroupsType } from '@components/options/ToolOptions';
+import SimpleRadio from '@components/options/SimpleRadio';
+import CheckboxWithDesc from '@components/options/CheckboxWithDesc';
+import ToolInputAndResult from '@components/ToolInputAndResult';
+import ToolExamples, {
+  CardExampleType
+} from '@components/examples/ToolExamples';
+import { ToolComponentProps } from '@tools/defineTool';
+import { FormikProps } from 'formik';
+import removeDuplicateLines, {
+  DuplicateRemovalMode,
+  DuplicateRemoverOptions,
+  NewlineOption
+} from './service';
+
+// Initial values for our form
+const initialValues: DuplicateRemoverOptions = {
+  mode: 'all',
+  newlines: 'filter',
+  sortLines: false,
+  trimTextLines: false
+};
+
+// Operation mode options
+const operationModes = [
+  {
+    title: 'Remove All Duplicate Lines',
+    description:
+      'If this option is selected, then all repeated lines across entire text are removed, starting from the second occurrence.',
+    value: 'all' as DuplicateRemovalMode
+  },
+  {
+    title: 'Remove Consecutive Duplicate Lines',
+    description:
+      'If this option is selected, then only consecutive repeated lines are removed.',
+    value: 'consecutive' as DuplicateRemovalMode
+  },
+  {
+    title: 'Leave Absolutely Unique Text Lines',
+    description:
+      'If this option is selected, then all lines that appear more than once are removed.',
+    value: 'unique' as DuplicateRemovalMode
+  }
+];
+
+// Newlines options
+const newlineOptions = [
+  {
+    title: 'Preserve All Newlines',
+    description: 'Leave all empty lines in the output.',
+    value: 'preserve' as NewlineOption
+  },
+  {
+    title: 'Filter All Newlines',
+    description: 'Process newlines as regular lines.',
+    value: 'filter' as NewlineOption
+  },
+  {
+    title: 'Delete All Newlines',
+    description: 'Before filtering uniques, remove all newlines.',
+    value: 'delete' as NewlineOption
+  }
+];
+
+// Example cards for demonstration
+const exampleCards: CardExampleType<typeof initialValues>[] = [
+  {
+    title: 'Remove Duplicate Items from List',
+    description:
+      'Removes duplicate items from a shopping list, keeping only the first occurrence of each item.',
+    sampleText: `Apples
+Bananas
+Milk
+Eggs
+Bread
+Milk
+Cheese
+Apples
+Yogurt`,
+    sampleResult: `Apples
+Bananas
+Milk
+Eggs
+Bread
+Cheese
+Yogurt`,
+    sampleOptions: {
+      ...initialValues,
+      mode: 'all',
+      newlines: 'filter',
+      sortLines: false,
+      trimTextLines: false
+    }
+  },
+  {
+    title: 'Clean Consecutive Duplicates',
+    description:
+      'Removes consecutive duplicates from log entries, which often happen when a system repeatedly logs the same error.',
+    sampleText: `[INFO] Application started
+[ERROR] Connection failed
+[ERROR] Connection failed
+[ERROR] Connection failed
+[INFO] Retrying connection
+[ERROR] Authentication error
+[ERROR] Authentication error
+[INFO] Connection established`,
+    sampleResult: `[INFO] Application started
+[ERROR] Connection failed
+[INFO] Retrying connection
+[ERROR] Authentication error
+[INFO] Connection established`,
+    sampleOptions: {
+      ...initialValues,
+      mode: 'consecutive',
+      newlines: 'filter',
+      sortLines: false,
+      trimTextLines: false
+    }
+  },
+  {
+    title: 'Extract Unique Entries Only',
+    description:
+      'Filters a list to keep only entries that appear exactly once, removing any duplicated items entirely.',
+    sampleText: `Red
+Blue
+Green
+Blue
+Yellow
+Purple
+Red
+Orange`,
+    sampleResult: `Green
+Yellow
+Purple
+Orange`,
+    sampleOptions: {
+      ...initialValues,
+      mode: 'unique',
+      newlines: 'filter',
+      sortLines: false,
+      trimTextLines: false
+    }
+  },
+  {
+    title: 'Sort and Clean Data',
+    description:
+      'Removes duplicate items from a list, trims whitespace, and sorts the results alphabetically.',
+    sampleText: `  Apple
+Banana
+ Cherry
+Apple
+   Banana
+Dragonfruit
+ Elderberry `,
+    sampleResult: `Apple
+Banana
+Cherry
+Dragonfruit
+Elderberry`,
+    sampleOptions: {
+      ...initialValues,
+      mode: 'all',
+      newlines: 'filter',
+      sortLines: true,
+      trimTextLines: true
+    }
+  }
+];
+
+export default function RemoveDuplicateLines({ title }: ToolComponentProps) {
+  const [input, setInput] = useState<string>('');
+  const [result, setResult] = useState<string>('');
+  const formRef = useRef<FormikProps<typeof initialValues>>(null);
+
+  const computeExternal = (
+    optionsValues: typeof initialValues,
+    inputText: string
+  ) => {
+    setResult(removeDuplicateLines(inputText, optionsValues));
+  };
+
+  const getGroups: GetGroupsType<typeof initialValues> = ({
+    values,
+    updateField
+  }) => [
+    {
+      title: 'Operation Mode',
+      component: operationModes.map(({ title, description, value }) => (
+        <SimpleRadio
+          key={value}
+          checked={value === values.mode}
+          title={title}
+          description={description}
+          onClick={() => updateField('mode', value)}
+        />
+      ))
+    },
+    {
+      title: 'Newlines, Tabs and Spaces',
+      component: [
+        ...newlineOptions.map(({ title, description, value }) => (
+          <SimpleRadio
+            key={value}
+            checked={value === values.newlines}
+            title={title}
+            description={description}
+            onClick={() => updateField('newlines', value)}
+          />
+        )),
+        <CheckboxWithDesc
+          key="trimTextLines"
+          checked={values.trimTextLines}
+          title="Trim Text Lines"
+          description="Before filtering uniques, remove tabs and spaces from the beginning and end of all lines."
+          onChange={(checked) => updateField('trimTextLines', checked)}
+        />
+      ]
+    },
+    {
+      title: 'Sort Lines',
+      component: [
+        <CheckboxWithDesc
+          key="sortLines"
+          checked={values.sortLines}
+          title="Sort the Output Lines"
+          description="After removing the duplicates, sort the unique lines."
+          onChange={(checked) => updateField('sortLines', checked)}
+        />
+      ]
+    }
+  ];
+
+  return (
+    <Box>
+      <ToolInputAndResult
+        input={<ToolTextInput value={input} onChange={setInput} />}
+        result={
+          <ToolTextResult title={'Text without duplicates'} value={result} />
+        }
+      />
+      <ToolOptions
+        compute={computeExternal}
+        getGroups={getGroups}
+        initialValues={initialValues}
+        input={input}
+      />
+      <ToolExamples
+        title={title}
+        exampleCards={exampleCards}
+        getGroups={getGroups}
+        formRef={formRef}
+        setInput={setInput}
+      />
+    </Box>
+  );
+}

src/pages/tools/string/remove-duplicate-lines/meta.ts

@@ -0,0 +1,13 @@
+import { defineTool } from '@tools/defineTool';
+import { lazy } from 'react';
+
+export const tool = defineTool('string', {
+  name: 'Remove duplicate lines',
+  path: 'remove-duplicate-lines',
+  icon: 'pepicons-print:duplicate-off',
+  description:
+    "Load your text in the input form on the left and you'll instantly get text with no duplicate lines in the output area. Powerful, free, and fast. Load text lines – get unique text lines",
+  shortDescription: 'Quickly delete all repeated lines from text',
+  keywords: ['remove', 'duplicate', 'lines'],
+  component: lazy(() => import('./index'))
+});

src/pages/tools/string/remove-duplicate-lines/remove-duplicate-lines.service.test.ts

@@ -0,0 +1,200 @@
+import { describe, expect, it } from 'vitest';
+import removeDuplicateLines, { DuplicateRemoverOptions } from './service';
+
+describe('removeDuplicateLines function', () => {
+  // Test for 'all' duplicate removal mode
+  describe('mode: all', () => {
+    it('should remove all duplicates keeping first occurrence', () => {
+      const input = 'line1\nline2\nline1\nline3\nline2';
+      const options: DuplicateRemoverOptions = {
+        mode: 'all',
+        newlines: 'filter',
+        sortLines: false,
+        trimTextLines: false
+      };
+      const result = removeDuplicateLines(input, options);
+      expect(result).toBe('line1\nline2\nline3');
+    });
+
+    it('should handle case-sensitive duplicates correctly', () => {
+      const input = 'Line1\nline1\nLine2\nline2';
+      const options: DuplicateRemoverOptions = {
+        mode: 'all',
+        newlines: 'filter',
+        sortLines: false,
+        trimTextLines: false
+      };
+      const result = removeDuplicateLines(input, options);
+      expect(result).toBe('Line1\nline1\nLine2\nline2');
+    });
+  });
+
+  // Test for 'consecutive' duplicate removal mode
+  describe('mode: consecutive', () => {
+    it('should remove only consecutive duplicates', () => {
+      const input = 'line1\nline1\nline2\nline3\nline3\nline1';
+      const options: DuplicateRemoverOptions = {
+        mode: 'consecutive',
+        newlines: 'filter',
+        sortLines: false,
+        trimTextLines: false
+      };
+      const result = removeDuplicateLines(input, options);
+      expect(result).toBe('line1\nline2\nline3\nline1');
+    });
+  });
+
+  // Test for 'unique' duplicate removal mode
+  describe('mode: unique', () => {
+    it('should keep only lines that appear exactly once', () => {
+      const input = 'line1\nline2\nline1\nline3\nline4\nline4';
+      const options: DuplicateRemoverOptions = {
+        mode: 'unique',
+        newlines: 'filter',
+        sortLines: false,
+        trimTextLines: false
+      };
+      const result = removeDuplicateLines(input, options);
+      expect(result).toBe('line2\nline3');
+    });
+  });
+
+  // Test for newlines handling
+  describe('newlines option', () => {
+    it('should filter newlines when newlines is set to filter', () => {
+      const input = 'line1\n\nline2\n\n\nline3';
+      const options: DuplicateRemoverOptions = {
+        mode: 'all',
+        newlines: 'filter',
+        sortLines: false,
+        trimTextLines: false
+      };
+      const result = removeDuplicateLines(input, options);
+      expect(result).toBe('line1\n\nline2\nline3');
+    });
+
+    it('should delete newlines when newlines is set to delete', () => {
+      const input = 'line1\n\nline2\n\n\nline3';
+      const options: DuplicateRemoverOptions = {
+        mode: 'all',
+        newlines: 'delete',
+        sortLines: false,
+        trimTextLines: false
+      };
+      const result = removeDuplicateLines(input, options);
+      expect(result).toBe('line1\nline2\nline3');
+    });
+
+    it('should preserve newlines when newlines is set to preserve', () => {
+      const input = 'line1\n\nline2\n\nline2\nline3';
+      const options: DuplicateRemoverOptions = {
+        mode: 'all',
+        newlines: 'preserve',
+        sortLines: false,
+        trimTextLines: false
+      };
+      const result = removeDuplicateLines(input, options);
+      // This test needs careful consideration of the expected behavior
+      expect(result).not.toContain('line2\nline2');
+      expect(result).toContain('line1');
+      expect(result).toContain('line2');
+      expect(result).toContain('line3');
+    });
+  });
+
+  // Test for sorting
+  describe('sortLines option', () => {
+    it('should sort lines when sortLines is true', () => {
+      const input = 'line3\nline1\nline2';
+      const options: DuplicateRemoverOptions = {
+        mode: 'all',
+        newlines: 'filter',
+        sortLines: true,
+        trimTextLines: false
+      };
+      const result = removeDuplicateLines(input, options);
+      expect(result).toBe('line1\nline2\nline3');
+    });
+  });
+
+  // Test for trimming
+  describe('trimTextLines option', () => {
+    it('should trim lines when trimTextLines is true', () => {
+      const input = '  line1  \n  line2  \nline3';
+      const options: DuplicateRemoverOptions = {
+        mode: 'all',
+        newlines: 'filter',
+        sortLines: false,
+        trimTextLines: true
+      };
+      const result = removeDuplicateLines(input, options);
+      expect(result).toBe('line1\nline2\nline3');
+    });
+
+    it('should consider trimmed lines as duplicates', () => {
+      const input = '  line1  \nline1\n  line2\nline2  ';
+      const options: DuplicateRemoverOptions = {
+        mode: 'all',
+        newlines: 'filter',
+        sortLines: false,
+        trimTextLines: true
+      };
+      const result = removeDuplicateLines(input, options);
+      expect(result).toBe('line1\nline2');
+    });
+  });
+
+  // Combined scenarios
+  describe('combined options', () => {
+    it('should handle all options together correctly', () => {
+      const input = '  line3  \nline1\n\nline3\nline2\nline1';
+      const options: DuplicateRemoverOptions = {
+        mode: 'all',
+        newlines: 'delete',
+        sortLines: true,
+        trimTextLines: true
+      };
+      const result = removeDuplicateLines(input, options);
+      expect(result).toBe('line1\nline2\nline3');
+    });
+  });
+
+  // Edge cases
+  describe('edge cases', () => {
+    it('should handle empty input', () => {
+      const input = '';
+      const options: DuplicateRemoverOptions = {
+        mode: 'all',
+        newlines: 'filter',
+        sortLines: false,
+        trimTextLines: false
+      };
+      const result = removeDuplicateLines(input, options);
+      expect(result).toBe('');
+    });
+
+    it('should handle input with only newlines', () => {
+      const input = '\n\n\n';
+      const options: DuplicateRemoverOptions = {
+        mode: 'all',
+        newlines: 'filter',
+        sortLines: false,
+        trimTextLines: false
+      };
+      const result = removeDuplicateLines(input, options);
+      expect(result).toBe('');
+    });
+
+    it('should handle input with only whitespace', () => {
+      const input = '   \n  \n    ';
+      const options: DuplicateRemoverOptions = {
+        mode: 'all',
+        newlines: 'filter',
+        sortLines: false,
+        trimTextLines: true
+      };
+      const result = removeDuplicateLines(input, options);
+      expect(result).toBe('');
+    });
+  });
+});

src/pages/tools/string/remove-duplicate-lines/service.ts

@@ -0,0 +1,88 @@
+export type NewlineOption = 'preserve' | 'filter' | 'delete';
+export type DuplicateRemovalMode = 'all' | 'consecutive' | 'unique';
+
+export interface DuplicateRemoverOptions {
+  mode: DuplicateRemovalMode;
+  newlines: NewlineOption;
+  sortLines: boolean;
+  trimTextLines: boolean;
+}
+
+/**
+ * Removes duplicate lines from text based on specified options
+ * @param text The input text to process
+ * @param options Configuration options for text processing
+ * @returns Processed text with duplicates removed according to options
+ */
+export default function removeDuplicateLines(
+  text: string,
+  options: DuplicateRemoverOptions
+): string {
+  // Split the text into individual lines
+  let lines = text.split('\n');
+
+  // Process newlines based on option
+  if (options.newlines === 'delete') {
+    // Remove all empty lines
+    lines = lines.filter((line) => line.trim() !== '');
+  }
+
+  // Trim lines if option is selected
+  if (options.trimTextLines) {
+    lines = lines.map((line) => line.trim());
+  }
+
+  // Remove duplicates based on mode
+  let processedLines: string[] = [];
+
+  if (options.mode === 'all') {
+    // Remove all duplicates, keeping only first occurrence
+    const seen = new Set<string>();
+    processedLines = lines.filter((line) => {
+      if (seen.has(line)) {
+        return false;
+      }
+      seen.add(line);
+      return true;
+    });
+  } else if (options.mode === 'consecutive') {
+    // Remove only consecutive duplicates
+    processedLines = lines.filter((line, index, arr) => {
+      return index === 0 || line !== arr[index - 1];
+    });
+  } else if (options.mode === 'unique') {
+    // Leave only absolutely unique lines
+    const lineCount = new Map<string, number>();
+    lines.forEach((line) => {
+      lineCount.set(line, (lineCount.get(line) || 0) + 1);
+    });
+
+    processedLines = lines.filter((line) => lineCount.get(line) === 1);
+  }
+
+  // Sort lines if option is selected
+  if (options.sortLines) {
+    processedLines.sort();
+  }
+
+  // Process newlines for output
+  if (options.newlines === 'filter') {
+    // Process newlines as regular lines (already done by default)
+  } else if (options.newlines === 'preserve') {
+    // Make sure empty lines are preserved in the output
+    processedLines = text.split('\n').map((line) => {
+      if (line.trim() === '') return line;
+      return processedLines.includes(line) ? line : '';
+    });
+  }
+
+  return processedLines.join('\n');
+}
+
+// Example usage:
+// const result = removeDuplicateLines(inputText, {
+//   mode: 'all',
+//   newlines: 'filter',
+//   sortLines: false,
+//   trimTextLines: true
+// });

src/tools/defineTool.tsx

@@ -6,7 +6,7 @@ interface ToolOptions {
   path: string;
   component: LazyExoticComponent<JSXElementConstructor<ToolComponentProps>>;
   keywords: string[];
-  icon?: IconifyIcon | string;
+  icon: IconifyIcon | string;
   name: string;
   description: string;
   shortDescription: string;
@@ -26,7 +26,7 @@ export interface DefinedTool {
   name: string;
   description: string;
   shortDescription: string;
-  icon?: IconifyIcon | string;
+  icon: IconifyIcon | string;
   keywords: string[];
   component: () => JSX.Element;
 }