feat: remove duplicate lines

src/components/Hero.tsx

@@ -7,6 +7,7 @@ import { DefinedTool } from '@tools/defineTool';
 import { filterTools, tools } from '@tools/index';
 import { useNavigate } from 'react-router-dom';
 import _ from 'lodash';
+import { Icon } from '@iconify/react';
 
 const exampleTools: { label: string; url: string }[] = [
   {
@@ -97,10 +98,13 @@ export default function Hero() {
             {...props}
             onClick={() => navigate('/' + option.path)}
           >
-            <Box>
-              <Typography fontWeight={'bold'}>{option.name}</Typography>
-              <Typography fontSize={12}>{option.shortDescription}</Typography>
-            </Box>
+            <Stack direction={'row'} spacing={2} alignItems={'center'}>
+              <Icon fontSize={20} icon={option.icon} />
+              <Box>
+                <Typography fontWeight={'bold'}>{option.name}</Typography>
+                <Typography fontSize={12}>{option.shortDescription}</Typography>
+              </Box>
+            </Stack>
           </Box>
         )}
         onKeyDown={(event) => {

src/pages/tools/string/index.ts

@@ -1,3 +1,4 @@
+import { tool as stringRemoveDuplicateLines } from './remove-duplicate-lines/meta';
 import { tool as stringReverse } from './reverse/meta';
 import { tool as stringRandomizeCase } from './randomize-case/meta';
 import { tool as stringUppercase } from './uppercase/meta';
@@ -11,6 +12,7 @@ import { tool as stringJoin } from './join/meta';
 export const stringTools = [
   stringSplit,
   stringJoin,
+  stringRemoveDuplicateLines,
   stringToMorse
   // stringReverse,
   // stringRandomizeCase,

src/pages/tools/string/remove-duplicate-lines/index.tsx

@@ -0,0 +1,260 @@
+import { Box } from '@mui/material';
+import React, { useRef, useState } from 'react';
+import ToolTextInput from '@components/input/ToolTextInput';
+import ToolTextResult from '@components/result/ToolTextResult';
+import ToolOptions, { GetGroupsType } from '@components/options/ToolOptions';
+import SimpleRadio from '@components/options/SimpleRadio';
+import CheckboxWithDesc from '@components/options/CheckboxWithDesc';
+import ToolInputAndResult from '@components/ToolInputAndResult';
+import ToolExamples, {
+  CardExampleType
+} from '@components/examples/ToolExamples';
+import { ToolComponentProps } from '@tools/defineTool';
+import { FormikProps } from 'formik';
+import removeDuplicateLines, {
+  DuplicateRemovalMode,
+  DuplicateRemoverOptions,
+  NewlineOption
+} from './service';
+
+// Initial values for our form
+const initialValues: DuplicateRemoverOptions = {
+  mode: 'all',
+  newlines: 'filter',
+  sortLines: false,
+  trimTextLines: false
+};
+
+// Operation mode options
+const operationModes = [
+  {
+    title: 'Remove All Duplicate Lines',
+    description:
+      'If this option is selected, then all repeated lines across entire text are removed, starting from the second occurrence.',
+    value: 'all' as DuplicateRemovalMode
+  },
+  {
+    title: 'Remove Consecutive Duplicate Lines',
+    description:
+      'If this option is selected, then only consecutive repeated lines are removed.',
+    value: 'consecutive' as DuplicateRemovalMode
+  },
+  {
+    title: 'Leave Absolutely Unique Text Lines',
+    description:
+      'If this option is selected, then all lines that appear more than once are removed.',
+    value: 'unique' as DuplicateRemovalMode
+  }
+];
+
+// Newlines options
+const newlineOptions = [
+  {
+    title: 'Preserve All Newlines',
+    description: 'Leave all empty lines in the output.',
+    value: 'preserve' as NewlineOption
+  },
+  {
+    title: 'Filter All Newlines',
+    description: 'Process newlines as regular lines.',
+    value: 'filter' as NewlineOption
+  },
+  {
+    title: 'Delete All Newlines',
+    description: 'Before filtering uniques, remove all newlines.',
+    value: 'delete' as NewlineOption
+  }
+];
+
+// Example cards for demonstration
+const exampleCards: CardExampleType<typeof initialValues>[] = [
+  {
+    title: 'Remove Duplicate Items from List',
+    description:
+      'Removes duplicate items from a shopping list, keeping only the first occurrence of each item.',
+    sampleText: `Apples
+Bananas
+Milk
+Eggs
+Bread
+Milk
+Cheese
+Apples
+Yogurt`,
+    sampleResult: `Apples
+Bananas
+Milk
+Eggs
+Bread
+Cheese
+Yogurt`,
+    sampleOptions: {
+      ...initialValues,
+      mode: 'all',
+      newlines: 'filter',
+      sortLines: false,
+      trimTextLines: false
+    }
+  },
+  {
+    title: 'Clean Consecutive Duplicates',
+    description:
+      'Removes consecutive duplicates from log entries, which often happen when a system repeatedly logs the same error.',
+    sampleText: `[INFO] Application started
+[ERROR] Connection failed
+[ERROR] Connection failed
+[ERROR] Connection failed
+[INFO] Retrying connection
+[ERROR] Authentication error
+[ERROR] Authentication error
+[INFO] Connection established`,
+    sampleResult: `[INFO] Application started
+[ERROR] Connection failed
+[INFO] Retrying connection
+[ERROR] Authentication error
+[INFO] Connection established`,
+    sampleOptions: {
+      ...initialValues,
+      mode: 'consecutive',
+      newlines: 'filter',
+      sortLines: false,
+      trimTextLines: false
+    }
+  },
+  {
+    title: 'Extract Unique Entries Only',
+    description:
+      'Filters a list to keep only entries that appear exactly once, removing any duplicated items entirely.',
+    sampleText: `Red
+Blue
+Green
+Blue
+Yellow
+Purple
+Red
+Orange`,
+    sampleResult: `Green
+Yellow
+Purple
+Orange`,
+    sampleOptions: {
+      ...initialValues,
+      mode: 'unique',
+      newlines: 'filter',
+      sortLines: false,
+      trimTextLines: false
+    }
+  },
+  {
+    title: 'Sort and Clean Data',
+    description:
+      'Removes duplicate items from a list, trims whitespace, and sorts the results alphabetically.',
+    sampleText: `  Apple
+Banana
+ Cherry
+Apple
+   Banana
+Dragonfruit
+ Elderberry `,
+    sampleResult: `Apple
+Banana
+Cherry
+Dragonfruit
+Elderberry`,
+    sampleOptions: {
+      ...initialValues,
+      mode: 'all',
+      newlines: 'filter',
+      sortLines: true,
+      trimTextLines: true
+    }
+  }
+];
+
+export default function RemoveDuplicateLines({ title }: ToolComponentProps) {
+  const [input, setInput] = useState<string>('');
+  const [result, setResult] = useState<string>('');
+  const formRef = useRef<FormikProps<typeof initialValues>>(null);
+
+  const computeExternal = (
+    optionsValues: typeof initialValues,
+    inputText: string
+  ) => {
+    setResult(removeDuplicateLines(inputText, optionsValues));
+  };
+
+  const getGroups: GetGroupsType<typeof initialValues> = ({
+    values,
+    updateField
+  }) => [
+    {
+      title: 'Operation Mode',
+      component: operationModes.map(({ title, description, value }) => (
+        <SimpleRadio
+          key={value}
+          checked={value === values.mode}
+          title={title}
+          description={description}
+          onClick={() => updateField('mode', value)}
+        />
+      ))
+    },
+    {
+      title: 'Newlines, Tabs and Spaces',
+      component: [
+        ...newlineOptions.map(({ title, description, value }) => (
+          <SimpleRadio
+            key={value}
+            checked={value === values.newlines}
+            title={title}
+            description={description}
+            onClick={() => updateField('newlines', value)}
+          />
+        )),
+        <CheckboxWithDesc
+          key="trimTextLines"
+          checked={values.trimTextLines}
+          title="Trim Text Lines"
+          description="Before filtering uniques, remove tabs and spaces from the beginning and end of all lines."
+          onChange={(checked) => updateField('trimTextLines', checked)}
+        />
+      ]
+    },
+    {
+      title: 'Sort Lines',
+      component: [
+        <CheckboxWithDesc
+          key="sortLines"
+          checked={values.sortLines}
+          title="Sort the Output Lines"
+          description="After removing the duplicates, sort the unique lines."
+          onChange={(checked) => updateField('sortLines', checked)}
+        />
+      ]
+    }
+  ];
+
+  return (
+    <Box>
+      <ToolInputAndResult
+        input={<ToolTextInput value={input} onChange={setInput} />}
+        result={
+          <ToolTextResult title={'Text without duplicates'} value={result} />
+        }
+      />
+      <ToolOptions
+        compute={computeExternal}
+        getGroups={getGroups}
+        initialValues={initialValues}
+        input={input}
+      />
+      <ToolExamples
+        title={title}
+        exampleCards={exampleCards}
+        getGroups={getGroups}
+        formRef={formRef}
+        setInput={setInput}
+      />
+    </Box>
+  );
+}

src/pages/tools/string/remove-duplicate-lines/meta.ts

@@ -0,0 +1,13 @@
+import { defineTool } from '@tools/defineTool';
+import { lazy } from 'react';
+
+export const tool = defineTool('string', {
+  name: 'Remove duplicate lines',
+  path: 'remove-duplicate-lines',
+  icon: 'pepicons-print:duplicate-off',
+  description:
+    "Load your text in the input form on the left and you'll instantly get text with no duplicate lines in the output area. Powerful, free, and fast. Load text lines – get unique text lines",
+  shortDescription: 'Quickly delete all repeated lines from text',
+  keywords: ['remove', 'duplicate', 'lines'],
+  component: lazy(() => import('./index'))
+});

src/pages/tools/string/remove-duplicate-lines/remove-duplicate-lines.service.test.ts

@@ -0,0 +1,200 @@
+import { describe, expect, it } from 'vitest';
+import removeDuplicateLines, { DuplicateRemoverOptions } from './service';
+
+describe('removeDuplicateLines function', () => {
+  // Test for 'all' duplicate removal mode
+  describe('mode: all', () => {
+    it('should remove all duplicates keeping first occurrence', () => {
+      const input = 'line1\nline2\nline1\nline3\nline2';
+      const options: DuplicateRemoverOptions = {
+        mode: 'all',
+        newlines: 'filter',
+        sortLines: false,
+        trimTextLines: false
+      };
+      const result = removeDuplicateLines(input, options);
+      expect(result).toBe('line1\nline2\nline3');
+    });
+
+    it('should handle case-sensitive duplicates correctly', () => {
+      const input = 'Line1\nline1\nLine2\nline2';
+      const options: DuplicateRemoverOptions = {
+        mode: 'all',
+        newlines: 'filter',
+        sortLines: false,
+        trimTextLines: false
+      };
+      const result = removeDuplicateLines(input, options);
+      expect(result).toBe('Line1\nline1\nLine2\nline2');
+    });
+  });
+
+  // Test for 'consecutive' duplicate removal mode
+  describe('mode: consecutive', () => {
+    it('should remove only consecutive duplicates', () => {
+      const input = 'line1\nline1\nline2\nline3\nline3\nline1';
+      const options: DuplicateRemoverOptions = {
+        mode: 'consecutive',
+        newlines: 'filter',
+        sortLines: false,
+        trimTextLines: false
+      };
+      const result = removeDuplicateLines(input, options);
+      expect(result).toBe('line1\nline2\nline3\nline1');
+    });
+  });
+
+  // Test for 'unique' duplicate removal mode
+  describe('mode: unique', () => {
+    it('should keep only lines that appear exactly once', () => {
+      const input = 'line1\nline2\nline1\nline3\nline4\nline4';
+      const options: DuplicateRemoverOptions = {
+        mode: 'unique',
+        newlines: 'filter',
+        sortLines: false,
+        trimTextLines: false
+      };
+      const result = removeDuplicateLines(input, options);
+      expect(result).toBe('line2\nline3');
+    });
+  });
+
+  // Test for newlines handling
+  describe('newlines option', () => {
+    it('should filter newlines when newlines is set to filter', () => {
+      const input = 'line1\n\nline2\n\n\nline3';
+      const options: DuplicateRemoverOptions = {
+        mode: 'all',
+        newlines: 'filter',
+        sortLines: false,
+        trimTextLines: false
+      };
+      const result = removeDuplicateLines(input, options);
+      expect(result).toBe('line1\n\nline2\nline3');
+    });
+
+    it('should delete newlines when newlines is set to delete', () => {
+      const input = 'line1\n\nline2\n\n\nline3';
+      const options: DuplicateRemoverOptions = {
+        mode: 'all',
+        newlines: 'delete',
+        sortLines: false,
+        trimTextLines: false
+      };
+      const result = removeDuplicateLines(input, options);
+      expect(result).toBe('line1\nline2\nline3');
+    });
+
+    it('should preserve newlines when newlines is set to preserve', () => {
+      const input = 'line1\n\nline2\n\nline2\nline3';
+      const options: DuplicateRemoverOptions = {
+        mode: 'all',
+        newlines: 'preserve',
+        sortLines: false,
+        trimTextLines: false
+      };
+      const result = removeDuplicateLines(input, options);
+      // This test needs careful consideration of the expected behavior
+      expect(result).not.toContain('line2\nline2');
+      expect(result).toContain('line1');
+      expect(result).toContain('line2');
+      expect(result).toContain('line3');
+    });
+  });
+
+  // Test for sorting
+  describe('sortLines option', () => {
+    it('should sort lines when sortLines is true', () => {
+      const input = 'line3\nline1\nline2';
+      const options: DuplicateRemoverOptions = {
+        mode: 'all',
+        newlines: 'filter',
+        sortLines: true,
+        trimTextLines: false
+      };
+      const result = removeDuplicateLines(input, options);
+      expect(result).toBe('line1\nline2\nline3');
+    });
+  });
+
+  // Test for trimming
+  describe('trimTextLines option', () => {
+    it('should trim lines when trimTextLines is true', () => {
+      const input = '  line1  \n  line2  \nline3';
+      const options: DuplicateRemoverOptions = {
+        mode: 'all',
+        newlines: 'filter',
+        sortLines: false,
+        trimTextLines: true
+      };
+      const result = removeDuplicateLines(input, options);
+      expect(result).toBe('line1\nline2\nline3');
+    });
+
+    it('should consider trimmed lines as duplicates', () => {
+      const input = '  line1  \nline1\n  line2\nline2  ';
+      const options: DuplicateRemoverOptions = {
+        mode: 'all',
+        newlines: 'filter',
+        sortLines: false,
+        trimTextLines: true
+      };
+      const result = removeDuplicateLines(input, options);
+      expect(result).toBe('line1\nline2');
+    });
+  });
+
+  // Combined scenarios
+  describe('combined options', () => {
+    it('should handle all options together correctly', () => {
+      const input = '  line3  \nline1\n\nline3\nline2\nline1';
+      const options: DuplicateRemoverOptions = {
+        mode: 'all',
+        newlines: 'delete',
+        sortLines: true,
+        trimTextLines: true
+      };
+      const result = removeDuplicateLines(input, options);
+      expect(result).toBe('line1\nline2\nline3');
+    });
+  });
+
+  // Edge cases
+  describe('edge cases', () => {
+    it('should handle empty input', () => {
+      const input = '';
+      const options: DuplicateRemoverOptions = {
+        mode: 'all',
+        newlines: 'filter',
+        sortLines: false,
+        trimTextLines: false
+      };
+      const result = removeDuplicateLines(input, options);
+      expect(result).toBe('');
+    });
+
+    it('should handle input with only newlines', () => {
+      const input = '\n\n\n';
+      const options: DuplicateRemoverOptions = {
+        mode: 'all',
+        newlines: 'filter',
+        sortLines: false,
+        trimTextLines: false
+      };
+      const result = removeDuplicateLines(input, options);
+      expect(result).toBe('');
+    });
+
+    it('should handle input with only whitespace', () => {
+      const input = '   \n  \n    ';
+      const options: DuplicateRemoverOptions = {
+        mode: 'all',
+        newlines: 'filter',
+        sortLines: false,
+        trimTextLines: true
+      };
+      const result = removeDuplicateLines(input, options);
+      expect(result).toBe('');
+    });
+  });
+});

src/pages/tools/string/remove-duplicate-lines/service.ts

@@ -0,0 +1,88 @@
+export type NewlineOption = 'preserve' | 'filter' | 'delete';
+export type DuplicateRemovalMode = 'all' | 'consecutive' | 'unique';
+
+export interface DuplicateRemoverOptions {
+  mode: DuplicateRemovalMode;
+  newlines: NewlineOption;
+  sortLines: boolean;
+  trimTextLines: boolean;
+}
+
+/**
+ * Removes duplicate lines from text based on specified options
+ * @param text The input text to process
+ * @param options Configuration options for text processing
+ * @returns Processed text with duplicates removed according to options
+ */
+export default function removeDuplicateLines(
+  text: string,
+  options: DuplicateRemoverOptions
+): string {
+  // Split the text into individual lines
+  let lines = text.split('\n');
+
+  // Process newlines based on option
+  if (options.newlines === 'delete') {
+    // Remove all empty lines
+    lines = lines.filter((line) => line.trim() !== '');
+  }
+
+  // Trim lines if option is selected
+  if (options.trimTextLines) {
+    lines = lines.map((line) => line.trim());
+  }
+
+  // Remove duplicates based on mode
+  let processedLines: string[] = [];
+
+  if (options.mode === 'all') {
+    // Remove all duplicates, keeping only first occurrence
+    const seen = new Set<string>();
+    processedLines = lines.filter((line) => {
+      if (seen.has(line)) {
+        return false;
+      }
+      seen.add(line);
+      return true;
+    });
+  } else if (options.mode === 'consecutive') {
+    // Remove only consecutive duplicates
+    processedLines = lines.filter((line, index, arr) => {
+      return index === 0 || line !== arr[index - 1];
+    });
+  } else if (options.mode === 'unique') {
+    // Leave only absolutely unique lines
+    const lineCount = new Map<string, number>();
+    lines.forEach((line) => {
+      lineCount.set(line, (lineCount.get(line) || 0) + 1);
+    });
+
+    processedLines = lines.filter((line) => lineCount.get(line) === 1);
+  }
+
+  // Sort lines if option is selected
+  if (options.sortLines) {
+    processedLines.sort();
+  }
+
+  // Process newlines for output
+  if (options.newlines === 'filter') {
+    // Process newlines as regular lines (already done by default)
+  } else if (options.newlines === 'preserve') {
+    // Make sure empty lines are preserved in the output
+    processedLines = text.split('\n').map((line) => {
+      if (line.trim() === '') return line;
+      return processedLines.includes(line) ? line : '';
+    });
+  }
+
+  return processedLines.join('\n');
+}
+
+// Example usage:
+// const result = removeDuplicateLines(inputText, {
+//   mode: 'all',
+//   newlines: 'filter',
+//   sortLines: false,
+//   trimTextLines: true
+// });

src/tools/defineTool.tsx

@@ -6,7 +6,7 @@ interface ToolOptions {
   path: string;
   component: LazyExoticComponent<JSXElementConstructor<ToolComponentProps>>;
   keywords: string[];
-  icon?: IconifyIcon | string;
+  icon: IconifyIcon | string;
   name: string;
   description: string;
   shortDescription: string;
@@ -26,7 +26,7 @@ export interface DefinedTool {
   name: string;
   description: string;
   shortDescription: string;
-  icon?: IconifyIcon | string;
+  icon: IconifyIcon | string;
   keywords: string[];
   component: () => JSX.Element;
 }