Updated screenshot

README.md

@@ -1,6 +1,7 @@
 # xxSherly
 
-A fork of [Sherly](https://github.com/BlyDoesCoding/Sherly), using [xxHash](https://github.com/Cyan4973/xxHash).
+A fork of [Sherly](https://github.com/BlyDoesCoding/Sherly), using [xxHash](https://github.com/Cyan4973/xxHash).  
+This fork is faster, but has less features and may produce false-positives.
 
 ![](./images/screenshot.png)
 
@@ -8,22 +9,16 @@ A fork of [Sherly](https://github.com/BlyDoesCoding/Sherly), using [xxHash](http
 
 Sherly is a Multithreaded Duplicate File Finder for your Terminal, written in java. You can Easily find duplicate Images, videos as well as any other type of Data. That can be helpful if you run on small storage or just want to keep regular housekeeping.
 
-This fork uses [xxHash](https://github.com/Cyan4973/xxHash) instead of MD5 for performance reasons (see [Speed comparison](#speed-comparison)).
-Note that xxHash is not a cryptographic hash function and therefore may produce collisions. That's why the checksum is composed of the xxHash Digest and the filesize.
+Instead of md5, this fork uses [xxHash](https://github.com/Cyan4973/xxHash) + the filesize to find duplicates, for performance reasons (see [Speed comparison](#speed-comparison)).
+Note that xxHash is not a cryptographic hash function and therefore may produce collisions (false-positives). For this reason, since version 2.1, the program no longer offers the option to delete duplicates. You should delete them by yourself.
 
 ## Usage
 
-```console
-Usage: sherly -f inputfolder1 inputfolder2 inputfolder3 [options]...
- 
-   -h / -help             show this
-   -f / -folder           all the folders you want to scan for (see example above!)
-   -c / -color            enable colored messages
-   -t / -threads          override default Thread number (default is usually number of cores * 2)
-   -p / -progress         enable progress indicator
-   -d / -delete           delete all dups except one without asking first
-   -n / -noinput          skip all user input
-   -debug                 debug stuff
+```
+usage: xxSherly.jar [options] folder1 folder2 ...
+ -c,--color     enable colored output
+ -h,--help      show this help message
+ -v,--verbose   more verbose output
 ```
 
 ## Build
@@ -46,15 +41,19 @@ mvn package assembly:single
 I let Sherly and xxSherly find duplicates in my Music Library (containing `.wav` files) using the following commands:
 
 ```bash
+# Sherly v1.1.4
 time java -jar Bin/sherly.jar -n -f ~/Music/
-time java -jar target/xxSherly-1.0-jar-with-dependencies.jar -n -f ~/Music/
+# xxSherly v2.1
+time java -jar target/xxSherly-2.1-jar-with-dependencies.jar ~/Music/
+# xxSherly v3.0
+time java -jar target/xxSherly-3.0-jar-with-dependencies.jar ~/Music/
 ```
 
 The timings are measured using the Linux tool `time` (`real`).
 
-|           |        Sherly |        xxSherly |
-| --------: | ------------: | --------------: |
-|  1st run  |        4.055s |          2.561s |
-|  2nd run  |        4.055s |          2.304s |
-|  3rd run  |        4.066s |          2.549s |
-|  **avg**  |    **4.059s** |      **2.471s** |
+|           | Sherly v1.1.4 | xxSherly v2.1 | xxSherly v3.0 |
+| --------: | ------------: | ------------: | ------------: |
+|  1st run  |        4.055s |        2.554s |        2.086s |
+|  2nd run  |        4.055s |        2.554s |        2.109s |
+|  3rd run  |        4.066s |        2.556s |        2.092s |
+|  **avg**  |    **4.059s** |    **2.555s** |    **2.096s** |

pom.xml

@@ -6,7 +6,7 @@
 
   <groupId>net.chaoticbyte.xxsherly</groupId>
   <artifactId>xxSherly</artifactId>
-  <version>1.0</version>
+  <version>3.0</version>
 
   <name>xxSherly</name>
   <!-- FIXME change it to the project's website -->
@@ -25,6 +25,11 @@
       <artifactId>commons-codec</artifactId>
       <version>1.15</version>
     </dependency>
+    <dependency>
+      <groupId>commons-cli</groupId>
+      <artifactId>commons-cli</artifactId>
+      <version>1.5.0</version>
+    </dependency>
   </dependencies>
 
   <build>

src/main/java/net/chaoticbyte/xxsherly/App.java

@@ -1,213 +1,165 @@
 package net.chaoticbyte.xxsherly;
 
+import java.io.File;
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
-import java.nio.file.Paths;
 import java.util.ArrayList;
-import java.util.HashMap;
 import java.util.List;
-import java.util.Scanner;
-import java.util.concurrent.TimeUnit;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.DefaultParser;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
 
 public class App {
 
-    public static int completedThreads = 0;
-    public static int progress = 0;
-    public static HashMap<String, List<Path>> fileMap = new HashMap<>();
+    public static final String usageHelp = "xxSherly.jar [options] folder1 folder2 ...";
+
+    public static boolean doTheColorThingy = false;
+    public static boolean verbose = false;
 
     public static void main(String[] args) throws InterruptedException {
-        boolean doTheColorThingy = false;
-        boolean showProgress = false;
-        boolean deleteDups = false;
-        boolean recordFolder = false;
-        boolean recordThreads = false;
-        int saidThreads = 0;
-        boolean showDebug = false;
-        boolean noInput = false;
-        boolean help = false;
 
-        List<String> paths = new ArrayList<>();
+        // CLI
 
-        for(String i : args) {
-            if (recordFolder) {
-                if(Files.isDirectory(Paths.get(i))) {
-                    paths.add(i);
-                } else {recordFolder = false;}
+        List<File> folderList = new ArrayList<>();
+        boolean displayHelp = false;
+
+        HelpFormatter helpFormatter = new HelpFormatter();
+
+        Options commandlineOptions = new Options();
+        commandlineOptions.addOption("c", "color", false, "enable colored output");
+        commandlineOptions.addOption("v", "verbose", false, "more verbose output");
+        commandlineOptions.addOption("h", "help", false, "show this help message");
+
+        try {
+            CommandLine arguments = new DefaultParser().parse(commandlineOptions, args, false);
+            // Get folder paths
+            for (String folderArgument : arguments.getArgList()) {
+                File folder = new File(folderArgument);
+                if (folder.isDirectory() && folder.canRead()) folderList.add(folder);
+                else System.err.println(folderArgument + " is not a folder or isn't readable.");
             }
-            if (recordThreads) {
-                saidThreads = Integer.parseInt(i);
-                recordThreads = false;
-            }
-            if (i.equalsIgnoreCase("-c") || i.equalsIgnoreCase("-color")) { doTheColorThingy = true;}
-            if (i.equalsIgnoreCase("-p") || i.equalsIgnoreCase("-progress")) { showProgress = true;}
-            if (i.equalsIgnoreCase("-f") || i.equalsIgnoreCase("-folder")) { recordFolder = true;}
-            if (i.equalsIgnoreCase("-t") || i.equalsIgnoreCase("-threads")) { recordThreads = true;}
-            if (i.equalsIgnoreCase("-d") || i.equalsIgnoreCase("-delete")) { deleteDups = true;}
-            if (i.equalsIgnoreCase("-n") || i.equalsIgnoreCase("-noinput")) { noInput = true; }
-            if (i.equalsIgnoreCase("-h") || i.equalsIgnoreCase("-help")) { help = true;}
-            if (i.equalsIgnoreCase("-debug")) { showDebug = true;}
-
+            // Get arguments & options
+            doTheColorThingy = arguments.hasOption("c");
+            verbose = arguments.hasOption("v");
+            displayHelp = arguments.hasOption("h");
         }
-
-        if (help) {
-            System.out.println("Usage: sherly -f inputfolder1 inputfolder2 inputfolder3 [options]...");
-            System.out.println(" ");
-            System.out.println("   -h / -help             show this");
-            System.out.println("   -f / -folder           all the folders you want to scan for (see example above!)");
-            System.out.println("   -c / -color            enable colored messages");
-            System.out.println("   -t / -threads          override default Thread number (default is usually number of cores * 2)");
-            System.out.println("   -p / -progress         enable progress indicator");
-            System.out.println("   -d / -delete           delete all dups except one without asking first");
-            System.out.println("   -n / -noinput          skip all user input");
-            System.out.println("   -debug                 debug stuff");
-            return;
-        }
-        if (paths.size() == 0) {
-            System.out.println("Aborted, no Folders Found!");
+        catch (ParseException | NumberFormatException e) {
+            helpFormatter.printHelp(usageHelp, commandlineOptions);
+            System.err.println();
+            System.err.println(e.getMessage());
             return;
         }
 
-        if (showDebug) {
-            System.out.println("Folders: " + paths.size());
-            System.out.println("Color: " + doTheColorThingy);
-            System.out.println("Delete: " + deleteDups);
-            System.out.println("Progressbar: " + showProgress);
-            System.out.println("Commanded Threads " + saidThreads);
+        if (displayHelp) {
+            helpFormatter.printHelp(usageHelp, commandlineOptions);
+            return;
         }
 
-        List<Path> pathList = new ArrayList<>();
-        List<Path> allFiles = new ArrayList<>();
+        if (folderList.size() < 1) {
+            System.err.println("No valid folders specified.");
+            helpFormatter.printHelp(usageHelp, commandlineOptions);
+            return;
+        }
 
-        for (String folder : paths) {
-            try (Stream<Path> stream = Files.walk(Paths.get(folder))) {
-                pathList = stream.map(Path::normalize).filter(Files::isRegularFile).collect(Collectors.toList());
-            } catch (IOException e) {
-                e.printStackTrace();
+        if (verbose) {
+            System.out.println("Arguments:");;
+            System.out.println("  Folders:  " + folderList.size());
+            System.out.println("  Color:    " + doTheColorThingy);
+        }
+
+        // Find all files
+        List<File> files = new ArrayList<>();
+        for (File folder : folderList) {
+            try (Stream<Path> stream = Files.walk(folder.toPath())) {
+                List<Path> filePaths = stream
+                    .filter(Files::isReadable)
+                    .filter(Files::isRegularFile)
+                    .filter(f -> !Files.isSymbolicLink(f))
+                    .collect(Collectors.toList());
+                filePaths.forEach((filePath) -> {
+                    files.add(filePath.toFile());
+                });
+            }
+            catch (IOException e) {
+                System.out.println(e.getMessage());
+                return;
             }
-            allFiles.addAll(pathList);
         }
+        int nFiles = files.size();
+        if (verbose) System.out.println("Files: " + nFiles);
 
-        // calculations for multithreading
-        //The number of Cores or better said Threads that can be used
-        int availableThreads = Runtime.getRuntime().availableProcessors();
-        if (saidThreads != 0) {availableThreads = saidThreads;}
+        // Calculate Hashes
 
-        //just the number of All Files in all Folders taken from the Args
-        int filesToBeDone = allFiles.size();
+        ConcurrentHashMap<String, List<File>> fileMap = new ConcurrentHashMap<>();
 
-        //Every Thread that is going to be started gets a range of files
-        //They are seperated and are called sections
-        int sections = filesToBeDone / availableThreads;
+        files.parallelStream().forEach(file -> {
 
-        for (int i = 1; i <= availableThreads; i++) {
-
-            List<Path> sectionedList = new ArrayList<>();
-
-            //Here the different Threads are being started
-            //Usually the separation gives the first threads the same number of files to be working on and the last one is given all the files that could not be separetated
-            if (i == availableThreads) {
-                for (int x = (sections * i) - (sections); x < filesToBeDone; x++) {
-                    sectionedList.add(allFiles.get(x));
-                }
-            } else {
-                for (int x = (sections * i) - (sections); x < (sections * i); x++) {
-                    sectionedList.add(allFiles.get(x));
+            List<File> fileArray = new ArrayList<>();
+            assert fileArray != null;
+            fileArray.add(file);
 
+            // Generate Checksum
+            try {
+                String checksum = FileChecksum.getChecksum(file);
+                if (fileMap.containsKey(checksum)) {
+                    fileArray.addAll(fileMap.get(checksum));
+                    fileMap.put(checksum, fileArray);
+                } else {
+                    fileMap.put(checksum, fileArray);
                 }
             }
-
-            //Start Multithreading
-            //sectionedList gives the thread their Assigned Part of Files
-            ThreadedCompare threadedCompare = new ThreadedCompare(sectionedList);
-            threadedCompare.start();
-
-        }
-
-        //this updates if necessary the Progress bar and checks for Finished threads
-
-        while (completedThreads < availableThreads) {
-            TimeUnit.MILLISECONDS.sleep(250);
-
-            if (showProgress && doTheColorThingy) {
-                System.out.print(ConsoleColors.BLUE_BOLD + "Progress: " + ConsoleColors.GREEN_BOLD + progress + " / " + filesToBeDone + " | " + (progress * 100 / filesToBeDone) + "%" + ConsoleColors.RESET + "\r");
-            } else if (showProgress) {
-                System.out.print("Progress: " + progress + " / " + filesToBeDone + " | " + (progress * 100 / filesToBeDone) + "%" + "\r");
+            catch (IOException e) {
+                System.err.println("An exception occured while processing the file " + file.getPath());
+                System.err.println(e.getMessage());
             }
-        }
+        });
 
         ArrayList<String> toRemove = new ArrayList<String>();
         for (String checksum: fileMap.keySet()) {
-            if (App.fileMap.get(checksum).size() == 1) {
+            if (fileMap.get(checksum).size() == 1) {
                 toRemove.add(checksum);
             }
         }
         fileMap.keySet().removeAll(toRemove);
 
-        // now everything is finished and the Filemap (hashmap with all Dups) can be printed out in a nice view
+        // Now everything is finished and the Filemap (hashmap with all Dups) can be printed out in a nice view
 
-        for (String checksum: fileMap.keySet()) {
-
-            if (doTheColorThingy) {
-                System.out.println(ConsoleColors.BLUE_BOLD + checksum + ConsoleColors.CYAN_BOLD + "\t--> " + ConsoleColors.GREEN_BOLD + fileMap.get(checksum) + ConsoleColors.RESET);
-
-            } else {
-                System.out.println(checksum +"\t--> " + fileMap.get(checksum));
+        if (fileMap.size() > 0) {
+            System.out.println();
+            for (String checksum: fileMap.keySet()) {
+                if (doTheColorThingy) {
+                    System.out.println(
+                        ConsoleColors.BLUE_BOLD + checksum
+                        + ConsoleColors.CYAN_BOLD + "\t--> "
+                        + ConsoleColors.GREEN_BOLD + fileMap.get(checksum)
+                        + ConsoleColors.RESET);
+                } else System.out.println(checksum +"\t--> " + fileMap.get(checksum));
             }
-
+            System.out.println();
         }
 
-        List<Path> allTheFilesWillBeDeleted = new ArrayList<>();
+        // Count redundant files and bytes
 
+        int toBeDeleted = 0;
         long bytes = 0;
-
-        for (String md5: fileMap.keySet()) {
-            App.fileMap.get(md5).remove(0);
-            for (Path file: App.fileMap.get(md5)) {
-                if (file != null) {
-                    bytes += file.toFile().length();
-                }
+        for (String checksum: fileMap.keySet()) {
+            fileMap.get(checksum).remove(0);
+            for (File file: fileMap.get(checksum)) {
+                if (file != null) bytes += file.length();
             }
-            allTheFilesWillBeDeleted.addAll(App.fileMap.get(md5));
-        }
-
-        if (deleteDups) {
-            delete(allTheFilesWillBeDeleted);
-        } else if (!noInput) {
-            ask(doTheColorThingy, bytes, allTheFilesWillBeDeleted);
+            toBeDeleted++;
         }
 
+        if (doTheColorThingy) {
+            String color = ConsoleColors.RED_BOLD;
+            if (toBeDeleted < 1) color = ConsoleColors.GREEN_BOLD;
+            System.out.println(color + (bytes / 1000000.0) + " redundant MB in " + toBeDeleted + " file(s) found." + ConsoleColors.RESET);
+        } else System.out.println((bytes / 1000000.0) + " redundant MB in " + toBeDeleted + " file(s) found.");
     }
-
-    // print files and ask user
-    public static void ask(boolean color, long bytes, List<Path> deleteThem) {
-        if (color) {
-            System.out.println(ConsoleColors.RED_BOLD + (bytes / 8000000) + " unnecessary MB in " + deleteThem.size() + " Files found, do you want to Delete them? Y / N" + ConsoleColors.RESET);
-        } else {
-            System.out.println((bytes / 8000000) + " unnecessary MB in " + deleteThem.size() + " Files found, do you want to Delete them? Y / N");
-        }
-        Scanner input = new Scanner(System.in);
-        String answer = input.next();
-        if (answer.toLowerCase().contains("y")) {
-            delete(deleteThem);
-            input.close();
-
-        } else if (answer.toLowerCase().contains("n")) {
-            input.close();
-            return;
-        } else {
-            ask(color, bytes, deleteThem);
-        }
-        input.close();
-    }
-
-    public static void delete(List<Path> deleteThem) {
-        for (Path file : deleteThem) {
-            if (file != null) {file.toFile().delete();}
-
-        }
-    }
-
 }

src/main/java/net/chaoticbyte/xxsherly/FileChecksum.java

@@ -0,0 +1,31 @@
+package net.chaoticbyte.xxsherly;
+
+import java.io.*;
+import java.util.zip.Checksum;
+import org.apache.commons.codec.digest.XXHash32;
+
+public class FileChecksum {
+
+    //this is used to get the MD5 String of one of the files (one of them is just fine since they both have the same value)
+    public static String getChecksum (File file) throws IOException {
+
+        String digest = "";
+
+        // Calculate xxHash32 and add it's hexadecimal presentation to the digest
+        Checksum xxHash = new XXHash32();
+        FileInputStream inputStream = new FileInputStream(file);
+        byte[] dataBytes = new byte[1024];
+        int unread = 0;
+        while ((unread = inputStream.read(dataBytes)) != -1) {
+            xxHash.update(dataBytes, 0, unread);
+        }
+        inputStream.close();
+        digest += Long.toHexString(xxHash.getValue());
+
+        // Add File length to the digest
+        digest += Long.toHexString(file.length());
+
+        // return result
+        return digest;
+    }
+}

src/main/java/net/chaoticbyte/xxsherly/ThreadedCompare.java

@@ -1,65 +0,0 @@
-package net.chaoticbyte.xxsherly;
-
-import java.io.*;
-import java.nio.file.Path;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.zip.Checksum;
-import org.apache.commons.codec.digest.XXHash32;
-
-public class ThreadedCompare extends Thread {
-
-    private final List<Path> pathsToCompareTo;
-
-    public ThreadedCompare (List<Path> pathsToCompareTo) {
-        this.pathsToCompareTo = pathsToCompareTo;
-    }
-
-    @Override
-    public void run() {
-        for (Path file : pathsToCompareTo) {
-            List<Path> fileArray = new ArrayList<>();
-            assert fileArray != null;
-            fileArray.add(file);
-            String checksum;
-            try {
-                checksum = getChecksum(file.toFile());
-            } catch (IOException e) {
-                throw new RuntimeException(e);
-            }
-            if (App.fileMap.containsKey(checksum)) {
-                fileArray.addAll(App.fileMap.get(checksum));
-                App.fileMap.put(checksum, fileArray);
-            } else {
-                App.fileMap.put(checksum, fileArray);
-            }
-            App.progress++;
-        }
-        App.completedThreads++;
-    }
-
-    //this is used to get the MD5 String of one of the files (one of them is just fine since they both have the same value)
-    private String getChecksum (File file) throws IOException {
-
-        String digest = "";
-
-        // Calculate xxHash32 and add it's hexadecimal presentation to the digest
-
-        Checksum xxHash = new XXHash32();
-        FileInputStream inputStream = new FileInputStream(file);
-        byte[] dataBytes = new byte[1024];
-        int unread = 0;
-        while ((unread = inputStream.read(dataBytes)) != -1) {
-            xxHash.update(dataBytes, 0, unread);
-        }
-        inputStream.close();
-        digest += Long.toHexString(xxHash.getValue());
-
-        // Add File length to the digest
-
-        digest += Long.toHexString(file.length());
-
-        // return result
-        return digest;
-    }
-}