ChaoticByte/xxSherly
Archived
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Changed commandline arguments and how they are parsed, refactored the code, updated the README, improved output

Browse source
This commit is contained in:
Julian Müller (ChaoticByte) 2023-05-06 20:01:17 +02:00
parent 6ded43f515
commit 6f81a25573
4 changed files with 191 additions and 171 deletions
Download patch file Download diff file Expand all files Collapse all files

23
README.md
View file

@ -13,17 +13,16 @@ Note that xxHash is not a cryptographic hash function and therefore may produce
## Usage ## Usage
```console ```
Usage: sherly -f inputfolder1 inputfolder2 inputfolder3 [options]... usage: xxSherly.jar [options] folder1 folder2 ...
-c,--color enable colored output
-h / -help show this -d,--delete delete all dups except one, without asking first
-f / -folder all the folders you want to scan for (see example above!) -h,--help show this help message
-c / -color enable colored messages -n,--noinput skip all user input
-t / -threads override default Thread number (default is usually number of cores * 2) -p,--progress enable progress indicator
-p / -progress enable progress indicator -t,--threads <arg> override default thread number (defaults to the
-d / -delete delete all dups except one without asking first number of cores)
-n / -noinput skip all user input -v,--verbose more verbose output
-debug debug stuff
``` ```
## Build ## Build
@ -43,7 +42,7 @@ mvn package assembly:single
## Speed comparison ## Speed comparison
I let Sherly and xxSherly find duplicates in my Music Library (containing `.wav` files) using the following commands: I let Sherly v1.1.4 and xxSherly v1.0 find duplicates in my Music Library (containing `.wav` files) using the following commands:
```bash ```bash
time java -jar Bin/sherly.jar -n -f ~/Music/ time java -jar Bin/sherly.jar -n -f ~/Music/

5
pom.xml
View file

@ -25,6 +25,11 @@
<artifactId>commons-codec</artifactId> <artifactId>commons-codec</artifactId>
<version>1.15</version> <version>1.15</version>
</dependency> </dependency>
<dependency>
<groupId>commons-cli</groupId>
<artifactId>commons-cli</artifactId>
<version>1.5.0</version>
</dependency>
</dependencies> </dependencies>
<build> <build>

298
src/main/java/net/chaoticbyte/xxsherly/App.java
View file

@ -1,9 +1,9 @@
package net.chaoticbyte.xxsherly; package net.chaoticbyte.xxsherly;
import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.nio.file.Files; import java.nio.file.Files;
import java.nio.file.Path; import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
@ -11,130 +11,142 @@ import java.util.Scanner;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import java.util.stream.Stream; import java.util.stream.Stream;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.DefaultParser;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
public class App { public class App {
public static final String usageHelp = "xxSherly.jar [options] folder1 folder2 ...";
public static int completedThreads = 0; public static int completedThreads = 0;
public static int progress = 0; public static int progress = 0;
public static HashMap<String, List<Path>> fileMap = new HashMap<>(); public static HashMap<String, List<File>> fileMap = new HashMap<>();
public static boolean doTheColorThingy = false;
public static void main(String[] args) throws InterruptedException { public static void main(String[] args) throws InterruptedException {
boolean doTheColorThingy = false;
// Arguments
List<File> folderList = new ArrayList<>();
boolean showProgress = false; boolean showProgress = false;
boolean deleteDups = false; boolean deleteDups = false;
boolean recordFolder = false; boolean verbose = false;
boolean recordThreads = false;
int saidThreads = 0;
boolean showDebug = false;
boolean noInput = false; boolean noInput = false;
boolean help = false; boolean displayHelp = false;
int requestedThreads = 0;
List<String> paths = new ArrayList<>(); // CLI
for(String i : args) { HelpFormatter helpFormatter = new HelpFormatter();
if (recordFolder) {
if(Files.isDirectory(Paths.get(i))) { Options commandlineOptions = new Options();
paths.add(i); commandlineOptions.addOption("c", "color", false, "enable colored output");
} else {recordFolder = false;} commandlineOptions.addOption("t", "threads", true, "override default thread number (defaults to the number of cores)");
commandlineOptions.addOption("p", "progress", false, "enable progress indicator");
commandlineOptions.addOption("d", "delete", false, "delete all dups except one, without asking first");
commandlineOptions.addOption("n", "noinput", false, "skip all user input");
commandlineOptions.addOption("v", "verbose", false, "more verbose output");
commandlineOptions.addOption("h", "help", false, "show this help message");
try {
CommandLine arguments = new DefaultParser().parse(commandlineOptions, args, false);
// Get folder paths
for (String folderArgument : arguments.getArgList()) {
File folder = new File(folderArgument);
if (folder.isDirectory() && folder.canRead()) folderList.add(folder);
else System.err.println(folderArgument + " is not a folder or isn't readable.");
} }
if (recordThreads) { // Get arguments & options
saidThreads = Integer.parseInt(i); doTheColorThingy = arguments.hasOption("c");
recordThreads = false; showProgress = arguments.hasOption("p");
} deleteDups = arguments.hasOption("d");
if (i.equalsIgnoreCase("-c") || i.equalsIgnoreCase("-color")) { doTheColorThingy = true;} verbose = arguments.hasOption("v");
if (i.equalsIgnoreCase("-p") || i.equalsIgnoreCase("-progress")) { showProgress = true;} noInput = arguments.hasOption("n");
if (i.equalsIgnoreCase("-f") || i.equalsIgnoreCase("-folder")) { recordFolder = true;} displayHelp = arguments.hasOption("h");
if (i.equalsIgnoreCase("-t") || i.equalsIgnoreCase("-threads")) { recordThreads = true;} requestedThreads = Integer.parseInt(arguments.getOptionValue("t", "0"));
if (i.equalsIgnoreCase("-d") || i.equalsIgnoreCase("-delete")) { deleteDups = true;}
if (i.equalsIgnoreCase("-n") || i.equalsIgnoreCase("-noinput")) { noInput = true; }
if (i.equalsIgnoreCase("-h") || i.equalsIgnoreCase("-help")) { help = true;}
if (i.equalsIgnoreCase("-debug")) { showDebug = true;}
} }
catch (ParseException | NumberFormatException e) {
if (help) { helpFormatter.printHelp(usageHelp, commandlineOptions);
System.out.println("Usage: sherly -f inputfolder1 inputfolder2 inputfolder3 [options]..."); System.err.println();
System.out.println(" "); System.err.println(e.getMessage());
System.out.println(" -h / -help show this");
System.out.println(" -f / -folder all the folders you want to scan for (see example above!)");
System.out.println(" -c / -color enable colored messages");
System.out.println(" -t / -threads override default Thread number (default is usually number of cores * 2)");
System.out.println(" -p / -progress enable progress indicator");
System.out.println(" -d / -delete delete all dups except one without asking first");
System.out.println(" -n / -noinput skip all user input");
System.out.println(" -debug debug stuff");
return;
}
if (paths.size() == 0) {
System.out.println("Aborted, no Folders Found!");
return; return;
} }
if (showDebug) { if (displayHelp) {
System.out.println("Folders: " + paths.size()); helpFormatter.printHelp(usageHelp, commandlineOptions);
System.out.println("Color: " + doTheColorThingy); return;
System.out.println("Delete: " + deleteDups);
System.out.println("Progressbar: " + showProgress);
System.out.println("Commanded Threads " + saidThreads);
} }
List<Path> pathList = new ArrayList<>(); if (folderList.size() < 1) {
List<Path> allFiles = new ArrayList<>(); System.err.println("No valid folders specified.");
helpFormatter.printHelp(usageHelp, commandlineOptions);
for (String folder : paths) { return;
try (Stream<Path> stream = Files.walk(Paths.get(folder))) {
pathList = stream.map(Path::normalize).filter(Files::isRegularFile).collect(Collectors.toList());
} catch (IOException e) {
e.printStackTrace();
}
allFiles.addAll(pathList);
} }
// calculations for multithreading if (verbose) {
//The number of Cores or better said Threads that can be used System.out.println("Arguments:");;
int availableThreads = Runtime.getRuntime().availableProcessors(); System.out.println(" Folders: " + folderList.size());
if (saidThreads != 0) {availableThreads = saidThreads;} System.out.println(" Color: " + doTheColorThingy);
System.out.println(" Delete: " + deleteDups);
System.out.println(" Progress: " + showProgress);
}
//just the number of All Files in all Folders taken from the Args // Calculations for multithreading
int filesToBeDone = allFiles.size(); // The number of Cores or better said Threads that can be used
int availableProcessors = Runtime.getRuntime().availableProcessors();
int nThreads = availableProcessors;
if (requestedThreads > 0) nThreads = requestedThreads;
if (verbose) System.out.println("Threads: " + nThreads);
//Every Thread that is going to be started gets a range of files // Find all files
//They are seperated and are called sections List<File> files = new ArrayList<>();
int sections = filesToBeDone / availableThreads; for (File folder : folderList) {
try (Stream<Path> stream = Files.walk(folder.toPath())) {
for (int i = 1; i <= availableThreads; i++) { List<Path> filePaths = stream
.filter(Files::isReadable)
List<Path> sectionedList = new ArrayList<>(); .filter(Files::isRegularFile)
.filter(f -> !Files.isSymbolicLink(f))
//Here the different Threads are being started .collect(Collectors.toList());
//Usually the separation gives the first threads the same number of files to be working on and the last one is given all the files that could not be separetated filePaths.forEach((filePath) -> {
if (i == availableThreads) { files.add(filePath.toFile());
for (int x = (sections * i) - (sections); x < filesToBeDone; x++) { });
sectionedList.add(allFiles.get(x));
}
} else {
for (int x = (sections * i) - (sections); x < (sections * i); x++) {
sectionedList.add(allFiles.get(x));
}
} }
catch (IOException e) {
System.out.println(e.getMessage());
return;
}
}
int nFiles = files.size();
if (verbose) System.out.println("Files: " + nFiles);
//Start Multithreading // Every Thread that is going to be started gets a range of files
//sectionedList gives the thread their Assigned Part of Files // They are seperated and are called sections
int sections = nFiles / nThreads;
for (int i = 1; i <= nThreads; i++) {
List<File> sectionedList = new ArrayList<>();
// Here the different Threads are being started
// Usually the separation gives the first threads the same number of files to be working on and the last one is given all the files that could not be separetated
if (i == nThreads) for (int x = (sections * i) - (sections); x < nFiles; x++) {
sectionedList.add(files.get(x));
} else for (int x = (sections * i) - (sections); x < (sections * i); x++) {
sectionedList.add(files.get(x));
}
// Start Multithreading
// sectionedList gives the thread their Assigned Part of Files
ThreadedCompare threadedCompare = new ThreadedCompare(sectionedList); ThreadedCompare threadedCompare = new ThreadedCompare(sectionedList);
threadedCompare.start(); threadedCompare.start();
} }
//this updates if necessary the Progress bar and checks for Finished threads // This updates if necessary the Progress bar and checks for Finished threads
while (completedThreads < nThreads) {
while (completedThreads < availableThreads) {
TimeUnit.MILLISECONDS.sleep(250); TimeUnit.MILLISECONDS.sleep(250);
if (showProgress && doTheColorThingy) { if (showProgress && doTheColorThingy) {
System.out.print(ConsoleColors.BLUE_BOLD + "Progress: " + ConsoleColors.GREEN_BOLD + progress + " / " + filesToBeDone + " | " + (progress * 100 / filesToBeDone) + "%" + ConsoleColors.RESET + "\r"); System.out.print(ConsoleColors.BLUE_BOLD + "Progress: " + ConsoleColors.GREEN_BOLD + progress + " / " + nFiles + " | " + (progress * 100 / nFiles) + "%" + ConsoleColors.RESET + "\r");
} else if (showProgress) { } else if (showProgress) {
System.out.print("Progress: " + progress + " / " + filesToBeDone + " | " + (progress * 100 / filesToBeDone) + "%" + "\r"); System.out.print("Progress: " + progress + " / " + nFiles + " | " + (progress * 100 / nFiles) + "%" + "\r");
} }
} }
@ -146,68 +158,70 @@ public class App {
} }
fileMap.keySet().removeAll(toRemove); fileMap.keySet().removeAll(toRemove);
// now everything is finished and the Filemap (hashmap with all Dups) can be printed out in a nice view // Now everything is finished and the Filemap (hashmap with all Dups) can be printed out in a nice view
if (fileMap.size() > 0) System.out.println();
for (String checksum: fileMap.keySet()) { for (String checksum: fileMap.keySet()) {
if (doTheColorThingy) { if (doTheColorThingy) {
System.out.println(ConsoleColors.BLUE_BOLD + checksum + ConsoleColors.CYAN_BOLD + "\t--> " + ConsoleColors.GREEN_BOLD + fileMap.get(checksum) + ConsoleColors.RESET); System.out.println(
ConsoleColors.BLUE_BOLD + checksum
} else { + ConsoleColors.CYAN_BOLD + "\t--> "
System.out.println(checksum +"\t--> " + fileMap.get(checksum)); + ConsoleColors.GREEN_BOLD + fileMap.get(checksum)
} + ConsoleColors.RESET);
} else System.out.println(checksum +"\t--> " + fileMap.get(checksum));
} }
if (fileMap.size() > 0) System.out.println();
List<Path> allTheFilesWillBeDeleted = new ArrayList<>(); List<File> toBeDeleted = new ArrayList<>();
long bytes = 0; long bytes = 0;
for (String checksum: fileMap.keySet()) {
for (String md5: fileMap.keySet()) { App.fileMap.get(checksum).remove(0);
App.fileMap.get(md5).remove(0); for (File file: App.fileMap.get(checksum)) {
for (Path file: App.fileMap.get(md5)) { if (file != null) bytes += file.length();
if (file != null) {
bytes += file.toFile().length();
}
} }
allTheFilesWillBeDeleted.addAll(App.fileMap.get(md5)); toBeDeleted.addAll(App.fileMap.get(checksum));
} }
if (doTheColorThingy) {
String color = ConsoleColors.RED_BOLD;
if (fileMap.size() < 1) color = ConsoleColors.GREEN_BOLD;
System.out.println(color + (bytes / 1000000.0) + " unnecessary MB in " + toBeDeleted.size() + " file(s) found." + ConsoleColors.RESET);
} else System.out.println((bytes / 1000000.0) + " unnecessary MB in " + toBeDeleted.size() + " file(s) found.");
// Don't go further if there is nothing to delete
if (fileMap.size() < 1) return;
if (deleteDups) { if (deleteDups) {
delete(allTheFilesWillBeDeleted); System.out.println();
delete(toBeDeleted);
} else if (!noInput) { } else if (!noInput) {
ask(doTheColorThingy, bytes, allTheFilesWillBeDeleted); // Ask if the user wants to delete the file
} Scanner input = new Scanner(System.in);
while (true) {
} if (doTheColorThingy) System.out.print(ConsoleColors.RED_BOLD + "Do you want to delete them? [y/n] " + ConsoleColors.RESET);
else System.out.print("Do you want to delete them? [y/n] ");
// print files and ask user String answer = input.next();
public static void ask(boolean color, long bytes, List<Path> deleteThem) { if (answer.toLowerCase().contains("y")) {
if (color) { System.out.println();
System.out.println(ConsoleColors.RED_BOLD + (bytes / 8000000) + " unnecessary MB in " + deleteThem.size() + " Files found, do you want to Delete them? Y / N" + ConsoleColors.RESET); delete(toBeDeleted);
} else { break;
System.out.println((bytes / 8000000) + " unnecessary MB in " + deleteThem.size() + " Files found, do you want to Delete them? Y / N"); }
} else if (answer.toLowerCase().contains("n")) break;
Scanner input = new Scanner(System.in); }
String answer = input.next();
if (answer.toLowerCase().contains("y")) {
delete(deleteThem);
input.close(); input.close();
} else if (answer.toLowerCase().contains("n")) {
input.close();
return;
} else {
ask(color, bytes, deleteThem);
}
input.close();
}
public static void delete(List<Path> deleteThem) {
for (Path file : deleteThem) {
if (file != null) {file.toFile().delete();}
} }
} }
public static void delete(List<File> fileList) {
for (File file : fileList) if (file != null) {
if (file.delete()) {
if (doTheColorThingy) System.out.println(ConsoleColors.RED_BOLD + "Deleted " + file.toPath() + ConsoleColors.RESET);
else System.out.println("Deleted " + file.toPath());
}
else {
if (doTheColorThingy) System.err.println(ConsoleColors.RED_BOLD + "Couldn't delete " + ConsoleColors.RESET + file.toPath());
else System.err.println("Couldn't delete " + file.toPath());
}
}
}
} }

36
src/main/java/net/chaoticbyte/xxsherly/ThreadedCompare.java
View file

@ -1,7 +1,6 @@
package net.chaoticbyte.xxsherly; package net.chaoticbyte.xxsherly;
import java.io.*; import java.io.*;
import java.nio.file.Path;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.zip.Checksum; import java.util.zip.Checksum;
@ -9,30 +8,35 @@ import org.apache.commons.codec.digest.XXHash32;
public class ThreadedCompare extends Thread { public class ThreadedCompare extends Thread {
private final List<Path> pathsToCompareTo; private final List<File> filesToCompare;
public ThreadedCompare (List<Path> pathsToCompareTo) { public ThreadedCompare (List<File> pathsToCompare_) {
this.pathsToCompareTo = pathsToCompareTo; this.filesToCompare = pathsToCompare_;
} }
@Override @Override
public void run() { public void run() {
for (Path file : pathsToCompareTo) { for (File file : filesToCompare) {
List<Path> fileArray = new ArrayList<>();
List<File> fileArray = new ArrayList<>();
assert fileArray != null; assert fileArray != null;
fileArray.add(file); fileArray.add(file);
String checksum;
// Generate Checksum
try { try {
checksum = getChecksum(file.toFile()); String checksum = getChecksum(file);
} catch (IOException e) { if (App.fileMap.containsKey(checksum)) {
throw new RuntimeException(e); fileArray.addAll(App.fileMap.get(checksum));
App.fileMap.put(checksum, fileArray);
} else {
App.fileMap.put(checksum, fileArray);
}
} }
if (App.fileMap.containsKey(checksum)) { catch (IOException e) {
fileArray.addAll(App.fileMap.get(checksum)); System.err.println("An exception occured while processing the file " + file.getPath());
App.fileMap.put(checksum, fileArray); System.err.println(e.getMessage());
} else {
App.fileMap.put(checksum, fileArray);
} }
App.progress++; App.progress++;
} }
App.completedThreads++; App.completedThreads++;
@ -44,7 +48,6 @@ public class ThreadedCompare extends Thread {
String digest = ""; String digest = "";
// Calculate xxHash32 and add it's hexadecimal presentation to the digest // Calculate xxHash32 and add it's hexadecimal presentation to the digest
Checksum xxHash = new XXHash32(); Checksum xxHash = new XXHash32();
FileInputStream inputStream = new FileInputStream(file); FileInputStream inputStream = new FileInputStream(file);
byte[] dataBytes = new byte[1024]; byte[] dataBytes = new byte[1024];
@ -56,7 +59,6 @@ public class ThreadedCompare extends Thread {
digest += Long.toHexString(xxHash.getValue()); digest += Long.toHexString(xxHash.getValue());
// Add File length to the digest // Add File length to the digest
digest += Long.toHexString(file.length()); digest += Long.toHexString(file.length());
// return result // return result