commit e0623bc399ccab32c45200839b1fb1bb82dfe3e1 Author: bly Date: Tue Jan 3 00:50:03 2023 +0100 First Creation diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..f68d109 --- /dev/null +++ b/.gitignore @@ -0,0 +1,29 @@ +### IntelliJ IDEA ### +out/ +!**/src/main/**/out/ +!**/src/test/**/out/ + +### Eclipse ### +.apt_generated +.classpath +.factorypath +.project +.settings +.springBeans +.sts4-cache +bin/ +!**/src/main/**/bin/ +!**/src/test/**/bin/ + +### NetBeans ### +/nbproject/private/ +/nbbuild/ +/dist/ +/nbdist/ +/.nb-gradle/ + +### VS Code ### +.vscode/ + +### Mac OS ### +.DS_Store \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml new file mode 100644 index 0000000..7464918 --- /dev/null +++ b/.idea/misc.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 0000000..bfa89a1 --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 0000000..94a25f7 --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/.idea/workspace.xml b/.idea/workspace.xml new file mode 100644 index 0000000..84508ae --- /dev/null +++ b/.idea/workspace.xml @@ -0,0 +1,50 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + 1672703116568 + + + + \ No newline at end of file diff --git a/Sherly.iml b/Sherly.iml new file mode 100644 index 0000000..c90834f --- /dev/null +++ b/Sherly.iml @@ -0,0 +1,11 @@ + + + + + + + + + + + \ No newline at end of file diff --git a/src/ConsoleColors.java b/src/ConsoleColors.java new file mode 100644 index 0000000..65f546a --- /dev/null +++ b/src/ConsoleColors.java @@ -0,0 +1,77 @@ +//Just regular Colors to make it a bit prettier + +public class ConsoleColors { + // Reset + public static final String RESET = "\033[0m"; // Text Reset + + // Regular Colors + public static final String BLACK = "\033[0;30m"; // BLACK + public static final String RED = "\033[0;31m"; // RED + public static final String GREEN = "\033[0;32m"; // GREEN + public static final String YELLOW = "\033[0;33m"; // YELLOW + public static final String BLUE = "\033[0;34m"; // BLUE + public static final String PURPLE = "\033[0;35m"; // PURPLE + public static final String CYAN = "\033[0;36m"; // CYAN + public static final String WHITE = "\033[0;37m"; // WHITE + + // Bold + public static final String BLACK_BOLD = "\033[1;30m"; // BLACK + public static final String RED_BOLD = "\033[1;31m"; // RED + public static final String GREEN_BOLD = "\033[1;32m"; // GREEN + public static final String YELLOW_BOLD = "\033[1;33m"; // YELLOW + public static final String BLUE_BOLD = "\033[1;34m"; // BLUE + public static final String PURPLE_BOLD = "\033[1;35m"; // PURPLE + public static final String CYAN_BOLD = "\033[1;36m"; // CYAN + public static final String WHITE_BOLD = "\033[1;37m"; // WHITE + + // Underline + public static final String BLACK_UNDERLINED = "\033[4;30m"; // BLACK + public static final String RED_UNDERLINED = "\033[4;31m"; // RED + public static final String GREEN_UNDERLINED = "\033[4;32m"; // GREEN + public static final String YELLOW_UNDERLINED = "\033[4;33m"; // YELLOW + public static final String BLUE_UNDERLINED = "\033[4;34m"; // BLUE + public static final String PURPLE_UNDERLINED = "\033[4;35m"; // PURPLE + public static final String CYAN_UNDERLINED = "\033[4;36m"; // CYAN + public static final String WHITE_UNDERLINED = "\033[4;37m"; // WHITE + + // Background + public static final String BLACK_BACKGROUND = "\033[40m"; // BLACK + public static final String RED_BACKGROUND = "\033[41m"; // RED + public static final String GREEN_BACKGROUND = "\033[42m"; // GREEN + public static final String YELLOW_BACKGROUND = "\033[43m"; // YELLOW + public static final String BLUE_BACKGROUND = "\033[44m"; // BLUE + public static final String PURPLE_BACKGROUND = "\033[45m"; // PURPLE + public static final String CYAN_BACKGROUND = "\033[46m"; // CYAN + public static final String WHITE_BACKGROUND = "\033[47m"; // WHITE + + // High Intensity + public static final String BLACK_BRIGHT = "\033[0;90m"; // BLACK + public static final String RED_BRIGHT = "\033[0;91m"; // RED + public static final String GREEN_BRIGHT = "\033[0;92m"; // GREEN + public static final String YELLOW_BRIGHT = "\033[0;93m"; // YELLOW + public static final String BLUE_BRIGHT = "\033[0;94m"; // BLUE + public static final String PURPLE_BRIGHT = "\033[0;95m"; // PURPLE + public static final String CYAN_BRIGHT = "\033[0;96m"; // CYAN + public static final String WHITE_BRIGHT = "\033[0;97m"; // WHITE + + // Bold High Intensity + public static final String BLACK_BOLD_BRIGHT = "\033[1;90m"; // BLACK + public static final String RED_BOLD_BRIGHT = "\033[1;91m"; // RED + public static final String GREEN_BOLD_BRIGHT = "\033[1;92m"; // GREEN + public static final String YELLOW_BOLD_BRIGHT = "\033[1;93m";// YELLOW + public static final String BLUE_BOLD_BRIGHT = "\033[1;94m"; // BLUE + public static final String PURPLE_BOLD_BRIGHT = "\033[1;95m";// PURPLE + public static final String CYAN_BOLD_BRIGHT = "\033[1;96m"; // CYAN + public static final String WHITE_BOLD_BRIGHT = "\033[1;97m"; // WHITE + + // High Intensity backgrounds + public static final String BLACK_BACKGROUND_BRIGHT = "\033[0;100m";// BLACK + public static final String RED_BACKGROUND_BRIGHT = "\033[0;101m";// RED + public static final String GREEN_BACKGROUND_BRIGHT = "\033[0;102m";// GREEN + public static final String YELLOW_BACKGROUND_BRIGHT = "\033[0;103m";// YELLOW + public static final String BLUE_BACKGROUND_BRIGHT = "\033[0;104m";// BLUE + public static final String PURPLE_BACKGROUND_BRIGHT = "\033[0;105m"; // PURPLE + public static final String CYAN_BACKGROUND_BRIGHT = "\033[0;106m"; // CYAN + public static final String WHITE_BACKGROUND_BRIGHT = "\033[0;107m"; // WHITE +} + diff --git a/src/Main.java b/src/Main.java new file mode 100644 index 0000000..18412ab --- /dev/null +++ b/src/Main.java @@ -0,0 +1,130 @@ +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +public class Main { + + public static int completedThreads = 0; + public static int progress = 0; + + public static HashMap> fileMap = new HashMap<>(); + public static void main(String[] args) throws InterruptedException { + + boolean doTheColorThingy = false; + boolean showProgress = false; + + boolean recordFolder = false; + + List paths = new ArrayList<>(); + + for(String i : args) + { + if (recordFolder) { + if(Files.isDirectory(Paths.get(i))) { + paths.add(i); + } else {recordFolder = false;} + } + if (i.equalsIgnoreCase("-c") || i.equalsIgnoreCase("-color")) { doTheColorThingy = true;} + if (i.equalsIgnoreCase("-p") || i.equalsIgnoreCase("-progress")) { showProgress = true;} + if (i.equalsIgnoreCase("-f") || i.equalsIgnoreCase("-folder")) { recordFolder = true;} + + } + + if (paths.size() == 0) { + System.out.println("Aborted, no Folders Found!"); + return; + } + + + + /*System.out.println("Folders: " + paths.size()); + System.out.println("Color: " + doTheColorThingy); + System.out.println("Progressbar: " + showProgress); */ + + List pathList = new ArrayList<>(); + List allFiles = new ArrayList<>(); + + for (String folder : paths) { + try (Stream stream = Files.walk(Paths.get(folder))) { + pathList = stream.map(Path::normalize).filter(Files::isRegularFile).collect(Collectors.toList()); + } catch (IOException e) { + e.printStackTrace(); + } + allFiles.addAll(pathList); + } + + + // calculations for multithreading + + //The number of Cores or better said Threads that can be used + int availableThreads = Runtime.getRuntime().availableProcessors(); + + //just the number of All Files in all Folders taken from the Args + int filesToBeDone = allFiles.size(); + + //Every Thread that is going to be started gets a range of files + //They are seperated and are called sections + int sections = filesToBeDone / availableThreads; + + for (int i = 1; i <= availableThreads; i++) { + + List sectionedList = new ArrayList<>(); + + + //Here the different Threads are being started + //Usually the separation gives the first threads the same number of files to be working on and the last one is given all the files that could not be separetated + if (i == availableThreads) { + for (int x = (sections * i) - (sections); x < filesToBeDone; x++) { + sectionedList.add(allFiles.get(x)); + + } + } else { + for (int x = (sections * i) - (sections); x < (sections * i); x++) { + sectionedList.add(allFiles.get(x)); + + } + } + //Start Multithreading + + ThreadedCompare threadedCompare = new ThreadedCompare(sectionedList, allFiles); + threadedCompare.start(); + + } + //this updates if necessary the Progress bar and checks for Finished threads + while (completedThreads < availableThreads) { + TimeUnit.SECONDS.sleep(1); + + if (showProgress && doTheColorThingy) { + System.out.print(ConsoleColors.BLUE_BOLD + "Progress: " + ConsoleColors.GREEN_BOLD + progress + " / " + filesToBeDone + ConsoleColors.RESET + "\r"); + } else if (showProgress) { + System.out.print("Progress: " + progress + " / " + filesToBeDone + "\r"); + } + + } + + //now everything is finished and the filemap (hashmap with all dups) can be printed out in a nice view + //System.out.println(fileMap); + + for (String md5: fileMap.keySet()) { + + if (doTheColorThingy) { + System.out.println(ConsoleColors.BLUE_BOLD + md5 + ConsoleColors.CYAN_BOLD + " --> " + ConsoleColors.GREEN_BOLD + fileMap.get(md5) + ConsoleColors.RESET); + + } else { + System.out.println(md5 +" --> " + fileMap.get(md5)); + } + + } + + + + } + +} diff --git a/src/ThreadedCompare.java b/src/ThreadedCompare.java new file mode 100644 index 0000000..22c2d24 --- /dev/null +++ b/src/ThreadedCompare.java @@ -0,0 +1,79 @@ +import java.io.*; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.List; +public class ThreadedCompare extends Thread { + private final List pathsToCompareTo; + private final List pathsToBeCompared; + public ThreadedCompare (List pathsToBeCompared, List pathsToCompareTo) { + this.pathsToBeCompared = pathsToBeCompared; + this.pathsToCompareTo = pathsToCompareTo; + } + @Override + public void run() { + + //Compare every File + for (Path file1 : pathsToBeCompared) { + for (Path file2 : pathsToCompareTo) { + try { + if (sameContent(file1, file2)) { + List bothPaths = new ArrayList<>(); + bothPaths.add(file1); + bothPaths.add(file2); + + //here it is trying to add the values in the HashMap + if (Main.fileMap.containsKey(getMD5Sum(file1.toFile()))) { + if (!Main.fileMap.get(getMD5Sum(file1.toFile())).contains(file1)) { + Main.fileMap.get(getMD5Sum(file1.toFile())).add(file1); + } + if (!Main.fileMap.get(getMD5Sum(file1.toFile())).contains(file2)) { + Main.fileMap.get(getMD5Sum(file1.toFile())).add(file2); + } + } else { + Main.fileMap.put(getMD5Sum(file1.toFile()), bothPaths); + } + } + } catch (IOException e) { + throw new RuntimeException(e); + } + } + //Update the Progress that can be found in The Main Class + Main.progress += 1; + } + //Update the thread Completion Counter that can be found in the Main Class + Main.completedThreads += 1; + } + private boolean sameContent(Path file1, Path file2) throws IOException { + if (file1 != file2) { + return Files.mismatch(file1, file2) == -1; + } + return false; + } + + //this is used to get the MD5 String of one of the files (one of them is just fine since they both have the same value) + private String getMD5Sum (File file) throws IOException { + MessageDigest messageDigest = null; + try { + messageDigest = MessageDigest.getInstance("MD5"); + } catch (NoSuchAlgorithmException e) { + throw new RuntimeException(e); + } + FileInputStream fis = new FileInputStream(file); + + byte[] dataBytes = new byte[1024]; + + int unread = 0; + while ((unread = fis.read(dataBytes)) != -1) { + messageDigest.update(dataBytes, 0, unread); + } + byte[] mdbytes = messageDigest.digest(); + StringBuilder sb = new StringBuilder(); + for (byte mdbyte : mdbytes) { + sb.append(Integer.toString((mdbyte & 0xff) + 0x100, 16).substring(1)); + } + return sb.toString(); + } +}