First Creation
This commit is contained in:
bly
commit
e0623bc399
9 changed files with 396 additions and 0 deletions
29
.gitignore
vendored
Normal file
29
.gitignore
vendored
Normal file
|
|
@ -0,0 +1,29 @@
|
|||
### IntelliJ IDEA ###
|
||||
out/
|
||||
!**/src/main/**/out/
|
||||
!**/src/test/**/out/
|
||||
|
||||
### Eclipse ###
|
||||
.apt_generated
|
||||
.classpath
|
||||
.factorypath
|
||||
.project
|
||||
.settings
|
||||
.springBeans
|
||||
.sts4-cache
|
||||
bin/
|
||||
!**/src/main/**/bin/
|
||||
!**/src/test/**/bin/
|
||||
|
||||
### NetBeans ###
|
||||
/nbproject/private/
|
||||
/nbbuild/
|
||||
/dist/
|
||||
/nbdist/
|
||||
/.nb-gradle/
|
||||
|
||||
### VS Code ###
|
||||
.vscode/
|
||||
|
||||
### Mac OS ###
|
||||
.DS_Store
|
||||
6
.idea/misc.xml
generated
Normal file
6
.idea/misc.xml
generated
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project version="4">
|
||||
<component name="ProjectRootManager" version="2" languageLevel="JDK_19" default="true" project-jdk-name="openjdk-19" project-jdk-type="JavaSDK">
|
||||
<output url="file://$PROJECT_DIR$/out" />
|
||||
</component>
|
||||
</project>
|
||||
8
.idea/modules.xml
generated
Normal file
8
.idea/modules.xml
generated
Normal file
|
|
@ -0,0 +1,8 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project version="4">
|
||||
<component name="ProjectModuleManager">
|
||||
<modules>
|
||||
<module fileurl="file://$PROJECT_DIR$/Sherly.iml" filepath="$PROJECT_DIR$/Sherly.iml" />
|
||||
</modules>
|
||||
</component>
|
||||
</project>
|
||||
6
.idea/vcs.xml
generated
Normal file
6
.idea/vcs.xml
generated
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project version="4">
|
||||
<component name="VcsDirectoryMappings">
|
||||
<mapping directory="$PROJECT_DIR$" vcs="Git" />
|
||||
</component>
|
||||
</project>
|
||||
50
.idea/workspace.xml
generated
Normal file
50
.idea/workspace.xml
generated
Normal file
|
|
@ -0,0 +1,50 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project version="4">
|
||||
<component name="ChangeListManager">
|
||||
<list default="true" id="9b57ac51-c870-474b-9dfd-64a5fc490635" name="Changes" comment="">
|
||||
<change afterPath="$PROJECT_DIR$/.gitignore" afterDir="false" />
|
||||
<change afterPath="$PROJECT_DIR$/.idea/misc.xml" afterDir="false" />
|
||||
<change afterPath="$PROJECT_DIR$/.idea/modules.xml" afterDir="false" />
|
||||
<change afterPath="$PROJECT_DIR$/.idea/vcs.xml" afterDir="false" />
|
||||
<change afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
|
||||
<change afterPath="$PROJECT_DIR$/Sherly.iml" afterDir="false" />
|
||||
<change afterPath="$PROJECT_DIR$/src/ConsoleColors.java" afterDir="false" />
|
||||
<change afterPath="$PROJECT_DIR$/src/Main.java" afterDir="false" />
|
||||
<change afterPath="$PROJECT_DIR$/src/ThreadedCompare.java" afterDir="false" />
|
||||
</list>
|
||||
<option name="SHOW_DIALOG" value="false" />
|
||||
<option name="HIGHLIGHT_CONFLICTS" value="true" />
|
||||
<option name="HIGHLIGHT_NON_ACTIVE_CHANGELIST" value="false" />
|
||||
<option name="LAST_RESOLUTION" value="IGNORE" />
|
||||
</component>
|
||||
<component name="FileTemplateManagerImpl">
|
||||
<option name="RECENT_TEMPLATES">
|
||||
<list>
|
||||
<option value="Class" />
|
||||
</list>
|
||||
</option>
|
||||
</component>
|
||||
<component name="ProjectId" id="2Jn9jTd9EocdVDpvvPMYxSrBjXm" />
|
||||
<component name="ProjectLevelVcsManager" settingsEditedManually="true" />
|
||||
<component name="ProjectViewState">
|
||||
<option name="hideEmptyMiddlePackages" value="true" />
|
||||
<option name="showLibraryContents" value="true" />
|
||||
</component>
|
||||
<component name="PropertiesComponent"><![CDATA[{
|
||||
"keyToString": {
|
||||
"RunOnceActivity.OpenProjectViewOnStart": "true",
|
||||
"RunOnceActivity.ShowReadmeOnStart": "true"
|
||||
}
|
||||
}]]></component>
|
||||
<component name="SpellCheckerSettings" RuntimeDictionaries="0" Folders="0" CustomDictionaries="0" DefaultDictionary="application-level" UseSingleDictionary="true" transferred="true" />
|
||||
<component name="TaskManager">
|
||||
<task active="true" id="Default" summary="Default task">
|
||||
<changelist id="9b57ac51-c870-474b-9dfd-64a5fc490635" name="Changes" comment="" />
|
||||
<created>1672703116568</created>
|
||||
<option name="number" value="Default" />
|
||||
<option name="presentableId" value="Default" />
|
||||
<updated>1672703116568</updated>
|
||||
</task>
|
||||
<servers />
|
||||
</component>
|
||||
</project>
|
||||
11
Sherly.iml
Normal file
11
Sherly.iml
Normal file
|
|
@ -0,0 +1,11 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<module type="JAVA_MODULE" version="4">
|
||||
<component name="NewModuleRootManager" inherit-compiler-output="true">
|
||||
<exclude-output />
|
||||
<content url="file://$MODULE_DIR$">
|
||||
<sourceFolder url="file://$MODULE_DIR$/src" isTestSource="false" />
|
||||
</content>
|
||||
<orderEntry type="inheritedJdk" />
|
||||
<orderEntry type="sourceFolder" forTests="false" />
|
||||
</component>
|
||||
</module>
|
||||
77
src/ConsoleColors.java
Normal file
77
src/ConsoleColors.java
Normal file
|
|
@ -0,0 +1,77 @@
|
|||
//Just regular Colors to make it a bit prettier
|
||||
|
||||
public class ConsoleColors {
|
||||
// Reset
|
||||
public static final String RESET = "\033[0m"; // Text Reset
|
||||
|
||||
// Regular Colors
|
||||
public static final String BLACK = "\033[0;30m"; // BLACK
|
||||
public static final String RED = "\033[0;31m"; // RED
|
||||
public static final String GREEN = "\033[0;32m"; // GREEN
|
||||
public static final String YELLOW = "\033[0;33m"; // YELLOW
|
||||
public static final String BLUE = "\033[0;34m"; // BLUE
|
||||
public static final String PURPLE = "\033[0;35m"; // PURPLE
|
||||
public static final String CYAN = "\033[0;36m"; // CYAN
|
||||
public static final String WHITE = "\033[0;37m"; // WHITE
|
||||
|
||||
// Bold
|
||||
public static final String BLACK_BOLD = "\033[1;30m"; // BLACK
|
||||
public static final String RED_BOLD = "\033[1;31m"; // RED
|
||||
public static final String GREEN_BOLD = "\033[1;32m"; // GREEN
|
||||
public static final String YELLOW_BOLD = "\033[1;33m"; // YELLOW
|
||||
public static final String BLUE_BOLD = "\033[1;34m"; // BLUE
|
||||
public static final String PURPLE_BOLD = "\033[1;35m"; // PURPLE
|
||||
public static final String CYAN_BOLD = "\033[1;36m"; // CYAN
|
||||
public static final String WHITE_BOLD = "\033[1;37m"; // WHITE
|
||||
|
||||
// Underline
|
||||
public static final String BLACK_UNDERLINED = "\033[4;30m"; // BLACK
|
||||
public static final String RED_UNDERLINED = "\033[4;31m"; // RED
|
||||
public static final String GREEN_UNDERLINED = "\033[4;32m"; // GREEN
|
||||
public static final String YELLOW_UNDERLINED = "\033[4;33m"; // YELLOW
|
||||
public static final String BLUE_UNDERLINED = "\033[4;34m"; // BLUE
|
||||
public static final String PURPLE_UNDERLINED = "\033[4;35m"; // PURPLE
|
||||
public static final String CYAN_UNDERLINED = "\033[4;36m"; // CYAN
|
||||
public static final String WHITE_UNDERLINED = "\033[4;37m"; // WHITE
|
||||
|
||||
// Background
|
||||
public static final String BLACK_BACKGROUND = "\033[40m"; // BLACK
|
||||
public static final String RED_BACKGROUND = "\033[41m"; // RED
|
||||
public static final String GREEN_BACKGROUND = "\033[42m"; // GREEN
|
||||
public static final String YELLOW_BACKGROUND = "\033[43m"; // YELLOW
|
||||
public static final String BLUE_BACKGROUND = "\033[44m"; // BLUE
|
||||
public static final String PURPLE_BACKGROUND = "\033[45m"; // PURPLE
|
||||
public static final String CYAN_BACKGROUND = "\033[46m"; // CYAN
|
||||
public static final String WHITE_BACKGROUND = "\033[47m"; // WHITE
|
||||
|
||||
// High Intensity
|
||||
public static final String BLACK_BRIGHT = "\033[0;90m"; // BLACK
|
||||
public static final String RED_BRIGHT = "\033[0;91m"; // RED
|
||||
public static final String GREEN_BRIGHT = "\033[0;92m"; // GREEN
|
||||
public static final String YELLOW_BRIGHT = "\033[0;93m"; // YELLOW
|
||||
public static final String BLUE_BRIGHT = "\033[0;94m"; // BLUE
|
||||
public static final String PURPLE_BRIGHT = "\033[0;95m"; // PURPLE
|
||||
public static final String CYAN_BRIGHT = "\033[0;96m"; // CYAN
|
||||
public static final String WHITE_BRIGHT = "\033[0;97m"; // WHITE
|
||||
|
||||
// Bold High Intensity
|
||||
public static final String BLACK_BOLD_BRIGHT = "\033[1;90m"; // BLACK
|
||||
public static final String RED_BOLD_BRIGHT = "\033[1;91m"; // RED
|
||||
public static final String GREEN_BOLD_BRIGHT = "\033[1;92m"; // GREEN
|
||||
public static final String YELLOW_BOLD_BRIGHT = "\033[1;93m";// YELLOW
|
||||
public static final String BLUE_BOLD_BRIGHT = "\033[1;94m"; // BLUE
|
||||
public static final String PURPLE_BOLD_BRIGHT = "\033[1;95m";// PURPLE
|
||||
public static final String CYAN_BOLD_BRIGHT = "\033[1;96m"; // CYAN
|
||||
public static final String WHITE_BOLD_BRIGHT = "\033[1;97m"; // WHITE
|
||||
|
||||
// High Intensity backgrounds
|
||||
public static final String BLACK_BACKGROUND_BRIGHT = "\033[0;100m";// BLACK
|
||||
public static final String RED_BACKGROUND_BRIGHT = "\033[0;101m";// RED
|
||||
public static final String GREEN_BACKGROUND_BRIGHT = "\033[0;102m";// GREEN
|
||||
public static final String YELLOW_BACKGROUND_BRIGHT = "\033[0;103m";// YELLOW
|
||||
public static final String BLUE_BACKGROUND_BRIGHT = "\033[0;104m";// BLUE
|
||||
public static final String PURPLE_BACKGROUND_BRIGHT = "\033[0;105m"; // PURPLE
|
||||
public static final String CYAN_BACKGROUND_BRIGHT = "\033[0;106m"; // CYAN
|
||||
public static final String WHITE_BACKGROUND_BRIGHT = "\033[0;107m"; // WHITE
|
||||
}
|
||||
|
||||
130
src/Main.java
Normal file
130
src/Main.java
Normal file
|
|
@ -0,0 +1,130 @@
|
|||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.Paths;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
public class Main {
|
||||
|
||||
public static int completedThreads = 0;
|
||||
public static int progress = 0;
|
||||
|
||||
public static HashMap<String, List<Path>> fileMap = new HashMap<>();
|
||||
public static void main(String[] args) throws InterruptedException {
|
||||
|
||||
boolean doTheColorThingy = false;
|
||||
boolean showProgress = false;
|
||||
|
||||
boolean recordFolder = false;
|
||||
|
||||
List<String> paths = new ArrayList<>();
|
||||
|
||||
for(String i : args)
|
||||
{
|
||||
if (recordFolder) {
|
||||
if(Files.isDirectory(Paths.get(i))) {
|
||||
paths.add(i);
|
||||
} else {recordFolder = false;}
|
||||
}
|
||||
if (i.equalsIgnoreCase("-c") || i.equalsIgnoreCase("-color")) { doTheColorThingy = true;}
|
||||
if (i.equalsIgnoreCase("-p") || i.equalsIgnoreCase("-progress")) { showProgress = true;}
|
||||
if (i.equalsIgnoreCase("-f") || i.equalsIgnoreCase("-folder")) { recordFolder = true;}
|
||||
|
||||
}
|
||||
|
||||
if (paths.size() == 0) {
|
||||
System.out.println("Aborted, no Folders Found!");
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*System.out.println("Folders: " + paths.size());
|
||||
System.out.println("Color: " + doTheColorThingy);
|
||||
System.out.println("Progressbar: " + showProgress); */
|
||||
|
||||
List<Path> pathList = new ArrayList<>();
|
||||
List<Path> allFiles = new ArrayList<>();
|
||||
|
||||
for (String folder : paths) {
|
||||
try (Stream<Path> stream = Files.walk(Paths.get(folder))) {
|
||||
pathList = stream.map(Path::normalize).filter(Files::isRegularFile).collect(Collectors.toList());
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
allFiles.addAll(pathList);
|
||||
}
|
||||
|
||||
|
||||
// calculations for multithreading
|
||||
|
||||
//The number of Cores or better said Threads that can be used
|
||||
int availableThreads = Runtime.getRuntime().availableProcessors();
|
||||
|
||||
//just the number of All Files in all Folders taken from the Args
|
||||
int filesToBeDone = allFiles.size();
|
||||
|
||||
//Every Thread that is going to be started gets a range of files
|
||||
//They are seperated and are called sections
|
||||
int sections = filesToBeDone / availableThreads;
|
||||
|
||||
for (int i = 1; i <= availableThreads; i++) {
|
||||
|
||||
List<Path> sectionedList = new ArrayList<>();
|
||||
|
||||
|
||||
//Here the different Threads are being started
|
||||
//Usually the separation gives the first threads the same number of files to be working on and the last one is given all the files that could not be separetated
|
||||
if (i == availableThreads) {
|
||||
for (int x = (sections * i) - (sections); x < filesToBeDone; x++) {
|
||||
sectionedList.add(allFiles.get(x));
|
||||
|
||||
}
|
||||
} else {
|
||||
for (int x = (sections * i) - (sections); x < (sections * i); x++) {
|
||||
sectionedList.add(allFiles.get(x));
|
||||
|
||||
}
|
||||
}
|
||||
//Start Multithreading
|
||||
|
||||
ThreadedCompare threadedCompare = new ThreadedCompare(sectionedList, allFiles);
|
||||
threadedCompare.start();
|
||||
|
||||
}
|
||||
//this updates if necessary the Progress bar and checks for Finished threads
|
||||
while (completedThreads < availableThreads) {
|
||||
TimeUnit.SECONDS.sleep(1);
|
||||
|
||||
if (showProgress && doTheColorThingy) {
|
||||
System.out.print(ConsoleColors.BLUE_BOLD + "Progress: " + ConsoleColors.GREEN_BOLD + progress + " / " + filesToBeDone + ConsoleColors.RESET + "\r");
|
||||
} else if (showProgress) {
|
||||
System.out.print("Progress: " + progress + " / " + filesToBeDone + "\r");
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
//now everything is finished and the filemap (hashmap with all dups) can be printed out in a nice view
|
||||
//System.out.println(fileMap);
|
||||
|
||||
for (String md5: fileMap.keySet()) {
|
||||
|
||||
if (doTheColorThingy) {
|
||||
System.out.println(ConsoleColors.BLUE_BOLD + md5 + ConsoleColors.CYAN_BOLD + " --> " + ConsoleColors.GREEN_BOLD + fileMap.get(md5) + ConsoleColors.RESET);
|
||||
|
||||
} else {
|
||||
System.out.println(md5 +" --> " + fileMap.get(md5));
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
79
src/ThreadedCompare.java
Normal file
79
src/ThreadedCompare.java
Normal file
|
|
@ -0,0 +1,79 @@
|
|||
import java.io.*;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.security.MessageDigest;
|
||||
import java.security.NoSuchAlgorithmException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
public class ThreadedCompare extends Thread {
|
||||
private final List<Path> pathsToCompareTo;
|
||||
private final List<Path> pathsToBeCompared;
|
||||
public ThreadedCompare (List<Path> pathsToBeCompared, List<Path> pathsToCompareTo) {
|
||||
this.pathsToBeCompared = pathsToBeCompared;
|
||||
this.pathsToCompareTo = pathsToCompareTo;
|
||||
}
|
||||
@Override
|
||||
public void run() {
|
||||
|
||||
//Compare every File
|
||||
for (Path file1 : pathsToBeCompared) {
|
||||
for (Path file2 : pathsToCompareTo) {
|
||||
try {
|
||||
if (sameContent(file1, file2)) {
|
||||
List<Path> bothPaths = new ArrayList<>();
|
||||
bothPaths.add(file1);
|
||||
bothPaths.add(file2);
|
||||
|
||||
//here it is trying to add the values in the HashMap
|
||||
if (Main.fileMap.containsKey(getMD5Sum(file1.toFile()))) {
|
||||
if (!Main.fileMap.get(getMD5Sum(file1.toFile())).contains(file1)) {
|
||||
Main.fileMap.get(getMD5Sum(file1.toFile())).add(file1);
|
||||
}
|
||||
if (!Main.fileMap.get(getMD5Sum(file1.toFile())).contains(file2)) {
|
||||
Main.fileMap.get(getMD5Sum(file1.toFile())).add(file2);
|
||||
}
|
||||
} else {
|
||||
Main.fileMap.put(getMD5Sum(file1.toFile()), bothPaths);
|
||||
}
|
||||
}
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
//Update the Progress that can be found in The Main Class
|
||||
Main.progress += 1;
|
||||
}
|
||||
//Update the thread Completion Counter that can be found in the Main Class
|
||||
Main.completedThreads += 1;
|
||||
}
|
||||
private boolean sameContent(Path file1, Path file2) throws IOException {
|
||||
if (file1 != file2) {
|
||||
return Files.mismatch(file1, file2) == -1;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
//this is used to get the MD5 String of one of the files (one of them is just fine since they both have the same value)
|
||||
private String getMD5Sum (File file) throws IOException {
|
||||
MessageDigest messageDigest = null;
|
||||
try {
|
||||
messageDigest = MessageDigest.getInstance("MD5");
|
||||
} catch (NoSuchAlgorithmException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
FileInputStream fis = new FileInputStream(file);
|
||||
|
||||
byte[] dataBytes = new byte[1024];
|
||||
|
||||
int unread = 0;
|
||||
while ((unread = fis.read(dataBytes)) != -1) {
|
||||
messageDigest.update(dataBytes, 0, unread);
|
||||
}
|
||||
byte[] mdbytes = messageDigest.digest();
|
||||
StringBuilder sb = new StringBuilder();
|
||||
for (byte mdbyte : mdbytes) {
|
||||
sb.append(Integer.toString((mdbyte & 0xff) + 0x100, 16).substring(1));
|
||||
}
|
||||
return sb.toString();
|
||||
}
|
||||
}
|
||||
Reference in a new issue