Browse Source

Working on graalvm native-image and memory optim

master
Delta503 7 months ago
parent
commit
8c039e0a0f
  1. 51
      dependency-reduced-pom.xml
  2. 3
      org.hostliststools.cli.app.baseapp.build_artifacts.txt
  3. 295
      pom.xml
  4. 443
      src/main/java/org/hostliststools/cli/app/BaseApp.java
  5. 515
      src/main/java/org/hostliststools/cli/io/BaseFile.java
  6. 161
      src/main/java/org/hostliststools/cli/io/BaseNet.java
  7. 99
      src/main/java/org/hostliststools/cli/io/BaseSettings.java
  8. 29
      src/main/java/org/hostliststools/cli/io/Communicats.java
  9. 48
      src/main/java/org/hostliststools/cli/io/Serialization.java
  10. 206
      src/main/java/org/hostliststools/cli/logic/BaseTools.java
  11. 124
      src/main/java/org/hostliststools/cli/logic/HostList.java
  12. 344
      src/main/java/org/hostliststools/cli/objects/Git.java
  13. 886
      src/main/java/org/hostliststools/cli/process/BaseFunctions.java
  14. 408
      src/main/java/org/hostliststools/cli/process/BaseMethods.java
  15. 158
      src/main/java/org/hostliststools/cli/wrapper/GitWrapper.java

51
dependency-reduced-pom.xml

@ -5,7 +5,7 @@
<artifactId>hostliststools-cli</artifactId>
<name>hostliststools-cli</name>
<version>0.2.1</version>
<description>CLI version of hostliststools to manage hostlists (for pihole/adguard)</description>
<description>CLI version of hostliststools to manage hostlists (for pihole/adguard/dnscrypt-proxy)</description>
<build>
<plugins>
<plugin>
@ -41,16 +41,59 @@
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.graalvm.nativeimage</groupId>
<artifactId>native-image-maven-plugin</artifactId>
<version>${graalvm.version}</version>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>native-image</goal>
</goals>
</execution>
</executions>
<configuration>
<imageName>htcli</imageName>
<mainClass>org.hostliststools.cli.app.BaseApp</mainClass>
<skip>false</skip>
<buildArgs>
<buildArg>-H:EnableURLProtocols=http,https</buildArg>
<buildArg>--allow-incomplete-classpath</buildArg>
<buildArg>--no-fallback</buildArg>
<buildArg>--verbose</buildArg>
</buildArgs>
<filters>
<filter>
<artifact>*:*</artifact>
<excludes>
<exclude>META-INF/*.SF</exclude>
<exclude>META-INF/*.DSA</exclude>
<exclude>META-INF/*.RSA</exclude>
</excludes>
</filter>
</filters>
</configuration>
</plugin>
</plugins>
</build>
<dependencies>
<dependency>
<groupId>org.graalvm.sdk</groupId>
<artifactId>graal-sdk</artifactId>
<version>21.2.0</version>
<scope>provided</scope>
</dependency>
</dependencies>
<properties>
<mvndep.version>3.1.2</mvndep.version>
<tongfeiprogressbar.version>0.9.0</tongfeiprogressbar.version>
<maven.compiler.target>11</maven.compiler.target>
<mvnjar.version>3.2.0</mvnjar.version>
<commonslang3.version>3.11</commonslang3.version>
<commonscli.version>1.4</commonscli.version>
<maven.compiler.source>11</maven.compiler.source>
<graalvm.version>21.2.0</graalvm.version>
<tongfeiprogressbar.version>0.9.0</tongfeiprogressbar.version>
<maven.compiler.target>11</maven.compiler.target>
<mvnjar.version>3.2.0</mvnjar.version>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<commonsio.version>2.8.0</commonsio.version>
<mavenShadePlugin.version>3.2.4</mavenShadePlugin.version>

3
org.hostliststools.cli.app.baseapp.build_artifacts.txt

@ -0,0 +1,3 @@
[EXECUTABLE]
org.hostliststools.cli.app.baseapp

295
pom.xml

@ -1,129 +1,174 @@
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>org.hostliststools.cli</groupId>
<artifactId>hostliststools-cli</artifactId>
<version>0.2.1</version>
<name>hostliststools-cli</name>
<description>CLI version of hostliststools to manage hostlists (for pihole/adguard)</description>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<maven.compiler.source>11</maven.compiler.source>
<maven.compiler.target>11</maven.compiler.target>
<mavenShadePlugin.version>3.2.4</mavenShadePlugin.version>
<mvnjar.version>3.2.0</mvnjar.version>
<mvndep.version>3.1.2</mvndep.version>
<commonsio.version>2.8.0</commonsio.version>
<commonslang3.version>3.11</commonslang3.version>
<commonscli.version>1.4</commonscli.version>
<!-- <jgit.version>5.10.0.202012080955-r</jgit.version>
<jgitapache.version>5.10.0.202012080955-r</jgitapache.version>-->
<htmlunit.version>2.50.0</htmlunit.version>
<tongfeiprogressbar.version>0.9.0</tongfeiprogressbar.version>
</properties>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>${mavenShadePlugin.version}</version>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
<configuration>
<transformers>
<transformer
implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
<manifestEntries>
<Main-Class>org.hostliststools.cli.app.BaseApp</Main-Class>
<Build-Number>1.0</Build-Number>
</manifestEntries>
</transformer>
</transformers>
<filters>
<filter>
<artifact>*:*</artifact>
<excludes>
<exclude>META-INF/*.SF</exclude>
<exclude>META-INF/*.DSA</exclude>
<exclude>META-INF/*.RSA</exclude>
</excludes>
</filter>
</filters>
<release>11</release>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
<dependencies>
<!-- https://mvnrepository.com/artifact/commons-io/commons-io -->
<!-- Apache Licensed -->
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<version>${commonsio.version}</version>
</dependency>
<!-- https://mvnrepository.com/artifact/me.tongfei/progressbar -->
<!-- MIT Licensed -->
<dependency>
<groupId>me.tongfei</groupId>
<artifactId>progressbar</artifactId>
<version>${tongfeiprogressbar.version}</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.apache.commons/commons-lang3 -->
<!-- Apache Licensed -->
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
<version>${commonslang3.version}</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.eclipse.jgit/org.eclipse.jgit
BSD EDL Licensed! Check it out
<dependency>
<groupId>org.eclipse.jgit</groupId>
<artifactId>org.eclipse.jgit</artifactId>
<version>${jgit.version}</version>
</dependency> -->
<!-- https://mvnrepository.com/artifact/commons-cli/commons-cli -->
<!-- Apache Licensed -->
<dependency>
<groupId>commons-cli</groupId>
<artifactId>commons-cli</artifactId>
<version>${commonscli.version}</version>
</dependency>
<!-- https://github.com/HtmlUnit/htmlunit -->
<!-- Apache Licensed -->
<dependency>
<groupId>net.sourceforge.htmlunit</groupId>
<artifactId>htmlunit</artifactId>
<version>${htmlunit.version}</version>
</dependency>
<!-- <dependency>
<groupId>org.eclipse.jgit</groupId>
<artifactId>org.eclipse.jgit.ssh.apache</artifactId>
<version>${jgitapache.version}</version>
</dependency>-->
</dependencies>
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>org.hostliststools.cli</groupId>
<artifactId>hostliststools-cli</artifactId>
<version>0.2.1</version>
<name>hostliststools-cli</name>
<description>CLI version of hostliststools to manage hostlists (for pihole/adguard/dnscrypt-proxy) </description>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<maven.compiler.source>11</maven.compiler.source>
<maven.compiler.target>11</maven.compiler.target>
<mavenShadePlugin.version>3.2.4</mavenShadePlugin.version>
<mvnjar.version>3.2.0</mvnjar.version>
<mvndep.version>3.1.2</mvndep.version>
<commonsio.version>2.8.0</commonsio.version>
<commonslang3.version>3.11</commonslang3.version>
<commonscli.version>1.4</commonscli.version>
<graalvm.version>21.2.0</graalvm.version>
<!-- <jgit.version>5.10.0.202012080955-r</jgit.version>
<jgitapache.version>5.10.0.202012080955-r</jgitapache.version>-->
<htmlunit.version>2.50.0</htmlunit.version>
<tongfeiprogressbar.version>0.9.0</tongfeiprogressbar.version>
</properties>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>${mavenShadePlugin.version}</version>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
<configuration>
<transformers>
<transformer
implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
<manifestEntries>
<Main-Class>org.hostliststools.cli.app.BaseApp</Main-Class>
<Build-Number>1.0</Build-Number>
</manifestEntries>
</transformer>
</transformers>
<filters>
<filter>
<artifact>*:*</artifact>
<excludes>
<exclude>META-INF/*.SF</exclude>
<exclude>META-INF/*.DSA</exclude>
<exclude>META-INF/*.RSA</exclude>
</excludes>
</filter>
</filters>
<release>11</release>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.graalvm.nativeimage</groupId>
<artifactId>native-image-maven-plugin</artifactId>
<version>${graalvm.version}</version>
<executions>
<execution>
<goals>
<goal>native-image</goal>
</goals>
<phase>package</phase>
</execution>
</executions>
<configuration>
<imageName>htcli</imageName>
<mainClass>org.hostliststools.cli.app.BaseApp</mainClass>
<skip>false</skip>
<buildArgs>
<buildArg> -H:EnableURLProtocols=http,https</buildArg>
<buildArg> --allow-incomplete-classpath</buildArg>
<buildArg> --no-fallback</buildArg>
<buildArg> --verbose</buildArg>
</buildArgs>
<filters>
<filter>
<artifact>*:*</artifact>
<excludes>
<exclude>META-INF/*.SF</exclude>
<exclude>META-INF/*.DSA</exclude>
<exclude>META-INF/*.RSA</exclude>
</excludes>
</filter>
</filters>
</configuration>
</plugin>
</plugins>
</build>
<dependencies>
<!-- https://mvnrepository.com/artifact/commons-io/commons-io -->
<!-- Apache Licensed -->
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<version>${commonsio.version}</version>
</dependency>
<!-- https://mvnrepository.com/artifact/me.tongfei/progressbar -->
<!-- MIT Licensed -->
<dependency>
<groupId>me.tongfei</groupId>
<artifactId>progressbar</artifactId>
<version>${tongfeiprogressbar.version}</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.apache.commons/commons-lang3 -->
<!-- Apache Licensed -->
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
<version>${commonslang3.version}</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.eclipse.jgit/org.eclipse.jgit
BSD EDL Licensed! Check it out
<dependency>
<groupId>org.eclipse.jgit</groupId>
<artifactId>org.eclipse.jgit</artifactId>
<version>${jgit.version}</version>
</dependency> -->
<!-- https://mvnrepository.com/artifact/commons-cli/commons-cli -->
<!-- Apache Licensed -->
<dependency>
<groupId>commons-cli</groupId>
<artifactId>commons-cli</artifactId>
<version>${commonscli.version}</version>
</dependency>
<!-- https://github.com/HtmlUnit/htmlunit -->
<!-- Apache Licensed -->
<dependency>
<groupId>net.sourceforge.htmlunit</groupId>
<artifactId>htmlunit</artifactId>
<version>${htmlunit.version}</version>
</dependency>
<!-- <dependency>
<groupId>org.eclipse.jgit</groupId>
<artifactId>org.eclipse.jgit.ssh.apache</artifactId>
<version>${jgitapache.version}</version>
</dependency>-->
<dependency>
<groupId>org.graalvm.sdk</groupId>
<artifactId>graal-sdk</artifactId>
<version>${graalvm.version}</version>
<scope>provided</scope>
</dependency>
</dependencies>
</project>

443
src/main/java/org/hostliststools/cli/app/BaseApp.java

@ -5,7 +5,6 @@ import java.io.IOException;
import java.net.URISyntaxException;
import java.text.SimpleDateFormat;
import java.util.Date;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.DefaultParser;
@ -22,248 +21,248 @@ import org.hostliststools.cli.process.BaseMethods;
public class BaseApp {
public static boolean skipHeading = false;
public static boolean skipSources = false;
public static String fillwithString = null;
public static Date date = new Date();
public static SimpleDateFormat dateformatter = new SimpleDateFormat("dd-MM-yyyy_HH:mm:ss");
public static boolean skipHeading = false;
public static boolean skipSources = false;
public static String fillwithString = null;
public static Date date = new Date();
public static SimpleDateFormat dateformatter = new SimpleDateFormat("dd-MM-yyyy_HH:mm:ss");
public static String outputFileName = BaseSettings.OUT_FILENAME + "_" + dateformatter.format(date) + ".txt";
public static String outputFileName = BaseSettings.OUT_FILENAME + "_" + dateformatter.format(date) + ".txt";
public static void main(String[] args) throws URISyntaxException {
public static void main(String[] args) throws URISyntaxException {
BaseFile.init();
Options cliOptions = new Options();
BaseFile.init();
Options cliOptions = new Options();
// Big letters program management
cliOptions.addOption("X", "print-settings", false, "reads variables and prints it for user");
cliOptions.addOption("S", "git-set-remote", true, "set git remote path");
// cliOptions.addOption("P", "git-push", false, "push product file to repo");
// cliOptions.addOption("A", "git-remote-add", true, "connect local repository
// with remote one");
//// cliOptions.addOption("I", "git-init", true, "connect local repository with
// remote one");
cliOptions.addOption("D", "download", false, "downloads hostlists");
cliOptions.addOption("R", "read", false, "reads hostlists directly from web");
// cliOptions.addOption("F", "diff", false, "check diffrences"); //Leave for
// hash module implementation
cliOptions.addOption("M", "merge", false, "merges and cleans the lists from local files");
cliOptions.addOption("H", "skip-heading", false, "don't include heading");
cliOptions.addOption("K", "skip-sources", false, "don't include sources");
// cliOptions.addOption("T", "hostlist-template", false, "generate /etc/hosts
// file scheme");
// cliOptions.addOption("Q", "dnsmasq", true, "convert to dnsmasq format");
cliOptions.addOption("F", "fill-with", true, "adds prefix and space before records like 0.0.0.0 example.com");
cliOptions.addOption("I", "print-stats", false, "prints statistic about line processing and adds them to README");
cliOptions.addOption("E", "save-excluded", false, "saves excluded sets of badly formated lines");
// cliOptions.addOption("T", "test", true, "only for developer, don't use !!!");
// Small letters
cliOptions.addOption("h", "help", false, "print avaliable options");
// cliOptions.addOption("b", "build-list", true, "build own list with desired
// structure");
cliOptions.addOption("w", "set-workpace", true,
"executes all tasks on pointed workspace and saves it as last used");
cliOptions.addOption("o", "output-filename", true,
"specify merged filename once, default: Merged_$Date_$Time.txt ");
cliOptions.addOption("i", "input", true, "specify custom input folder sourcelist");
cliOptions.addOption("p", "permissive", false, "disable recover rules with ;000 ");
cliOptions.addOption("j", "download-jobs", true, "paralell download jobs");
cliOptions.addOption("c", "clean-environment", false, "removes files from $workspace/downloaded, $workspace/out ");
// Big letters program management
cliOptions.addOption("X", "print-settings", false, "reads variables and prints it for user");
cliOptions.addOption("S", "git-set-remote", true, "set git remote path");
// cliOptions.addOption("P", "git-push", false, "push product file to repo");
// cliOptions.addOption("A", "git-remote-add", true, "connect local repository
// with remote one");
//// cliOptions.addOption("I", "git-init", true, "connect local repository with
// remote one");
cliOptions.addOption("D", "download", false, "downloads hostlists");
cliOptions.addOption("R", "read", false, "reads hostlists directly from web");
// cliOptions.addOption("F", "diff", false, "check diffrences"); //Leave for
// hash module implementation
cliOptions.addOption("M", "merge", false, "merges and cleans the lists from local files");
cliOptions.addOption("H", "skip-heading", false, "don't include heading");
cliOptions.addOption("K", "skip-sources", false, "don't include sources");
// cliOptions.addOption("T", "hostlist-template", false, "generate /etc/hosts
// file scheme");
// cliOptions.addOption("Q", "dnsmasq", true, "convert to dnsmasq format");
cliOptions.addOption("F", "fill-with", true, "adds prefix and space before records like 0.0.0.0 example.com");
cliOptions.addOption("I", "print-stats", false, "prints statistic about line processing and adds them to README");
cliOptions.addOption("E", "save-excluded", false, "saves excluded sets of badly formated lines");
// cliOptions.addOption("T", "test", true, "only for developer, don't use !!!");
// Small letters
cliOptions.addOption("h", "help", false, "print avaliable options");
// cliOptions.addOption("b", "build-list", true, "build own list with desired
// structure");
cliOptions.addOption("w", "set-workpace", true,
"executes all tasks on pointed workspace and saves it as last used");
cliOptions.addOption("o", "output-filename", true,
"specify merged filename once, default: Merged_$Date_$Time.txt ");
cliOptions.addOption("i", "input", true, "specify custom input folder sourcelist");
cliOptions.addOption("p", "permissive", false, "disable recover rules with ;000 ");
cliOptions.addOption("j", "download-jobs", true, "paralell download jobs");
cliOptions.addOption("c", "clean-environment", false, "removes files from $workspace/downloaded, $workspace/out ");
CommandLineParser parser = new DefaultParser();
HelpFormatter formatter = new HelpFormatter();
// String latestDir;
String srcDir = BaseSettings.SRC_LISTS_DIR;
CommandLineParser parser = new DefaultParser();
HelpFormatter formatter = new HelpFormatter();
// String latestDir;
String srcDir = BaseSettings.SRC_LISTS_DIR;
try {
CommandLine cmd = parser.parse(cliOptions, args);
try {
CommandLine cmd = parser.parse(cliOptions, args);
if (cmd.hasOption("c")) {
FileUtils.cleanDirectory(BaseSettings.DOWNLOADED_LISTS_DIR_ASFILE);
FileUtils.cleanDirectory(BaseSettings.OUT_DIR_ASFILE);
FileUtils.cleanDirectory(BaseSettings.GIT_DIR_ASFILE);
java.lang.System.gc();
}
if (cmd.hasOption("c")) {
FileUtils.cleanDirectory(BaseSettings.DOWNLOADED_LISTS_DIR_ASFILE);
FileUtils.cleanDirectory(BaseSettings.OUT_DIR_ASFILE);
FileUtils.cleanDirectory(BaseSettings.GIT_DIR_ASFILE);
java.lang.System.gc();
}
if (cmd.hasOption("h")) {
formatter.printHelp("java -jar hostlistools-base.jar", cliOptions, true);
}
if (cmd.hasOption("p")) {
BaseSettings.isPermissive = true;
}
if (cmd.hasOption("i")) {
srcDir = cmd.getOptionValue("i");
}
if (cmd.hasOption("w")) {
File newWorkspace = new File(cmd.getOptionValue("w") + ".hostliststools/");
BaseSettings.LAST_USED_WORKSPACE = cmd.getOptionValue("w") + ".hostliststools/";
BaseSettings.settingsProperties.replace("LAST_USED_WORKSPACE", BaseSettings.LAST_USED_WORKSPACE);
if (!newWorkspace.exists()) {
new File(cmd.getOptionValue("w")).mkdir();
BaseFile.saveSettingsToFile(BaseSettings.SETTINGS_DIR + "settings.txt", true);
BaseFile.init();
}
}
if (cmd.hasOption("X")) {
System.out.println("Current Workspace: " + BaseSettings.LAST_USED_WORKSPACE);
System.out.println("Download Jobs: " + BaseSettings.DOWNLOAD_JOBS);
System.out.println("Log to file: " + BaseSettings.LOG_TO_FILE);
}
if (cmd.hasOption("h")) {
formatter.printHelp("java -jar hostlistools-base.jar", cliOptions, true);
}
if (cmd.hasOption("p")) {
BaseSettings.isPermissive = true;
}
if (cmd.hasOption("i")) {
srcDir = cmd.getOptionValue("i");
}
if (cmd.hasOption("w")) {
File newWorkspace = new File(cmd.getOptionValue("w") + ".hostliststools/");
BaseSettings.LAST_USED_WORKSPACE = cmd.getOptionValue("w") + ".hostliststools/";
BaseSettings.settingsProperties.replace("LAST_USED_WORKSPACE", BaseSettings.LAST_USED_WORKSPACE);
if (!newWorkspace.exists()) {
new File(cmd.getOptionValue("w")).mkdir();
BaseFile.saveSettingsToFile(BaseSettings.SETTINGS_DIR + "settings.txt", true);
BaseFile.init();
}
}
if (cmd.hasOption("X")) {
System.out.println("Current Workspace: " + BaseSettings.LAST_USED_WORKSPACE);
System.out.println("Download Jobs: " + BaseSettings.DOWNLOAD_JOBS);
System.out.println("Log to file: " + BaseSettings.LOG_TO_FILE);
}
if (cmd.hasOption("o")) {
BaseSettings.OUT_FILENAME = cmd.getOptionValue("o");
outputFileName = BaseSettings.OUT_FILENAME + "_" + dateformatter.format(date) + ".txt";
}
if (cmd.hasOption("o")) {
BaseSettings.OUT_FILENAME = cmd.getOptionValue("o");
outputFileName = BaseSettings.OUT_FILENAME + "_" + dateformatter.format(date) + ".txt";
}
if (cmd.hasOption("H")) {
skipHeading = true;
}
if (cmd.hasOption("K")) {
skipSources = true;
}
if (cmd.hasOption("H")) {
skipHeading = true;
}
if (cmd.hasOption("K")) {
skipSources = true;
}
if (cmd.hasOption("F")) {
fillwithString = cmd.getOptionValue("F");
}
if (cmd.hasOption("F")) {
fillwithString = cmd.getOptionValue("F");
}
if (cmd.hasOption("j")) {
System.out.println();
BaseSettings.DOWNLOAD_JOBS = Integer.parseInt(cmd.getOptionValue("j"));
BaseSettings.settingsProperties.replace("DOWNLOAD_JOBS", BaseSettings.DOWNLOAD_JOBS);
}
if (cmd.hasOption("Q")) {
BaseFile.saveMergedSetToOneFile(BaseTools.clearAndStoreAllRecords(
BaseMethods.generateSRCListForLocal(BaseSettings.DOWNLOADED_LISTS_DIR, "Generating file list"),
BaseMethods.readFolderToSetOfStrings(BaseSettings.EXPRESSIONS_DIR, "Generating Unwanted Expressions list",
BaseFunctions.removeCommentedLines(), BaseFunctions.removeEmptyLines()),
BaseMethods.readFolderToSetOfStrings(BaseSettings.WHITELIST_DIR, "Rading whitelisted records",
BaseFunctions.removeCommentedLines(), BaseFunctions.removeEmptyLines()),
fillwithString), BaseSettings.OUT_DIR, outputFileName, skipHeading, skipSources);
if (cmd.hasOption("j")) {
System.out.println();
BaseSettings.DOWNLOAD_JOBS = Integer.parseInt(cmd.getOptionValue("j"));
BaseSettings.settingsProperties.replace("DOWNLOAD_JOBS", BaseSettings.DOWNLOAD_JOBS);
}
if (cmd.hasOption("Q")) {
BaseFile.saveMergedSetToOneFile(BaseTools.clearAndStoreAllRecords(
BaseMethods.generateSRCListForLocal(BaseSettings.DOWNLOADED_LISTS_DIR, "Generating file list"),
BaseMethods.readFolderToSetOfStrings(BaseSettings.EXPRESSIONS_DIR, "Generating Unwanted Expressions list",
BaseFunctions.removeCommentedLines(), BaseFunctions.removeEmptyLines()),
BaseMethods.readFolderToSetOfStrings(BaseSettings.WHITELIST_DIR, "Rading whitelisted records",
BaseFunctions.removeCommentedLines(), BaseFunctions.removeEmptyLines()),
fillwithString), BaseSettings.OUT_DIR, outputFileName, skipHeading, skipSources);
}
}
if (cmd.hasOption("D")) {
// DOWNLOAD
BaseNet.downloadAllHostLists(BaseMethods.readFolderToListOfStrings(srcDir, "Reading source"),
BaseSettings.DOWNLOADED_LISTS_DIR,
Integer.parseInt(BaseSettings.settingsProperties.getProperty("DOWNLOAD_JOBS")), " ");
java.lang.System.gc();
}
if (cmd.hasOption("D")) {
// DOWNLOAD
BaseNet.downloadAllHostLists(BaseMethods.readFolderToListOfStrings(srcDir, "Reading source"),
BaseSettings.DOWNLOADED_LISTS_DIR,
Integer.parseInt(BaseSettings.settingsProperties.getProperty("DOWNLOAD_JOBS")), " ");
java.lang.System.gc();
}
if (cmd.hasOption("M")) {
BaseFile.saveMergedSetToOneFile(BaseTools.clearAndStoreAllRecords(
BaseMethods.generateSRCListForLocal(BaseSettings.DOWNLOADED_LISTS_DIR, "Generating file list"),
BaseMethods.readFolderToSetOfStrings(BaseSettings.EXPRESSIONS_DIR, "Generating Unwanted Expressions list",
BaseFunctions.removeCommentedLines(), BaseFunctions.removeEmptyLines()),
BaseMethods.readFolderToSetOfStrings(BaseSettings.WHITELIST_DIR, "Rading whitelisted records",
BaseFunctions.removeCommentedLines(), BaseFunctions.removeEmptyLines()),
fillwithString), BaseSettings.OUT_DIR, outputFileName, skipHeading, skipSources);
}
if (cmd.hasOption("M")) {
BaseFile.saveMergedSetToOneFile(BaseTools.clearAndStoreAllRecords(
BaseMethods.generateSRCListForLocal(BaseSettings.DOWNLOADED_LISTS_DIR, "Generating file list"),
BaseMethods.readFolderToSetOfStrings(BaseSettings.EXPRESSIONS_DIR, "Generating Unwanted Expressions list",
BaseFunctions.removeCommentedLines(), BaseFunctions.removeEmptyLines()),
BaseMethods.readFolderToSetOfStrings(BaseSettings.WHITELIST_DIR, "Rading whitelisted records",
BaseFunctions.removeCommentedLines(), BaseFunctions.removeEmptyLines()),
fillwithString), BaseSettings.OUT_DIR, outputFileName, skipHeading, skipSources);
}
if (cmd.hasOption("R")) {
BaseFile.saveMergedSetToOneFile(
BaseTools.clearAndStoreAllRecords(BaseMethods.generateSRCListForDirect(srcDir, "Generating src list"),
BaseMethods.readFolderToSetOfStrings(BaseSettings.EXPRESSIONS_DIR,
"Generating Unwanted Expressions list", BaseFunctions.removeCommentedLines(),
BaseFunctions.removeEmptyLines()),
BaseMethods.readFolderToSetOfStrings(BaseSettings.WHITELIST_DIR, "Rading whitelisted records",
BaseFunctions.removeCommentedLines(), BaseFunctions.removeEmptyLines()),
fillwithString),
BaseSettings.OUT_DIR, outputFileName, skipHeading, skipSources);
java.lang.System.gc();
}
if (cmd.hasOption("I")) {
String stats = BaseTools.genStats();
System.out.println(stats);
BaseFile.overrideReadme(stats);
}
if (cmd.hasOption("E")) {
if (!BaseFunctions.commentedLineSet.isEmpty()) {
BaseFile.saveSetToOneFile(BaseFunctions.commentedLineSet, BaseSettings.OUT_DIR, "EXCLUDED-commented.txt");
}
if (!BaseFunctions.semicolonedLineSet.isEmpty()) {
BaseFile.saveSetToOneFile(BaseFunctions.semicolonedLineSet, BaseSettings.OUT_DIR,
"RECOVERED-semicoloned.txt");
}
if (!BaseFunctions.slashedLineSet.isEmpty()) {
BaseFile.saveSetToOneFile(BaseFunctions.slashedLineSet, BaseSettings.OUT_DIR, "RECOVERED-slashed.txt");
}
if (!BaseFunctions.dashStartedSet.isEmpty()) {
BaseFile.saveSetToOneFile(BaseFunctions.dashStartedSet, BaseSettings.OUT_DIR, "EXCLUDED-dashed.txt");
}
if (!BaseFunctions.dollarStartedSet.isEmpty()) {
BaseFile.saveSetToOneFile(BaseFunctions.dollarStartedSet, BaseSettings.OUT_DIR, "EXCLUDED-dollared.txt");
}
if (!BaseFunctions.dotStartedSet.isEmpty()) {
BaseFile.saveSetToOneFile(BaseFunctions.dotStartedSet, BaseSettings.OUT_DIR, "EXCLUDED-dotted.txt");
}
if (!BaseFunctions.lineSetWithCommentsSet.isEmpty()) {
BaseFile.saveSetToOneFile(BaseFunctions.lineSetWithCommentsSet, BaseSettings.OUT_DIR,
"RECOVERED-afterline-commented.txt");
}
if (!BaseFunctions.notURLSet.isEmpty()) {
BaseFile.saveSetToOneFile(BaseFunctions.notURLSet, BaseSettings.OUT_DIR, "EXCLUDED-notURLs.txt");
}
if (!BaseFunctions.dnsmasqSet.isEmpty()) {
BaseFile.saveSetToOneFile(BaseFunctions.dnsmasqSet, BaseSettings.OUT_DIR, "RECOVERED-dnsmasq.txt");
}
if (cmd.hasOption("R")) {
BaseFile.saveMergedSetToOneFile(
BaseTools.clearAndStoreAllRecords(BaseMethods.generateSRCListForDirect(srcDir, "Generating src list"),
BaseMethods.readFolderToSetOfStrings(BaseSettings.EXPRESSIONS_DIR,
"Generating Unwanted Expressions list", BaseFunctions.removeCommentedLines(),
BaseFunctions.removeEmptyLines()),
BaseMethods.readFolderToSetOfStrings(BaseSettings.WHITELIST_DIR, "Rading whitelisted records",
BaseFunctions.removeCommentedLines(), BaseFunctions.removeEmptyLines()),
fillwithString),
BaseSettings.OUT_DIR, outputFileName, skipHeading, skipSources);
java.lang.System.gc();
}
if (cmd.hasOption("I")) {
String stats = BaseTools.genStats();
System.out.println(stats);
BaseFile.overrideReadme(stats);
}
if (cmd.hasOption("E")) {
if (!BaseFunctions.commentedLineSet.isEmpty()) {
BaseFile.saveSetToOneFile(BaseFunctions.commentedLineSet, BaseSettings.OUT_DIR, "EXCLUDED-commented.txt");
}
if (!BaseFunctions.semicolonedLineSet.isEmpty()) {
BaseFile.saveSetToOneFile(BaseFunctions.semicolonedLineSet, BaseSettings.OUT_DIR,
"RECOVERED-semicoloned.txt");
}
if (!BaseFunctions.slashedLineSet.isEmpty()) {
BaseFile.saveSetToOneFile(BaseFunctions.slashedLineSet, BaseSettings.OUT_DIR, "RECOVERED-slashed.txt");
}
if (!BaseFunctions.dashStartedSet.isEmpty()) {
BaseFile.saveSetToOneFile(BaseFunctions.dashStartedSet, BaseSettings.OUT_DIR, "EXCLUDED-dashed.txt");
}
if (!BaseFunctions.dollarStartedSet.isEmpty()) {
BaseFile.saveSetToOneFile(BaseFunctions.dollarStartedSet, BaseSettings.OUT_DIR, "EXCLUDED-dollared.txt");
}
if (!BaseFunctions.dotStartedSet.isEmpty()) {
BaseFile.saveSetToOneFile(BaseFunctions.dotStartedSet, BaseSettings.OUT_DIR, "EXCLUDED-dotted.txt");
}
if (!BaseFunctions.lineSetWithCommentsSet.isEmpty()) {
BaseFile.saveSetToOneFile(BaseFunctions.lineSetWithCommentsSet, BaseSettings.OUT_DIR,
"RECOVERED-afterline-commented.txt");
}
if (!BaseFunctions.notURLSet.isEmpty()) {
BaseFile.saveSetToOneFile(BaseFunctions.notURLSet, BaseSettings.OUT_DIR, "EXCLUDED-notURLs.txt");
}
if (!BaseFunctions.dnsmasqSet.isEmpty()) {
BaseFile.saveSetToOneFile(BaseFunctions.dnsmasqSet, BaseSettings.OUT_DIR, "RECOVERED-dnsmasq.txt");
}
}
}
if (cmd.hasOption("T")) {
// GitWrapper.gitDebugCloneShallow(BaseSettings.GIT_DIR_ASFILE);
}
// if (cmd.hasOption("I")) {
// GitWrapper.gitInit(BaseSettings.GIT_DIR_ASFILE);
// GitWrapper.gitRemoteAdd(BaseSettings.GIT_DIR_ASFILE,
// cmd.getOptionValues("I"));
// GitWrapper.gitPush(BaseSettings.GIT_DIR_ASFILE);
// }
//
if (cmd.hasOption("T")) {
// GitWrapper.gitDebugCloneShallow(BaseSettings.GIT_DIR_ASFILE);
}
// if (cmd.hasOption("I")) {
// GitWrapper.gitInit(BaseSettings.GIT_DIR_ASFILE);
// GitWrapper.gitRemoteAdd(BaseSettings.GIT_DIR_ASFILE,
// cmd.getOptionValues("I"));
// GitWrapper.gitPush(BaseSettings.GIT_DIR_ASFILE);
// }
//
//
// if (cmd.hasOption("C")) {
// GitWrapper.gitAddToIndex(BaseSettings.GIT_DIR_ASFILE, ".");
// GitWrapper.gitCommit(BaseSettings.GIT_DIR_ASFILE, "-m \" Update on merge file
// \" ");
// }
// if (cmd.hasOption("P")) {
// GitWrapper.gitPush(BaseSettings.GIT_DIR_ASFILE);
// }
if (cmd.hasOption("S")) {
BaseNet.gitConnect(cmd.getOptionValue("S"));
}
//
// if (cmd.hasOption("C")) {
// GitWrapper.gitAddToIndex(BaseSettings.GIT_DIR_ASFILE, ".");
// GitWrapper.gitCommit(BaseSettings.GIT_DIR_ASFILE, "-m \" Update on merge file
// \" ");
// }
// if (cmd.hasOption("P")) {
// GitWrapper.gitPush(BaseSettings.GIT_DIR_ASFILE);
// }
if (cmd.hasOption("S")) {
BaseNet.gitConnect(cmd.getOptionValue("S"));
}
// if (cmd.hasOption("P")) {
// // Cannot use native git wrapper because processes ares not beeing
// synchronized...
// // Cannot use jgit, as it's too complex with weak documentation
// // so there's need to create system specific script... It's a shame for Java
// Runtime r = Runtime.getRuntime();
// Process p = r.exec("bash " + BaseSettings.SCRIPTS_LISTS_DIR + "bash-git.sh");
// p.waitFor();
// try (BufferedReader b = new BufferedReader(new
// InputStreamReader(p.getInputStream()));) {
// b.lines().forEach(System.out::println);
// }
//
//// GitWrapper.passCommandLinuxUnified(BaseSettings.SCRIPTS_LISTS_DIR_ASFILE,
// "bash linux-git.sh");
// }
if (args.length == 0) {
formatter.printHelp("java -jar hostlistools-cli.jar", cliOptions, true);
return;
}
// if (cmd.hasOption("P")) {
// // Cannot use native git wrapper because processes ares not beeing
// synchronized...
// // Cannot use jgit, as it's too complex with weak documentation
// // so there's need to create system specific script... It's a shame for Java
// Runtime r = Runtime.getRuntime();
// Process p = r.exec("bash " + BaseSettings.SCRIPTS_LISTS_DIR + "bash-git.sh");
// p.waitFor();
// try (BufferedReader b = new BufferedReader(new
// InputStreamReader(p.getInputStream()));) {
// b.lines().forEach(System.out::println);
// }
//
//// GitWrapper.passCommandLinuxUnified(BaseSettings.SCRIPTS_LISTS_DIR_ASFILE,
// "bash linux-git.sh");
// }
if (args.length == 0) {
formatter.printHelp("java -jar hostlistools-cli.jar", cliOptions, true);
return;
}
} catch (ParseException | IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
// TODO Auto-generated catch block
} catch (ParseException | IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
// TODO Auto-generated catch block
// TODO Auto-generated catch block
BaseFile.saveSettingsToFile(BaseSettings.SETTINGS_DIR + "settings.txt", true);
java.lang.System.gc();
}
// TODO Auto-generated catch block
BaseFile.saveSettingsToFile(BaseSettings.SETTINGS_DIR + "settings.txt", true);
java.lang.System.gc();
}
}

515
src/main/java/org/hostliststools/cli/io/BaseFile.java

@ -17,284 +17,323 @@ import java.util.List;
import java.util.Set;
import java.util.regex.Pattern;
import java.util.stream.Stream;
import me.tongfei.progressbar.ProgressBar;
import org.apache.commons.lang3.StringUtils;
import org.hostliststools.cli.app.BaseApp;
import org.hostliststools.cli.logic.BaseTools;
import org.hostliststools.cli.process.BaseFunctions;
import org.hostliststools.cli.process.BaseMethods;
import me.tongfei.progressbar.ProgressBar;
import org.hostliststools.cli.wrapper.GitWrapper;
public class BaseFile {
public static void init() throws URISyntaxException {
readSettingFromFile(searchForSettingFile());
// readSerialized();
setProperties();
}
// private static void readSerialized() {
//
// try {
// if (new File(BaseSettings.SETTINGS_DIR + "u.dat").exists()) {
// BaseApp.git = Git.open(new File(BaseSettings.GIT_DIR + ".git"));
// BaseApp.mainGitUser = (org.hostliststools.cli.objects.Git) Serialization
// .deserializeObj(BaseSettings.SETTINGS_DIR + "u.dat");
// }
// } catch (IOException e) {
// // TODO Auto-generated catch block
// e.printStackTrace();
// }
// }
public static void readSettingFromFile(String localWorkdir) {
try (FileReader fReader = new FileReader(localWorkdir); BufferedReader bReader = new BufferedReader(fReader);) {
bReader.lines().filter(BaseFunctions.removeCommentedLines()).filter(BaseFunctions.removeEmptyLines())
.forEach(setting -> {
String settingName = StringUtils.substringBefore(setting, "=");
String settingValue = StringUtils.substringAfter(setting, "=");
for (Field field : BaseSettings.class.getDeclaredFields()) {
if (settingName.equals(field.getName())) {
BaseSettings.settingsProperties.setProperty(settingName, settingValue);
// System.out.println("Setting property: " + settingName + " = " + settingValue); // DEBUG
} else {
// System.out.println("No such setting: " + setting);
}
public static void init() throws URISyntaxException {
newReadSettingFromFile();
// setProperties();
}
// public static void readSettingFromFile(String localWorkdir) {
// try (FileReader fReader = new FileReader(localWorkdir); BufferedReader bReader = new BufferedReader(fReader);) {
// bReader.lines().filter(BaseFunctions.removeCommentedLines()).filter(BaseFunctions.removeEmptyLines())
// .forEach(setting -> {
// String settingName = StringUtils.substringBefore(setting, "=");
// String settingValue = StringUtils.substringAfter(setting, "=");
//
// for (Field field : BaseSettings.class.getDeclaredFields()) {
// if (settingName.equals(field.getName())) {
// BaseSettings.settingsProperties.setProperty(settingName, settingValue);
// // System.out.println("Setting property: " + settingName + " = " + settingValue); // DEBUG
// } else {
// // System.out.println("No such setting: " + setting);
// }
//
// }
// });
//
// BaseSettings.WORKDIR = BaseSettings.settingsProperties.getProperty("LAST_USED_WORKSPACE");
// setRestFolderVars();
// System.out.println("DEBUG: " + BaseSettings.WORKDIR);
//
// } catch (IOException e1) {
// BaseMethods.printErrorMessageAndExitWithState1(e1);
// }
//
// }
public static void newReadSettingFromFile() {
BaseSettings.WORKDIR = "/home/" + System.getProperty("user.name") + "/.hostliststools/";
// BaseSettings.WORKDIR = BaseSettings.LAST_USED_WORKSPACE;
setRestFolderVars();
String filePath = BaseSettings.SETTINGS_DIR + "settings.txt";
File settingsFile = new File(filePath);
if (!new File(BaseSettings.WORKDIR).exists()) {
System.out.println("No workspace in," + filePath + " creating...");
BaseSettings.workdirSet_ASFILE.forEach(File::mkdir);
BaseSettings.LAST_USED_WORKSPACE = BaseSettings.WORKDIR;
filePath = BaseSettings.SETTINGS_DIR + "settings.txt";
settingsFile = new File(filePath);
try {
settingsFile.createNewFile();
// TODO add properties to allow below method to execute because now properties
// are empty so file is not being overwritten
BaseSettings.settingsProperties.setProperty("DOWNLOAD_JOBS", "4");
BaseSettings.settingsProperties.setProperty("LOG_TO_FILE", "true");
BaseSettings.settingsProperties.setProperty("LAST_USED_WORKSPACE", BaseSettings.WORKDIR);
BaseSettings.settingsProperties.setProperty("GIT_REMOTE_PATH", "unknown");
BaseSettings.settingsProperties.setProperty("OUT_FILENAME", "merged");
saveSettingsToFile(filePath, true);
saveStringToFile(GitWrapper.bashScript, BaseSettings.SCRIPTS_LISTS_DIR, "bash-git.sh");
setProperties();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
} else {
System.out.println("DEBUG: " + BaseSettings.WORKDIR);
try (FileReader fReader = new FileReader(filePath); BufferedReader bReader = new BufferedReader(fReader);) {
bReader.lines().filter(BaseFunctions.removeCommentedLines()).filter(BaseFunctions.removeEmptyLines())
.forEach(setting -> {
String settingName = StringUtils.substringBefore(setting, "=");
String settingValue = StringUtils.substringAfter(setting, "=");
System.out.println(settingName + " " + settingValue);
for (Field field : BaseSettings.class.getDeclaredFields()) {
if (settingName.equals(field.getName())) {
System.out.println(settingName + " " + settingValue);
BaseSettings.settingsProperties.setProperty(settingName, settingValue);
// System.out.println("Setting property: " + settingName + " = " + settingValue); // DEBUG
} else {
// System.out.println("No such setting: " + setting);
}
}
});
setProperties();
BaseSettings.WORKDIR = BaseSettings.LAST_USED_WORKSPACE;
setRestFolderVars();
} catch (IOException e1) {
BaseMethods.printErrorMessageAndExitWithState1(e1);
}
}
});
BaseSettings.WORKDIR = BaseSettings.settingsProperties.getProperty("LAST_USED_WORKSPACE");
setRestFolderVars();
System.out.println("DEBUG: " + BaseSettings.WORKDIR);
} catch (IOException e1) {
BaseMethods.printErrorMessageAndExitWithState1(e1);
System.out.println("DEBUG: " + BaseSettings.WORKDIR);
}
}
public static String searchForSettingFile() throws URISyntaxException {
if (BaseSettings.WORKDIR == null) {
String runningJarPath = BaseApp.class.getProtectionDomain().getCodeSource().getLocation().toURI().getPath();
String[] splitted = runningJarPath.split("/");
StringBuilder sBuilder = new StringBuilder();
for (int i = 0; i < splitted.length - 1; i++) {
sBuilder.append(splitted[i] + "/");
}
String rmLastLine = sBuilder.toString();
BaseSettings.WORKDIR = rmLastLine + ".hostliststools/";
} else {
BaseSettings.WORKDIR = BaseSettings.LAST_USED_WORKSPACE;
// public static String searchForSettingFile() throws URISyntaxException {
// if (BaseSettings.WORKDIR == null) {
//
// String runningJarPath = BaseApp.class.getProtectionDomain().getCodeSource().getLocation().toURI().getPath();
// String[] splitted = runningJarPath.split("/");
// StringBuilder sBuilder = new StringBuilder();
// for (int i = 0; i < splitted.length - 1; i++) {
// sBuilder.append(splitted[i] + "/");
// }
// String rmLastLine = sBuilder.toString();
// BaseSettings.WORKDIR = rmLastLine + ".hostliststools/";
// } else {
// BaseSettings.WORKDIR = BaseSettings.LAST_USED_WORKSPACE;
// }
//
// setRestFolderVars();
// String filePath = BaseSettings.SETTINGS_DIR + "settings.txt";
// File settingsFile = new File(filePath);
// if (!new File(BaseSettings.WORKDIR).exists()) {
// System.out.println("No workspace, creating...");
//
// BaseSettings.workdirSet_ASFILE.forEach(file -> {
// file.mkdir();
// });
// BaseSettings.LAST_USED_WORKSPACE = BaseSettings.WORKDIR;
// filePath = BaseSettings.SETTINGS_DIR + "settings.txt";
// settingsFile = new File(filePath);
// try {
// settingsFile.createNewFile();
// // TODO add properties to allow below method to execute because now properties
// // are empty so file is not being overwritten
// BaseSettings.settingsProperties.setProperty("DOWNLOAD_JOBS", "4");
// BaseSettings.settingsProperties.setProperty("LOG_TO_FILE", "true");
// BaseSettings.settingsProperties.setProperty("LAST_USED_WORKSPACE", BaseSettings.WORKDIR);
// BaseSettings.settingsProperties.setProperty("GIT_REMOTE_PATH", "unknown");
// BaseSettings.settingsProperties.setProperty("OUT_FILENAME", "merged");
// saveSettingsToFile(filePath, true);
// saveStringToFile(GitWrapper.bashScript, BaseSettings.SCRIPTS_LISTS_DIR, "bash-git.sh");
// } catch (IOException e) {
// // TODO Auto-generated catch block
// e.printStackTrace();
// }
// }
// return filePath;
//
// }
private static void setRestFolderVars() {
BaseSettings.SETTINGS_DIR = BaseSettings.WORKDIR + "settings/";
BaseSettings.SCRIPTS_LISTS_DIR = BaseSettings.WORKDIR + "scripts/";
BaseSettings.DOWNLOADED_LISTS_DIR = BaseSettings.WORKDIR + "downloaded/";
BaseSettings.EXPRESSIONS_DIR = BaseSettings.WORKDIR + "unwantedExpressions/";
BaseSettings.LOG_DIR = BaseSettings.WORKDIR + "log/";
BaseSettings.OUT_DIR = BaseSettings.WORKDIR + "out/";
BaseSettings.SRC_LISTS_DIR = BaseSettings.WORKDIR + "src/";
BaseSettings.WHITELIST_DIR = BaseSettings.WORKDIR + "whiteList/";
BaseSettings.BLACKLIST_DIR = BaseSettings.WORKDIR + "blacklist/";
BaseSettings.GIT_DIR = BaseSettings.WORKDIR + "git/";
BaseSettings.GIT_DIR_ARCHIVE = BaseSettings.GIT_DIR + "archive/";
BaseSettings.GIT_DIR_SRC = BaseSettings.GIT_DIR + "src/";
BaseSettings.WORKDIR_ASFILE = new File(BaseSettings.WORKDIR);
BaseSettings.SCRIPTS_LISTS_DIR_ASFILE = new File(BaseSettings.SCRIPTS_LISTS_DIR);
BaseSettings.OUT_DIR_ASFILE = new File(BaseSettings.OUT_DIR);
BaseSettings.DOWNLOADED_LISTS_DIR_ASFILE = new File(BaseSettings.DOWNLOADED_LISTS_DIR);
BaseSettings.SETTINGS_DIR_ASFILE = new File(BaseSettings.SETTINGS_DIR);
BaseSettings.SRC_LISTS_DIR_ASFILE = new File(BaseSettings.SRC_LISTS_DIR);
BaseSettings.LOG_DIR_ASFILE = new File(BaseSettings.LOG_DIR);
BaseSettings.EXPRESSIONS_DIR_ASFILE = new File(BaseSettings.EXPRESSIONS_DIR);
BaseSettings.WHITELIST_DIR_ASFILE = new File(BaseSettings.WHITELIST_DIR);
BaseSettings.BLACKLIST_DIR_ASFILE = new File(BaseSettings.BLACKLIST_DIR);
BaseSettings.GIT_DIR_ASFILE = new File(BaseSettings.GIT_DIR);
BaseSettings.GIT_DIR_SRC_ASFILE = new File(BaseSettings.GIT_DIR_SRC);
BaseSettings.GIT_DIR_ARCHIVE_ASFILE = new File(BaseSettings.GIT_DIR_ARCHIVE);
BaseSettings.workdirSet_ASFILE = List.of(BaseSettings.WORKDIR_ASFILE, BaseSettings.SETTINGS_DIR_ASFILE,
BaseSettings.SCRIPTS_LISTS_DIR_ASFILE, BaseSettings.OUT_DIR_ASFILE,
BaseSettings.DOWNLOADED_LISTS_DIR_ASFILE, BaseSettings.SRC_LISTS_DIR_ASFILE,
BaseSettings.LOG_DIR_ASFILE, BaseSettings.EXPRESSIONS_DIR_ASFILE, BaseSettings.WHITELIST_DIR_ASFILE,
BaseSettings.GIT_DIR_ASFILE, BaseSettings.GIT_DIR_SRC_ASFILE, BaseSettings.GIT_DIR_ARCHIVE_ASFILE,
BaseSettings.BLACKLIST_DIR_ASFILE);
BaseSettings.workdirSet = List.of(BaseSettings.WORKDIR, BaseSettings.SCRIPTS_LISTS_DIR, BaseSettings.OUT_DIR,
BaseSettings.DOWNLOADED_LISTS_DIR, BaseSettings.SETTINGS_DIR, BaseSettings.SRC_LISTS_DIR,
BaseSettings.LOG_DIR, BaseSettings.EXPRESSIONS_DIR, BaseSettings.WHITELIST_DIR, BaseSettings.GIT_DIR,
BaseSettings.GIT_DIR_SRC, BaseSettings.GIT_DIR_ARCHIVE, BaseSettings.BLACKLIST_DIR);
}
setRestFolderVars();
String filePath = BaseSettings.SETTINGS_DIR + "settings.txt";
File settingsFile = new File(filePath);
if (!new File(BaseSettings.WORKDIR).exists()) {
System.out.println("No workspace, creating...");
BaseSettings.workdirSet_ASFILE.forEach(file -> {
file.mkdir();
});
BaseSettings.LAST_USED_WORKSPACE = BaseSettings.WORKDIR;
filePath = BaseSettings.SETTINGS_DIR + "settings.txt";
settingsFile = new File(filePath);
try {
settingsFile.createNewFile();
// TODO add properties to allow below method to execute because now properties
// are empty so file is not being overwritten
BaseSettings.settingsProperties.setProperty("DOWNLOAD_JOBS", "4");
BaseSettings.settingsProperties.setProperty("LOG_TO_FILE", "true");
BaseSettings.settingsProperties.setProperty("LAST_USED_WORKSPACE", BaseSettings.WORKDIR);
BaseSettings.settingsProperties.setProperty("GIT_REMOTE_PATH", "unknown");
BaseSettings.settingsProperties.setProperty("OUT_FILENAME", "merged");
saveSettingsToFile(filePath, true);
saveStringToFile(GitWrapper.bashScript, BaseSettings.SCRIPTS_LISTS_DIR, "bash-git.sh");
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
return filePath;
}
private static void setRestFolderVars() {
BaseSettings.SETTINGS_DIR = BaseSettings.WORKDIR + "settings/";
BaseSettings.SCRIPTS_LISTS_DIR = BaseSettings.WORKDIR + "scripts/";
BaseSettings.DOWNLOADED_LISTS_DIR = BaseSettings.WORKDIR + "downloaded/";
BaseSettings.EXPRESSIONS_DIR = BaseSettings.WORKDIR + "unwantedExpressions/";
BaseSettings.LOG_DIR = BaseSettings.WORKDIR + "log/";
BaseSettings.OUT_DIR = BaseSettings.WORKDIR + "out/";
BaseSettings.SRC_LISTS_DIR = BaseSettings.WORKDIR + "src/";
BaseSettings.WHITELIST_DIR = BaseSettings.WORKDIR + "whiteList/";
BaseSettings.BLACKLIST_DIR = BaseSettings.WORKDIR + "blacklist/";
BaseSettings.GIT_DIR = BaseSettings.WORKDIR + "git/";
BaseSettings.GIT_DIR_ARCHIVE = BaseSettings.GIT_DIR + "archive/";
BaseSettings.GIT_DIR_SRC = BaseSettings.GIT_DIR + "src/";
BaseSettings.WORKDIR_ASFILE = new File(BaseSettings.WORKDIR);
BaseSettings.SCRIPTS_LISTS_DIR_ASFILE = new File(BaseSettings.SCRIPTS_LISTS_DIR);
BaseSettings.OUT_DIR_ASFILE = new File(BaseSettings.OUT_DIR);
BaseSettings.DOWNLOADED_LISTS_DIR_ASFILE = new File(BaseSettings.DOWNLOADED_LISTS_DIR);
BaseSettings.SETTINGS_DIR_ASFILE = new File(BaseSettings.SETTINGS_DIR);
BaseSettings.SRC_LISTS_DIR_ASFILE = new File(BaseSettings.SRC_LISTS_DIR);
BaseSettings.LOG_DIR_ASFILE = new File(BaseSettings.LOG_DIR);
BaseSettings.EXPRESSIONS_DIR_ASFILE = new File(BaseSettings.EXPRESSIONS_DIR);
BaseSettings.WHITELIST_DIR_ASFILE = new File(BaseSettings.WHITELIST_DIR);
BaseSettings.BLACKLIST_DIR_ASFILE = new File(BaseSettings.BLACKLIST_DIR);
BaseSettings.GIT_DIR_ASFILE = new File(BaseSettings.GIT_DIR);
BaseSettings.GIT_DIR_SRC_ASFILE = new File(BaseSettings.GIT_DIR_SRC);
BaseSettings.GIT_DIR_ARCHIVE_ASFILE = new File(BaseSettings.GIT_DIR_ARCHIVE);
BaseSettings.workdirSet_ASFILE = List.of(BaseSettings.WORKDIR_ASFILE, BaseSettings.SETTINGS_DIR_ASFILE,
BaseSettings.SCRIPTS_LISTS_DIR_ASFILE, BaseSettings.OUT_DIR_ASFILE,
BaseSettings.DOWNLOADED_LISTS_DIR_ASFILE, BaseSettings.SRC_LISTS_DIR_ASFILE,
BaseSettings.LOG_DIR_ASFILE, BaseSettings.EXPRESSIONS_DIR_ASFILE, BaseSettings.WHITELIST_DIR_ASFILE,
BaseSettings.GIT_DIR_ASFILE, BaseSettings.GIT_DIR_SRC_ASFILE, BaseSettings.GIT_DIR_ARCHIVE_ASFILE,
BaseSettings.BLACKLIST_DIR_ASFILE);
BaseSettings.workdirSet = List.of(BaseSettings.WORKDIR, BaseSettings.SCRIPTS_LISTS_DIR, BaseSettings.OUT_DIR,
BaseSettings.DOWNLOADED_LISTS_DIR, BaseSettings.SETTINGS_DIR, BaseSettings.SRC_LISTS_DIR,
BaseSettings.LOG_DIR, BaseSettings.EXPRESSIONS_DIR, BaseSettings.WHITELIST_DIR, BaseSettings.GIT_DIR,
BaseSettings.GIT_DIR_SRC, BaseSettings.GIT_DIR_ARCHIVE, BaseSettings.BLACKLIST_DIR);
}
private static void setProperties() {
BaseSettings.LOG_TO_FILE = Boolean.parseBoolean(BaseSettings.settingsProperties.getProperty("LOG_TO_FILE"));
BaseSettings.DOWNLOAD_JOBS = Integer.parseInt(BaseSettings.settingsProperties.getProperty("DOWNLOAD_JOBS"));
BaseSettings.LAST_USED_WORKSPACE = BaseSettings.settingsProperties.getProperty("LAST_USED_WORKSPACE");
BaseSettings.GIT_REMOTE_PATH = BaseSettings.settingsProperties.getProperty("GIT_REMOTE_PATH");
BaseSettings.OUT_FILENAME = BaseSettings.settingsProperties.getProperty("OUT_FILENAME");
}
public static void saveMergedSetToOneFile(Set<String> records, String outFolder, String fileName,
boolean skipHeading, boolean skipSources) throws IOException {
File outFile = new File(outFolder, fileName);
if (!skipHeading) {
outFile.createNewFile();
if (BaseNet.srcListList.isEmpty()) {
BaseNet.srcListList = BaseMethods.readFolderToArrayListOfStrings(BaseSettings.SRC_LISTS_DIR,
"Generating source list");
}
BaseTools.generateIntro(BaseNet.srcListList, outFile, skipSources, records.size());
private static void setProperties() {
BaseSettings.LOG_TO_FILE = Boolean.parseBoolean(BaseSettings.settingsProperties.getProperty("LOG_TO_FILE"));
BaseSettings.DOWNLOAD_JOBS = Integer.parseInt(BaseSettings.settingsProperties.getProperty("DOWNLOAD_JOBS"));
BaseSettings.LAST_USED_WORKSPACE = BaseSettings.settingsProperties.getProperty("LAST_USED_WORKSPACE");
BaseSettings.GIT_REMOTE_PATH = BaseSettings.settingsProperties.getProperty("GIT_REMOTE_PATH");
BaseSettings.OUT_FILENAME = BaseSettings.settingsProperties.getProperty("OUT_FILENAME");
}
ProgressBar.wrap(records.stream(), "Saving to file ").forEach(BaseFunctions.saveAllRecordsToOneFile(outFile));
java.lang.System.gc();
}
public static void saveSetToOneFile(Set<String> records, String outFolder, String fileName) throws IOException {
File outFile = new File(outFolder, fileName);
outFile.createNewFile();
public static void saveMergedSetToOneFile(Set<String> records, String outFolder, String fileName,
boolean skipHeading, boolean skipSources) throws IOException {
File outFile = new File(outFolder, fileName);
if (!skipHeading) {
outFile.createNewFile();
if (BaseNet.srcListList.isEmpty()) {
BaseNet.srcListList = BaseMethods.readFolderToArrayListOfStrings(BaseSettings.SRC_LISTS_DIR,
"Generating source list");
}
BaseTools.generateIntro(BaseNet.srcListList, outFile, skipSources, records.size());
}
ProgressBar.wrap(records.stream(), "Saving to file ").forEach(BaseFunctions.saveAllRecordsToOneFile(outFile));
java.lang.System.gc();
}
ProgressBar.wrap(records.stream(), "Saving to file ").forEach(BaseFunctions.saveAllRecordsToOneFile(outFile, records));
java.lang.System.gc();
}
public static void saveSettingsToFile(String fileName, boolean override) {
public static void saveSetToOneFile(Set<String> records, String outFolder, String fileName) throws IOException {
File outFile = new File(outFolder, fileName);
outFile.createNewFile();
File file = new File(fileName);
if (override) {
file.delete();
try {
file.createNewFile();
} catch (IOException e) {
e.printStackTrace();
}
ProgressBar.wrap(records.stream(), "Saving to file ").forEach(BaseFunctions.saveAllRecordsToOneFile(outFile, records));
java.lang.System.gc();
}
BaseSettings.settingsProperties.forEach((name, prop) -> {
try (FileWriter fw = new FileWriter(file, true); BufferedWriter bw = new BufferedWriter(fw);) {
public static void saveSettingsToFile(String fileName, boolean override) {
bw.write(name.toString());
bw.write("=" );
bw.write(prop.toString());
bw.write("\n\n");
File file = new File(fileName);
if (override) {
file.delete();
try {
file.createNewFile();
} catch (IOException e) {
e.printStackTrace();
}
}
BaseSettings.settingsProperties.forEach((name, prop) -> {
try (FileWriter fw = new FileWriter(file, true); BufferedWriter bw = new BufferedWriter(fw);) {
} catch (IOException e) {
e.printStackTrace();
// TODO: handle exception
}
});
bw.write(name.toString());
bw.write("=");
bw.write(prop.toString());
bw.write("\n\n");
}
} catch (IOException e) {
e.printStackTrace();
// TODO: handle exception
}
});
public static void gitActionCopyResult() {
}
ProgressBar pBar = new ProgressBar("Copying", 2);
public static void gitActionCopyResult() {
File destFile = new File(BaseSettings.GIT_DIR + BaseSettings.OUT_FILENAME);
try {
destFile.createNewFile();
} catch (IOException e1) {
// TODO Auto-generated catch block
e1.printStackTrace();
}
try (Stream<Path> paths = Files.walk(Paths.get(BaseSettings.OUT_DIR_ASFILE.toURI()), 2)) {
paths.filter(Files::isRegularFile).forEach(pat -> {
String pattern1 = "^" + Pattern.quote(BaseSettings.OUT_FILENAME) + "(.*?)";
boolean matches1 = Pattern.matches(pattern1, pat.getFileName().toString().toLowerCase());
File sourceFile = new File(pat.toAbsolutePath().toString());
pBar.step();
if (matches1) {
Communicats.printFoundMatching(BaseSettings.OUT_FILENAME);
try (FileChannel source = new RandomAccessFile(sourceFile, "rw").getChannel();
FileChannel destination = new RandomAccessFile(destFile, "rw").getChannel();) {
long position = 0;
long count = source.size();
source.transferTo(position, count, destination);
pBar.step();
} catch (IOException e) {
ProgressBar pBar = new ProgressBar("Copying", 2);
File destFile = new File(BaseSettings.GIT_DIR + BaseSettings.OUT_FILENAME);
try {
destFile.createNewFile();
} catch (IOException e1) {
// TODO Auto-generated catch block
e1.printStackTrace();
}
try (Stream<Path> paths = Files.walk(Paths.get(BaseSettings.OUT_DIR_ASFILE.toURI()), 2)) {
paths.filter(Files::isRegularFile).forEach(pat -> {
String pattern1 = "^" + Pattern.quote(BaseSettings.OUT_FILENAME) + "(.*?)";
boolean matches1 = Pattern.matches(pattern1, pat.getFileName().toString().toLowerCase());
File sourceFile = new File(pat.toAbsolutePath().toString());
pBar.step();
if (matches1) {
Communicats.printFoundMatching(BaseSettings.OUT_FILENAME);
try (FileChannel source = new RandomAccessFile(sourceFile, "rw").getChannel();
FileChannel destination = new RandomAccessFile(destFile, "rw").getChannel();) {