Implementing logic, added first test

.gitignore

@@ -21,3 +21,4 @@
 # Go workspace file
 go.work
 
+hstl

LICENSE

@@ -1,16 +0,0 @@
-Copyright (c) year copyright holder. All Rights Reserved.
-
-Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
-
-1.
-Redistribution of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
-
-2.
-Redistribution in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
-
-3.
-Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-YOU ACKNOWLEDGE THAT THIS SOFTWARE IS NOT DESIGNED, LICENSED OR INTENDED FOR USE IN THE DESIGN, CONSTRUCTION, OPERATION OR MAINTENANCE OF ANY MILITARY FACILITY.

README.md

@@ -1,3 +1,23 @@
 # hstl
+Ultimate tool for managing hostlists written in go
 
-Ultimate tool for managing hostlists written in go
+# Status
+ALPHA
+
+# Description
+Manipulate hostlists:
+- clean from non-url
+- remove/add prefixes like 127.0.0.1
+- remove/add hostlist scheme
+- convert between formats
+- merge lists into one
+
+# Motivation
+There are dozens of lists in different formats.
+
+Setting up pfblockernng/adguard/pihole with > 10 lists (in my case ~88 is pain) especially if some of recods are broken.
+
+My frined actually solved it by scripting with sed, but I want some more control over it.
+
+# Technical details
+Used parallel lazy streams (something like java-8 streams / .NET linq)

cmd/root.go

@@ -0,0 +1,51 @@
+/*
+Copyright © 2023 NAME HERE <EMAIL ADDRESS>
+*/
+package cmd
+
+import (
+	"os"
+
+	"git.itmodulo.eu/go/hstl/cmd/transform"
+	"github.com/spf13/cobra"
+)
+
+// rootCmd represents the base command when called without any subcommands
+var rootCmd = &cobra.Command{
+	Use:   "hstl",
+	Short: "Ultimate tool for managing hostlists",
+	Long: `
+This tool allows to modify your hostlists by
+merging them, removing duplicates, removing broken ones
+it can also transform them into different formats.
+	`,
+	// Uncomment the following line if your bare application
+	// has an action associated with it:
+	// Run: func(cmd *cobra.Command, args []string) { },
+}
+
+// Execute adds all child commands to the root command and sets flags appropriately.
+// This is called by main.main(). It only needs to happen once to the rootCmd.
+func Execute() {
+	err := rootCmd.Execute()
+	if err != nil {
+		os.Exit(1)
+	}
+}
+
+func init() {
+	// Here you will define your flags and configuration settings.
+	// Cobra supports persistent flags, which, if defined here,
+	// will be global for your application.
+
+	// rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is $HOME/.hstl.yaml)")
+
+	// Cobra also supports local flags, which will only run
+	// when this action is called directly.
+	rootCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle")
+	addSubcommandPalettes()
+}
+
+func addSubcommandPalettes() {
+	rootCmd.AddCommand(transform.TransformCmd)
+}

cmd/transform/transform.go

@@ -0,0 +1,29 @@
+/*
+Copyright © 2023 NAME HERE <EMAIL ADDRESS>
+*/
+package transform
+
+import (
+	"github.com/spf13/cobra"
+)
+
+// transformCmd represents the transform command
+var TransformCmd = &cobra.Command{
+	Use:   "transform",
+	Short: "Will transform your hostlist in specified way",
+	Long: `
+`,
+}
+
+func init() {
+
+	// Here you will define your flags and configuration settings.
+
+	// Cobra supports Persistent Flags which will work for this command
+	// and all subcommands, e.g.:
+	// transformCmd.PersistentFlags().String("foo", "", "A help for foo")
+
+	// Cobra supports local flags which will only run when this command
+	// is called directly, e.g.:
+	// transformCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle")
+}

go.mod

@@ -0,0 +1,14 @@
+module git.itmodulo.eu/go/hstl
+
+go 1.20
+
+require (
+	github.com/deckarep/golang-set/v2 v2.3.0
+	github.com/jucardi/go-streams/v2 v2.0.0
+	github.com/spf13/cobra v1.7.0
+)
+
+require (
+	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+)

go.sum

@@ -0,0 +1,18 @@
+github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/deckarep/golang-set/v2 v2.3.0 h1:qs18EKUfHm2X9fA50Mr/M5hccg2tNnVqsiBImnyDs0g=
+github.com/deckarep/golang-set/v2 v2.3.0/go.mod h1:VAky9rY/yGXJOLEDv3OMci+7wtDpOF4IN+y82NBOac4=
+github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
+github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
+github.com/jucardi/go-streams/v2 v2.0.0 h1:Z107d2RGhT2QQimJQEwf893tfIuguHTags5So7Zo2Q4=
+github.com/jucardi/go-streams/v2 v2.0.0/go.mod h1:ovpcmqHJB55MiRu/ouRrHIsbCVwrKE5u8ADZ+ylz5Lk=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I=
+github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0=
+github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
+github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

main.go

@@ -0,0 +1,12 @@
+/*
+Copyright © 2023 itmodulo itmodulo@disroot.org
+*/
+package main
+
+import (
+	"git.itmodulo.eu/go/hstl/cmd"
+)
+
+func main() {
+	cmd.Execute()
+}

main_test.go

@@ -0,0 +1,83 @@
+package main
+
+import (
+	"fmt"
+	"strings"
+	"testing"
+
+	. "git.itmodulo.eu/go/hstl/processing"
+	. "git.itmodulo.eu/go/hstl/vars"
+
+	// . "github.com/deckarep/golang-set/v2"
+	. "github.com/jucardi/go-streams/v2/streams"
+)
+
+var str string = `# Malicious or purchaseable domains found in videogame manuals, title screens, or game driver/patch sites
+127.0.0.1 acclaimsports.com
+127.0.0.1 acclaimmaxsports.com
+127.0.0.1 hip-games.com
+127.0.0.1 thq.com.au
+127.0.0.1 gp32.com
+127.0.0.1 kidnkid.com
+127.0.0.1 b-shooter.com # dupa
+127.0.0.1 b-shooter.com
+127.0.0.1 b-shooter.com
+127.0.0.1 b-shooter.com
+127.0.0.1 b-shooter.com
+0.0.0.0 b-shooter.com
+127.0.0.1 b-shooter.com
+127.0.0.1 b-shooter.com
+127.0.0.1 b-shooter.com
+127.0.0.1 b-shooter.com
+127.0.0.1 100-9.com`
+
+// func TestPredicates(test *testing.T) {
+
+// 	slc := strings.Split(str, "\n")
+// 	set := treeset.NewWithStringComparator()
+
+// 	var addtoset = func(i interface{}) { set.Add(i) }
+
+// 	From(slc).
+// 		Where(IsCommented).
+// 		Select(SplitBySpaceOrTab).
+// 		Where(MatchesInbuiltExpression).
+// 		ForEach(addtoset)
+
+// 	if set.Size() == 0 {
+// 		test.FailNow()
+// 	}
+// 	set.Each(print)
+
+// }
+var PrintPredicate = func(word string) bool { fmt.Println("|", word, "|"); return false }
+
+func TestPredicates(test *testing.T) {
+
+	slc := strings.Split(str, "\n")
+
+	var afterMapping = Map[string, string](
+		Map[string, string](
+			FromArray[string](slc, THREADS).
+				Except(IsCommented).
+				ToIterable(),
+			RmCommentsAfterRec).
+			Stream().
+			ToIterable(),
+		SplitBySpaceOrTab).
+		Stream().
+		SetThreads(THREADS).
+		Except(IsCommented).
+		Except(MatchesInbuiltExpression).
+		Except(IsEmpty).
+		Except(IsCommented).
+		ToList().Distinct().ToArray()
+	// ParallelForEach(addtoset, THREADS, true)
+	// var map2 = Map[string, string](afterMapping)
+	// x.Each(PrintPredicate)
+	for _, i := range afterMapping {
+		println(i)
+	}
+	println()
+
+}

processing/predicates.go

@@ -0,0 +1,84 @@
+package processing
+
+import (
+	"regexp"
+	"strings"
+
+	. "git.itmodulo.eu/go/hstl/utils"
+)
+
+var ExpressionsSlice = []string{"::",
+	"127.0.0.1",
+	"localhost",
+	"localhost.localdomain",
+	"local",
+	"255.255.255.255",
+	"::1",
+	"ip6-localhost",
+	"ip6-loopback",
+	"fe80::1%lo0",
+	"ff00::0",
+	"fe00::0",
+	"ip6-localnet",
+	"ip6-mcastprefix",
+	"ff02::1",
+	"ff02::2",
+	"ip6-allrouters",
+	"ff02::3",
+	"0.0.0.0",
+	"ip6-allhosts",
+	"ip6-allnodes",
+	"broadcasthost",
+	""}
+
+// var Prefixes = []string{}
+
+// APPROVED
+
+var IsCommented = func(word string) bool { return strings.HasPrefix(word, "#") }
+
+var IsEmpty = func(word string) bool {
+	if word == "" || word == "\n" {
+		return true
+	}
+	return false
+}
+var SplitBySpaceOrTab = func(word string) string {
+	var splitted []string
+	if strings.Contains(word, "\t ") {
+		splitted = strings.Split(word, "\t ")
+	} else if strings.Contains(word, " \t") {
+		splitted = strings.Split(word, " \t")
+	} else if strings.Contains(word, " ") {
+		splitted = strings.Split(word, " ")
+	} else if strings.Contains(word, "\t") {
+		splitted = strings.Split(word, "\t")
+	} else {
+		return word
+	}
+	return splitted[1]
+}
+
+var MatchesInbuiltExpression = func(word string) bool {
+	for _, expr := range ExpressionsSlice {
+		if word == expr {
+			return true
+		}
+	}
+	return false
+}
+
+// TODO
+var RmCommentsAfterRec = func(word string) string {
+	if strings.Contains(word, " #") || strings.Contains(word, "\t"+"#") {
+		re := regexp.MustCompile(`address=/.*?\..*?/#[ |\t]#.*?$`)
+		if re.MatchString(word) {
+			return word[:(FindLastIndexOfCharInString(word, "#") - 1)]
+		} else {
+			// Self-made substring after: Find index of # as char and return a slice, -1 is because of space
+			return word[:(FindFirstIndexOfCharInString(word, "#"))-1]
+		}
+	}
+	return word
+
+}

utils/stringutils.go

@@ -0,0 +1,42 @@
+package utils
+
+import "strconv"
+
+func FindFirstIndexOfCharInString(toFind string, x string) int {
+
+	str := []rune(toFind)
+	z := []rune(x)
+	y := z[0]
+
+	for i := 0; i < len(str); i++ {
+		if str[i] == y {
+
+			return i
+		}
+	}
+	return 0
+
+}
+func FindLastIndexOfCharInString(toFind string, x string) int {
+
+	str := []rune(toFind)
+	z := []rune(x)
+	y := z[0]
+
+	for i := len(str) - 1; i >= 0; i-- {
+		if str[i] == y {
+			return i
+		} else {
+		}
+	}
+	return 0
+
+}
+
+func AddPrefixZero(num int) string {
+
+	if num < 10 {
+		return strconv.Itoa(0) + strconv.Itoa(num)
+	}
+	return strconv.Itoa(num)
+}

vars/const.go

@@ -0,0 +1,3 @@
+package vars
+
+const THREADS int = 8