feat: add support for variable strings (#386)

* add tsv utilities * move logstash utilities to its own file * add test for tsv utilities * add template for variable strings * add test for variable strings template * rename field for clarity * update template field names * remove inline field set for clarity * add template documentation * rename field for clarity * add omitempty for custom-data set * change parameter-based configuration to struct-based for clarity * add option for including tsv-file * add template-group-by-custom-data to plugin template * add sid-list to plugin template * ignore coverage on internal dpluger package * add missing identifier block * fix: enable category field checking * tidy up es7 collect method * fix: fix empty parsed PluginSID simplify tsvref upsert plugin add test for tsvref upsert plugin * remove old tsvref upsert method * remove extra newline * doc: add in-code documentaiton * fix in-code documentation * change variable names for readability * fix: add variable strings support for non-collect run * add parsable implementation * fix: add variable strings support for collect-type run * add sid-list to viper binding * fix: fix incorrect template reference * add custom-data into plugin-sid set * add test for es7-client * change dpluger create plugin flow to match variable strings update * fix incorrect template indentation
defenxor · Mar 1, 2022 · 95c1131 · 95c1131
1 parent b5fb195
commit 95c1131
Show file tree

Hide file tree

Showing 14 changed files with 1,172 additions and 335 deletions.
diff --git a/.codecov.yml b/.codecov.yml
@@ -10,3 +10,5 @@ coverage:
         target: auto            # specify the target coverage for each commit status
         threshold: 1%          # allowed to drop X% and still result in a "success" commit status
 
+ignore:
+  - "internal/pkg/dpluger"
diff --git a/cmd/dpluger/run.go b/cmd/dpluger/run.go
@@ -17,9 +17,11 @@ func init() {
 	runCmd.Flags().BoolP("skipTLSVerify", "s", false, "whether to skip ES server certificate verification (when using HTTPS)")
 	runCmd.Flags().BoolP("usePipeline", "p", false, "whether to generate plugin that is suitable for logstash pipeline to pipeline configuration")
 	runCmd.Flags().BoolP("validate", "v", true, "Check whether each referred ES field exists on the target index")
+	runCmd.Flags().StringP("sid-list", "f", "", "optional, Plugin SID list file to use for generating logstash plugin (tsv-formatted). If set, dpluger will not generate any .tsv file, and assumes that you already have a .tsv file containing list of plugin SID, either by previous dpluger run or created manually")
 	viper.BindPFlag("validate", runCmd.Flags().Lookup("validate"))
 	viper.BindPFlag("skipTLSVerify", runCmd.Flags().Lookup("skipTLSVerify"))
 	viper.BindPFlag("usePipeline", runCmd.Flags().Lookup("usePipeline"))
+	viper.BindPFlag("sid-list", runCmd.Flags().Lookup("sid-list"))
 
 	rootCmd.AddCommand(runCmd)
 }
@@ -33,6 +35,7 @@ var runCmd = &cobra.Command{
 		validate := viper.GetBool("validate")
 		skipTLSVerify := viper.GetBool("skipTLSVerify")
 		usePipeline := viper.GetBool("usePipeline")
+		SIDListFile := viper.GetString("sid-list")
 
 		if skipTLSVerify {
 			http.DefaultTransport.(*http.Transport).TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
@@ -44,13 +47,23 @@ var runCmd = &cobra.Command{
 		if err := log.Setup(true); err != nil {
 			exit("Cannot setup logger", err)
 		}
+
 		plugin, err := dpluger.Parse(config)
 		if err != nil {
 			exit("Cannot parse config file", err)
 		}
-		if err := dpluger.CreatePlugin(plugin, config, progName, validate, usePipeline); err != nil {
+
+		if err := dpluger.CreatePlugin(dpluger.CreatePluginConfig{
+			Plugin:      plugin,
+			ConfigFile:  config,
+			Creator:     progName,
+			Validate:    validate,
+			UsePipeline: usePipeline,
+			SIDListFile: SIDListFile,
+		}); err != nil {
 			exit("Error encountered while running config file", err)
 		}
+
 		fmt.Println("Logstash conf file created.")
 	},
 }
diff --git a/internal/pkg/dpluger/dirmaker.go b/internal/pkg/dpluger/dirmaker.go
@@ -31,7 +31,7 @@ import (
 )
 
 type tsvEntries struct {
-	records []pluginSIDRef
+	records []PluginSID
 }
 
 // CreateDirective starts directive creation
@@ -66,13 +66,13 @@ func createDirective(in io.Reader, dirs siem.Directives, kingdom, titleTemplate
 
 	parser := tsv.NewParser(in)
 
-	defaultRef := pluginSIDRef{
+	defaultRef := PluginSID{
 		Kingdom: kingdom,
 	}
 
 	entries := tsvEntries{}
 	for {
-		var ref pluginSIDRef
+		var ref PluginSID
 		ok := parser.Read(&ref, defaultRef)
 		if !ok {
 			break