defenxor · rkspx · Mar 15, 2022 · Mar 11, 2022 · Mar 15, 2022 · Mar 15, 2022
@@ -417,7 +417,13 @@ func collectPair(plugin Plugin, confFile, esFilter string, validate bool) (tsvRe
 	)
 
 	sidSource := strings.Replace(plugin.Fields.PluginSID, "es:", "", 1)
-	titleSource, err := checkKeyword(ctx, plugin.Index, strings.Replace(plugin.Fields.Title, "es:", "", 1))
+	sidSource, err = checkKeyword(ctx, plugin.Index, sidSource)
+	if err != nil {
+		return c, err
+	}
+
+	titleSource := strings.Replace(plugin.Fields.Title, "es:", "", 1)
+	titleSource, err = checkKeyword(ctx, plugin.Index, titleSource)
 	if err != nil {
 		return c, err
 	}
@@ -475,7 +481,7 @@ func collectPair(plugin Plugin, confFile, esFilter string, validate bool) (tsvRe
 
 	fmt.Printf("Collecting unique entries for field '%s' and '%s' on index '%s' ... ", titleSource, sidSource, plugin.Index)
 	if esFilter != "" {
-		fmt.Printf("Limiting collection with term '%s'\n", esFilter)
+		fmt.Printf("Limiting collection with term '%s' ", esFilter)
 	}
 
 	fmt.Println("OK")
@@ -490,7 +496,8 @@ func collectSID(plugin Plugin, confFile, esFilter string, validate bool) (tsvRef
 		err error
 	)
 
-	sidSource, err := checkKeyword(ctx, plugin.Index, strings.Replace(plugin.Fields.PluginSID, "collect:", "", 1))
+	sidSource := strings.Replace(plugin.Fields.PluginSID, "collect:", "", 1)
+	sidSource, err = checkKeyword(ctx, plugin.Index, sidSource)
 	if err != nil {
 		return c, err
 	}

@@ -49,12 +49,11 @@ func (es *es6Client) CollectPair(plugin Plugin, confFile, sidSource, esFilter, t
 
 	query := elastic6.NewBoolQuery()
 	if esFilter != "" {
-		coll := strings.Split(esFilter, ";")
-		for _, v := range coll {
-			s := strings.Split(v, "=")
+		filters := strings.Split(esFilter, ";")
+		for _, filter := range filters {
+			s := strings.Split(filter, "=")
 			if len(s) != 2 {
-				err = errors.New("Cannot split the ES filter term")
-				return
+				return tsvRef{}, fmt.Errorf("invalid ES filter term, '%s', expected pair of strings with '=' delimitier", filter)
 			}
 			query = query.Must(elastic6.NewTermQuery(s[0], s[1]))
 		}
@@ -72,46 +71,49 @@ func (es *es6Client) CollectPair(plugin Plugin, confFile, sidSource, esFilter, t
 	if err != nil {
 		return
 	}
-	agg, found := searchResult.Aggregations.Terms("finalAgg")
+
+	roots, found := searchResult.Aggregations.Terms("finalAgg")
 	if !found {
 		err = errors.New("cannot find aggregation finalAgg in ES query result")
 		return
 	}
-	count := len(agg.Buckets)
+	count := len(roots.Buckets)
 	if count == 0 {
 		err = errors.New("cannot find matching entry in field " + sidSource + " on index " + plugin.Index)
 		return
 	}
-	fmt.Println("Found", count, "uniq "+sidSource+".")
+
+	fmt.Printf("found %d unique '%s'\n", count, sidSource)
 	nID, err := strconv.Atoi(plugin.Fields.PluginID)
 	if err != nil {
 		return
 	}
 
-	for _, lvl1Bucket := range agg.Buckets {
-		subterm, found := lvl1Bucket.Terms("subterm")
+	for _, rootBucket := range roots.Buckets {
+		sidlist, found := rootBucket.Terms("subterm")
 		if !found {
 			continue
 		}
-		for _, lvl2Bucket := range subterm.Buckets {
-			sKey := lvl1Bucket.Key.(string)
-			nKey, err := toInt(lvl2Bucket.Key)
+
+		for _, sidBucket := range sidlist.Buckets {
+			root := rootBucket.Key.(string)
+			sid, err := toInt(sidBucket.Key)
 			if err != nil {
-				return c, fmt.Errorf("invalid sid aggregation key, %s", err.Error())
+				return c, fmt.Errorf("invalid signature ID, %s", err.Error())
 			}
 			// fmt.Println("item1:", sKey, "item2:", nKey)
 			if shouldCollectCategory {
-				subSubTerm, found2 := lvl1Bucket.Terms("subSubTerm")
+				subSubTerm, found2 := rootBucket.Terms("subSubTerm")
 				if !found2 {
 					continue
 				}
 				for _, lvl3Bucket := range subSubTerm.Buckets {
 					sCat := lvl3Bucket.Key.(string)
-					_ = c.upsert(plugin.Name, nID, &nKey, sCat, sKey)
+					_ = c.upsert(plugin.Name, nID, &sid, sCat, root)
 					break
 				}
 			} else {
-				_ = c.upsert(plugin.Name, nID, &nKey, categorySource, sKey)
+				_ = c.upsert(plugin.Name, nID, &sid, categorySource, root)
 			}
 			break
 		}
@@ -132,12 +134,11 @@ func (es *es6Client) Collect(plugin Plugin, confFile, sidSource, esFilter, categ
 
 	query := elastic6.NewBoolQuery()
 	if esFilter != "" {
-		coll := strings.Split(esFilter, ";")
-		for _, v := range coll {
-			s := strings.Split(v, "=")
+		filters := strings.Split(esFilter, ";")
+		for _, filter := range filters {
+			s := strings.Split(filter, "=")
 			if len(s) != 2 {
-				err = errors.New("Cannot split the ES filter term")
-				return
+				return tsvRef{}, fmt.Errorf("invalid ES filter term, '%s', expected pair of strings with '=' delimitier", filter)
 			}
 			query = query.Must(elastic6.NewTermQuery(s[0], s[1]))
 		}
@@ -152,9 +153,11 @@ func (es *es6Client) Collect(plugin Plugin, confFile, sidSource, esFilter, categ
 		Aggregation("uniqTerm", terms).
 		Pretty(true).
 		Do(ctx)
+
 	if err != nil {
 		return
 	}
+
 	agg, found := searchResult.Aggregations.Terms("uniqTerm")
 	if !found {
 		err = errors.New("cannot find aggregation uniqTerm in ES query result")
@@ -165,7 +168,8 @@ func (es *es6Client) Collect(plugin Plugin, confFile, sidSource, esFilter, categ
 		err = errors.New("cannot find matching entry in field " + sidSource + " on index " + plugin.Index)
 		return
 	}
-	fmt.Println("Found", count, "uniq "+sidSource+".")
+
+	fmt.Printf("found %d unique '%s'\n", count, sidSource)
 	newSID := 1
 	nID, err := strconv.Atoi(plugin.Fields.PluginID)
 	if err != nil {
@@ -223,7 +227,6 @@ func (es *es6Client) FieldType(ctx context.Context, index string, field string)
 	m, err := elastic6.NewGetFieldMappingService(es.client).
 		Field(field).
 		Index(index).
-		Type("_doc").
 		Do(ctx)
 
 	if err != nil {
@@ -232,15 +235,19 @@ func (es *es6Client) FieldType(ctx context.Context, index string, field string)
 
 	var fiedMapping map[string]interface{}
 	var ok bool
+MAPPING_SEARCH:
 	for _, v := range m {
-		fm, exist := v.(map[string]interface{})["mappings"].(map[string]interface{})["_doc"].(map[string]interface{})[field]
-		if !exist {
+		fm, fmok := v.(map[string]interface{})["mappings"].(map[string]interface{})
+		if !fmok {
 			continue
 		}
 
-		fiedMapping, ok = fm.(map[string]interface{})
-		if ok {
-			break
+		for _, val := range fm {
+			// get the first child that has the mapping for the field
+			fiedMapping, ok = val.(map[string]interface{})[field].(map[string]interface{})
+			if ok {
+				break MAPPING_SEARCH
+			}
 		}
 
 	}

@@ -104,9 +104,9 @@ func (es *es7Client) CollectPair(plugin Plugin, confFile, sidSource, esFilter, t
 
 		for _, sidBucket := range SIDs.Buckets {
 			rootKey := root.Key.(string)
-			pluginSID, err := toInt(sidBucket.Key)
+			sid, err := toInt(sidBucket.Key)
 			if err != nil {
-				return ref, fmt.Errorf("invalid SID aggregation key, %s", err.Error())
+				return ref, fmt.Errorf("invalid signature ID, %s", err.Error())
 			}
 
 			if shouldCollectCategory {
@@ -117,12 +117,12 @@ func (es *es7Client) CollectPair(plugin Plugin, confFile, sidSource, esFilter, t
 
 				for _, categoryBucket := range categories.Buckets {
 					category := categoryBucket.Key.(string)
-					ref.upsert(plugin.Name, pluginID, &pluginSID, category, rootKey)
+					ref.upsert(plugin.Name, pluginID, &sid, category, rootKey)
 					break
 				}
 
 			} else {
-				ref.upsert(plugin.Name, pluginID, &pluginSID, categorySource, rootKey)
+				ref.upsert(plugin.Name, pluginID, &sid, categorySource, rootKey)
 			}
 
 			break

@@ -205,7 +205,6 @@ func (c *tsvRef) initWithConfig(configFile string) {
 func (c *tsvRef) init(pluginName string, configFile string) {
 	c.SIDs = make(map[int]PluginSID)
 	c.setFilename(pluginName, path.Dir(configFile))
-	// f, err := os.OpenFile(c.filename, os.O_RDONLY, 0600)
 	f, err := os.OpenFile(c.filename, os.O_RDONLY, 0600)
 	if err != nil {
 		return

@@ -255,7 +255,7 @@ func toInt(v interface{}) (int, error) {
 	case string:
 		n, err := strconv.ParseInt(t, 10, 64)
 		if err != nil {
-			return 0, err
+			return 0, fmt.Errorf("expecting numeric value, got '%s'", t)
 		}
 
 		if n >= 0 && n < math.MaxInt32 {
@@ -265,5 +265,5 @@ func toInt(v interface{}) (int, error) {
 		return 0, ErrIntValueExceedBoundary
 	}
 
-	return 0, fmt.Errorf("invalid value type, %T", v)
+	return 0, fmt.Errorf("invalid numeric value type, %T", v)
 }
@@ -4,10 +4,12 @@ command -v go >/dev/null || { echo 'cannot find go command in $PATH'; exit 1; }
 
 cmd=${1}
 
-# cgoflag should be 0 or 1
-cgoflag=${2}
-xtraflag=${3}
-[ "$cgoflag" = "" ] && cgoflag=0
+goos=${2}
+[ -z $goos ] && goos=darwin
+
+goarch=${3}
+[ -z $goarch ] && goarch=arm64
+
 
 [ -z $cmd ] && cmd=$(find ./cmd/ -maxdepth 1 ! -path ./cmd/ -type d)
 
@@ -16,6 +18,6 @@ now=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
 
 for c in $cmd; do
   [ ! -d $c ] && echo $c directory doesnt exist, skipping. && continue
-  echo building $c ver=${ver} buildtime=${now}
-  GOFLAGS="-mod=vendor" CGO_ENABLED=${cgoflag} GOOS=darwin GOARCH=arm64 go build ${xtraflag} -a -ldflags "-s -w -X main.version=${ver} -X main.buildTime=${now} -extldflags '-static'" $c
+  echo building $c ver=${ver} buildtime=${now} for $goos/$goarch
+  GOFLAGS="-mod=vendor" CGO_ENABLED=${cgoflag} GOOS=$goos GOARCH=$goarch go build ${xtraflag} -a -ldflags "-s -w -X main.version=${ver} -X main.buildTime=${now} -extldflags '-static'" $c
 done