Made random statistics work properly through the detection system

Author: frikky

db-connector.go

@@ -12453,7 +12453,7 @@ func UpdateDetectionStats(ctx context.Context, cacheData CacheKeyData) {
 	}
 
 	detectionStatname := fmt.Sprintf("detection_rule_%s", strings.TrimSpace(strings.ToLower(strings.ReplaceAll(ruleName, " ", "_"))))
-	IncrementCache(ctx, cacheData.OrgId, detectionStatname, 10)
+	IncrementCache(ctx, cacheData.OrgId, detectionStatname, 1)
 	if debug {
 		log.Printf("[DEBUG] Incremented detection stat '%s' for org %s", detectionStatname, cacheData.OrgId)
 	}

detection.go

@@ -1,17 +1,18 @@
 package shuffle
 
 import (
-	"context"
-	"encoding/json"
 	"fmt"
-	"io/ioutil"
 	"log"
+	"context"
 	"net/http"
+	"io/ioutil"
+	"crypto/sha1"
+	"encoding/json"
 
-	"errors"
 	"sort"
-	"strings"
 	"time"
+	"errors"
+	"strings"
 
 	uuid "github.com/satori/go.uuid"
 	"gopkg.in/yaml.v2"
@@ -562,15 +563,43 @@ func HandleDetectionAutoConnect(resp http.ResponseWriter, request *http.Request)
 
 	log.Printf("[AUDIT] User '%s' (%s) is trying to detection-connect to %s", user.Username, user.Id, strings.ToUpper(detectionType))
 
+	// Uses the same system we are using in the ai.go standard workflow creation
 	workflow := Workflow{}
-	if detectionType == "siem" {
+	if detectionType == "siem" || detectionType == "sigma" {
+		categoryAction := CategoryAction{
+			Label: "Ingest Tickets_webhook",
+			Category: "cases",
+		}
+
+		seedString := fmt.Sprintf("%s_%s", user.ActiveOrg.Id, categoryAction.Label)
+		hash := sha1.New()
+		hash.Write([]byte(seedString))
+		hashBytes := hash.Sum(nil)
+
+		uuidBytes := make([]byte, 16)
+		copy(uuidBytes, hashBytes)
+		workflowId := uuid.Must(uuid.FromBytes(uuidBytes)).String()
 
 		ctx := GetContext(request)
-		workflow, err = ConfigureDetectionWorkflow(ctx, user.ActiveOrg.Id, "TENZIR-SIGMA")
-		if err != nil {
-			log.Printf("[ERROR] Failed to create Sigma handling workflow: %s", err)
+		foundWorkflow, err := GetWorkflow(ctx, workflowId)
+		if err != nil || workflow.ID == "" {
+			log.Printf("[WARNING] Failed to get workflow by ID '%s' in GenerateSingulWorkflows: %s", workflowId, err)
+			//initialising = true
+			newWorkflow, err := GetDefaultWorkflowByType(*foundWorkflow, user.ActiveOrg.Id, categoryAction)
+			if err != nil {
+				log.Printf("[ERROR] Failed to get default workflow in GenerateSingulWorkflows: %s", err)
+				resp.WriteHeader(http.StatusInternalServerError)
+				resp.Write([]byte(`{"success": false, "reason": "Failed to get default workflow for this category. Please contact [email protected]"}`))
+				return
+			}
+
+			workflow = newWorkflow
+		} else {
+			workflow = *foundWorkflow
 		}
 
+		workflow.ID = workflowId
+
 		log.Printf("[DEBUG] Sending orborus request to start Sigma handling IF an available environment is found.")
 
 		execType := "START_TENZIR"
@@ -942,6 +971,7 @@ func ConfigureDetectionWorkflow(ctx context.Context, orgId, workflowType string)
 
 	// FIXME: Add a changeout for ANY schemaless node to use the correct
 	// action in it
+	workflow.BackgroundProcessing = true
 	log.Printf("[DEBUG] Saving workflow for org %s", orgId)
 	err = SetWorkflow(ctx, workflow, workflow.ID)
 	if err != nil {

stats.go

@@ -433,7 +433,7 @@ func GetSpecificStats(resp http.ResponseWriter, request *http.Request) {
 	statEntries = mergedEntries
 
 	// Check if entries exist for the last X statDays
-	// Backfill any missing ones
+	// Backfill any missing ones so that the number is correct
 	if len(statEntries) < statDays {
 		// Find the missing days
 		missingDays := []time.Time{}
@@ -468,16 +468,43 @@ func GetSpecificStats(resp http.ResponseWriter, request *http.Request) {
 		statEntries = append(statEntries, toAppend...)
 	}
 
+	// Append cache for right now as it may not be in the DB yet 
+	for statEntryIndex, statEntry := range statEntries {
+		if statEntry.Date.Day() == time.Now().Day() && statEntry.Date.Month() == time.Now().Month() && statEntry.Date.Year() == time.Now().Year() {
+			for _, addition := range info.Additions {
+				if addition.Key != statsKey {
+					continue
+				}
+
+				key := fmt.Sprintf("cache_%s_%s", orgId, addition.Key)
+				cacheItem, err := GetCache(ctx, key)
+				if err == nil {
+					parsedItem := []byte(cacheItem.([]uint8))
+					increment, err := strconv.Atoi(string(parsedItem))
+					if err == nil {
+						statEntries[statEntryIndex].Value += int64(increment)
+						totalValue += int(increment)
+					}
+				}
+
+				break
+			}
+		}
+	}
+
 	// Sort statentries by date
 	sort.Slice(statEntries, func(i, j int) bool {
 		return statEntries[i].Date.Before(statEntries[j].Date)
 	})
 
+	// For debugging stats that don't show up by injecting them 
+	/*
 	if debug && totalValue == 0 {
 		log.Printf("[DEBUG] Found %d entries for '%s' with 0 in data. Force-adding data to first entry.", len(statEntries), statsKey)
 		chosenIndex := rand.Intn(len(statEntries))
 		statEntries[chosenIndex].Value = int64(rand.Intn(10) + 1)
 	}
+	*/
 
 	marshalledEntries, err := json.Marshal(statEntries)
 	if err != nil {
@@ -631,6 +658,24 @@ func HandleGetStatistics(resp http.ResponseWriter, request *http.Request) {
 		}
 	}
 
+	for additionCnt, addition := range info.Additions {
+
+		key := fmt.Sprintf("cache_%s_%s", orgId, addition.Key)
+		cacheItem, err = GetCache(ctx, key)
+		if err == nil {
+			parsedItem := []byte(cacheItem.([]uint8))
+			increment, err := strconv.Atoi(string(parsedItem))
+			if err == nil {
+				info.Additions[additionCnt].Value += int64(increment)
+			}
+		}
+
+		// In case a lot of use
+		if additionCnt > 10 {
+			break
+		}
+	}
+
 	if len(statsKey) > 0 {
 		log.Printf("[INFO] Should get stats for key %s", statsKey)
 	}
@@ -1232,7 +1277,6 @@ func IncrementCache(ctx context.Context, orgId, dataType string, amount ...int)
 
 	} else {
 		// Get the cache, but use requestCache instead of memcache
-		//log.Printf("[DEBUG] Incrementing cache for %s with amount %d", key, incrementAmount)
 		foundItem := 1
 		item, err := GetCache(ctx, key)
 		if err != nil {
@@ -1269,7 +1313,7 @@ func IncrementCache(ctx context.Context, orgId, dataType string, amount ...int)
 
 		if foundItem >= int(dbDumpInterval) {
 			// Memcache dump first to keep the counter going for other executions
-			go SetCache(ctx, key, []byte(fmt.Sprintf("%x", 0)), 86400)
+			go SetCache(context.Background(), key, []byte(fmt.Sprintf("%x", 0)), 86400)
 			IncrementCacheDump(ctx, orgId, dataType, foundItem)
 
 			//log.Printf("[DEBUG] Dumping cache for %s with amount %d", key, foundItem)