Merge branch 'main' into fix-minor-issues

Author: frikky

blobs.go

@@ -3,21 +3,31 @@ package shuffle
 func GetPublicDetections() []DetectionResponse {
 	return []DetectionResponse{
 		DetectionResponse{
-			Title: "Sigma SIEM Detections",
+			Title:             "Sigma SIEM Detections",
 			DetectionName:     "Sigma",
 			Category:          "SIEM",
 			DetectionInfo:     []DetectionFileInfo{},
 			FolderDisabled:    false,
 			IsConnectorActive: false,
-			DownloadRepo: 	   "https://github.com/satti-hari-krishna-reddy/shuffle_sigma",
+			DownloadRepo:      "https://github.com/shuffle/security-rules",
 		},
 		DetectionResponse{
-			Title: "Sublime Email Detection",
+			Title:             "Sublime Email Detection",
 			DetectionName:     "Sublime",
 			Category:          "Email",
 			DetectionInfo:     []DetectionFileInfo{},
 			FolderDisabled:    false,
 			IsConnectorActive: false,
+			DownloadRepo:      "https://github.com/shuffle/security-rules",
+		},
+		DetectionResponse{
+			Title:             "File Detection",
+			DetectionName:     "Yara",
+			Category:          "Files",
+			DetectionInfo:     []DetectionFileInfo{},
+			FolderDisabled:    false,
+			IsConnectorActive: false,
+			DownloadRepo:      "https://github.com/shuffle/security-rules",
 		},
 	}
 }
@@ -302,29 +312,8 @@ func GetUsecaseData() string {
 				"blogpost": "https://medium.com/shuffle-automation/introducing-shuffle-an-open-source-soar-platform-part-1-58a529de7d12",
 				"reference_image": "/images/detectionframework.png",
                 "items": {}
-            },
-            {
-				"type": "cases",
-				"last": "cases",
-                "name": "2-way Ticket synchronization",
-				"priority": 20,
-                "items": {}
-            },
-            {
-                "name": "ChatOps",
-				"priority": 70,
-				"type": "communication",
-				"last": "cases",
-                "items": {}
-            },
-            {
-                "name": "Threat Intel received",
-				"priority": 50,
-				"type": "intel",
-				"last": "cases",
-                "items": {}
-            }
-        ]
+            }        
+		]
     },
     {
         "name": "2. Enrich",
@@ -441,79 +430,31 @@ func GetUsecaseData() string {
         "color": "#4885ed",
         "list": [
             {
-                "name": "Eradicate malware",
-				"priority": 90,
-				"type": "intel",
-				"last": "edr",
-                "items": {}
-            },
-            {
-                "name": "Quarantine host(s)",
-				"priority": 90,
+                "name": "Isolate Host",
+				"old_name": "Quarantine host(s)",
+				"priority": 80,
 				"type": "edr",
                 "items": {}
             },
             {
-                "name": "Update Outdated Software",
-				"priority": 70,
-				"type": "assets",
-                "items": {}
-            },
-            {
-                "name": "Block IPs, URLs, Domains and Hashes",
-				"priority": 90,
+                "name": "Block an IP",
+				"old_name": "Block IPs, URLs, Domains and Hashes",
+				"priority": 75,
 				"type": "network",
                 "items": {}
             },
             {
-                "name": "Trigger scans",
-				"priority": 50,
-				"type": "assets",
-                "items": {}
-            },
-            {
-                "name": "Update indicators (FW, EDR, SIEM...)",
-				"priority": 50,
-				"type": "intel",
-				"last": "siem",
-                "items": {}
-            },
-            {
-                "name": "Autoblock activity when threat intel is received",
+                "name": "Kill a process",
 				"priority": 50,
-				"type": "intel",
-				"last": "iam",
-                "items": {}
-            },
-            {
-                "name": "Lock/Delete/Reset account",
-				"priority": 50,
-				"type": "iam",
-                "items": {}
-            },
-            {
-                "name": "Lock vault",
-				"priority": 50,
-				"type": "iam",
+				"type": "edr",
                 "items": {}
             },
             {
-                "name": "Increase authentication",
-				"priority": 50,
+                "name": "Lock account",
+				"old_name": "Lock/Delete/Reset account",
+				"priority": 70,
 				"type": "iam",
                 "items": {}
-            },
-            {
-                "name": "Get policies from assets",
-				"priority": 50,
-				"type": "assets",
-                "items": {}
-            },
-            {
-                "name": "Run ansible scripts",
-				"type": "assets",
-				"priority": 50,
-                "items": {}
             }
         ]
     },

codegen.go

@@ -16,14 +16,17 @@ import (
 	"sort"
 	"strconv"
 	"strings"
+	"net/http"
 
 	"cloud.google.com/go/storage"
 	"github.com/frikky/kin-openapi/openapi3"
+	docker "github.com/docker/docker/client"
 
 	//"github.com/satori/go.uuid"
 	"gopkg.in/yaml.v2"
 )
 
+var downloadedImages = []string{}
 var pythonAllowed = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_"
 var pythonReplacements = map[string]string{
 	"[": "",
@@ -3771,3 +3774,132 @@ func RemoveJsonValues(input []byte, depth int64) ([]byte, string, error) {
 
 	return input, keyToken, nil
 }
+
+func DownloadDockerImageBackend(topClient *http.Client, imageName string) error {
+	// Check environment SHUFFLE_AUTO_IMAGE_DOWNLOAD
+	if os.Getenv("SHUFFLE_AUTO_IMAGE_DOWNLOAD") == "false" {
+		log.Printf("[DEBUG] SHUFFLE_AUTO_IMAGE_DOWNLOAD is false. Not downloading image %s", imageName)
+		return nil
+	}
+
+	if ArrayContains(downloadedImages, imageName) && project.Environment == "worker" {
+		log.Printf("[DEBUG] Image %s already downloaded - not re-downloading. This only applies to workers.", imageName)
+		return nil
+	}
+
+	baseUrl := os.Getenv("BASE_URL")
+	log.Printf("[DEBUG] Trying to download image %s from backend %s as it doesn't exist. All images: %#v", imageName, baseUrl, downloadedImages)
+
+	if !ArrayContains(downloadedImages, imageName) {
+		downloadedImages = append(downloadedImages, imageName)
+	}
+
+	data := fmt.Sprintf(`{"name": "%s"}`, imageName)
+	dockerImgUrl := fmt.Sprintf("%s/api/v1/get_docker_image", baseUrl)
+
+	req, err := http.NewRequest(
+		"POST",
+		dockerImgUrl,
+		bytes.NewBuffer([]byte(data)),
+	)
+
+	// Specific to the worker
+	authorization := os.Getenv("AUTHORIZATION")
+	if len(authorization) > 0 {
+		req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", authorization))
+	} else {
+		// Specific to Orborus auth (org + auth) -> environment auth
+		authorization = os.Getenv("AUTH")
+		if len(authorization) > 0 {
+			log.Printf("[DEBUG] Found Orborus environment auth - adding to header.")
+			req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", authorization))
+
+			org := os.Getenv("ORG")
+			if len(org) > 0 {
+				req.Header.Add("Org-Id", org)
+			}
+
+		} else {
+			log.Printf("[WARNING] No auth found - running backend download without it.")
+		}
+	}
+
+	newresp, err := topClient.Do(req)
+	if err != nil {
+		log.Printf("[ERROR] Failed download request for %s: %s", imageName, err)
+		return err
+	}
+
+	defer newresp.Body.Close()
+	if newresp.StatusCode != 200 {
+		log.Printf("[ERROR] Docker download for image %s (backend) StatusCode (1): %d", imageName, newresp.StatusCode)
+		return errors.New(fmt.Sprintf("Failed to get image - status code %d", newresp.StatusCode))
+	}
+
+	newImageName := strings.Replace(imageName, "/", "_", -1)
+	newFileName := newImageName + ".tar"
+
+	tar, err := os.Create(newFileName)
+	if err != nil {
+		log.Printf("[WARNING] Failed creating file: %s", err)
+		return err
+	}
+
+	defer tar.Close()
+	_, err = io.Copy(tar, newresp.Body)
+	if err != nil {
+		log.Printf("[WARNING] Failed response body copying: %s", err)
+		return err
+	}
+	tar.Seek(0, 0)
+
+	dockercli, err := docker.NewEnvClient()
+	if err != nil {
+		log.Printf("[ERROR] Unable to create docker client (3): %s", err)
+		return err
+	}
+
+	defer dockercli.Close()
+
+	imageLoadResponse, err := dockercli.ImageLoad(context.Background(), tar, true)
+	if err != nil {
+		log.Printf("[ERROR] Error loading images: %s", err)
+		return err
+	}
+
+	defer imageLoadResponse.Body.Close()
+	body, err := ioutil.ReadAll(imageLoadResponse.Body)
+	if err != nil {
+		log.Printf("[ERROR] Error reading: %s", err)
+		return err
+	}
+
+	if strings.Contains(string(body), "no such file") {
+		return errors.New(string(body))
+	}
+
+	os.Remove(newFileName)
+
+	if strings.Contains(strings.ToLower(string(body)), "error") {
+		log.Printf("[ERROR] Error loading image %s: %s", imageName, string(body))
+		return errors.New(string(body))
+	}
+
+	baseTag := strings.Split(imageName, ":")
+	if len(baseTag) > 1 {
+		tag := baseTag[1]
+		log.Printf("[DEBUG] Creating tag copies of downloaded containers from tag %s", tag)
+
+		// Remapping
+		ctx := context.Background()
+		dockercli.ImageTag(ctx, imageName, fmt.Sprintf("frikky/shuffle:%s", tag))
+		dockercli.ImageTag(ctx, imageName, fmt.Sprintf("registry.hub.docker.com/frikky/shuffle:%s", tag))
+
+		downloadedImages = append(downloadedImages, fmt.Sprintf("frikky/shuffle:%s", tag))
+		downloadedImages = append(downloadedImages, fmt.Sprintf("registry.hub.docker.com/frikky/shuffle:%s", tag))
+
+	}
+
+	log.Printf("[INFO] Successfully loaded image %s: %s", imageName, string(body))
+	return nil
+}