Standalone http server support

Dockerfile

@@ -24,9 +24,7 @@ RUN go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.27.1 \
 WORKDIR /home/deepfence/src/YaRadare
 COPY . .
 RUN make clean \
-    && make all \
-    && cd /home/deepfence \
-    && git clone https://github.com/deepfence/yara-rules
+    && make all 
 
 FROM debian:bullseye
 MAINTAINER DeepFence
@@ -43,8 +41,6 @@ RUN apt-get update && apt-get -qq -y --no-install-recommends install libjansson4
     && curl -fsSLO https://download.docker.com/linux/static/stable/x86_64/docker-${DOCKERVERSION}.tgz \
     && tar xzvf docker-${DOCKERVERSION}.tgz --strip 1 -C /usr/local/bin docker/docker \
     && rm docker-${DOCKERVERSION}.tgz
-WORKDIR /home/deepfence/rules
-COPY --from=builder /home/deepfence/yara-rules .
 WORKDIR /home/deepfence/usr
 COPY --from=builder /usr/local/yara.tar.gz /usr/local/yara.tar.gz
 COPY --from=builder /home/deepfence/src/YaRadare/YaRadare .

core/session.go

@@ -50,6 +50,16 @@ func (s *Session) InitThreads() {
 	runtime.GOMAXPROCS(*s.Options.Threads + 1)
 }
 
+func AddSessionRules(rulesSession *Session) *Session {
+	rules, err := compile(filescan, rulesSession)
+	if err != nil {
+		rulesSession.Log.Error("compiling rules issue: %s", err)
+		os.Exit(1)
+	}
+	rulesSession.YaraRules = rules
+	return rulesSession
+}
+
 func GetSession() *Session {
 	sessionSync.Do(func() {
 		session = &Session{
@@ -76,12 +86,7 @@ func GetSession() *Session {
 
 		session.Start()
 
-		rules, err := compile(filescan, session)
-		if err != nil {
-			session.Log.Error("compiling rules issue: %s", err)
-			os.Exit(1)
-		}
-		session.YaraRules = rules
+
 	})
 
 	return session

main.go

@@ -25,8 +25,22 @@ package main
 // ------------------------------------------------------------------------------
 
 import (
+	"archive/tar"
+	"compress/gzip"
+	"crypto/sha256"
+	"encoding/json"
 	"flag"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/url"
+	"net/http"
+	"os"
+	"path"
+	"path/filepath"
 	"strings"
+	"sync"
+	"time"
 
 	"github.com/deepfence/YaRadare/core"
 	"github.com/deepfence/YaRadare/output"
@@ -35,6 +49,30 @@ import (
 	"github.com/fatih/color"
 )
 
+type YaraRuleDetail struct {
+	Built    time.Time `json:"built"`
+	Version  int       `json:"version"`
+	URL      string    `json:"url"`
+	Checksum string    `json:"checksum"`
+}
+
+type YaraRuleListingV3 struct {
+	V3 []YaraRuleDetail `json:"3"`
+}
+
+type YaraRuleListing struct {
+	Available YaraRuleListingV3 `json:"available"`
+}
+
+type YaraRuleUpdater struct {
+	yaraRuleListingJson  YaraRuleListing
+	yaraRulePath         string
+	downloadYaraRulePath string
+	currentFileChecksum  string
+	currentFilePath      string
+	sync.RWMutex
+}
+
 const (
 	PLUGIN_NAME = "MalwareScanner"
 )
@@ -174,9 +212,245 @@ func runOnce() {
 	}
 }
 
+func sha256sum(filePath string) (string, error) {
+	file, err := os.Open(filePath)
+	if err != nil {
+		return "", err
+	}
+	defer file.Close()
+
+	hash := sha256.New()
+	if _, err := io.Copy(hash, file); err != nil {
+		return "", err
+	}
+	return fmt.Sprintf("sha256:%x", hash.Sum(nil)), nil
+}
+
+func fileExists(path string) bool {
+	_, err := os.Stat(path)
+	if err == nil {
+		return true
+	}
+	return false
+}
+
+func NewYaraRuleUpdater() (error, *YaraRuleUpdater) {
+	updater := &YaraRuleUpdater{
+		yaraRuleListingJson:  YaraRuleListing{},
+		yaraRulePath:         path.Join(*core.GetSession().Options.RulesPath, "metaListingData.json"),
+		downloadYaraRulePath: "",
+	}
+	if fileExists(updater.yaraRulePath) {
+		content, err := ioutil.ReadFile(updater.yaraRulePath)
+		if err != nil {
+			return err, nil
+		}
+		err = json.Unmarshal(content, &updater)
+		if err != nil {
+			return err, nil
+		}
+	}
+	return nil, updater
+}
+
+func untar(dst string, r io.Reader) error {
+
+	gzr, err := gzip.NewReader(r)
+	if err != nil {
+		return err
+	}
+	defer gzr.Close()
+	tr := tar.NewReader(gzr)
+	for {
+		header, err := tr.Next()
+		switch {
+		// if no more files are found return
+		case err == io.EOF:
+			return nil
+		// return any other error
+		case err != nil:
+			return err
+		// if the header is nil, just skip it (not sure how this happens)
+		case header == nil:
+			continue
+		}
+		// the target location where the dir/file should be created
+		target := filepath.Join(dst, header.Name)
+		fmt.Println("the target is", target)
+		// the following switch could also be done using fi.Mode(), not sure if there
+		// a benefit of using one vs. the other.
+		// fi := header.FileInfo()
+
+		// check the file type
+		switch header.Typeflag {
+		// if its a dir and it doesn't exist create it
+		// if it's a file create it
+		case tar.TypeReg:
+			fmt.Println("the j is", header.Name)
+			if strings.Contains(header.Name ,".yar") {
+				f, err := os.OpenFile(target, os.O_CREATE|os.O_RDWR, os.FileMode(header.Mode))
+				fmt.Println("the target is", header.Name)
+				if err != nil {
+					return err
+				}
+				// copy over contents
+				if _, err := io.Copy(f, tr); err != nil {
+					return err
+				}
+
+				// manually close here after each file operation; defering would cause each file close
+				// to wait until all operations have completed.
+				f.Close()
+			}
+
+		}
+	}
+}
+
+func downloadFile(dUrl string, dest string) (error,string){
+	fmt.Println("the dynamic url is",dUrl)
+	fullUrlFile := dUrl
+
+    // Build fileName from fullPath
+    fileURL, err := url.Parse(fullUrlFile)
+    if err != nil {
+        return err, ""
+    }
+	fmt.Println("the dynamic url is",fileURL)
+    path := fileURL.Path
+    segments := strings.Split(path, "/")
+    fileName := segments[len(segments)-1]
+
+    // Create blank file
+    file, err := os.Create(filepath.Join(dest,fileName))
+    if err != nil {
+        return err, ""
+    }
+    client := http.Client{
+        CheckRedirect: func(r *http.Request, via []*http.Request) error {
+            r.URL.Opaque = r.URL.Path
+            return nil
+        },
+    }
+    // Put content on file
+    resp, err := client.Get(fullUrlFile)
+	fmt.Println("the dynamic url is",fileName)
+    if err != nil {
+        return err, ""
+    }
+    defer resp.Body.Close()
+
+    size, err := io.Copy(file, resp.Body)
+	fmt.Println("copied size", size)
+	if err != nil {
+        return err, ""
+    }
+    fmt.Println("the dynamic url is",fileURL)
+    defer file.Close()
+	return nil,fileName
+
+}
+
+func writeToFile(dUrl string, dest string) error{
+	fullUrlFile := dUrl
+
+    // Build fileName from fullPath
+    fileURL, err := url.Parse(fullUrlFile)
+    if err != nil {
+        return err
+    }
+    path := fileURL.Path
+    segments := strings.Split(path, "/")
+    fileName := segments[len(segments)-1]
+
+    // Create blank file
+    file, err := os.Create(filepath.Join(dest,fileName))
+    if err != nil {
+        return err
+    }
+    client := http.Client{
+        CheckRedirect: func(r *http.Request, via []*http.Request) error {
+            r.URL.Opaque = r.URL.Path
+            return nil
+        },
+    }
+    // Put content on file
+    resp, err := client.Get(fullUrlFile)
+    if err != nil {
+        return err
+    }
+    defer resp.Body.Close()
+
+    size, err := io.Copy(file, resp.Body)
+	fmt.Println("copied size", size)
+	if err != nil {
+        return err
+    }
+
+    defer file.Close()
+	 return nil
+
+}
+
 func main() {
 	flag.Parse()
-	core.GetSession().Log.Info("server inside23 port", *session.Options)
+	core.GetSession().Log.Error("reached here")
+	ch := make(chan bool)
+	go func() {
+		err, yaraRuleUpdater := NewYaraRuleUpdater()
+		if err != nil {
+			core.GetSession().Log.Error("main: failed to serve: %v", err)
+		}
+		downloadError,_ := downloadFile("https://threat-intel.deepfence.io/yara-rules/listing.json",*core.GetSession().Options.ConfigPath)
+		if downloadError != nil {
+			core.GetSession().Log.Error("main: failed to serve: %v", downloadError)
+		}
+		core.GetSession().Log.Error("reached here 2")
+		content, err := ioutil.ReadFile(filepath.Join(*core.GetSession().Options.ConfigPath,"/listing.json"))
+		if err != nil {
+			core.GetSession().Log.Error("main: failed to serve: %v", err)
+		}
+		var yaraRuleListingJson YaraRuleListing
+		err = json.Unmarshal(content, &yaraRuleListingJson)
+		if err != nil {
+			core.GetSession().Log.Error("main: failed to serve: %v", err)
+		}
+		if len(yaraRuleListingJson.Available.V3) > 0 {
+			core.GetSession().Log.Error("reached here 4 %v", yaraRuleListingJson.Available.V3[0].Checksum)
+			if yaraRuleListingJson.Available.V3[0].Checksum != yaraRuleUpdater.currentFileChecksum {
+				yaraRuleUpdater.currentFileChecksum = yaraRuleListingJson.Available.V3[0].Checksum
+				file, _ := json.MarshalIndent(yaraRuleUpdater, "", " ")
+				ioutil.WriteFile(path.Join(*core.GetSession().Options.RulesPath, "metaListingData.json"),file,0644)
+				downloadError,fileName := downloadFile(yaraRuleListingJson.Available.V3[0].URL,*core.GetSession().Options.ConfigPath)
+				fmt.Println("reached here 5 times", fileName)
+
+				if downloadError != nil {
+					core.GetSession().Log.Error("main: failed to serve: %v", downloadError)
+				}
+				if fileExists(filepath.Join(*core.GetSession().Options.ConfigPath,fileName)) {
+					fmt.Println("the file exists")
+					readFile, readErr := os.Open(filepath.Join(*core.GetSession().Options.ConfigPath,fileName))
+					if readErr != nil {
+						core.GetSession().Log.Error("main: failed to serve: %v", readErr)
+					}
+					unTarErr := untar(*core.GetSession().Options.RulesPath,readFile)
+					if unTarErr != nil {
+						core.GetSession().Log.Error("main: failed to serve: %v", unTarErr)
+					}
+					defer readFile.Close()
+
+				}
+
+			}
+		}
+
+		ch <- true
+
+	}()
+	<-ch
+	fmt.Println("server inside23 port", *session.Options)
+
+	//core.AddSessionRules(core.GetSession())
 	if *session.Options.SocketPath != "" {
 		err := server.RunServer(*session.Options.SocketPath, PLUGIN_NAME)
 		if err != nil {