add hash sum file and profiler

cmd/datasheet-converter.go

@@ -6,8 +6,8 @@ import (
 	"errors"
 	"flag"
 	"fmt"
-	"github.com/ake-persson/mapslice-json"
 	"github.com/new-world-tools/extracter/datasheet"
+	"github.com/new-world-tools/extracter/profiler"
 	workerpool "github.com/zelenin/go-worker-pool"
 	"log"
 	"math"
@@ -25,6 +25,7 @@ var (
 	pool      *workerpool.Pool
 	outputDir string
 	format    string
+	pr        *profiler.Profiler
 )
 
 const (
@@ -38,6 +39,8 @@ var formats = map[string]bool{
 }
 
 func main() {
+	pr = profiler.New()
+
 	inputDirPtr := flag.String("input", ".\\assets", "directory path")
 	outputDirPtr := flag.String("output", ".\\assets\\datasheets", "directory path")
 	formatPtr := flag.String("format", "csv", "csv or json")
@@ -105,6 +108,8 @@ func main() {
 	}
 
 	pool.Wait()
+
+	log.Printf("PeakMemory: %0.1fMb Duration: %s", float64(pr.GetPeakMemory())/1024/1024, pr.GetDuration().String())
 }
 
 func addTask(id int64, file *datasheet.DataSheetFile) {

cmd/pak-extracter.go

@@ -1,16 +1,19 @@
 package main
 
 import (
+	"crypto/sha1"
 	"errors"
 	"flag"
 	"fmt"
+	"github.com/new-world-tools/extracter/hash"
 	"github.com/new-world-tools/extracter/pak"
-	"github.com/new-world-tools/go-oodle"
+	"github.com/new-world-tools/extracter/profiler"
 	workerpool "github.com/zelenin/go-worker-pool"
 	"io"
 	"log"
 	"os"
 	"path/filepath"
+	"sort"
 	"strings"
 )
 
@@ -20,13 +23,18 @@ const (
 )
 
 var (
-	pool      *workerpool.Pool
-	filters   map[string]bool
-	assetsDir string
-	outputDir string
+	pool         *workerpool.Pool
+	filters      map[string]bool
+	assetsDir    string
+	outputDir    string
+	hashSumFile  string
+	hashRegistry *hash.Registry
+	pr           *profiler.Profiler
 )
 
 func main() {
+	pr = profiler.New()
+
 	_, err := os.Stat("oo2core_9_win64.dll")
 	if os.IsNotExist(err) {
 		err := oodle.Download()
@@ -39,6 +47,7 @@ func main() {
 	outputDirPtr := flag.String("output", "./extract", "directory path")
 	filterPtr := flag.String("filter", "", "comma separated file extensions")
 	threadsPtr := flag.Int("threads", defaultThreads, fmt.Sprintf("1-%d", maxThreads))
+	hashSumFilePtr := flag.String("hash", "", "hash sum path")
 	flag.Parse()
 
 	assetsDir, err = filepath.Abs(filepath.Clean(*assetsDirPtr))
@@ -68,6 +77,15 @@ func main() {
 	}
 	log.Printf("The number of threads is set to %d", threads)
 
+	hashSumFile = *hashSumFilePtr
+	if hashSumFile != "" {
+		hashSumFile, err = filepath.Abs(filepath.Clean(hashSumFile))
+		if err != nil {
+			log.Fatalf("filepath.Abs: %s", err)
+		}
+		hashRegistry = hash.NewRegistry()
+	}
+
 	err = os.MkdirAll(outputDir, 0755)
 	if err != nil {
 		log.Fatalf("MkdirAll: %s", err)
@@ -113,6 +131,30 @@ func main() {
 	}
 
 	pool.Wait()
+
+	if hashSumFile != "" {
+		log.Printf("Writing %s", hashSumFile)
+
+		hashes := hashRegistry.Hashes()
+		sort.Slice(hashes, func(i, j int) bool {
+			return hashes[i].FileName < hashes[j].FileName
+		})
+
+		hashSumsFile, err := os.Create(hashSumFile)
+		if err != nil {
+			log.Fatalf("os.Create: %s", err)
+		}
+		defer hashSumsFile.Close()
+
+		for _, fileHash := range hashes {
+			_, err = hashSumsFile.WriteString(fmt.Sprintf("%x *%s\n", fileHash.Hash, fileHash.FileName))
+			if err != nil {
+				log.Fatalf("hashSumsFile.WriteString: %s", err)
+			}
+		}
+	}
+
+	log.Printf("PeakMemory: %0.1fMb Duration: %s", float64(pr.GetPeakMemory())/1024/1024, pr.GetDuration().String())
 }
 
 func addTask(id int64, pakFile *pak.Pak, file *pak.File) {
@@ -133,15 +175,29 @@ func addTask(id int64, pakFile *pak.Pak, file *pak.File) {
 		}
 		defer dest.Close()
 
-		reader, err := file.Decompress()
+		decompressReader, err := file.Decompress()
 		if err != nil {
 			return err
 		}
-		defer reader.Close()
+		defer decompressReader.Close()
 
-		_, err = io.Copy(dest, reader)
-		if err != nil {
-			return err
+		if hashSumFile == "" {
+			reader := decompressReader
+
+			_, err = io.Copy(dest, reader)
+			if err != nil {
+				return err
+			}
+		} else {
+			hasher := sha1.New()
+			reader := io.TeeReader(decompressReader, hasher)
+
+			_, err = io.Copy(dest, reader)
+			if err != nil {
+				return err
+			}
+
+			hashRegistry.Add(file.Name, hasher.Sum(nil))
 		}
 
 		return nil

hash/registry.go

@@ -0,0 +1,38 @@
+package hash
+
+import (
+	"sync"
+)
+
+type Hash struct {
+	FileName string
+	Hash     []byte
+}
+
+type Registry struct {
+	hashes []*Hash
+	mu     sync.Mutex
+}
+
+func NewRegistry() *Registry {
+	return &Registry{
+		hashes: []*Hash{},
+	}
+}
+
+func (registry *Registry) Add(fileName string, hash []byte) {
+	registry.mu.Lock()
+	defer registry.mu.Unlock()
+
+	registry.hashes = append(registry.hashes, &Hash{
+		FileName: fileName,
+		Hash:     hash,
+	})
+}
+
+func (registry *Registry) Hashes() []*Hash {
+	registry.mu.Lock()
+	defer registry.mu.Unlock()
+
+	return registry.hashes
+}

profiler/profiler.go

@@ -0,0 +1,67 @@
+package profiler
+
+import (
+	"runtime"
+	"sync"
+	"time"
+)
+
+func New() *Profiler {
+	profiler := &Profiler{
+		start: time.Now(),
+	}
+
+	profiler.loadCurrentMemory()
+
+	go profiler.poller()
+
+	return profiler
+}
+
+type Profiler struct {
+	peakMemory    uint64
+	currentMemory uint64
+	start         time.Time
+	mu            sync.Mutex
+}
+
+func (profiler *Profiler) GetCurrentMemory() uint64 {
+	profiler.mu.Lock()
+	defer profiler.mu.Unlock()
+
+	return profiler.currentMemory
+}
+
+func (profiler *Profiler) GetPeakMemory() uint64 {
+	profiler.mu.Lock()
+	defer profiler.mu.Unlock()
+
+	return profiler.peakMemory
+}
+
+func (profiler *Profiler) GetDuration() time.Duration {
+	profiler.mu.Lock()
+	defer profiler.mu.Unlock()
+
+	return time.Now().Sub(profiler.start)
+}
+
+func (profiler *Profiler) loadCurrentMemory() {
+	profiler.mu.Lock()
+	defer profiler.mu.Unlock()
+
+	var memory runtime.MemStats
+	runtime.ReadMemStats(&memory)
+
+	profiler.currentMemory = memory.Sys
+	if profiler.currentMemory > profiler.peakMemory {
+		profiler.peakMemory = profiler.currentMemory
+	}
+}
+
+func (profiler *Profiler) poller() {
+	ticker := time.NewTicker(1 * time.Second)
+	for range ticker.C {
+		profiler.loadCurrentMemory()
+	}
+}