Skip to content

Commit

Permalink
v0.0.5
Browse files Browse the repository at this point in the history
  • Loading branch information
bobby96333 committed Jan 26, 2024
1 parent 1f6a1bf commit d4840ff
Show file tree
Hide file tree
Showing 6 changed files with 237 additions and 129 deletions.
12 changes: 12 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,18 @@ test group match
cat demo/demo.log|ggrep --group 'LogId\[[:logid:]\]' --color=always|head
```

# Print full group of log

```shell
ggrep --smart-logid --print-group --grep 'LOG2' --full-match --color always demo/demo.log
```
output:
```text
20240119 LogId[111] LOG1
20240119 LogId[111] LOG2
20240119 LogId[111] LOG3
```


# install
```shell
Expand Down
2 changes: 1 addition & 1 deletion build.sh
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
#!/bin/bash
VERSION=v0.0.3
VERSION=v0.0.5

echo $VERSION

Expand Down
151 changes: 62 additions & 89 deletions command.go
Original file line number Diff line number Diff line change
Expand Up @@ -43,8 +43,13 @@ type command struct {
fullMatch bool
recursive bool
ignoreCase bool
printGroup bool
smartLogId bool
lineIndex int
perlModel bool
color string
colorFormat ColorFormatFunc
paramHelp bool
}

const DefaultCutLineSize = 1024 * 1024 * 5
Expand Down Expand Up @@ -74,11 +79,17 @@ func (c *command) ready() {
flag.BoolVar(&c.fullMatch, "full-match", false, "Only output checked all grep expression in group, will be output")
flag.BoolVar(&c.ignoreCase, "i", false, "ignore case in match")
flag.BoolVar(&c.fullMatch, "F", false, "Only output checked all grep expression in group, will be output")
flag.BoolVar(&c.printGroup, "print-group", false, "print full content of group")
flag.BoolVar(&c.smartLogId, "smart-logid", false, "identify LogId in log content automatically")
flag.BoolVar(&c.perlModel, "perl", false, "as close to Perl as possible")
flag.BoolVar(&c.perlModel, "P", false, "as close to Perl as possible")
//support auto todo
flag.StringVar(&c.color, "color", "", "will highlight the matched word when you set always")
//todo
//flag.BoolVar(&c.recursive, "recursive", false, "Recursively search subdirectories listed")

flag.BoolVar(&c.paramHelp, "help", false, "show usage of ggrep")
flag.BoolVar(&c.paramHelp, "h", false, "show udage of ggrep --help(shorthand)")
}

func (c *command) valid() error {
Expand All @@ -98,31 +109,47 @@ func (c *command) valid() error {
} else {
c.errLog = log.New(os.Stderr, "[err]", log.Lmsgprefix)
}
if c.group == "" {
if c.smartLogId {
c.groupRegex = regexp.MustCompile("(?i)logid[^a-z0-9_-]+[0-9a-z_-]+")
}
if c.group == "" && !c.smartLogId {
return errors.New("no found parameter: group")
}
c.regexps = make([]*regexp.Regexp, len(c.greps))
var err error
if c.groupRegex, err = c.ggrepRegex(c.group); err == nil {
for i, grep := range c.greps {
if c.regexps[i], err = c.ggrepRegex(grep); err != nil {
return fmt.Errorf("grep (%s) expression mistake:%w", grep, err)
}
for i, grep := range c.greps {
if c.regexps[i], err = c.ggrepRegex(grep); err != nil {
return fmt.Errorf("grep (%s) expression mistake:%w", grep, err)
}
} else {
return fmt.Errorf("group expression mistake:%w", err)
}
if !c.smartLogId {
var err error
c.groupRegex, err = regexp.CompilePOSIX(c.group)
util.CheckPanic(err)
}
return nil
}
func (c *command) ggrepRegex(exp string) (*regexp.Regexp, error) {
if c.perlModel {
if c.ignoreCase {
exp = "(?i)" + exp
}
exp = strings.ReplaceAll(exp, "[[:logid:]]", "[0-9a-zA-z_\\-]+")
return regexp.Compile(exp)
}
exp = strings.ReplaceAll(exp, "[[:logid:]]", "[0-9a-zA-z_\\-]+")
if c.ignoreCase {
exp = "(?i)" + exp
exp = strings.ToLower(exp)
}
exp = strings.ReplaceAll(exp, "[:logid:]", "[0-9a-zA-z_\\-]+")
return regexp.CompilePOSIX(exp)

}

func (c *command) run() {
if c.paramHelp {
flag.Usage()
return
}
if err := c.valid(); err != nil {
c.errLog.Printf("valid faild,%s", err.Error())
os.Exit(1)
Expand All @@ -134,6 +161,9 @@ func (c *command) run() {
runtime.GOMAXPROCS(c.parallelCnt)
}
start := time.Now()
if c.color == "always" {
c.colorFormat = GrepColorRedFormat
}
c.verboseLogf("ggroup prepared, parameters:%+v", *c)
c.groupWorkers = cyclemap.New[string, int](c.cacheGroupSize, false)
c.outChan = make(chan string, 100)
Expand All @@ -142,14 +172,9 @@ func (c *command) run() {
waitGroup := &sync.WaitGroup{}
waitGroup.Add(c.parallelCnt)
c.verboseLogf("waitgroup count:%d", c.parallelCnt)
if len(c.greps) > 0 {
for i := 0; i < c.parallelCnt; i++ {
c.workerChans[i] = make(chan *inputdata, 100)
go c.startMatchWorker(ctx, waitGroup, i)
}
} else {
c.workerChans[0] = make(chan *inputdata, 100)
go c.startDirectWorker(ctx, waitGroup, 0)
for i := 0; i < c.parallelCnt; i++ {
c.workerChans[i] = make(chan *inputdata, 100)
go c.startMatchWorker(ctx, waitGroup, i)
}
var finishedChan = make(chan struct{})
go c.startPrint(ctx, finishedChan)
Expand Down Expand Up @@ -201,7 +226,8 @@ func (c *command) readFile(ctx context.Context, fi int) {
c.verboseLogf("log:%s", string(bs))
}
line := string(bs)
groupIndexes := c.groupRegex.FindStringIndex(line)
var groupIndexes []int
groupIndexes = c.groupRegex.FindStringIndex(line)
if len(groupIndexes) > 1 {
group := line[groupIndexes[0]:groupIndexes[1]]
if groupColorFormat != nil {
Expand All @@ -220,89 +246,36 @@ func (c *command) readFile(ctx context.Context, fi int) {
}
}

func (c *command) CheckGrepMatch(ctx context.Context, matchIndex uint, line string) []int {

if c.orderlyMatch {
if int(matchIndex) < len(c.regexps) {
reg := c.regexps[matchIndex]
return reg.FindStringIndex(line)
}
} else {
for i := 0; i < len(c.regexps); i++ {
if !util.IsTrue(matchIndex, i) {
reg := c.regexps[i]
return reg.FindStringIndex(line)
}
}
}
if len(c.regexps) < 1 {
return []int{0, 0}
}
c.errLog.Println("unknow error #202401222101")
os.Exit(1)
return nil
}

func (c *command) startDirectWorker(ctx context.Context, waitGroup *sync.WaitGroup, goIndex int) {
defer func() {
waitGroup.Done()
}()
for {
if lineInfo, ok := <-c.workerChans[goIndex]; ok {
c.outChan <- lineInfo.line
} else {
c.verboseLogf("directWorker end")
break
}
}
}

func (c *command) startMatchWorker(ctx context.Context, waitGroup *sync.WaitGroup, goIndex int) {
c.verboseLogf("orderly scan worker %d started", goIndex)
var colorFormat ColorFormatFunc
if c.color == "always" {
colorFormat = GrepColorRedFormat
}

var wokerCycleMaps = cyclemap.New[string, *groupbuff.GroupBuff](c.cacheGroupSize/c.parallelCnt, false)
if !c.fullMatch {
wokerCycleMaps.SetListenRemoveFunc(func(group string, item *groupbuff.GroupBuff) {
wokerCycleMaps.SetListenRemoveFunc(func(group string, item *groupbuff.GroupBuff) {
if (!c.fullMatch && item.MatchIndex() > 0) || item.FullMatched() {
c.outChan <- item.String()
})
}
}
})
defer func() {
if !c.fullMatch {
iter := wokerCycleMaps.Iter()
for item, ok := iter.First(); ok; item, ok = iter.Next() {
iter := wokerCycleMaps.Iter()
for item, ok := iter.First(); ok; item, ok = iter.Next() {
if (!c.fullMatch && item.MatchIndex() > 0) || item.FullMatched() {
c.outChan <- item.String()
}
}
waitGroup.Done()
c.verboseLogf("orderly scan worker %d finished", goIndex)
}()
var outFunc matchFunc
if len(c.greps) == 0 {
outFunc = matchDirectOut
} else if c.orderlyMatch {
outFunc = matchOrderlyOut
} else {
outFunc = matchUnOrderlyOut
}
for {
if lineInfo, ok := <-c.workerChans[goIndex]; ok {
line := lineInfo.line
group := lineInfo.group
groupItem, existsGroup := wokerCycleMaps.Get(group)

var matchIndex uint = 0
if existsGroup {
matchIndex = groupItem.MatchIndex()
}
if indexes := c.CheckGrepMatch(ctx, matchIndex, line); len(indexes) > 1 {
if colorFormat != nil {
line = colorFormat(line, indexes)
}
if !existsGroup {
groupItem = groupbuff.NewItem(goIndex, c.orderlyMatch, false)
wokerCycleMaps.Set(group, groupItem)
}

if bs, finished := groupItem.Write(line, matchIndex, uint(len(c.regexps)), c.mergeLine); finished {
wokerCycleMaps.Remove(group)
c.outChan <- string(bs)
}
}
outFunc(c, lineInfo, wokerCycleMaps)
} else {
c.verboseLogf("worker %d canceled", goIndex)
break
Expand Down
134 changes: 134 additions & 0 deletions command_matcher.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,134 @@
package main

import (
"github.com/lingdor/cyclemap"
"github.com/lingdor/ggrep/groupbuff"
"github.com/lingdor/ggrep/util"
"strings"
)

type matchFunc func(c *command, lineInfo *inputdata, wokerCycleMaps *cyclemap.CycleMap[string, *groupbuff.GroupBuff])

func matchOrderlyOut(c *command, lineInfo *inputdata, wokerCycleMaps *cyclemap.CycleMap[string, *groupbuff.GroupBuff]) {
line := lineInfo.line
group := lineInfo.group
groupItem, existsGroup := wokerCycleMaps.Get(lineInfo.group)
var matchIndex uint = 0
var isOutputted = false
if existsGroup {
matchIndex = groupItem.MatchIndex()
}
seekline := line
if c.ignoreCase {
seekline = strings.ToLower(seekline)
}
if int(matchIndex) < len(c.regexps) {
reg := c.regexps[matchIndex]
if indexes := reg.FindStringIndex(seekline); len(indexes) > 1 {
if c.colorFormat != nil {
line = c.colorFormat(line, indexes)
}
if !existsGroup {
groupItem = groupbuff.NewItem()
wokerCycleMaps.Set(group, groupItem)
}
groupItem.Increment()
isOutputted = true
//check buff size, and break todo
groupItem.Write([]byte(line), c.mergeLine)
if groupItem.MatchIndex() >= uint(len(c.greps)) {
groupItem.SetFullMatched(true)
if !c.printGroup {
wokerCycleMaps.Remove(group)
c.outChan <- groupItem.String()
}
}
return
}
}

if c.printGroup && !isOutputted {
if !existsGroup {
groupItem = groupbuff.NewItem()
wokerCycleMaps.Set(group, groupItem)
}
groupItem.Write([]byte(line), c.mergeLine)
}
}

func matchDirectOut(c *command, lineInfo *inputdata, wokerCycleMaps *cyclemap.CycleMap[string, *groupbuff.GroupBuff]) {

c.outChan <- lineInfo.line

}
func matchUnOrderlyOut(c *command, lineInfo *inputdata, wokerCycleMaps *cyclemap.CycleMap[string, *groupbuff.GroupBuff]) {
line := lineInfo.line
group := lineInfo.group
groupItem, existsGroup := wokerCycleMaps.Get(lineInfo.group)
var matchIndex uint = 0
var isOutputted = false
if existsGroup {
matchIndex = groupItem.MatchIndex()
}
seekline := line
if c.ignoreCase {
seekline = strings.ToLower(seekline)
}
for i := 0; i < len(c.regexps); i++ {
if !util.IsTrue(matchIndex, i) {
reg := c.regexps[i]
if indexes := reg.FindStringIndex(seekline); len(indexes) > 1 {
if c.colorFormat != nil {
line = c.colorFormat(line, indexes)
}
if !existsGroup {
groupItem = groupbuff.NewItem()
wokerCycleMaps.Set(group, groupItem)
}
isOutputted = true
matchIndex = util.SetTrue(matchIndex, int(matchIndex))
groupItem.SetIndex(matchIndex)
groupItem.Write([]byte(line), c.mergeLine)
full := util.FullIntBinary(len(c.regexps))
if matchIndex == full {
groupItem.SetFullMatched(true)
if !c.printGroup {
wokerCycleMaps.Remove(group)
c.outChan <- groupItem.String()
}
}
return
}
}
}
if c.printGroup && !isOutputted {
if !existsGroup {
groupItem = groupbuff.NewItem()
wokerCycleMaps.Set(group, groupItem)
}
groupItem.Write([]byte(line), c.mergeLine)
}
}

//func (c *command) CheckGrepMatch(ctx context.Context, matchIndex uint, line string) []int {
// if c.orderlyMatch {
// if int(matchIndex) < len(c.regexps) {
// reg := c.regexps[matchIndex]
// return reg.FindStringIndex(line)
// }
// } else {
// for i := 0; i < len(c.regexps); i++ {
// if !util.IsTrue(matchIndex, i) {
// reg := c.regexps[i]
// return reg.FindStringIndex(line)
// }
// }
// }
// if len(c.regexps) < 1 {
// return []int{0, 0}
// }
// c.errLog.Println("unknow error #202401222101")
// os.Exit(1)
// return nil
//
//}
Loading

0 comments on commit d4840ff

Please sign in to comment.