From 5e87a9f9cc2703dde6be6dc345ad0334fbc7643b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=A3=8E=E5=90=B9=E6=88=91=E5=B7=B2=E6=95=A3?= Date: Fri, 21 Oct 2022 09:41:18 +0800 Subject: [PATCH] =?UTF-8?q?=E9=87=8D=E6=96=B0=E5=AE=9E=E7=8E=B0=E4=BA=86?= =?UTF-8?q?=20wry=20=E6=A0=BC=E5=BC=8F=E6=95=B0=E6=8D=AE=E5=BA=93=E8=A7=A3?= =?UTF-8?q?=E6=9E=90=EF=BC=8C=E5=8C=85=E6=8B=AC=20qqwry=20=E5=92=8C=20zxip?= =?UTF-8?q?v6wry=EF=BC=8C=E6=94=AF=E6=8C=81=E5=B9=B6=E5=8F=91=E6=9F=A5?= =?UTF-8?q?=E8=AF=A2=20=E4=BF=AE=E5=A4=8Dwry=E6=96=87=E4=BB=B6=E4=B8=8D?= =?UTF-8?q?=E5=AE=8C=E6=95=B4=E6=97=B6crash=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 对 nali 的数据库和配置文件目录进行了规范，具体见 [文档-工作目录](https://github.com/zu1k/nali#%E5%B7%A5%E4%BD%9C%E7%9B%AE%E5%BD%95) 优先使用环境变量 NALI_HOME、NALI_CONFIG_HOME、NALI_DB_HOME 指定的目录 未指定 nali 特定环境变量的情况下默认使用 XDG 规范，配置文件目录在 $XDG_CONFIG_HOME/nali，数据库文件目录在 $XDG_DATA_HOME/nali 若未检测到 XDG 相关环境变量，将根据平台使用默认目录，具体见 XDG Base Directory 中 XDG_CONFIG_HOME 和 XDG_DATA_HOME 部分 初次运行此版本将会进行目录和数据的迁移，将 ~/.nali 下的配置文件和数据库转移到相应目录，并删除 ~/.nali 目录 支持使用环境变量指定的代理下载数据库，thanks to @jingjingxyk #126 修复了 pipe mtr 时无法获取内容和格式错乱的问题，thanks to @mzz2017 #132 , fix #12, #61, #85, #115, #123. cache map 使用并发安全的版本，thanks to @lhcn #125 升级 Go 版本到 1.19，更新了依赖 不再支持 ip2region 旧数据库格式，目前仅支持 ip2region xdb 格式 去除了已过时的数据库下载代码 从 git 历史记录中去除了数据库文件 修复自动迁移导致生成空配置文件的bug 更新纯真IP数据库的下载地址 --- .github/workflows/go.yml | 2 +- Makefile | 213 ++++++++++++----------- cmd/root.go | 184 ++++++++++---------- cmd/update.go | 60 +++---- go.mod | 4 +- go.sum | 8 +- internal/config/config.go | 82 ++++----- internal/constant/path.go | 82 +++++---- internal/constant/version.go | 14 +- internal/db/cache.go | 32 ++-- internal/db/db.go | 166 +++++++++--------- internal/db/default.go | 178 +++++++++---------- internal/db/type.go | 282 +++++++++++++++---------------- internal/db/update.go | 188 +++++++++------------ internal/migration/mirgration.go | 6 + internal/migration/v4.go | 51 ++++++ internal/migration/v6.go | 43 +++++ main.go | 5 +- pkg/cdn/cdn.go | 212 +++++++++++++---------- pkg/cdn/update.go | 31 ---- pkg/common/const.go | 8 - pkg/common/dbtool.go | 8 - pkg/common/httpclient.go | 116 ++++++------- pkg/common/savefile.go | 42 ++--- pkg/common/scan.go | 26 +++ pkg/common/struct.go | 99 ----------- pkg/dbif/db.go | 70 ++++---- pkg/download/download.go | 48 +++--- pkg/entity/entity.go | 150 ++++++++-------- pkg/entity/parse.go | 144 ++++++++-------- pkg/entity/parse_test.go | 15 -- pkg/geoip/geoip.go | 137 ++++++++------- pkg/ip2location/ip2location.go | 122 ++++++------- pkg/ip2region/ip2region.go | 128 +++++++------- pkg/ip2region/update.go | 29 ---- pkg/ipip/ipip.go | 112 ++++++------ pkg/qqwry/qqwry.go | 237 ++++++++++---------------- pkg/qqwry/update.go | 72 -------- pkg/re/re.go | 27 +-- pkg/re/re_test.go | 92 ++++++---- pkg/wry/index.go | 63 +++++++ pkg/wry/parse.go | 47 ++++++ pkg/wry/wry.go | 116 +++++++++++++ pkg/zxipv6wry/update.go | 184 ++++++++++---------- pkg/zxipv6wry/zxipv6wry.go | 215 ++++++++++------------- 45 files changed, 2101 insertions(+), 2049 deletions(-) create mode 100644 internal/migration/mirgration.go create mode 100644 internal/migration/v4.go create mode 100644 internal/migration/v6.go delete mode 100644 pkg/cdn/update.go delete mode 100644 pkg/common/const.go delete mode 100644 pkg/common/dbtool.go create mode 100644 pkg/common/scan.go delete mode 100644 pkg/common/struct.go delete mode 100644 pkg/entity/parse_test.go delete mode 100644 pkg/ip2region/update.go delete mode 100644 pkg/qqwry/update.go create mode 100644 pkg/wry/index.go create mode 100644 pkg/wry/parse.go create mode 100644 pkg/wry/wry.go diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 1ac8dadc..1c4020a3 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -9,7 +9,7 @@ jobs: - name: Setup Go uses: actions/setup-go@v3 with: - go-version: '^1.18' + go-version: '^1.19' - name: Check out code into the Go module directory uses: actions/checkout@v3 diff --git a/Makefile b/Makefile index a647d1fd..6eac396f 100644 --- a/Makefile +++ b/Makefile @@ -1,108 +1,105 @@ -NAME=nali -BINDIR=bin -VERSION=$(shell git describe --tags || echo "unknown version") -BUILDTIME=$(shell date -u) -GOBUILD=CGO_ENABLED=0 go build -trimpath -ldflags '-X "github.com/zu1k/nali/internal/constant.Version=$(VERSION)" \ - -X "github.com/zu1k/nali/internal/constant.BuildTime=$(BUILDTIME)" \ - -w -s' - -PLATFORM_LIST = \ - darwin-arm64 \ - darwin-amd64 \ - linux-386 \ - linux-amd64 \ - linux-armv5 \ - linux-armv6 \ - linux-armv7 \ - linux-armv8 \ - linux-mips-softfloat \ - linux-mips-hardfloat \ - linux-mipsle-softfloat \ - linux-mipsle-hardfloat \ - linux-mips64 \ - linux-mips64le \ - freebsd-386 \ - freebsd-amd64 - -WINDOWS_ARCH_LIST = \ - windows-386 \ - windows-amd64 - -all: linux-amd64 darwin-amd64 windows-amd64 # Most used - -docker: - $(GOBUILD) -o $(BINDIR)/$(NAME)-$@ - -darwin-arm64: - GOARCH=arm64 GOOS=darwin $(GOBUILD) -o $(BINDIR)/$(NAME)-$@ - -darwin-amd64: - GOARCH=amd64 GOOS=darwin $(GOBUILD) -o $(BINDIR)/$(NAME)-$@ - -linux-386: - GOARCH=386 GOOS=linux $(GOBUILD) -o $(BINDIR)/$(NAME)-$@ - -linux-amd64: - GOARCH=amd64 GOOS=linux $(GOBUILD) -o $(BINDIR)/$(NAME)-$@ - -linux-armv5: - GOARCH=arm GOOS=linux GOARM=5 $(GOBUILD) -o $(BINDIR)/$(NAME)-$@ - -linux-armv6: - GOARCH=arm GOOS=linux GOARM=6 $(GOBUILD) -o $(BINDIR)/$(NAME)-$@ - -linux-armv7: - GOARCH=arm GOOS=linux GOARM=7 $(GOBUILD) -o $(BINDIR)/$(NAME)-$@ - -linux-armv8: - GOARCH=arm64 GOOS=linux $(GOBUILD) -o $(BINDIR)/$(NAME)-$@ - -linux-mips-softfloat: - GOARCH=mips GOMIPS=softfloat GOOS=linux $(GOBUILD) -o $(BINDIR)/$(NAME)-$@ - -linux-mips-hardfloat: - GOARCH=mips GOMIPS=hardfloat GOOS=linux $(GOBUILD) -o $(BINDIR)/$(NAME)-$@ - -linux-mipsle-softfloat: - GOARCH=mipsle GOMIPS=softfloat GOOS=linux $(GOBUILD) -o $(BINDIR)/$(NAME)-$@ - -linux-mipsle-hardfloat: - GOARCH=mipsle GOMIPS=hardfloat GOOS=linux $(GOBUILD) -o $(BINDIR)/$(NAME)-$@ - -linux-mips64: - GOARCH=mips64 GOOS=linux $(GOBUILD) -o $(BINDIR)/$(NAME)-$@ - -linux-mips64le: - GOARCH=mips64le GOOS=linux $(GOBUILD) -o $(BINDIR)/$(NAME)-$@ - -freebsd-386: - GOARCH=386 GOOS=freebsd $(GOBUILD) -o $(BINDIR)/$(NAME)-$@ - -freebsd-amd64: - GOARCH=amd64 GOOS=freebsd $(GOBUILD) -o $(BINDIR)/$(NAME)-$@ - -windows-386: - GOARCH=386 GOOS=windows $(GOBUILD) -o $(BINDIR)/$(NAME)-$@.exe - -windows-amd64: - GOARCH=amd64 GOOS=windows $(GOBUILD) -o $(BINDIR)/$(NAME)-$@.exe - -gz_releases=$(addsuffix .gz, $(PLATFORM_LIST)) -zip_releases=$(addsuffix .zip, $(WINDOWS_ARCH_LIST)) - -$(gz_releases): %.gz : % - chmod +x $(BINDIR)/$(NAME)-$(basename $@) - gzip -f -S -$(VERSION).gz $(BINDIR)/$(NAME)-$(basename $@) - -$(zip_releases): %.zip : % - zip -m -j $(BINDIR)/$(NAME)-$(basename $@)-$(VERSION).zip $(BINDIR)/$(NAME)-$(basename $@).exe - -all-arch: $(PLATFORM_LIST) $(WINDOWS_ARCH_LIST) - -releases: $(gz_releases) $(zip_releases) - -sha256sum: - cd $(BINDIR); for file in *; do sha256sum $$file > $$file.sha256; done - -clean: - rm $(BINDIR)/* +NAME=nali +BINDIR=bin +VERSION=$(shell git describe --tags || echo "unknown version") +GOBUILD=CGO_ENABLED=0 go build -trimpath -ldflags '-X "github.com/zu1k/nali/internal/constant.Version=$(VERSION)" -w -s' + +PLATFORM_LIST = \ + darwin-arm64 \ + darwin-amd64 \ + linux-386 \ + linux-amd64 \ + linux-armv5 \ + linux-armv6 \ + linux-armv7 \ + linux-armv8 \ + linux-mips-softfloat \ + linux-mips-hardfloat \ + linux-mipsle-softfloat \ + linux-mipsle-hardfloat \ + linux-mips64 \ + linux-mips64le \ + freebsd-386 \ + freebsd-amd64 + +WINDOWS_ARCH_LIST = \ + windows-386 \ + windows-amd64 + +all: linux-amd64 darwin-amd64 windows-amd64 # Most used + +docker: + $(GOBUILD) -o $(BINDIR)/$(NAME)-$@ + +darwin-arm64: + GOARCH=arm64 GOOS=darwin $(GOBUILD) -o $(BINDIR)/$(NAME)-$@ + +darwin-amd64: + GOARCH=amd64 GOOS=darwin $(GOBUILD) -o $(BINDIR)/$(NAME)-$@ + +linux-386: + GOARCH=386 GOOS=linux $(GOBUILD) -o $(BINDIR)/$(NAME)-$@ + +linux-amd64: + GOARCH=amd64 GOOS=linux $(GOBUILD) -o $(BINDIR)/$(NAME)-$@ + +linux-armv5: + GOARCH=arm GOOS=linux GOARM=5 $(GOBUILD) -o $(BINDIR)/$(NAME)-$@ + +linux-armv6: + GOARCH=arm GOOS=linux GOARM=6 $(GOBUILD) -o $(BINDIR)/$(NAME)-$@ + +linux-armv7: + GOARCH=arm GOOS=linux GOARM=7 $(GOBUILD) -o $(BINDIR)/$(NAME)-$@ + +linux-armv8: + GOARCH=arm64 GOOS=linux $(GOBUILD) -o $(BINDIR)/$(NAME)-$@ + +linux-mips-softfloat: + GOARCH=mips GOMIPS=softfloat GOOS=linux $(GOBUILD) -o $(BINDIR)/$(NAME)-$@ + +linux-mips-hardfloat: + GOARCH=mips GOMIPS=hardfloat GOOS=linux $(GOBUILD) -o $(BINDIR)/$(NAME)-$@ + +linux-mipsle-softfloat: + GOARCH=mipsle GOMIPS=softfloat GOOS=linux $(GOBUILD) -o $(BINDIR)/$(NAME)-$@ + +linux-mipsle-hardfloat: + GOARCH=mipsle GOMIPS=hardfloat GOOS=linux $(GOBUILD) -o $(BINDIR)/$(NAME)-$@ + +linux-mips64: + GOARCH=mips64 GOOS=linux $(GOBUILD) -o $(BINDIR)/$(NAME)-$@ + +linux-mips64le: + GOARCH=mips64le GOOS=linux $(GOBUILD) -o $(BINDIR)/$(NAME)-$@ + +freebsd-386: + GOARCH=386 GOOS=freebsd $(GOBUILD) -o $(BINDIR)/$(NAME)-$@ + +freebsd-amd64: + GOARCH=amd64 GOOS=freebsd $(GOBUILD) -o $(BINDIR)/$(NAME)-$@ + +windows-386: + GOARCH=386 GOOS=windows $(GOBUILD) -o $(BINDIR)/$(NAME)-$@.exe + +windows-amd64: + GOARCH=amd64 GOOS=windows $(GOBUILD) -o $(BINDIR)/$(NAME)-$@.exe + +gz_releases=$(addsuffix .gz, $(PLATFORM_LIST)) +zip_releases=$(addsuffix .zip, $(WINDOWS_ARCH_LIST)) + +$(gz_releases): %.gz : % + chmod +x $(BINDIR)/$(NAME)-$(basename $@) + gzip -f -S -$(VERSION).gz $(BINDIR)/$(NAME)-$(basename $@) + +$(zip_releases): %.zip : % + zip -m -j $(BINDIR)/$(NAME)-$(basename $@)-$(VERSION).zip $(BINDIR)/$(NAME)-$(basename $@).exe + +all-arch: $(PLATFORM_LIST) $(WINDOWS_ARCH_LIST) + +releases: $(gz_releases) $(zip_releases) + +sha256sum: + cd $(BINDIR); for file in *; do sha256sum $$file > $$file.sha256; done + +clean: + rm $(BINDIR)/* diff --git a/cmd/root.go b/cmd/root.go index 26d3de5c..7893b22e 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -1,91 +1,93 @@ -package cmd - -import ( - "bufio" - "fmt" - "log" - "os" - "strings" - - "github.com/fatih/color" - "github.com/spf13/cobra" - "golang.org/x/text/encoding/simplifiedchinese" - "golang.org/x/text/transform" - - "github.com/zu1k/nali/internal/constant" - "github.com/zu1k/nali/pkg/entity" -) - -var rootCmd = &cobra.Command{ - Use: "nali", - Short: "An offline tool for querying IP geographic information", - Long: `An offline tool for querying IP geographic information. - -Find document on: https://github.com/zu1k/nali - -#1 Query a simple IP address - - $ nali 1.2.3.4 - - or use pipe - - $ echo IP 6.6.6.6 | nali - -#2 Query multiple IP addresses - - $ nali 1.2.3.4 4.3.2.1 123.23.3.0 - -#3 Interactive query - - $ nali - 123.23.23.23 - 123.23.23.23 [越南 越南邮电集团公司] - quit - -#4 Use with dig - - $ dig nali.zu1k.com +short | nali - -#5 Use with nslookup - - $ nslookup nali.zu1k.com 8.8.8.8 | nali - -#6 Use with any other program - - bash abc.sh | nali - -#7 IPV6 support -`, - Version: constant.Version, - Args: cobra.MinimumNArgs(0), - Run: func(cmd *cobra.Command, args []string) { - gbk, _ := cmd.Flags().GetBool("gbk") - - if len(args) == 0 { - stdin := bufio.NewScanner(os.Stdin) - for stdin.Scan() { - line := stdin.Text() - if gbk { - line, _, _ = transform.String(simplifiedchinese.GBK.NewDecoder(), line) - } - if line == "quit" || line == "exit" { - return - } - _, _ = fmt.Fprintf(color.Output, "%s\n", entity.ParseLine(line).ColorString()) - } - } else { - _, _ = fmt.Fprintf(color.Output, "%s\n", entity.ParseLine(strings.Join(args, " ")).ColorString()) - } - }, -} - -// Execute parse subcommand and run -func Execute() { - if err := rootCmd.Execute(); err != nil { - log.Fatal(err.Error()) - } -} - -func init() { - rootCmd.Flags().Bool("gbk", false, "Use GBK decoder") -} +package cmd + +import ( + "bufio" + "fmt" + "log" + "os" + "strings" + + "github.com/fatih/color" + "github.com/spf13/cobra" + "golang.org/x/text/encoding/simplifiedchinese" + "golang.org/x/text/transform" + + "github.com/zu1k/nali/internal/constant" + "github.com/zu1k/nali/pkg/common" + "github.com/zu1k/nali/pkg/entity" +) + +var rootCmd = &cobra.Command{ + Use: "nali", + Short: "An offline tool for querying IP geographic information", + Long: `An offline tool for querying IP geographic information. + +Find document on: https://github.com/zu1k/nali + +#1 Query a simple IP address + + $ nali 1.2.3.4 + + or use pipe + + $ echo IP 6.6.6.6 | nali + +#2 Query multiple IP addresses + + $ nali 1.2.3.4 4.3.2.1 123.23.3.0 + +#3 Interactive query + + $ nali + 123.23.23.23 + 123.23.23.23 [越南 越南邮电集团公司] + quit + +#4 Use with dig + + $ dig nali.zu1k.com +short | nali + +#5 Use with nslookup + + $ nslookup nali.zu1k.com 8.8.8.8 | nali + +#6 Use with any other program + + bash abc.sh | nali + +#7 IPV6 support +`, + Version: constant.Version, + Args: cobra.MinimumNArgs(0), + Run: func(cmd *cobra.Command, args []string) { + gbk, _ := cmd.Flags().GetBool("gbk") + + if len(args) == 0 { + stdin := bufio.NewScanner(os.Stdin) + stdin.Split(common.ScanLines) + for stdin.Scan() { + line := stdin.Text() + if gbk { + line, _, _ = transform.String(simplifiedchinese.GBK.NewDecoder(), line) + } + if line := strings.TrimSpace(line); line == "quit" || line == "exit" { + return + } + _, _ = fmt.Fprintf(color.Output, "%s", entity.ParseLine(line).ColorString()) + } + } else { + _, _ = fmt.Fprintf(color.Output, "%s\n", entity.ParseLine(strings.Join(args, " ")).ColorString()) + } + }, +} + +// Execute parse subcommand and run +func Execute() { + if err := rootCmd.Execute(); err != nil { + log.Fatal(err.Error()) + } +} + +func init() { + rootCmd.Flags().Bool("gbk", false, "Use GBK decoder") +} diff --git a/cmd/update.go b/cmd/update.go index 4721234f..28cbd675 100644 --- a/cmd/update.go +++ b/cmd/update.go @@ -1,30 +1,30 @@ -package cmd - -import ( - "strings" - - "github.com/zu1k/nali/internal/db" - - "github.com/spf13/cobra" -) - -// updateCmd represents the update command -var updateCmd = &cobra.Command{ - Use: "update", - Short: "update qqwry, zxipv6wry, ip2region ip database and cdn", - Long: `update qqwry, zxipv6wry, ip2region ip database and cdn. Use commas to separate`, - Example: "nali update --db qqwry,cdn", - Run: func(cmd *cobra.Command, args []string) { - DBs, _ := cmd.Flags().GetString("db") - var DBNameArray []string - if DBs != "" { - DBNameArray = strings.Split(DBs, ",") - } - db.UpdateDB(DBNameArray...) - }, -} - -func init() { - rootCmd.AddCommand(updateCmd) - rootCmd.PersistentFlags().String("db", "", "choose db you want to update") -} +package cmd + +import ( + "strings" + + "github.com/zu1k/nali/internal/db" + + "github.com/spf13/cobra" +) + +// updateCmd represents the update command +var updateCmd = &cobra.Command{ + Use: "update", + Short: "update qqwry, zxipv6wry, ip2region ip database and cdn", + Long: `update qqwry, zxipv6wry, ip2region ip database and cdn. Use commas to separate`, + Example: "nali update --db qqwry,cdn", + Run: func(cmd *cobra.Command, args []string) { + DBs, _ := cmd.Flags().GetString("db") + var DBNameArray []string + if DBs != "" { + DBNameArray = strings.Split(DBs, ",") + } + db.UpdateDB(DBNameArray...) + }, +} + +func init() { + rootCmd.AddCommand(updateCmd) + rootCmd.PersistentFlags().String("db", "", "choose db you want to update") +} diff --git a/go.mod b/go.mod index bb06ee0a..f6320301 100644 --- a/go.mod +++ b/go.mod @@ -3,11 +3,13 @@ module github.com/zu1k/nali go 1.18 require ( + github.com/adrg/xdg v0.4.0 github.com/fatih/color v1.13.0 github.com/gin-gonic/gin v1.7.7 + github.com/google/martian v2.1.0+incompatible github.com/ip2location/ip2location-go/v9 v9.2.0 github.com/ipipdotnet/ipdb-go v1.3.1 - github.com/lionsoul2014/ip2region v2.2.0-release+incompatible + github.com/lionsoul2014/ip2region/binding/golang v0.0.0-20221017063954-c5c24655eb63 github.com/oschwald/geoip2-golang v1.7.0 github.com/saracen/go7z v0.0.0-20191010121135-9c09b6bd7fda github.com/spf13/cobra v1.4.0 diff --git a/go.sum b/go.sum index 2d62e672..d201a3f6 100644 --- a/go.sum +++ b/go.sum @@ -38,6 +38,8 @@ cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3f dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/adrg/xdg v0.4.0 h1:RzRqFcjH4nE5C6oTAxhBtoE2IRyjBSa62SCbyPidvls= +github.com/adrg/xdg v0.4.0/go.mod h1:N6ag73EX4wyxeaoeHctc1mas01KZgsj5tYiAIwqJE/E= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= @@ -118,6 +120,7 @@ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= @@ -165,8 +168,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= -github.com/lionsoul2014/ip2region v2.2.0-release+incompatible h1:1qp9iks+69h7IGLazAplzS9Ca14HAxuD5c0rbFdPGy4= -github.com/lionsoul2014/ip2region v2.2.0-release+incompatible/go.mod h1:+ZBN7PBoh5gG6/y0ZQ85vJDBe21WnfbRrQQwTfliJJI= +github.com/lionsoul2014/ip2region/binding/golang v0.0.0-20221017063954-c5c24655eb63 h1:29A0rIOPpaXCQNjb2eNhf/fAxCFxS0KpRbJVcPL63fQ= +github.com/lionsoul2014/ip2region/binding/golang v0.0.0-20221017063954-c5c24655eb63/go.mod h1:bChUKvbKVC3zL/lLLIcu6alhQaL8uWD/DA+jRdyggdI= github.com/magiconair/properties v1.8.6 h1:5ibWZ6iY0NctNGWo87LalDlEZ6R41TqbbDamhfG/Qzo= github.com/magiconair/properties v1.8.6/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= @@ -375,6 +378,7 @@ golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a h1:dGzPydgVsqGcTRVwiLJ1jVbufYwmzD3LfVPLKsKg+0k= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= diff --git a/internal/config/config.go b/internal/config/config.go index cdede7a7..09c0d1ae 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -1,41 +1,41 @@ -package config - -import ( - "log" - - "github.com/spf13/viper" - "github.com/zu1k/nali/internal/db" -) - -func ReadConfig(basePath string) { - viper.SetDefault("databases", db.GetDefaultDBList()) - viper.SetDefault("selected.ipv4", "qqwry") - viper.SetDefault("selected.ipv6", "zxipv6wry") - viper.SetDefault("selected.cdn", "cdn") - viper.SetDefault("selected.lang", "zh-CN") - - viper.SetConfigName("config") - viper.SetConfigType("yaml") - viper.AddConfigPath(basePath) - err := viper.ReadInConfig() - if err != nil { - err = viper.SafeWriteConfig() - if err != nil { - panic(err) - } - } - - _ = viper.BindEnv("selected.ipv4", "NALI_DB_IP4") - _ = viper.BindEnv("selected.ipv6", "NALI_DB_IP6") - _ = viper.BindEnv("selected.cdn", "NALI_DB_CDN") - _ = viper.BindEnv("selected.lang", "NALI_LANG") - - dbList := db.List{} - err = viper.UnmarshalKey("databases", &dbList) - if err != nil { - log.Fatalln("Config invalid:", err) - } - - db.NameDBMap.From(dbList) - db.TypeDBMap.From(dbList) -} +package config + +import ( + "log" + + "github.com/spf13/viper" + "github.com/zu1k/nali/internal/db" +) + +func ReadConfig(basePath string) { + viper.SetDefault("databases", db.GetDefaultDBList()) + viper.SetDefault("selected.ipv4", "qqwry") + viper.SetDefault("selected.ipv6", "zxipv6wry") + viper.SetDefault("selected.cdn", "cdn") + viper.SetDefault("selected.lang", "zh-CN") + + viper.SetConfigName("config") + viper.SetConfigType("yaml") + viper.AddConfigPath(basePath) + err := viper.ReadInConfig() + if err != nil { + err = viper.SafeWriteConfig() + if err != nil { + panic(err) + } + } + + _ = viper.BindEnv("selected.ipv4", "NALI_DB_IP4") + _ = viper.BindEnv("selected.ipv6", "NALI_DB_IP6") + _ = viper.BindEnv("selected.cdn", "NALI_DB_CDN") + _ = viper.BindEnv("selected.lang", "NALI_LANG") + + dbList := db.List{} + err = viper.UnmarshalKey("databases", &dbList) + if err != nil { + log.Fatalln("Config invalid:", err) + } + + db.NameDBMap.From(dbList) + db.TypeDBMap.From(dbList) +} diff --git a/internal/constant/path.go b/internal/constant/path.go index 65c537f4..bc8eca60 100644 --- a/internal/constant/path.go +++ b/internal/constant/path.go @@ -1,33 +1,49 @@ -package constant - -import ( - "log" - "os" - "path/filepath" -) - -var ( - // WorkDirPath database home path - WorkDirPath string -) - -func init() { - WorkDirPath = os.Getenv("NALI_HOME") - if WorkDirPath == "" { - WorkDirPath = os.Getenv("NALI_DB_HOME") - } - if WorkDirPath == "" { - homeDir, err := os.UserHomeDir() - if err != nil { - panic(err) - } - WorkDirPath = filepath.Join(homeDir, ".nali") - } - if _, err := os.Stat(WorkDirPath); os.IsNotExist(err) { - if err := os.MkdirAll(WorkDirPath, 0777); err != nil { - log.Fatal("can not create", WorkDirPath, ", use bin dir instead") - } - } - - _ = os.Chdir(WorkDirPath) -} +package constant + +import ( + "log" + "os" + "path/filepath" + + "github.com/adrg/xdg" +) + +var ( + ConfigDirPath string + DataDirPath string +) + +func init() { + if naliHome := os.Getenv("NALI_HOME"); len(naliHome) != 0 { + ConfigDirPath = naliHome + DataDirPath = naliHome + } else { + ConfigDirPath = os.Getenv("NALI_CONFIG_HOME") + if len(ConfigDirPath) == 0 { + ConfigDirPath = filepath.Join(xdg.ConfigHome, "nali") + } + + DataDirPath = os.Getenv("NALI_DB_HOME") + if len(DataDirPath) == 0 { + DataDirPath = filepath.Join(xdg.DataHome, "nali") + } + } + + prepareDir(ConfigDirPath) + prepareDir(DataDirPath) + + _ = os.Chdir(DataDirPath) +} + +func prepareDir(dir string) { + stat, err := os.Stat(dir) + if err != nil && os.IsNotExist(err) { + if err := os.MkdirAll(dir, 0755); err != nil { + log.Fatal("can not create config dir:", dir) + } + } else { + if !stat.IsDir() { + log.Fatal("path already exists, but not a dir:", dir) + } + } +} diff --git a/internal/constant/version.go b/internal/constant/version.go index 19c52cb9..7af1c7ee 100644 --- a/internal/constant/version.go +++ b/internal/constant/version.go @@ -1,8 +1,6 @@ -package constant - -var ( - // Version like 1.0.1 - Version = "unknown version" - // BuildTime like 2020-01-01 - BuildTime = "unknown time" -) +package constant + +var ( + // Version like 1.0.1 + Version = "unknown version" +) diff --git a/internal/db/cache.go b/internal/db/cache.go index b53dee9c..bf17dc3d 100644 --- a/internal/db/cache.go +++ b/internal/db/cache.go @@ -1,14 +1,18 @@ -package db - -import "github.com/zu1k/nali/pkg/dbif" - -var ( - dbNameCache = make(map[string]dbif.DB) - dbTypeCache = make(map[dbif.QueryType]dbif.DB) - queryCache = make(map[string]string) -) - -var ( - NameDBMap = make(NameMap) - TypeDBMap = make(TypeMap) -) +package db + +import ( + "sync" + + "github.com/zu1k/nali/pkg/dbif" +) + +var ( + dbNameCache = make(map[string]dbif.DB) + dbTypeCache = make(map[dbif.QueryType]dbif.DB) + queryCache = sync.Map{} +) + +var ( + NameDBMap = make(NameMap) + TypeDBMap = make(TypeMap) +) diff --git a/internal/db/db.go b/internal/db/db.go index ae2c14da..6d75a6f2 100644 --- a/internal/db/db.go +++ b/internal/db/db.go @@ -1,83 +1,83 @@ -package db - -import ( - "log" - "strings" - - "github.com/spf13/viper" - - "github.com/zu1k/nali/pkg/cdn" - "github.com/zu1k/nali/pkg/dbif" - "github.com/zu1k/nali/pkg/geoip" - "github.com/zu1k/nali/pkg/qqwry" - "github.com/zu1k/nali/pkg/zxipv6wry" -) - -func GetDB(typ dbif.QueryType) (db dbif.DB) { - if db, found := dbTypeCache[typ]; found { - return db - } - - lang := viper.GetString("selected.lang") - if lang == "" { - lang = "zh-CN" - } - - var err error - switch typ { - case dbif.TypeIPv4: - selected := viper.GetString("selected.ipv4") - if selected != "" { - db = getDbByName(selected).get() - break - } - - if lang == "zh-CN" { - db, err = qqwry.NewQQwry(getDbByName("qqwry").File) - } else { - db, err = geoip.NewGeoIP(getDbByName("geoip").File) - } - case dbif.TypeIPv6: - selected := viper.GetString("selected.ipv6") - if selected != "" { - db = getDbByName(selected).get() - break - } - - if lang == "zh-CN" { - db, err = zxipv6wry.NewZXwry(getDbByName("zxipv6wry").File) - } else { - db, err = geoip.NewGeoIP(getDbByName("geoip").File) - } - case dbif.TypeDomain: - selected := viper.GetString("selected.cdn") - if selected != "" { - db = getDbByName(selected).get() - break - } - - db, err = cdn.NewCDN(getDbByName("cdn").File) - default: - panic("Query type not supported!") - } - - if err != nil || db == nil { - log.Fatalln("Database init failed:", err) - } - - dbTypeCache[typ] = db - return -} - -func Find(typ dbif.QueryType, query string) string { - if result, found := queryCache[query]; found { - return result - } - result, err := GetDB(typ).Find(query) - if err != nil { - return "" - } - r := strings.Trim(result.String(), " ") - queryCache[query] = r - return r -} +package db + +import ( + "log" + "strings" + + "github.com/spf13/viper" + + "github.com/zu1k/nali/pkg/cdn" + "github.com/zu1k/nali/pkg/dbif" + "github.com/zu1k/nali/pkg/geoip" + "github.com/zu1k/nali/pkg/qqwry" + "github.com/zu1k/nali/pkg/zxipv6wry" +) + +func GetDB(typ dbif.QueryType) (db dbif.DB) { + if db, found := dbTypeCache[typ]; found { + return db + } + + lang := viper.GetString("selected.lang") + if lang == "" { + lang = "zh-CN" + } + + var err error + switch typ { + case dbif.TypeIPv4: + selected := viper.GetString("selected.ipv4") + if selected != "" { + db = getDbByName(selected).get() + break + } + + if lang == "zh-CN" { + db, err = qqwry.NewQQwry(getDbByName("qqwry").File) + } else { + db, err = geoip.NewGeoIP(getDbByName("geoip").File) + } + case dbif.TypeIPv6: + selected := viper.GetString("selected.ipv6") + if selected != "" { + db = getDbByName(selected).get() + break + } + + if lang == "zh-CN" { + db, err = zxipv6wry.NewZXwry(getDbByName("zxipv6wry").File) + } else { + db, err = geoip.NewGeoIP(getDbByName("geoip").File) + } + case dbif.TypeDomain: + selected := viper.GetString("selected.cdn") + if selected != "" { + db = getDbByName(selected).get() + break + } + + db, err = cdn.NewCDN(getDbByName("cdn").File) + default: + panic("Query type not supported!") + } + + if err != nil || db == nil { + log.Fatalln("Database init failed:", err) + } + + dbTypeCache[typ] = db + return +} + +func Find(typ dbif.QueryType, query string) string { + if result, found := queryCache.Load(query); found { + return result.(string) + } + result, err := GetDB(typ).Find(query) + if err != nil { + return "" + } + r := strings.Trim(result.String(), " ") + queryCache.Store(query, r) + return r +} diff --git a/internal/db/default.go b/internal/db/default.go index 15cb5f4e..cb259a61 100644 --- a/internal/db/default.go +++ b/internal/db/default.go @@ -1,88 +1,90 @@ -package db - -import ( - "github.com/zu1k/nali/pkg/cdn" - "github.com/zu1k/nali/pkg/ip2region" -) - -func GetDefaultDBList() List { - return List{ - &DB{ - Name: "qqwry", - NameAlias: []string{ - "chunzhen", - }, - Format: FormatQQWry, - File: "qqwry.dat", - Languages: LanguagesZH, - Types: TypesIPv4, - }, - &DB{ - Name: "zxipv6wry", - NameAlias: []string{ - "zxipv6", - "zx", - }, - Format: FormatZXIPv6Wry, - File: "zxipv6wry.db", - Languages: LanguagesZH, - Types: TypesIPv6, - }, - &DB{ - Name: "geoip", - NameAlias: []string{ - "geoip2", - "geolite", - "geolite2", - }, - Format: FormatMMDB, - File: "GeoLite2-City.mmdb", - Languages: LanguagesAll, - Types: TypesIP, - }, - &DB{ - Name: "dbip", - NameAlias: []string{ - "db-ip", - }, - Format: FormatMMDB, - File: "dbip.mmdb", - Languages: LanguagesAll, - Types: TypesIP, - }, - &DB{ - Name: "ipip", - Format: FormatIPIP, - File: "ipipfree.ipdb", - Languages: LanguagesZH, - Types: TypesIP, - }, - &DB{ - Name: "ip2region", - NameAlias: []string{ - "i2r", - }, - Format: FormatIP2Region, - File: "ip2region.db", - Languages: LanguagesZH, - Types: TypesIPv4, - DownloadUrls: ip2region.DownloadUrls, - }, - &DB{ - Name: "ip2location", - Format: FormatIP2Location, - File: "IP2LOCATION-LITE-DB3.IPV6.BIN", - Languages: LanguagesEN, - Types: TypesIP, - }, - - &DB{ - Name: "cdn", - Format: FormatCDNSkkYml, - File: "cdn.yml", - Languages: LanguagesZH, - Types: TypesCDN, - DownloadUrls: cdn.DownloadUrls, - }, - } -} +package db + +import ( + "github.com/zu1k/nali/pkg/cdn" + "github.com/zu1k/nali/pkg/ip2region" + "github.com/zu1k/nali/pkg/qqwry" +) + +func GetDefaultDBList() List { + return List{ + &DB{ + Name: "qqwry", + NameAlias: []string{ + "chunzhen", + }, + Format: FormatQQWry, + File: "qqwry.dat", + Languages: LanguagesZH, + Types: TypesIPv4, + DownloadUrls: qqwry.DownloadUrls, + }, + &DB{ + Name: "zxipv6wry", + NameAlias: []string{ + "zxipv6", + "zx", + }, + Format: FormatZXIPv6Wry, + File: "zxipv6wry.db", + Languages: LanguagesZH, + Types: TypesIPv6, + }, + &DB{ + Name: "geoip", + NameAlias: []string{ + "geoip2", + "geolite", + "geolite2", + }, + Format: FormatMMDB, + File: "GeoLite2-City.mmdb", + Languages: LanguagesAll, + Types: TypesIP, + }, + &DB{ + Name: "dbip", + NameAlias: []string{ + "db-ip", + }, + Format: FormatMMDB, + File: "dbip.mmdb", + Languages: LanguagesAll, + Types: TypesIP, + }, + &DB{ + Name: "ipip", + Format: FormatIPIP, + File: "ipipfree.ipdb", + Languages: LanguagesZH, + Types: TypesIP, + }, + &DB{ + Name: "ip2region", + NameAlias: []string{ + "i2r", + }, + Format: FormatIP2Region, + File: "ip2region.xdb", + Languages: LanguagesZH, + Types: TypesIPv4, + DownloadUrls: ip2region.DownloadUrls, + }, + &DB{ + Name: "ip2location", + Format: FormatIP2Location, + File: "IP2LOCATION-LITE-DB3.IPV6.BIN", + Languages: LanguagesEN, + Types: TypesIP, + }, + + &DB{ + Name: "cdn", + Format: FormatCDNYml, + File: "cdn.yml", + Languages: LanguagesZH, + Types: TypesCDN, + DownloadUrls: cdn.DownloadUrls, + }, + } +} diff --git a/internal/db/type.go b/internal/db/type.go index ffdcf02d..5db927bf 100644 --- a/internal/db/type.go +++ b/internal/db/type.go @@ -1,141 +1,141 @@ -package db - -import ( - "log" - - "github.com/zu1k/nali/pkg/cdn" - "github.com/zu1k/nali/pkg/dbif" - "github.com/zu1k/nali/pkg/geoip" - "github.com/zu1k/nali/pkg/ip2location" - "github.com/zu1k/nali/pkg/ip2region" - "github.com/zu1k/nali/pkg/ipip" - "github.com/zu1k/nali/pkg/qqwry" - "github.com/zu1k/nali/pkg/zxipv6wry" -) - -type DB struct { - Name string - NameAlias []string `yaml:"name-alias,omitempty" mapstructure:"name-alias"` - Format Format - File string - - Languages []string - Types []Type - - DownloadUrls []string `yaml:"download-urls,omitempty" mapstructure:"download-urls"` -} - -func (d *DB) get() (db dbif.DB) { - if db, found := dbNameCache[d.Name]; found { - return db - } - - filePath := d.File - - var err error - switch d.Format { - case FormatQQWry: - db, err = qqwry.NewQQwry(filePath) - case FormatZXIPv6Wry: - db, err = zxipv6wry.NewZXwry(filePath) - case FormatIPIP: - db, err = ipip.NewIPIP(filePath) - case FormatMMDB: - db, err = geoip.NewGeoIP(filePath) - case FormatIP2Region: - db, err = ip2region.NewIp2Region(filePath) - case FormatIP2Location: - db, err = ip2location.NewIP2Location(filePath) - case FormatCDNSkkYml: - db, err = cdn.NewCDN(filePath) - default: - panic("DB format not supported!") - } - - if err != nil || db == nil { - log.Fatalln("Database init failed:", err) - } - - dbNameCache[d.Name] = db - return -} - -type Format string - -const ( - FormatMMDB Format = "mmdb" - FormatQQWry = "qqwry" - FormatZXIPv6Wry = "zxipv6wry" - FormatIPIP = "ipip" - FormatIP2Region = "ip2region" - FormatIP2Location = "ip2location" - - FormatCDNSkkYml = "cdn-skk-yml" -) - -var ( - LanguagesAll = []string{"ALL"} - LanguagesZH = []string{"zh-CN"} - LanguagesEN = []string{"en"} -) - -type Type string - -const ( - TypeIPv4 Type = "IPv4" - TypeIPv6 = "IPv6" - TypeCDN = "CDN" -) - -var ( - TypesAll = []Type{TypeIPv4, TypeIPv6, TypeCDN} - TypesIP = []Type{TypeIPv4, TypeIPv6} - TypesIPv4 = []Type{TypeIPv4} - TypesIPv6 = []Type{TypeIPv6} - TypesCDN = []Type{TypeCDN} -) - -type List []*DB -type NameMap map[string]*DB -type TypeMap map[Type][]*DB - -func (m *NameMap) From(dbs List) { - for _, db := range dbs { - (*m)[db.Name] = db - - if alias := db.NameAlias; alias != nil { - for _, aName := range alias { - (*m)[aName] = db - } - } - } -} - -func (m *TypeMap) From(dbs List) { - for _, db := range dbs { - for _, typ := range db.Types { - dbsInType := (*m)[typ] - if dbsInType == nil { - dbsInType = []*DB{db} - } else { - dbsInType = append(dbsInType, db) - } - (*m)[typ] = dbsInType - } - } -} - -func getDbByName(name string) (db *DB) { - if dbInfo, found := NameDBMap[name]; found { - return dbInfo - } - - defaultNameDBMap := NameMap{} - defaultNameDBMap.From(GetDefaultDBList()) - if dbInfo, found := defaultNameDBMap[name]; found { - return dbInfo - } - - log.Fatalf("DB with name %s not found!\n", name) - return -} +package db + +import ( + "log" + + "github.com/zu1k/nali/pkg/cdn" + "github.com/zu1k/nali/pkg/dbif" + "github.com/zu1k/nali/pkg/geoip" + "github.com/zu1k/nali/pkg/ip2location" + "github.com/zu1k/nali/pkg/ip2region" + "github.com/zu1k/nali/pkg/ipip" + "github.com/zu1k/nali/pkg/qqwry" + "github.com/zu1k/nali/pkg/zxipv6wry" +) + +type DB struct { + Name string + NameAlias []string `yaml:"name-alias,omitempty" mapstructure:"name-alias"` + Format Format + File string + + Languages []string + Types []Type + + DownloadUrls []string `yaml:"download-urls,omitempty" mapstructure:"download-urls"` +} + +func (d *DB) get() (db dbif.DB) { + if db, found := dbNameCache[d.Name]; found { + return db + } + + filePath := d.File + + var err error + switch d.Format { + case FormatQQWry: + db, err = qqwry.NewQQwry(filePath) + case FormatZXIPv6Wry: + db, err = zxipv6wry.NewZXwry(filePath) + case FormatIPIP: + db, err = ipip.NewIPIP(filePath) + case FormatMMDB: + db, err = geoip.NewGeoIP(filePath) + case FormatIP2Region: + db, err = ip2region.NewIp2Region(filePath) + case FormatIP2Location: + db, err = ip2location.NewIP2Location(filePath) + case FormatCDNYml: + db, err = cdn.NewCDN(filePath) + default: + panic("DB format not supported!") + } + + if err != nil || db == nil { + log.Fatalln("Database init failed:", err) + } + + dbNameCache[d.Name] = db + return +} + +type Format string + +const ( + FormatMMDB Format = "mmdb" + FormatQQWry = "qqwry" + FormatZXIPv6Wry = "zxipv6wry" + FormatIPIP = "ipip" + FormatIP2Region = "ip2region" + FormatIP2Location = "ip2location" + + FormatCDNYml = "cdn-yml" +) + +var ( + LanguagesAll = []string{"ALL"} + LanguagesZH = []string{"zh-CN"} + LanguagesEN = []string{"en"} +) + +type Type string + +const ( + TypeIPv4 Type = "IPv4" + TypeIPv6 = "IPv6" + TypeCDN = "CDN" +) + +var ( + TypesAll = []Type{TypeIPv4, TypeIPv6, TypeCDN} + TypesIP = []Type{TypeIPv4, TypeIPv6} + TypesIPv4 = []Type{TypeIPv4} + TypesIPv6 = []Type{TypeIPv6} + TypesCDN = []Type{TypeCDN} +) + +type List []*DB +type NameMap map[string]*DB +type TypeMap map[Type][]*DB + +func (m *NameMap) From(dbs List) { + for _, db := range dbs { + (*m)[db.Name] = db + + if alias := db.NameAlias; alias != nil { + for _, aName := range alias { + (*m)[aName] = db + } + } + } +} + +func (m *TypeMap) From(dbs List) { + for _, db := range dbs { + for _, typ := range db.Types { + dbsInType := (*m)[typ] + if dbsInType == nil { + dbsInType = []*DB{db} + } else { + dbsInType = append(dbsInType, db) + } + (*m)[typ] = dbsInType + } + } +} + +func getDbByName(name string) (db *DB) { + if dbInfo, found := NameDBMap[name]; found { + return dbInfo + } + + defaultNameDBMap := NameMap{} + defaultNameDBMap.From(GetDefaultDBList()) + if dbInfo, found := defaultNameDBMap[name]; found { + return dbInfo + } + + log.Fatalf("DB with name %s not found!\n", name) + return +} diff --git a/internal/db/update.go b/internal/db/update.go index a5b2548c..d7320c27 100644 --- a/internal/db/update.go +++ b/internal/db/update.go @@ -1,109 +1,79 @@ -package db - -import ( - "log" - "strings" - "time" - - "github.com/zu1k/nali/pkg/cdn" - "github.com/zu1k/nali/pkg/download" - "github.com/zu1k/nali/pkg/ip2region" - "github.com/zu1k/nali/pkg/qqwry" - "github.com/zu1k/nali/pkg/zxipv6wry" -) - -func UpdateDB(dbNames ...string) { - if len(dbNames) == 0 { - dbNames = DbNameListForUpdate - } - - done := make(map[string]struct{}) - for _, dbName := range dbNames { - update, name := getUpdateFuncByName(dbName) - if _, found := done[name]; !found { - done[name] = struct{}{} - if err := update(); err != nil { - continue - } - } - } -} - -var DbNameListForUpdate = []string{ - "qqwry", - "zxipv6wry", - "ip2region", - "cdn", -} - -func getUpdateFuncByName(name string) (func() error, string) { - name = strings.TrimSpace(name) - if db := getDbByName(name); db != nil { - // direct download if download-url not null - if len(db.DownloadUrls) > 0 { - return func() error { - log.Printf("正在下载最新 %s 数据库...\n", db.Name) - _, err := download.Download(db.File, db.DownloadUrls...) - if err != nil { - log.Printf("%s 数据库下载失败: %s\n", db.Name, db.File) - log.Println("error:", err) - return err - } else { - log.Printf("%s 数据库下载成功: %s\n", db.Name, db.File) - return nil - } - }, string(db.Format) - } - - // intenel download func - switch db.Format { - case FormatQQWry: - return func() error { - log.Println("正在下载最新 纯真 IPv4数据库...") - _, err := qqwry.Download(getDbByName("qqwry").File) - if err != nil { - log.Println("数据库 QQWry 下载失败:", err) - } - return err - }, FormatQQWry - case FormatZXIPv6Wry: - return func() error { - log.Println("正在下载最新 ZX IPv6数据库...") - _, err := zxipv6wry.Download(getDbByName("zxipv6wry").File) - if err != nil { - log.Println("数据库 ZXIPv6Wry 下载失败:", err) - } - return err - }, FormatZXIPv6Wry - case FormatIP2Region: - return func() error { - log.Println("正在下载最新 Ip2Region 数据库...") - _, err := ip2region.Download(getDbByName("ip2region").File) - if err != nil { - log.Println("数据库 Ip2Region 下载失败:", err) - } - return err - }, FormatZXIPv6Wry - case FormatCDNSkkYml: - return func() error { - log.Println("正在下载最新 CDN服务提供商数据库...") - _, err := cdn.Download(getDbByName("cdn").File) - if err != nil { - log.Println("数据库 CDN 下载失败:", err) - } - return err - }, FormatZXIPv6Wry - default: - return func() error { - log.Println("暂不支持该类型数据库的自动更新") - log.Println("可通过指定数据库的 download-urls 从特定链接下载数据库文件") - return nil - }, time.Now().String() - } - } else { - return func() error { - log.Fatalln("该名称的数据库未找到：", name) - return nil - }, time.Now().String() - } -} +package db + +import ( + "log" + "strings" + "time" + + "github.com/zu1k/nali/pkg/download" + "github.com/zu1k/nali/pkg/zxipv6wry" +) + +func UpdateDB(dbNames ...string) { + if len(dbNames) == 0 { + dbNames = DbNameListForUpdate + } + + done := make(map[string]struct{}) + for _, dbName := range dbNames { + update, name := getUpdateFuncByName(dbName) + if _, found := done[name]; !found { + done[name] = struct{}{} + if err := update(); err != nil { + continue + } + } + } +} + +var DbNameListForUpdate = []string{ + "qqwry", + "zxipv6wry", + "ip2region", + "cdn", +} + +func getUpdateFuncByName(name string) (func() error, string) { + name = strings.TrimSpace(name) + if db := getDbByName(name); db != nil { + // direct download if download-url not null + if len(db.DownloadUrls) > 0 { + return func() error { + log.Printf("正在下载最新 %s 数据库...\n", db.Name) + _, err := download.Download(db.File, db.DownloadUrls...) + if err != nil { + log.Printf("%s 数据库下载失败: %s\n", db.Name, db.File) + log.Println("error:", err) + return err + } else { + log.Printf("%s 数据库下载成功: %s\n", db.Name, db.File) + return nil + } + }, string(db.Format) + } + + // intenel download func + switch db.Format { + case FormatZXIPv6Wry: + return func() error { + log.Println("正在下载最新 ZX IPv6数据库...") + _, err := zxipv6wry.Download(getDbByName("zxipv6wry").File) + if err != nil { + log.Println("数据库 ZXIPv6Wry 下载失败:", err) + } + return err + }, FormatZXIPv6Wry + default: + return func() error { + log.Println("暂不支持该类型数据库的自动更新") + log.Println("可通过指定数据库的 download-urls 从特定链接下载数据库文件") + return nil + }, time.Now().String() + } + } else { + return func() error { + log.Fatalln("该名称的数据库未找到：", name) + return nil + }, time.Now().String() + } +} diff --git a/internal/migration/mirgration.go b/internal/migration/mirgration.go new file mode 100644 index 00000000..2d4b3c64 --- /dev/null +++ b/internal/migration/mirgration.go @@ -0,0 +1,6 @@ +package migration + +func init() { + migration2v4() + migration2v6() +} diff --git a/internal/migration/v4.go b/internal/migration/v4.go new file mode 100644 index 00000000..fd1fc822 --- /dev/null +++ b/internal/migration/v4.go @@ -0,0 +1,51 @@ +package migration + +import ( + "log" + + "github.com/spf13/viper" + "github.com/zu1k/nali/internal/constant" + "github.com/zu1k/nali/internal/db" + "github.com/zu1k/nali/pkg/cdn" + "github.com/zu1k/nali/pkg/ip2region" +) + +func migration2v4() { + viper.SetConfigName("config") + viper.SetConfigType("yaml") + viper.AddConfigPath(constant.ConfigDirPath) + + err := viper.ReadInConfig() + if err != nil { + return + } + + dbList := db.List{} + err = viper.UnmarshalKey("databases", &dbList) + if err != nil { + log.Fatalln("Config invalid:", err) + } + + needOverwrite := false + for _, adb := range dbList { + if adb.Name == "ip2region" && adb.File != "ip2region.xdb" { + needOverwrite = true + adb.File = "ip2region.xdb" + adb.DownloadUrls = ip2region.DownloadUrls + } + + if adb.Name == "cdn" && adb.Format != "cdn-yml" { + needOverwrite = true + adb.Format = "cdn-yml" + adb.DownloadUrls = cdn.DownloadUrls + } + } + + if needOverwrite { + viper.Set("databases", dbList) + err = viper.WriteConfig() + if err != nil { + log.Println(err) + } + } +} diff --git a/internal/migration/v6.go b/internal/migration/v6.go new file mode 100644 index 00000000..4fa05106 --- /dev/null +++ b/internal/migration/v6.go @@ -0,0 +1,43 @@ +package migration + +import ( + "os" + "path/filepath" + + "github.com/google/martian/log" + "github.com/zu1k/nali/internal/constant" +) + +func migration2v6() { + homeDir, err := os.UserHomeDir() + if err != nil { + return + } + oldDefaultWorkPath := filepath.Join(homeDir, ".nali") + _, err = os.Stat(oldDefaultWorkPath) + if err == nil { + println("Old data directories are detected and will attempt to migrate automatically") + + oldDefaultConfigPath := filepath.Join(oldDefaultWorkPath, "config.yaml") + stat, err := os.Stat(oldDefaultConfigPath) + if err == nil { + if stat.Mode().IsRegular() { + _ = os.Rename(oldDefaultConfigPath, filepath.Join(constant.ConfigDirPath, "config.yaml")) + } + } + + files, err := os.ReadDir(oldDefaultWorkPath) + if err == nil { + for _, file := range files { + if file.Type().IsRegular() { + _ = os.Rename(filepath.Join(oldDefaultWorkPath, file.Name()), filepath.Join(constant.DataDirPath, file.Name())) + } + } + } + + err = os.RemoveAll(oldDefaultWorkPath) + if err != nil { + log.Errorf("Auto migration failed: %s\n", err) + } + } +} diff --git a/main.go b/main.go index 9297af61..9e0e6495 100644 --- a/main.go +++ b/main.go @@ -1,12 +1,13 @@ package main import ( + "github.com/zu1k/nali/internal/constant" "github.com/zu1k/nali/cmd" "github.com/zu1k/nali/internal/config" - "github.com/zu1k/nali/internal/constant" + _ "github.com/zu1k/nali/internal/migration" ) func main() { - config.ReadConfig(constant.WorkDirPath) + config.ReadConfig(constant.ConfigDirPath) cmd.Execute() } diff --git a/pkg/cdn/cdn.go b/pkg/cdn/cdn.go index 7b2bcfad..5db5590a 100644 --- a/pkg/cdn/cdn.go +++ b/pkg/cdn/cdn.go @@ -1,93 +1,119 @@ -package cdn - -import ( - "errors" - "fmt" - "io/ioutil" - "log" - "os" - "strings" - - "gopkg.in/yaml.v2" -) - -type CDN struct { - Data CDNDist -} - -type CDNDist map[string]CDNResult - -type CDNResult struct { - Name string `yaml:"name"` - Link string `yaml:"link"` -} - -func (r CDNResult) String() string { - return r.Name -} - -func NewCDN(filePath string) (*CDN, error) { - cdnDist := make(CDNDist) - cdnData := make([]byte, 0) - - _, err := os.Stat(filePath) - if err != nil && os.IsNotExist(err) { - log.Println("文件不存在，尝试从网络获取最新CDN数据库") - cdnData, err = Download(filePath) - if err != nil { - return nil, err - } - } else { - cdnFile, err := os.OpenFile(filePath, os.O_RDONLY, 0400) - if err != nil { - return nil, err - } - defer cdnFile.Close() - - cdnData, err = ioutil.ReadAll(cdnFile) - if err != nil { - return nil, err - } - } - - err = yaml.Unmarshal(cdnData, &cdnDist) - if err != nil { - return nil, err - } - return &CDN{Data: cdnDist}, nil -} - -func (db CDN) Find(query string, params ...string) (result fmt.Stringer, err error) { - baseCname := parseBaseCname(query) - for _, domain := range baseCname { - if domain != "" { - cdnResult, found := db.Data[domain] - if found { - return cdnResult, nil - } - } - - if strings.Contains(domain, "kunlun") { - return CDNResult{ - Name: "阿里云 CDN", - }, nil - } - } - - return nil, errors.New("not found") -} - -func parseBaseCname(domain string) (result []string) { - parts := strings.Split(domain, ".") - size := len(parts) - if size == 0 { - return []string{} - } - domain = parts[size-1] - result = append(result, domain) - for i := len(parts) - 2; i >= 0; i-- { - domain = parts[i] + "." + domain - result = append(result, domain) - } - return result -} +package cdn + +import ( + "errors" + "fmt" + "io/ioutil" + "log" + "os" + "regexp" + "strings" + + "github.com/zu1k/nali/pkg/download" + "github.com/zu1k/nali/pkg/re" + "gopkg.in/yaml.v2" +) + +var DownloadUrls = []string{ + "https://cdn.jsdelivr.net/gh/4ft35t/cdn/src/cdn.yml", + "https://raw.githubusercontent.com/4ft35t/cdn/master/src/cdn.yml", + "https://raw.githubusercontent.com/SukkaLab/cdn/master/src/cdn.yml", +} + +type CDN struct { + Map map[string]CDNResult + ReMap []CDNReTuple +} + +type CDNReTuple struct { + *regexp.Regexp + CDNResult +} + +type CDNResult struct { + Name string `yaml:"name"` + Link string `yaml:"link"` +} + +func (r CDNResult) String() string { + return r.Name +} + +func NewCDN(filePath string) (*CDN, error) { + fileData := make([]byte, 0) + _, err := os.Stat(filePath) + if err != nil && os.IsNotExist(err) { + log.Println("文件不存在，尝试从网络获取最新CDN数据库") + fileData, err = download.Download(filePath, DownloadUrls...) + if err != nil { + return nil, err + } + } else { + cdnFile, err := os.OpenFile(filePath, os.O_RDONLY, 0400) + if err != nil { + return nil, err + } + defer cdnFile.Close() + + fileData, err = ioutil.ReadAll(cdnFile) + if err != nil { + return nil, err + } + } + + cdnMap := make(map[string]CDNResult) + err = yaml.Unmarshal(fileData, &cdnMap) + if err != nil { + return nil, err + } + cdnReMap := make([]CDNReTuple, 0) + for k, v := range cdnMap { + if re.MaybeRegexp(k) { + rex, err := regexp.Compile(k) + if err != nil { + log.Printf("[CDN Database] entry %s not a valid regexp", k) + } + cdnReMap = append(cdnReMap, CDNReTuple{ + Regexp: rex, + CDNResult: v, + }) + } + } + + return &CDN{Map: cdnMap, ReMap: cdnReMap}, nil +} + +func (db CDN) Find(query string, params ...string) (result fmt.Stringer, err error) { + baseCname := parseBaseCname(query) + for _, domain := range baseCname { + if domain != "" { + cdnResult, found := db.Map[domain] + if found { + return cdnResult, nil + } + } + + for _, entry := range db.ReMap { + if entry.Regexp.MatchString(domain) { + return entry.CDNResult, nil + } + } + } + + return nil, errors.New("not found") +} + +func parseBaseCname(domain string) (result []string) { + parts := strings.Split(domain, ".") + size := len(parts) + if size == 0 { + return []string{} + } + domain = parts[size-1] + result = append(result, domain) + for i := len(parts) - 2; i >= 0; i-- { + domain = parts[i] + "." + domain + result = append(result, domain) + } + return result +} diff --git a/pkg/cdn/update.go b/pkg/cdn/update.go deleted file mode 100644 index 42be0ea0..00000000 --- a/pkg/cdn/update.go +++ /dev/null @@ -1,31 +0,0 @@ -package cdn - -import ( - "log" - - "github.com/zu1k/nali/pkg/common" -) - -var DownloadUrls = []string{ - "https://cdn.jsdelivr.net/gh/SukkaLab/cdn/src/cdn.yml", - "https://raw.githubusercontent.com/SukkaLab/cdn/master/src/cdn.yml", - "https://cdn.jsdelivr.net/gh/4ft35t/cdn/src/cdn.yml", - "https://raw.githubusercontent.com/4ft35t/cdn/master/src/cdn.yml", -} - -// Deprecated: This will be removed from 0.5.0, use package download instead -func Download(filePath ...string) (data []byte, err error) { - data, err = common.GetHttpClient().Get(DownloadUrls...) - if err != nil { - log.Printf("CDN数据库下载失败，请手动下载解压后保存到本地: %s \n", filePath) - log.Println("下载链接：", DownloadUrls) - return - } - - if len(filePath) == 1 { - if err := common.SaveFile(filePath[0], data); err == nil { - log.Printf("已将最新的 CDN数据库 保存到本地: %s \n", filePath) - } - } - return -} diff --git a/pkg/common/const.go b/pkg/common/const.go deleted file mode 100644 index c1fb9502..00000000 --- a/pkg/common/const.go +++ /dev/null @@ -1,8 +0,0 @@ -package common - -const ( - // RedirectMode1 [IP][0x01][国家和地区信息的绝对偏移地址] - RedirectMode1 = 0x01 - // RedirectMode2 [IP][0x02][信息的绝对偏移][...] or [IP][国家][...] - RedirectMode2 = 0x02 -) diff --git a/pkg/common/dbtool.go b/pkg/common/dbtool.go deleted file mode 100644 index 69fe3bfc..00000000 --- a/pkg/common/dbtool.go +++ /dev/null @@ -1,8 +0,0 @@ -package common - -func ByteToUInt32(data []byte) uint32 { - i := uint32(data[0]) & 0xff - i |= (uint32(data[1]) << 8) & 0xff00 - i |= (uint32(data[2]) << 16) & 0xff0000 - return i -} diff --git a/pkg/common/httpclient.go b/pkg/common/httpclient.go index 4be834ea..2d5418bc 100644 --- a/pkg/common/httpclient.go +++ b/pkg/common/httpclient.go @@ -1,57 +1,59 @@ -package common - -import ( - "io/ioutil" - "net/http" - "time" -) - -const UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36" - -type HttpClient struct { - *http.Client -} - -var httpClient *HttpClient - -func init() { - httpClient = &HttpClient{http.DefaultClient} - httpClient.Timeout = time.Second * 30 - httpClient.Transport = &http.Transport{ - TLSHandshakeTimeout: time.Second * 5, - IdleConnTimeout: time.Second * 20, - ResponseHeaderTimeout: time.Second * 20, - ExpectContinueTimeout: time.Second * 20, - } -} - -func GetHttpClient() *HttpClient { - c := *httpClient - return &c -} - -func (c *HttpClient) Get(urls ...string) (body []byte, err error) { - var req *http.Request - var resp *http.Response - - for _, url := range urls { - req, err = http.NewRequest(http.MethodGet, url, nil) - if err != nil { - continue - } - req.Header.Set("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8") - req.Header.Set("User-Agent", UserAgent) - resp, err = c.Do(req) - - if err == nil && resp != nil && resp.StatusCode == 200 { - defer resp.Body.Close() - body, err = ioutil.ReadAll(resp.Body) - if err != nil { - continue - } - return - } - } - - return nil, err -} +package common + +import ( + "io/ioutil" + "log" + "net/http" + "time" +) + +const UserAgent = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.0.0 Safari/537.36" + +type HttpClient struct { + *http.Client +} + +var httpClient *HttpClient + +func init() { + httpClient = &HttpClient{http.DefaultClient} + httpClient.Timeout = time.Second * 60 + httpClient.Transport = &http.Transport{ + TLSHandshakeTimeout: time.Second * 5, + IdleConnTimeout: time.Second * 10, + ResponseHeaderTimeout: time.Second * 10, + ExpectContinueTimeout: time.Second * 20, + Proxy: http.ProxyFromEnvironment, + } +} + +func GetHttpClient() *HttpClient { + c := *httpClient + return &c +} + +func (c *HttpClient) Get(urls ...string) (body []byte, err error) { + var req *http.Request + var resp *http.Response + + for _, url := range urls { + req, err = http.NewRequest(http.MethodGet, url, nil) + if err != nil { + log.Println(err) + continue + } + req.Header.Set("User-Agent", UserAgent) + resp, err = c.Do(req) + + if err == nil && resp != nil && resp.StatusCode == 200 { + defer resp.Body.Close() + body, err = ioutil.ReadAll(resp.Body) + if err != nil { + continue + } + return + } + } + + return nil, err +} diff --git a/pkg/common/savefile.go b/pkg/common/savefile.go index f8f603e2..0f3b9bc3 100644 --- a/pkg/common/savefile.go +++ b/pkg/common/savefile.go @@ -1,21 +1,21 @@ -package common - -import ( - "io/ioutil" - "log" - "os" -) - -func SaveFile(path string, data []byte) (err error) { - // Remove file if exist - _, err = os.Stat(path) - if err == nil { - err = os.Remove(path) - if err != nil { - log.Fatalln("旧文件删除失败", err.Error()) - } - } - - // save file - return ioutil.WriteFile(path, data, 0644) -} +package common + +import ( + "io/ioutil" + "log" + "os" +) + +func SaveFile(path string, data []byte) (err error) { + // Remove file if exist + _, err = os.Stat(path) + if err == nil { + err = os.Remove(path) + if err != nil { + log.Fatalln("旧文件删除失败", err.Error()) + } + } + + // save file + return ioutil.WriteFile(path, data, 0644) +} diff --git a/pkg/common/scan.go b/pkg/common/scan.go new file mode 100644 index 00000000..390254b8 --- /dev/null +++ b/pkg/common/scan.go @@ -0,0 +1,26 @@ +package common + +import ( + "bytes" +) + +// ScanLines scan lines but keep the suffix \r and \n +func ScanLines(data []byte, atEOF bool) (advance int, token []byte, err error) { + if atEOF && len(data) == 0 { + return 0, nil, nil + } + + if i := bytes.IndexByte(data, '\n'); i >= 0 { + return i + 1, data[:i+1], nil + } + if i := bytes.IndexByte(data, '\r'); i >= 0 { + return i + 1, data[:i+1], nil + } + + // If we're at EOF, we have a final, non-terminated line. Return it. + if atEOF { + return len(data), data, nil + } + // Request more data. + return 0, nil, nil +} diff --git a/pkg/common/struct.go b/pkg/common/struct.go deleted file mode 100644 index 6a6c5096..00000000 --- a/pkg/common/struct.go +++ /dev/null @@ -1,99 +0,0 @@ -package common - -import ( - "fmt" - "os" -) - -// FileData: info of database file -type FileData struct { - Data []byte - FilePath string - FileBase *os.File -} - -// IPDB common ip database -type IPDB struct { - Data *FileData - Offset uint32 - IPNum uint32 -} - -// setOffset 设置偏移量 -func (db *IPDB) SetOffset(offset uint32) { - db.Offset = offset -} - -// readString 获取字符串 -func (db *IPDB) ReadString(offset uint32) []byte { - db.SetOffset(offset) - data := make([]byte, 0, 30) - buf := make([]byte, 1) - for { - buf = db.ReadData(1) - if buf[0] == 0 { - break - } - data = append(data, buf[0]) - } - return data -} - -// readData 从文件中读取数据 -func (db *IPDB) ReadData(length uint32, offset ...uint32) (rs []byte) { - if len(offset) > 0 { - db.SetOffset(offset[0]) - } - - end := db.Offset + length - dataNum := uint32(len(db.Data.Data)) - if db.Offset > dataNum { - return nil - } - - if end > dataNum { - end = dataNum - } - rs = db.Data.Data[db.Offset:end] - db.Offset = end - return -} - -// readMode 获取偏移值类型 -func (db *IPDB) ReadMode(offset uint32) byte { - mode := db.ReadData(1, offset) - return mode[0] -} - -// ReadUInt24 -func (db *IPDB) ReadUInt24() uint32 { - buf := db.ReadData(3) - return ByteToUInt32(buf) -} - -// readArea 读取区域 -func (db *IPDB) ReadArea(offset uint32) []byte { - mode := db.ReadMode(offset) - if mode == RedirectMode1 || mode == RedirectMode2 { - areaOffset := db.ReadUInt24() - if areaOffset == 0 { - return []byte("") - } - return db.ReadString(areaOffset) - } - return db.ReadString(offset) -} - -func GetMiddleOffset(start uint32, end uint32, indexLen uint32) uint32 { - records := ((end - start) / indexLen) >> 1 - return start + records*indexLen -} - -type Result struct { - Country string - Area string -} - -func (r Result) String() string { - return fmt.Sprintf("%s %s", r.Country, r.Area) -} diff --git a/pkg/dbif/db.go b/pkg/dbif/db.go index 7e50d4fa..90c4bdd4 100644 --- a/pkg/dbif/db.go +++ b/pkg/dbif/db.go @@ -1,35 +1,35 @@ -package dbif - -import ( - "fmt" - - "github.com/zu1k/nali/pkg/cdn" - "github.com/zu1k/nali/pkg/geoip" - "github.com/zu1k/nali/pkg/ip2location" - "github.com/zu1k/nali/pkg/ip2region" - "github.com/zu1k/nali/pkg/ipip" - "github.com/zu1k/nali/pkg/qqwry" - "github.com/zu1k/nali/pkg/zxipv6wry" -) - -type QueryType uint - -const ( - TypeIPv4 = iota - TypeIPv6 - TypeDomain -) - -type DB interface { - Find(query string, params ...string) (result fmt.Stringer, err error) -} - -var ( - _ DB = &qqwry.QQwry{} - _ DB = &zxipv6wry.ZXwry{} - _ DB = &ipip.IPIPFree{} - _ DB = &geoip.GeoIP{} - _ DB = &ip2region.Ip2Region{} - _ DB = &ip2location.IP2Location{} - _ DB = &cdn.CDN{} -) +package dbif + +import ( + "fmt" + + "github.com/zu1k/nali/pkg/cdn" + "github.com/zu1k/nali/pkg/geoip" + "github.com/zu1k/nali/pkg/ip2location" + "github.com/zu1k/nali/pkg/ip2region" + "github.com/zu1k/nali/pkg/ipip" + "github.com/zu1k/nali/pkg/qqwry" + "github.com/zu1k/nali/pkg/zxipv6wry" +) + +type QueryType uint + +const ( + TypeIPv4 = iota + TypeIPv6 + TypeDomain +) + +type DB interface { + Find(query string, params ...string) (result fmt.Stringer, err error) +} + +var ( + _ DB = &qqwry.QQwry{} + _ DB = &zxipv6wry.ZXwry{} + _ DB = &ipip.IPIPFree{} + _ DB = &geoip.GeoIP{} + _ DB = &ip2region.Ip2Region{} + _ DB = &ip2location.IP2Location{} + _ DB = &cdn.CDN{} +) diff --git a/pkg/download/download.go b/pkg/download/download.go index 35f46e52..5719c45f 100644 --- a/pkg/download/download.go +++ b/pkg/download/download.go @@ -1,25 +1,23 @@ -package download - -import ( - "log" - - "github.com/zu1k/nali/pkg/common" -) - -func Download(filePath string, urls ...string) (data []byte, err error) { - _ = urls[0] - - data, err = common.GetHttpClient().Get(urls...) - if err != nil { - log.Printf("文件下载失败，请手动下载解压后保存到本地: %s \n", filePath) - log.Println("下载链接：", urls) - return - } - - //if len(filePath) == 1 { - if err := common.SaveFile(filePath, data); err == nil { - log.Println("文件下载成功:", filePath) - } - //} - return -} +package download + +import ( + "log" + + "github.com/zu1k/nali/pkg/common" +) + +func Download(filePath string, urls ...string) (data []byte, err error) { + _ = urls[0] + + data, err = common.GetHttpClient().Get(urls...) + if err != nil { + log.Printf("文件下载失败，请手动下载解压后保存到本地: %s \n", filePath) + log.Println("下载链接：", urls) + return + } + + if err := common.SaveFile(filePath, data); err == nil { + log.Println("文件下载成功:", filePath) + } + return +} diff --git a/pkg/entity/entity.go b/pkg/entity/entity.go index cdce8b76..8b6b647a 100644 --- a/pkg/entity/entity.go +++ b/pkg/entity/entity.go @@ -1,75 +1,75 @@ -package entity - -import ( - "strings" - - "github.com/fatih/color" - "github.com/zu1k/nali/pkg/dbif" -) - -type EntityType uint - -const ( - TypeIPv4 = dbif.TypeIPv4 - TypeIPv6 = dbif.TypeIPv6 - TypeDomain = dbif.TypeDomain - - TypePlain = 100 -) - -type Entity struct { - Loc []int // s[Loc[0]:Loc[1]] - Type EntityType - - Text string - Info string -} - -func (e Entity) ParseInfo() error { - return nil -} - -type Entities []*Entity - -func (es Entities) Len() int { - return len(es) -} - -func (es Entities) Less(i, j int) bool { - return es[i].Loc[0] < es[j].Loc[0] -} - -func (es Entities) Swap(i, j int) { - es[i], es[j] = es[j], es[i] -} - -func (es Entities) String() string { - var result strings.Builder - for _, entity := range es { - result.WriteString(entity.Text) - if entity.Type != TypePlain && len(entity.Info) > 0 { - result.WriteString("[" + entity.Info + "] ") - } - } - return result.String() -} - -func (es Entities) ColorString() string { - var line strings.Builder - for _, e := range es { - s := e.Text - switch e.Type { - case TypeIPv4: - s = color.GreenString(e.Text) - case TypeIPv6: - s = color.BlueString(e.Text) - case TypeDomain: - s = color.YellowString(e.Text) - } - if e.Type != TypePlain && len(e.Info) > 0 { - s += " [" + color.RedString(e.Info) + "] " - } - line.WriteString(s) - } - return line.String() -} +package entity + +import ( + "strings" + + "github.com/fatih/color" + "github.com/zu1k/nali/pkg/dbif" +) + +type EntityType uint + +const ( + TypeIPv4 = dbif.TypeIPv4 + TypeIPv6 = dbif.TypeIPv6 + TypeDomain = dbif.TypeDomain + + TypePlain = 100 +) + +type Entity struct { + Loc []int // s[Loc[0]:Loc[1]] + Type EntityType + + Text string + Info string +} + +func (e Entity) ParseInfo() error { + return nil +} + +type Entities []*Entity + +func (es Entities) Len() int { + return len(es) +} + +func (es Entities) Less(i, j int) bool { + return es[i].Loc[0] < es[j].Loc[0] +} + +func (es Entities) Swap(i, j int) { + es[i], es[j] = es[j], es[i] +} + +func (es Entities) String() string { + var result strings.Builder + for _, entity := range es { + result.WriteString(entity.Text) + if entity.Type != TypePlain && len(entity.Info) > 0 { + result.WriteString("[" + entity.Info + "] ") + } + } + return result.String() +} + +func (es Entities) ColorString() string { + var line strings.Builder + for _, e := range es { + s := e.Text + switch e.Type { + case TypeIPv4: + s = color.GreenString(e.Text) + case TypeIPv6: + s = color.BlueString(e.Text) + case TypeDomain: + s = color.YellowString(e.Text) + } + if e.Type != TypePlain && len(e.Info) > 0 { + s += " [" + color.RedString(e.Info) + "] " + } + line.WriteString(s) + } + return line.String() +} diff --git a/pkg/entity/parse.go b/pkg/entity/parse.go index b811e1f6..bdd1d6cf 100644 --- a/pkg/entity/parse.go +++ b/pkg/entity/parse.go @@ -1,72 +1,72 @@ -package entity - -import ( - "net/netip" - "sort" - - "github.com/zu1k/nali/internal/db" - "github.com/zu1k/nali/pkg/dbif" - "github.com/zu1k/nali/pkg/re" -) - -// ParseLine parse a line into entities -func ParseLine(line string) Entities { - ip4sLoc := re.IPv4Re.FindAllStringIndex(line, -1) - ip6sLoc := re.IPv6Re.FindAllStringIndex(line, -1) - domainsLoc := re.DomainRe.FindAllStringIndex(line, -1) - - tmp := make(Entities, 0, len(ip4sLoc)+len(ip6sLoc)+len(domainsLoc)) - for _, e := range ip4sLoc { - tmp = append(tmp, &Entity{ - Loc: e, - Type: TypeIPv4, - Text: line[e[0]:e[1]], - }) - } - for _, e := range ip6sLoc { - text := line[e[0]:e[1]] - if ip, _ := netip.ParseAddr(text); !ip.Is4In6() { - tmp = append(tmp, &Entity{ - Loc: e, - Type: TypeIPv6, - Text: text, - }) - } - } - for _, e := range domainsLoc { - tmp = append(tmp, &Entity{ - Loc: e, - Type: TypeDomain, - Text: line[e[0]:e[1]], - }) - } - - sort.Sort(tmp) - es := make(Entities, 0, len(tmp)) - - idx := 0 - for _, e := range tmp { - start := e.Loc[0] - if start >= idx { - if start > idx { - es = append(es, &Entity{ - Loc: []int{idx, start}, - Type: TypePlain, - Text: line[idx:start], - }) - } - e.Info = db.Find(dbif.QueryType(e.Type), e.Text) - es = append(es, e) - idx = e.Loc[1] - } - } - if total := len(line); idx < total { - es = append(es, &Entity{ - Loc: []int{idx, total}, - Type: TypePlain, - Text: line[idx:total], - }) - } - - return es -} +package entity + +import ( + "net/netip" + "sort" + + "github.com/zu1k/nali/internal/db" + "github.com/zu1k/nali/pkg/dbif" + "github.com/zu1k/nali/pkg/re" +) + +// ParseLine parse a line into entities +func ParseLine(line string) Entities { + ip4sLoc := re.IPv4Re.FindAllStringIndex(line, -1) + ip6sLoc := re.IPv6Re.FindAllStringIndex(line, -1) + domainsLoc := re.DomainRe.FindAllStringIndex(line, -1) + + tmp := make(Entities, 0, len(ip4sLoc)+len(ip6sLoc)+len(domainsLoc)) + for _, e := range ip4sLoc { + tmp = append(tmp, &Entity{ + Loc: e, + Type: TypeIPv4, + Text: line[e[0]:e[1]], + }) + } + for _, e := range ip6sLoc { + text := line[e[0]:e[1]] + if ip, _ := netip.ParseAddr(text); !ip.Is4In6() { + tmp = append(tmp, &Entity{ + Loc: e, + Type: TypeIPv6, + Text: text, + }) + } + } + for _, e := range domainsLoc { + tmp = append(tmp, &Entity{ + Loc: e, + Type: TypeDomain, + Text: line[e[0]:e[1]], + }) + } + + sort.Sort(tmp) + es := make(Entities, 0, len(tmp)) + + idx := 0 + for _, e := range tmp { + start := e.Loc[0] + if start >= idx { + if start > idx { + es = append(es, &Entity{ + Loc: []int{idx, start}, + Type: TypePlain, + Text: line[idx:start], + }) + } + e.Info = db.Find(dbif.QueryType(e.Type), e.Text) + es = append(es, e) + idx = e.Loc[1] + } + } + if total := len(line); idx < total { + es = append(es, &Entity{ + Loc: []int{idx, total}, + Type: TypePlain, + Text: line[idx:total], + }) + } + + return es +} diff --git a/pkg/entity/parse_test.go b/pkg/entity/parse_test.go deleted file mode 100644 index 197b48ff..00000000 --- a/pkg/entity/parse_test.go +++ /dev/null @@ -1,15 +0,0 @@ -package entity - -import ( - "fmt" - "testing" -) - -func TestParse(t *testing.T) { - fmt.Println(ParseLine("2001:0db8:85a3:0000:0000:8a2e:0370:7334 baidu.com 1.2.3.4 baidu.com")) - fmt.Println(ParseLine("a.cn.b.com.c.org d.com")) -} - -func TestColorPrint(t *testing.T) { - fmt.Println(ParseLine("2001:0db8:85a3:0000:0000:8a2e:0370:7334 baidu.com 1.2.3.4 baidu.com").ColorString()) -} diff --git a/pkg/geoip/geoip.go b/pkg/geoip/geoip.go index 3655312f..2a0f72f1 100644 --- a/pkg/geoip/geoip.go +++ b/pkg/geoip/geoip.go @@ -1,69 +1,68 @@ -package geoip - -import ( - "errors" - "fmt" - "log" - "net" - "os" - - "github.com/spf13/viper" - - "github.com/oschwald/geoip2-golang" -) - -// GeoIP2 -type GeoIP struct { - db *geoip2.Reader -} - -// new geoip from database file -func NewGeoIP(filePath string) (*GeoIP, error) { - // 判断文件是否存在 - _, err := os.Stat(filePath) - if err != nil && os.IsNotExist(err) { - log.Println("文件不存在，请自行下载 Geoip2 City库，并保存在", filePath) - return nil, err - } else { - db, err := geoip2.Open(filePath) - if err != nil { - log.Fatal(err) - } - return &GeoIP{db: db}, nil - } -} - -func (g GeoIP) Find(query string, params ...string) (result fmt.Stringer, err error) { - ip := net.ParseIP(query) - if ip == nil { - return nil, errors.New("Query should be valid IP") - } - record, err := g.db.City(ip) - if err != nil { - return - } - - lang := viper.GetString("selected.lang") - if lang == "" { - lang = "zh-CN" - } - - result = Result{ - Country: record.Country.Names[lang], - City: record.City.Names[lang], - } - return -} - -type Result struct { - Country string - City string -} - -func (r Result) String() string { - if r.City == "" { - return r.Country - } else { - return fmt.Sprintf("%s %s", r.Country, r.City) - } -} +package geoip + +import ( + "errors" + "fmt" + "log" + "net" + "os" + + "github.com/oschwald/geoip2-golang" + "github.com/spf13/viper" +) + +// GeoIP2 +type GeoIP struct { + db *geoip2.Reader +} + +// new geoip from database file +func NewGeoIP(filePath string) (*GeoIP, error) { + // 判断文件是否存在 + _, err := os.Stat(filePath) + if err != nil && os.IsNotExist(err) { + log.Println("文件不存在，请自行下载 Geoip2 City库，并保存在", filePath) + return nil, err + } else { + db, err := geoip2.Open(filePath) + if err != nil { + log.Fatal(err) + } + return &GeoIP{db: db}, nil + } +} + +func (g GeoIP) Find(query string, params ...string) (result fmt.Stringer, err error) { + ip := net.ParseIP(query) + if ip == nil { + return nil, errors.New("Query should be valid IP") + } + record, err := g.db.City(ip) + if err != nil { + return + } + + lang := viper.GetString("selected.lang") + if lang == "" { + lang = "zh-CN" + } + + result = Result{ + Country: record.Country.Names[lang], + City: record.City.Names[lang], + } + return +} + +type Result struct { + Country string + City string +} + +func (r Result) String() string { + if r.City == "" { + return r.Country + } else { + return fmt.Sprintf("%s %s", r.Country, r.City) + } +} diff --git a/pkg/ip2location/ip2location.go b/pkg/ip2location/ip2location.go index 40dfd165..1e1cd1e7 100644 --- a/pkg/ip2location/ip2location.go +++ b/pkg/ip2location/ip2location.go @@ -1,61 +1,61 @@ -package ip2location - -import ( - "errors" - "fmt" - "log" - "net" - "os" - - "github.com/ip2location/ip2location-go/v9" -) - -// IP2Location -type IP2Location struct { - db *ip2location.DB -} - -// new IP2Location from database file -func NewIP2Location(filePath string) (*IP2Location, error) { - _, err := os.Stat(filePath) - if err != nil && os.IsNotExist(err) { - log.Println("文件不存在，请自行下载 IP2Location 库，并保存在", filePath) - return nil, err - } else { - db, err := ip2location.OpenDB(filePath) - - if err != nil { - log.Fatal(err) - } - return &IP2Location{db: db}, nil - } -} - -func (x IP2Location) Find(query string, params ...string) (result fmt.Stringer, err error) { - ip := net.ParseIP(query) - if ip == nil { - return nil, errors.New("Query should be valid IP") - } - record, err := x.db.Get_all(ip.String()) - - if err != nil { - return - } - - result = Result{ - Country: record.Country_long, - Region: record.Region, - City: record.City, - } - return -} - -type Result struct { - Country string - Region string - City string -} - -func (r Result) String() string { - return fmt.Sprintf("%s %s %s", r.Country, r.Region, r.City) -} +package ip2location + +import ( + "errors" + "fmt" + "log" + "net" + "os" + + "github.com/ip2location/ip2location-go/v9" +) + +// IP2Location +type IP2Location struct { + db *ip2location.DB +} + +// new IP2Location from database file +func NewIP2Location(filePath string) (*IP2Location, error) { + _, err := os.Stat(filePath) + if err != nil && os.IsNotExist(err) { + log.Println("文件不存在，请自行下载 IP2Location 库，并保存在", filePath) + return nil, err + } else { + db, err := ip2location.OpenDB(filePath) + + if err != nil { + log.Fatal(err) + } + return &IP2Location{db: db}, nil + } +} + +func (x IP2Location) Find(query string, params ...string) (result fmt.Stringer, err error) { + ip := net.ParseIP(query) + if ip == nil { + return nil, errors.New("Query should be valid IP") + } + record, err := x.db.Get_all(ip.String()) + + if err != nil { + return + } + + result = Result{ + Country: record.Country_long, + Region: record.Region, + City: record.City, + } + return +} + +type Result struct { + Country string + Region string + City string +} + +func (r Result) String() string { + return fmt.Sprintf("%s %s %s", r.Country, r.Region, r.City) +} diff --git a/pkg/ip2region/ip2region.go b/pkg/ip2region/ip2region.go index 53cbc150..ae6dcc12 100644 --- a/pkg/ip2region/ip2region.go +++ b/pkg/ip2region/ip2region.go @@ -1,59 +1,69 @@ -package ip2region - -import ( - "fmt" - "log" - "os" - "strings" - - "github.com/lionsoul2014/ip2region/binding/golang/ip2region" - "github.com/zu1k/nali/pkg/common" -) - -type Ip2Region struct { - db *ip2region.Ip2Region -} - -func NewIp2Region(filePath string) (*Ip2Region, error) { - _, err := os.Stat(filePath) - if err != nil && os.IsNotExist(err) { - log.Println("文件不存在，尝试从网络获取最新 ip2region 库") - _, err = Download(filePath) - if err != nil { - return nil, err - } - } - - region, err := ip2region.New(filePath) - if err != nil { - return nil, err - } - - return &Ip2Region{ - db: region, - }, nil -} - -func (db Ip2Region) Find(query string, params ...string) (result fmt.Stringer, err error) { - ip, err := db.db.MemorySearch(query) - if err != nil { - return nil, err - } - - area := "" - if ip.Province != "0" { - area = ip.Province - } - if ip.City != "0" && strings.EqualFold(ip.City, ip.Province) { - area = area + " " + ip.Province - } - if ip.ISP != "0" { - area = area + " " + ip.ISP - } - - result = common.Result{ - Country: ip.Country, - Area: area, - } - return result, nil -} +package ip2region + +import ( + "errors" + "fmt" + "io/ioutil" + "log" + "os" + "strings" + + "github.com/zu1k/nali/pkg/download" + "github.com/zu1k/nali/pkg/wry" + + "github.com/lionsoul2014/ip2region/binding/golang/xdb" +) + +var DownloadUrls = []string{ + "https://cdn.jsdelivr.net/gh/lionsoul2014/ip2region/data/ip2region.xdb", + "https://raw.githubusercontent.com/lionsoul2014/ip2region/master/data/ip2region.xdb", +} + +type Ip2Region struct { + seacher *xdb.Searcher +} + +func NewIp2Region(filePath string) (*Ip2Region, error) { + _, err := os.Stat(filePath) + if err != nil && os.IsNotExist(err) { + log.Println("文件不存在，尝试从网络获取最新 ip2region 库") + _, err = download.Download(filePath, DownloadUrls...) + if err != nil { + return nil, err + } + } + + f, err := os.OpenFile(filePath, os.O_RDONLY, 0400) + if err != nil { + return nil, err + } + defer f.Close() + + data, err := ioutil.ReadAll(f) + if err != nil { + return nil, err + } + searcher, err := xdb.NewWithBuffer(data) + if err != nil { + fmt.Printf("无法解析 ip2region xdb 数据库: %s\n", err) + return nil, err + } + return &Ip2Region{ + seacher: searcher, + }, nil +} + +func (db Ip2Region) Find(query string, params ...string) (result fmt.Stringer, err error) { + if db.seacher != nil { + res, err := db.seacher.SearchByStr(query) + if err != nil { + return nil, err + } else { + return wry.Result{ + Country: strings.ReplaceAll(res, "|0", ""), + }, nil + } + } + + return nil, errors.New("ip2region 未初始化") +} diff --git a/pkg/ip2region/update.go b/pkg/ip2region/update.go deleted file mode 100644 index de5054e5..00000000 --- a/pkg/ip2region/update.go +++ /dev/null @@ -1,29 +0,0 @@ -package ip2region - -import ( - "log" - - "github.com/zu1k/nali/pkg/common" -) - -var DownloadUrls = []string{ - "https://cdn.jsdelivr.net/gh/lionsoul2014/ip2region/data/ip2region.db", - "https://raw.githubusercontent.com/lionsoul2014/ip2region/master/data/ip2region.db", -} - -// Deprecated: This will be removed from 0.5.0, use package download instead -func Download(filePath ...string) (data []byte, err error) { - data, err = common.GetHttpClient().Get(DownloadUrls...) - if err != nil { - log.Printf("CDN数据库下载失败，请手动下载解压后保存到本地: %s \n", filePath) - log.Println("下载链接：", DownloadUrls) - return - } - - if len(filePath) == 1 { - if err := common.SaveFile(filePath[0], data); err == nil { - log.Println("已将最新的 ip2region 保存到本地:", filePath) - } - } - return -} diff --git a/pkg/ipip/ipip.go b/pkg/ipip/ipip.go index aafe28ca..ec537ed1 100644 --- a/pkg/ipip/ipip.go +++ b/pkg/ipip/ipip.go @@ -1,56 +1,56 @@ -package ipip - -import ( - "fmt" - "log" - "os" - - "github.com/ipipdotnet/ipdb-go" -) - -type IPIPFree struct { - *ipdb.City -} - -func NewIPIP(filePath string) (*IPIPFree, error) { - _, err := os.Stat(filePath) - if err != nil && os.IsNotExist(err) { - log.Printf("IPIP数据库不存在，请手动下载解压后保存到本地: %s \n", filePath) - log.Println("下载链接： https://www.ipip.net/product/ip.html") - return nil, err - } else { - db, err := ipdb.NewCity(filePath) - if err != nil { - return nil, err - } - return &IPIPFree{City: db}, nil - } -} - -type Result struct { - Country string - Region string - City string -} - -func (r Result) String() string { - if r.City == "" { - return fmt.Sprintf("%s %s", r.Country, r.Region) - } - return fmt.Sprintf("%s %s %s", r.Country, r.Region, r.City) -} - -func (db IPIPFree) Find(query string, params ...string) (result fmt.Stringer, err error) { - info, err := db.FindInfo(query, "CN") - if err != nil || info == nil { - return nil, err - } else { - // info contains more info - result = Result{ - Country: info.CountryName, - Region: info.RegionName, - City: info.CityName, - } - return - } -} +package ipip + +import ( + "fmt" + "log" + "os" + + "github.com/ipipdotnet/ipdb-go" +) + +type IPIPFree struct { + *ipdb.City +} + +func NewIPIP(filePath string) (*IPIPFree, error) { + _, err := os.Stat(filePath) + if err != nil && os.IsNotExist(err) { + log.Printf("IPIP数据库不存在，请手动下载解压后保存到本地: %s \n", filePath) + log.Println("下载链接： https://www.ipip.net/product/ip.html") + return nil, err + } else { + db, err := ipdb.NewCity(filePath) + if err != nil { + return nil, err + } + return &IPIPFree{City: db}, nil + } +} + +type Result struct { + Country string + Region string + City string +} + +func (r Result) String() string { + if r.City == "" { + return fmt.Sprintf("%s %s", r.Country, r.Region) + } + return fmt.Sprintf("%s %s %s", r.Country, r.Region, r.City) +} + +func (db IPIPFree) Find(query string, params ...string) (result fmt.Stringer, err error) { + info, err := db.FindInfo(query, "CN") + if err != nil || info == nil { + return nil, err + } else { + // info contains more info + result = Result{ + Country: info.CountryName, + Region: info.RegionName, + City: info.CityName, + } + return + } +} diff --git a/pkg/qqwry/qqwry.go b/pkg/qqwry/qqwry.go index bd5a835e..9b590015 100644 --- a/pkg/qqwry/qqwry.go +++ b/pkg/qqwry/qqwry.go @@ -1,145 +1,92 @@ -package qqwry - -import ( - "encoding/binary" - "errors" - "fmt" - "io/ioutil" - "log" - "net" - "os" - "strings" - - "github.com/zu1k/nali/pkg/common" - "golang.org/x/text/encoding/simplifiedchinese" -) - -type QQwry struct { - common.IPDB -} - -// NewQQwry new database from path -func NewQQwry(filePath string) (*QQwry, error) { - var fileData []byte - var fileInfo common.FileData - - _, err := os.Stat(filePath) - if err != nil && os.IsNotExist(err) { - log.Println("文件不存在，尝试从网络获取最新纯真 IP 库") - fileData, err = Download(filePath) - if err != nil { - return nil, err - } - } else { - fileInfo.FileBase, err = os.OpenFile(filePath, os.O_RDONLY, 0400) - if err != nil { - return nil, err - } - defer fileInfo.FileBase.Close() - - fileData, err = ioutil.ReadAll(fileInfo.FileBase) - if err != nil { - return nil, err - } - } - fileInfo.Data = fileData - - buf := fileInfo.Data[0:8] - start := binary.LittleEndian.Uint32(buf[:4]) - end := binary.LittleEndian.Uint32(buf[4:]) - - return &QQwry{ - IPDB: common.IPDB{ - Data: &fileInfo, - IPNum: (end-start)/7 + 1, - }, - }, nil -} - -func (db QQwry) Find(query string, params ...string) (result fmt.Stringer, err error) { - ip := net.ParseIP(query) - if ip == nil { - return nil, errors.New("Query should be IPv4") - } - ip4 := ip.To4() - if ip4 == nil { - return nil, errors.New("Query should be IPv4") - } - ip4uint := binary.BigEndian.Uint32(ip4) - - offset := db.searchIndex(ip4uint) - if offset <= 0 { - return nil, errors.New("Query not valid") - } - - var gbkCountry []byte - var gbkArea []byte - - mode := db.ReadMode(offset + 4) - switch mode { - case common.RedirectMode1: // [IP][0x01][国家和地区信息的绝对偏移地址] - countryOffset := db.ReadUInt24() - mode = db.ReadMode(countryOffset) - if mode == common.RedirectMode2 { - c := db.ReadUInt24() - gbkCountry = db.ReadString(c) - countryOffset += 4 - } else { - gbkCountry = db.ReadString(countryOffset) - countryOffset += uint32(len(gbkCountry) + 1) - } - gbkArea = db.ReadArea(countryOffset) - case common.RedirectMode2: - countryOffset := db.ReadUInt24() - gbkCountry = db.ReadString(countryOffset) - gbkArea = db.ReadArea(offset + 8) - default: - gbkCountry = db.ReadString(offset + 4) - gbkArea = db.ReadArea(offset + uint32(5+len(gbkCountry))) - } - - enc := simplifiedchinese.GBK.NewDecoder() - country, _ := enc.String(string(gbkCountry)) - area, _ := enc.String(string(gbkArea)) - - result = common.Result{ - Country: strings.ReplaceAll(country, " CZ88.NET", ""), - Area: strings.ReplaceAll(area, " CZ88.NET", ""), - } - return result, nil -} - -// searchIndex 查找索引位置 -func (db *QQwry) searchIndex(ip uint32) uint32 { - header := db.ReadData(8, 0) - - start := binary.LittleEndian.Uint32(header[:4]) - end := binary.LittleEndian.Uint32(header[4:]) - - buf := make([]byte, 7) - mid := uint32(0) - ipUint := uint32(0) - - for { - mid = common.GetMiddleOffset(start, end, 7) - buf = db.ReadData(7, mid) - ipUint = binary.LittleEndian.Uint32(buf[:4]) - - if end-start == 7 { - offset := common.ByteToUInt32(buf[4:]) - buf = db.ReadData(7) - if ip < binary.LittleEndian.Uint32(buf[:4]) { - return offset - } - return 0 - } - - if ipUint > ip { - end = mid - } else if ipUint < ip { - start = mid - } else if ipUint == ip { - return common.ByteToUInt32(buf[4:]) - } - } -} +package qqwry + +import ( + "encoding/binary" + "errors" + "fmt" + "io" + "log" + "net" + "os" + + "github.com/zu1k/nali/pkg/download" + "github.com/zu1k/nali/pkg/wry" +) + +var DownloadUrls = []string{ + "https://99wry.cf/qqwry.dat", +} + +type QQwry struct { + wry.IPDB[uint32] +} + +// NewQQwry new database from path +func NewQQwry(filePath string) (*QQwry, error) { + var fileData []byte + + _, err := os.Stat(filePath) + if err != nil && os.IsNotExist(err) { + log.Println("文件不存在，尝试从网络获取最新纯真 IP 库") + fileData, err = download.Download(filePath, DownloadUrls...) + if err != nil { + return nil, err + } + } else { + fileBase, err := os.OpenFile(filePath, os.O_RDONLY, 0400) + if err != nil { + return nil, err + } + defer fileBase.Close() + + fileData, err = io.ReadAll(fileBase) + if err != nil { + return nil, err + } + } + + if len(fileData) < 8 { + log.Fatalln("纯真 IP 库存在错误，请重新下载") + } + + header := fileData[0:8] + start := binary.LittleEndian.Uint32(header[:4]) + end := binary.LittleEndian.Uint32(header[4:]) + + if uint32(len(fileData)) < end+7 { + log.Fatalln("纯真 IP 库存在错误，请重新下载") + } + + return &QQwry{ + IPDB: wry.IPDB[uint32]{ + Data: fileData, + + OffLen: 3, + IPLen: 4, + IPCnt: (end-start)/7 + 1, + IdxStart: start, + IdxEnd: end, + }, + }, nil +} + +func (db QQwry) Find(query string, params ...string) (result fmt.Stringer, err error) { + ip := net.ParseIP(query) + if ip == nil { + return nil, errors.New("query should be IPv4") + } + ip4 := ip.To4() + if ip4 == nil { + return nil, errors.New("query should be IPv4") + } + ip4uint := binary.BigEndian.Uint32(ip4) + + offset := db.SearchIndexV4(ip4uint) + if offset <= 0 { + return nil, errors.New("query not valid") + } + + reader := wry.NewReader(db.Data) + reader.Parse(offset + 4) + return reader.Result.DecodeGBK(), nil +} diff --git a/pkg/qqwry/update.go b/pkg/qqwry/update.go deleted file mode 100644 index 6a3ba851..00000000 --- a/pkg/qqwry/update.go +++ /dev/null @@ -1,72 +0,0 @@ -package qqwry - -import ( - "bytes" - "compress/zlib" - "encoding/binary" - "io/ioutil" - "log" - - "github.com/zu1k/nali/pkg/common" -) - -func Download(filePath ...string) (data []byte, err error) { - data, err = downloadAndDecrypt() - if err != nil { - log.Printf("纯真IP库下载失败，请手动下载解压后保存到本地: %s \n", filePath) - log.Println("下载链接： https://qqwry.mirror.noc.one/qqwry.rar") - return - } - - if len(filePath) == 1 { - if err := common.SaveFile(filePath[0], data); err == nil { - log.Println("已将最新的 纯真IP库 保存到本地:", filePath) - } - } - return -} - -const ( - mirror = "https://qqwry.mirror.noc.one/qqwry.rar" - key = "https://qqwry.mirror.noc.one/copywrite.rar" -) - -func downloadAndDecrypt() (data []byte, err error) { - data, err = common.GetHttpClient().Get(mirror) - if err != nil { - return nil, err - } - - key, err := getCopyWriteKey() - if err != nil { - return nil, err - } - - return unRar(data, key) -} - -func unRar(data []byte, key uint32) ([]byte, error) { - for i := 0; i < 0x200; i++ { - key = key * 0x805 - key++ - key = key & 0xff - - data[i] = byte(uint32(data[i]) ^ key) - } - - reader, err := zlib.NewReader(bytes.NewReader(data)) - if err != nil { - return nil, err - } - - return ioutil.ReadAll(reader) -} - -func getCopyWriteKey() (uint32, error) { - body, err := common.GetHttpClient().Get(key) - if err != nil { - return 0, err - } - - return binary.LittleEndian.Uint32(body[5*4:]), nil -} diff --git a/pkg/re/re.go b/pkg/re/re.go index 5d07c0c1..780b159b 100644 --- a/pkg/re/re.go +++ b/pkg/re/re.go @@ -1,10 +1,17 @@ -package re - -import "regexp" - -var ( - DomainRe = regexp.MustCompile(`([a-zA-Z0-9][-a-zA-Z0-9]{0,62}\.)+([a-zA-Z][-a-zA-Z]{0,62})`) - - IPv4Re = regexp.MustCompile(`(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)){3}`) - IPv6Re = regexp.MustCompile(`fe80:(:[0-9a-fA-F]{1,4}){0,4}(%\w+)?|([0-9a-fA-F]{1,4}:){7}[0-9a-fA-F]{1,4}|::[fF]{4}:(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)){3}|(([0-9a-fA-F]{1,4}:){0,6}[0-9a-fA-F]{1,4})?::(([0-9a-fA-F]{1,4}:){0,6}[0-9a-fA-F]{1,4})?`) -) +package re + +import ( + "regexp" + "strings" +) + +var ( + DomainRe = regexp.MustCompile(`([a-zA-Z0-9][-a-zA-Z0-9]{0,62}\.)+([a-zA-Z][-a-zA-Z]{0,62})`) + + IPv4Re = regexp.MustCompile(`(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)){3}`) + IPv6Re = regexp.MustCompile(`fe80:(:[0-9a-fA-F]{1,4}){0,4}(%\w+)?|([0-9a-fA-F]{1,4}:){7}[0-9a-fA-F]{1,4}|::[fF]{4}:(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)){3}|(([0-9a-fA-F]{1,4}:){0,6}[0-9a-fA-F]{1,4})?::(([0-9a-fA-F]{1,4}:){0,6}[0-9a-fA-F]{1,4})?`) +) + +func MaybeRegexp(s string) bool { + return strings.ContainsAny(s, "[]{}()?") +} diff --git a/pkg/re/re_test.go b/pkg/re/re_test.go index 4907b5ab..4760ca1b 100644 --- a/pkg/re/re_test.go +++ b/pkg/re/re_test.go @@ -1,37 +1,55 @@ -package re - -import ( - "fmt" - "testing" -) - -var domainList = []string{ - "a.a.qiniudns.com", - "a.com.qiniudns.com", - "a.com.cn.qiniudns.com", - "看这里：a.com.cn.qiniudns.com行不行", -} - -func TestDomainRe(t *testing.T) { - for _, domain := range domainList { - if !DomainRe.MatchString(domain) { - t.Error(domain) - t.Fail() - } - fmt.Println(DomainRe.FindAllString(domain, -1)) - } -} - -var validIPv6List = []string{ - "::ffff:104.26.11.119", -} - -func TestIPv6Re(t *testing.T) { - for _, ip := range validIPv6List { - if !IPv6Re.MatchString(ip) { - t.Error(ip) - t.Fail() - } - fmt.Println(IPv6Re.FindAllString(ip, -1)) - } -} +package re + +import ( + "fmt" + "testing" +) + +var domainList = []string{ + "a.a.qiniudns.com", + "a.com.qiniudns.com", + "a.com.cn.qiniudns.com", + "看这里：a.com.cn.qiniudns.com行不行", +} + +func TestDomainRe(t *testing.T) { + for _, domain := range domainList { + if !DomainRe.MatchString(domain) { + t.Error(domain) + t.Fail() + } + fmt.Println(DomainRe.FindAllString(domain, -1)) + } +} + +var validIPv6List = []string{ + "::ffff:104.26.11.119", +} + +func TestIPv6Re(t *testing.T) { + for _, ip := range validIPv6List { + if !IPv6Re.MatchString(ip) { + t.Error(ip) + t.Fail() + } + fmt.Println(IPv6Re.FindAllString(ip, -1)) + } +} + +var maybeRegexList = []string{ + "[a-z]*\\.example.com", + "kunlun[^.]+.com", + "gtm-a[1-7]b[1-9].com", +} + +func TestMaybeRegexp(t *testing.T) { + if MaybeRegexp("abc.com") { + t.Fail() + } + + for _, str := range maybeRegexList { + if !MaybeRegexp(str) { + t.Fail() + } + } +} diff --git a/pkg/wry/index.go b/pkg/wry/index.go new file mode 100644 index 00000000..ef0b3b09 --- /dev/null +++ b/pkg/wry/index.go @@ -0,0 +1,63 @@ +package wry + +import ( + "encoding/binary" +) + +func (db *IPDB[uint32]) SearchIndexV4(ip uint32) uint32 { + ipLen := db.IPLen + entryLen := uint32(db.OffLen + db.IPLen) + + buf := make([]byte, entryLen) + l, r, mid, ipc := db.IdxStart, db.IdxEnd, uint32(0), uint32(0) + + for { + mid = (r-l)/entryLen/2*entryLen + l + buf = db.Data[mid : mid+entryLen] + ipc = uint32(binary.LittleEndian.Uint32(buf[:ipLen])) + + if r-l == entryLen { + if ip >= uint32(binary.LittleEndian.Uint32(db.Data[r:r+uint32(ipLen)])) { + buf = db.Data[r : r+entryLen] + } + return uint32(Bytes3ToUint32(buf[ipLen:entryLen])) + } + + if ipc > ip { + r = mid + } else if ipc < ip { + l = mid + } else if ipc == ip { + return uint32(Bytes3ToUint32(buf[ipLen:entryLen])) + } + } +} + +func (db *IPDB[uint64]) SearchIndexV6(ip uint64) uint32 { + ipLen := db.IPLen + entryLen := uint64(db.OffLen + db.IPLen) + + buf := make([]byte, entryLen) + l, r, mid, ipc := db.IdxStart, db.IdxEnd, uint64(0), uint64(0) + + for { + mid = (r-l)/entryLen/2*entryLen + l + buf = db.Data[mid : mid+entryLen] + ipc = uint64(binary.LittleEndian.Uint64(buf[:ipLen])) + + if r-l == entryLen { + if ip >= uint64(binary.LittleEndian.Uint64(db.Data[r:r+uint64(ipLen)])) { + buf = db.Data[r : r+entryLen] + } + return Bytes3ToUint32(buf[ipLen:entryLen]) + } + + if ipc > ip { + r = mid + } else if ipc < ip { + l = mid + } else if ipc == ip { + return Bytes3ToUint32(buf[ipLen:entryLen]) + } + } +} diff --git a/pkg/wry/parse.go b/pkg/wry/parse.go new file mode 100644 index 00000000..6cdea0ff --- /dev/null +++ b/pkg/wry/parse.go @@ -0,0 +1,47 @@ +package wry + +const ( + // RedirectMode1 [IP][0x01][国家和地区信息的绝对偏移地址] + RedirectMode1 = 0x01 + // RedirectMode2 [IP][0x02][信息的绝对偏移][...] or [IP][国家][...] + RedirectMode2 = 0x02 +) + +func (r *Reader) Parse(offset uint32) { + if offset != 0 { + r.seekAbs(offset) + } + + switch r.readMode() { + case RedirectMode1: + r.readOffset(true) + r.Parse(0) + case RedirectMode2: + r.Result.Country = r.parseRedMode2() + r.Result.Area = r.readArea() + default: + r.seekBack() + r.Result.Country = r.readString(true) + r.Result.Area = r.readArea() + } +} + +func (r *Reader) parseRedMode2() string { + r.readOffset(true) + str := r.readString(false) + r.seekBack() + return str +} + +func (r *Reader) readArea() string { + mode := r.readMode() + if mode == RedirectMode1 || mode == RedirectMode2 { + offset := r.readOffset(true) + if offset == 0 { + return "" + } + } else { + r.seekBack() + } + return r.readString(false) +} diff --git a/pkg/wry/wry.go b/pkg/wry/wry.go new file mode 100644 index 00000000..506354fa --- /dev/null +++ b/pkg/wry/wry.go @@ -0,0 +1,116 @@ +package wry + +import ( + "bytes" + "fmt" + "strings" + + "golang.org/x/text/encoding/simplifiedchinese" +) + +// IPDB common ip database +type IPDB[T ~uint32 | ~uint64] struct { + Data []byte + + OffLen uint8 + IPLen uint8 + IPCnt T + IdxStart T + IdxEnd T +} + +type Reader struct { + s []byte + i uint32 // current reading index + l uint32 // last reading index + + Result Result +} + +func NewReader(data []byte) Reader { + return Reader{s: data, i: 0, l: 0, Result: Result{ + Country: "", + Area: "", + }} +} + +func (r *Reader) seekAbs(offset uint32) { + r.l = r.i + r.i = offset +} + +func (r *Reader) seek(offset int64) { + r.l = r.i + r.i = uint32(int64(r.i) + offset) +} + +// seekBack: seek to last index, can only call once +func (r *Reader) seekBack() { + r.i = r.l +} + +func (r *Reader) read(length uint32) []byte { + rs := make([]byte, length) + copy(rs, r.s[r.i:]) + r.l = r.i + r.i += length + return rs +} + +func (r *Reader) readMode() (mode byte) { + mode = r.s[r.i] + r.l = r.i + r.i += 1 + return +} + +// readOffset: read 3 bytes as uint32 offset +func (r *Reader) readOffset(follow bool) uint32 { + buf := r.read(3) + offset := Bytes3ToUint32(buf) + if follow { + r.l = r.i + r.i = offset + } + return offset +} + +func (r *Reader) readString(seek bool) string { + length := bytes.IndexByte(r.s[r.i:], 0) + str := string(r.s[r.i : r.i+uint32(length)]) + if seek { + r.l = r.i + r.i += uint32(length) + 1 + } + return str +} + +type Result struct { + Country string + Area string +} + +func (r *Result) DecodeGBK() *Result { + enc := simplifiedchinese.GBK.NewDecoder() + r.Country, _ = enc.String(r.Country) + r.Area, _ = enc.String(r.Area) + return r +} + +func (r *Result) Trim() *Result { + r.Country = strings.TrimSpace(strings.ReplaceAll(r.Country, "CZ88.NET", "")) + r.Area = strings.TrimSpace(strings.ReplaceAll(r.Area, "CZ88.NET", "")) + return r +} + +func (r Result) String() string { + r.Trim() + return strings.TrimSpace(fmt.Sprintf("%s %s", r.Country, r.Area)) +} + +func Bytes3ToUint32(data []byte) uint32 { + i := uint32(data[0]) & 0xff + i |= (uint32(data[1]) << 8) & 0xff00 + i |= (uint32(data[2]) << 16) & 0xff0000 + return i +} diff --git a/pkg/zxipv6wry/update.go b/pkg/zxipv6wry/update.go index 082665ef..f25ca576 100644 --- a/pkg/zxipv6wry/update.go +++ b/pkg/zxipv6wry/update.go @@ -1,92 +1,92 @@ -package zxipv6wry - -import ( - "io" - "io/ioutil" - "log" - "os" - - "github.com/saracen/go7z" - "github.com/zu1k/nali/pkg/common" -) - -func Download(filePath ...string) (data []byte, err error) { - data, err = getData() - if err != nil { - log.Printf("ZX IPv6数据库下载失败，请手动下载解压后保存到本地: %s \n", filePath) - log.Println("下载链接： https://ip.zxinc.org/ip.7z") - return - } - - if len(filePath) == 1 { - if err := common.SaveFile(filePath[0], data); err == nil { - log.Println("已将最新的 ZX IPv6数据库 保存到本地:", filePath) - } - } - return -} - -const ( - zx = "https://ip.zxinc.org/ip.7z" -) - -func getData() (data []byte, err error) { - data, err = common.GetHttpClient().Get(zx) - - file7z, err := ioutil.TempFile("", "*") - if err != nil { - return nil, err - } - defer os.Remove(file7z.Name()) - if err := ioutil.WriteFile(file7z.Name(), data, 0644); err == nil { - return Un7z(file7z.Name()) - } - return -} - -func Un7z(filePath string) (data []byte, err error) { - sz, err := go7z.OpenReader(filePath) - if err != nil { - return nil, err - } - defer sz.Close() - - fileNoNeed, err := ioutil.TempFile("", "*") - if err != nil { - return nil, err - } - fileNeed, err := ioutil.TempFile("", "*") - if err != nil { - return nil, err - } - - if err != nil { - return nil, err - } - for { - hdr, err := sz.Next() - if err == io.EOF { - break // End of archive - } - if err != nil { - return nil, err - } - - if hdr.Name == "ipv6wry.db" { - if _, err := io.Copy(fileNeed, sz); err != nil { - log.Fatalln("ZX ipv6数据库解压出错：", err.Error()) - } - } else { - if _, err := io.Copy(fileNoNeed, sz); err != nil { - log.Fatalln("ZX ipv6数据库解压出错：", err.Error()) - } - } - } - err = fileNoNeed.Close() - if err != nil { - return nil, err - } - defer os.Remove(fileNoNeed.Name()) - defer os.Remove(fileNeed.Name()) - return ioutil.ReadFile(fileNeed.Name()) -} +package zxipv6wry + +import ( + "io" + "io/ioutil" + "log" + "os" + + "github.com/saracen/go7z" + "github.com/zu1k/nali/pkg/common" +) + +func Download(filePath ...string) (data []byte, err error) { + data, err = getData() + if err != nil { + log.Printf("ZX IPv6数据库下载失败，请手动下载解压后保存到本地: %s \n", filePath) + log.Println("下载链接： https://ip.zxinc.org/ip.7z") + return + } + + if len(filePath) == 1 { + if err := common.SaveFile(filePath[0], data); err == nil { + log.Println("已将最新的 ZX IPv6数据库 保存到本地:", filePath) + } + } + return +} + +const ( + zx = "https://ip.zxinc.org/ip.7z" +) + +func getData() (data []byte, err error) { + data, err = common.GetHttpClient().Get(zx) + + file7z, err := os.CreateTemp("", "*") + if err != nil { + return nil, err + } + defer os.Remove(file7z.Name()) + if err := os.WriteFile(file7z.Name(), data, 0644); err == nil { + return Un7z(file7z.Name()) + } + return +} + +func Un7z(filePath string) (data []byte, err error) { + sz, err := go7z.OpenReader(filePath) + if err != nil { + return nil, err + } + defer sz.Close() + + fileNoNeed, err := os.CreateTemp("", "*") + if err != nil { + return nil, err + } + fileNeed, err := os.CreateTemp("", "*") + if err != nil { + return nil, err + } + + if err != nil { + return nil, err + } + for { + hdr, err := sz.Next() + if err == io.EOF { + break // IdxEnd of archive + } + if err != nil { + return nil, err + } + + if hdr.Name == "ipv6wry.db" { + if _, err := io.Copy(fileNeed, sz); err != nil { + log.Fatalln("ZX ipv6数据库解压出错：", err.Error()) + } + } else { + if _, err := io.Copy(fileNoNeed, sz); err != nil { + log.Fatalln("ZX ipv6数据库解压出错：", err.Error()) + } + } + } + err = fileNoNeed.Close() + if err != nil { + return nil, err + } + defer os.Remove(fileNoNeed.Name()) + defer os.Remove(fileNeed.Name()) + return ioutil.ReadFile(fileNeed.Name()) +} diff --git a/pkg/zxipv6wry/zxipv6wry.go b/pkg/zxipv6wry/zxipv6wry.go index cc9f21df..d9650f74 100644 --- a/pkg/zxipv6wry/zxipv6wry.go +++ b/pkg/zxipv6wry/zxipv6wry.go @@ -1,128 +1,87 @@ -package zxipv6wry - -import ( - "encoding/binary" - "errors" - "fmt" - "io/ioutil" - "log" - "math/big" - "net" - "os" - "strings" - - "github.com/zu1k/nali/pkg/common" -) - -type ZXwry struct { - common.IPDB -} - -func NewZXwry(filePath string) (*ZXwry, error) { - var fileData []byte - var fileInfo common.FileData - - _, err := os.Stat(filePath) - if err != nil && os.IsNotExist(err) { - log.Println("文件不存在，尝试从网络获取最新ZX IPv6数据库") - fileData, err = Download(filePath) - if err != nil { - return nil, err - } - } else { - fileInfo.FileBase, err = os.OpenFile(filePath, os.O_RDONLY, 0400) - if err != nil { - return nil, err - } - defer fileInfo.FileBase.Close() - - fileData, err = ioutil.ReadAll(fileInfo.FileBase) - if err != nil { - return nil, err - } - } - - fileInfo.Data = fileData - - return &ZXwry{ - IPDB: common.IPDB{ - Data: &fileInfo, - }, - }, nil -} - -func (db ZXwry) Find(query string, params ...string) (result fmt.Stringer, err error) { - ip := net.ParseIP(query) - if ip == nil { - return nil, errors.New("Query should be IPv6") - } - ip6 := ip.To16() - if ip6 == nil { - return nil, errors.New("Query should be IPv6") - } - - tp := big.NewInt(0) - op := big.NewInt(0) - tp.SetBytes(ip6) - op.SetString("18446744073709551616", 10) - op.Div(tp, op) - tp.SetString("FFFFFFFFFFFFFFFF", 16) - op.And(op, tp) - - ipv6 := op.Uint64() - offset := db.searchIndex(ipv6) - country, area := db.getAddr(offset) - - result = common.Result{ - Country: strings.ReplaceAll(country, " CZ88.NET", ""), - Area: strings.ReplaceAll(area, " CZ88.NET", ""), - } - return result, nil -} - -func (db *ZXwry) getAddr(offset uint32) (string, string) { - mode := db.ReadMode(offset) - if mode == common.RedirectMode1 { - offset = db.ReadUInt24() - return db.getAddr(offset) - } - realOffset := db.Offset - 1 - c1 := db.ReadArea(realOffset) - if mode == common.RedirectMode2 { - db.Offset = 4 + realOffset - } else { - db.Offset = realOffset + uint32(1+len(c1)) - } - c2 := db.ReadArea(db.Offset) - return string(c1), string(c2) -} - -func (db *ZXwry) searchIndex(ip uint64) uint32 { - header := db.ReadData(16, 8) - start := binary.LittleEndian.Uint32(header[8:]) - counts := binary.LittleEndian.Uint32(header[:8]) - end := start + counts*11 - - buf := make([]byte, 11) - - for { - mid := common.GetMiddleOffset(start, end, 11) - buf = db.ReadData(11, mid) - ipBytes := binary.LittleEndian.Uint64(buf[:8]) - - if end-start == 11 { - if ip >= binary.LittleEndian.Uint64(db.ReadData(8, end)) { - buf = db.ReadData(11, end) - } - return common.ByteToUInt32(buf[8:]) - } - - if ipBytes > ip { - end = mid - } else if ipBytes < ip { - start = mid - } else if ipBytes == ip { - return common.ByteToUInt32(buf[8:]) - } - } -} +package zxipv6wry + +import ( + "encoding/binary" + "errors" + "fmt" + "io" + "log" + "net" + "os" + + "github.com/zu1k/nali/pkg/wry" +) + +type ZXwry struct { + wry.IPDB[uint64] +} + +func NewZXwry(filePath string) (*ZXwry, error) { + var fileData []byte + + _, err := os.Stat(filePath) + if err != nil && os.IsNotExist(err) { + log.Println("文件不存在，尝试从网络获取最新ZX IPv6数据库") + fileData, err = Download(filePath) + if err != nil { + return nil, err + } + } else { + fileBase, err := os.OpenFile(filePath, os.O_RDONLY, 0400) + if err != nil { + return nil, err + } + defer fileBase.Close() + + fileData, err = io.ReadAll(fileBase) + if err != nil { + return nil, err + } + } + + if len(fileData) < 24 { + log.Fatalln("ZX IPv6数据库存在错误，请重新下载") + } + + header := fileData[:24] + offLen := header[6] + ipLen := header[7] + + start := binary.LittleEndian.Uint64(header[16:24]) + counts := binary.LittleEndian.Uint64(header[8:16]) + end := start + counts*11 + + if uint64(len(fileData)) < end { + log.Fatalln("ZX IPv6数据库存在错误，请重新下载") + } + + return &ZXwry{ + IPDB: wry.IPDB[uint64]{ + Data: fileData, + + OffLen: offLen, + IPLen: ipLen, + IPCnt: counts, + IdxStart: start, + IdxEnd: end, + }, + }, nil +} + +func (db *ZXwry) Find(query string, _ ...string) (result fmt.Stringer, err error) { + ip := net.ParseIP(query) + if ip == nil { + return nil, errors.New("query should be IPv6") + } + ip6 := ip.To16() + if ip6 == nil { + return nil, errors.New("query should be IPv6") + } + ip6 = ip6[:8] + ipu64 := binary.BigEndian.Uint64(ip6) + + offset := db.SearchIndexV6(ipu64) + reader := wry.NewReader(db.Data) + reader.Parse(offset) + return reader.Result, nil +}