chore: support ETag for update geo

adapter/provider/parser.go

@@ -111,7 +111,7 @@ func ParseProxyProvider(name string, mapping map[string]any) (types.ProxyProvide
 				return nil, fmt.Errorf("%w: %s", errSubPath, path)
 			}
 		}
-		vehicle = resource.NewHTTPVehicle(schema.URL, path, schema.Proxy, schema.Header)
+		vehicle = resource.NewHTTPVehicle(schema.URL, path, schema.Proxy, schema.Header, resource.DefaultHttpTimeout)
 	default:
 		return nil, fmt.Errorf("%w: %s", errVehicleType, schema.Type)
 	}

component/resource/fetcher.go

@@ -3,7 +3,6 @@ package resource
 import (
 	"context"
 	"os"
-	"path/filepath"
 	"time"
 
 	types "github.com/metacubex/mihomo/constant/provider"
@@ -13,11 +12,6 @@ import (
 	"github.com/samber/lo"
 )
 
-var (
-	fileMode os.FileMode = 0o666
-	dirMode  os.FileMode = 0o755
-)
-
 type Parser[V any] func([]byte) (V, error)
 
 type Fetcher[V any] struct {
@@ -118,7 +112,7 @@ func (f *Fetcher[V]) loadBuf(buf []byte, hash types.HashType, updateFile bool) (
 	}
 
 	if updateFile {
-		if err = safeWrite(f.vehicle.Path(), buf); err != nil {
+		if err = f.vehicle.Write(buf); err != nil {
 			return lo.Empty[V](), false, err
 		}
 	}
@@ -205,18 +199,6 @@ func (f *Fetcher[V]) updateWithLog() {
 	return
 }
 
-func safeWrite(path string, buf []byte) error {
-	dir := filepath.Dir(path)
-
-	if _, err := os.Stat(dir); os.IsNotExist(err) {
-		if err := os.MkdirAll(dir, dirMode); err != nil {
-			return err
-		}
-	}
-
-	return os.WriteFile(path, buf, fileMode)
-}
-
 func NewFetcher[V any](name string, interval time.Duration, vehicle types.Vehicle, parser Parser[V], onUpdate func(V)) *Fetcher[V] {
 	ctx, cancel := context.WithCancel(context.Background())
 	return &Fetcher[V]{

component/resource/vehicle.go

@@ -6,13 +6,33 @@ import (
 	"io"
 	"net/http"
 	"os"
+	"path/filepath"
 	"time"
 
 	mihomoHttp "github.com/metacubex/mihomo/component/http"
 	"github.com/metacubex/mihomo/component/profile/cachefile"
 	types "github.com/metacubex/mihomo/constant/provider"
 )
 
+const (
+	DefaultHttpTimeout = time.Second * 20
+
+	fileMode os.FileMode = 0o666
+	dirMode  os.FileMode = 0o755
+)
+
+func safeWrite(path string, buf []byte) error {
+	dir := filepath.Dir(path)
+
+	if _, err := os.Stat(dir); os.IsNotExist(err) {
+		if err := os.MkdirAll(dir, dirMode); err != nil {
+			return err
+		}
+	}
+
+	return os.WriteFile(path, buf, fileMode)
+}
+
 type FileVehicle struct {
 	path string
 }
@@ -42,15 +62,20 @@ func (f *FileVehicle) Proxy() string {
 	return ""
 }
 
+func (f *FileVehicle) Write(buf []byte) error {
+	return safeWrite(f.path, buf)
+}
+
 func NewFileVehicle(path string) *FileVehicle {
 	return &FileVehicle{path: path}
 }
 
 type HTTPVehicle struct {
-	url    string
-	path   string
-	proxy  string
-	header http.Header
+	url     string
+	path    string
+	proxy   string
+	header  http.Header
+	timeout time.Duration
 }
 
 func (h *HTTPVehicle) Url() string {
@@ -69,8 +94,12 @@ func (h *HTTPVehicle) Proxy() string {
 	return h.proxy
 }
 
+func (h *HTTPVehicle) Write(buf []byte) error {
+	return safeWrite(h.path, buf)
+}
+
 func (h *HTTPVehicle) Read(ctx context.Context, oldHash types.HashType) (buf []byte, hash types.HashType, err error) {
-	ctx, cancel := context.WithTimeout(ctx, time.Second*20)
+	ctx, cancel := context.WithTimeout(ctx, h.timeout)
 	defer cancel()
 	header := h.header
 	setIfNoneMatch := false
@@ -107,11 +136,12 @@ func (h *HTTPVehicle) Read(ctx context.Context, oldHash types.HashType) (buf []b
 	return
 }
 
-func NewHTTPVehicle(url string, path string, proxy string, header http.Header) *HTTPVehicle {
+func NewHTTPVehicle(url string, path string, proxy string, header http.Header, timeout time.Duration) *HTTPVehicle {
 	return &HTTPVehicle{
-		url:    url,
-		path:   path,
-		proxy:  proxy,
-		header: header,
+		url:     url,
+		path:    path,
+		proxy:   proxy,
+		header:  header,
+		timeout: timeout,
 	}
 }

component/updater/update_geo.go

@@ -13,7 +13,9 @@ import (
 	"github.com/metacubex/mihomo/component/geodata"
 	_ "github.com/metacubex/mihomo/component/geodata/standard"
 	"github.com/metacubex/mihomo/component/mmdb"
+	"github.com/metacubex/mihomo/component/resource"
 	C "github.com/metacubex/mihomo/constant"
+	P "github.com/metacubex/mihomo/constant/provider"
 	"github.com/metacubex/mihomo/log"
 
 	"github.com/oschwald/maxminddb-golang"
@@ -43,73 +45,122 @@ func SetGeoUpdateInterval(newGeoUpdateInterval int) {
 }
 
 func UpdateMMDB() (err error) {
-	defer mmdb.ReloadIP()
-	data, err := downloadForBytes(geodata.MmdbUrl())
+	vehicle := resource.NewHTTPVehicle(geodata.MmdbUrl(), C.Path.MMDB(), "", nil, defaultHttpTimeout)
+	var oldHash P.HashType
+	if buf, err := os.ReadFile(vehicle.Path()); err == nil {
+		oldHash = P.MakeHash(buf)
+	}
+	data, hash, err := vehicle.Read(context.Background(), oldHash)
 	if err != nil {
 		return fmt.Errorf("can't download MMDB database file: %w", err)
 	}
+	if oldHash.Equal(hash) { // same hash, ignored
+		return nil
+	}
+	if len(data) == 0 {
+		return fmt.Errorf("can't download MMDB database file: no data")
+	}
+
 	instance, err := maxminddb.FromBytes(data)
 	if err != nil {
 		return fmt.Errorf("invalid MMDB database file: %s", err)
 	}
 	_ = instance.Close()
 
+	defer mmdb.ReloadIP()
 	mmdb.IPInstance().Reader.Close() //  mmdb is loaded with mmap, so it needs to be closed before overwriting the file
-	if err = saveFile(data, C.Path.MMDB()); err != nil {
+	if err = vehicle.Write(data); err != nil {
 		return fmt.Errorf("can't save MMDB database file: %w", err)
 	}
 	return nil
 }
 
 func UpdateASN() (err error) {
-	defer mmdb.ReloadASN()
-	data, err := downloadForBytes(geodata.ASNUrl())
+	vehicle := resource.NewHTTPVehicle(geodata.ASNUrl(), C.Path.ASN(), "", nil, defaultHttpTimeout)
+	var oldHash P.HashType
+	if buf, err := os.ReadFile(vehicle.Path()); err == nil {
+		oldHash = P.MakeHash(buf)
+	}
+	data, hash, err := vehicle.Read(context.Background(), oldHash)
 	if err != nil {
 		return fmt.Errorf("can't download ASN database file: %w", err)
 	}
+	if oldHash.Equal(hash) { // same hash, ignored
+		return nil
+	}
+	if len(data) == 0 {
+		return fmt.Errorf("can't download ASN database file: no data")
+	}
 
 	instance, err := maxminddb.FromBytes(data)
 	if err != nil {
 		return fmt.Errorf("invalid ASN database file: %s", err)
 	}
 	_ = instance.Close()
 
+	defer mmdb.ReloadASN()
 	mmdb.ASNInstance().Reader.Close() //  mmdb is loaded with mmap, so it needs to be closed before overwriting the file
-	if err = saveFile(data, C.Path.ASN()); err != nil {
+	if err = vehicle.Write(data); err != nil {
 		return fmt.Errorf("can't save ASN database file: %w", err)
 	}
 	return nil
 }
 
 func UpdateGeoIp() (err error) {
-	defer geodata.ClearGeoIPCache()
 	geoLoader, err := geodata.GetGeoDataLoader("standard")
-	data, err := downloadForBytes(geodata.GeoIpUrl())
+
+	vehicle := resource.NewHTTPVehicle(geodata.GeoIpUrl(), C.Path.GeoIP(), "", nil, defaultHttpTimeout)
+	var oldHash P.HashType
+	if buf, err := os.ReadFile(vehicle.Path()); err == nil {
+		oldHash = P.MakeHash(buf)
+	}
+	data, hash, err := vehicle.Read(context.Background(), oldHash)
 	if err != nil {
 		return fmt.Errorf("can't download GeoIP database file: %w", err)
 	}
+	if oldHash.Equal(hash) { // same hash, ignored
+		return nil
+	}
+	if len(data) == 0 {
+		return fmt.Errorf("can't download GeoIP database file: no data")
+	}
+
 	if _, err = geoLoader.LoadIPByBytes(data, "cn"); err != nil {
 		return fmt.Errorf("invalid GeoIP database file: %s", err)
 	}
-	if err = saveFile(data, C.Path.GeoIP()); err != nil {
+
+	defer geodata.ClearGeoIPCache()
+	if err = vehicle.Write(data); err != nil {
 		return fmt.Errorf("can't save GeoIP database file: %w", err)
 	}
 	return nil
 }
 
 func UpdateGeoSite() (err error) {
-	defer geodata.ClearGeoSiteCache()
 	geoLoader, err := geodata.GetGeoDataLoader("standard")
-	data, err := downloadForBytes(geodata.GeoSiteUrl())
+
+	vehicle := resource.NewHTTPVehicle(geodata.GeoSiteUrl(), C.Path.GeoSite(), "", nil, defaultHttpTimeout)
+	var oldHash P.HashType
+	if buf, err := os.ReadFile(vehicle.Path()); err == nil {
+		oldHash = P.MakeHash(buf)
+	}
+	data, hash, err := vehicle.Read(context.Background(), oldHash)
 	if err != nil {
 		return fmt.Errorf("can't download GeoSite database file: %w", err)
 	}
+	if oldHash.Equal(hash) { // same hash, ignored
+		return nil
+	}
+	if len(data) == 0 {
+		return fmt.Errorf("can't download GeoSite database file: no data")
+	}
 
 	if _, err = geoLoader.LoadSiteByBytes(data, "cn"); err != nil {
 		return fmt.Errorf("invalid GeoSite database file: %s", err)
 	}
 
-	if err = saveFile(data, C.Path.GeoSite()); err != nil {
+	defer geodata.ClearGeoSiteCache()
+	if err = vehicle.Write(data); err != nil {
 		return fmt.Errorf("can't save GeoSite database file: %w", err)
 	}
 	return nil

component/updater/update_ui.go

@@ -17,12 +17,12 @@ import (
 var (
 	ExternalUIURL  string
 	ExternalUIPath string
-	AutoUpdateUI   bool
+	AutoDownloadUI bool
 )
 
 var xdMutex sync.Mutex
 
-func UpdateUI() error {
+func DownloadUI() error {
 	xdMutex.Lock()
 	defer xdMutex.Unlock()
 

component/updater/utils.go

@@ -13,8 +13,10 @@ import (
 	"golang.org/x/exp/constraints"
 )
 
+const defaultHttpTimeout = time.Second * 90
+
 func downloadForBytes(url string) ([]byte, error) {
-	ctx, cancel := context.WithTimeout(context.Background(), time.Second*90)
+	ctx, cancel := context.WithTimeout(context.Background(), defaultHttpTimeout)
 	defer cancel()
 	resp, err := mihomoHttp.HttpRequest(ctx, url, http.MethodGet, nil, nil)
 	if err != nil {

config/config.go

@@ -701,7 +701,7 @@ func parseGeneral(cfg *RawConfig) (*General, error) {
 
 	// checkout externalUI exist
 	if cfg.ExternalUI != "" {
-		updater.AutoUpdateUI = true
+		updater.AutoDownloadUI = true
 		updater.ExternalUIPath = C.Path.Resolve(cfg.ExternalUI)
 	} else {
 		// default externalUI path
@@ -710,7 +710,7 @@ func parseGeneral(cfg *RawConfig) (*General, error) {
 
 	// checkout UIpath/name exist
 	if cfg.ExternalUIName != "" {
-		updater.AutoUpdateUI = true
+		updater.AutoDownloadUI = true
 		updater.ExternalUIPath = path.Join(updater.ExternalUIPath, cfg.ExternalUIName)
 	}
 

constant/provider/interface.go

@@ -33,6 +33,7 @@ func (v VehicleType) String() string {
 
 type Vehicle interface {
 	Read(ctx context.Context, oldHash HashType) (buf []byte, hash HashType, err error)
+	Write(buf []byte) error
 	Path() string
 	Url() string
 	Proxy() string

hub/executor/executor.go

@@ -381,13 +381,13 @@ func updateTunnels(tunnels []LC.Tunnel) {
 }
 
 func initExternalUI() {
-	if updater.AutoUpdateUI {
+	if updater.AutoDownloadUI {
 		dirEntries, _ := os.ReadDir(updater.ExternalUIPath)
 		if len(dirEntries) > 0 {
 			log.Infoln("UI already exists, skip downloading")
 		} else {
 			log.Infoln("External UI downloading ...")
-			updater.UpdateUI()
+			updater.DownloadUI()
 		}
 	}
 }

hub/route/upgrade.go

@@ -47,7 +47,7 @@ func upgradeCore(w http.ResponseWriter, r *http.Request) {
 }
 
 func updateUI(w http.ResponseWriter, r *http.Request) {
-	err := updater.UpdateUI()
+	err := updater.DownloadUI()
 	if err != nil {
 		log.Warnln("%s", err)
 		render.Status(r, http.StatusInternalServerError)

rules/provider/parse.go

@@ -53,7 +53,7 @@ func ParseRuleProvider(name string, mapping map[string]interface{}, parse func(t
 				return nil, fmt.Errorf("%w: %s", errSubPath, path)
 			}
 		}
-		vehicle = resource.NewHTTPVehicle(schema.URL, path, schema.Proxy, nil)
+		vehicle = resource.NewHTTPVehicle(schema.URL, path, schema.Proxy, nil, resource.DefaultHttpTimeout)
 	default:
 		return nil, fmt.Errorf("unsupported vehicle type: %s", schema.Type)
 	}