Skip to content

Commit 61e1153

Browse files
hossinasaadiRPRX
hossinasaadi
authored and
RPRX
committed
MPH domian matcher: Support building & using cache directly (instead of building from geosite.dat when Xray starts) (XTLS#5505)
Like XTLS#5488 (comment)
1 parent afcfdbc commit 61e1153

18 files changed

Lines changed: 987 additions & 160 deletions

‎app/dns/dns.go‎

Lines changed: 27 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -12,9 +12,11 @@ import (
1212
"sync"
1313
"time"
1414

15+
"github.com/xtls/xray-core/app/router"
1516
"github.com/xtls/xray-core/common"
1617
"github.com/xtls/xray-core/common/errors"
1718
"github.com/xtls/xray-core/common/net"
19+
"github.com/xtls/xray-core/common/platform"
1820
"github.com/xtls/xray-core/common/session"
1921
"github.com/xtls/xray-core/common/strmatcher"
2022
"github.com/xtls/xray-core/features/dns"
@@ -83,9 +85,31 @@ func New(ctx context.Context, config *Config) (*DNS, error) {
8385
return nil, errors.New("unexpected query strategy ", config.QueryStrategy)
8486
}
8587

86-
hosts, err := NewStaticHosts(config.StaticHosts)
87-
if err != nil {
88-
return nil, errors.New("failed to create hosts").Base(err)
88+
var hosts *StaticHosts
89+
mphLoaded := false
90+
domainMatcherPath := platform.NewEnvFlag(platform.MphCachePath).GetValue(func() string { return "" })
91+
if domainMatcherPath != "" {
92+
if f, err := os.Open(domainMatcherPath); err == nil {
93+
defer f.Close()
94+
if m, err := router.LoadGeoSiteMatcher(f, "HOSTS"); err == nil {
95+
f.Seek(0, 0)
96+
if hostIPs, err := router.LoadGeoSiteHosts(f); err == nil {
97+
if sh, err := NewStaticHostsFromCache(m, hostIPs); err == nil {
98+
hosts = sh
99+
mphLoaded = true
100+
errors.LogDebug(ctx, "MphDomainMatcher loaded from cache for DNS hosts, size: ", sh.matchers.Size())
101+
}
102+
}
103+
}
104+
}
105+
}
106+
107+
if !mphLoaded {
108+
sh, err := NewStaticHosts(config.StaticHosts)
109+
if err != nil {
110+
return nil, errors.New("failed to create hosts").Base(err)
111+
}
112+
hosts = sh
89113
}
90114

91115
var clients []*Client

‎app/dns/hosts.go‎

Lines changed: 48 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ import (
1414
// StaticHosts represents static domain-ip mapping in DNS server.
1515
type StaticHosts struct {
1616
ips [][]net.Address
17-
matchers *strmatcher.MatcherGroup
17+
matchers strmatcher.IndexMatcher
1818
}
1919

2020
// NewStaticHosts creates a new StaticHosts instance.
@@ -124,3 +124,50 @@ func (h *StaticHosts) lookup(domain string, option dns.IPOption, maxDepth int) (
124124
func (h *StaticHosts) Lookup(domain string, option dns.IPOption) ([]net.Address, error) {
125125
return h.lookup(domain, option, 5)
126126
}
127+
func NewStaticHostsFromCache(matcher strmatcher.IndexMatcher, hostIPs map[string][]string) (*StaticHosts, error) {
128+
sh := &StaticHosts{
129+
ips: make([][]net.Address, matcher.Size()+1),
130+
matchers: matcher,
131+
}
132+
133+
order := hostIPs["_ORDER"]
134+
var offset uint32
135+
136+
img, ok := matcher.(*strmatcher.IndexMatcherGroup)
137+
if !ok {
138+
// Single matcher (e.g. only manual or only one geosite)
139+
if len(order) > 0 {
140+
pattern := order[0]
141+
ips := parseIPs(hostIPs[pattern])
142+
for i := uint32(1); i <= matcher.Size(); i++ {
143+
sh.ips[i] = ips
144+
}
145+
}
146+
return sh, nil
147+
}
148+
149+
for i, m := range img.Matchers {
150+
if i < len(order) {
151+
pattern := order[i]
152+
ips := parseIPs(hostIPs[pattern])
153+
for j := uint32(1); j <= m.Size(); j++ {
154+
sh.ips[offset+j] = ips
155+
}
156+
offset += m.Size()
157+
}
158+
}
159+
return sh, nil
160+
}
161+
162+
func parseIPs(raw []string) []net.Address {
163+
addrs := make([]net.Address, 0, len(raw))
164+
for _, s := range raw {
165+
if len(s) > 1 && s[0] == '#' {
166+
rcode, _ := strconv.Atoi(s[1:])
167+
addrs = append(addrs, dns.RCodeError(rcode))
168+
} else {
169+
addrs = append(addrs, net.ParseAddress(s))
170+
}
171+
}
172+
return addrs
173+
}

‎app/dns/hosts_test.go‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,12 @@
11
package dns_test
22

33
import (
4+
"bytes"
45
"testing"
56

67
"github.com/google/go-cmp/cmp"
78
. "github.com/xtls/xray-core/app/dns"
9+
"github.com/xtls/xray-core/app/router"
810
"github.com/xtls/xray-core/common"
911
"github.com/xtls/xray-core/common/net"
1012
"github.com/xtls/xray-core/features/dns"
@@ -130,3 +132,57 @@ func TestStaticHosts(t *testing.T) {
130132
}
131133
}
132134
}
135+
func TestStaticHostsFromCache(t *testing.T) {
136+
sites := []*router.GeoSite{
137+
{
138+
CountryCode: "cloudflare-dns.com",
139+
Domain: []*router.Domain{
140+
{Type: router.Domain_Full, Value: "example.com"},
141+
},
142+
},
143+
{
144+
CountryCode: "geosite:cn",
145+
Domain: []*router.Domain{
146+
{Type: router.Domain_Domain, Value: "baidu.cn"},
147+
},
148+
},
149+
}
150+
deps := map[string][]string{
151+
"HOSTS": {"cloudflare-dns.com", "geosite:cn"},
152+
}
153+
hostIPs := map[string][]string{
154+
"cloudflare-dns.com": {"1.1.1.1"},
155+
"geosite:cn": {"2.2.2.2"},
156+
"_ORDER": {"cloudflare-dns.com", "geosite:cn"},
157+
}
158+
159+
var buf bytes.Buffer
160+
err := router.SerializeGeoSiteList(sites, deps, hostIPs, &buf)
161+
common.Must(err)
162+
163+
// Load matcher
164+
m, err := router.LoadGeoSiteMatcher(bytes.NewReader(buf.Bytes()), "HOSTS")
165+
common.Must(err)
166+
167+
// Load hostIPs
168+
f := bytes.NewReader(buf.Bytes())
169+
hips, err := router.LoadGeoSiteHosts(f)
170+
common.Must(err)
171+
172+
hosts, err := NewStaticHostsFromCache(m, hips)
173+
common.Must(err)
174+
175+
{
176+
ips, _ := hosts.Lookup("example.com", dns.IPOption{IPv4Enable: true})
177+
if len(ips) != 1 || ips[0].String() != "1.1.1.1" {
178+
t.Error("failed to lookup example.com from cache")
179+
}
180+
}
181+
182+
{
183+
ips, _ := hosts.Lookup("baidu.cn", dns.IPOption{IPv4Enable: true})
184+
if len(ips) != 1 || ips[0].String() != "2.2.2.2" {
185+
t.Error("failed to lookup baidu.cn from cache deps")
186+
}
187+
}
188+
}

‎app/dns/nameserver.go‎

Lines changed: 53 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -10,13 +10,27 @@ import (
1010
"github.com/xtls/xray-core/app/router"
1111
"github.com/xtls/xray-core/common/errors"
1212
"github.com/xtls/xray-core/common/net"
13+
"github.com/xtls/xray-core/common/platform"
14+
"github.com/xtls/xray-core/common/platform/filesystem"
1315
"github.com/xtls/xray-core/common/session"
1416
"github.com/xtls/xray-core/common/strmatcher"
1517
"github.com/xtls/xray-core/core"
1618
"github.com/xtls/xray-core/features/dns"
1719
"github.com/xtls/xray-core/features/routing"
1820
)
1921

22+
type mphMatcherWrapper struct {
23+
m strmatcher.IndexMatcher
24+
}
25+
26+
func (w *mphMatcherWrapper) Match(s string) bool {
27+
return w.m.Match(s) != nil
28+
}
29+
30+
func (w *mphMatcherWrapper) String() string {
31+
return "mph-matcher"
32+
}
33+
2034
// Server is the interface for Name Server.
2135
type Server interface {
2236
// Name of the Client.
@@ -132,29 +146,50 @@ func NewClient(
132146
var rules []string
133147
ruleCurr := 0
134148
ruleIter := 0
135-
for i, domain := range ns.PrioritizedDomain {
136-
ns.PrioritizedDomain[i] = nil
137-
domainRule, err := toStrMatcher(domain.Type, domain.Domain)
138-
if err != nil {
139-
errors.LogErrorInner(ctx, err, "failed to create domain matcher, ignore domain rule [type: ", domain.Type, ", domain: ", domain.Domain, "]")
140-
domainRule, _ = toStrMatcher(DomainMatchingType_Full, "hack.fix.index.for.illegal.domain.rule")
149+
150+
// Check if domain matcher cache is provided via environment
151+
domainMatcherPath := platform.NewEnvFlag(platform.MphCachePath).GetValue(func() string { return "" })
152+
var mphLoaded bool
153+
154+
if domainMatcherPath != "" && ns.Tag != "" {
155+
f, err := filesystem.NewFileReader(domainMatcherPath)
156+
if err == nil {
157+
defer f.Close()
158+
g, err := router.LoadGeoSiteMatcher(f, ns.Tag)
159+
if err == nil {
160+
errors.LogDebug(ctx, "MphDomainMatcher loaded from cache for ", ns.Tag, " dns tag)")
161+
updateDomainRule(&mphMatcherWrapper{m: g}, 0, *matcherInfos)
162+
rules = append(rules, "[MPH Cache]")
163+
mphLoaded = true
164+
}
141165
}
142-
originalRuleIdx := ruleCurr
143-
if ruleCurr < len(ns.OriginalRules) {
144-
rule := ns.OriginalRules[ruleCurr]
145-
if ruleCurr >= len(rules) {
146-
rules = append(rules, rule.Rule)
166+
}
167+
168+
if !mphLoaded {
169+
for i, domain := range ns.PrioritizedDomain {
170+
ns.PrioritizedDomain[i] = nil
171+
domainRule, err := toStrMatcher(domain.Type, domain.Domain)
172+
if err != nil {
173+
errors.LogErrorInner(ctx, err, "failed to create domain matcher, ignore domain rule [type: ", domain.Type, ", domain: ", domain.Domain, "]")
174+
domainRule, _ = toStrMatcher(DomainMatchingType_Full, "hack.fix.index.for.illegal.domain.rule")
147175
}
148-
ruleIter++
149-
if ruleIter >= int(rule.Size) {
150-
ruleIter = 0
176+
originalRuleIdx := ruleCurr
177+
if ruleCurr < len(ns.OriginalRules) {
178+
rule := ns.OriginalRules[ruleCurr]
179+
if ruleCurr >= len(rules) {
180+
rules = append(rules, rule.Rule)
181+
}
182+
ruleIter++
183+
if ruleIter >= int(rule.Size) {
184+
ruleIter = 0
185+
ruleCurr++
186+
}
187+
} else { // No original rule, generate one according to current domain matcher (majorly for compatibility with tests)
188+
rules = append(rules, domainRule.String())
151189
ruleCurr++
152190
}
153-
} else { // No original rule, generate one according to current domain matcher (majorly for compatibility with tests)
154-
rules = append(rules, domainRule.String())
155-
ruleCurr++
191+
updateDomainRule(domainRule, originalRuleIdx, *matcherInfos)
156192
}
157-
updateDomainRule(domainRule, originalRuleIdx, *matcherInfos)
158193
}
159194
ns.PrioritizedDomain = nil
160195
runtime.GC()

‎app/router/condition.go‎

Lines changed: 31 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ package router
22

33
import (
44
"context"
5+
"io"
56
"os"
67
"path/filepath"
78
"regexp"
@@ -52,7 +53,34 @@ var matcherTypeMap = map[Domain_Type]strmatcher.Type{
5253
}
5354

5455
type DomainMatcher struct {
55-
matchers strmatcher.IndexMatcher
56+
Matchers strmatcher.IndexMatcher
57+
}
58+
59+
func SerializeDomainMatcher(domains []*Domain, w io.Writer) error {
60+
61+
g := strmatcher.NewMphMatcherGroup()
62+
for _, d := range domains {
63+
matcherType, f := matcherTypeMap[d.Type]
64+
if !f {
65+
continue
66+
}
67+
68+
_, err := g.AddPattern(d.Value, matcherType)
69+
if err != nil {
70+
return err
71+
}
72+
}
73+
g.Build()
74+
// serialize
75+
return g.Serialize(w)
76+
}
77+
78+
func NewDomainMatcherFromBuffer(data []byte) (*strmatcher.MphMatcherGroup, error) {
79+
matcher, err := strmatcher.NewMphMatcherGroupFromBuffer(data)
80+
if err != nil {
81+
return nil, err
82+
}
83+
return matcher, nil
5684
}
5785

5886
func NewMphMatcherGroup(domains []*Domain) (*DomainMatcher, error) {
@@ -72,12 +100,12 @@ func NewMphMatcherGroup(domains []*Domain) (*DomainMatcher, error) {
72100
}
73101
g.Build()
74102
return &DomainMatcher{
75-
matchers: g,
103+
Matchers: g,
76104
}, nil
77105
}
78106

79107
func (m *DomainMatcher) ApplyDomain(domain string) bool {
80-
return len(m.matchers.Match(strings.ToLower(domain))) > 0
108+
return len(m.Matchers.Match(strings.ToLower(domain))) > 0
81109
}
82110

83111
// Apply implements Condition.

0 commit comments

Comments
 (0)