Compare commits

...

16 Commits

Author SHA1 Message Date
MkQtS
9d73db400e Feat: add support for partial include 2025-12-31 21:42:35 +08:00
MkQtS
c83bb86d06 Refactor: promote refMap 2025-12-31 20:30:16 +08:00
MkQtS
58f79fa579 Remove support for partial include
This reverts e640ac2783

It is problematic and I will implement a new one
2025-12-31 20:29:54 +08:00
MkQtS
969e6baad8 Refactor exportPlainTextList
- remove unnecessary variable
- improve readablity
2025-12-31 20:25:58 +08:00
MkQtS
bbd5b64219 Refactor parseEntry
- add value/attribute checker(check missing space)
- allow multiple spaces
- sort attributes
- improve readablity
2025-12-31 20:24:51 +08:00
MkQtS
6b10d69246 Refactor: reduce the use of strings.TrimSpace 2025-12-31 20:14:14 +08:00
bafometik
fd4895c71e Update google-ads (#3115) 2025-12-31 11:59:52 +08:00
jinqiang zhang
f105486f25 hetzner: add hetzner-cloud.de (#3114) 2025-12-30 17:26:25 +08:00
MkQtS
75f8ac0b8a chinaunicom: add 10155.com (#3113) 2025-12-29 20:58:28 +08:00
深鸣
12711270a8 connectivity-check: add redirect.archlinux.org (#3112) 2025-12-29 00:26:26 +08:00
深鸣
6ab293c1dc geolocation-!cn: add dmhy.org (#3111) 2025-12-28 18:17:21 +08:00
深鸣
f050db51f0 archlinux: add archlinuxcn.org (#3110) 2025-12-28 18:15:13 +08:00
深鸣
645014d31e category-dev: add godbolt.org (#3109) 2025-12-28 18:12:18 +08:00
TripleA
4b1cef4329 connectivity-check: add connectivity-check.ubuntu.com (#3108)
and sort
2025-12-27 21:57:59 +08:00
MkQtS
234cc35db2 Add more cn domains (#3107)
* cctv: add more domains

* chinamobile: add more domains

* add hikvision

* category-education-cn: add more domains

* category-entertainment-cn: add aisee.tv

* category-media-cn: add more domains

* geolocation-cn: add more domains
2025-12-26 20:06:02 +08:00
a052
9b1f374455 futu: add @cn attribute for some domains (#3105)
and add futuhkapp.com
2025-12-23 18:32:15 +08:00
16 changed files with 232 additions and 200 deletions

View File

@@ -1,2 +1,5 @@
archlinux.org archlinux.org
archlinuxarm.org archlinuxarm.org
# Unofficial
archlinuxcn.org

View File

@@ -91,6 +91,7 @@ getcomposer.org
getzola.org getzola.org
git-scm.com git-scm.com
gnu.org gnu.org
godbolt.org
# PuTTY and some software official websites. # PuTTY and some software official websites.
greenend.org.uk greenend.org.uk

View File

@@ -83,8 +83,17 @@ dearedu.com
dgjy.net dgjy.net
# 在职研究生教育信息网 # 在职研究生教育信息网
eduei.com eduei.com
# 中公
eoffcn.com
jinrongren.net
offcn.com
zggqzp.com
zgsydw.com
# 高考100 # 高考100
gk100.com gk100.com
# 华图
huatu.com
sydw8.com
# 中科大数据研究院豫ICP备19023555号 # 中科大数据研究院豫ICP备19023555号
ictbda.cn ictbda.cn
ictbda.com ictbda.com

View File

@@ -48,6 +48,8 @@ yeshen.com
17k.com 17k.com
# https://github.com/supzhang/epg # https://github.com/supzhang/epg
51zmt.top 51zmt.top
# 广东南方新媒体
aisee.tv
# 暴风影音 # 暴风影音
baofeng.com baofeng.com
baofeng.net baofeng.net

View File

@@ -147,6 +147,8 @@ ql1d.com
qingdaonews.com qingdaonews.com
# 前瞻网 # 前瞻网
qianzhan.com qianzhan.com
# 青岛之窗
qingdaochina.org
# 四川手机报 # 四川手机报
scsjb.cn scsjb.cn
# 安全内参 # 安全内参
@@ -157,6 +159,8 @@ sfccn.com
shangyexinzhi.com shangyexinzhi.com
# 奇客资讯网 # 奇客资讯网
solidot.org solidot.org
# 南方新闻网
southcn.com
# 少数派 # 少数派
sspai.com sspai.com
# 证券之星 # 证券之星

View File

@@ -1,22 +1,24 @@
cctv.cn cctv.cn
cctv.com cctv.com
cctvlib.cn
cctvlib.com.cn
cctvlibrary.cn
cctvlibrary.com.cn
cctvpic.com cctvpic.com
cctvpro.cn
cctvpro.com.cn
chinaepg.cn chinaepg.cn
chinalive.com chinalive.com
chinaott.com
citv.net.cn citv.net.cn
cnms.net.cn cnms.net.cn
cntv.cn cntv.cn
cntv.com.cn cntv.com.cn
cctvlib.cn
cctvlibrary.cn
cctvlib.com.cn
cctvlibrary.com.cn
cctvpro.cn
cctvpro.com.cn
cntvwb.cn cntvwb.cn
gjgbdszt.cn gjgbdszt.cn
gjgbdszt.com.cn gjgbdszt.com.cn
gjgbdszt.net.cn gjgbdszt.net.cn
icntvcdn.com
ipanda.cn ipanda.cn
ipanda.com ipanda.com
ipanda.com.cn ipanda.com.cn
@@ -24,11 +26,12 @@ ipanda.net
livechina.cn livechina.cn
livechina.com livechina.com
olympicchannelchina.cn olympicchannelchina.cn
ottcn.com
tvcc.cn tvcc.cn
tvcc.com.cn tvcc.com.cn
xn--fiq53l6wcx3kp9bc7joo6apn8a.cn # 中国广播电视总台.cn xn--fiq53l6wcx3kp9bc7joo6apn8a.cn # 中国广播电视总台.cn
xn--fiq53l90et9fpncc7joo6apn8a.cn # 中央广播电视总台.cn
xn--fiq53l6wcx3kp9bc7joo6apn8a.xn--fiqs8s # 中国广播电视总台.中国 xn--fiq53l6wcx3kp9bc7joo6apn8a.xn--fiqs8s # 中国广播电视总台.中国
xn--fiq53l90et9fpncc7joo6apn8a.cn # 中央广播电视总台.cn
xn--kprv4ewxfr9cpxcc7joo6apn8a.cn # 国家广播电视总台.cn xn--kprv4ewxfr9cpxcc7joo6apn8a.cn # 国家广播电视总台.cn
xn--kprv4ewxfr9cpxcc7joo6apn8a.xn--fiqs8s # 国家广播电视总台.中国 xn--kprv4ewxfr9cpxcc7joo6apn8a.xn--fiqs8s # 国家广播电视总台.中国
yangshipin.cn yangshipin.cn

View File

@@ -6,9 +6,11 @@ speedtest.hk.chinamobile.com @!cn
speedtestbb.hk.chinamobile.com @!cn speedtestbb.hk.chinamobile.com @!cn
# Migu # Migu
migucloud.com cmread.com
migu.cn
cmvideo.cn cmvideo.cn
migu.cn
migucloud.com
migufun.com
miguvideo.com miguvideo.com
# 中移互联 # 中移互联

View File

@@ -6,6 +6,7 @@ xn--8y0a063a
10010.com 10010.com
10010.com.cn 10010.com.cn
10155.com
114menhu.com 114menhu.com
169ol.com 169ol.com
chinaunicom.cn chinaunicom.cn

View File

@@ -1,11 +1,9 @@
# ArchLinux
ping.archlinux.org
# Apple # Apple
captive.apple.com captive.apple.com
# Android # Arch Linux
connectivitycheck.gstatic.com full:ping.archlinux.org
full:redirect.archlinux.org # Canonical Name of ping.archlinux.org
# Cloudflare # Cloudflare
cp.cloudflare.com cp.cloudflare.com
@@ -16,6 +14,9 @@ network-test.debian.org
# Firefox # Firefox
detectportal.firefox.com detectportal.firefox.com
# Google
connectivitycheck.gstatic.com
# Honor # Honor
full:connectivitycheck.platform.hihonorcloud.com full:connectivitycheck.platform.hihonorcloud.com
@@ -28,6 +29,9 @@ networkcheck.kde.org
# MIUI # MIUI
full:connect.rom.miui.com full:connect.rom.miui.com
# Ubuntu
full:connectivity-check.ubuntu.com
# Windows # Windows
msftconnecttest.com msftconnecttest.com
msftncsi.com msftncsi.com

View File

@@ -1,12 +1,13 @@
# 富途证券 # 富途证券
futuesop.com futuesop.com @cn
futufin.com futufin.com
futuhk.com futuhk.com
futuhkapp.com
futuhn.com futuhn.com
futuholdings.com futuholdings.com
futuniuniu.com futuniuniu.com @cn
futunn.com futunn.com @cn
futustatic.com futustatic.com @cn
# 富途 moomoo 证券 # 富途 moomoo 证券
moomoo.com moomoo.com

View File

@@ -291,6 +291,7 @@ include:demonoid
include:nyaa include:nyaa
include:piratebay include:piratebay
include:rarbg include:rarbg
dmhy.org
# User scripts # User scripts
greasyfork.org greasyfork.org

View File

@@ -334,6 +334,8 @@ bblmw.com
## BOSS 直聘 ## BOSS 直聘
bosszhipin.com bosszhipin.com
zhipin.com zhipin.com
## 天润融通 AI外呼 客服
clink.cn
## 当贝 ## 当贝
dangbei.com dangbei.com
dangbei.net dangbei.net
@@ -400,6 +402,7 @@ include:dingdatech # 叮哒出行(杭州金通互联科技有限公司)
include:dji include:dji
include:gree include:gree
include:haier include:haier
include:hikvision
include:honor include:honor
include:huawei include:huawei
include:hupun # 杭州湖畔网络技术有限公司 include:hupun # 杭州湖畔网络技术有限公司
@@ -428,12 +431,17 @@ include:zte
51togic.com # 泰捷WEBOX 51togic.com # 泰捷WEBOX
avlyun.com # 安天移动安全 avlyun.com # 安天移动安全
dreame.tech # 追觅科技
hisensehitachi.com # 海信日立官网 hisensehitachi.com # 海信日立官网
insta360.com # 影石Insta360
jamcz.com # 晨钟网络科技 jamcz.com # 晨钟网络科技
jmgo.com # 坚果投影
mediastory.cc # 闪电分镜
roborock.com # 石头科技 roborock.com # 石头科技
socpk.com # 极客湾移动芯片排行 socpk.com # 极客湾移动芯片排行
suzuki-china.com # 铃木中国官网 suzuki-china.com # 铃木中国官网
udesk.cn # 沃丰科技 udesk.cn # 沃丰科技
ysjf.com # 影视飓风
# 稿定(厦门)信息服务有限公司 # 稿定(厦门)信息服务有限公司
## 花瓣网 闽ICP备2021013850号 ## 花瓣网 闽ICP备2021013850号
@@ -637,7 +645,6 @@ aiops.com
aipai.com aipai.com
airchangan.com airchangan.com
airguilin.com airguilin.com
aisee.tv # 粤ICP备10217959号-12
aiskycn.com aiskycn.com
aizhan.com aizhan.com
aldwx.com # 京ICP备17030349号-1 aldwx.com # 京ICP备17030349号-1
@@ -933,7 +940,6 @@ houdao.com
houyuantuan.com houyuantuan.com
hteacher.net hteacher.net
huashengdaili.com huashengdaili.com
huatu.com
huize.com huize.com
hujia.org hujia.org
huochepiao.com huochepiao.com
@@ -984,7 +990,6 @@ jiazhao.com
jiemeng8.com jiemeng8.com
jikexueyuan.com jikexueyuan.com
jinianbi.com jinianbi.com
jinrongren.net
jinse.com jinse.com
jinti.com jinti.com
jisuxia.com jisuxia.com
@@ -1127,7 +1132,6 @@ nxog.top # 湘ICP备2022006601号-1
nzbdw.com nzbdw.com
oadz.com oadz.com
oeeee.com oeeee.com
offcn.com
officese.com officese.com
oicq88.com oicq88.com
okooo.com okooo.com
@@ -1140,7 +1144,6 @@ onlinedown.net
onlinesjtu.com onlinesjtu.com
onlylady.com onlylady.com
opdown.com opdown.com
ottcn.com # 津ICP备12004891号-2
oupeng.com oupeng.com
p2peye.com p2peye.com
p5w.net p5w.net
@@ -1172,7 +1175,6 @@ qiaobutang.com
qie.tv qie.tv
qifeiye.com qifeiye.com
qijiayoudao.net qijiayoudao.net
qingdaochina.org
qingsongchou.com qingsongchou.com
qiumibao.com qiumibao.com
qizuang.com qizuang.com

View File

@@ -7,11 +7,13 @@ adsense.com @ads
adsensecustomsearchads.com @ads adsensecustomsearchads.com @ads
adsenseformobileapps.com @ads adsenseformobileapps.com @ads
adservice.google.com @ads adservice.google.com @ads
adtrafficquality.google @ads
advertisercommunity.com @ads advertisercommunity.com @ads
advertiserscommunity.com @ads advertiserscommunity.com @ads
adwords-community.com @ads adwords-community.com @ads
adwords.com @ads adwords.com @ads
adwordsexpress.com @ads adwordsexpress.com @ads
analytics-alv.google.com @ads
app-analytics-services.com @ads app-analytics-services.com @ads
app-analytics-services-att.com @ads app-analytics-services-att.com @ads
app-measurement-cn.com @ads app-measurement-cn.com @ads
@@ -20,6 +22,7 @@ doubleclick-cn.net @ads
doubleclick.cn @ads doubleclick.cn @ads
doubleclick.com @ads doubleclick.com @ads
doubleclick.net @ads doubleclick.net @ads
fcmatch.google.com @ads
google-analytics-cn.com @ads google-analytics-cn.com @ads
google-analytics.com @ads google-analytics.com @ads
googleadapis.com @ads googleadapis.com @ads

View File

@@ -1,4 +1,5 @@
first-ns.de first-ns.de
hetzner-cloud.de
hetzner.cloud hetzner.cloud
hetzner.com hetzner.com
hetzner.company hetzner.company

11
data/hikvision Normal file
View File

@@ -0,0 +1,11 @@
# 海康威视
hik-cloud.com
hik-express.com
hikiot.com
hikmall.com
hikvision.com
# 萤石
eziot.com
ezviz.com
ys7.com

322
main.go
View File

@@ -29,6 +29,19 @@ const (
RuleTypeInclude string = "include" RuleTypeInclude string = "include"
) )
var (
TypeChecker = regexp.MustCompile(`^(domain|full|keyword|regexp|include)$`)
ValueChecker = regexp.MustCompile(`^[a-z0-9!\.-]+$`)
AttrChecker = regexp.MustCompile(`^[a-z0-9!-]+$`)
)
var (
refMap = make(map[string]*List)
plMap = make(map[string]*ParsedList)
finalMap = make(map[string]*List)
cirIncMap = make(map[string]bool) // Used for circular inclusion detection
)
type Entry struct { type Entry struct {
Type string Type string
Value string Value string
@@ -40,15 +53,21 @@ type List struct {
Entry []Entry Entry []Entry
} }
type Inclusion struct {
Source string
MustAttrs []*router.Domain_Attribute
BannedAttrs []*router.Domain_Attribute
}
type ParsedList struct { type ParsedList struct {
Name string Name string
Inclusion map[string]bool Inclusions []Inclusion
Entry []Entry Entry []Entry
} }
func (l *ParsedList) toPlainText(listName string) error { func (entryList *List) toPlainText() error {
var entryBytes []byte var entryBytes []byte
for _, entry := range l.Entry { for _, entry := range entryList.Entry {
var attrString string var attrString string
if entry.Attrs != nil { if entry.Attrs != nil {
for _, attr := range entry.Attrs { for _, attr := range entry.Attrs {
@@ -59,13 +78,13 @@ func (l *ParsedList) toPlainText(listName string) error {
// Entry output format is: type:domain.tld:@attr1,@attr2 // Entry output format is: type:domain.tld:@attr1,@attr2
entryBytes = append(entryBytes, []byte(entry.Type+":"+entry.Value+attrString+"\n")...) entryBytes = append(entryBytes, []byte(entry.Type+":"+entry.Value+attrString+"\n")...)
} }
if err := os.WriteFile(filepath.Join(*outputDir, listName+".txt"), entryBytes, 0644); err != nil { if err := os.WriteFile(filepath.Join(*outputDir, strings.ToLower(entryList.Name)+".txt"), entryBytes, 0644); err != nil {
return err return err
} }
return nil return nil
} }
func (l *ParsedList) toProto() (*router.GeoSite, error) { func (l *List) toProto() (*router.GeoSite, error) {
site := &router.GeoSite{ site := &router.GeoSite{
CountryCode: l.Name, CountryCode: l.Name,
} }
@@ -79,11 +98,6 @@ func (l *ParsedList) toProto() (*router.GeoSite, error) {
}) })
case RuleTypeRegexp: case RuleTypeRegexp:
// check regexp validity to avoid runtime error
_, err := regexp.Compile(entry.Value)
if err != nil {
return nil, fmt.Errorf("invalid regexp in list %s: %s", l.Name, entry.Value)
}
site.Domain = append(site.Domain, &router.Domain{ site.Domain = append(site.Domain, &router.Domain{
Type: router.Domain_Regex, Type: router.Domain_Regex,
Value: entry.Value, Value: entry.Value,
@@ -103,88 +117,72 @@ func (l *ParsedList) toProto() (*router.GeoSite, error) {
Value: entry.Value, Value: entry.Value,
Attribute: entry.Attrs, Attribute: entry.Attrs,
}) })
default:
return nil, fmt.Errorf("unknown domain type: %s", entry.Type)
} }
} }
return site, nil return site, nil
} }
func exportPlainTextList(list []string, refName string, pl *ParsedList) { func exportPlainTextList(exportFiles []string, entryList *List) {
for _, listName := range list { for _, exportfilename := range exportFiles {
if strings.EqualFold(refName, listName) { if strings.EqualFold(entryList.Name, exportfilename) {
if err := pl.toPlainText(strings.ToLower(refName)); err != nil { if err := entryList.toPlainText(); err != nil {
fmt.Println("Failed:", err) fmt.Println("Failed to exportPlainTextList:", err)
continue continue
} }
fmt.Printf("'%s' has been generated successfully.\n", listName) fmt.Printf("'%s' has been generated successfully.\n", exportfilename)
} }
} }
} }
func removeComment(line string) string { func parseEntry(line string) (Entry, error) {
idx := strings.Index(line, "#") var entry Entry
if idx == -1 { parts := strings.Fields(line)
return line
}
return strings.TrimSpace(line[:idx])
}
func parseDomain(domain string, entry *Entry) error { // Parse/Check type and value
kv := strings.Split(domain, ":") rawTypeVal := parts[0]
kv := strings.Split(rawTypeVal, ":")
if len(kv) == 1 { if len(kv) == 1 {
entry.Type = RuleTypeDomain entry.Type = RuleTypeDomain // Default type
entry.Value = strings.ToLower(kv[0]) entry.Value = strings.ToLower(rawTypeVal)
return nil } else if len(kv) == 2 {
}
if len(kv) == 2 {
entry.Type = strings.ToLower(kv[0]) entry.Type = strings.ToLower(kv[0])
if entry.Type == RuleTypeRegexp {
if strings.EqualFold(entry.Type, RuleTypeRegexp) {
entry.Value = kv[1] entry.Value = kv[1]
} else { } else {
entry.Value = strings.ToLower(kv[1]) entry.Value = strings.ToLower(kv[1])
} }
} else {
return nil return entry, fmt.Errorf("invalid format: %s", line)
}
if !TypeChecker.MatchString(entry.Type) {
return entry, fmt.Errorf("invalid type: %s", entry.Type)
}
if entry.Type == RuleTypeRegexp {
if _, err := regexp.Compile(entry.Value); err != nil {
return entry, fmt.Errorf("invalid regexp: %s", entry.Value)
}
} else if !ValueChecker.MatchString(entry.Value) {
return entry, fmt.Errorf("invalid value: %s", entry.Value)
} }
return fmt.Errorf("invalid format: %s", domain) // Parse/Check attributes
for _, part := range parts[1:] {
if !strings.HasPrefix(part, "@") {
return entry, fmt.Errorf("invalid attribute: %s", part)
} }
attrKey := strings.ToLower(part[1:]) // Trim attribute prefix `@` character
func parseAttribute(attr string) (*router.Domain_Attribute, error) { if !AttrChecker.MatchString(attrKey) {
var attribute router.Domain_Attribute return entry, fmt.Errorf("invalid attribute key: %s", attrKey)
if len(attr) == 0 || attr[0] != '@' {
return &attribute, fmt.Errorf("invalid attribute: %s", attr)
} }
entry.Attrs = append(entry.Attrs, &router.Domain_Attribute{
attribute.Key = strings.ToLower(attr[1:]) // Trim attribute prefix `@` character Key: attrKey,
attribute.TypedValue = &router.Domain_Attribute_BoolValue{BoolValue: true} TypedValue: &router.Domain_Attribute_BoolValue{BoolValue: true},
return &attribute, nil })
}
func parseEntry(line string) (Entry, error) {
line = strings.TrimSpace(line)
parts := strings.Split(line, " ")
var entry Entry
if len(parts) == 0 {
return entry, fmt.Errorf("empty entry")
}
if err := parseDomain(parts[0], &entry); err != nil {
return entry, err
}
for i := 1; i < len(parts); i++ {
attr, err := parseAttribute(parts[i])
if err != nil {
return entry, err
}
entry.Attrs = append(entry.Attrs, attr)
} }
// Sort attributes
sort.Slice(entry.Attrs, func(i, j int) bool {
return entry.Attrs[i].Key < entry.Attrs[j].Key
})
return entry, nil return entry, nil
} }
@@ -201,9 +199,13 @@ func Load(path string) (*List, error) {
} }
scanner := bufio.NewScanner(file) scanner := bufio.NewScanner(file)
for scanner.Scan() { for scanner.Scan() {
line := strings.TrimSpace(scanner.Text()) line := scanner.Text()
line = removeComment(line) // Remove comments
if len(line) == 0 { if idx := strings.Index(line, "#"); idx != -1 {
line = line[:idx]
}
line = strings.TrimSpace(line)
if line == "" {
continue continue
} }
entry, err := parseEntry(line) entry, err := parseEntry(line)
@@ -216,108 +218,77 @@ func Load(path string) (*List, error) {
return list, nil return list, nil
} }
func isMatchAttr(Attrs []*router.Domain_Attribute, includeKey string) bool { func ParseList(refList *List) (*ParsedList, error) {
isMatch := false pl := &ParsedList{Name: refList.Name}
mustMatch := true for _, entry := range refList.Entry {
matchName := includeKey
if strings.HasPrefix(includeKey, "!") {
isMatch = true
mustMatch = false
matchName = strings.TrimLeft(includeKey, "!")
}
for _, Attr := range Attrs {
attrName := Attr.Key
if mustMatch {
if matchName == attrName {
isMatch = true
break
}
} else {
if matchName == attrName {
isMatch = false
break
}
}
}
return isMatch
}
func createIncludeAttrEntrys(list *List, matchAttr *router.Domain_Attribute) []Entry {
newEntryList := make([]Entry, 0, len(list.Entry))
matchName := matchAttr.Key
for _, entry := range list.Entry {
matched := isMatchAttr(entry.Attrs, matchName)
if matched {
newEntryList = append(newEntryList, entry)
}
}
return newEntryList
}
func ParseList(list *List, ref map[string]*List) (*ParsedList, error) {
pl := &ParsedList{
Name: list.Name,
Inclusion: make(map[string]bool),
}
entryList := list.Entry
for {
newEntryList := make([]Entry, 0, len(entryList))
hasInclude := false
for _, entry := range entryList {
if entry.Type == RuleTypeInclude { if entry.Type == RuleTypeInclude {
refName := strings.ToUpper(entry.Value) inc := Inclusion{Source: strings.ToUpper(entry.Value)}
if entry.Attrs != nil {
for _, attr := range entry.Attrs { for _, attr := range entry.Attrs {
InclusionName := strings.ToUpper(refName + "@" + attr.Key) if strings.HasPrefix(attr.Key, "-") {
if pl.Inclusion[InclusionName] { inc.BannedAttrs = append(inc.BannedAttrs, &router.Domain_Attribute{
continue Key: attr.Key[1:], // Trim attribute prefix `-` character
} TypedValue: &router.Domain_Attribute_BoolValue{BoolValue: true},
pl.Inclusion[InclusionName] = true })
refList := ref[refName]
if refList == nil {
return nil, fmt.Errorf("list not found: %s", entry.Value)
}
attrEntrys := createIncludeAttrEntrys(refList, attr)
if len(attrEntrys) != 0 {
newEntryList = append(newEntryList, attrEntrys...)
}
}
} else { } else {
InclusionName := refName inc.MustAttrs = append(inc.MustAttrs, attr)
if pl.Inclusion[InclusionName] {
continue
} }
pl.Inclusion[InclusionName] = true
refList := ref[refName]
if refList == nil {
return nil, fmt.Errorf("list not found: %s", entry.Value)
} }
newEntryList = append(newEntryList, refList.Entry...) pl.Inclusions = append(pl.Inclusions, inc)
}
hasInclude = true
} else { } else {
newEntryList = append(newEntryList, entry) pl.Entry = append(pl.Entry, entry)
} }
} }
entryList = newEntryList
if !hasInclude {
break
}
}
pl.Entry = entryList
return pl, nil return pl, nil
} }
func isMatchAttrFilters(entry Entry, incFilter Inclusion) bool {
attrMap := make(map[string]bool)
for _, attr := range entry.Attrs {
attrMap[attr.Key] = true
}
for _, m := range incFilter.MustAttrs {
if !attrMap[m.Key] { return false }
}
for _, b := range incFilter.BannedAttrs {
if attrMap[b.Key] { return false }
}
return true
}
func ResolveList(pl *ParsedList) error {
//TODO: deduplicate
if _, pldone := finalMap[pl.Name]; pldone { return nil }
if cirIncMap[pl.Name] {
return fmt.Errorf("circular inclusion in: %s", pl.Name)
}
cirIncMap[pl.Name] = true
defer delete(cirIncMap, pl.Name)
finalList := &List{Name: pl.Name}
finalList.Entry = append(finalList.Entry, pl.Entry...)
for _, inc := range pl.Inclusions {
if err := ResolveList(plMap[inc.Source]); err != nil {
return err
}
for _, entry := range finalMap[inc.Source].Entry {
if isMatchAttrFilters(entry, inc) {
finalList.Entry = append(finalList.Entry, entry)
}
}
}
finalMap[pl.Name] = finalList
return nil
}
func main() { func main() {
flag.Parse() flag.Parse()
dir := *dataPath dir := *dataPath
fmt.Println("Use domain lists in", dir) fmt.Println("Use domain lists in", dir)
ref := make(map[string]*List) // Generate refMap
err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
if err != nil { if err != nil {
return err return err
@@ -329,7 +300,7 @@ func main() {
if err != nil { if err != nil {
return err return err
} }
ref[list.Name] = list refMap[list.Name] = list
return nil return nil
}) })
if err != nil { if err != nil {
@@ -337,6 +308,24 @@ func main() {
os.Exit(1) os.Exit(1)
} }
// Generate plMap
for refName, refList := range refMap {
pl, err := ParseList(refList)
if err != nil {
fmt.Println("Failed to ParseList:", err)
os.Exit(1)
}
plMap[refName] = pl
}
// Generate finalMap
for _, pl := range plMap {
if err := ResolveList(pl); err != nil {
fmt.Println("Failed to ResolveList:", err)
os.Exit(1)
}
}
// Create output directory if not exist // Create output directory if not exist
if _, err := os.Stat(*outputDir); os.IsNotExist(err) { if _, err := os.Stat(*outputDir); os.IsNotExist(err) {
if mkErr := os.MkdirAll(*outputDir, 0755); mkErr != nil { if mkErr := os.MkdirAll(*outputDir, 0755); mkErr != nil {
@@ -347,13 +336,8 @@ func main() {
protoList := new(router.GeoSiteList) protoList := new(router.GeoSiteList)
var existList []string var existList []string
for refName, list := range ref { for _, siteEntries := range finalMap {
pl, err := ParseList(list, ref) site, err := siteEntries.toProto()
if err != nil {
fmt.Println("Failed:", err)
os.Exit(1)
}
site, err := pl.toProto()
if err != nil { if err != nil {
fmt.Println("Failed:", err) fmt.Println("Failed:", err)
os.Exit(1) os.Exit(1)
@@ -363,7 +347,7 @@ func main() {
// Flatten and export plaintext list // Flatten and export plaintext list
if *exportLists != "" { if *exportLists != "" {
if existList != nil { if existList != nil {
exportPlainTextList(existList, refName, pl) exportPlainTextList(existList, siteEntries)
} else { } else {
exportedListSlice := strings.Split(*exportLists, ",") exportedListSlice := strings.Split(*exportLists, ",")
for _, exportedListName := range exportedListSlice { for _, exportedListName := range exportedListSlice {
@@ -376,7 +360,7 @@ func main() {
} }
} }
if existList != nil { if existList != nil {
exportPlainTextList(existList, refName, pl) exportPlainTextList(existList, siteEntries)
} }
} }
} }
@@ -389,11 +373,11 @@ func main() {
protoBytes, err := proto.Marshal(protoList) protoBytes, err := proto.Marshal(protoList)
if err != nil { if err != nil {
fmt.Println("Failed:", err) fmt.Println("Failed to Marshal:", err)
os.Exit(1) os.Exit(1)
} }
if err := os.WriteFile(filepath.Join(*outputDir, *outputName), protoBytes, 0644); err != nil { if err := os.WriteFile(filepath.Join(*outputDir, *outputName), protoBytes, 0644); err != nil {
fmt.Println("Failed:", err) fmt.Println("Failed to write output:", err)
os.Exit(1) os.Exit(1)
} else { } else {
fmt.Println(*outputName, "has been generated successfully.") fmt.Println(*outputName, "has been generated successfully.")