Compare commits

...

15 Commits

Author SHA1 Message Date
TripleA
aa4904fb6c Take-two update (#3256)
* Update 2K games domain list

Added new domains for 2K games

* Add rockstarnorth.com

* Add zynga domains

* Add take-two

* zynga: add zyngaplayersupport.com

* category-games-!cn: include take-two

---------

Co-authored-by: MkQtS <81752398+MkQtS@users.noreply.github.com>
2026-02-08 13:52:19 +08:00
Konstantin
2a8ef360ee Add domains to category-ru, netease, hdrezka, kodik, windsurf and yandex (#3238)
* add Aviasales

* add hdrezka

* add kodik

* add windsurf

* yandex: add yandex-pogoda.static-storage.net

* netease: add easebar.com
2026-02-07 23:45:47 +08:00
Qi Tianyi
c8717372da category-media-cn: add jin10 & tonghuashun domains (#3255) 2026-02-07 13:48:19 +08:00
Qi Tianyi
106baf8ba0 douyu: add edgesrv.com (#3254) 2026-02-06 23:25:57 +08:00
Qi Tianyi
d12634be01 geolocation-cn: add jisilu.cn (#3253)
集思录
2026-02-06 21:45:21 +08:00
Aethersailor
a8b474fac0 update category-ai-!cn (#3251)
add manus
2026-02-06 18:35:19 +08:00
⑨bingyin
f5227519b8 vercel: add skills.sh (#3250) 2026-02-06 15:34:15 +08:00
MkQtS
b20cf00e07 Add more cn domains (#3249)
* add growingio

* category-cdn-cn: add dfyun.com.cn

* category-collaborate-cn: add feihengip.com

* category-dev-cn: add aardio.com

* category-education-cn: add biyehome.net

* category-enterprise-query-platform-cn: add xinchacha domains

* category-media-cn: add more domains

* category-social-media-cn: add fanfou.com

* category-wiki-cn: add chaz.fun
2026-02-05 21:32:06 +08:00
jinqiang zhang
027b8b3409 dji: add djigate.com (#3248) 2026-02-05 20:20:39 +08:00
xd DG
535dc789b9 Add geosite:radiko (#3247)
* Add geosite:radiko

* Sort domains and include radiko in category-entertainment

---------

Co-authored-by: terada46 <mizukiloveu@gmail.com>
2026-02-05 17:30:18 +08:00
MkQtS
311b281000 improve codes (#3246) 2026-02-04 15:03:04 +08:00
秋野かえで
bfb35d7b68 split githubcopilot.com to github-copilot (#3245) 2026-02-04 14:34:55 +08:00
深鸣
daf4c10d0c category-entertainment-cn: add anitabi.cn (#3244) 2026-02-04 13:58:38 +08:00
深鸣
a188c2c058 geolocation-!cn: add osmand.net (#3243) 2026-02-04 13:57:46 +08:00
MkQtS
947556aa16 Improve codes (#3242)
* main.go: improve code

* main.go: move refMap from global variable to local

* main.go: allow tld to be a parent domain

* datdump: improve code
2026-02-03 22:38:18 +08:00
37 changed files with 260 additions and 163 deletions

View File

@@ -31,11 +31,21 @@ type DomainList struct {
} }
func (d *DomainRule) domain2String() string { func (d *DomainRule) domain2String() string {
dstring := d.Type + ":" + d.Value var dstr strings.Builder
if len(d.Attrs) != 0 { dstr.Grow(len(d.Type) + len(d.Value) + 10)
dstring += ":@" + strings.Join(d.Attrs, ",@") dstr.WriteString(d.Type)
dstr.WriteByte(':')
dstr.WriteString(d.Value)
for i, attr := range d.Attrs {
if i == 0 {
dstr.WriteByte(':')
} else {
dstr.WriteByte(',')
}
dstr.WriteByte('@')
dstr.WriteString(attr)
} }
return dstring return dstr.String()
} }
func loadGeosite(path string) ([]DomainList, map[string]*DomainList, error) { func loadGeosite(path string) ([]DomainList, map[string]*DomainList, error) {
@@ -82,10 +92,10 @@ func loadGeosite(path string) ([]DomainList, map[string]*DomainList, error) {
func exportSite(name string, domainListByName map[string]*DomainList) error { func exportSite(name string, domainListByName map[string]*DomainList) error {
domainList, ok := domainListByName[strings.ToUpper(name)] domainList, ok := domainListByName[strings.ToUpper(name)]
if !ok { if !ok {
return fmt.Errorf("list '%s' does not exist", name) return fmt.Errorf("list %q does not exist", name)
} }
if len(domainList.Rules) == 0 { if len(domainList.Rules) == 0 {
return fmt.Errorf("list '%s' is empty", name) return fmt.Errorf("list %q is empty", name)
} }
file, err := os.Create(filepath.Join(*outputDir, name+".yml")) file, err := os.Create(filepath.Join(*outputDir, name+".yml"))
if err != nil { if err != nil {
@@ -119,22 +129,16 @@ func exportAll(filename string, domainLists []DomainList) error {
return w.Flush() return w.Flush()
} }
func main() { func run() error {
flag.Parse() // Make sure output directory exists
if err := os.MkdirAll(*outputDir, 0755); err != nil {
// Create output directory if not exist return fmt.Errorf("failed to create output directory: %w", err)
if _, err := os.Stat(*outputDir); os.IsNotExist(err) {
if mkErr := os.MkdirAll(*outputDir, 0755); mkErr != nil {
fmt.Println("Failed to create output directory:", mkErr)
os.Exit(1)
}
} }
fmt.Printf("Loading %s...\n", *inputData) fmt.Printf("loading source data %q...\n", *inputData)
domainLists, domainListByName, err := loadGeosite(*inputData) domainLists, domainListByName, err := loadGeosite(*inputData)
if err != nil { if err != nil {
fmt.Println("Failed to loadGeosite:", err) return fmt.Errorf("failed to loadGeosite: %w", err)
os.Exit(1)
} }
var exportListSlice []string var exportListSlice []string
@@ -150,15 +154,24 @@ func main() {
for _, eplistname := range exportListSlice { for _, eplistname := range exportListSlice {
if strings.EqualFold(eplistname, "_all_") { if strings.EqualFold(eplistname, "_all_") {
if err := exportAll(filepath.Base(*inputData)+"_plain.yml", domainLists); err != nil { if err := exportAll(filepath.Base(*inputData)+"_plain.yml", domainLists); err != nil {
fmt.Println("Failed to exportAll:", err) fmt.Printf("failed to exportAll: %v\n", err)
continue continue
} }
} else { } else {
if err := exportSite(eplistname, domainListByName); err != nil { if err := exportSite(eplistname, domainListByName); err != nil {
fmt.Println("Failed to exportSite:", err) fmt.Printf("failed to exportSite: %v\n", err)
continue continue
} }
} }
fmt.Printf("list: '%s' has been exported successfully.\n", eplistname) fmt.Printf("list: %q has been exported successfully.\n", eplistname)
}
return nil
}
func main() {
flag.Parse()
if err := run(); err != nil {
fmt.Printf("Fatal error: %v\n", err)
os.Exit(1)
} }
} }

View File

@@ -1,4 +1,6 @@
2k.com 2k.com
2kcoretech.online 2kcoretech.online
2kct.onl
2kgames.com 2kgames.com
take2games.com 2ksports.com
2kweb.online

3
data/aviasales Normal file
View File

@@ -0,0 +1,3 @@
avs.io
aviasales.ru
aviasales.com

View File

@@ -64,7 +64,6 @@ adservice.sigmob.cn
adtechus.com adtechus.com
adtrue.com adtrue.com
adxprtz.com adxprtz.com
assets.growingio.com
cdn.advertserve.com cdn.advertserve.com
cdn.banclip.com cdn.banclip.com
cfts1tifqr.com cfts1tifqr.com

View File

@@ -3,6 +3,7 @@ include:category-ads
include:adjust include:adjust
include:clearbit include:clearbit
include:growingio
include:ogury include:ogury
include:openx include:openx
include:pubmatic include:pubmatic

View File

@@ -4,13 +4,16 @@ include:cerebras
include:comfy include:comfy
include:cursor include:cursor
include:elevenlabs include:elevenlabs
include:github-copilot
include:google-deepmind include:google-deepmind
include:groq include:groq
include:huggingface include:huggingface
include:liveperson include:liveperson
include:manus
include:openai include:openai
include:perplexity include:perplexity
include:poe include:poe
include:windsurf
include:xai include:xai
include:youmind include:youmind

View File

@@ -6,12 +6,14 @@ include:qiniu
include:upai include:upai
include:wangsu include:wangsu
## 创世云 # 创世云
chuangcache.com chuangcache.com
chuangcdn.com chuangcdn.com
## FUNCDN # 大风云CDN
dfyun.com.cn
# FUNCDN
funcdn.com funcdn.com
## 北京知道创宇信息技术股份有限公司 # 北京知道创宇信息技术股份有限公司
jiashule.com jiashule.com
jiasule.com jiasule.com
yunaq.com yunaq.com

View File

@@ -4,6 +4,8 @@
asklink.com asklink.com
## EasyTier ## EasyTier
easytier.cn easytier.cn
## 飞衡HTTP
feihengip.com
## Oray ## Oray
oray.com oray.com
oray.net oray.net

View File

@@ -20,6 +20,7 @@ include:tencent-dev
include:ubuntukylin include:ubuntukylin
include:unitychina include:unitychina
aardio.com
jinrishici.com jinrishici.com
openvela.com openvela.com
tipdm.org tipdm.org

View File

@@ -71,6 +71,8 @@ baicizhan.com
baicizhan.org baicizhan.org
bczcdn.com bczcdn.com
bczeducation.cn bczeducation.cn
# 毕业之家科研服务平台
biyehome.net
# Burning Vocabulary # Burning Vocabulary
burningvocabulary.cn burningvocabulary.cn
burningvocabulary.com burningvocabulary.com

View File

@@ -6,3 +6,7 @@ include:tianyancha
qichamao.com qichamao.com
qyyjt.cn qyyjt.cn
x315.com x315.com
# 信查查
xcc.cn
xinchacha.com

View File

@@ -54,6 +54,7 @@ include:pixiv
include:plutotv include:plutotv
include:pocketcasts include:pocketcasts
include:primevideo include:primevideo
include:radiko
include:roku include:roku
include:showtimeanytime include:showtimeanytime
include:sling include:sling

View File

@@ -50,6 +50,8 @@ yeshen.com
51zmt.top 51zmt.top
# 广东南方新媒体 # 广东南方新媒体
aisee.tv aisee.tv
# 动画巡礼
anitabi.cn
# 暴风影音 # 暴风影音
baofeng.com baofeng.com
baofeng.net baofeng.net

View File

@@ -1,4 +1,3 @@
include:2kgames
include:blizzard include:blizzard
include:bluearchive include:bluearchive
include:bohemia include:bohemia
@@ -30,12 +29,12 @@ include:pubg
include:purikonejp include:purikonejp
include:riot include:riot
include:roblox include:roblox
include:rockstar
include:snk include:snk
include:steam include:steam
include:steaminventoryhelper include:steaminventoryhelper
include:steamunlocked include:steamunlocked
include:supercell include:supercell
include:take-two
include:ubisoft include:ubisoft
include:vrchat include:vrchat
include:wbgames include:wbgames

View File

@@ -78,6 +78,15 @@ freebuf.com
geekpark.net geekpark.net
# 光明网 # 光明网
gmw.com gmw.com
# 硅谷网
guigu.org
# 同花顺
0033.cn
10jqka.com.cn
hexin.cn
hexin.com.cn
ths123.com
thsi.cn
# 和讯 # 和讯
hexun.com hexun.com
# 河南广播电视台/大象网 # 河南广播电视台/大象网
@@ -100,6 +109,8 @@ iresearch.cn
jianyuweb.com jianyuweb.com
wallstcn.com wallstcn.com
wallstreetcn.com wallstreetcn.com
# 金十数据
jin10.com
# 上观新闻/解放日报 # 上观新闻/解放日报
jfdaily.com jfdaily.com
shobserver.com shobserver.com
@@ -134,6 +145,9 @@ xinhuanet.com
xinhuaxmt.com xinhuaxmt.com
# 维科网 # 维科网
ofweek.com ofweek.com
# PChome电脑之家
pchome.net
pchpic.net
# PConline 太平洋科技 # PConline 太平洋科技
3conline.com 3conline.com
pconline.com.cn pconline.com.cn
@@ -184,10 +198,6 @@ techweb.com.cn
thecover.cn thecover.cn
# 澎湃新闻 # 澎湃新闻
thepaper.cn thepaper.cn
# 同花顺
10jqka.com.cn
thsi.cn
ths123.com
# 人人都是产品经理 # 人人都是产品经理
woshipm.com woshipm.com
# 新快报 # 新快报

View File

@@ -22,6 +22,7 @@ include:category-gov-ru
include:category-retail-ru include:category-retail-ru
# Public transportation # Public transportation
include:aviasales
include:mosmetro include:mosmetro
include:avito include:avito

View File

@@ -18,6 +18,9 @@ tieba.com
dandan818.com dandan818.com
dandanvoice.com dandanvoice.com
# 饭否
fanfou.com
# 脉脉 # 脉脉
maimai.cn maimai.cn
taou.com taou.com

View File

@@ -4,6 +4,9 @@ mbalib.com
sec-wiki.com sec-wiki.com
shidianbaike.com shidianbaike.com
# 叉子周 手机博物馆
chaz.fun
# huijiwiki # huijiwiki
huijistatic.com huijistatic.com
huijiwiki.com huijiwiki.com

View File

@@ -2,6 +2,7 @@ dji.com
dji.ink dji.ink
dji.net dji.net
djicdn.com djicdn.com
djigate.com
djiits.com djiits.com
djiops.com djiops.com
djiservice.org djiservice.org

View File

@@ -4,3 +4,4 @@ douyucdn.cn
douyucdn2.cn douyucdn2.cn
douyuscdn.com douyuscdn.com
douyutv.com douyutv.com
edgesrv.com

View File

@@ -271,6 +271,8 @@ ldoceonline.com
immersivetranslate.com # 沉浸式翻译 (国际版) immersivetranslate.com # 沉浸式翻译 (国际版)
## OriginLab (Graphing for Science and Engineering) ## OriginLab (Graphing for Science and Engineering)
originlab.com originlab.com
## OsmAnd
osmand.net
# Software development # Software development
include:category-dev include:category-dev

View File

@@ -23,6 +23,7 @@ include:category-social-media-cn
# Advertisment & Analytics # Advertisment & Analytics
include:getui include:getui
include:growingio
include:jiguang include:jiguang
# 神策数据 # 神策数据
@@ -163,6 +164,7 @@ ezpro.pro # 小马路由
itpub.net # ITPUB技术论坛 itpub.net # ITPUB技术论坛
itsk.com # IT天空论坛 itsk.com # IT天空论坛
jandan.net # 煎蛋 jandan.net # 煎蛋
jisilu.cn # 集思录
kafan.cn # 卡饭论坛 kafan.cn # 卡饭论坛
kanxue.com # 看雪 kanxue.com # 看雪
kdslife.com # 宽带山 kdslife.com # 宽带山
@@ -433,7 +435,7 @@ include:lenovo
include:meizu include:meizu
include:midea include:midea
include:narwal # 云鲸科技 include:narwal # 云鲸科技
include:netease include:netease @-!cn
include:oppo @-!cn include:oppo @-!cn
include:qihoo360 include:qihoo360
include:sumkoo #北京尚古创新科技有限公司 include:sumkoo #北京尚古创新科技有限公司
@@ -663,7 +665,6 @@ ycrx360.com
9ht.com 9ht.com
9xu.com 9xu.com
a9vg.com a9vg.com
aardio.com # 皖ICP备09012014号
acetaffy.club # 粤ICP备2022042304号 acetaffy.club # 粤ICP备2022042304号
adxvip.com adxvip.com
afzhan.com afzhan.com
@@ -719,7 +720,6 @@ bio-equip.com
biodiscover.com biodiscover.com
bishijie.com bishijie.com
bitecoin.com bitecoin.com
biyehome.net
bjcathay.com bjcathay.com
bobo.com bobo.com
bojianger.com bojianger.com
@@ -743,7 +743,6 @@ chachaba.com
changba.com changba.com
chaojituzi.net chaojituzi.net
chashebao.com chashebao.com
chaz.fun # 粤ICP备2022001828号-2
chazhengla.com chazhengla.com
chazidian.com chazidian.com
che168.com che168.com
@@ -879,7 +878,6 @@ fanli.com
fangxiaoer.com fangxiaoer.com
fanxian.com fanxian.com
fastapi.net fastapi.net
feihengip.com # 粤ICP备2023115330号-1
feihuo.com feihuo.com
feiniaomy.com feiniaomy.com
fengniao.com fengniao.com
@@ -903,7 +901,6 @@ gdrc.com
geektool.top # 极客Tool 蜀ICP备2024086015号-2 geektool.top # 极客Tool 蜀ICP备2024086015号-2
gezida.com gezida.com
gfan.com gfan.com
giocdn.com
globrand.com globrand.com
gm86.com gm86.com
gmz88.com gmz88.com
@@ -914,7 +911,6 @@ gongxiangcj.com
goosail.com goosail.com
goufw.com goufw.com
greenxiazai.com greenxiazai.com
growingio.com
gtags.net gtags.net
guabu.com guabu.com
guaiguai.com guaiguai.com
@@ -922,7 +918,6 @@ guanaitong.com
guanhaobio.com guanhaobio.com
guanyierp.com # 沪ICP备14043335号-8 guanyierp.com # 沪ICP备14043335号-8
gucheng.com gucheng.com
guigu.org
guoxinmac.com guoxinmac.com
gupzs.com gupzs.com
gushiwen.org gushiwen.org
@@ -1178,7 +1173,6 @@ p5w.net
paipaibang.com paipaibang.com
paopaoche.net paopaoche.net
pc6.com pc6.com
pchome.net
pcpop.com pcpop.com
peccn.com peccn.com
pgzs.com pgzs.com

View File

@@ -1,4 +1,5 @@
include:github-ads include:github-ads
include:github-copilot
include:npmjs include:npmjs
atom.io atom.io
@@ -14,7 +15,6 @@ github.dev
github.io github.io
githubapp.com githubapp.com
githubassets.com githubassets.com
githubcopilot.com
githubhackathon.com githubhackathon.com
githubnext.com githubnext.com
githubpreview.dev githubpreview.dev

1
data/github-copilot Normal file
View File

@@ -0,0 +1 @@
githubcopilot.com

7
data/growingio Normal file
View File

@@ -0,0 +1,7 @@
# 北京易数科技
datayi.cn
gio.ren
giocdn.com
growin.cn
growingio.cn
growingio.com

12
data/hdrezka Normal file
View File

@@ -0,0 +1,12 @@
hdrezka-home.tv
hdrezka.ac
hdrezka.ag
hdrezka.cm
hdrezka.co
hdrezka.me
hdrezka.org
hdrezka.site
rezka.ag
# hdrezka video CDN servers
stream.voidboost.cc

4
data/kodik Normal file
View File

@@ -0,0 +1,4 @@
kodik-storage.com
kodik.biz
kodik.cc
kodik.info

2
data/manus Normal file
View File

@@ -0,0 +1,2 @@
manus.im
manuscdn.com

View File

@@ -16,6 +16,7 @@ include:netease-ads
166.net 166.net
188.com 188.com
321fenx.com 321fenx.com
easebar.com @!cn
haowu.link haowu.link
icourse163.org icourse163.org
lf126.net lf126.net

5
data/radiko Normal file
View File

@@ -0,0 +1,5 @@
# radiko official access and streaming domains
radiko-cf.com
radiko.jp
smartstream.ne.jp

View File

@@ -1,4 +1,5 @@
rockstargames.com rockstargames.com
rockstarnorth.com
rsg.sc rsg.sc
full:gamedownloads-rockstargames-com.akamaized.net full:gamedownloads-rockstargames-com.akamaized.net

5
data/take-two Normal file
View File

@@ -0,0 +1,5 @@
include:2kgames
include:rockstar
include:zynga
take2games.com

View File

@@ -3,6 +3,7 @@ err.sh
hyper.is hyper.is
nextjs.org nextjs.org
now.sh now.sh
skills.sh
static.fun static.fun
title.sh title.sh
turborepo.org turborepo.org

4
data/windsurf Normal file
View File

@@ -0,0 +1,4 @@
codeium.com
codeiumdata.com
windsurf.build
windsurf.com

View File

@@ -60,3 +60,6 @@ yastatic.net
# Watching movies, included in the Yandex subscription # Watching movies, included in the Yandex subscription
kinopoisk.ru kinopoisk.ru
kinopoisk-ru.clstorage.net kinopoisk-ru.clstorage.net
# Weather service
full:yandex-pogoda.static-storage.net

3
data/zynga Normal file
View File

@@ -0,0 +1,3 @@
zynga.com
zyngaplayersupport.com
zyngapoker.com

189
main.go
View File

@@ -23,7 +23,6 @@ var (
) )
var ( var (
refMap = make(map[string][]*Entry)
plMap = make(map[string]*ParsedList) plMap = make(map[string]*ParsedList)
finalMap = make(map[string][]*Entry) finalMap = make(map[string][]*Entry)
cirIncMap = make(map[string]bool) // Used for circular inclusion detection cirIncMap = make(map[string]bool) // Used for circular inclusion detection
@@ -78,18 +77,14 @@ func makeProtoList(listName string, entries []*Entry) (*router.GeoSite, error) {
return site, nil return site, nil
} }
func writePlainList(exportedName string) error { func writePlainList(listname string, entries []*Entry) error {
targetList, exist := finalMap[strings.ToUpper(exportedName)] file, err := os.Create(filepath.Join(*outputDir, strings.ToLower(listname)+".txt"))
if !exist || len(targetList) == 0 {
return fmt.Errorf("list %q does not exist or is empty.", exportedName)
}
file, err := os.Create(filepath.Join(*outputDir, strings.ToLower(exportedName)+".txt"))
if err != nil { if err != nil {
return err return err
} }
defer file.Close() defer file.Close()
w := bufio.NewWriter(file) w := bufio.NewWriter(file)
for _, entry := range targetList { for _, entry := range entries {
fmt.Fprintln(w, entry.Plain) fmt.Fprintln(w, entry.Plain)
} }
return w.Flush() return w.Flush()
@@ -99,7 +94,7 @@ func parseEntry(line string) (Entry, error) {
var entry Entry var entry Entry
parts := strings.Fields(line) parts := strings.Fields(line)
if len(parts) == 0 { if len(parts) == 0 {
return entry, fmt.Errorf("empty line: %q", line) return entry, fmt.Errorf("empty line")
} }
// Parse type and value // Parse type and value
@@ -138,7 +133,7 @@ func parseEntry(line string) (Entry, error) {
} }
} }
// Parse/Check attributes and affiliations // Parse attributes and affiliations
for _, part := range parts[1:] { for _, part := range parts[1:] {
if strings.HasPrefix(part, "@") { if strings.HasPrefix(part, "@") {
attr := strings.ToLower(part[1:]) // Trim attribute prefix `@` character attr := strings.ToLower(part[1:]) // Trim attribute prefix `@` character
@@ -159,10 +154,21 @@ func parseEntry(line string) (Entry, error) {
// Sort attributes // Sort attributes
slices.Sort(entry.Attrs) slices.Sort(entry.Attrs)
// Formated plain entry: type:domain.tld:@attr1,@attr2 // Formated plain entry: type:domain.tld:@attr1,@attr2
entry.Plain = entry.Type + ":" + entry.Value var plain strings.Builder
if len(entry.Attrs) != 0 { plain.Grow(len(entry.Type) + len(entry.Value) + 10)
entry.Plain = entry.Plain + ":@" + strings.Join(entry.Attrs, ",@") plain.WriteString(entry.Type)
plain.WriteByte(':')
plain.WriteString(entry.Value)
for i, attr := range entry.Attrs {
if i == 0 {
plain.WriteByte(':')
} else {
plain.WriteByte(',')
}
plain.WriteByte('@')
plain.WriteString(attr)
} }
entry.Plain = plain.String()
return entry, nil return entry, nil
} }
@@ -200,25 +206,21 @@ func validateSiteName(name string) bool {
return true return true
} }
func loadData(path string) error { func loadData(path string) ([]*Entry, error) {
file, err := os.Open(path) file, err := os.Open(path)
if err != nil { if err != nil {
return err return nil, err
} }
defer file.Close() defer file.Close()
listName := strings.ToUpper(filepath.Base(path)) var entries []*Entry
if !validateSiteName(listName) {
return fmt.Errorf("invalid list name: %s", listName)
}
scanner := bufio.NewScanner(file) scanner := bufio.NewScanner(file)
lineIdx := 0 lineIdx := 0
for scanner.Scan() { for scanner.Scan() {
line := scanner.Text() line := scanner.Text()
lineIdx++ lineIdx++
// Remove comments
if idx := strings.Index(line, "#"); idx != -1 { if idx := strings.Index(line, "#"); idx != -1 {
line = line[:idx] line = line[:idx] // Remove comments
} }
line = strings.TrimSpace(line) line = strings.TrimSpace(line)
if line == "" { if line == "" {
@@ -226,11 +228,11 @@ func loadData(path string) error {
} }
entry, err := parseEntry(line) entry, err := parseEntry(line)
if err != nil { if err != nil {
return fmt.Errorf("error in %s at line %d: %v", path, lineIdx, err) return entries, fmt.Errorf("error in %q at line %d: %w", path, lineIdx, err)
} }
refMap[listName] = append(refMap[listName], &entry) entries = append(entries, &entry)
} }
return nil return entries, nil
} }
func parseList(refName string, refList []*Entry) error { func parseList(refName string, refList []*Entry) error {
@@ -242,7 +244,7 @@ func parseList(refName string, refList []*Entry) error {
for _, entry := range refList { for _, entry := range refList {
if entry.Type == dlc.RuleTypeInclude { if entry.Type == dlc.RuleTypeInclude {
if len(entry.Affs) != 0 { if len(entry.Affs) != 0 {
return fmt.Errorf("affiliation is not allowed for include:%s", entry.Value) return fmt.Errorf("affiliation is not allowed for include:%q", entry.Value)
} }
inc := &Inclusion{Source: entry.Value} inc := &Inclusion{Source: entry.Value}
for _, attr := range entry.Attrs { for _, attr := range entry.Attrs {
@@ -268,11 +270,31 @@ func parseList(refName string, refList []*Entry) error {
return nil return nil
} }
func polishList(roughMap *map[string]*Entry) []*Entry { func isMatchAttrFilters(entry *Entry, incFilter *Inclusion) bool {
finalList := make([]*Entry, 0, len(*roughMap)) if len(incFilter.MustAttrs) == 0 && len(incFilter.BanAttrs) == 0 {
queuingList := make([]*Entry, 0, len(*roughMap)) // Domain/full entries without attr return true
}
if len(entry.Attrs) == 0 {
return len(incFilter.MustAttrs) == 0
}
for _, m := range incFilter.MustAttrs {
if !slices.Contains(entry.Attrs, m) {
return false
}
}
for _, b := range incFilter.BanAttrs {
if slices.Contains(entry.Attrs, b) {
return false
}
}
return true
}
func polishList(roughMap map[string]*Entry) []*Entry {
finalList := make([]*Entry, 0, len(roughMap))
queuingList := make([]*Entry, 0, len(roughMap)) // Domain/full entries without attr
domainsMap := make(map[string]bool) domainsMap := make(map[string]bool)
for _, entry := range *roughMap { for _, entry := range roughMap {
switch entry.Type { // Bypass regexp, keyword and "full/domain with attr" switch entry.Type { // Bypass regexp, keyword and "full/domain with attr"
case dlc.RuleTypeRegexp: case dlc.RuleTypeRegexp:
finalList = append(finalList, entry) finalList = append(finalList, entry)
@@ -306,9 +328,6 @@ func polishList(roughMap *map[string]*Entry) []*Entry {
break break
} }
pd = pd[idx+1:] // Go for next parent pd = pd[idx+1:] // Go for next parent
if !strings.Contains(pd, ".") {
break
} // Not allow tld to be a parent
if domainsMap[pd] { if domainsMap[pd] {
isRedundant = true isRedundant = true
break break
@@ -331,32 +350,11 @@ func resolveList(pl *ParsedList) error {
} }
if cirIncMap[pl.Name] { if cirIncMap[pl.Name] {
return fmt.Errorf("circular inclusion in: %s", pl.Name) return fmt.Errorf("circular inclusion in: %q", pl.Name)
} }
cirIncMap[pl.Name] = true cirIncMap[pl.Name] = true
defer delete(cirIncMap, pl.Name) defer delete(cirIncMap, pl.Name)
isMatchAttrFilters := func(entry *Entry, incFilter *Inclusion) bool {
if len(incFilter.MustAttrs) == 0 && len(incFilter.BanAttrs) == 0 {
return true
}
if len(entry.Attrs) == 0 {
return len(incFilter.MustAttrs) == 0
}
for _, m := range incFilter.MustAttrs {
if !slices.Contains(entry.Attrs, m) {
return false
}
}
for _, b := range incFilter.BanAttrs {
if slices.Contains(entry.Attrs, b) {
return false
}
}
return true
}
roughMap := make(map[string]*Entry) // Avoid basic duplicates roughMap := make(map[string]*Entry) // Avoid basic duplicates
for _, dentry := range pl.Entries { // Add direct entries for _, dentry := range pl.Entries { // Add direct entries
roughMap[dentry.Plain] = dentry roughMap[dentry.Plain] = dentry
@@ -375,80 +373,75 @@ func resolveList(pl *ParsedList) error {
} }
} }
} }
finalMap[pl.Name] = polishList(&roughMap) finalMap[pl.Name] = polishList(roughMap)
return nil return nil
} }
func main() { func run() error {
flag.Parse()
dir := *dataPath dir := *dataPath
fmt.Println("Use domain lists in", dir) fmt.Printf("using domain lists data in %q\n", dir)
// Generate refMap // Generate refMap
err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { refMap := make(map[string][]*Entry)
err := filepath.WalkDir(dir, func(path string, d os.DirEntry, err error) error {
if err != nil { if err != nil {
return err return err
} }
if info.IsDir() { if d.IsDir() {
return nil return nil
} }
if err := loadData(path); err != nil { listName := strings.ToUpper(filepath.Base(path))
return err if !validateSiteName(listName) {
return fmt.Errorf("invalid list name: %q", listName)
} }
return nil refMap[listName], err = loadData(path)
return err
}) })
if err != nil { if err != nil {
fmt.Println("Failed to loadData:", err) return fmt.Errorf("failed to loadData: %w", err)
os.Exit(1)
} }
// Generate plMap // Generate plMap
for refName, refList := range refMap { for refName, refList := range refMap {
if err := parseList(refName, refList); err != nil { if err := parseList(refName, refList); err != nil {
fmt.Println("Failed to parseList:", err) return fmt.Errorf("failed to parseList %q: %w", refName, err)
os.Exit(1)
} }
} }
// Generate finalMap // Generate finalMap
for _, pl := range plMap { for plname, pl := range plMap {
if err := resolveList(pl); err != nil { if err := resolveList(pl); err != nil {
fmt.Println("Failed to resolveList:", err) return fmt.Errorf("failed to resolveList %q: %w", plname, err)
os.Exit(1)
} }
} }
// Create output directory if not exist // Make sure output directory exists
if _, err := os.Stat(*outputDir); os.IsNotExist(err) { if err := os.MkdirAll(*outputDir, 0755); err != nil {
if mkErr := os.MkdirAll(*outputDir, 0755); mkErr != nil { return fmt.Errorf("failed to create output directory: %w", err)
fmt.Println("Failed to create output directory:", mkErr)
os.Exit(1)
}
} }
// Export plaintext list // Export plaintext list
var exportListSlice []string for rawEpList := range strings.SplitSeq(*exportLists, ",") {
for raw := range strings.SplitSeq(*exportLists, ",") { if epList := strings.TrimSpace(rawEpList); epList != "" {
if trimmed := strings.TrimSpace(raw); trimmed != "" { entries, exist := finalMap[strings.ToUpper(epList)]
exportListSlice = append(exportListSlice, trimmed) if !exist || len(entries) == 0 {
fmt.Printf("list %q does not exist or is empty\n", epList)
continue
}
if err := writePlainList(epList, entries); err != nil {
fmt.Printf("failed to write list %q: %v\n", epList, err)
continue
}
fmt.Printf("list %q has been generated successfully.\n", epList)
} }
} }
for _, exportList := range exportListSlice {
if err := writePlainList(exportList); err != nil {
fmt.Println("Failed to write list:", err)
continue
}
fmt.Printf("list %q has been generated successfully.\n", exportList)
}
// Generate dat file // Generate dat file
protoList := new(router.GeoSiteList) protoList := new(router.GeoSiteList)
for siteName, siteEntries := range finalMap { for siteName, siteEntries := range finalMap {
site, err := makeProtoList(siteName, siteEntries) site, err := makeProtoList(siteName, siteEntries)
if err != nil { if err != nil {
fmt.Println("Failed to makeProtoList:", err) return fmt.Errorf("failed to makeProtoList %q: %w", siteName, err)
os.Exit(1)
} }
protoList.Entry = append(protoList.Entry, site) protoList.Entry = append(protoList.Entry, site)
} }
@@ -459,13 +452,19 @@ func main() {
protoBytes, err := proto.Marshal(protoList) protoBytes, err := proto.Marshal(protoList)
if err != nil { if err != nil {
fmt.Println("Failed to marshal:", err) return fmt.Errorf("failed to marshal: %w", err)
os.Exit(1)
} }
if err := os.WriteFile(filepath.Join(*outputDir, *outputName), protoBytes, 0644); err != nil { if err := os.WriteFile(filepath.Join(*outputDir, *outputName), protoBytes, 0644); err != nil {
fmt.Println("Failed to write output:", err) return fmt.Errorf("failed to write output: %w", err)
}
fmt.Printf("%q has been generated successfully.\n", *outputName)
return nil
}
func main() {
flag.Parse()
if err := run(); err != nil {
fmt.Printf("Fatal error: %v\n", err)
os.Exit(1) os.Exit(1)
} else {
fmt.Println(*outputName, "has been generated successfully.")
} }
} }