Compare commits

...

25 Commits

Author SHA1 Message Date
inf
f34f22819e category-dev: add ziglang.org (#3373) 2026-03-20 10:46:09 +08:00
ir0nmand0
baa1409cfb category-entertainment-ru: add beeline.tv (#3372)
Beeline TV (beeline.tv) is a Russian streaming service by VEON (Beeline).
Movies, TV series, and live TV channels for Russian-speaking audience.

Subdomains (covered by domain match): web-prod, rest, images, static, video.
External deps (mediavitrina.ru, vimpelcom.ru) already in category-ru.

Co-authored-by: Dima Dudukin <dima.dudukin.dev@gmail.com>
2026-03-19 18:12:53 +08:00
MkQtS
a22d247c5a qcloud: comment out useless regexp rules (#3371)
Overrided by other domain type rules, but cannot be optimized automatically.

They are actually useless and only affect performance.
2026-03-19 12:43:29 +08:00
yobarerukoto
d311bbe50b geolocation-cn: add gzyowin.com (#3369) 2026-03-18 19:29:43 +08:00
MkQtS
1db558b165 main.go: support to generate multiple custom dats (#3367)
This allows to remove any unwanted lists without modifying the domains
data, and you can generate multiple custom v2ray dat files in a single
command.

As long as the source data is consistent, any list remains in the trimed
dat contains the same rules comparing to the list in full dat.

Use the new option `datprofile` to specify the config json file path.
`outputname` will be ignored when `datprofile` is set.

Co-authored-by: database64128 <free122448@hotmail.com>
2026-03-18 18:32:05 +08:00
Konstantin
9ee0757263 Add Tilda domains (#3368)
* add tilda

* tilda: add to category-dev
2026-03-18 18:05:54 +08:00
MkQtS
714a061ba3 main.go: improve codes (#3366)
* main.go: improve codes

* main.go: add parseInclusion

- seprate from parseEntry
- not allow affiliation for inclusion
2026-03-18 15:58:47 +08:00
TripleA
5ff8142411 Update category-speedtest (#3365) 2026-03-18 00:01:38 +08:00
MkQtS
becbd7a8ad wordpress: add wordpress.net (#3364) 2026-03-16 19:05:25 +08:00
xchacha20-poly1305
cd2d66eb72 baidu: Add xdrtc.com (#3362) 2026-03-15 21:55:53 +08:00
MkQtS
4c4ad053ef nvidia: add nvidia.custhelp.com 2026-03-15 14:51:57 +08:00
MkQtS
6544f6d3a6 oracle: add custhelp.com 2026-03-15 14:51:57 +08:00
MkQtS
673a70c380 category-ai-!cn: add spicywriter.com (#3360) 2026-03-15 14:32:27 +08:00
TripleA
be078767c4 Update category-ip-geo-detect (#3359) 2026-03-15 14:21:52 +08:00
深鸣
15fde0da4b Add more cn domains (#3358) 2026-03-15 14:20:57 +08:00
pover0k
5dd4779425 nodeseek: add nodeget.com (#3357) 2026-03-15 14:16:38 +08:00
Mikhail
eeccde7239 kinopub: add ahc.ovh (#3353)
Add ahc.ovh entry as a sub domains mirror
2026-03-13 21:52:34 +08:00
Larin Sergey
235988ffe4 anthropic: add clau.de (#3350) 2026-03-12 15:14:55 +08:00
MkQtS
a45bf04e9b category-ai-!cn: add more domains (#3352) 2026-03-12 15:08:42 +08:00
MkQtS
f31764e263 annas-archive: add new domains (#3349)
listed in https://en.wikipedia.org/wiki/Anna's_Archive

close https://github.com/v2fly/domain-list-community/issues/3347
2026-03-11 21:45:55 +08:00
MkQtS
540b28d5e4 Add more cn domains (#3348)
* geolocation-cn: remove zsxq.com

already in category-social-media-cn

* category-dev-cn: add zuidaima.com

* geolocation-cn: remove zimuzu.io

not in cn and seems dead

* geolocation-cn: reorder zhaopin.com

* geolocation-cn: reorder airline domains

* geolocation-cn: reorder more airline domains

* geolocation-cn: add more airline domains

* sohu: add 56.com domains
2026-03-11 21:42:32 +08:00
MkQtS
9c06656430 Add more cn domains (#3346)
* geolocation-cn: remove zrblog.net

not in cn

* category-wiki-cn: add allhistory.com

* category-education-cn: add more domains

* netease: add yunxindns.com
2026-03-11 14:03:34 +08:00
letsky192
f524c3b6bf geolocation-!cn: add sinyalee.com (#3345) 2026-03-11 13:52:48 +08:00
MkQtS
3cdb699cac Add more cn domains (#3343)
* keep: add more domains

* geolocation-cn: add more payment domains
2026-03-09 13:51:10 +08:00
MkQtS
6bb4a68f2f category-dev: add cve.org (#3342) 2026-03-09 12:18:58 +08:00
30 changed files with 425 additions and 159 deletions

6
.gitignore vendored
View File

@@ -4,9 +4,9 @@
/domain-list-community
/domain-list-community.exe
# Generated dat file.
dlc.dat
# Generated dat files.
/*.dat
# Exported plaintext lists.
/*.yml
/*.txt
/*.yml

View File

@@ -1,5 +1,8 @@
annas-archive.gd
annas-archive.gl
annas-archive.in
annas-archive.li
annas-archive.org
annas-archive.pk
annas-archive.pm
annas-archive.se

View File

@@ -1,8 +1,9 @@
anthropic.com
clau.de
claude.ai
claude.com
claudeusercontent.com
claudemcpclient.com
claudeusercontent.com
# CDN
full:servd-anthropic-website.b-cdn.net

View File

@@ -64,6 +64,7 @@ shifen.com
smartapps.cn
tieba.com
tiebaimg.com
xdrtc.com
xianfae.com
xiaodutv.com
yoojia.com

View File

@@ -30,6 +30,7 @@ clipdrop.co
jasper.ai
chutes.ai
clawhub.ai
copilot.microsoft.com
devin.ai
diabrowser.com
@@ -37,9 +38,12 @@ diabrowser.engineering
dify.ai
duck.ai
gateway.ai.cloudflare.com
h2o.ai
kiro.dev
lovart.ai
meta.ai
mistral.ai
openart.ai
openclaw.ai
openrouter.ai
spicywriter.com

View File

@@ -54,6 +54,7 @@ include:stackexchange
include:strikingly
include:termux
include:thelinuxfoundation
include:tilda
include:unity
include:v8
@@ -68,6 +69,7 @@ centos.org
chocolatey.org
clojure.org
cnpmjs.org
cve.org
cygwin.com
cython.org
deno.com
@@ -151,4 +153,5 @@ wireshark.org
x.org
xposed.info
yarnpkg.com
ziglang.org
zsh.org

View File

@@ -25,3 +25,4 @@ jinrishici.com
openvela.com
pingcode.com
tipdm.org
zuidaima.com

View File

@@ -10,6 +10,8 @@ include:shanbay
include:xueersi
include:yuanfudao
edu.cn
# 雨课堂
include:yuketang
## 雨豆课堂
@@ -19,8 +21,6 @@ yushiyan.net
## 学堂在线
xuetangx.com
edu.cn
# 国家智慧教育公共服务平台
cbern.com.cn
smartedu.cn
@@ -118,6 +118,11 @@ llscdn.com
llsserver.com
# 马哥教育
magedu.com
# 墨墨背单词
maimemo.com
maimemostatus.com
# 墨墨记忆卡
markji.com
# CIS NanoMasters
neoscholar.com
neoschool.com
@@ -127,6 +132,8 @@ oldboyedu.com
pigai.org
# 公考知识库
saduck.top
# 外研在线
unipus.cn
# 未来云校
weilaiyunxiao.com
# 北京嘉瑞新创教育咨询有限公司

View File

@@ -57,10 +57,14 @@ anitabi.cn
# 暴风影音
baofeng.com
baofeng.net
# 街机地图
bemanicn.com
# 布咕阅读
bgwxc.com
# B站空降助手
bsbsb.top
# CdkeyNoGap
cdkeynogap.com
# 动漫之家 #703
dmzj.com
muwai.com

View File

@@ -11,6 +11,7 @@ include:okko
include:wink
24h.tv
amediateka.ru
beeline.tv
ivi.ru
premier.one
smotreshka.tv

View File

@@ -131,6 +131,8 @@ wtfismyip.com
# Subdomains/internal api used for ip-geo-detect
full:checkip.amazonaws.com
full:ipv4-check-perf.radar.cloudflare.com
full:ipv6-check-perf.radar.cloudflare.com
geoip.noc.gov.ru
ip.mail.ru
ip.nic.ru

View File

@@ -3,33 +3,62 @@ include:openspeedtest
cnspeedtest.cn @cn
fast.com
fastspeedtest.com
linkmeter.net
measurementlab.net
meter.net
nperf.com
openspeedtest.ru
speed.cloudflare.com
speed.dler.io
speed.ee
speed.hinet.net
speed.nccu.edu.tw
speed.neu6.edu.cn @cn
speed.nju.edu.cn @cn
speed.nuaa.edu.cn @cn
speed.qlu.edu.cn @cn
speed.ujs.edu.cn @cn
speed6.ujs.edu.cn @cn
speed2.hinet.net
speed5.ntu.edu.tw
speed6.ujs.edu.cn @cn
speedcheck.org
speedgeo.net
speedof.me
speedtest.cesnet.cz
speedtest.ch
speedtest.citylink.pro
speedtest.cn @cn
speedtest.co.za
speedtest.de
speedtest.dno-it.ru
speedtest.frontier.com
speedtest.im
speedtest.mail.ru
speedtest.mfcyun.com @cn
speedtest.net.in
speedtest.net.ua
speedtest.net.uk
speedtest.org
speedtest.rt.ru
speedtest.ru
speedtest.shaw.ca
speedtest.shu.edu.cn @cn
speedtest6.shu.edu.cn @cn
speedtest.su
speedtest.uz
speedtest.volia.com
speedtest.xaut.edu.cn @cn
speedtest.xfinity.com
speedtestcustom.com
test.ustc.edu.cn @cn
test6.ustc.edu.cn @cn
speedtest.xyz
speedtest24.ru
speedtest6.shu.edu.cn @cn
test.nju.edu.cn @cn
test.ustc.edu.cn @cn
test6.nju.edu.cn @cn
speed.nju.edu.cn @cn
test6.ustc.edu.cn @cn
testmy.net
testmyspeed.com
testskorosti.ru
xnfz.seu.edu.cn @cn
full:hk-global-bgp.hkg.speedtest.yecaoyun.com @!cn

View File

@@ -1,3 +1,4 @@
allhistory.com
fgo.wiki
jikipedia.com
mbalib.com

View File

@@ -16,6 +16,9 @@ miguvideo.com
# 中移金科
cmft.com.cn
# 中移支付 和包
cmpay.com
# 中移在线
cmcc-cs.cn
online-cmcc.cn

View File

@@ -1,10 +1,11 @@
include:ctexcel
189.cn
bestpay.cn
bestpay.com.cn
chinatelecom-h.com
chinatelecom.com.cn
chinatelecomglobal.com @!cn
chinatelecomhk.com @!cn
chntel.com
vnet.cn
chinatelecomhk.com @!cn

View File

@@ -18,6 +18,7 @@ cuguplus.com @!cn
cuniq.com @!cn
cuniq.sg @!cn
mychinaunicom.com
unicompayment.com
wo.cn
wo.com.cn
wo116114.com

View File

@@ -65,3 +65,6 @@ supercachenode.com
ultracachenode.com
woaihuoshan.com
wohst8.com
# 抖音支付
douyinpay.com

View File

@@ -326,3 +326,5 @@ include:tsquare
include:windy
include:xdty
include:xingrz
sinyalee.com

View File

@@ -45,6 +45,8 @@ include:taikang
## 航财通·校园付
cc-pay.cn
## 合利宝
helipay.com
## 徽商期货有限责任公司
hsqh.net
## 上海陆金所
@@ -53,6 +55,8 @@ lufax.com
lufaxcdn.com
## 银豹收银系统
pospal.cn
## 上海付费通
shfft.com
# CDN or SDWAN
include:category-cdn-cn
@@ -222,23 +226,65 @@ cma.cn
nmc.cn
# Public transportation
include:airchina # 中国国际航空
include:hainanairlines # 海南航空
## 中国国际航空
include:airchina
## 海南航空
include:hainanairlines
## 长安航空
airchangan.com
## 桂林航空
airguilin.com
## 中国东方航空
ceair.com
## 春秋航空
ch.com
china-sss.com
springairlines.com
## 华夏航空
chinaexpressair.com
## 中国南方航空
csair.com
## 中国联合航空
flycua.com
## 广西北部湾航空
gxairlines.com
## 河北航空
hbhk.com.cn
## 首都航空
jdair.net
## 吉祥航空
juneyaoair.com
## 长龙航空
loongair.cn
## 祥鹏航空
luckyair.net
## 山东航空
sda.cn
## 深圳航空
shenzhenair.com
## 四川航空
sichuanair.com
## 天津航空
tianjin-air.com
## 西藏航空
tibetairlines.com.cn
## 乌鲁木齐航空
urumqi-air.com
## 厦门航空
xiamenair.com
## 中国民航局
caac.gov.cn
## 信天游(航空电子客票验真)
travelskymobile.cn
## 航旅纵横
umetrip.com
umetrip.com.cn
umetrip.net.cn
ume-trip.cn
## 飞常准
variflight.com
# 中国民航局 caac.gov.cn
ceair.com # 中国东方航空
chinaexpressair.com # 华夏航空
csair.com # 中国南方航空
flycua.com # 中国联合航空
hbhk.com.cn # 河北航空
jdair.net # 首都航空
juneyaoair.com # 吉祥航空
luckyair.net # 祥鹏航空
sichuanair.com # 四川航空
tibetairlines.com.cn # 西藏航空
travelskymobile.cn # 信天游(航空电子客票验真)
variflight.com # 飞常准
xiamenair.com # 厦门航空
## Railway
12306.cn
95306.cn
@@ -253,11 +299,6 @@ kuajing84.com
mygolbs.com
## 宁停车
ningtingche.com
## 航旅纵横
umetrip.com
umetrip.com.cn
umetrip.net.cn
ume-trip.cn
# Science & Technology & Knowledge & Scholar & Press & E-learning & Education
include:category-education-cn
@@ -284,6 +325,10 @@ guoxuemi.com
zyh365.com
## 温州市图书馆
wzlib.cn
## 中国大百科全书
zgbk.com
## 浙江图书馆
zjlib.cn
# Services & Softwares
include:category-ai-cn
@@ -421,6 +466,7 @@ xbext.com
yinxiang.com
## 智联招聘
zhaopin.cn
zhaopin.com
# Tech companies & Orginations
include:aisiku # 北京艾斯酷科技有限公司
@@ -642,7 +688,6 @@ ycrx360.com
52z.com
53kf.com
55.com
56.com
5577.com
55bbs.com
58pic.com
@@ -688,8 +733,6 @@ aiemy.com
aiketour.com
aiops.com
aipai.com
airchangan.com
airguilin.com
aiskycn.com
aizhan.com
aldwx.com # 京ICP备17030349号-1
@@ -753,7 +796,6 @@ cecdc.com
cecport.com
centanet.com
cerambath.org
ch.com
chachaba.com
changba.com
chaojituzi.net
@@ -938,7 +980,6 @@ gupzs.com
gushiwen.org
guzhangting.com
gwdang.com
gxairlines.com
gxb.io
gzcqs.com
gzwanju.com
@@ -1296,7 +1337,6 @@ taoche.com
te5.com
tenxcloud.com
tianjimedia.com
tianjin-air.com
tianlailive.com
tietuku.com
tiexue.net
@@ -1332,7 +1372,6 @@ ucbug.com
uggame.com
uoko.com
urselect.com
urumqi-air.com
useso.com
uuu9.com
uzzf.com
@@ -1496,7 +1535,6 @@ zhangyoubao.com
zhanzhang.net
zhaodanji.com
zhaokao.net
zhaopin.com
zhcw.com
zhenai.com
zhibo8.cc
@@ -1507,15 +1545,11 @@ zhongkao.com
zhuangjiba.com
zhuanyewanjia.com
zhuayoukong.com
zimuzu.io
zixiaomao.com
zixuntop.com
zmengzhu.com
zpb365.com
zrblog.net
zsxq.com
zuhaowan.com
zuidaima.com
zuowen.com
zuowen8.com
zuowenwang.net
@@ -1591,3 +1625,6 @@ ao-x.ac.cn
# 万集科技 京ICP备18036282号-2
wanji.net.cn
# 广州市雅望互联网服务有限公司
gzyowin.com

View File

@@ -1,2 +1,5 @@
calorietech.com
gotokeep.com
keep.com
keep.com.cn
keepcdn.com

View File

@@ -3,6 +3,7 @@ kinopub.online
kpdl.link
# Mirror sites
ahc.ovh # sub domains mirror
gfw.ovh # sub domains mirror
mos-gorsud.co # kinopub domain to generate a mirror site through gfw.ovh

View File

@@ -32,6 +32,7 @@ static163.net
ydstatic.com
yeah.net
youdao.com
yunxindns.com
yunxinfw.com
zhuanzfx.com

View File

@@ -1,5 +1,6 @@
22112211.xyz
deepflood.com
nodeget.com
nodeimage.com
nodequality.com
nodeseek.com

View File

@@ -59,6 +59,7 @@ tegrazone.com
tegrazone.jp
tegrazone.kr
full:nvidia.custhelp.com
full:nvidia.tt.omtrdc.net
# NVIDIA 文件下载服务器中国镜像

View File

@@ -1,10 +1,14 @@
# All .oracle domains
oracle
include:addthis
include:java
ateam-oracle.com
bronto.com
covid19-rx.org
covid19rx.org
custhelp.com
oracle.com
oraclecloud.com
oraclefoundation.org
@@ -12,6 +16,3 @@ oracleimg.com
oracleinfinity.io
sun.com
virtualbox.org
include:addthis
include:java

View File

@@ -258,13 +258,14 @@ tdnsv14.net
tdnsv15.net
# myqcloud inside mainland China
regexp:\.(.+-)?ap-beijing(-.+)?\.myqcloud\.com$ #北京
regexp:\.(.+-)?ap-nanjing(-.+)?\.myqcloud\.com$ #
regexp:\.(.+-)?ap-shanghai(-.+)?\.myqcloud\.com$ #上海
regexp:\.(.+-)?ap-guangzhou(-.+)?\.myqcloud\.com$ #广州
regexp:\.(.+-)?ap-chengdu(-.+)?\.myqcloud\.com$ #成都
regexp:\.(.+-)?ap-chongqing(-.+)?\.myqcloud\.com$ #重庆
regexp:\.(.+-)?ap-shenzhen(-.+)?\.myqcloud\.com$ #深圳
# overrided by myqcloud.com
#regexp:\.(.+-)?ap-beijing(-.+)?\.myqcloud\.com$ #
#regexp:\.(.+-)?ap-nanjing(-.+)?\.myqcloud\.com$ #南京
#regexp:\.(.+-)?ap-shanghai(-.+)?\.myqcloud\.com$ #上海
#regexp:\.(.+-)?ap-guangzhou(-.+)?\.myqcloud\.com$ #广州
#regexp:\.(.+-)?ap-chengdu(-.+)?\.myqcloud\.com$ #成都
#regexp:\.(.+-)?ap-chongqing(-.+)?\.myqcloud\.com$ #重庆
#regexp:\.(.+-)?ap-shenzhen(-.+)?\.myqcloud\.com$ #深圳
# COS 使用到的非中国大陆的地域与可用区,参见 https://cloud.tencent.com/document/product/436/6224
ap-hongkong.myqcloud.com @!cn #中国香港
@@ -282,13 +283,14 @@ eu-frankfurt.myqcloud.com @!cn #法兰克福
eu-moscow.myqcloud.com @!cn #莫斯科
# tencentcos inside mainland China
regexp:\.(.+-)?ap-beijing(-.+)?\.tencentcos\.(cn|com(\.cn)?)$ #北京
regexp:\.(.+-)?ap-nanjing(-.+)?\.tencentcos\.(cn|com(\.cn)?)$ #
regexp:\.(.+-)?ap-shanghai(-.+)?\.tencentcos\.(cn|com(\.cn)?)$ #上海
regexp:\.(.+-)?ap-guangzhou(-.+)?\.tencentcos\.(cn|com(\.cn)?)$ #广州
regexp:\.(.+-)?ap-chengdu(-.+)?\.tencentcos\.(cn|com(\.cn)?)$ #成都
regexp:\.(.+-)?ap-chongqing(-.+)?\.tencentcos\.(cn|com(\.cn)?)$ #重庆
regexp:\.(.+-)?ap-shenzhen(-.+)?\.tencentcos\.(cn|com(\.cn)?)$ #深圳
# overrided by tencentcos.cn, tencentcos.com, tencentcos.com.cn
#regexp:\.(.+-)?ap-beijing(-.+)?\.tencentcos\.(cn|com(\.cn)?)$ #
#regexp:\.(.+-)?ap-nanjing(-.+)?\.tencentcos\.(cn|com(\.cn)?)$ #南京
#regexp:\.(.+-)?ap-shanghai(-.+)?\.tencentcos\.(cn|com(\.cn)?)$ #上海
#regexp:\.(.+-)?ap-guangzhou(-.+)?\.tencentcos\.(cn|com(\.cn)?)$ #广州
#regexp:\.(.+-)?ap-chengdu(-.+)?\.tencentcos\.(cn|com(\.cn)?)$ #成都
#regexp:\.(.+-)?ap-chongqing(-.+)?\.tencentcos\.(cn|com(\.cn)?)$ #重庆
#regexp:\.(.+-)?ap-shenzhen(-.+)?\.tencentcos\.(cn|com(\.cn)?)$ #深圳
# tencentcos outside mainland China
# regexp:.+\.ap-hongkong\.tencentcos\.(cn|com(\.cn)?)$ @!cn #中国香港

View File

@@ -9,6 +9,10 @@ itc.cn
sohu.com
sohucs.com
# 56网
56.com
56img.com
# Ads/tracking
adnet.sohu.com @ads
ads.sohu.com @ads

5
data/tilda Normal file
View File

@@ -0,0 +1,5 @@
tilda.cc
tilda.ru
tilda.ws
tildaapi.com
tildacdn.com

View File

@@ -1,6 +1,7 @@
videopress.com
w.org
wordpress.com
wordpress.net
wordpress.org
wordpress.tv
wp-themes.com

332
main.go
View File

@@ -2,6 +2,7 @@ package main
import (
"bufio"
"encoding/json"
"flag"
"fmt"
"os"
@@ -19,6 +20,7 @@ var (
dataPath = flag.String("datapath", "./data", "Path to your custom 'data' directory")
outputName = flag.String("outputname", "dlc.dat", "Name of the generated dat file")
outputDir = flag.String("outputdir", "./", "Directory to place all generated files")
datProfile = flag.String("datprofile", "", "Path of config file used to assemble custom dats")
exportLists = flag.String("exportlists", "", "Lists to be flattened and exported in plaintext format, separated by ',' comma")
)
@@ -47,7 +49,24 @@ type Processor struct {
cirIncMap map[string]bool
}
func makeProtoList(listName string, entries []*Entry) (*router.GeoSite, error) {
type GeoSites struct {
Sites []*router.GeoSite
SiteIdx map[string]int
}
type DatTask struct {
Name string `json:"name"`
Mode string `json:"mode"`
Lists []string `json:"lists"`
}
const (
ModeAll string = "all"
ModeAllowlist string = "allowlist"
ModeDenylist string = "denylist"
)
func makeProtoList(listName string, entries []*Entry) *router.GeoSite {
site := &router.GeoSite{
CountryCode: listName,
Domain: make([]*router.Domain, 0, len(entries)),
@@ -73,7 +92,91 @@ func makeProtoList(listName string, entries []*Entry) (*router.GeoSite, error) {
}
site.Domain = append(site.Domain, pdomain)
}
return site, nil
return site
}
func loadTasks(path string) ([]DatTask, error) {
f, err := os.Open(path)
if err != nil {
return nil, err
}
defer f.Close()
var tasks []DatTask
dec := json.NewDecoder(f)
if err := dec.Decode(&tasks); err != nil {
return nil, fmt.Errorf("failed to decode json: %w", err)
}
for i, t := range tasks {
if t.Name == "" {
return nil, fmt.Errorf("task[%d]: name is required", i)
}
switch t.Mode {
case ModeAll, ModeAllowlist, ModeDenylist:
default:
return nil, fmt.Errorf("task[%d] %q: invalid mode %q", i, t.Name, t.Mode)
}
}
return tasks, nil
}
func (gs *GeoSites) assembleDat(task DatTask) error {
datFileName := strings.ToLower(filepath.Base(task.Name))
geoSiteList := new(router.GeoSiteList)
switch task.Mode {
case ModeAll:
geoSiteList.Entry = gs.Sites
case ModeAllowlist:
allowedIdxes := make([]int, 0, len(task.Lists))
for _, list := range task.Lists {
if idx, ok := gs.SiteIdx[strings.ToUpper(list)]; ok {
allowedIdxes = append(allowedIdxes, idx)
} else {
return fmt.Errorf("list %q not found for allowlist task", list)
}
}
slices.Sort(allowedIdxes)
allowedlen := len(allowedIdxes)
if allowedlen == 0 {
return fmt.Errorf("allowlist needs at least one valid list")
}
geoSiteList.Entry = make([]*router.GeoSite, allowedlen)
for i, idx := range allowedIdxes {
geoSiteList.Entry[i] = gs.Sites[idx]
}
case ModeDenylist:
deniedMap := make(map[int]bool, len(task.Lists))
for _, list := range task.Lists {
if idx, ok := gs.SiteIdx[strings.ToUpper(list)]; ok {
deniedMap[idx] = true
} else {
fmt.Printf("[Warn] list %q not found in denylist task %q", list, task.Name)
}
}
deniedlen := len(deniedMap)
if deniedlen == 0 {
fmt.Printf("[Warn] nothing to deny in task %q", task.Name)
geoSiteList.Entry = gs.Sites
} else {
geoSiteList.Entry = make([]*router.GeoSite, 0, len(gs.Sites)-deniedlen)
for i, site := range gs.Sites {
if !deniedMap[i] {
geoSiteList.Entry = append(geoSiteList.Entry, site)
}
}
}
}
protoBytes, err := proto.Marshal(geoSiteList)
if err != nil {
return fmt.Errorf("failed to marshal: %w", err)
}
if err := os.WriteFile(filepath.Join(*outputDir, datFileName), protoBytes, 0644); err != nil {
return fmt.Errorf("failed to write file %q: %w", datFileName, err)
}
fmt.Printf("dat %q has been generated successfully\n", datFileName)
return nil
}
func writePlainList(listname string, entries []*Entry) error {
@@ -89,46 +192,28 @@ func writePlainList(listname string, entries []*Entry) error {
return w.Flush()
}
func parseEntry(line string) (*Entry, []string, error) {
entry := new(Entry)
parts := strings.Fields(line)
func parseEntry(typ, rule string) (*Entry, []string, error) {
entry := &Entry{Type: typ}
parts := strings.Fields(rule)
if len(parts) == 0 {
return entry, nil, fmt.Errorf("empty line")
return entry, nil, fmt.Errorf("empty domain rule")
}
// Parse type and value
typ, val, isTypeSpecified := strings.Cut(parts[0], ":")
typ = strings.ToLower(typ)
if !isTypeSpecified { // Default RuleType
if !validateDomainChars(typ) {
return entry, nil, fmt.Errorf("invalid domain: %q", typ)
// Parse value
switch entry.Type {
case dlc.RuleTypeRegexp:
if _, err := regexp.Compile(parts[0]); err != nil {
return entry, nil, fmt.Errorf("invalid regexp %q: %w", parts[0], err)
}
entry.Type = dlc.RuleTypeDomain
entry.Value = typ
} else {
switch typ {
case dlc.RuleTypeRegexp:
if _, err := regexp.Compile(val); err != nil {
return entry, nil, fmt.Errorf("invalid regexp %q: %w", val, err)
}
entry.Type = dlc.RuleTypeRegexp
entry.Value = val
case dlc.RuleTypeInclude:
entry.Type = dlc.RuleTypeInclude
entry.Value = strings.ToUpper(val)
if !validateSiteName(entry.Value) {
return entry, nil, fmt.Errorf("invalid included list name: %q", entry.Value)
}
case dlc.RuleTypeDomain, dlc.RuleTypeFullDomain, dlc.RuleTypeKeyword:
entry.Type = typ
entry.Value = strings.ToLower(val)
if !validateDomainChars(entry.Value) {
return entry, nil, fmt.Errorf("invalid domain: %q", entry.Value)
}
default:
return entry, nil, fmt.Errorf("invalid type: %q", typ)
entry.Value = parts[0]
case dlc.RuleTypeDomain, dlc.RuleTypeFullDomain, dlc.RuleTypeKeyword:
entry.Value = strings.ToLower(parts[0])
if !validateDomainChars(entry.Value) {
return entry, nil, fmt.Errorf("invalid domain: %q", entry.Value)
}
default:
return entry, nil, fmt.Errorf("unknown rule type: %q", entry.Type)
}
plen := len(entry.Type) + len(entry.Value) + 1
// Parse attributes and affiliations
var affs []string
@@ -140,6 +225,7 @@ func parseEntry(line string) (*Entry, []string, error) {
return entry, affs, fmt.Errorf("invalid attribute: %q", attr)
}
entry.Attrs = append(entry.Attrs, attr)
plen += 2 + len(attr)
case '&':
aff := strings.ToUpper(part[1:])
if !validateSiteName(aff) {
@@ -147,33 +233,70 @@ func parseEntry(line string) (*Entry, []string, error) {
}
affs = append(affs, aff)
default:
return entry, affs, fmt.Errorf("invalid attribute/affiliation: %q", part)
return entry, affs, fmt.Errorf("unknown field: %q", part)
}
}
if entry.Type != dlc.RuleTypeInclude {
slices.Sort(entry.Attrs) // Sort attributes
// Formated plain entry: type:domain.tld:@attr1,@attr2
var plain strings.Builder
plain.Grow(len(entry.Type) + len(entry.Value) + 10)
plain.WriteString(entry.Type)
plain.WriteByte(':')
plain.WriteString(entry.Value)
for i, attr := range entry.Attrs {
if i == 0 {
plain.WriteByte(':')
} else {
plain.WriteByte(',')
}
plain.WriteByte('@')
plain.WriteString(attr)
slices.Sort(entry.Attrs) // Sort attributes
// Formated plain entry: type:domain.tld:@attr1,@attr2
var plain strings.Builder
plain.Grow(plen)
plain.WriteString(entry.Type)
plain.WriteByte(':')
plain.WriteString(entry.Value)
for i, attr := range entry.Attrs {
if i == 0 {
plain.WriteByte(':')
} else {
plain.WriteByte(',')
}
entry.Plain = plain.String()
plain.WriteByte('@')
plain.WriteString(attr)
}
entry.Plain = plain.String()
return entry, affs, nil
}
func parseInclusion(rule string) (*Inclusion, error) {
parts := strings.Fields(rule)
if len(parts) == 0 {
return nil, fmt.Errorf("empty inclusion")
}
inc := &Inclusion{Source: strings.ToUpper(parts[0])}
if !validateSiteName(inc.Source) {
return inc, fmt.Errorf("invalid included list name: %q", inc.Source)
}
// Parse attributes
for _, part := range parts[1:] {
switch part[0] {
case '@':
attr := strings.ToLower(part[1:])
if attr[0] == '-' {
battr := attr[1:]
if !validateAttrChars(battr) {
return inc, fmt.Errorf("invalid ban attribute: %q", battr)
}
inc.BanAttrs = append(inc.BanAttrs, battr)
} else {
if !validateAttrChars(attr) {
return inc, fmt.Errorf("invalid must attribute: %q", attr)
}
inc.MustAttrs = append(inc.MustAttrs, attr)
}
case '&':
return inc, fmt.Errorf("affiliation is not allowed for inclusion")
default:
return inc, fmt.Errorf("unknown field: %q", part)
}
}
return inc, nil
}
func validateDomainChars(domain string) bool {
if domain == "" {
return false
}
for i := range domain {
c := domain[i]
if (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') || c == '.' || c == '-' {
@@ -185,9 +308,12 @@ func validateDomainChars(domain string) bool {
}
func validateAttrChars(attr string) bool {
if attr == "" {
return false
}
for i := range attr {
c := attr[i]
if (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') || c == '!' || c == '-' {
if (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') || c == '!' {
continue
}
return false
@@ -196,6 +322,9 @@ func validateAttrChars(attr string) bool {
}
func validateSiteName(name string) bool {
if name == "" {
return false
}
for i := range name {
c := name[i]
if (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '!' || c == '-' {
@@ -232,26 +361,23 @@ func (p *Processor) loadData(listName string, path string) error {
if line == "" {
continue
}
entry, affs, err := parseEntry(line)
if err != nil {
return fmt.Errorf("error in %q at line %d: %w", path, lineIdx, err)
typ, rule, isTypeSpecified := strings.Cut(line, ":")
if !isTypeSpecified { // Default RuleType
typ, rule = dlc.RuleTypeDomain, typ
} else {
typ = strings.ToLower(typ)
}
if entry.Type == dlc.RuleTypeInclude {
inc := &Inclusion{Source: entry.Value}
for _, attr := range entry.Attrs {
if attr[0] == '-' {
inc.BanAttrs = append(inc.BanAttrs, attr[1:])
} else {
inc.MustAttrs = append(inc.MustAttrs, attr)
}
}
for _, aff := range affs {
apl := p.getOrCreateParsedList(aff)
apl.Inclusions = append(apl.Inclusions, inc)
if typ == dlc.RuleTypeInclude {
inc, err := parseInclusion(rule)
if err != nil {
return fmt.Errorf("error in %q at line %d: %w", path, lineIdx, err)
}
pl.Inclusions = append(pl.Inclusions, inc)
} else {
entry, affs, err := parseEntry(typ, rule)
if err != nil {
return fmt.Errorf("error in %q at line %d: %w", path, lineIdx, err)
}
for _, aff := range affs {
apl := p.getOrCreateParsedList(aff)
apl.Entries = append(apl.Entries, entry)
@@ -259,7 +385,7 @@ func (p *Processor) loadData(listName string, path string) error {
pl.Entries = append(pl.Entries, entry)
}
}
return nil
return scanner.Err()
}
func isMatchAttrFilters(entry *Entry, incFilter *Inclusion) bool {
@@ -360,6 +486,9 @@ func (p *Processor) resolveList(plname string) error {
}
}
}
if len(roughMap) == 0 {
return fmt.Errorf("empty list")
}
p.finalMap[plname] = polishList(roughMap)
return nil
}
@@ -387,13 +516,15 @@ func run() error {
return fmt.Errorf("failed to loadData: %w", err)
}
// Generate finalMap
processor.finalMap = make(map[string][]*Entry, len(processor.plMap))
sitesCount := len(processor.plMap)
processor.finalMap = make(map[string][]*Entry, sitesCount)
processor.cirIncMap = make(map[string]bool)
for plname := range processor.plMap {
if err := processor.resolveList(plname); err != nil {
return fmt.Errorf("failed to resolveList %q: %w", plname, err)
}
}
processor.plMap = nil
// Make sure output directory exists
if err := os.MkdirAll(*outputDir, 0755); err != nil {
@@ -403,47 +534,58 @@ func run() error {
for rawEpList := range strings.SplitSeq(*exportLists, ",") {
if epList := strings.TrimSpace(rawEpList); epList != "" {
entries, exist := processor.finalMap[strings.ToUpper(epList)]
if !exist || len(entries) == 0 {
fmt.Printf("list %q does not exist or is empty\n", epList)
if !exist {
fmt.Printf("[Warn] list %q does not exist\n", epList)
continue
}
if err := writePlainList(epList, entries); err != nil {
fmt.Printf("failed to write list %q: %v\n", epList, err)
fmt.Printf("[Error] failed to write list %q: %v\n", epList, err)
continue
}
fmt.Printf("list %q has been generated successfully.\n", epList)
fmt.Printf("list %q has been generated successfully\n", epList)
}
}
// Generate dat file
protoList := new(router.GeoSiteList)
for siteName, siteEntries := range processor.finalMap {
site, err := makeProtoList(siteName, siteEntries)
if err != nil {
return fmt.Errorf("failed to makeProtoList %q: %w", siteName, err)
}
protoList.Entry = append(protoList.Entry, site)
// Generate proto sites
gs := &GeoSites{
Sites: make([]*router.GeoSite, 0, sitesCount),
SiteIdx: make(map[string]int, sitesCount),
}
// Sort protoList so the marshaled list is reproducible
slices.SortFunc(protoList.Entry, func(a, b *router.GeoSite) int {
for siteName, siteEntries := range processor.finalMap {
gs.Sites = append(gs.Sites, makeProtoList(siteName, siteEntries))
}
processor = nil
// Sort proto sites so the generated file is reproducible
slices.SortFunc(gs.Sites, func(a, b *router.GeoSite) int {
return strings.Compare(a.CountryCode, b.CountryCode)
})
for i := range sitesCount {
gs.SiteIdx[gs.Sites[i].CountryCode] = i
}
protoBytes, err := proto.Marshal(protoList)
if err != nil {
return fmt.Errorf("failed to marshal: %w", err)
// Load tasks and generate dat files
var tasks []DatTask
if *datProfile == "" {
tasks = []DatTask{{Name: *outputName, Mode: ModeAll}}
} else {
var err error
tasks, err = loadTasks(*datProfile)
if err != nil {
return fmt.Errorf("failed to loadTasks %q: %v", *datProfile, err)
}
}
if err := os.WriteFile(filepath.Join(*outputDir, *outputName), protoBytes, 0644); err != nil {
return fmt.Errorf("failed to write output: %w", err)
for _, task := range tasks {
if err := gs.assembleDat(task); err != nil {
fmt.Printf("[Error] failed to assembleDat %q: %v", task.Name, err)
}
}
fmt.Printf("%q has been generated successfully.\n", *outputName)
return nil
}
func main() {
flag.Parse()
if err := run(); err != nil {
fmt.Printf("Fatal error: %v\n", err)
fmt.Printf("[Fatal] critical error: %v\n", err)
os.Exit(1)
}
}