Compare commits

...

19 Commits

Author SHA1 Message Date
MkQtS
db9c0fe466 main.go: improve codes (#3290) 2026-02-19 17:24:13 +08:00
Konstantin
109a50f048 Add more ru bank/financial domains (#3264) 2026-02-19 14:36:27 +08:00
EvanLiu2000
b258a6594e Update missav & category-porn (#3289) 2026-02-19 14:30:57 +08:00
Andrey Cherepkov
8d5ef88729 Update Russian websites (#3269)
* add category-bank-ru and category-finance-ru

* category-ecommerce-ru: include avito

* category-entertainment: add megogo.net

* add kinopoisk

* category-entertainment-ru: add more domains

* category-gov-ru: add more domains

* category-retail-ru: add more domains

* add category-travel-ru

* mailru-group: add more domains

* category-ru: add more domains

* Update category-retail-ru: remove duplicates as they are in x5

* Update category-travel-ru: return taxsee to the ru category

* Update category-ru: add taxsee

* cleanup and reorder

---------

Co-authored-by: MkQtS <81752398+MkQtS@users.noreply.github.com>
2026-02-19 14:10:09 +08:00
MkQtS
491d185657 category-cryptocurrency: add bitcoincore.org (#3287) 2026-02-17 21:16:32 +08:00
Emik
d8852e85ea Add yostar (#3283) 2026-02-17 20:54:50 +08:00
MkQtS
fb9754a371 add zscaler (#3285) 2026-02-17 18:07:56 +08:00
MkQtS
a5fac50ae6 add anker (#3284) 2026-02-17 18:06:03 +08:00
TripleA
94d33011d4 create tld-ru (#3280) 2026-02-17 16:52:16 +08:00
MkQtS
f1688340cd category-cryptocurrency: add walletconnect.org (#3281)
Co-authored-by: sinalphabeta <53267553+sinalphabeta@users.noreply.github.com>
2026-02-15 14:38:51 +08:00
sinalphabeta
515262667d Update category-cryptocurrency (#3277) 2026-02-15 13:16:55 +08:00
MkQtS
06af8b92a1 Add airbnb (#3279)
* add airbnb

* airbnb: add more domains

---------

Co-authored-by: Andrey Cherepkov <4404804+ACherepkov1989@users.noreply.github.com>
2026-02-15 10:58:53 +08:00
MkQtS
be01a25015 Add 2gis (#3278)
* add 2gis

* 2gis: add more domains

---------

Co-authored-by: Andrey Cherepkov <4404804+ACherepkov1989@users.noreply.github.com>
2026-02-15 10:58:33 +08:00
MkQtS
98d161e89a Create pull_request_template.md (#3276)
[skip ci]
2026-02-14 21:19:33 +08:00
NetworK
8523573ba1 telegram: add ton.org (#3273) 2026-02-14 09:00:22 +08:00
jinqiang zhang
94997e5bd6 add grapheneos (#3272) 2026-02-13 22:43:43 +08:00
mcdemon05
939492e1b7 ubisoft: add more domains (#3201) 2026-02-13 22:01:28 +08:00
fanymagnet
d059d7c595 Add HDrezka App information and new domain (#3271) 2026-02-13 21:31:49 +08:00
unixty
8c42635f44 kinopub: add kpdl.link (#3262)
short link for downloading apk
2026-02-13 21:10:05 +08:00
35 changed files with 553 additions and 162 deletions

14
.github/pull_request_template.md vendored Normal file
View File

@@ -0,0 +1,14 @@
<!--
Thanks for your contribution!
Please check the following items: (not mandatory)
- Adjacent rules should be sorted alphabetically
- It's not encouraged to create new list/file containing too few (one or two) rules
- Newly added list should be included in relevant categories if possible
- Subdomains are unnecessary as they are overridden by the parent domain. e.g. `[domain:]example.com` overrides `[domain:]app.example.com`
- Description for your changes is welcome (why the change is necessary, data source of added domains, potential impacts, etc.)
-->

15
data/2gis Normal file
View File

@@ -0,0 +1,15 @@
# https://crt.sh/?id=24119301132
2gis.ae
2gis.am
2gis.az
2gis.by
2gis.com
2gis.com.cy
2gis.cz
2gis.ge
2gis.kg
2gis.kz
2gis.ru
2gis.tj
2gis.ua
2gis.uz

86
data/airbnb Normal file
View File

@@ -0,0 +1,86 @@
# https://crt.sh/?id=23815053677
# https://crt.sh/?id=24021548393
# https://crt.sh/?id=6327797853
abb.travel
accomable.com
airbnb.ae
airbnb.al
airbnb.am
airbnb.at
airbnb.az
airbnb.ba
airbnb.be
airbnb.ca
airbnb.cat
airbnb.ch
airbnb.cl
airbnb.cn
airbnb.co.cr
airbnb.co.id
airbnb.co.il
airbnb.co.in
airbnb.co.kr
airbnb.co.nz
airbnb.co.uk
airbnb.co.ve
airbnb.co.za
airbnb.com
airbnb.com.ar
airbnb.com.au
airbnb.com.bo
airbnb.com.br
airbnb.com.bz
airbnb.com.co
airbnb.com.ec
airbnb.com.ee
airbnb.com.gt
airbnb.com.hk
airbnb.com.hn
airbnb.com.hr
airbnb.com.kh
airbnb.com.mt
airbnb.com.my
airbnb.com.ni
airbnb.com.pa
airbnb.com.pe
airbnb.com.ph
airbnb.com.py
airbnb.com.ro
airbnb.com.sg
airbnb.com.sv
airbnb.com.tr
airbnb.com.tw
airbnb.com.ua
airbnb.com.vn
airbnb.cz
airbnb.de
airbnb.dk
airbnb.es
airbnb.fi
airbnb.fr
airbnb.gr
airbnb.gy
airbnb.hu
airbnb.ie
airbnb.is
airbnb.it
airbnb.jp
airbnb.la
airbnb.lt
airbnb.lu
airbnb.lv
airbnb.me
airbnb.mx
airbnb.nl
airbnb.no
airbnb.org
airbnb.pl
airbnb.pt
airbnb.rs
airbnb.ru
airbnb.se
airbnb.si
airbnb.tools
airbnb.travel
muscache.com
vamo.com

12
data/anker Normal file
View File

@@ -0,0 +1,12 @@
anker-in.com @!cn
anker.com @!cn
anker.com.cn
ankersolix.com @!cn
ankerwork.cn
eufy.com @!cn
eufycn.com
eufylife.com @!cn
eufymake.com @!cn
eufymake.com.cn
soundcore.com @!cn
soundcore.com.cn

40
data/category-bank-ru Normal file
View File

@@ -0,0 +1,40 @@
# Alfa Bank
alfabank.ru
# GazpromBank
gazprombank.ru
gpb.ru
# Mts dengi
dbo-dengi.online
mtsdengi.ru
# PSB Bank
psbank.ru
# RosBank
bankline.ru
rosbank.ru
# RUSSIA Bank
abr.ru
# Russian Agricultural Bank
rshb.ru
# SberBank
sber.ru
sberbank.com
sberbank.ru
# T-Bank (Tinkoff)
cdn-tinkoff.ru
tbank-online.com
tbank.ru
# Tochka bank
tochka-tech.com
tochka.com
# VTB Bank
vtb.ru

View File

@@ -1,5 +1,6 @@
# This list contains Tech & Cyber companies and organizations outside China mainland.
include:2gis
include:acer
include:adobe
include:alibabacloud
@@ -91,6 +92,7 @@ include:westerndigital
include:wisekey
include:yahoo
include:yandex
include:zscaler
# Remote Control
include:category-remote-control

View File

@@ -28,6 +28,7 @@ aave.com
bingx.com
bitbank.cc
bitcoin.org
bitcoincore.org
bitfinex.com
bitget.com
bitmex.com
@@ -55,6 +56,7 @@ dogecoin.com
dydx.exchange
fundingrates.xyz
gemini.com
gmgn.ai
guardarian.com
hashflow.com
infura.io
@@ -77,5 +79,7 @@ truefi.io
unisat.io
uniswap.org
walletconnect.com
walletconnect.org
web3modal.org
wintermute.com
zapper.fi

View File

@@ -1,6 +1,7 @@
include:category-ecommerce-ru
include:adidas
include:airbnb
include:bestbuy
include:booking
include:books

View File

@@ -1,2 +1,3 @@
include:avito
include:ozon
include:wildberries

View File

@@ -89,6 +89,7 @@ joox.com
kpacg.com
linetv.tw
megaphone.fm
megogo.net
mikanani.me
moov.hk
odysee.com

View File

@@ -1,3 +1,22 @@
# CDNvideo is a leading CDN provider in Russia and the CIS
cdnvideo.ru
trbcdn.net
# Information about films and series
kinorium.com
# Streaming services for watching TV series and movies
include:kinopoisk
include:okko
include:rutube
include:wink
24h.tv
amediateka.ru
ivi.ru
premier.one
smotreshka.tv
start.ru
tvigle.ru
viju.ru
# Video hosting
include:rutube

View File

@@ -2,6 +2,7 @@
include:category-bank-ir
include:category-bank-jp
include:category-bank-mm
include:category-bank-ru
include:fibank
include:futu

View File

@@ -41,6 +41,7 @@ include:vrchat
include:wbgames
include:xbox
include:ynoproject
include:yostar
dinopoloclub.com
dodi-repacks.download

View File

@@ -15,7 +15,13 @@ edu.ru
# Other Federal Resources
cbr.ru # Central Bank of Russia
cikrf.ru # Central Electoral Commission of the Russian Federation
ebs.ru # Unified Biometric System
goskey.ru # GosKey - an electronic signature on a smartphone
izbirkom.ru # Information on ongoing elections and referendums
kremlin.ru # Online representation of the President of Russia
nalog.ru # Federal Tax Service
xn--80ajghhoc2aj1c8b.xn--p1ai # Honest Sign - State Labeling System
#
# Regional sites and Public Services
@@ -149,3 +155,6 @@ chukotka.ru # Chukotka Autonomous Area
jamal.ru # Yamal-Nenets Autonomous Area
surgut.ru # Khanty-Mansi Autonomous Area Yugra
yamal.ru # Yamal-Nenets Autonomous Area
# Regional health services
zdrav10.ru # Republic of Karelia

View File

@@ -4,6 +4,7 @@ include:creativecommons
include:csis
include:emojipedia
include:f-droid
include:grapheneos
include:ieee
include:metabrainz
include:nist

View File

@@ -2223,6 +2223,8 @@ heavyfetish.com
hegre.com
heiguab.top
heijidi.life
heiliao.com
heiliao88.com
heise360181.buzz
heise360182.buzz
helixstudios.net
@@ -6157,6 +6159,7 @@ regexp:(^|\.)tqav[1-9][0-9]\.com$
regexp:(^|\.)tt[1-2][0-9]\.tv$
regexp:(^|\.)ttghg[1-9][0-9]\.xyz$
regexp:(^|\.)tttv([1-9][0-9]?|100)\.com$
regexp:(^|\.)twav[1-9]\.xyz$
regexp:(^|\.)twseb([1-9][0-9]?)?\.com$
regexp:(^|\.)uu[a-z][1-9][0-9]?\.com$
regexp:(^|\.)whtdh0[1-3]\.cc$

View File

@@ -1,6 +1,69 @@
# ABC of Taste
av.ru
# Auchan
auchan.ru
# Bristol
bristol.ru
# Children's World
detmir.ru
# Dixie
dixy.ru
# Dodo Pizza
dodois.com
dodois.io
dodopizza.com
dodopizza.ru
dodostatic.net
# Dostaevsky
dostaevsky.ru
# Eurasia
evrasia.rest
# Fast food restaurant chain "Vkusno i tochka"
vkusnoitochka.ru
# Fix Price
fix-price.com
# Lenta
lenta.com
lenta.tech
# Magnet
magnit.ru
# Metro
metro-cc.ru
# Pyaterochka, Perekrestok, Chizhik
include:x5
# Red and White
krasnoeibeloe.ru
# Rostics
rostics.ru
uni.rest
unirest.tech
# Sakura
ilovesakura.ru
# Spar
myspar.ru
# Sushi Wok
sushiwok.ru
# Vinlab
winelab.ru
# VkusVill
vkusvill.ru

View File

@@ -1,47 +1,57 @@
# Russian websites
# Russian TLDs
moscow
tatar
ru
su
xn--80adxhks
xn--80asehdb
xn--80aswg
xn--c1avg
xn--d1acj3b
xn--p1acf
xn--p1ai
ru.com
ru.net
include:tld-ru
include:category-ecommerce-ru
include:category-entertainment-ru
include:category-gov-ru
include:category-retail-ru
include:category-travel-ru
# Public transportation
include:aviasales
include:mosmetro
include:avito
# Well-known companies
include:mailru-group
include:x5
include:yandex
# Dodo Pizza
dodopizza.ru
dodopizza.com
dodostatic.net
dodois.com
dodois.io
# Bank & Finance & Insurance & Securities
include:category-bank-ru
# Credit History Bureaus
credistory.ru
nbki.ru
# Currency and stock exchanges
moex.com # Moscow Stock Exchange
spvb.ru # Saint-Petersburg Stock Exchange
# Financial marketplace
banki.ru
finuslugi.ru
# Investment
sistema-capital.com
# Mir payment system
mirpayonline.ru
# National Payment Card System
nspk.ru
# Tipping service
netmonet.co
tips.tips
# Rostics
rostics.ru
uni.rest
unirest.tech
# Telecom operators
beeline.ru
dom.ru
megafon.ru
mts.ru
mymts.ru
rostelecom.ru
rt.ru
t2.ru
tele2.ru
yota.ru
trbcdn.net
tbank-online.com
taxsee.com
# Other domains
gazfond-pn.ru # Non-state pension fund GAZFOND pension savings
litres.ru # E-book and audiobook service
meteoinfo.ru # Hydrometeorological Center of Russia
ngenix.net # NGENIX is a Russian provider of acceleration and security services for public web resources
pochta.ru # Russian post
qms.ru # Russian internet speed testing service
rustore.ru # RuStore is a Russian mobile app store for Android
taxsee.com # Taxi for business (self-employed drivers)

38
data/category-travel-ru Normal file
View File

@@ -0,0 +1,38 @@
# Aviasales
include:aviasales
# BlaBlaCar
blablacar.ru
# Bus stations
avtovokzaly.ru
# Ostrovok
ostrovok.ru
# Othello from 2GIS
otello.ru
# Public transport and ticketing
include:mosmetro
full:bilet.nspk.ru
full:metro.spb.ru
full:mos.transport.vtb.ru
full:podorozhnik.spb.ru
full:transport.mos.ru
gorodpay.ru
rrtp.ru
sbertroika.ru
# Russian Railways
rzd-bonus.ru
rzd.ru
# Tutu
tutu.ru
# Yandex Rent
full:arenda.yandex.ru
# Yandex Taxi
full:taxi.yandex.ru

View File

@@ -422,6 +422,7 @@ zhaopin.cn
include:aisiku # 北京艾斯酷科技有限公司
include:akiko # 秋子酱科技
include:alibaba @-!cn
include:anker @-!cn
include:baidu
include:beisen
include:bluepoch

16
data/grapheneos Normal file
View File

@@ -0,0 +1,16 @@
attestation.app
grapheneos.app
grapheneos.ca
grapheneos.com
grapheneos.dev
grapheneos.foundation
grapheneos.info
grapheneos.net
grapheneos.network
grapheneos.online
grapheneos.org
grapheneos.ovh
grapheneos.page
grapheneos.social
seamlessupdate.app
vanadium.app

View File

@@ -8,5 +8,8 @@ hdrezka.org
hdrezka.site
rezka.ag
# HDrezka App for Android
hdrzk.org
# hdrezka video CDN servers
stream.voidboost.cc

2
data/kinopoisk Normal file
View File

@@ -0,0 +1,2 @@
kinopoisk.ru
kinopoisk-ru.clstorage.net

View File

@@ -1,5 +1,6 @@
kino.pub
kinopub.online
kpdl.link
# Mirror sites
gfw.ovh # sub domains mirror

View File

@@ -5,5 +5,7 @@ include:vk
boosty.to
donationalerts.com
max.ru
memealerts.com
oneme.ru
tamtam.chat

View File

@@ -1,5 +1,6 @@
missav.ai
missav.com
missav.live
missav.uno
missav.vip
missav.ws

View File

@@ -15,5 +15,6 @@ telegram.org
telegram.space
telesco.pe
tg.dev
ton.org
tx.me
usercontent.dev

View File

@@ -18,6 +18,7 @@
# xn--j6w193g # 香港 Hong Kong SAR, People's Republic of China
include:google-registry-tld
include:tld-ru
# Used by U.S. federal government
gov
@@ -209,7 +210,6 @@ qa # Qatar
re # Réunion Island
ro # Romania
rs # Serbia
ru # Russia
rw # Rwanda
sa # Saudi Arabia
sb # Solomon Islands
@@ -229,7 +229,6 @@ so # Somaliland
sr # Suriname
ss # South Sudan
st # São Tomé and Príncipe
su # The Union of Soviet Socialist Republic
sv # El Salvador
sx # Sint Maarten
sy # Syria
@@ -329,7 +328,6 @@ okinawa # Okinawa, Japan
osaka # Osaka, Japan
ryukyu # Ryukyu Islands, Japan
taipei # Taipei, Republic of China (Taiwan)
tatar # Tatar peoples and places
tokyo # Tokyo, Japan
yokohama # Yokohama, Japan
@@ -358,7 +356,6 @@ istanbul # İstanbul, Turkey
koeln # Cologne, Germany
london # London, United Kingdom
madrid # Madrid, Spain
moscow # Moscow, Russia
nrw # North Rhine-Westphalia, Germany
paris # Paris, France
ruhr # Ruhr, Germany
@@ -467,7 +464,6 @@ shell # Shell Information Technology International Inc
shia # Shia Muslim community
style # though the comment actually seems to be about .tattoo
sucks # Vox Populi Registry
tatar # Limited Liability Company "Coordination Center of Regional Domain of Tatarstan Republic"
tattoo # Tattoo
vana # Scripps Networks Interactive, Inc.
virgin # Virgin Enterprises Limited
@@ -557,7 +553,6 @@ xn--mgbai9azgqp6j # پاکستان Pakistan
xn--mgbai9a5eva00b # پاكستان Pakistan
xn--ygbi2ammx # فلسطين Palestine
xn--wgbl6a # قطر Qatar
xn--p1ai # рф Russian Federation
xn--mgberp4a5d4ar # السعودية Saudi Arabia
xn--mgberp4a5d4a87g # السعودیة Saudi Arabia
xn--mgbqly7c0a67fbc # السعودیۃ Saudi Arabia

12
data/tld-ru Normal file
View File

@@ -0,0 +1,12 @@
# Russian TLDs
moscow
ru
su
tatar
xn--80adxhks
xn--80asehdb
xn--80aswg
xn--c1avg
xn--d1acj3b
xn--p1acf
xn--p1ai

View File

@@ -6,5 +6,29 @@ ubisoftconnect.com
uplay.com
full:ubisoft-uplay-savegames.s3.amazonaws.com
full:ubisoft-orbit-savegames.s3.amazonaws.com
full:ubistatic1-a.akamaihd.net
full:ubistatic2-a.akamaihd.net
full:ubistatic3-a.akamaihd.net
full:ubistatic4-a.akamaihd.net
full:ubistatic5-a.akamaihd.net
full:ubistatic6-a.akamaihd.net
full:ubistatic7-a.akamaihd.net
full:ubistatic8-a.akamaihd.net
full:ubistatic9-a.akamaihd.net
full:ubistatic10-a.akamaihd.net
full:ubistatic11-a.akamaihd.net
full:ubistatic12-a.akamaihd.net
full:ubistatic13-a.akamaihd.net
full:ubistatic14-a.akamaihd.net
full:ubistatic15-a.akamaihd.net
full:ubistatic16-a.akamaihd.net
full:ubistatic17-a.akamaihd.net
full:ubistatic18-a.akamaihd.net
full:ubistatic19-a.akamaihd.net
full:ubistatic20-a.akamaihd.net
full:ubistatic21-a.akamaihd.net
full:ubistatic22-a.akamaihd.net
full:ubistatic23-a.akamaihd.net
full:ubistatic24-a.akamaihd.net
full:ubisoft.siteintercept.qualtrics.com
full:uplaypc-s-ubisoft.cdn.ubi.com @cn

View File

@@ -1,3 +1,2 @@
ngenix.net
restream-media.net
wink.ru

View File

@@ -58,8 +58,7 @@ yastat.net
yastatic.net
# Watching movies, included in the Yandex subscription
kinopoisk.ru
kinopoisk-ru.clstorage.net
include:kinopoisk
# Weather service
full:yandex-pogoda.static-storage.net

12
data/yostar Normal file
View File

@@ -0,0 +1,12 @@
# Yostar
yo-star.com
yostar.cn @cn
yostar.co.jp
yostar.net
yostarplat.com
# Yostar Pictures
yostar-pictures.co.jp
# Stella Sora
stargazer-games.com
stellasora.global
stellasora.jp

18
data/zscaler Normal file
View File

@@ -0,0 +1,18 @@
# https://crt.sh/?id=23639384976
securecloudtransformation.com
securitypreview.com
threatlabz.com
zdxbeta.net
zdxcloud.net
zpagov.net
zpath.com
zscaler.com
zscaler.jp
zscaler.net
zscalerbeta.net
zscalergov.net
zscalerone.net
zscalershift.com
zscalerthree.net
zscalertwo.net
zscloud.net

222
main.go
View File

@@ -22,18 +22,11 @@ var (
exportLists = flag.String("exportlists", "", "Lists to be flattened and exported in plaintext format, separated by ',' comma")
)
var (
plMap = make(map[string]*ParsedList)
finalMap = make(map[string][]*Entry)
cirIncMap = make(map[string]bool) // Used for circular inclusion detection
)
type Entry struct {
Type string
Value string
Attrs []string
Plain string
Affs []string
}
type Inclusion struct {
@@ -48,6 +41,12 @@ type ParsedList struct {
Entries []*Entry
}
type Processor struct {
plMap map[string]*ParsedList
finalMap map[string][]*Entry
cirIncMap map[string]bool
}
func makeProtoList(listName string, entries []*Entry) (*router.GeoSite, error) {
site := &router.GeoSite{
CountryCode: listName,
@@ -90,29 +89,27 @@ func writePlainList(listname string, entries []*Entry) error {
return w.Flush()
}
func parseEntry(line string) (Entry, error) {
var entry Entry
func parseEntry(line string) (*Entry, []string, error) {
entry := new(Entry)
parts := strings.Fields(line)
if len(parts) == 0 {
return entry, fmt.Errorf("empty line")
return entry, nil, fmt.Errorf("empty line")
}
// Parse type and value
v := parts[0]
colonIndex := strings.Index(v, ":")
if colonIndex == -1 {
entry.Type = dlc.RuleTypeDomain // Default type
entry.Value = strings.ToLower(v)
if !validateDomainChars(entry.Value) {
return entry, fmt.Errorf("invalid domain: %q", entry.Value)
typ, val, isTypeSpecified := strings.Cut(parts[0], ":")
typ = strings.ToLower(typ)
if !isTypeSpecified { // Default RuleType
if !validateDomainChars(typ) {
return entry, nil, fmt.Errorf("invalid domain: %q", typ)
}
entry.Type = dlc.RuleTypeDomain
entry.Value = typ
} else {
typ := strings.ToLower(v[:colonIndex])
val := v[colonIndex+1:]
switch typ {
case dlc.RuleTypeRegexp:
if _, err := regexp.Compile(val); err != nil {
return entry, fmt.Errorf("invalid regexp %q: %w", val, err)
return entry, nil, fmt.Errorf("invalid regexp %q: %w", val, err)
}
entry.Type = dlc.RuleTypeRegexp
entry.Value = val
@@ -120,57 +117,60 @@ func parseEntry(line string) (Entry, error) {
entry.Type = dlc.RuleTypeInclude
entry.Value = strings.ToUpper(val)
if !validateSiteName(entry.Value) {
return entry, fmt.Errorf("invalid include list name: %q", entry.Value)
return entry, nil, fmt.Errorf("invalid included list name: %q", entry.Value)
}
case dlc.RuleTypeDomain, dlc.RuleTypeFullDomain, dlc.RuleTypeKeyword:
entry.Type = typ
entry.Value = strings.ToLower(val)
if !validateDomainChars(entry.Value) {
return entry, fmt.Errorf("invalid domain: %q", entry.Value)
return entry, nil, fmt.Errorf("invalid domain: %q", entry.Value)
}
default:
return entry, fmt.Errorf("invalid type: %q", typ)
return entry, nil, fmt.Errorf("invalid type: %q", typ)
}
}
// Parse attributes and affiliations
var affs []string
for _, part := range parts[1:] {
if strings.HasPrefix(part, "@") {
attr := strings.ToLower(part[1:]) // Trim attribute prefix `@` character
switch part[0] {
case '@':
attr := strings.ToLower(part[1:])
if !validateAttrChars(attr) {
return entry, fmt.Errorf("invalid attribute: %q", attr)
return entry, affs, fmt.Errorf("invalid attribute: %q", attr)
}
entry.Attrs = append(entry.Attrs, attr)
} else if strings.HasPrefix(part, "&") {
aff := strings.ToUpper(part[1:]) // Trim affiliation prefix `&` character
case '&':
aff := strings.ToUpper(part[1:])
if !validateSiteName(aff) {
return entry, fmt.Errorf("invalid affiliation: %q", aff)
return entry, affs, fmt.Errorf("invalid affiliation: %q", aff)
}
entry.Affs = append(entry.Affs, aff)
} else {
return entry, fmt.Errorf("invalid attribute/affiliation: %q", part)
affs = append(affs, aff)
default:
return entry, affs, fmt.Errorf("invalid attribute/affiliation: %q", part)
}
}
// Sort attributes
slices.Sort(entry.Attrs)
// Formated plain entry: type:domain.tld:@attr1,@attr2
var plain strings.Builder
plain.Grow(len(entry.Type) + len(entry.Value) + 10)
plain.WriteString(entry.Type)
plain.WriteByte(':')
plain.WriteString(entry.Value)
for i, attr := range entry.Attrs {
if i == 0 {
plain.WriteByte(':')
} else {
plain.WriteByte(',')
}
plain.WriteByte('@')
plain.WriteString(attr)
}
entry.Plain = plain.String()
return entry, nil
if entry.Type != dlc.RuleTypeInclude {
slices.Sort(entry.Attrs) // Sort attributes
// Formated plain entry: type:domain.tld:@attr1,@attr2
var plain strings.Builder
plain.Grow(len(entry.Type) + len(entry.Value) + 10)
plain.WriteString(entry.Type)
plain.WriteByte(':')
plain.WriteString(entry.Value)
for i, attr := range entry.Attrs {
if i == 0 {
plain.WriteByte(':')
} else {
plain.WriteByte(',')
}
plain.WriteByte('@')
plain.WriteString(attr)
}
entry.Plain = plain.String()
}
return entry, affs, nil
}
func validateDomainChars(domain string) bool {
@@ -206,62 +206,54 @@ func validateSiteName(name string) bool {
return true
}
func loadData(path string) ([]*Entry, error) {
func (p *Processor) getOrCreateParsedList(name string) *ParsedList {
pl, exist := p.plMap[name]
if !exist {
pl = &ParsedList{Name: name}
p.plMap[name] = pl
}
return pl
}
func (p *Processor) loadData(listName string, path string) error {
file, err := os.Open(path)
if err != nil {
return nil, err
return err
}
defer file.Close()
var entries []*Entry
pl := p.getOrCreateParsedList(listName)
scanner := bufio.NewScanner(file)
lineIdx := 0
for scanner.Scan() {
line := scanner.Text()
lineIdx++
if idx := strings.Index(line, "#"); idx != -1 {
line = line[:idx] // Remove comments
}
line, _, _ := strings.Cut(scanner.Text(), "#") // Remove comments
line = strings.TrimSpace(line)
if line == "" {
continue
}
entry, err := parseEntry(line)
entry, affs, err := parseEntry(line)
if err != nil {
return entries, fmt.Errorf("error in %q at line %d: %w", path, lineIdx, err)
return fmt.Errorf("error in %q at line %d: %w", path, lineIdx, err)
}
entries = append(entries, &entry)
}
return entries, nil
}
func parseList(refName string, refList []*Entry) error {
pl, _ := plMap[refName]
if pl == nil {
pl = &ParsedList{Name: refName}
plMap[refName] = pl
}
for _, entry := range refList {
if entry.Type == dlc.RuleTypeInclude {
if len(entry.Affs) != 0 {
return fmt.Errorf("affiliation is not allowed for include:%q", entry.Value)
}
inc := &Inclusion{Source: entry.Value}
for _, attr := range entry.Attrs {
if strings.HasPrefix(attr, "-") {
inc.BanAttrs = append(inc.BanAttrs, attr[1:]) // Trim attribute prefix `-` character
if attr[0] == '-' {
inc.BanAttrs = append(inc.BanAttrs, attr[1:])
} else {
inc.MustAttrs = append(inc.MustAttrs, attr)
}
}
for _, aff := range affs {
apl := p.getOrCreateParsedList(aff)
apl.Inclusions = append(apl.Inclusions, inc)
}
pl.Inclusions = append(pl.Inclusions, inc)
} else {
for _, aff := range entry.Affs {
apl, _ := plMap[aff]
if apl == nil {
apl = &ParsedList{Name: aff}
plMap[aff] = apl
}
for _, aff := range affs {
apl := p.getOrCreateParsedList(aff)
apl.Entries = append(apl.Entries, entry)
}
pl.Entries = append(pl.Entries, entry)
@@ -296,9 +288,7 @@ func polishList(roughMap map[string]*Entry) []*Entry {
domainsMap := make(map[string]bool)
for _, entry := range roughMap {
switch entry.Type { // Bypass regexp, keyword and "full/domain with attr"
case dlc.RuleTypeRegexp:
finalList = append(finalList, entry)
case dlc.RuleTypeKeyword:
case dlc.RuleTypeRegexp, dlc.RuleTypeKeyword:
finalList = append(finalList, entry)
case dlc.RuleTypeDomain:
domainsMap[entry.Value] = true
@@ -323,11 +313,11 @@ func polishList(roughMap map[string]*Entry) []*Entry {
pd = "." + pd // So that `domain:example.org` overrides `full:example.org`
}
for {
idx := strings.Index(pd, ".")
if idx == -1 {
var hasParent bool
_, pd, hasParent = strings.Cut(pd, ".") // Go for next parent
if !hasParent {
break
}
pd = pd[idx+1:] // Go for next parent
if domainsMap[pd] {
isRedundant = true
break
@@ -344,36 +334,38 @@ func polishList(roughMap map[string]*Entry) []*Entry {
return finalList
}
func resolveList(pl *ParsedList) error {
if _, pldone := finalMap[pl.Name]; pldone {
func (p *Processor) resolveList(plname string) error {
if _, pldone := p.finalMap[plname]; pldone {
return nil
}
if cirIncMap[pl.Name] {
return fmt.Errorf("circular inclusion in: %q", pl.Name)
pl, plexist := p.plMap[plname]
if !plexist {
return fmt.Errorf("list %q not found", plname)
}
cirIncMap[pl.Name] = true
defer delete(cirIncMap, pl.Name)
if p.cirIncMap[plname] {
return fmt.Errorf("circular inclusion in: %q", plname)
}
p.cirIncMap[plname] = true
defer delete(p.cirIncMap, plname)
roughMap := make(map[string]*Entry) // Avoid basic duplicates
for _, dentry := range pl.Entries { // Add direct entries
roughMap[dentry.Plain] = dentry
}
for _, inc := range pl.Inclusions {
incPl, exist := plMap[inc.Source]
if !exist {
return fmt.Errorf("list %q includes a non-existent list: %q", pl.Name, inc.Source)
if _, exist := p.plMap[inc.Source]; !exist {
return fmt.Errorf("list %q includes a non-existent list: %q", plname, inc.Source)
}
if err := resolveList(incPl); err != nil {
if err := p.resolveList(inc.Source); err != nil {
return err
}
for _, ientry := range finalMap[inc.Source] {
for _, ientry := range p.finalMap[inc.Source] {
if isMatchAttrFilters(ientry, inc) { // Add included entries
roughMap[ientry.Plain] = ientry
}
}
}
finalMap[pl.Name] = polishList(roughMap)
p.finalMap[plname] = polishList(roughMap)
return nil
}
@@ -381,8 +373,8 @@ func run() error {
dir := *dataPath
fmt.Printf("using domain lists data in %q\n", dir)
// Generate refMap
refMap := make(map[string][]*Entry)
// Generate plMap
processor := &Processor{plMap: make(map[string]*ParsedList)}
err := filepath.WalkDir(dir, func(path string, d os.DirEntry, err error) error {
if err != nil {
return err
@@ -394,23 +386,16 @@ func run() error {
if !validateSiteName(listName) {
return fmt.Errorf("invalid list name: %q", listName)
}
refMap[listName], err = loadData(path)
return err
return processor.loadData(listName, path)
})
if err != nil {
return fmt.Errorf("failed to loadData: %w", err)
}
// Generate plMap
for refName, refList := range refMap {
if err := parseList(refName, refList); err != nil {
return fmt.Errorf("failed to parseList %q: %w", refName, err)
}
}
// Generate finalMap
for plname, pl := range plMap {
if err := resolveList(pl); err != nil {
processor.finalMap = make(map[string][]*Entry, len(processor.plMap))
processor.cirIncMap = make(map[string]bool)
for plname := range processor.plMap {
if err := processor.resolveList(plname); err != nil {
return fmt.Errorf("failed to resolveList %q: %w", plname, err)
}
}
@@ -419,11 +404,10 @@ func run() error {
if err := os.MkdirAll(*outputDir, 0755); err != nil {
return fmt.Errorf("failed to create output directory: %w", err)
}
// Export plaintext list
// Export plaintext lists
for rawEpList := range strings.SplitSeq(*exportLists, ",") {
if epList := strings.TrimSpace(rawEpList); epList != "" {
entries, exist := finalMap[strings.ToUpper(epList)]
entries, exist := processor.finalMap[strings.ToUpper(epList)]
if !exist || len(entries) == 0 {
fmt.Printf("list %q does not exist or is empty\n", epList)
continue
@@ -438,7 +422,7 @@ func run() error {
// Generate dat file
protoList := new(router.GeoSiteList)
for siteName, siteEntries := range finalMap {
for siteName, siteEntries := range processor.finalMap {
site, err := makeProtoList(siteName, siteEntries)
if err != nil {
return fmt.Errorf("failed to makeProtoList %q: %w", siteName, err)