Compare commits

...

14 Commits

Author SHA1 Message Date
MkQtS
91da593233 apple: add aod-ssl.itunes.apple.com with cn attr (#3226) 2026-01-28 16:51:48 +08:00
TripleA
9f1c6b6922 Add Bohemia Interactive and Battleye domains (#3223) 2026-01-28 16:41:32 +08:00
MkQtS
b3bae7de8f Update category-ads (#3222)
* remove ads attr from openaicom.imgix.net

imgix.net is serving for pictures, not ads/tracking

* category-ads: include more ad domains
2026-01-28 13:07:34 +08:00
Jinzhe Zeng
4e9b28f951 add crixet.com to openai (#3221)
Crixet has been acquired by OpenAI, per https://crixet.com
2026-01-28 11:49:57 +08:00
xiyao
3c0a538219 samsung: add ospserver.net (#3219)
Samsung OneUI update server
2026-01-27 16:52:46 +08:00
MkQtS
2160230ef9 terabox: add more domains (#3218) 2026-01-27 15:24:47 +08:00
MkQtS
5c38f34456 Add cmd/datdump/main.go (#3213)
* Feat: add a new datdump tool

* Refactor: address code review comments

* Refactor: remove export all from main program

use datdump instead

* Refactor: allow spaces in exportlists

e.g. `--exportlists="lista, listb"`

* all: cleanup

* apply review suggestion

---------

Co-authored-by: database64128 <free122448@hotmail.com>
2026-01-24 23:11:35 +08:00
Zeehan2005
8e62b9b541 Enhance README with additional attribute @cn details (#3212)
Expanded the explanation of attributes in the README to include domains available in China mainland.

[skip ci]
2026-01-24 16:05:37 +08:00
EpLiar
85edae7ba1 Add new Binance API endpoint 'binanceru.net' (#3210) 2026-01-23 14:57:08 +08:00
MkQtS
1bd07b2e76 Support to export all lists to a plain yml (#3211)
* Refactor: improve deduplicate

* Feat: support to export all lists to a plain yml

use: `--exportlists=_all_`

* Docs: add links for dlc plain yml

[skip ci]
2026-01-23 14:56:35 +08:00
Kusu
614a880a55 okx: add okx.cab (#3209) 2026-01-22 22:14:50 +08:00
MkQtS
676832d14a Improve value checkers and docs (#3208)
* Refactor: improve value checkers

* Docs: small improvements

[skip ci]
2026-01-22 18:46:53 +08:00
MkQtS
a2f08a142c Docs: update for selective inclusion and affiliations (#3207)
[skip ci]
2026-01-22 14:30:10 +08:00
MkQtS
2359ad7f8e Add eneba (#3205) 2026-01-22 11:40:44 +08:00
18 changed files with 369 additions and 102 deletions

View File

@@ -33,15 +33,17 @@ jobs:
echo "TAG_NAME=$(date +%Y%m%d%H%M%S)" >> $GITHUB_ENV echo "TAG_NAME=$(date +%Y%m%d%H%M%S)" >> $GITHUB_ENV
shell: bash shell: bash
- name: Build dlc.dat file - name: Build dlc.dat and plain lists
run: | run: |
cd code || exit 1 cd code || exit 1
go run ./ --outputdir=../ --exportlists=category-ads-all,tld-cn,cn,tld-\!cn,geolocation-\!cn,apple,icloud go run ./ --outputdir=../ --exportlists=category-ads-all,tld-cn,cn,tld-\!cn,geolocation-\!cn,apple,icloud
go run ./cmd/datdump/main.go --inputdata=../dlc.dat --outputdir=../ --exportlists=_all_
cd ../ && rm -rf code cd ../ && rm -rf code
- name: Generate dlc.dat sha256 hash - name: Generate dlc.dat sha256 hash
run: | run: |
sha256sum dlc.dat > dlc.dat.sha256sum sha256sum dlc.dat > dlc.dat.sha256sum
sha256sum dlc.dat_plain.yml > dlc.dat_plain.yml.sha256sum
- name: Generate Zip - name: Generate Zip
run: | run: |
@@ -66,6 +68,6 @@ jobs:
- name: Release and upload assets - name: Release and upload assets
run: | run: |
gh release create ${{ env.TAG_NAME }} --generate-notes --latest --title ${{ env.RELEASE_NAME }} ./dlc.dat ./dlc.dat.* gh release create ${{ env.TAG_NAME }} --generate-notes --latest --title ${{ env.RELEASE_NAME }} ./dlc.dat ./dlc.dat.* ./dlc.dat_plain.yml ./dlc.dat_plain.yml.*
env: env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

1
.gitignore vendored
View File

@@ -8,4 +8,5 @@
dlc.dat dlc.dat
# Exported plaintext lists. # Exported plaintext lists.
/*.yml
/*.txt /*.txt

View File

@@ -10,12 +10,12 @@ This project is not opinionated. In other words, it does NOT endorse, claim or i
- **dlc.dat**[https://github.com/v2fly/domain-list-community/releases/latest/download/dlc.dat](https://github.com/v2fly/domain-list-community/releases/latest/download/dlc.dat) - **dlc.dat**[https://github.com/v2fly/domain-list-community/releases/latest/download/dlc.dat](https://github.com/v2fly/domain-list-community/releases/latest/download/dlc.dat)
- **dlc.dat.sha256sum**[https://github.com/v2fly/domain-list-community/releases/latest/download/dlc.dat.sha256sum](https://github.com/v2fly/domain-list-community/releases/latest/download/dlc.dat.sha256sum) - **dlc.dat.sha256sum**[https://github.com/v2fly/domain-list-community/releases/latest/download/dlc.dat.sha256sum](https://github.com/v2fly/domain-list-community/releases/latest/download/dlc.dat.sha256sum)
- **dlc.dat_plain.yml**[https://github.com/v2fly/domain-list-community/releases/latest/download/dlc.dat_plain.yml](https://github.com/v2fly/domain-list-community/releases/latest/download/dlc.dat_plain.yml)
- **dlc.dat_plain.yml.sha256sum**[https://github.com/v2fly/domain-list-community/releases/latest/download/dlc.dat_plain.yml.sha256sum](https://github.com/v2fly/domain-list-community/releases/latest/download/dlc.dat_plain.yml.sha256sum)
## Notice ## Notice
Rules with `@!cn` attribute has been cast out from cn lists. `geosite:geolocation-cn@!cn` is no longer available. Rules with `@!cn` attribute has been cast out from cn lists. `geosite:geolocation-cn@!cn` is no longer available. Check [#390](https://github.com/v2fly/domain-list-community/issues/390), [#3119](https://github.com/v2fly/domain-list-community/pull/3119) and [#3198](https://github.com/v2fly/domain-list-community/pull/3198) for more information.
Check [#390](https://github.com/v2fly/domain-list-community/issues/390), [#3119](https://github.com/v2fly/domain-list-community/pull/3119) and [#3198](https://github.com/v2fly/domain-list-community/pull/3198) for more information.
Please report if you have any problems or questions. Please report if you have any problems or questions.
@@ -93,38 +93,45 @@ All data are under `data` directory. Each file in the directory represents a sub
# comments # comments
include:another-file include:another-file
domain:google.com @attr1 @attr2 domain:google.com @attr1 @attr2
full:analytics.google.com @ads
keyword:google keyword:google
regexp:www\.google\.com$ regexp:^odd[1-7]\.example\.org(\.[a-z]{2})?$
full:www.google.com
``` ```
**Syntax:** **Syntax:**
> [!NOTE]
> Adding new `regexp` and `keyword` rules is discouraged because it is easy to use them incorrectly, and proxy software cannot efficiently match these types of rules.
> [!NOTE]
> The following types of rules are **NOT** fully compatible with the ones that defined by user in V2Ray config file. Do **Not** copy and paste directly. > The following types of rules are **NOT** fully compatible with the ones that defined by user in V2Ray config file. Do **Not** copy and paste directly.
- Comment begins with `#`. It may begin anywhere in the file. The content in the line after `#` is treated as comment and ignored in production. - Comment begins with `#`. It may begin anywhere in the file. The content in the line after `#` is treated as comment and ignored in production.
- Inclusion begins with `include:`, followed by the file name of an existing file in the same directory.
- Subdomain begins with `domain:`, followed by a valid domain name. The prefix `domain:` may be omitted. - Subdomain begins with `domain:`, followed by a valid domain name. The prefix `domain:` may be omitted.
- Keyword begins with `keyword:`, followed by a string.
- Regular expression begins with `regexp:`, followed by a valid regular expression (per Golang's standard).
- Full domain begins with `full:`, followed by a complete and valid domain name. - Full domain begins with `full:`, followed by a complete and valid domain name.
- Domains (including `domain`, `keyword`, `regexp` and `full`) may have one or more attributes. Each attribute begins with `@` and followed by the name of the attribute. - Keyword begins with `keyword:`, followed by a substring of a valid domain name.
- Regular expression begins with `regexp:`, followed by a valid regular expression (per Golang's standard).
> **Note:** Adding new `regexp` and `keyword` rules is discouraged because it is easy to use them incorrectly, and proxy software cannot efficiently match these types of rules. - Domain rules (including `domain`, `full`, `keyword`, and `regexp`) may have none, one or more attributes. Each attribute begins with `@` and followed by the name of the attribute. Attributes will remain available in final lists and `dlc.dat`.
- Domain rules may have none, one or more affiliations, which additionally adds the domain rule into the affiliated target list. Each affiliation begins with `&` and followed by the name of the target list (nomatter whether the target has a dedicated file in data path). This is a method for data management, and will not remain in the final lists or `dlc.dat`.
- Inclusion begins with `include:`, followed by the name of another valid domain list. A simple `include:listb` in file `lista` means adding all domain rules of `listb` into `lista`. Inclusions with attributes stands for selective inclusion. `include:listb @attr1 @-attr2` means only adding those domain rules *with* `@attr1` **and** *without* `@attr2`. This is a special type for data management, and will not remain in the final lists or `dlc.dat`.
## How it works ## How it works
The entire `data` directory will be built into an external `geosite` file for Project V. Each file in the directory represents a section in the generated file. The entire `data` directory will be built into an external `geosite` file for Project V. Each file in the directory represents a section in the generated file.
To generate a section: **General steps:**
1. Remove all the comments in the file. 1. Read files in the data path (ignore all comments and empty lines).
2. Replace `include:` lines with the actual content of the file. 2. Parse and resolve source data, turn affiliations and inclusions into actual domain rules in proper lists.
3. Omit all empty lines. 3. Deduplicate and sort rules in every list.
4. Generate each `domain:` line into a [sub-domain routing rule](https://github.com/v2fly/v2ray-core/blob/master/app/router/routercommon/common.proto#L21). 4. Export desired plain text lists.
5. Generate each `full:` line into a [full domain routing rule](https://github.com/v2fly/v2ray-core/blob/master/app/router/routercommon/common.proto#L23). 5. Generate `dlc.dat`:
6. Generate each `keyword:` line into a [plain domain routing rule](https://github.com/v2fly/v2ray-core/blob/master/app/router/routercommon/common.proto#L17). - turn each `domain:` line into a [sub-domain routing rule](https://github.com/v2fly/v2ray-core/blob/master/app/router/routercommon/common.proto#L21).
7. Generate each `regexp:` line into a [regex domain routing rule](https://github.com/v2fly/v2ray-core/blob/master/app/router/routercommon/common.proto#L19). - turn each `full:` line into a [full domain routing rule](https://github.com/v2fly/v2ray-core/blob/master/app/router/routercommon/common.proto#L23).
- turn each `keyword:` line into a [plain domain routing rule](https://github.com/v2fly/v2ray-core/blob/master/app/router/routercommon/common.proto#L17).
- turn each `regexp:` line into a [regex domain routing rule](https://github.com/v2fly/v2ray-core/blob/master/app/router/routercommon/common.proto#L19).
Read [main.go](./main.go) for details.
## How to organize domains ## How to organize domains
@@ -134,7 +141,7 @@ Theoretically any string can be used as the name, as long as it is a valid file
### Attributes ### Attributes
Attribute is useful for sub-group of domains, especially for filtering purpose. For example, the list of `google` domains may contains its main domains, as well as domains that serve ads. The ads domains may be marked by attribute `@ads`, and can be used as `geosite:google@ads` in V2Ray routing. Attribute is useful for sub-group of domains, especially for filtering purpose. For example, the list of `google` may contains its main domains, as well as domains that serve ads. The ads domains may be marked by attribute `@ads`, and can be used as `geosite:google@ads` in V2Ray routing. Domains and services that originate from outside China mainland but have access point in China mainland, may be marked by attribute `@cn`.
## Contribution guideline ## Contribution guideline

164
cmd/datdump/main.go Normal file
View File

@@ -0,0 +1,164 @@
package main
import (
"bufio"
"flag"
"fmt"
"os"
"path/filepath"
"strings"
"github.com/v2fly/domain-list-community/internal/dlc"
router "github.com/v2fly/v2ray-core/v5/app/router/routercommon"
"google.golang.org/protobuf/proto"
)
var (
inputData = flag.String("inputdata", "dlc.dat", "Name of the geosite dat file")
outputDir = flag.String("outputdir", "./", "Directory to place all generated files")
exportLists = flag.String("exportlists", "", "Lists to be exported, separated by ',' (empty for _all_)")
)
type DomainRule struct {
Type string
Value string
Attrs []string
}
type DomainList struct {
Name string
Rules []DomainRule
}
func (d *DomainRule) domain2String() string {
dstring := d.Type + ":" + d.Value
if len(d.Attrs) != 0 {
dstring += ":@" + strings.Join(d.Attrs, ",@")
}
return dstring
}
func loadGeosite(path string) ([]DomainList, map[string]*DomainList, error) {
data, err := os.ReadFile(path)
if err != nil {
return nil, nil, fmt.Errorf("failed to read geosite file: %w", err)
}
vgeositeList := new(router.GeoSiteList)
if err := proto.Unmarshal(data, vgeositeList); err != nil {
return nil, nil, fmt.Errorf("failed to unmarshal: %w", err)
}
domainLists := make([]DomainList, len(vgeositeList.Entry))
domainListByName := make(map[string]*DomainList, len(vgeositeList.Entry))
for i, vsite := range vgeositeList.Entry {
rules := make([]DomainRule, 0, len(vsite.Domain))
for _, vdomain := range vsite.Domain {
rule := DomainRule{Value: vdomain.Value}
switch vdomain.Type {
case router.Domain_RootDomain:
rule.Type = dlc.RuleTypeDomain
case router.Domain_Regex:
rule.Type = dlc.RuleTypeRegexp
case router.Domain_Plain:
rule.Type = dlc.RuleTypeKeyword
case router.Domain_Full:
rule.Type = dlc.RuleTypeFullDomain
default:
return nil, nil, fmt.Errorf("invalid rule type: %+v", vdomain.Type)
}
for _, vattr := range vdomain.Attribute {
rule.Attrs = append(rule.Attrs, vattr.Key)
}
rules = append(rules, rule)
}
domainLists[i] = DomainList{
Name: strings.ToUpper(vsite.CountryCode),
Rules: rules,
}
domainListByName[domainLists[i].Name] = &domainLists[i]
}
return domainLists, domainListByName, nil
}
func exportSite(name string, domainListByName map[string]*DomainList) error {
domainList, ok := domainListByName[strings.ToUpper(name)]
if !ok {
return fmt.Errorf("list '%s' does not exist", name)
}
if len(domainList.Rules) == 0 {
return fmt.Errorf("list '%s' is empty", name)
}
file, err := os.Create(filepath.Join(*outputDir, name+".yml"))
if err != nil {
return err
}
defer file.Close()
w := bufio.NewWriter(file)
fmt.Fprintf(w, "%s:\n", name)
for _, domain := range domainList.Rules {
fmt.Fprintf(w, " - %q\n", domain.domain2String())
}
return w.Flush()
}
func exportAll(filename string, domainLists []DomainList) error {
file, err := os.Create(filepath.Join(*outputDir, filename))
if err != nil {
return err
}
defer file.Close()
w := bufio.NewWriter(file)
w.WriteString("lists:\n")
for _, domainList := range domainLists {
fmt.Fprintf(w, " - name: %s\n", strings.ToLower(domainList.Name))
fmt.Fprintf(w, " length: %d\n", len(domainList.Rules))
w.WriteString(" rules:\n")
for _, domain := range domainList.Rules {
fmt.Fprintf(w, " - %q\n", domain.domain2String())
}
}
return w.Flush()
}
func main() {
flag.Parse()
// Create output directory if not exist
if _, err := os.Stat(*outputDir); os.IsNotExist(err) {
if mkErr := os.MkdirAll(*outputDir, 0755); mkErr != nil {
fmt.Println("Failed to create output directory:", mkErr)
os.Exit(1)
}
}
fmt.Printf("Loading %s...\n", *inputData)
domainLists, domainListByName, err := loadGeosite(*inputData)
if err != nil {
fmt.Println("Failed to loadGeosite:", err)
os.Exit(1)
}
var exportListSlice []string
for raw := range strings.SplitSeq(*exportLists, ",") {
if trimmed := strings.TrimSpace(raw); trimmed != "" {
exportListSlice = append(exportListSlice, trimmed)
}
}
if len(exportListSlice) == 0 {
exportListSlice = []string{"_all_"}
}
for _, eplistname := range exportListSlice {
if strings.EqualFold(eplistname, "_all_") {
if err := exportAll(filepath.Base(*inputData)+"_plain.yml", domainLists); err != nil {
fmt.Println("Failed to exportAll:", err)
continue
}
} else {
if err := exportSite(eplistname, domainListByName); err != nil {
fmt.Println("Failed to exportSite:", err)
continue
}
}
fmt.Printf("list: '%s' has been exported successfully.\n", eplistname)
}
}

View File

@@ -756,6 +756,7 @@ full:amp-api-edge.apps.apple.com @cn
full:amp-api-search-edge.apps.apple.com @cn full:amp-api-search-edge.apps.apple.com @cn
full:amp-api.apps.apple.com @cn full:amp-api.apps.apple.com @cn
full:amp-api.music.apple.com @cn full:amp-api.music.apple.com @cn
full:aod-ssl.itunes.apple.com @cn
full:aod.itunes.apple.com @cn full:aod.itunes.apple.com @cn
full:api-edge.apps.apple.com @cn full:api-edge.apps.apple.com @cn
full:apptrailers.itunes.apple.com @cn full:apptrailers.itunes.apple.com @cn

View File

@@ -28,6 +28,7 @@ binancezh.top
# API # API
binanceapi.com binanceapi.com
binanceru.net
bnbstatic.com bnbstatic.com
bntrace.com bntrace.com
nftstatic.com nftstatic.com

9
data/bohemia Normal file
View File

@@ -0,0 +1,9 @@
arma3.com
armaplatform.com
bistudio.com
bohemia.net
dayz.com
makearmanotwar.com
silicagame.com
vigorgame.com
ylands.com

View File

@@ -14,12 +14,15 @@ include:bytedance-ads
include:category-ads-ir include:category-ads-ir
include:cctv @ads include:cctv @ads
include:clearbit-ads include:clearbit-ads
include:disney @ads
include:dmm-ads include:dmm-ads
include:duolingo-ads include:duolingo-ads
include:emogi-ads include:emogi-ads
include:flurry-ads include:flurry-ads
include:gamersky @ads
include:google-ads include:google-ads
include:growingio-ads include:growingio-ads
include:hetzner @ads
include:hiido-ads include:hiido-ads
include:hotjar-ads include:hotjar-ads
include:hunantv-ads include:hunantv-ads
@@ -37,13 +40,18 @@ include:netease-ads
include:newrelic-ads include:newrelic-ads
include:ogury-ads include:ogury-ads
include:ookla-speedtest-ads include:ookla-speedtest-ads
include:openai @ads
include:openx-ads include:openx-ads
include:picacg @ads include:picacg @ads
include:pikpak @ads
include:pixiv @ads
include:pocoiq-ads include:pocoiq-ads
include:pubmatic-ads include:pubmatic-ads
include:qihoo360-ads include:qihoo360-ads
include:samsung @ads
include:segment-ads include:segment-ads
include:sina-ads include:sina-ads
include:snap @ads
include:sohu-ads include:sohu-ads
include:spotify-ads include:spotify-ads
include:supersonic-ads include:supersonic-ads
@@ -52,6 +60,7 @@ include:tappx-ads
include:television-ads include:television-ads
include:tencent-ads include:tencent-ads
include:tendcloud @ads include:tendcloud @ads
include:twitter @ads
include:uberads-ads include:uberads-ads
include:umeng-ads include:umeng-ads
include:unity-ads include:unity-ads

View File

@@ -20,7 +20,6 @@ mgid.com @ads
ns1p.net @ads ns1p.net @ads
pubmatic.com @ads pubmatic.com @ads
sigmob.com @ads sigmob.com @ads
snapads.com @ads
spotxchange.com @ads spotxchange.com @ads
unimhk.com @ads unimhk.com @ads
upapi.net @ads upapi.net @ads

View File

@@ -2,6 +2,9 @@ include:playcover
include:fflogs include:fflogs
include:trackernetwork include:trackernetwork
# Anti-Cheat
battleye.com
# Android Emulator # Android Emulator
bluestacks.com bluestacks.com
ldmnq.com @cn ldmnq.com @cn
@@ -16,5 +19,5 @@ prts.plus
heavenlywind.cc @cn heavenlywind.cc @cn
poi.moe poi.moe
# Steam++ / Watt Toolkit
steampp.net @cn steampp.net @cn

View File

@@ -1,10 +1,12 @@
include:2kgames include:2kgames
include:blizzard include:blizzard
include:bluearchive include:bluearchive
include:bohemia
include:curseforge include:curseforge
include:cygames include:cygames
include:ea include:ea
include:embark include:embark
include:eneba
include:epicgames include:epicgames
include:escapefromtarkov include:escapefromtarkov
include:faceit include:faceit

2
data/eneba Normal file
View File

@@ -0,0 +1,2 @@
eneba.com
eneba.games

View File

@@ -1,8 +1,9 @@
okex.com okex.com
okx.com
okx-dns.com okx-dns.com
okx-dns1.com okx-dns1.com
okx-dns2.com okx-dns2.com
okx.cab
okx.com
# OKC Browser # OKC Browser
oklink.com @cn oklink.com @cn

View File

@@ -1,6 +1,7 @@
# Main domain # Main domain
chatgpt.com
chat.com chat.com
chatgpt.com
crixet.com
oaistatic.com oaistatic.com
oaiusercontent.com oaiusercontent.com
openai.com openai.com
@@ -10,13 +11,13 @@ sora.com
openai.com.cdn.cloudflare.net openai.com.cdn.cloudflare.net
full:openaiapi-site.azureedge.net full:openaiapi-site.azureedge.net
full:openaicom-api-bdcpf8c6d2e9atf6.z01.azurefd.net full:openaicom-api-bdcpf8c6d2e9atf6.z01.azurefd.net
full:openaicom.imgix.net
full:openaicomproductionae4b.blob.core.windows.net full:openaicomproductionae4b.blob.core.windows.net
full:production-openaicom-storage.azureedge.net full:production-openaicom-storage.azureedge.net
regexp:^chatgpt-async-webps-prod-\S+-\d+\.webpubsub\.azure\.com$ regexp:^chatgpt-async-webps-prod-\S+-\d+\.webpubsub\.azure\.com$
# tracking # tracking
full:o33249.ingest.sentry.io @ads full:o33249.ingest.sentry.io @ads
full:openaicom.imgix.net @ads
full:browser-intake-datadoghq.com @ads full:browser-intake-datadoghq.com @ads
# Advanced Voice # Advanced Voice

View File

@@ -8,6 +8,7 @@ galaxyappstore.com
galaxymobile.jp galaxymobile.jp
game-platform.net game-platform.net
knoxemm.com knoxemm.com
ospserver.net
samsung.com samsung.com
samsungads.com @ads samsungads.com @ads
samsungapps.com samsungapps.com

View File

@@ -1,2 +1,7 @@
1024terabox.com
bestclouddrive.com
freeterabox.com
nephobox.com
terabox.com terabox.com
terabox1024.com
teraboxcdn.com teraboxcdn.com

9
internal/dlc/dlc.go Normal file
View File

@@ -0,0 +1,9 @@
package dlc
const (
RuleTypeDomain string = "domain"
RuleTypeFullDomain string = "full"
RuleTypeKeyword string = "keyword"
RuleTypeRegexp string = "regexp"
RuleTypeInclude string = "include"
)

188
main.go
View File

@@ -10,6 +10,7 @@ import (
"slices" "slices"
"strings" "strings"
"github.com/v2fly/domain-list-community/internal/dlc"
router "github.com/v2fly/v2ray-core/v5/app/router/routercommon" router "github.com/v2fly/v2ray-core/v5/app/router/routercommon"
"google.golang.org/protobuf/proto" "google.golang.org/protobuf/proto"
) )
@@ -21,21 +22,6 @@ var (
exportLists = flag.String("exportlists", "", "Lists to be flattened and exported in plaintext format, separated by ',' comma") exportLists = flag.String("exportlists", "", "Lists to be flattened and exported in plaintext format, separated by ',' comma")
) )
const (
RuleTypeDomain string = "domain"
RuleTypeFullDomain string = "full"
RuleTypeKeyword string = "keyword"
RuleTypeRegexp string = "regexp"
RuleTypeInclude string = "include"
)
var (
TypeChecker = regexp.MustCompile(`^(domain|full|keyword|regexp|include)$`)
ValueChecker = regexp.MustCompile(`^[a-z0-9!\.-]+$`)
AttrChecker = regexp.MustCompile(`^[a-z0-9!-]+$`)
SiteChecker = regexp.MustCompile(`^[A-Z0-9!-]+$`)
)
var ( var (
refMap = make(map[string][]*Entry) refMap = make(map[string][]*Entry)
plMap = make(map[string]*ParsedList) plMap = make(map[string]*ParsedList)
@@ -78,13 +64,13 @@ func makeProtoList(listName string, entries []*Entry) (*router.GeoSite, error) {
} }
switch entry.Type { switch entry.Type {
case RuleTypeDomain: case dlc.RuleTypeDomain:
pdomain.Type = router.Domain_RootDomain pdomain.Type = router.Domain_RootDomain
case RuleTypeRegexp: case dlc.RuleTypeRegexp:
pdomain.Type = router.Domain_Regex pdomain.Type = router.Domain_Regex
case RuleTypeKeyword: case dlc.RuleTypeKeyword:
pdomain.Type = router.Domain_Plain pdomain.Type = router.Domain_Plain
case RuleTypeFullDomain: case dlc.RuleTypeFullDomain:
pdomain.Type = router.Domain_Full pdomain.Type = router.Domain_Full
} }
site.Domain = append(site.Domain, pdomain) site.Domain = append(site.Domain, pdomain)
@@ -95,7 +81,7 @@ func makeProtoList(listName string, entries []*Entry) (*router.GeoSite, error) {
func writePlainList(exportedName string) error { func writePlainList(exportedName string) error {
targetList, exist := finalMap[strings.ToUpper(exportedName)] targetList, exist := finalMap[strings.ToUpper(exportedName)]
if !exist || len(targetList) == 0 { if !exist || len(targetList) == 0 {
return fmt.Errorf("'%s' list does not exist or is empty.", exportedName) return fmt.Errorf("list %q does not exist or is empty.", exportedName)
} }
file, err := os.Create(filepath.Join(*outputDir, strings.ToLower(exportedName)+".txt")) file, err := os.Create(filepath.Join(*outputDir, strings.ToLower(exportedName)+".txt"))
if err != nil { if err != nil {
@@ -112,51 +98,62 @@ func writePlainList(exportedName string) error {
func parseEntry(line string) (Entry, error) { func parseEntry(line string) (Entry, error) {
var entry Entry var entry Entry
parts := strings.Fields(line) parts := strings.Fields(line)
if len(parts) == 0 {
return entry, fmt.Errorf("empty line: %q", line)
}
// Parse type and value // Parse type and value
rawTypeVal := parts[0] v := parts[0]
kv := strings.Split(rawTypeVal, ":") colonIndex := strings.Index(v, ":")
if len(kv) == 1 { if colonIndex == -1 {
entry.Type = RuleTypeDomain // Default type entry.Type = dlc.RuleTypeDomain // Default type
entry.Value = strings.ToLower(rawTypeVal) entry.Value = strings.ToLower(v)
} else if len(kv) == 2 { if !validateDomainChars(entry.Value) {
entry.Type = strings.ToLower(kv[0]) return entry, fmt.Errorf("invalid domain: %q", entry.Value)
if entry.Type == RuleTypeRegexp {
entry.Value = kv[1]
} else {
entry.Value = strings.ToLower(kv[1])
} }
} else { } else {
return entry, fmt.Errorf("invalid format: %s", line) typ := strings.ToLower(v[:colonIndex])
val := v[colonIndex+1:]
switch typ {
case dlc.RuleTypeRegexp:
if _, err := regexp.Compile(val); err != nil {
return entry, fmt.Errorf("invalid regexp %q: %w", val, err)
} }
// Check type and value entry.Type = dlc.RuleTypeRegexp
if !TypeChecker.MatchString(entry.Type) { entry.Value = val
return entry, fmt.Errorf("invalid type: %s", entry.Type) case dlc.RuleTypeInclude:
entry.Type = dlc.RuleTypeInclude
entry.Value = strings.ToUpper(val)
if !validateSiteName(entry.Value) {
return entry, fmt.Errorf("invalid include list name: %q", entry.Value)
} }
if entry.Type == RuleTypeRegexp { case dlc.RuleTypeDomain, dlc.RuleTypeFullDomain, dlc.RuleTypeKeyword:
if _, err := regexp.Compile(entry.Value); err != nil { entry.Type = typ
return entry, fmt.Errorf("invalid regexp: %s", entry.Value) entry.Value = strings.ToLower(val)
if !validateDomainChars(entry.Value) {
return entry, fmt.Errorf("invalid domain: %q", entry.Value)
}
default:
return entry, fmt.Errorf("invalid type: %q", typ)
} }
} else if !ValueChecker.MatchString(entry.Value) {
return entry, fmt.Errorf("invalid value: %s", entry.Value)
} }
// Parse/Check attributes and affiliations // Parse/Check attributes and affiliations
for _, part := range parts[1:] { for _, part := range parts[1:] {
if strings.HasPrefix(part, "@") { if strings.HasPrefix(part, "@") {
attr := strings.ToLower(part[1:]) // Trim attribute prefix `@` character attr := strings.ToLower(part[1:]) // Trim attribute prefix `@` character
if !AttrChecker.MatchString(attr) { if !validateAttrChars(attr) {
return entry, fmt.Errorf("invalid attribute key: %s", attr) return entry, fmt.Errorf("invalid attribute: %q", attr)
} }
entry.Attrs = append(entry.Attrs, attr) entry.Attrs = append(entry.Attrs, attr)
} else if strings.HasPrefix(part, "&") { } else if strings.HasPrefix(part, "&") {
aff := strings.ToUpper(part[1:]) // Trim affiliation prefix `&` character aff := strings.ToUpper(part[1:]) // Trim affiliation prefix `&` character
if !SiteChecker.MatchString(aff) { if !validateSiteName(aff) {
return entry, fmt.Errorf("invalid affiliation key: %s", aff) return entry, fmt.Errorf("invalid affiliation: %q", aff)
} }
entry.Affs = append(entry.Affs, aff) entry.Affs = append(entry.Affs, aff)
} else { } else {
return entry, fmt.Errorf("invalid attribute/affiliation: %s", part) return entry, fmt.Errorf("invalid attribute/affiliation: %q", part)
} }
} }
// Sort attributes // Sort attributes
@@ -170,6 +167,39 @@ func parseEntry(line string) (Entry, error) {
return entry, nil return entry, nil
} }
func validateDomainChars(domain string) bool {
for i := range domain {
c := domain[i]
if (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') || c == '.' || c == '-' {
continue
}
return false
}
return true
}
func validateAttrChars(attr string) bool {
for i := range attr {
c := attr[i]
if (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') || c == '!' || c == '-' {
continue
}
return false
}
return true
}
func validateSiteName(name string) bool {
for i := range name {
c := name[i]
if (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '!' || c == '-' {
continue
}
return false
}
return true
}
func loadData(path string) error { func loadData(path string) error {
file, err := os.Open(path) file, err := os.Open(path)
if err != nil { if err != nil {
@@ -178,7 +208,7 @@ func loadData(path string) error {
defer file.Close() defer file.Close()
listName := strings.ToUpper(filepath.Base(path)) listName := strings.ToUpper(filepath.Base(path))
if !SiteChecker.MatchString(listName) { if !validateSiteName(listName) {
return fmt.Errorf("invalid list name: %s", listName) return fmt.Errorf("invalid list name: %s", listName)
} }
scanner := bufio.NewScanner(file) scanner := bufio.NewScanner(file)
@@ -210,11 +240,11 @@ func parseList(refName string, refList []*Entry) error {
plMap[refName] = pl plMap[refName] = pl
} }
for _, entry := range refList { for _, entry := range refList {
if entry.Type == RuleTypeInclude { if entry.Type == dlc.RuleTypeInclude {
if len(entry.Affs) != 0 { if len(entry.Affs) != 0 {
return fmt.Errorf("affiliation is not allowed for include:%s", entry.Value) return fmt.Errorf("affiliation is not allowed for include:%s", entry.Value)
} }
inc := &Inclusion{Source: strings.ToUpper(entry.Value)} inc := &Inclusion{Source: entry.Value}
for _, attr := range entry.Attrs { for _, attr := range entry.Attrs {
if strings.HasPrefix(attr, "-") { if strings.HasPrefix(attr, "-") {
inc.BanAttrs = append(inc.BanAttrs, attr[1:]) // Trim attribute prefix `-` character inc.BanAttrs = append(inc.BanAttrs, attr[1:]) // Trim attribute prefix `-` character
@@ -244,18 +274,18 @@ func polishList(roughMap *map[string]*Entry) []*Entry {
domainsMap := make(map[string]bool) domainsMap := make(map[string]bool)
for _, entry := range *roughMap { for _, entry := range *roughMap {
switch entry.Type { // Bypass regexp, keyword and "full/domain with attr" switch entry.Type { // Bypass regexp, keyword and "full/domain with attr"
case RuleTypeRegexp: case dlc.RuleTypeRegexp:
finalList = append(finalList, entry) finalList = append(finalList, entry)
case RuleTypeKeyword: case dlc.RuleTypeKeyword:
finalList = append(finalList, entry) finalList = append(finalList, entry)
case RuleTypeDomain: case dlc.RuleTypeDomain:
domainsMap[entry.Value] = true domainsMap[entry.Value] = true
if len(entry.Attrs) != 0 { if len(entry.Attrs) != 0 {
finalList = append(finalList, entry) finalList = append(finalList, entry)
} else { } else {
queuingList = append(queuingList, entry) queuingList = append(queuingList, entry)
} }
case RuleTypeFullDomain: case dlc.RuleTypeFullDomain:
if len(entry.Attrs) != 0 { if len(entry.Attrs) != 0 {
finalList = append(finalList, entry) finalList = append(finalList, entry)
} else { } else {
@@ -266,12 +296,19 @@ func polishList(roughMap *map[string]*Entry) []*Entry {
// Remove redundant subdomains for full/domain without attr // Remove redundant subdomains for full/domain without attr
for _, qentry := range queuingList { for _, qentry := range queuingList {
isRedundant := false isRedundant := false
pd := qentry.Value // Parent domain pd := qentry.Value // To be parent domain
if qentry.Type == dlc.RuleTypeFullDomain {
pd = "." + pd // So that `domain:example.org` overrides `full:example.org`
}
for { for {
idx := strings.Index(pd, ".") idx := strings.Index(pd, ".")
if idx == -1 { break } if idx == -1 {
break
}
pd = pd[idx+1:] // Go for next parent pd = pd[idx+1:] // Go for next parent
if !strings.Contains(pd, ".") { break } // Not allow tld to be a parent if !strings.Contains(pd, ".") {
break
} // Not allow tld to be a parent
if domainsMap[pd] { if domainsMap[pd] {
isRedundant = true isRedundant = true
break break
@@ -289,7 +326,9 @@ func polishList(roughMap *map[string]*Entry) []*Entry {
} }
func resolveList(pl *ParsedList) error { func resolveList(pl *ParsedList) error {
if _, pldone := finalMap[pl.Name]; pldone { return nil } if _, pldone := finalMap[pl.Name]; pldone {
return nil
}
if cirIncMap[pl.Name] { if cirIncMap[pl.Name] {
return fmt.Errorf("circular inclusion in: %s", pl.Name) return fmt.Errorf("circular inclusion in: %s", pl.Name)
@@ -298,14 +337,22 @@ func resolveList(pl *ParsedList) error {
defer delete(cirIncMap, pl.Name) defer delete(cirIncMap, pl.Name)
isMatchAttrFilters := func(entry *Entry, incFilter *Inclusion) bool { isMatchAttrFilters := func(entry *Entry, incFilter *Inclusion) bool {
if len(incFilter.MustAttrs) == 0 && len(incFilter.BanAttrs) == 0 { return true } if len(incFilter.MustAttrs) == 0 && len(incFilter.BanAttrs) == 0 {
if len(entry.Attrs) == 0 { return len(incFilter.MustAttrs) == 0 } return true
}
if len(entry.Attrs) == 0 {
return len(incFilter.MustAttrs) == 0
}
for _, m := range incFilter.MustAttrs { for _, m := range incFilter.MustAttrs {
if !slices.Contains(entry.Attrs, m) { return false } if !slices.Contains(entry.Attrs, m) {
return false
}
} }
for _, b := range incFilter.BanAttrs { for _, b := range incFilter.BanAttrs {
if slices.Contains(entry.Attrs, b) { return false } if slices.Contains(entry.Attrs, b) {
return false
}
} }
return true return true
} }
@@ -317,7 +364,7 @@ func resolveList(pl *ParsedList) error {
for _, inc := range pl.Inclusions { for _, inc := range pl.Inclusions {
incPl, exist := plMap[inc.Source] incPl, exist := plMap[inc.Source]
if !exist { if !exist {
return fmt.Errorf("list '%s' includes a non-existent list: '%s'", pl.Name, inc.Source) return fmt.Errorf("list %q includes a non-existent list: %q", pl.Name, inc.Source)
} }
if err := resolveList(incPl); err != nil { if err := resolveList(incPl); err != nil {
return err return err
@@ -375,21 +422,24 @@ func main() {
// Create output directory if not exist // Create output directory if not exist
if _, err := os.Stat(*outputDir); os.IsNotExist(err) { if _, err := os.Stat(*outputDir); os.IsNotExist(err) {
if mkErr := os.MkdirAll(*outputDir, 0755); mkErr != nil { if mkErr := os.MkdirAll(*outputDir, 0755); mkErr != nil {
fmt.Println("Failed:", mkErr) fmt.Println("Failed to create output directory:", mkErr)
os.Exit(1) os.Exit(1)
} }
} }
// Export plaintext list // Export plaintext list
if *exportLists != "" { var exportListSlice []string
exportedListSlice := strings.Split(*exportLists, ",") for raw := range strings.SplitSeq(*exportLists, ",") {
for _, exportedList := range exportedListSlice { if trimmed := strings.TrimSpace(raw); trimmed != "" {
if err := writePlainList(exportedList); err != nil { exportListSlice = append(exportListSlice, trimmed)
}
}
for _, exportList := range exportListSlice {
if err := writePlainList(exportList); err != nil {
fmt.Println("Failed to write list:", err) fmt.Println("Failed to write list:", err)
continue continue
} }
fmt.Printf("list: '%s' has been generated successfully.\n", exportedList) fmt.Printf("list %q has been generated successfully.\n", exportList)
}
} }
// Generate dat file // Generate dat file
@@ -397,7 +447,7 @@ func main() {
for siteName, siteEntries := range finalMap { for siteName, siteEntries := range finalMap {
site, err := makeProtoList(siteName, siteEntries) site, err := makeProtoList(siteName, siteEntries)
if err != nil { if err != nil {
fmt.Println("Failed:", err) fmt.Println("Failed to makeProtoList:", err)
os.Exit(1) os.Exit(1)
} }
protoList.Entry = append(protoList.Entry, site) protoList.Entry = append(protoList.Entry, site)