mirror of
https://github.com/v2fly/domain-list-community.git
synced 2026-03-09 21:16:36 +07:00
Compare commits
12 Commits
2026012202
...
2026012803
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
4e9b28f951 | ||
|
|
3c0a538219 | ||
|
|
2160230ef9 | ||
|
|
5c38f34456 | ||
|
|
8e62b9b541 | ||
|
|
85edae7ba1 | ||
|
|
1bd07b2e76 | ||
|
|
614a880a55 | ||
|
|
676832d14a | ||
|
|
a2f08a142c | ||
|
|
2359ad7f8e | ||
|
|
330592feff |
6
.github/workflows/build.yml
vendored
6
.github/workflows/build.yml
vendored
@@ -33,15 +33,17 @@ jobs:
|
|||||||
echo "TAG_NAME=$(date +%Y%m%d%H%M%S)" >> $GITHUB_ENV
|
echo "TAG_NAME=$(date +%Y%m%d%H%M%S)" >> $GITHUB_ENV
|
||||||
shell: bash
|
shell: bash
|
||||||
|
|
||||||
- name: Build dlc.dat file
|
- name: Build dlc.dat and plain lists
|
||||||
run: |
|
run: |
|
||||||
cd code || exit 1
|
cd code || exit 1
|
||||||
go run ./ --outputdir=../ --exportlists=category-ads-all,tld-cn,cn,tld-\!cn,geolocation-\!cn,apple,icloud
|
go run ./ --outputdir=../ --exportlists=category-ads-all,tld-cn,cn,tld-\!cn,geolocation-\!cn,apple,icloud
|
||||||
|
go run ./cmd/datdump/main.go --inputdata=../dlc.dat --outputdir=../ --exportlists=_all_
|
||||||
cd ../ && rm -rf code
|
cd ../ && rm -rf code
|
||||||
|
|
||||||
- name: Generate dlc.dat sha256 hash
|
- name: Generate dlc.dat sha256 hash
|
||||||
run: |
|
run: |
|
||||||
sha256sum dlc.dat > dlc.dat.sha256sum
|
sha256sum dlc.dat > dlc.dat.sha256sum
|
||||||
|
sha256sum dlc.dat_plain.yml > dlc.dat_plain.yml.sha256sum
|
||||||
|
|
||||||
- name: Generate Zip
|
- name: Generate Zip
|
||||||
run: |
|
run: |
|
||||||
@@ -66,6 +68,6 @@ jobs:
|
|||||||
|
|
||||||
- name: Release and upload assets
|
- name: Release and upload assets
|
||||||
run: |
|
run: |
|
||||||
gh release create ${{ env.TAG_NAME }} --generate-notes --latest --title ${{ env.RELEASE_NAME }} ./dlc.dat ./dlc.dat.*
|
gh release create ${{ env.TAG_NAME }} --generate-notes --latest --title ${{ env.RELEASE_NAME }} ./dlc.dat ./dlc.dat.* ./dlc.dat_plain.yml ./dlc.dat_plain.yml.*
|
||||||
env:
|
env:
|
||||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||||
|
|||||||
1
.gitignore
vendored
1
.gitignore
vendored
@@ -8,4 +8,5 @@
|
|||||||
dlc.dat
|
dlc.dat
|
||||||
|
|
||||||
# Exported plaintext lists.
|
# Exported plaintext lists.
|
||||||
|
/*.yml
|
||||||
/*.txt
|
/*.txt
|
||||||
|
|||||||
47
README.md
47
README.md
@@ -10,12 +10,12 @@ This project is not opinionated. In other words, it does NOT endorse, claim or i
|
|||||||
|
|
||||||
- **dlc.dat**:[https://github.com/v2fly/domain-list-community/releases/latest/download/dlc.dat](https://github.com/v2fly/domain-list-community/releases/latest/download/dlc.dat)
|
- **dlc.dat**:[https://github.com/v2fly/domain-list-community/releases/latest/download/dlc.dat](https://github.com/v2fly/domain-list-community/releases/latest/download/dlc.dat)
|
||||||
- **dlc.dat.sha256sum**:[https://github.com/v2fly/domain-list-community/releases/latest/download/dlc.dat.sha256sum](https://github.com/v2fly/domain-list-community/releases/latest/download/dlc.dat.sha256sum)
|
- **dlc.dat.sha256sum**:[https://github.com/v2fly/domain-list-community/releases/latest/download/dlc.dat.sha256sum](https://github.com/v2fly/domain-list-community/releases/latest/download/dlc.dat.sha256sum)
|
||||||
|
- **dlc.dat_plain.yml**:[https://github.com/v2fly/domain-list-community/releases/latest/download/dlc.dat_plain.yml](https://github.com/v2fly/domain-list-community/releases/latest/download/dlc.dat_plain.yml)
|
||||||
|
- **dlc.dat_plain.yml.sha256sum**:[https://github.com/v2fly/domain-list-community/releases/latest/download/dlc.dat_plain.yml.sha256sum](https://github.com/v2fly/domain-list-community/releases/latest/download/dlc.dat_plain.yml.sha256sum)
|
||||||
|
|
||||||
## Notice
|
## Notice
|
||||||
|
|
||||||
Rules with `@!cn` attribute has been cast out from cn lists. `geosite:geolocation-cn@!cn` is no longer available.
|
Rules with `@!cn` attribute has been cast out from cn lists. `geosite:geolocation-cn@!cn` is no longer available. Check [#390](https://github.com/v2fly/domain-list-community/issues/390), [#3119](https://github.com/v2fly/domain-list-community/pull/3119) and [#3198](https://github.com/v2fly/domain-list-community/pull/3198) for more information.
|
||||||
|
|
||||||
Check [#390](https://github.com/v2fly/domain-list-community/issues/390), [#3119](https://github.com/v2fly/domain-list-community/pull/3119) and [#3198](https://github.com/v2fly/domain-list-community/pull/3198) for more information.
|
|
||||||
|
|
||||||
Please report if you have any problems or questions.
|
Please report if you have any problems or questions.
|
||||||
|
|
||||||
@@ -93,38 +93,45 @@ All data are under `data` directory. Each file in the directory represents a sub
|
|||||||
# comments
|
# comments
|
||||||
include:another-file
|
include:another-file
|
||||||
domain:google.com @attr1 @attr2
|
domain:google.com @attr1 @attr2
|
||||||
|
full:analytics.google.com @ads
|
||||||
keyword:google
|
keyword:google
|
||||||
regexp:www\.google\.com$
|
regexp:^odd[1-7]\.example\.org(\.[a-z]{2})?$
|
||||||
full:www.google.com
|
|
||||||
```
|
```
|
||||||
|
|
||||||
**Syntax:**
|
**Syntax:**
|
||||||
|
|
||||||
|
> [!NOTE]
|
||||||
|
> Adding new `regexp` and `keyword` rules is discouraged because it is easy to use them incorrectly, and proxy software cannot efficiently match these types of rules.
|
||||||
|
|
||||||
|
> [!NOTE]
|
||||||
> The following types of rules are **NOT** fully compatible with the ones that defined by user in V2Ray config file. Do **Not** copy and paste directly.
|
> The following types of rules are **NOT** fully compatible with the ones that defined by user in V2Ray config file. Do **Not** copy and paste directly.
|
||||||
|
|
||||||
- Comment begins with `#`. It may begin anywhere in the file. The content in the line after `#` is treated as comment and ignored in production.
|
- Comment begins with `#`. It may begin anywhere in the file. The content in the line after `#` is treated as comment and ignored in production.
|
||||||
- Inclusion begins with `include:`, followed by the file name of an existing file in the same directory.
|
|
||||||
- Subdomain begins with `domain:`, followed by a valid domain name. The prefix `domain:` may be omitted.
|
- Subdomain begins with `domain:`, followed by a valid domain name. The prefix `domain:` may be omitted.
|
||||||
- Keyword begins with `keyword:`, followed by a string.
|
|
||||||
- Regular expression begins with `regexp:`, followed by a valid regular expression (per Golang's standard).
|
|
||||||
- Full domain begins with `full:`, followed by a complete and valid domain name.
|
- Full domain begins with `full:`, followed by a complete and valid domain name.
|
||||||
- Domains (including `domain`, `keyword`, `regexp` and `full`) may have one or more attributes. Each attribute begins with `@` and followed by the name of the attribute.
|
- Keyword begins with `keyword:`, followed by a substring of a valid domain name.
|
||||||
|
- Regular expression begins with `regexp:`, followed by a valid regular expression (per Golang's standard).
|
||||||
> **Note:** Adding new `regexp` and `keyword` rules is discouraged because it is easy to use them incorrectly, and proxy software cannot efficiently match these types of rules.
|
- Domain rules (including `domain`, `full`, `keyword`, and `regexp`) may have none, one or more attributes. Each attribute begins with `@` and followed by the name of the attribute. Attributes will remain available in final lists and `dlc.dat`.
|
||||||
|
- Domain rules may have none, one or more affiliations, which additionally adds the domain rule into the affiliated target list. Each affiliation begins with `&` and followed by the name of the target list (nomatter whether the target has a dedicated file in data path). This is a method for data management, and will not remain in the final lists or `dlc.dat`.
|
||||||
|
- Inclusion begins with `include:`, followed by the name of another valid domain list. A simple `include:listb` in file `lista` means adding all domain rules of `listb` into `lista`. Inclusions with attributes stands for selective inclusion. `include:listb @attr1 @-attr2` means only adding those domain rules *with* `@attr1` **and** *without* `@attr2`. This is a special type for data management, and will not remain in the final lists or `dlc.dat`.
|
||||||
|
|
||||||
## How it works
|
## How it works
|
||||||
|
|
||||||
The entire `data` directory will be built into an external `geosite` file for Project V. Each file in the directory represents a section in the generated file.
|
The entire `data` directory will be built into an external `geosite` file for Project V. Each file in the directory represents a section in the generated file.
|
||||||
|
|
||||||
To generate a section:
|
**General steps:**
|
||||||
|
|
||||||
1. Remove all the comments in the file.
|
1. Read files in the data path (ignore all comments and empty lines).
|
||||||
2. Replace `include:` lines with the actual content of the file.
|
2. Parse and resolve source data, turn affiliations and inclusions into actual domain rules in proper lists.
|
||||||
3. Omit all empty lines.
|
3. Deduplicate and sort rules in every list.
|
||||||
4. Generate each `domain:` line into a [sub-domain routing rule](https://github.com/v2fly/v2ray-core/blob/master/app/router/routercommon/common.proto#L21).
|
4. Export desired plain text lists.
|
||||||
5. Generate each `full:` line into a [full domain routing rule](https://github.com/v2fly/v2ray-core/blob/master/app/router/routercommon/common.proto#L23).
|
5. Generate `dlc.dat`:
|
||||||
6. Generate each `keyword:` line into a [plain domain routing rule](https://github.com/v2fly/v2ray-core/blob/master/app/router/routercommon/common.proto#L17).
|
- turn each `domain:` line into a [sub-domain routing rule](https://github.com/v2fly/v2ray-core/blob/master/app/router/routercommon/common.proto#L21).
|
||||||
7. Generate each `regexp:` line into a [regex domain routing rule](https://github.com/v2fly/v2ray-core/blob/master/app/router/routercommon/common.proto#L19).
|
- turn each `full:` line into a [full domain routing rule](https://github.com/v2fly/v2ray-core/blob/master/app/router/routercommon/common.proto#L23).
|
||||||
|
- turn each `keyword:` line into a [plain domain routing rule](https://github.com/v2fly/v2ray-core/blob/master/app/router/routercommon/common.proto#L17).
|
||||||
|
- turn each `regexp:` line into a [regex domain routing rule](https://github.com/v2fly/v2ray-core/blob/master/app/router/routercommon/common.proto#L19).
|
||||||
|
|
||||||
|
Read [main.go](./main.go) for details.
|
||||||
|
|
||||||
## How to organize domains
|
## How to organize domains
|
||||||
|
|
||||||
@@ -134,7 +141,7 @@ Theoretically any string can be used as the name, as long as it is a valid file
|
|||||||
|
|
||||||
### Attributes
|
### Attributes
|
||||||
|
|
||||||
Attribute is useful for sub-group of domains, especially for filtering purpose. For example, the list of `google` domains may contains its main domains, as well as domains that serve ads. The ads domains may be marked by attribute `@ads`, and can be used as `geosite:google@ads` in V2Ray routing.
|
Attribute is useful for sub-group of domains, especially for filtering purpose. For example, the list of `google` may contains its main domains, as well as domains that serve ads. The ads domains may be marked by attribute `@ads`, and can be used as `geosite:google@ads` in V2Ray routing. Domains and services that originate from outside China mainland but have access point in China mainland, may be marked by attribute `@cn`.
|
||||||
|
|
||||||
## Contribution guideline
|
## Contribution guideline
|
||||||
|
|
||||||
|
|||||||
164
cmd/datdump/main.go
Normal file
164
cmd/datdump/main.go
Normal file
@@ -0,0 +1,164 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bufio"
|
||||||
|
"flag"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/v2fly/domain-list-community/internal/dlc"
|
||||||
|
router "github.com/v2fly/v2ray-core/v5/app/router/routercommon"
|
||||||
|
"google.golang.org/protobuf/proto"
|
||||||
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
inputData = flag.String("inputdata", "dlc.dat", "Name of the geosite dat file")
|
||||||
|
outputDir = flag.String("outputdir", "./", "Directory to place all generated files")
|
||||||
|
exportLists = flag.String("exportlists", "", "Lists to be exported, separated by ',' (empty for _all_)")
|
||||||
|
)
|
||||||
|
|
||||||
|
type DomainRule struct {
|
||||||
|
Type string
|
||||||
|
Value string
|
||||||
|
Attrs []string
|
||||||
|
}
|
||||||
|
|
||||||
|
type DomainList struct {
|
||||||
|
Name string
|
||||||
|
Rules []DomainRule
|
||||||
|
}
|
||||||
|
|
||||||
|
func (d *DomainRule) domain2String() string {
|
||||||
|
dstring := d.Type + ":" + d.Value
|
||||||
|
if len(d.Attrs) != 0 {
|
||||||
|
dstring += ":@" + strings.Join(d.Attrs, ",@")
|
||||||
|
}
|
||||||
|
return dstring
|
||||||
|
}
|
||||||
|
|
||||||
|
func loadGeosite(path string) ([]DomainList, map[string]*DomainList, error) {
|
||||||
|
data, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, fmt.Errorf("failed to read geosite file: %w", err)
|
||||||
|
}
|
||||||
|
vgeositeList := new(router.GeoSiteList)
|
||||||
|
if err := proto.Unmarshal(data, vgeositeList); err != nil {
|
||||||
|
return nil, nil, fmt.Errorf("failed to unmarshal: %w", err)
|
||||||
|
}
|
||||||
|
domainLists := make([]DomainList, len(vgeositeList.Entry))
|
||||||
|
domainListByName := make(map[string]*DomainList, len(vgeositeList.Entry))
|
||||||
|
for i, vsite := range vgeositeList.Entry {
|
||||||
|
rules := make([]DomainRule, 0, len(vsite.Domain))
|
||||||
|
for _, vdomain := range vsite.Domain {
|
||||||
|
rule := DomainRule{Value: vdomain.Value}
|
||||||
|
switch vdomain.Type {
|
||||||
|
case router.Domain_RootDomain:
|
||||||
|
rule.Type = dlc.RuleTypeDomain
|
||||||
|
case router.Domain_Regex:
|
||||||
|
rule.Type = dlc.RuleTypeRegexp
|
||||||
|
case router.Domain_Plain:
|
||||||
|
rule.Type = dlc.RuleTypeKeyword
|
||||||
|
case router.Domain_Full:
|
||||||
|
rule.Type = dlc.RuleTypeFullDomain
|
||||||
|
default:
|
||||||
|
return nil, nil, fmt.Errorf("invalid rule type: %+v", vdomain.Type)
|
||||||
|
}
|
||||||
|
for _, vattr := range vdomain.Attribute {
|
||||||
|
rule.Attrs = append(rule.Attrs, vattr.Key)
|
||||||
|
}
|
||||||
|
rules = append(rules, rule)
|
||||||
|
}
|
||||||
|
domainLists[i] = DomainList{
|
||||||
|
Name: strings.ToUpper(vsite.CountryCode),
|
||||||
|
Rules: rules,
|
||||||
|
}
|
||||||
|
domainListByName[domainLists[i].Name] = &domainLists[i]
|
||||||
|
}
|
||||||
|
return domainLists, domainListByName, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func exportSite(name string, domainListByName map[string]*DomainList) error {
|
||||||
|
domainList, ok := domainListByName[strings.ToUpper(name)]
|
||||||
|
if !ok {
|
||||||
|
return fmt.Errorf("list '%s' does not exist", name)
|
||||||
|
}
|
||||||
|
if len(domainList.Rules) == 0 {
|
||||||
|
return fmt.Errorf("list '%s' is empty", name)
|
||||||
|
}
|
||||||
|
file, err := os.Create(filepath.Join(*outputDir, name+".yml"))
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer file.Close()
|
||||||
|
w := bufio.NewWriter(file)
|
||||||
|
fmt.Fprintf(w, "%s:\n", name)
|
||||||
|
for _, domain := range domainList.Rules {
|
||||||
|
fmt.Fprintf(w, " - %q\n", domain.domain2String())
|
||||||
|
}
|
||||||
|
return w.Flush()
|
||||||
|
}
|
||||||
|
|
||||||
|
func exportAll(filename string, domainLists []DomainList) error {
|
||||||
|
file, err := os.Create(filepath.Join(*outputDir, filename))
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer file.Close()
|
||||||
|
w := bufio.NewWriter(file)
|
||||||
|
w.WriteString("lists:\n")
|
||||||
|
for _, domainList := range domainLists {
|
||||||
|
fmt.Fprintf(w, " - name: %s\n", strings.ToLower(domainList.Name))
|
||||||
|
fmt.Fprintf(w, " length: %d\n", len(domainList.Rules))
|
||||||
|
w.WriteString(" rules:\n")
|
||||||
|
for _, domain := range domainList.Rules {
|
||||||
|
fmt.Fprintf(w, " - %q\n", domain.domain2String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return w.Flush()
|
||||||
|
}
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
flag.Parse()
|
||||||
|
|
||||||
|
// Create output directory if not exist
|
||||||
|
if _, err := os.Stat(*outputDir); os.IsNotExist(err) {
|
||||||
|
if mkErr := os.MkdirAll(*outputDir, 0755); mkErr != nil {
|
||||||
|
fmt.Println("Failed to create output directory:", mkErr)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fmt.Printf("Loading %s...\n", *inputData)
|
||||||
|
domainLists, domainListByName, err := loadGeosite(*inputData)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Println("Failed to loadGeosite:", err)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
|
||||||
|
var exportListSlice []string
|
||||||
|
for raw := range strings.SplitSeq(*exportLists, ",") {
|
||||||
|
if trimmed := strings.TrimSpace(raw); trimmed != "" {
|
||||||
|
exportListSlice = append(exportListSlice, trimmed)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(exportListSlice) == 0 {
|
||||||
|
exportListSlice = []string{"_all_"}
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, eplistname := range exportListSlice {
|
||||||
|
if strings.EqualFold(eplistname, "_all_") {
|
||||||
|
if err := exportAll(filepath.Base(*inputData)+"_plain.yml", domainLists); err != nil {
|
||||||
|
fmt.Println("Failed to exportAll:", err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
if err := exportSite(eplistname, domainListByName); err != nil {
|
||||||
|
fmt.Println("Failed to exportSite:", err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
}
|
||||||
|
fmt.Printf("list: '%s' has been exported successfully.\n", eplistname)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -28,6 +28,7 @@ binancezh.top
|
|||||||
|
|
||||||
# API
|
# API
|
||||||
binanceapi.com
|
binanceapi.com
|
||||||
|
binanceru.net
|
||||||
bnbstatic.com
|
bnbstatic.com
|
||||||
bntrace.com
|
bntrace.com
|
||||||
nftstatic.com
|
nftstatic.com
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ include:curseforge
|
|||||||
include:cygames
|
include:cygames
|
||||||
include:ea
|
include:ea
|
||||||
include:embark
|
include:embark
|
||||||
|
include:eneba
|
||||||
include:epicgames
|
include:epicgames
|
||||||
include:escapefromtarkov
|
include:escapefromtarkov
|
||||||
include:faceit
|
include:faceit
|
||||||
|
|||||||
2
data/eneba
Normal file
2
data/eneba
Normal file
@@ -0,0 +1,2 @@
|
|||||||
|
eneba.com
|
||||||
|
eneba.games
|
||||||
3
data/okx
3
data/okx
@@ -1,8 +1,9 @@
|
|||||||
okex.com
|
okex.com
|
||||||
okx.com
|
|
||||||
okx-dns.com
|
okx-dns.com
|
||||||
okx-dns1.com
|
okx-dns1.com
|
||||||
okx-dns2.com
|
okx-dns2.com
|
||||||
|
okx.cab
|
||||||
|
okx.com
|
||||||
|
|
||||||
# OKC Browser
|
# OKC Browser
|
||||||
oklink.com @cn
|
oklink.com @cn
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
# Main domain
|
# Main domain
|
||||||
chatgpt.com
|
|
||||||
chat.com
|
chat.com
|
||||||
|
chatgpt.com
|
||||||
|
crixet.com
|
||||||
oaistatic.com
|
oaistatic.com
|
||||||
oaiusercontent.com
|
oaiusercontent.com
|
||||||
openai.com
|
openai.com
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ galaxyappstore.com
|
|||||||
galaxymobile.jp
|
galaxymobile.jp
|
||||||
game-platform.net
|
game-platform.net
|
||||||
knoxemm.com
|
knoxemm.com
|
||||||
|
ospserver.net
|
||||||
samsung.com
|
samsung.com
|
||||||
samsungads.com @ads
|
samsungads.com @ads
|
||||||
samsungapps.com
|
samsungapps.com
|
||||||
|
|||||||
@@ -1,2 +1,7 @@
|
|||||||
|
1024terabox.com
|
||||||
|
bestclouddrive.com
|
||||||
|
freeterabox.com
|
||||||
|
nephobox.com
|
||||||
terabox.com
|
terabox.com
|
||||||
|
terabox1024.com
|
||||||
teraboxcdn.com
|
teraboxcdn.com
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
include:askdiandian
|
include:askdiandian
|
||||||
|
|
||||||
|
rednotecdn.com
|
||||||
xhscdn.com
|
xhscdn.com
|
||||||
xhscdn.net
|
xhscdn.net
|
||||||
xhslink.com
|
xhslink.com
|
||||||
|
|||||||
9
internal/dlc/dlc.go
Normal file
9
internal/dlc/dlc.go
Normal file
@@ -0,0 +1,9 @@
|
|||||||
|
package dlc
|
||||||
|
|
||||||
|
const (
|
||||||
|
RuleTypeDomain string = "domain"
|
||||||
|
RuleTypeFullDomain string = "full"
|
||||||
|
RuleTypeKeyword string = "keyword"
|
||||||
|
RuleTypeRegexp string = "regexp"
|
||||||
|
RuleTypeInclude string = "include"
|
||||||
|
)
|
||||||
200
main.go
200
main.go
@@ -10,6 +10,7 @@ import (
|
|||||||
"slices"
|
"slices"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
|
"github.com/v2fly/domain-list-community/internal/dlc"
|
||||||
router "github.com/v2fly/v2ray-core/v5/app/router/routercommon"
|
router "github.com/v2fly/v2ray-core/v5/app/router/routercommon"
|
||||||
"google.golang.org/protobuf/proto"
|
"google.golang.org/protobuf/proto"
|
||||||
)
|
)
|
||||||
@@ -21,21 +22,6 @@ var (
|
|||||||
exportLists = flag.String("exportlists", "", "Lists to be flattened and exported in plaintext format, separated by ',' comma")
|
exportLists = flag.String("exportlists", "", "Lists to be flattened and exported in plaintext format, separated by ',' comma")
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
|
||||||
RuleTypeDomain string = "domain"
|
|
||||||
RuleTypeFullDomain string = "full"
|
|
||||||
RuleTypeKeyword string = "keyword"
|
|
||||||
RuleTypeRegexp string = "regexp"
|
|
||||||
RuleTypeInclude string = "include"
|
|
||||||
)
|
|
||||||
|
|
||||||
var (
|
|
||||||
TypeChecker = regexp.MustCompile(`^(domain|full|keyword|regexp|include)$`)
|
|
||||||
ValueChecker = regexp.MustCompile(`^[a-z0-9!\.-]+$`)
|
|
||||||
AttrChecker = regexp.MustCompile(`^[a-z0-9!-]+$`)
|
|
||||||
SiteChecker = regexp.MustCompile(`^[A-Z0-9!-]+$`)
|
|
||||||
)
|
|
||||||
|
|
||||||
var (
|
var (
|
||||||
refMap = make(map[string][]*Entry)
|
refMap = make(map[string][]*Entry)
|
||||||
plMap = make(map[string]*ParsedList)
|
plMap = make(map[string]*ParsedList)
|
||||||
@@ -66,7 +52,7 @@ type ParsedList struct {
|
|||||||
func makeProtoList(listName string, entries []*Entry) (*router.GeoSite, error) {
|
func makeProtoList(listName string, entries []*Entry) (*router.GeoSite, error) {
|
||||||
site := &router.GeoSite{
|
site := &router.GeoSite{
|
||||||
CountryCode: listName,
|
CountryCode: listName,
|
||||||
Domain: make([]*router.Domain, 0, len(entries)),
|
Domain: make([]*router.Domain, 0, len(entries)),
|
||||||
}
|
}
|
||||||
for _, entry := range entries {
|
for _, entry := range entries {
|
||||||
pdomain := &router.Domain{Value: entry.Value}
|
pdomain := &router.Domain{Value: entry.Value}
|
||||||
@@ -78,13 +64,13 @@ func makeProtoList(listName string, entries []*Entry) (*router.GeoSite, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
switch entry.Type {
|
switch entry.Type {
|
||||||
case RuleTypeDomain:
|
case dlc.RuleTypeDomain:
|
||||||
pdomain.Type = router.Domain_RootDomain
|
pdomain.Type = router.Domain_RootDomain
|
||||||
case RuleTypeRegexp:
|
case dlc.RuleTypeRegexp:
|
||||||
pdomain.Type = router.Domain_Regex
|
pdomain.Type = router.Domain_Regex
|
||||||
case RuleTypeKeyword:
|
case dlc.RuleTypeKeyword:
|
||||||
pdomain.Type = router.Domain_Plain
|
pdomain.Type = router.Domain_Plain
|
||||||
case RuleTypeFullDomain:
|
case dlc.RuleTypeFullDomain:
|
||||||
pdomain.Type = router.Domain_Full
|
pdomain.Type = router.Domain_Full
|
||||||
}
|
}
|
||||||
site.Domain = append(site.Domain, pdomain)
|
site.Domain = append(site.Domain, pdomain)
|
||||||
@@ -95,9 +81,9 @@ func makeProtoList(listName string, entries []*Entry) (*router.GeoSite, error) {
|
|||||||
func writePlainList(exportedName string) error {
|
func writePlainList(exportedName string) error {
|
||||||
targetList, exist := finalMap[strings.ToUpper(exportedName)]
|
targetList, exist := finalMap[strings.ToUpper(exportedName)]
|
||||||
if !exist || len(targetList) == 0 {
|
if !exist || len(targetList) == 0 {
|
||||||
return fmt.Errorf("'%s' list does not exist or is empty.", exportedName)
|
return fmt.Errorf("list %q does not exist or is empty.", exportedName)
|
||||||
}
|
}
|
||||||
file, err := os.Create(filepath.Join(*outputDir, strings.ToLower(exportedName) + ".txt"))
|
file, err := os.Create(filepath.Join(*outputDir, strings.ToLower(exportedName)+".txt"))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -112,51 +98,62 @@ func writePlainList(exportedName string) error {
|
|||||||
func parseEntry(line string) (Entry, error) {
|
func parseEntry(line string) (Entry, error) {
|
||||||
var entry Entry
|
var entry Entry
|
||||||
parts := strings.Fields(line)
|
parts := strings.Fields(line)
|
||||||
|
if len(parts) == 0 {
|
||||||
|
return entry, fmt.Errorf("empty line: %q", line)
|
||||||
|
}
|
||||||
|
|
||||||
// Parse type and value
|
// Parse type and value
|
||||||
rawTypeVal := parts[0]
|
v := parts[0]
|
||||||
kv := strings.Split(rawTypeVal, ":")
|
colonIndex := strings.Index(v, ":")
|
||||||
if len(kv) == 1 {
|
if colonIndex == -1 {
|
||||||
entry.Type = RuleTypeDomain // Default type
|
entry.Type = dlc.RuleTypeDomain // Default type
|
||||||
entry.Value = strings.ToLower(rawTypeVal)
|
entry.Value = strings.ToLower(v)
|
||||||
} else if len(kv) == 2 {
|
if !validateDomainChars(entry.Value) {
|
||||||
entry.Type = strings.ToLower(kv[0])
|
return entry, fmt.Errorf("invalid domain: %q", entry.Value)
|
||||||
if entry.Type == RuleTypeRegexp {
|
|
||||||
entry.Value = kv[1]
|
|
||||||
} else {
|
|
||||||
entry.Value = strings.ToLower(kv[1])
|
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
return entry, fmt.Errorf("invalid format: %s", line)
|
typ := strings.ToLower(v[:colonIndex])
|
||||||
}
|
val := v[colonIndex+1:]
|
||||||
// Check type and value
|
switch typ {
|
||||||
if !TypeChecker.MatchString(entry.Type) {
|
case dlc.RuleTypeRegexp:
|
||||||
return entry, fmt.Errorf("invalid type: %s", entry.Type)
|
if _, err := regexp.Compile(val); err != nil {
|
||||||
}
|
return entry, fmt.Errorf("invalid regexp %q: %w", val, err)
|
||||||
if entry.Type == RuleTypeRegexp {
|
}
|
||||||
if _, err := regexp.Compile(entry.Value); err != nil {
|
entry.Type = dlc.RuleTypeRegexp
|
||||||
return entry, fmt.Errorf("invalid regexp: %s", entry.Value)
|
entry.Value = val
|
||||||
|
case dlc.RuleTypeInclude:
|
||||||
|
entry.Type = dlc.RuleTypeInclude
|
||||||
|
entry.Value = strings.ToUpper(val)
|
||||||
|
if !validateSiteName(entry.Value) {
|
||||||
|
return entry, fmt.Errorf("invalid include list name: %q", entry.Value)
|
||||||
|
}
|
||||||
|
case dlc.RuleTypeDomain, dlc.RuleTypeFullDomain, dlc.RuleTypeKeyword:
|
||||||
|
entry.Type = typ
|
||||||
|
entry.Value = strings.ToLower(val)
|
||||||
|
if !validateDomainChars(entry.Value) {
|
||||||
|
return entry, fmt.Errorf("invalid domain: %q", entry.Value)
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
return entry, fmt.Errorf("invalid type: %q", typ)
|
||||||
}
|
}
|
||||||
} else if !ValueChecker.MatchString(entry.Value) {
|
|
||||||
return entry, fmt.Errorf("invalid value: %s", entry.Value)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Parse/Check attributes and affiliations
|
// Parse/Check attributes and affiliations
|
||||||
for _, part := range parts[1:] {
|
for _, part := range parts[1:] {
|
||||||
if strings.HasPrefix(part, "@") {
|
if strings.HasPrefix(part, "@") {
|
||||||
attr := strings.ToLower(part[1:]) // Trim attribute prefix `@` character
|
attr := strings.ToLower(part[1:]) // Trim attribute prefix `@` character
|
||||||
if !AttrChecker.MatchString(attr) {
|
if !validateAttrChars(attr) {
|
||||||
return entry, fmt.Errorf("invalid attribute key: %s", attr)
|
return entry, fmt.Errorf("invalid attribute: %q", attr)
|
||||||
}
|
}
|
||||||
entry.Attrs = append(entry.Attrs, attr)
|
entry.Attrs = append(entry.Attrs, attr)
|
||||||
} else if strings.HasPrefix(part, "&") {
|
} else if strings.HasPrefix(part, "&") {
|
||||||
aff := strings.ToUpper(part[1:]) // Trim affiliation prefix `&` character
|
aff := strings.ToUpper(part[1:]) // Trim affiliation prefix `&` character
|
||||||
if !SiteChecker.MatchString(aff) {
|
if !validateSiteName(aff) {
|
||||||
return entry, fmt.Errorf("invalid affiliation key: %s", aff)
|
return entry, fmt.Errorf("invalid affiliation: %q", aff)
|
||||||
}
|
}
|
||||||
entry.Affs = append(entry.Affs, aff)
|
entry.Affs = append(entry.Affs, aff)
|
||||||
} else {
|
} else {
|
||||||
return entry, fmt.Errorf("invalid attribute/affiliation: %s", part)
|
return entry, fmt.Errorf("invalid attribute/affiliation: %q", part)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
// Sort attributes
|
// Sort attributes
|
||||||
@@ -170,6 +167,39 @@ func parseEntry(line string) (Entry, error) {
|
|||||||
return entry, nil
|
return entry, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func validateDomainChars(domain string) bool {
|
||||||
|
for i := range domain {
|
||||||
|
c := domain[i]
|
||||||
|
if (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') || c == '.' || c == '-' {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
func validateAttrChars(attr string) bool {
|
||||||
|
for i := range attr {
|
||||||
|
c := attr[i]
|
||||||
|
if (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') || c == '!' || c == '-' {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
func validateSiteName(name string) bool {
|
||||||
|
for i := range name {
|
||||||
|
c := name[i]
|
||||||
|
if (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '!' || c == '-' {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
func loadData(path string) error {
|
func loadData(path string) error {
|
||||||
file, err := os.Open(path)
|
file, err := os.Open(path)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -178,7 +208,7 @@ func loadData(path string) error {
|
|||||||
defer file.Close()
|
defer file.Close()
|
||||||
|
|
||||||
listName := strings.ToUpper(filepath.Base(path))
|
listName := strings.ToUpper(filepath.Base(path))
|
||||||
if !SiteChecker.MatchString(listName) {
|
if !validateSiteName(listName) {
|
||||||
return fmt.Errorf("invalid list name: %s", listName)
|
return fmt.Errorf("invalid list name: %s", listName)
|
||||||
}
|
}
|
||||||
scanner := bufio.NewScanner(file)
|
scanner := bufio.NewScanner(file)
|
||||||
@@ -210,11 +240,11 @@ func parseList(refName string, refList []*Entry) error {
|
|||||||
plMap[refName] = pl
|
plMap[refName] = pl
|
||||||
}
|
}
|
||||||
for _, entry := range refList {
|
for _, entry := range refList {
|
||||||
if entry.Type == RuleTypeInclude {
|
if entry.Type == dlc.RuleTypeInclude {
|
||||||
if len(entry.Affs) != 0 {
|
if len(entry.Affs) != 0 {
|
||||||
return fmt.Errorf("affiliation is not allowed for include:%s", entry.Value)
|
return fmt.Errorf("affiliation is not allowed for include:%s", entry.Value)
|
||||||
}
|
}
|
||||||
inc := &Inclusion{Source: strings.ToUpper(entry.Value)}
|
inc := &Inclusion{Source: entry.Value}
|
||||||
for _, attr := range entry.Attrs {
|
for _, attr := range entry.Attrs {
|
||||||
if strings.HasPrefix(attr, "-") {
|
if strings.HasPrefix(attr, "-") {
|
||||||
inc.BanAttrs = append(inc.BanAttrs, attr[1:]) // Trim attribute prefix `-` character
|
inc.BanAttrs = append(inc.BanAttrs, attr[1:]) // Trim attribute prefix `-` character
|
||||||
@@ -244,18 +274,18 @@ func polishList(roughMap *map[string]*Entry) []*Entry {
|
|||||||
domainsMap := make(map[string]bool)
|
domainsMap := make(map[string]bool)
|
||||||
for _, entry := range *roughMap {
|
for _, entry := range *roughMap {
|
||||||
switch entry.Type { // Bypass regexp, keyword and "full/domain with attr"
|
switch entry.Type { // Bypass regexp, keyword and "full/domain with attr"
|
||||||
case RuleTypeRegexp:
|
case dlc.RuleTypeRegexp:
|
||||||
finalList = append(finalList, entry)
|
finalList = append(finalList, entry)
|
||||||
case RuleTypeKeyword:
|
case dlc.RuleTypeKeyword:
|
||||||
finalList = append(finalList, entry)
|
finalList = append(finalList, entry)
|
||||||
case RuleTypeDomain:
|
case dlc.RuleTypeDomain:
|
||||||
domainsMap[entry.Value] = true
|
domainsMap[entry.Value] = true
|
||||||
if len(entry.Attrs) != 0 {
|
if len(entry.Attrs) != 0 {
|
||||||
finalList = append(finalList, entry)
|
finalList = append(finalList, entry)
|
||||||
} else {
|
} else {
|
||||||
queuingList = append(queuingList, entry)
|
queuingList = append(queuingList, entry)
|
||||||
}
|
}
|
||||||
case RuleTypeFullDomain:
|
case dlc.RuleTypeFullDomain:
|
||||||
if len(entry.Attrs) != 0 {
|
if len(entry.Attrs) != 0 {
|
||||||
finalList = append(finalList, entry)
|
finalList = append(finalList, entry)
|
||||||
} else {
|
} else {
|
||||||
@@ -266,12 +296,19 @@ func polishList(roughMap *map[string]*Entry) []*Entry {
|
|||||||
// Remove redundant subdomains for full/domain without attr
|
// Remove redundant subdomains for full/domain without attr
|
||||||
for _, qentry := range queuingList {
|
for _, qentry := range queuingList {
|
||||||
isRedundant := false
|
isRedundant := false
|
||||||
pd := qentry.Value // Parent domain
|
pd := qentry.Value // To be parent domain
|
||||||
|
if qentry.Type == dlc.RuleTypeFullDomain {
|
||||||
|
pd = "." + pd // So that `domain:example.org` overrides `full:example.org`
|
||||||
|
}
|
||||||
for {
|
for {
|
||||||
idx := strings.Index(pd, ".")
|
idx := strings.Index(pd, ".")
|
||||||
if idx == -1 { break }
|
if idx == -1 {
|
||||||
|
break
|
||||||
|
}
|
||||||
pd = pd[idx+1:] // Go for next parent
|
pd = pd[idx+1:] // Go for next parent
|
||||||
if !strings.Contains(pd, ".") { break } // Not allow tld to be a parent
|
if !strings.Contains(pd, ".") {
|
||||||
|
break
|
||||||
|
} // Not allow tld to be a parent
|
||||||
if domainsMap[pd] {
|
if domainsMap[pd] {
|
||||||
isRedundant = true
|
isRedundant = true
|
||||||
break
|
break
|
||||||
@@ -289,7 +326,9 @@ func polishList(roughMap *map[string]*Entry) []*Entry {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func resolveList(pl *ParsedList) error {
|
func resolveList(pl *ParsedList) error {
|
||||||
if _, pldone := finalMap[pl.Name]; pldone { return nil }
|
if _, pldone := finalMap[pl.Name]; pldone {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
if cirIncMap[pl.Name] {
|
if cirIncMap[pl.Name] {
|
||||||
return fmt.Errorf("circular inclusion in: %s", pl.Name)
|
return fmt.Errorf("circular inclusion in: %s", pl.Name)
|
||||||
@@ -298,14 +337,22 @@ func resolveList(pl *ParsedList) error {
|
|||||||
defer delete(cirIncMap, pl.Name)
|
defer delete(cirIncMap, pl.Name)
|
||||||
|
|
||||||
isMatchAttrFilters := func(entry *Entry, incFilter *Inclusion) bool {
|
isMatchAttrFilters := func(entry *Entry, incFilter *Inclusion) bool {
|
||||||
if len(incFilter.MustAttrs) == 0 && len(incFilter.BanAttrs) == 0 { return true }
|
if len(incFilter.MustAttrs) == 0 && len(incFilter.BanAttrs) == 0 {
|
||||||
if len(entry.Attrs) == 0 { return len(incFilter.MustAttrs) == 0 }
|
return true
|
||||||
|
}
|
||||||
|
if len(entry.Attrs) == 0 {
|
||||||
|
return len(incFilter.MustAttrs) == 0
|
||||||
|
}
|
||||||
|
|
||||||
for _, m := range incFilter.MustAttrs {
|
for _, m := range incFilter.MustAttrs {
|
||||||
if !slices.Contains(entry.Attrs, m) { return false }
|
if !slices.Contains(entry.Attrs, m) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
}
|
}
|
||||||
for _, b := range incFilter.BanAttrs {
|
for _, b := range incFilter.BanAttrs {
|
||||||
if slices.Contains(entry.Attrs, b) { return false }
|
if slices.Contains(entry.Attrs, b) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
}
|
}
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
@@ -317,7 +364,7 @@ func resolveList(pl *ParsedList) error {
|
|||||||
for _, inc := range pl.Inclusions {
|
for _, inc := range pl.Inclusions {
|
||||||
incPl, exist := plMap[inc.Source]
|
incPl, exist := plMap[inc.Source]
|
||||||
if !exist {
|
if !exist {
|
||||||
return fmt.Errorf("list '%s' includes a non-existent list: '%s'", pl.Name, inc.Source)
|
return fmt.Errorf("list %q includes a non-existent list: %q", pl.Name, inc.Source)
|
||||||
}
|
}
|
||||||
if err := resolveList(incPl); err != nil {
|
if err := resolveList(incPl); err != nil {
|
||||||
return err
|
return err
|
||||||
@@ -375,29 +422,32 @@ func main() {
|
|||||||
// Create output directory if not exist
|
// Create output directory if not exist
|
||||||
if _, err := os.Stat(*outputDir); os.IsNotExist(err) {
|
if _, err := os.Stat(*outputDir); os.IsNotExist(err) {
|
||||||
if mkErr := os.MkdirAll(*outputDir, 0755); mkErr != nil {
|
if mkErr := os.MkdirAll(*outputDir, 0755); mkErr != nil {
|
||||||
fmt.Println("Failed:", mkErr)
|
fmt.Println("Failed to create output directory:", mkErr)
|
||||||
os.Exit(1)
|
os.Exit(1)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Export plaintext list
|
// Export plaintext list
|
||||||
if *exportLists != "" {
|
var exportListSlice []string
|
||||||
exportedListSlice := strings.Split(*exportLists, ",")
|
for raw := range strings.SplitSeq(*exportLists, ",") {
|
||||||
for _, exportedList := range exportedListSlice {
|
if trimmed := strings.TrimSpace(raw); trimmed != "" {
|
||||||
if err := writePlainList(exportedList); err != nil {
|
exportListSlice = append(exportListSlice, trimmed)
|
||||||
fmt.Println("Failed to write list:", err)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
fmt.Printf("list: '%s' has been generated successfully.\n", exportedList)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
for _, exportList := range exportListSlice {
|
||||||
|
if err := writePlainList(exportList); err != nil {
|
||||||
|
fmt.Println("Failed to write list:", err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
fmt.Printf("list %q has been generated successfully.\n", exportList)
|
||||||
|
}
|
||||||
|
|
||||||
// Generate dat file
|
// Generate dat file
|
||||||
protoList := new(router.GeoSiteList)
|
protoList := new(router.GeoSiteList)
|
||||||
for siteName, siteEntries := range finalMap {
|
for siteName, siteEntries := range finalMap {
|
||||||
site, err := makeProtoList(siteName, siteEntries)
|
site, err := makeProtoList(siteName, siteEntries)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
fmt.Println("Failed:", err)
|
fmt.Println("Failed to makeProtoList:", err)
|
||||||
os.Exit(1)
|
os.Exit(1)
|
||||||
}
|
}
|
||||||
protoList.Entry = append(protoList.Entry, site)
|
protoList.Entry = append(protoList.Entry, site)
|
||||||
|
|||||||
Reference in New Issue
Block a user