Add more cn domains (#3380)

* weiphone: add weiphone.net

* category-education-cn: add more domains

* category-media-cn: add more domains

* geolocation-cn: add more domains

.gitignore

@@ -4,9 +4,9 @@
 /domain-list-community
 /domain-list-community.exe
 
-# Generated dat file.
-dlc.dat
+# Generated dat files.
+/*.dat
 
 # Exported plaintext lists.
-/*.yml
 /*.txt
+/*.yml

README.md

@@ -86,6 +86,8 @@ Each file in the `data` directory can be used as a rule in this format: `geosite
 
 Run `go run ./ --help` for more usage information.
 
+For anyone who wants to generate custom `.dat` files, you may read [#3370](https://github.com/v2fly/domain-list-community/discussions/3370).
+
 ## Structure of data
 
 All data are under `data` directory. Each file in the directory represents a sub-list of domains, named by the file name. File content is in the following format.
@@ -105,7 +107,7 @@ regexp:^odd[1-7]\.example\.org(\.[a-z]{2})?$
 > Adding new `regexp` and `keyword` rules is discouraged because it is easy to use them incorrectly, and proxy software cannot efficiently match these types of rules.
 
 > [!NOTE]
-> The following types of rules are **NOT** fully compatible with the ones that defined by user in V2Ray config file. Do **Not** copy and paste directly.
+> The following types of rules are **NOT** fully compatible with the ones that defined by user in V2Ray config file. Do **NOT** copy and paste directly.
 
 - Comment begins with `#`. It may begin anywhere in the file. The content in the line after `#` is treated as comment and ignored in production.
 - Subdomain begins with `domain:`, followed by a valid domain name. The prefix `domain:` may be omitted.
@@ -114,7 +116,7 @@ regexp:^odd[1-7]\.example\.org(\.[a-z]{2})?$
 - Regular expression begins with `regexp:`, followed by a valid regular expression (per Golang's standard).
 - Domain rules (including `domain`, `full`, `keyword`, and `regexp`) may have none, one or more attributes. Each attribute begins with `@` and followed by the name of the attribute. Attributes will remain available in final lists and `dlc.dat`.
 - Domain rules may have none, one or more affiliations, which additionally adds the domain rule into the affiliated target list. Each affiliation begins with `&` and followed by the name of the target list (nomatter whether the target has a dedicated file in data path). This is a method for data management, and will not remain in the final lists or `dlc.dat`.
-- Inclusion begins with `include:`, followed by the name of another valid domain list. A simple `include:listb` in file `lista` means adding all domain rules of `listb` into `lista`. Inclusions with attributes stands for selective inclusion. `include:listb @attr1 @-attr2` means only adding those domain rules *with* `@attr1` **and** *without* `@attr2`. This is a special type for data management, and will not remain in the final lists or `dlc.dat`.
+- Inclusion begins with `include:`, followed by the name of another valid domain list. `include:listb` in file `lista` means adding all domain rules of `listb` into `lista`. Inclusions with attributes stand for selective inclusion. `include:listb @attr1 @-attr2` means only adding those domain rules *with* `@attr1` **and** *without* `@attr2`. This is a special type for data management, and will not remain in the final lists or `dlc.dat`.
 
 ## How it works
 

data/category-acg

@@ -1,4 +1,5 @@
 # Game
+bestdori.com
 colorfulstage.com
 
 # Idol

data/category-dev

@@ -54,6 +54,7 @@ include:stackexchange
 include:strikingly
 include:termux
 include:thelinuxfoundation
+include:tilda
 include:unity
 include:v8
 
@@ -152,4 +153,5 @@ wireshark.org
 x.org
 xposed.info
 yarnpkg.com
+ziglang.org
 zsh.org

data/category-education-cn

@@ -91,6 +91,7 @@ eoffcn.com
 jinrongren.net
 offcn.com
 zggqzp.com
+zgjsks.com
 zgsydw.com
 # 高考100
 gk100.com
@@ -105,6 +106,9 @@ xir.cn
 # 日语测试
 j-test.com
 nattest-china.com
+# 菁优网
+jyeoo.com
+jyeoo.net
 # 课播云校
 keboyunxiao.com
 # 课堂派

data/category-electronic-cn

@@ -17,11 +17,10 @@ bouffalolab.com
 cxmt.com
 
 # 乐鑫信息科技
+#include:espressif
 esp8266.cn
-esp8266.com
 esp8266.com.cn
 espressif.cn
-espressif.com
 espressif.com.cn
 
 # 华秋电子

data/category-entertainment-ru

@@ -11,6 +11,7 @@ include:okko
 include:wink
 24h.tv
 amediateka.ru
+beeline.tv
 ivi.ru
 premier.one
 smotreshka.tv

data/category-ip-geo-detect

@@ -97,6 +97,7 @@ myexternalip.com
 myip.com
 myip.ms
 myip.ru
+myip.wtf
 myipaddress.com
 myiplookup.com
 mylocation.org
@@ -134,6 +135,7 @@ full:checkip.amazonaws.com
 full:ipv4-check-perf.radar.cloudflare.com
 full:ipv6-check-perf.radar.cloudflare.com
 geoip.noc.gov.ru
+ip.hetzner.com
 ip.mail.ru
 ip.nic.ru
 ip.tyk.nu

data/category-media-cn

@@ -135,6 +135,10 @@ kksmg.com
 statickksmg.com
 # 晚点
 latepost.com
+# 雷科技
+leikeji.com
+# 雷峰网
+leiphone.com
 # 今日观察网
 miercn.com
 # 快科技
@@ -206,14 +210,21 @@ techweb.com.cn
 thecover.cn
 # 澎湃新闻
 thepaper.cn
+# 潮新闻/浙江日报
+8531.cn
+tidenews.com.cn
 # 人人都是产品经理
 woshipm.com
+# 新华日报
+xhby.net
 # 西陆网
 xilu.com
 # 新快报
 xkb.com.cn
 # 第一财经
 yicai.com
+# 扬子晚报
+yzwb.net
 # ZEALER
 zealer.com
 # 知产力

data/geolocation-cn

@@ -222,8 +222,12 @@ mingxuan.store
 gov.cn
 ## 中国气象局
 cma.cn
+## 政务和公益机构域名注册管理中心
+conac.cn
 ## 中央气象台
 nmc.cn
+## 中国互联网联合辟谣平台
+piyao.org.cn
 
 # Public transportation
 ## 中国国际航空
@@ -321,14 +325,14 @@ webterren.com
 ## 国学大师网
 guoxuedashi.com
 guoxuemi.com
-## 志愿汇
-zyh365.com
 ## 温州市图书馆
 wzlib.cn
 ## 中国大百科全书
 zgbk.com
 ## 浙江图书馆
 zjlib.cn
+## 志愿汇
+zyh365.com
 
 # Services & Softwares
 include:category-ai-cn
@@ -1116,8 +1120,6 @@ lascn.net
 lawxp.com
 leangoo.com
 leanote.com
-leikeji.com
-leiphone.com
 leiting.com
 leju.com
 leturich.org
@@ -1398,7 +1400,6 @@ weand.com
 weavatar.com
 weicaifu.com
 weilaicaijing.com
-weiphone.net
 weixing.com
 weiyangx.com
 welltrend-edu.com
@@ -1527,7 +1528,6 @@ zastatic.com
 zczj.com
 zdfans.com
 zgjm.org
-zgjsks.com
 zglxw.com
 zgzcw.com
 zhanbuba.com
@@ -1625,3 +1625,6 @@ ao-x.ac.cn
 
 # 万集科技 京ICP备18036282号-2
 wanji.net.cn
+
+# 广州市雅望互联网服务有限公司
+gzyowin.com

data/qcloud

@@ -258,13 +258,14 @@ tdnsv14.net
 tdnsv15.net
 
 # myqcloud inside mainland China
-regexp:\.(.+-)?ap-beijing(-.+)?\.myqcloud\.com$             #北京
-regexp:\.(.+-)?ap-nanjing(-.+)?\.myqcloud\.com$             #南京
-regexp:\.(.+-)?ap-shanghai(-.+)?\.myqcloud\.com$            #上海
-regexp:\.(.+-)?ap-guangzhou(-.+)?\.myqcloud\.com$           #广州
-regexp:\.(.+-)?ap-chengdu(-.+)?\.myqcloud\.com$             #成都
-regexp:\.(.+-)?ap-chongqing(-.+)?\.myqcloud\.com$           #重庆
-regexp:\.(.+-)?ap-shenzhen(-.+)?\.myqcloud\.com$            #深圳
+# overrided by myqcloud.com
+#regexp:\.(.+-)?ap-beijing(-.+)?\.myqcloud\.com$             #北京
+#regexp:\.(.+-)?ap-nanjing(-.+)?\.myqcloud\.com$             #南京
+#regexp:\.(.+-)?ap-shanghai(-.+)?\.myqcloud\.com$            #上海
+#regexp:\.(.+-)?ap-guangzhou(-.+)?\.myqcloud\.com$           #广州
+#regexp:\.(.+-)?ap-chengdu(-.+)?\.myqcloud\.com$             #成都
+#regexp:\.(.+-)?ap-chongqing(-.+)?\.myqcloud\.com$           #重庆
+#regexp:\.(.+-)?ap-shenzhen(-.+)?\.myqcloud\.com$            #深圳
 
 # COS 使用到的非中国大陆的地域与可用区，参见 https://cloud.tencent.com/document/product/436/6224
 ap-hongkong.myqcloud.com @!cn       #中国香港
@@ -282,13 +283,14 @@ eu-frankfurt.myqcloud.com @!cn      #法兰克福
 eu-moscow.myqcloud.com @!cn         #莫斯科
 
 # tencentcos inside mainland China
-regexp:\.(.+-)?ap-beijing(-.+)?\.tencentcos\.(cn|com(\.cn)?)$   #北京
-regexp:\.(.+-)?ap-nanjing(-.+)?\.tencentcos\.(cn|com(\.cn)?)$   #南京
-regexp:\.(.+-)?ap-shanghai(-.+)?\.tencentcos\.(cn|com(\.cn)?)$  #上海
-regexp:\.(.+-)?ap-guangzhou(-.+)?\.tencentcos\.(cn|com(\.cn)?)$ #广州
-regexp:\.(.+-)?ap-chengdu(-.+)?\.tencentcos\.(cn|com(\.cn)?)$   #成都
-regexp:\.(.+-)?ap-chongqing(-.+)?\.tencentcos\.(cn|com(\.cn)?)$ #重庆
-regexp:\.(.+-)?ap-shenzhen(-.+)?\.tencentcos\.(cn|com(\.cn)?)$  #深圳
+# overrided by tencentcos.cn, tencentcos.com, tencentcos.com.cn
+#regexp:\.(.+-)?ap-beijing(-.+)?\.tencentcos\.(cn|com(\.cn)?)$   #北京
+#regexp:\.(.+-)?ap-nanjing(-.+)?\.tencentcos\.(cn|com(\.cn)?)$   #南京
+#regexp:\.(.+-)?ap-shanghai(-.+)?\.tencentcos\.(cn|com(\.cn)?)$  #上海
+#regexp:\.(.+-)?ap-guangzhou(-.+)?\.tencentcos\.(cn|com(\.cn)?)$ #广州
+#regexp:\.(.+-)?ap-chengdu(-.+)?\.tencentcos\.(cn|com(\.cn)?)$   #成都
+#regexp:\.(.+-)?ap-chongqing(-.+)?\.tencentcos\.(cn|com(\.cn)?)$ #重庆
+#regexp:\.(.+-)?ap-shenzhen(-.+)?\.tencentcos\.(cn|com(\.cn)?)$  #深圳
 
 # tencentcos outside mainland China
 # regexp:.+\.ap-hongkong\.tencentcos\.(cn|com(\.cn)?)$ @!cn       #中国香港

data/tilda

@@ -0,0 +1,5 @@
+tilda.cc
+tilda.ru
+tilda.ws
+tildaapi.com
+tildacdn.com

data/weiphone

@@ -1,3 +1,4 @@
 feng.com
 fengimg.com
+weiphone.net
 wfdata.club

main.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"bufio"
+	"encoding/json"
 	"flag"
 	"fmt"
 	"os"
@@ -19,6 +20,7 @@ var (
 	dataPath    = flag.String("datapath", "./data", "Path to your custom 'data' directory")
 	outputName  = flag.String("outputname", "dlc.dat", "Name of the generated dat file")
 	outputDir   = flag.String("outputdir", "./", "Directory to place all generated files")
+	datProfile  = flag.String("datprofile", "", "Path of config file used to assemble custom dats")
 	exportLists = flag.String("exportlists", "", "Lists to be flattened and exported in plaintext format, separated by ',' comma")
 )
 
@@ -47,6 +49,23 @@ type Processor struct {
 	cirIncMap map[string]bool
 }
 
+type GeoSites struct {
+	Sites   []*router.GeoSite
+	SiteIdx map[string]int
+}
+
+type DatTask struct {
+	Name  string   `json:"name"`
+	Mode  string   `json:"mode"`
+	Lists []string `json:"lists"`
+}
+
+const (
+	ModeAll       string = "all"
+	ModeAllowlist string = "allowlist"
+	ModeDenylist  string = "denylist"
+)
+
 func makeProtoList(listName string, entries []*Entry) *router.GeoSite {
 	site := &router.GeoSite{
 		CountryCode: listName,
@@ -76,6 +95,90 @@ func makeProtoList(listName string, entries []*Entry) *router.GeoSite {
 	return site
 }
 
+func loadTasks(path string) ([]DatTask, error) {
+	f, err := os.Open(path)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+
+	var tasks []DatTask
+	dec := json.NewDecoder(f)
+	if err := dec.Decode(&tasks); err != nil {
+		return nil, fmt.Errorf("failed to decode json: %w", err)
+	}
+	for i, t := range tasks {
+		if t.Name == "" {
+			return nil, fmt.Errorf("task[%d]: name is required", i)
+		}
+		switch t.Mode {
+		case ModeAll, ModeAllowlist, ModeDenylist:
+		default:
+			return nil, fmt.Errorf("task[%d] %q: invalid mode %q", i, t.Name, t.Mode)
+		}
+	}
+	return tasks, nil
+}
+
+func (gs *GeoSites) assembleDat(task DatTask) error {
+	datFileName := strings.ToLower(filepath.Base(task.Name))
+	geoSiteList := new(router.GeoSiteList)
+
+	switch task.Mode {
+	case ModeAll:
+		geoSiteList.Entry = gs.Sites
+	case ModeAllowlist:
+		allowedIdxes := make([]int, 0, len(task.Lists))
+		for _, list := range task.Lists {
+			if idx, ok := gs.SiteIdx[strings.ToUpper(list)]; ok {
+				allowedIdxes = append(allowedIdxes, idx)
+			} else {
+				return fmt.Errorf("list %q not found for allowlist task", list)
+			}
+		}
+		slices.Sort(allowedIdxes)
+		allowedlen := len(allowedIdxes)
+		if allowedlen == 0 {
+			return fmt.Errorf("allowlist needs at least one valid list")
+		}
+		geoSiteList.Entry = make([]*router.GeoSite, allowedlen)
+		for i, idx := range allowedIdxes {
+			geoSiteList.Entry[i] = gs.Sites[idx]
+		}
+	case ModeDenylist:
+		deniedMap := make(map[int]bool, len(task.Lists))
+		for _, list := range task.Lists {
+			if idx, ok := gs.SiteIdx[strings.ToUpper(list)]; ok {
+				deniedMap[idx] = true
+			} else {
+				fmt.Printf("[Warn] list %q not found in denylist task %q", list, task.Name)
+			}
+		}
+		deniedlen := len(deniedMap)
+		if deniedlen == 0 {
+			fmt.Printf("[Warn] nothing to deny in task %q", task.Name)
+			geoSiteList.Entry = gs.Sites
+		} else {
+			geoSiteList.Entry = make([]*router.GeoSite, 0, len(gs.Sites)-deniedlen)
+			for i, site := range gs.Sites {
+				if !deniedMap[i] {
+					geoSiteList.Entry = append(geoSiteList.Entry, site)
+				}
+			}
+		}
+	}
+
+	protoBytes, err := proto.Marshal(geoSiteList)
+	if err != nil {
+		return fmt.Errorf("failed to marshal: %w", err)
+	}
+	if err := os.WriteFile(filepath.Join(*outputDir, datFileName), protoBytes, 0644); err != nil {
+		return fmt.Errorf("failed to write file %q: %w", datFileName, err)
+	}
+	fmt.Printf("dat %q has been generated successfully\n", datFileName)
+	return nil
+}
+
 func writePlainList(listname string, entries []*Entry) error {
 	file, err := os.Create(filepath.Join(*outputDir, strings.ToLower(listname)+".txt"))
 	if err != nil {
@@ -443,25 +546,39 @@ func run() error {
 		}
 	}
 
-	// Generate dat file
-	protoList := &router.GeoSiteList{Entry: make([]*router.GeoSite, 0, sitesCount)}
+	// Generate proto sites
+	gs := &GeoSites{
+		Sites:   make([]*router.GeoSite, 0, sitesCount),
+		SiteIdx: make(map[string]int, sitesCount),
+	}
 	for siteName, siteEntries := range processor.finalMap {
-		protoList.Entry = append(protoList.Entry, makeProtoList(siteName, siteEntries))
+		gs.Sites = append(gs.Sites, makeProtoList(siteName, siteEntries))
 	}
 	processor = nil
-	// Sort protoList so the marshaled list is reproducible
-	slices.SortFunc(protoList.Entry, func(a, b *router.GeoSite) int {
+	// Sort proto sites so the generated file is reproducible
+	slices.SortFunc(gs.Sites, func(a, b *router.GeoSite) int {
 		return strings.Compare(a.CountryCode, b.CountryCode)
 	})
+	for i := range sitesCount {
+		gs.SiteIdx[gs.Sites[i].CountryCode] = i
+	}
 
-	protoBytes, err := proto.Marshal(protoList)
-	if err != nil {
-		return fmt.Errorf("failed to marshal: %w", err)
+	// Load tasks and generate dat files
+	var tasks []DatTask
+	if *datProfile == "" {
+		tasks = []DatTask{{Name: *outputName, Mode: ModeAll}}
+	} else {
+		var err error
+		tasks, err = loadTasks(*datProfile)
+		if err != nil {
+			return fmt.Errorf("failed to loadTasks %q: %v", *datProfile, err)
+		}
 	}
-	if err := os.WriteFile(filepath.Join(*outputDir, *outputName), protoBytes, 0644); err != nil {
-		return fmt.Errorf("failed to write output: %w", err)
+	for _, task := range tasks {
+		if err := gs.assembleDat(task); err != nil {
+			fmt.Printf("[Error] failed to assembleDat %q: %v", task.Name, err)
+		}
 	}
-	fmt.Printf("%q has been generated successfully\n", *outputName)
 	return nil
 }