main.go: improve codes (#3366)

* main.go: improve codes

* main.go: add parseInclusion

- seprate from parseEntry
- not allow affiliation for inclusion

data/baidu

@@ -64,6 +64,7 @@ shifen.com
 smartapps.cn
 tieba.com
 tiebaimg.com
+xdrtc.com
 xianfae.com
 xiaodutv.com
 yoojia.com

data/category-ai-!cn

@@ -46,3 +46,4 @@ mistral.ai
 openart.ai
 openclaw.ai
 openrouter.ai
+spicywriter.com

data/category-education-cn

@@ -10,6 +10,8 @@ include:shanbay
 include:xueersi
 include:yuanfudao
 
+edu.cn
+
 # 雨课堂
 include:yuketang
 ## 雨豆课堂
@@ -19,8 +21,6 @@ yushiyan.net
 ## 学堂在线
 xuetangx.com
 
-edu.cn
-
 # 国家智慧教育公共服务平台
 cbern.com.cn
 smartedu.cn
@@ -132,6 +132,8 @@ oldboyedu.com
 pigai.org
 # 公考知识库
 saduck.top
+# 外研在线
+unipus.cn
 # 未来云校
 weilaiyunxiao.com
 # 北京嘉瑞新创教育咨询有限公司

data/category-entertainment-cn

@@ -57,10 +57,14 @@ anitabi.cn
 # 暴风影音
 baofeng.com
 baofeng.net
+# 街机地图
+bemanicn.com
 # 布咕阅读
 bgwxc.com
 # B站空降助手
 bsbsb.top
+# CdkeyNoGap
+cdkeynogap.com
 # 动漫之家 #703
 dmzj.com
 muwai.com

data/category-ip-geo-detect

@@ -131,6 +131,8 @@ wtfismyip.com
 
 # Subdomains/internal api used for ip-geo-detect
 full:checkip.amazonaws.com
+full:ipv4-check-perf.radar.cloudflare.com
+full:ipv6-check-perf.radar.cloudflare.com
 geoip.noc.gov.ru
 ip.mail.ru
 ip.nic.ru

data/category-speedtest

@@ -3,33 +3,62 @@ include:openspeedtest
 
 cnspeedtest.cn @cn
 fast.com
+fastspeedtest.com
+linkmeter.net
 measurementlab.net
+meter.net
 nperf.com
+openspeedtest.ru
 speed.cloudflare.com
 speed.dler.io
+speed.ee
 speed.hinet.net
 speed.nccu.edu.tw
 speed.neu6.edu.cn @cn
+speed.nju.edu.cn @cn
 speed.nuaa.edu.cn @cn
 speed.qlu.edu.cn @cn
 speed.ujs.edu.cn @cn
-speed6.ujs.edu.cn @cn
 speed2.hinet.net
 speed5.ntu.edu.tw
+speed6.ujs.edu.cn @cn
+speedcheck.org
+speedgeo.net
+speedof.me
+speedtest.cesnet.cz
+speedtest.ch
+speedtest.citylink.pro
 speedtest.cn @cn
+speedtest.co.za
+speedtest.de
+speedtest.dno-it.ru
+speedtest.frontier.com
 speedtest.im
+speedtest.mail.ru
 speedtest.mfcyun.com @cn
+speedtest.net.in
+speedtest.net.ua
+speedtest.net.uk
+speedtest.org
+speedtest.rt.ru
+speedtest.ru
+speedtest.shaw.ca
 speedtest.shu.edu.cn @cn
-speedtest6.shu.edu.cn @cn
+speedtest.su
+speedtest.uz
+speedtest.volia.com
 speedtest.xaut.edu.cn @cn
 speedtest.xfinity.com
-speedtestcustom.com
-test.ustc.edu.cn @cn
-test6.ustc.edu.cn @cn
+speedtest.xyz
+speedtest24.ru
+speedtest6.shu.edu.cn @cn
 test.nju.edu.cn @cn
+test.ustc.edu.cn @cn
 test6.nju.edu.cn @cn
-speed.nju.edu.cn @cn
+test6.ustc.edu.cn @cn
 testmy.net
+testmyspeed.com
+testskorosti.ru
 xnfz.seu.edu.cn @cn
 
 full:hk-global-bgp.hkg.speedtest.yecaoyun.com @!cn

data/geolocation-cn

@@ -325,6 +325,10 @@ guoxuemi.com
 zyh365.com
 ## 温州市图书馆
 wzlib.cn
+## 中国大百科全书
+zgbk.com
+## 浙江图书馆
+zjlib.cn
 
 # Services & Softwares
 include:category-ai-cn

data/kinopub

@@ -3,6 +3,7 @@ kinopub.online
 kpdl.link
 
 # Mirror sites
+ahc.ovh # sub domains mirror
 gfw.ovh # sub domains mirror
 mos-gorsud.co # kinopub domain to generate a mirror site through gfw.ovh
 

data/nodeseek

@@ -1,5 +1,6 @@
 22112211.xyz
 deepflood.com
+nodeget.com
 nodeimage.com
 nodequality.com
 nodeseek.com

data/nvidia

@@ -59,6 +59,7 @@ tegrazone.com
 tegrazone.jp
 tegrazone.kr
 
+full:nvidia.custhelp.com
 full:nvidia.tt.omtrdc.net
 
 # NVIDIA 文件下载服务器中国镜像

data/oracle

@@ -1,10 +1,14 @@
 # All .oracle domains
 oracle
 
+include:addthis
+include:java
+
 ateam-oracle.com
 bronto.com
 covid19-rx.org
 covid19rx.org
+custhelp.com
 oracle.com
 oraclecloud.com
 oraclefoundation.org
@@ -12,6 +16,3 @@ oracleimg.com
 oracleinfinity.io
 sun.com
 virtualbox.org
-
-include:addthis
-include:java

data/wordpress

@@ -1,6 +1,7 @@
 videopress.com
 w.org
 wordpress.com
+wordpress.net
 wordpress.org
 wordpress.tv
 wp-themes.com

main.go

@@ -47,7 +47,7 @@ type Processor struct {
 	cirIncMap map[string]bool
 }
 
-func makeProtoList(listName string, entries []*Entry) (*router.GeoSite, error) {
+func makeProtoList(listName string, entries []*Entry) *router.GeoSite {
 	site := &router.GeoSite{
 		CountryCode: listName,
 		Domain:      make([]*router.Domain, 0, len(entries)),
@@ -73,7 +73,7 @@ func makeProtoList(listName string, entries []*Entry) (*router.GeoSite, error) {
 		}
 		site.Domain = append(site.Domain, pdomain)
 	}
-	return site, nil
+	return site
 }
 
 func writePlainList(listname string, entries []*Entry) error {
@@ -89,46 +89,28 @@ func writePlainList(listname string, entries []*Entry) error {
 	return w.Flush()
 }
 
-func parseEntry(line string) (*Entry, []string, error) {
-	entry := new(Entry)
-	parts := strings.Fields(line)
+func parseEntry(typ, rule string) (*Entry, []string, error) {
+	entry := &Entry{Type: typ}
+	parts := strings.Fields(rule)
 	if len(parts) == 0 {
-		return entry, nil, fmt.Errorf("empty line")
+		return entry, nil, fmt.Errorf("empty domain rule")
 	}
-
-	// Parse type and value
-	typ, val, isTypeSpecified := strings.Cut(parts[0], ":")
-	typ = strings.ToLower(typ)
-	if !isTypeSpecified { // Default RuleType
-		if !validateDomainChars(typ) {
-			return entry, nil, fmt.Errorf("invalid domain: %q", typ)
+	// Parse value
+	switch entry.Type {
+	case dlc.RuleTypeRegexp:
+		if _, err := regexp.Compile(parts[0]); err != nil {
+			return entry, nil, fmt.Errorf("invalid regexp %q: %w", parts[0], err)
 		}
-		entry.Type = dlc.RuleTypeDomain
-		entry.Value = typ
-	} else {
-		switch typ {
-		case dlc.RuleTypeRegexp:
-			if _, err := regexp.Compile(val); err != nil {
-				return entry, nil, fmt.Errorf("invalid regexp %q: %w", val, err)
-			}
-			entry.Type = dlc.RuleTypeRegexp
-			entry.Value = val
-		case dlc.RuleTypeInclude:
-			entry.Type = dlc.RuleTypeInclude
-			entry.Value = strings.ToUpper(val)
-			if !validateSiteName(entry.Value) {
-				return entry, nil, fmt.Errorf("invalid included list name: %q", entry.Value)
-			}
-		case dlc.RuleTypeDomain, dlc.RuleTypeFullDomain, dlc.RuleTypeKeyword:
-			entry.Type = typ
-			entry.Value = strings.ToLower(val)
-			if !validateDomainChars(entry.Value) {
-				return entry, nil, fmt.Errorf("invalid domain: %q", entry.Value)
-			}
-		default:
-			return entry, nil, fmt.Errorf("invalid type: %q", typ)
+		entry.Value = parts[0]
+	case dlc.RuleTypeDomain, dlc.RuleTypeFullDomain, dlc.RuleTypeKeyword:
+		entry.Value = strings.ToLower(parts[0])
+		if !validateDomainChars(entry.Value) {
+			return entry, nil, fmt.Errorf("invalid domain: %q", entry.Value)
 		}
+	default:
+		return entry, nil, fmt.Errorf("unknown rule type: %q", entry.Type)
 	}
+	plen := len(entry.Type) + len(entry.Value) + 1
 
 	// Parse attributes and affiliations
 	var affs []string
@@ -140,6 +122,7 @@ func parseEntry(line string) (*Entry, []string, error) {
 				return entry, affs, fmt.Errorf("invalid attribute: %q", attr)
 			}
 			entry.Attrs = append(entry.Attrs, attr)
+			plen += 2 + len(attr)
 		case '&':
 			aff := strings.ToUpper(part[1:])
 			if !validateSiteName(aff) {
@@ -147,33 +130,70 @@ func parseEntry(line string) (*Entry, []string, error) {
 			}
 			affs = append(affs, aff)
 		default:
-			return entry, affs, fmt.Errorf("invalid attribute/affiliation: %q", part)
+			return entry, affs, fmt.Errorf("unknown field: %q", part)
 		}
 	}
 
-	if entry.Type != dlc.RuleTypeInclude {
-		slices.Sort(entry.Attrs) // Sort attributes
-		// Formated plain entry: type:domain.tld:@attr1,@attr2
-		var plain strings.Builder
-		plain.Grow(len(entry.Type) + len(entry.Value) + 10)
-		plain.WriteString(entry.Type)
-		plain.WriteByte(':')
-		plain.WriteString(entry.Value)
-		for i, attr := range entry.Attrs {
-			if i == 0 {
-				plain.WriteByte(':')
-			} else {
-				plain.WriteByte(',')
-			}
-			plain.WriteByte('@')
-			plain.WriteString(attr)
+	slices.Sort(entry.Attrs) // Sort attributes
+	// Formated plain entry: type:domain.tld:@attr1,@attr2
+	var plain strings.Builder
+	plain.Grow(plen)
+	plain.WriteString(entry.Type)
+	plain.WriteByte(':')
+	plain.WriteString(entry.Value)
+	for i, attr := range entry.Attrs {
+		if i == 0 {
+			plain.WriteByte(':')
+		} else {
+			plain.WriteByte(',')
 		}
-		entry.Plain = plain.String()
+		plain.WriteByte('@')
+		plain.WriteString(attr)
 	}
+	entry.Plain = plain.String()
 	return entry, affs, nil
 }
 
+func parseInclusion(rule string) (*Inclusion, error) {
+	parts := strings.Fields(rule)
+	if len(parts) == 0 {
+		return nil, fmt.Errorf("empty inclusion")
+	}
+	inc := &Inclusion{Source: strings.ToUpper(parts[0])}
+	if !validateSiteName(inc.Source) {
+		return inc, fmt.Errorf("invalid included list name: %q", inc.Source)
+	}
+
+	// Parse attributes
+	for _, part := range parts[1:] {
+		switch part[0] {
+		case '@':
+			attr := strings.ToLower(part[1:])
+			if attr[0] == '-' {
+				battr := attr[1:]
+				if !validateAttrChars(battr) {
+					return inc, fmt.Errorf("invalid ban attribute: %q", battr)
+				}
+				inc.BanAttrs = append(inc.BanAttrs, battr)
+			} else {
+				if !validateAttrChars(attr) {
+					return inc, fmt.Errorf("invalid must attribute: %q", attr)
+				}
+				inc.MustAttrs = append(inc.MustAttrs, attr)
+			}
+		case '&':
+			return inc, fmt.Errorf("affiliation is not allowed for inclusion")
+		default:
+			return inc, fmt.Errorf("unknown field: %q", part)
+		}
+	}
+	return inc, nil
+}
+
 func validateDomainChars(domain string) bool {
+	if domain == "" {
+		return false
+	}
 	for i := range domain {
 		c := domain[i]
 		if (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') || c == '.' || c == '-' {
@@ -185,9 +205,12 @@ func validateDomainChars(domain string) bool {
 }
 
 func validateAttrChars(attr string) bool {
+	if attr == "" {
+		return false
+	}
 	for i := range attr {
 		c := attr[i]
-		if (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') || c == '!' || c == '-' {
+		if (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') || c == '!' {
 			continue
 		}
 		return false
@@ -196,6 +219,9 @@ func validateAttrChars(attr string) bool {
 }
 
 func validateSiteName(name string) bool {
+	if name == "" {
+		return false
+	}
 	for i := range name {
 		c := name[i]
 		if (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '!' || c == '-' {
@@ -232,26 +258,23 @@ func (p *Processor) loadData(listName string, path string) error {
 		if line == "" {
 			continue
 		}
-		entry, affs, err := parseEntry(line)
-		if err != nil {
-			return fmt.Errorf("error in %q at line %d: %w", path, lineIdx, err)
+		typ, rule, isTypeSpecified := strings.Cut(line, ":")
+		if !isTypeSpecified { // Default RuleType
+			typ, rule = dlc.RuleTypeDomain, typ
+		} else {
+			typ = strings.ToLower(typ)
 		}
-
-		if entry.Type == dlc.RuleTypeInclude {
-			inc := &Inclusion{Source: entry.Value}
-			for _, attr := range entry.Attrs {
-				if attr[0] == '-' {
-					inc.BanAttrs = append(inc.BanAttrs, attr[1:])
-				} else {
-					inc.MustAttrs = append(inc.MustAttrs, attr)
-				}
-			}
-			for _, aff := range affs {
-				apl := p.getOrCreateParsedList(aff)
-				apl.Inclusions = append(apl.Inclusions, inc)
+		if typ == dlc.RuleTypeInclude {
+			inc, err := parseInclusion(rule)
+			if err != nil {
+				return fmt.Errorf("error in %q at line %d: %w", path, lineIdx, err)
 			}
 			pl.Inclusions = append(pl.Inclusions, inc)
 		} else {
+			entry, affs, err := parseEntry(typ, rule)
+			if err != nil {
+				return fmt.Errorf("error in %q at line %d: %w", path, lineIdx, err)
+			}
 			for _, aff := range affs {
 				apl := p.getOrCreateParsedList(aff)
 				apl.Entries = append(apl.Entries, entry)
@@ -259,7 +282,7 @@ func (p *Processor) loadData(listName string, path string) error {
 			pl.Entries = append(pl.Entries, entry)
 		}
 	}
-	return nil
+	return scanner.Err()
 }
 
 func isMatchAttrFilters(entry *Entry, incFilter *Inclusion) bool {
@@ -360,6 +383,9 @@ func (p *Processor) resolveList(plname string) error {
 			}
 		}
 	}
+	if len(roughMap) == 0 {
+		return fmt.Errorf("empty list")
+	}
 	p.finalMap[plname] = polishList(roughMap)
 	return nil
 }
@@ -387,13 +413,15 @@ func run() error {
 		return fmt.Errorf("failed to loadData: %w", err)
 	}
 	// Generate finalMap
-	processor.finalMap = make(map[string][]*Entry, len(processor.plMap))
+	sitesCount := len(processor.plMap)
+	processor.finalMap = make(map[string][]*Entry, sitesCount)
 	processor.cirIncMap = make(map[string]bool)
 	for plname := range processor.plMap {
 		if err := processor.resolveList(plname); err != nil {
 			return fmt.Errorf("failed to resolveList %q: %w", plname, err)
 		}
 	}
+	processor.plMap = nil
 
 	// Make sure output directory exists
 	if err := os.MkdirAll(*outputDir, 0755); err != nil {
@@ -403,27 +431,24 @@ func run() error {
 	for rawEpList := range strings.SplitSeq(*exportLists, ",") {
 		if epList := strings.TrimSpace(rawEpList); epList != "" {
 			entries, exist := processor.finalMap[strings.ToUpper(epList)]
-			if !exist || len(entries) == 0 {
-				fmt.Printf("list %q does not exist or is empty\n", epList)
+			if !exist {
+				fmt.Printf("[Warn] list %q does not exist\n", epList)
 				continue
 			}
 			if err := writePlainList(epList, entries); err != nil {
-				fmt.Printf("failed to write list %q: %v\n", epList, err)
+				fmt.Printf("[Error] failed to write list %q: %v\n", epList, err)
 				continue
 			}
-			fmt.Printf("list %q has been generated successfully.\n", epList)
+			fmt.Printf("list %q has been generated successfully\n", epList)
 		}
 	}
 
 	// Generate dat file
-	protoList := new(router.GeoSiteList)
+	protoList := &router.GeoSiteList{Entry: make([]*router.GeoSite, 0, sitesCount)}
 	for siteName, siteEntries := range processor.finalMap {
-		site, err := makeProtoList(siteName, siteEntries)
-		if err != nil {
-			return fmt.Errorf("failed to makeProtoList %q: %w", siteName, err)
-		}
-		protoList.Entry = append(protoList.Entry, site)
+		protoList.Entry = append(protoList.Entry, makeProtoList(siteName, siteEntries))
 	}
+	processor = nil
 	// Sort protoList so the marshaled list is reproducible
 	slices.SortFunc(protoList.Entry, func(a, b *router.GeoSite) int {
 		return strings.Compare(a.CountryCode, b.CountryCode)
@@ -436,14 +461,14 @@ func run() error {
 	if err := os.WriteFile(filepath.Join(*outputDir, *outputName), protoBytes, 0644); err != nil {
 		return fmt.Errorf("failed to write output: %w", err)
 	}
-	fmt.Printf("%q has been generated successfully.\n", *outputName)
+	fmt.Printf("%q has been generated successfully\n", *outputName)
 	return nil
 }
 
 func main() {
 	flag.Parse()
 	if err := run(); err != nil {
-		fmt.Printf("Fatal error: %v\n", err)
+		fmt.Printf("[Fatal] critical error: %v\n", err)
 		os.Exit(1)
 	}
 }