Chore: refine code (#3067)

Co-authored-by: Loyalsoldier <10487845+Loyalsoldier@users.noreply.github.com>

data/category-ads

@@ -38,6 +38,7 @@ include:newrelic-ads
 include:ogury-ads
 include:ookla-speedtest-ads
 include:openx-ads
+include:picacg-ads
 include:pocoiq-ads
 include:pubmatic-ads
 include:qihoo360-ads

data/category-ru

@@ -23,5 +23,6 @@ include:mailru-group
 include:okko
 include:ozon
 include:rutube
+include:wink
 include:x5
 include:yandex

data/ctyun

@@ -0,0 +1,69 @@
+# 天翼云
+# 京ICP备2021034386号
+ctadns.cn
+bjctyiptv.cn
+cqctyiptv.cn
+ctacdn.cn
+ctaigw.cn
+ctbcdn.com
+ctcdn.cn
+ctcdn.com.cn
+ctcdnov.net
+ctcloudzos.cn
+ctcns.cn
+ctdcdn.com
+ctdns.cn
+ctdns.com.cn
+ctdns.net
+ctecdn.cn
+ctecx.cn
+ctgcdn.com
+cthcdn.cn
+cthcdn.com
+cthcdn.net
+ctlcdn.cn
+ctlcdn.com
+ctlcdn.net
+ctmcdn.cn
+ctovcdn.com
+ctrender.com
+ctwcdn.cn
+ctxcdn.cn
+ctxcdn.com
+ctxcdn.net
+ctxirang.cn
+ctxirang.com
+ctycdn.cn
+ctycdn.net
+ctycdn.net.cn
+ctydoh.cn
+ctyecx.cn
+ctyiptv.cn
+ctyun.cn
+ctyun.com.cn
+ctyuncdn.cn
+ctyuncs.cn
+ctyuninner.com
+ctyunmds.cn
+ctyunwaf.cn
+ctyunwaf.com
+ctyunwaf1.com
+ctyunwaf3.cn
+ctyunxs.cn
+ctyunzos.cn
+ctzcdn.cn
+ctzcdn.com
+edgecloudx.cn
+faasapp.cn
+faasdev.cn
+fjctyiptv.cn
+gdctyiptv.cn
+gsctyiptv.cn
+gsjtyiptv.cn
+gzctyiptv.cn
+jsctyiptv.cn
+modelers.cn
+scctyiptv.cn
+snctyiptv.cn
+ynctyiptv.cn
+ynjtyiptv.cn

data/geolocation-cn

@@ -54,6 +54,7 @@ include:aws-cn
 include:baishancloud
 include:bootcdn
 include:cloudflare-cn
+include:ctyun
 include:dwion
 include:maocloud
 include:qingcloud

data/google-ads

@@ -49,6 +49,7 @@ partnerad.l.google.com @ads
 urchin.com @ads
 
 full:analytics.google.com @ads
-full:fundingchoicesmessages.google.com @ads
+# https://github.com/AdguardTeam/FiltersRegistry/pull/1154
+# full:fundingchoicesmessages.google.com @ads
 
 regexp:^adservice\.google\.([a-z]{2}|com?)(\.[a-z]{2})?$ @ads

data/ookla-speedtest

@@ -13,3 +13,6 @@ webtest.net
 full:www.speedtest.net.cdn.cloudflare.net
 
 include:ookla-speedtest-ads
+
+# SpeedTest Node
+ookla-speedtest-central.hgconair.hgc.com.hk # HGC Global Communications 香港環電

data/picacg

@@ -1,3 +1,5 @@
+include:picacg-ads
+
 bikaa.xyz
 bikac.xyz
 bikaios.xyz
@@ -10,6 +12,5 @@ picacomic.xyz
 wikawika.xyz
 
 # Image Resource Domain like `img.diwodiwo.xyz` `s3.diwodiwo.xyz` `storage.diwodiwo.xyz` `storage-b.diwodiwo.xyz`
-regexp:^([a-z0-9-]+\.)*(?!ad-display\.|ad-channel\.)[a-z0-9-]+\.diwodiwo\.xyz$
-ad-channel.diwodiwo.xyz @ads
-ad-display.diwodiwo.xyz @ads
+diwodiwo.xyz
+

data/picacg-ads

@@ -0,0 +1,2 @@
+full:ad-channel.diwodiwo.xyz @ads
+full:ad-display.diwodiwo.xyz @ads

data/umeng-ads

@@ -1,6 +1,24 @@
 aaid.umeng.com @ads
 alog.umeng.com @ads
 alog.umengcloud.com @ads
+alogs.umeng.com @ads
+alogus.umeng.com @ads
+ar.umeng.com @ads
+aspect-upush.umeng.com @ads
+audid.umeng.com @ads
+ccs.umeng.com @ads
+cnlogs.umeng.com @ads
+cnlogs.umengcloud.com @ads
+# https://github.com/TG-Twilight/AWAvenue-Ads-Rule/issues/185
+# errlog.umeng.com @ads
+# errnewlog.umeng.com @ads
+new-aaid.umeng.com @ads
+new-aaid.umeng.com.gds.alibabadns.com @ads
+oc.umeng.com @ads
+plbslog.umeng.com @ads
+resolve.umeng.com @ads
+ulogs.umeng.com @ads
+ulogs.umengcloud.com @ads
 utoken.umeng.com @ads
 
 # CNZZ

data/wink

@@ -0,0 +1,3 @@
+ngenix.net
+restream-media.net
+wink.ru

main.go

@@ -2,11 +2,11 @@ package main
 
 import (
 	"bufio"
-	"errors"
 	"flag"
 	"fmt"
 	"os"
 	"path/filepath"
+	"regexp"
 	"sort"
 	"strconv"
 	"strings"
@@ -22,6 +22,14 @@ var (
 	exportLists = flag.String("exportlists", "", "Lists to be flattened and exported in plaintext format, separated by ',' comma")
 )
 
+const (
+	RuleTypeDomain     string = "domain"
+	RuleTypeFullDomain string = "full"
+	RuleTypeKeyword    string = "keyword"
+	RuleTypeRegexp     string = "regexp"
+	RuleTypeInclude    string = "include"
+)
+
 type Entry struct {
 	Type  string
 	Value string
@@ -64,32 +72,41 @@ func (l *ParsedList) toProto() (*router.GeoSite, error) {
 	}
 	for _, entry := range l.Entry {
 		switch entry.Type {
-		case "domain":
+		case RuleTypeDomain:
 			site.Domain = append(site.Domain, &router.Domain{
 				Type:      router.Domain_RootDomain,
 				Value:     entry.Value,
 				Attribute: entry.Attrs,
 			})
-		case "regexp":
+
+		case RuleTypeRegexp:
+			// check regexp validity to avoid runtime error
+			_, err := regexp.Compile(entry.Value)
+			if err != nil {
+				return nil, fmt.Errorf("invalid regexp in list %s: %s", l.Name, entry.Value)
+			}
 			site.Domain = append(site.Domain, &router.Domain{
 				Type:      router.Domain_Regex,
 				Value:     entry.Value,
 				Attribute: entry.Attrs,
 			})
-		case "keyword":
+
+		case RuleTypeKeyword:
 			site.Domain = append(site.Domain, &router.Domain{
 				Type:      router.Domain_Plain,
 				Value:     entry.Value,
 				Attribute: entry.Attrs,
 			})
-		case "full":
+
+		case RuleTypeFullDomain:
 			site.Domain = append(site.Domain, &router.Domain{
 				Type:      router.Domain_Full,
 				Value:     entry.Value,
 				Attribute: entry.Attrs,
 			})
+
 		default:
-			return nil, errors.New("unknown domain type: " + entry.Type)
+			return nil, fmt.Errorf("unknown domain type: %s", entry.Type)
 		}
 	}
 	return site, nil
@@ -99,7 +116,7 @@ func exportPlainTextList(list []string, refName string, pl *ParsedList) {
 	for _, listName := range list {
 		if strings.EqualFold(refName, listName) {
 			if err := pl.toPlainText(strings.ToLower(refName)); err != nil {
-				fmt.Println("Failed: ", err)
+				fmt.Println("Failed:", err)
 				continue
 			}
 			fmt.Printf("'%s' has been generated successfully.\n", listName)
@@ -118,24 +135,30 @@ func removeComment(line string) string {
 func parseDomain(domain string, entry *Entry) error {
 	kv := strings.Split(domain, ":")
 	if len(kv) == 1 {
-		entry.Type = "domain"
+		entry.Type = RuleTypeDomain
 		entry.Value = strings.ToLower(kv[0])
 		return nil
 	}
 
 	if len(kv) == 2 {
 		entry.Type = strings.ToLower(kv[0])
-		entry.Value = strings.ToLower(kv[1])
+
+		if strings.EqualFold(entry.Type, RuleTypeRegexp) {
+			entry.Value = kv[1]
+		} else {
+			entry.Value = strings.ToLower(kv[1])
+		}
+
 		return nil
 	}
 
-	return errors.New("Invalid format: " + domain)
+	return fmt.Errorf("invalid format: %s", domain)
 }
 
 func parseAttribute(attr string) (*router.Domain_Attribute, error) {
 	var attribute router.Domain_Attribute
 	if len(attr) == 0 || attr[0] != '@' {
-		return &attribute, errors.New("invalid attribute: " + attr)
+		return &attribute, fmt.Errorf("invalid attribute: %s", attr)
 	}
 
 	// Trim attribute prefix `@` character
@@ -148,7 +171,7 @@ func parseAttribute(attr string) (*router.Domain_Attribute, error) {
 		attribute.Key = strings.ToLower(parts[0])
 		intv, err := strconv.Atoi(parts[1])
 		if err != nil {
-			return &attribute, errors.New("invalid attribute: " + attr + ": " + err.Error())
+			return &attribute, fmt.Errorf("invalid attribute: %s: %v", attr, err)
 		}
 		attribute.TypedValue = &router.Domain_Attribute_IntValue{IntValue: int64(intv)}
 	}
@@ -161,7 +184,7 @@ func parseEntry(line string) (Entry, error) {
 
 	var entry Entry
 	if len(parts) == 0 {
-		return entry, errors.New("empty entry")
+		return entry, fmt.Errorf("empty entry")
 	}
 
 	if err := parseDomain(parts[0], &entry); err != nil {
@@ -255,7 +278,7 @@ func ParseList(list *List, ref map[string]*List) (*ParsedList, error) {
 		newEntryList := make([]Entry, 0, len(entryList))
 		hasInclude := false
 		for _, entry := range entryList {
-			if entry.Type == "include" {
+			if entry.Type == RuleTypeInclude {
 				refName := strings.ToUpper(entry.Value)
 				if entry.Attrs != nil {
 					for _, attr := range entry.Attrs {
@@ -267,7 +290,7 @@ func ParseList(list *List, ref map[string]*List) (*ParsedList, error) {
 
 						refList := ref[refName]
 						if refList == nil {
-							return nil, errors.New(entry.Value + " not found.")
+							return nil, fmt.Errorf("list not found: %s", entry.Value)
 						}
 						attrEntrys := createIncludeAttrEntrys(refList, attr)
 						if len(attrEntrys) != 0 {
@@ -282,7 +305,7 @@ func ParseList(list *List, ref map[string]*List) (*ParsedList, error) {
 					pl.Inclusion[InclusionName] = true
 					refList := ref[refName]
 					if refList == nil {
-						return nil, errors.New(entry.Value + " not found.")
+						return nil, fmt.Errorf("list not found: %s", entry.Value)
 					}
 					newEntryList = append(newEntryList, refList.Entry...)
 				}
@@ -323,14 +346,14 @@ func main() {
 		return nil
 	})
 	if err != nil {
-		fmt.Println("Failed: ", err)
+		fmt.Println("Failed:", err)
 		os.Exit(1)
 	}
 
 	// Create output directory if not exist
 	if _, err := os.Stat(*outputDir); os.IsNotExist(err) {
 		if mkErr := os.MkdirAll(*outputDir, 0755); mkErr != nil {
-			fmt.Println("Failed: ", mkErr)
+			fmt.Println("Failed:", mkErr)
 			os.Exit(1)
 		}
 	}
@@ -340,12 +363,12 @@ func main() {
 	for refName, list := range ref {
 		pl, err := ParseList(list, ref)
 		if err != nil {
-			fmt.Println("Failed: ", err)
+			fmt.Println("Failed:", err)
 			os.Exit(1)
 		}
 		site, err := pl.toProto()
 		if err != nil {
-			fmt.Println("Failed: ", err)
+			fmt.Println("Failed:", err)
 			os.Exit(1)
 		}
 		protoList.Entry = append(protoList.Entry, site)
@@ -383,7 +406,7 @@ func main() {
 		os.Exit(1)
 	}
 	if err := os.WriteFile(filepath.Join(*outputDir, *outputName), protoBytes, 0644); err != nil {
-		fmt.Println("Failed: ", err)
+		fmt.Println("Failed:", err)
 		os.Exit(1)
 	} else {
 		fmt.Println(*outputName, "has been generated successfully.")