Chore: refine code (#3067)

Co-authored-by: Loyalsoldier <10487845+Loyalsoldier@users.noreply.github.com>

data/category-ads

@@ -38,6 +38,7 @@ include:newrelic-ads
 include:ogury-ads
 include:ookla-speedtest-ads
 include:openx-ads
+include:picacg-ads
 include:pocoiq-ads
 include:pubmatic-ads
 include:qihoo360-ads

data/category-ru

@@ -23,5 +23,6 @@ include:mailru-group
 include:okko
 include:ozon
 include:rutube
+include:wink
 include:x5
 include:yandex

data/ctyun

@@ -0,0 +1,69 @@
+# 天翼云
+# 京ICP备2021034386号
+ctadns.cn
+bjctyiptv.cn
+cqctyiptv.cn
+ctacdn.cn
+ctaigw.cn
+ctbcdn.com
+ctcdn.cn
+ctcdn.com.cn
+ctcdnov.net
+ctcloudzos.cn
+ctcns.cn
+ctdcdn.com
+ctdns.cn
+ctdns.com.cn
+ctdns.net
+ctecdn.cn
+ctecx.cn
+ctgcdn.com
+cthcdn.cn
+cthcdn.com
+cthcdn.net
+ctlcdn.cn
+ctlcdn.com
+ctlcdn.net
+ctmcdn.cn
+ctovcdn.com
+ctrender.com
+ctwcdn.cn
+ctxcdn.cn
+ctxcdn.com
+ctxcdn.net
+ctxirang.cn
+ctxirang.com
+ctycdn.cn
+ctycdn.net
+ctycdn.net.cn
+ctydoh.cn
+ctyecx.cn
+ctyiptv.cn
+ctyun.cn
+ctyun.com.cn
+ctyuncdn.cn
+ctyuncs.cn
+ctyuninner.com
+ctyunmds.cn
+ctyunwaf.cn
+ctyunwaf.com
+ctyunwaf1.com
+ctyunwaf3.cn
+ctyunxs.cn
+ctyunzos.cn
+ctzcdn.cn
+ctzcdn.com
+edgecloudx.cn
+faasapp.cn
+faasdev.cn
+fjctyiptv.cn
+gdctyiptv.cn
+gsctyiptv.cn
+gsjtyiptv.cn
+gzctyiptv.cn
+jsctyiptv.cn
+modelers.cn
+scctyiptv.cn
+snctyiptv.cn
+ynctyiptv.cn
+ynjtyiptv.cn

data/geolocation-cn

@@ -54,6 +54,7 @@ include:aws-cn
 include:baishancloud
 include:bootcdn
 include:cloudflare-cn
+include:ctyun
 include:dwion
 include:maocloud
 include:qingcloud

data/google-ads

@@ -49,6 +49,7 @@ partnerad.l.google.com @ads
 urchin.com @ads
 
 full:analytics.google.com @ads
-full:fundingchoicesmessages.google.com @ads
+# https://github.com/AdguardTeam/FiltersRegistry/pull/1154
+# full:fundingchoicesmessages.google.com @ads
 
 regexp:^adservice\.google\.([a-z]{2}|com?)(\.[a-z]{2})?$ @ads

data/picacg

@@ -1,3 +1,5 @@
+include:picacg-ads
+
 bikaa.xyz
 bikac.xyz
 bikaios.xyz
@@ -10,6 +12,5 @@ picacomic.xyz
 wikawika.xyz
 
 # Image Resource Domain like `img.diwodiwo.xyz` `s3.diwodiwo.xyz` `storage.diwodiwo.xyz` `storage-b.diwodiwo.xyz`
-regexp:^([a-z0-9-]+\.)*(?!ad-display\.|ad-channel\.)[a-z0-9-]+\.diwodiwo\.xyz$
+diwodiwo.xyz
-ad-channel.diwodiwo.xyz @ads
+
-ad-display.diwodiwo.xyz @ads

data/picacg-ads

@@ -0,0 +1,2 @@
+full:ad-channel.diwodiwo.xyz @ads
+full:ad-display.diwodiwo.xyz @ads

data/thescoregroup

@@ -4,18 +4,13 @@
 60plusmilfs.com
 analqts.com
 ashleysageellison.com
-asiancoochies.com
 autumn-jade.com
 beascoremodel.com
-bigboobalexya.com
 bigboobbundle.com
-bigboobdaria.com
 bigboobspov.com
-bigboobvanessay.com
 bigtitangelawhite.com
 bigtithitomi.com
 bigtithooker.com
-bigtitkatiethornton.com
 bigtitterrynova.com
 bigtitvenera.com
 blackandstacked.com
@@ -28,13 +23,11 @@ bustydustystash.com
 bustyinescudna.com
 bustykellykay.com
 bustykerrymarie.com
-bustylezzies.com
 bustylornamorgan.com
 bustymerilyn.com
 bustyoldsluts.com
 bustysammieblack.com
 cherrybrady.com
-chicksonblackdicks.com
 chloesworld.com
 christymarks.com
 cock4stepmom.com
@@ -44,7 +37,6 @@ crystalgunnsworld.com
 daylenerio.com
 desiraesworld.com
 dianepoppos.com
-ebonythots.com
 eboobstore.com
 evanottyvideos.com
 feedherfuckher.com
@@ -53,19 +45,14 @@ getscorecash.com
 grannygetsafacial.com
 grannylovesbbc.com
 grannylovesyoungcock.com
-hairycoochies.com
 homealonemilfs.com
-hornyasianmilfs.com
 ibonedyourmom.com
 ifuckedtheboss.com
-jessicaturner.co.uk
 jessicaturner.com
 joanabliss.com
 juliamiles.com
 karinahart.com
 karlajames.com
-latinacoochies.com
-latinmommas.com
 leannecrowvideos.com
 legsex.com
 linseysworld.com
@@ -86,7 +73,6 @@ pickinguppussy.com
 pornloser.com
 pornmegaload.com
 reneerossvideos.com
-roxired.com
 sarennasworld.com
 scoreclassics.com
 scoregroup.com

data/umeng-ads

@@ -1,6 +1,24 @@
 aaid.umeng.com @ads
 alog.umeng.com @ads
 alog.umengcloud.com @ads
+alogs.umeng.com @ads
+alogus.umeng.com @ads
+ar.umeng.com @ads
+aspect-upush.umeng.com @ads
+audid.umeng.com @ads
+ccs.umeng.com @ads
+cnlogs.umeng.com @ads
+cnlogs.umengcloud.com @ads
+# https://github.com/TG-Twilight/AWAvenue-Ads-Rule/issues/185
+# errlog.umeng.com @ads
+# errnewlog.umeng.com @ads
+new-aaid.umeng.com @ads
+new-aaid.umeng.com.gds.alibabadns.com @ads
+oc.umeng.com @ads
+plbslog.umeng.com @ads
+resolve.umeng.com @ads
+ulogs.umeng.com @ads
+ulogs.umengcloud.com @ads
 utoken.umeng.com @ads
 
 # CNZZ

data/wink

@@ -0,0 +1,3 @@
+ngenix.net
+restream-media.net
+wink.ru

main.go

@@ -2,11 +2,11 @@ package main
 
 import (
 	"bufio"
-	"errors"
 	"flag"
 	"fmt"
 	"os"
 	"path/filepath"
+	"regexp"
 	"sort"
 	"strconv"
 	"strings"
@@ -22,6 +22,14 @@ var (
 	exportLists = flag.String("exportlists", "", "Lists to be flattened and exported in plaintext format, separated by ',' comma")
 )
 
+const (
+	RuleTypeDomain     string = "domain"
+	RuleTypeFullDomain string = "full"
+	RuleTypeKeyword    string = "keyword"
+	RuleTypeRegexp     string = "regexp"
+	RuleTypeInclude    string = "include"
+)
+
 type Entry struct {
 	Type  string
 	Value string
@@ -64,32 +72,41 @@ func (l *ParsedList) toProto() (*router.GeoSite, error) {
 	}
 	for _, entry := range l.Entry {
 		switch entry.Type {
-		case "domain":
+		case RuleTypeDomain:
 			site.Domain = append(site.Domain, &router.Domain{
 				Type:      router.Domain_RootDomain,
 				Value:     entry.Value,
 				Attribute: entry.Attrs,
 			})
-		case "regexp":
+
+		case RuleTypeRegexp:
+			// check regexp validity to avoid runtime error
+			_, err := regexp.Compile(entry.Value)
+			if err != nil {
+				return nil, fmt.Errorf("invalid regexp in list %s: %s", l.Name, entry.Value)
+			}
 			site.Domain = append(site.Domain, &router.Domain{
 				Type:      router.Domain_Regex,
 				Value:     entry.Value,
 				Attribute: entry.Attrs,
 			})
-		case "keyword":
+
+		case RuleTypeKeyword:
 			site.Domain = append(site.Domain, &router.Domain{
 				Type:      router.Domain_Plain,
 				Value:     entry.Value,
 				Attribute: entry.Attrs,
 			})
-		case "full":
+
+		case RuleTypeFullDomain:
 			site.Domain = append(site.Domain, &router.Domain{
 				Type:      router.Domain_Full,
 				Value:     entry.Value,
 				Attribute: entry.Attrs,
 			})
+
 		default:
-			return nil, errors.New("unknown domain type: " + entry.Type)
+			return nil, fmt.Errorf("unknown domain type: %s", entry.Type)
 		}
 	}
 	return site, nil
@@ -118,24 +135,30 @@ func removeComment(line string) string {
 func parseDomain(domain string, entry *Entry) error {
 	kv := strings.Split(domain, ":")
 	if len(kv) == 1 {
-		entry.Type = "domain"
+		entry.Type = RuleTypeDomain
 		entry.Value = strings.ToLower(kv[0])
 		return nil
 	}
 
 	if len(kv) == 2 {
 		entry.Type = strings.ToLower(kv[0])
+
+		if strings.EqualFold(entry.Type, RuleTypeRegexp) {
+			entry.Value = kv[1]
+		} else {
 			entry.Value = strings.ToLower(kv[1])
+		}
+
 		return nil
 	}
 
-	return errors.New("Invalid format: " + domain)
+	return fmt.Errorf("invalid format: %s", domain)
 }
 
 func parseAttribute(attr string) (*router.Domain_Attribute, error) {
 	var attribute router.Domain_Attribute
 	if len(attr) == 0 || attr[0] != '@' {
-		return &attribute, errors.New("invalid attribute: " + attr)
+		return &attribute, fmt.Errorf("invalid attribute: %s", attr)
 	}
 
 	// Trim attribute prefix `@` character
@@ -148,7 +171,7 @@ func parseAttribute(attr string) (*router.Domain_Attribute, error) {
 		attribute.Key = strings.ToLower(parts[0])
 		intv, err := strconv.Atoi(parts[1])
 		if err != nil {
-			return &attribute, errors.New("invalid attribute: " + attr + ": " + err.Error())
+			return &attribute, fmt.Errorf("invalid attribute: %s: %v", attr, err)
 		}
 		attribute.TypedValue = &router.Domain_Attribute_IntValue{IntValue: int64(intv)}
 	}
@@ -161,7 +184,7 @@ func parseEntry(line string) (Entry, error) {
 
 	var entry Entry
 	if len(parts) == 0 {
-		return entry, errors.New("empty entry")
+		return entry, fmt.Errorf("empty entry")
 	}
 
 	if err := parseDomain(parts[0], &entry); err != nil {
@@ -255,7 +278,7 @@ func ParseList(list *List, ref map[string]*List) (*ParsedList, error) {
 		newEntryList := make([]Entry, 0, len(entryList))
 		hasInclude := false
 		for _, entry := range entryList {
-			if entry.Type == "include" {
+			if entry.Type == RuleTypeInclude {
 				refName := strings.ToUpper(entry.Value)
 				if entry.Attrs != nil {
 					for _, attr := range entry.Attrs {
@@ -267,7 +290,7 @@ func ParseList(list *List, ref map[string]*List) (*ParsedList, error) {
 
 						refList := ref[refName]
 						if refList == nil {
-							return nil, errors.New(entry.Value + " not found.")
+							return nil, fmt.Errorf("list not found: %s", entry.Value)
 						}
 						attrEntrys := createIncludeAttrEntrys(refList, attr)
 						if len(attrEntrys) != 0 {
@@ -282,7 +305,7 @@ func ParseList(list *List, ref map[string]*List) (*ParsedList, error) {
 					pl.Inclusion[InclusionName] = true
 					refList := ref[refName]
 					if refList == nil {
-						return nil, errors.New(entry.Value + " not found.")
+						return nil, fmt.Errorf("list not found: %s", entry.Value)
 					}
 					newEntryList = append(newEntryList, refList.Entry...)
 				}