Chore: refine code (#3067)

Co-authored-by: Loyalsoldier <10487845+Loyalsoldier@users.noreply.github.com>

.github/workflows/test-pr.yml

@@ -34,7 +34,7 @@ jobs:
           mv dlc.dat TEST-${{ github.run_number }}-dlc.dat
 
       - name: Upload TEST-${{ github.run_number }}-dlc.dat
-        uses: actions/upload-artifact@v5
+        uses: actions/upload-artifact@v6
         with:
           name: TEST-${{ github.run_number }}-dlc.dat
           path: TEST-${{ github.run_number }}-dlc.dat

data/18comic

@@ -5,11 +5,47 @@
 18comic.company
 18comic.org
 18comic.vip
+asjmapihost.cc
+cdnblackmyth.club
+cdnmhws.cc
+cdnmhwscc.vip
+cdnuc.vip
+cdnxxx-proxy.co
+cdnxxx-proxy.xyz
+jm-comic2.cc
+jm18c-bbm.cc
+jm18c-bbm.net
+jm18c-uoi.net
+jm365.work
+jm365.xyz
+jmapibranch1.cc
+jmapibranch2.cc
+jmapibranch3.cc
+jmapinode.biz
+jmapinode.vip
+jmapinode.xyz
+jmapinode1.top
+jmapinode2.top
+jmapinode3.top
+jmapinodeudzn.net
+jmapinodeudzn.xyz
+jmapiproxy1.cc
+jmapiproxy1.monster
+jmapiproxy2.cc
+jmapiproxy3.cc
+jmapiproxy4.cc
+jmapiproxyxxx.vip
+jmcomic.group
 jmcomic.ltd
 jmcomic.me
 jmcomic.mobi
 jmcomic.moe
+jmcomic.rocks
+jmcomic-fb.vip
+jmcomic-zzz.one
+jmcomic-zzz.org
 jmcomic1.city
 jmcomic1.me
 jmcomic1.mobi
+jmcomic1.rocks
 jmcomic2.moe

data/9game

@@ -0,0 +1 @@
+9game.cn

data/alibaba

@@ -10,6 +10,7 @@ taobao
 # All .tmall domains
 tmall
 
+include:9game
 include:alibaba-ads
 include:alibabacloud
 include:aliyun
@@ -25,7 +26,6 @@ include:youku
 25pp.com
 56che.com
 95095.com
-9game.cn
 ali-health.com
 aliapp.org
 alibaba-inc.com

data/alibaba-ads

@@ -1,3 +1,4 @@
+include:amap-ads
 include:youku-ads
 
 a.alimama.cn @ads
@@ -10,7 +11,6 @@ adash.man.aliyuncs.com @ads
 adashbc.m.taobao.com @ads
 adashbc.ut.taobao.com @ads
 adashx.m.taobao.com @ads
-adashx.ut.amap.com @ads
 adashx.ut.ele.me @ads
 adashx4ae.ut.taobao.com @ads
 adashx4yt.m.taobao.com @ads
@@ -38,7 +38,6 @@ retcode.taobao.com @ads
 tce.alicdn.com @ads
 tns.simba.taobao.com @ads
 track.lc.quark.cn @ads
-v6-adashx.ut.amap.com @ads
 v6-adashx.ut.cainiao.com @ads
 v6-adashx.ut.ele.me @ads
 v6-adashx.ut.taobao.com @ads

data/amap

@@ -1,3 +1,5 @@
+include:amap-ads
+
 a-map.cn
 a-map.co
 a-map.link

data/amap-ads

@@ -0,0 +1,5 @@
+adashx.ut.amap.com @ads
+dualstack-logs.amap.com @ads
+logs.amap.com @ads
+optimus-ads.amap.com @ads
+v6-adashx.ut.amap.com @ads

data/baishancloud

@@ -0,0 +1,3 @@
+bsclink.cn
+qingcdn.com
+trpcdn.net

data/category-ads

@@ -30,6 +30,7 @@ include:kuaishou-ads
 include:kugou-ads
 include:letv-ads
 include:meta-ads
+include:microsoft-ads
 include:mopub-ads
 include:mxplayer-ads
 include:netease-ads
@@ -37,6 +38,7 @@ include:newrelic-ads
 include:ogury-ads
 include:ookla-speedtest-ads
 include:openx-ads
+include:picacg-ads
 include:pocoiq-ads
 include:pubmatic-ads
 include:qihoo360-ads
@@ -57,7 +59,7 @@ include:xhamster-ads
 include:xiaomi-ads
 include:ximalaya-ads
 include:yahoo-ads
-include:youku-ads
+include:zhihu-ads
 
 # Other domains for ads serving
 51.la

data/category-ai-!cn

@@ -31,6 +31,7 @@ diabrowser.com
 dify.ai
 duck.ai
 gateway.ai.cloudflare.com
+kiro.dev
 meta.ai
 mistral.ai
 openart.ai

data/category-education-cn

@@ -53,6 +53,11 @@ qgzb.net
 zbbm.gov.cn
 zbbm.net
 
+# 中国考试教育网
+# 京ICP备05064772号
+neea.cn
+# neea.edu.cn
+
 # 云校
 91yunxiao.com
 # 爱云校/好分数

data/category-entertainment

@@ -83,6 +83,7 @@ egta.com
 fireside.fm
 internationalsaimoe.com
 joox.com
+kpacg.com
 linetv.tw
 megaphone.fm
 mikanani.me
@@ -100,3 +101,71 @@ yande.re
 amp-api.podcast.apple.com
 bag.itunes.apple.com
 bookeeper.itunes.apple.com
+
+# Entertainment oriented media
+## 4Gamer.net
+4gamer.net
+## 4Gamers
+4gamers.com.tw
+## 秋葉原綜研
+akiba-souken.com
+## Appget
+appget.com
+## AppMedia
+appmedia.jp
+## Automaton
+automaton-media.com
+## 电faminicogamer
+denfaminicogamer.jp
+## ASCII Media Works
+dengekionline.com
+## E-ROAD
+eroge-road.com
+## ファミ通
+famitsu.com
+## 遊戲基地
+gamebase.com.tw
+## gamebiz
+gamebiz.jp
+## IID, Inc.
+gamebusiness.jp
+gamespark.jp
+inside-games.jp
+## GameMeca
+gamemeca.com
+## Gameover有機網
+gameover.com.hk
+## Gamer
+gamer.ne.jp
+onlinegamer.jp
+## Great Game
+greatgame.asia
+## HobbiGame
+hobbigame.com
+## IGN
+ign.com
+ignimg.com
+## Made with Unity
+madewithunity.jp
+## Media Clip
+mediaclip.jp
+## 電脳世界のひみつ基地
+maedahiroyuki.com
+## MANTANWEB
+mantan-web.jp
+## Moepedia
+moepedia.net
+## QooApp
+qoo-app.com
+## Real Sound
+realsound.jp
+## Saiga NAK
+saiganak.com
+## The Games Daily
+tgdaily.co.kr
+## Thisisgame
+thisisgame.com
+## Wanuxi
+wanuxi.com
+## よろず〜
+yorozoonews.jp

data/category-entertainment-cn

@@ -46,12 +46,8 @@ yeshen.com
 
 # 17K小说
 17k.com
-# 3DM游戏网
-3dmgame.com
 # https://github.com/supzhang/epg
 51zmt.top
-# 游侠网
-ali213.net
 # 暴风影音
 baofeng.com
 baofeng.net
@@ -109,3 +105,47 @@ zhanqi.tv
 zhulang.com
 # 纵横小说
 zongheng.com
+
+# 娱乐资讯媒体
+
+include:tgbus
+include:vgtime
+
+## 17173网
+17173.com
+## 18183网
+18183.com
+## 3DM游戏网
+3dmgame.com
+## 游侠网
+ali213.net
+## 竞核
+cgames.com
+## 触乐
+chuapp.com
+## DataEye
+dataeye.com
+## GameDiary
+gamediary.cn
+## GameLook
+gamelook.com.cn
+## GameRes游资网
+gameres.com
+## 游戏价值论
+gamewower.com
+## 机核
+gcores.com
+## IGN中国
+ign.com.cn
+## 手游那点事
+nadianshi.com
+## 游戏日报
+news.yxrb.net
+## 游戏机实用技术
+ucg.cn
+## 游戏茶馆
+youxichaguan.com
+## 游戏葡萄
+youxiputao.com
+## 游研社
+yystv.cn

data/category-games-cn

@@ -1,4 +1,5 @@
 include:4399
+include:9game
 include:bilibili-game
 include:bluepoch-games
 include:cowlevel
@@ -6,13 +7,13 @@ include:herogame
 include:kurogames
 include:mihoyo-cn
 include:tencent-games
-include:tgbus
 include:tiancity
-include:vgtime
 include:vrzwk
 include:xd
 include:yokaverse
 
+# 北京奇客创想科技有限公司
+7k7k.com
 # 刀锋盒子 皖B2-20190103-4
 9xgame.com
 # 《异象回声》游戏官网 沪ICP备2023010411号-1

data/category-media

@@ -1,3 +1,4 @@
+# Entertainment(games) oriented media should be added in category-entertainment[-cn]
 include:category-tech-media
 
 include:6park
@@ -90,10 +91,12 @@ chinaeconomicreview.com
 cmmedia.com.tw
 cna.com.tw
 cnyes.com
+cool3c.com
 coolloud.org.tw
 cup.com.hk
 cw.com.tw
 d100.net
+donga.com
 ebc.net.tw
 ettoday.net
 eventsinfocus.org
@@ -101,6 +104,7 @@ factwire.org
 fortuneinsight.com
 freebeacon.com
 gbnews.uk
+getnews.jp
 hk01.com
 hkcitizenmedia.com
 hkcnews.com
@@ -109,11 +113,13 @@ hkgpao.com
 hongkongfp.com
 inmediahk.net
 inside.com.tw
+itmedia.co.jp
 jfengtime.com
 landofhope.tv
 limedia.tw
 localpresshk.com
 ltsports.com.tw
+macaodaily.com
 mdnkids.com
 mirrormedia.com.tw
 mirrormedia.mg
@@ -157,6 +163,7 @@ upmedia.mg
 vjmedia.com.hk
 warroom.org
 washingtonpost.com
+watch.impress.co.jp
 wealth.com.tw
 weiyuksj.com
 wenzhao.ca

data/category-media-cn

@@ -70,8 +70,6 @@ dzwww.com
 expreview.com
 # FreeBuf 网络安全行业门户
 freebuf.com
-# 机核
-gcores.com
 # 极客公园
 geekpark.net
 # 光明网
@@ -88,6 +86,8 @@ huanqiucdn.cn
 # 虎嗅
 huxiu.com
 huxiucdn.com
+# 电脑报
+icpcw.com
 # 中国IDC圈
 idcquan.com
 # 艾瑞咨询
@@ -115,6 +115,8 @@ kksmg.com
 statickksmg.com
 # 晚点
 latepost.com
+# 快科技
+mydrivers.com
 # 南岳新闻网
 nanyuenews.com
 # 每日经济新闻报社
@@ -174,6 +176,8 @@ thsi.cn
 ths123.com
 # 人人都是产品经理
 woshipm.com
+# 新快报
+xkb.com.cn
 # 第一财经
 yicai.com
 # ZEALER

data/category-ru

@@ -23,5 +23,6 @@ include:mailru-group
 include:okko
 include:ozon
 include:rutube
+include:wink
 include:x5
 include:yandex

data/ctyun

@@ -0,0 +1,69 @@
+# 天翼云
+# 京ICP备2021034386号
+ctadns.cn
+bjctyiptv.cn
+cqctyiptv.cn
+ctacdn.cn
+ctaigw.cn
+ctbcdn.com
+ctcdn.cn
+ctcdn.com.cn
+ctcdnov.net
+ctcloudzos.cn
+ctcns.cn
+ctdcdn.com
+ctdns.cn
+ctdns.com.cn
+ctdns.net
+ctecdn.cn
+ctecx.cn
+ctgcdn.com
+cthcdn.cn
+cthcdn.com
+cthcdn.net
+ctlcdn.cn
+ctlcdn.com
+ctlcdn.net
+ctmcdn.cn
+ctovcdn.com
+ctrender.com
+ctwcdn.cn
+ctxcdn.cn
+ctxcdn.com
+ctxcdn.net
+ctxirang.cn
+ctxirang.com
+ctycdn.cn
+ctycdn.net
+ctycdn.net.cn
+ctydoh.cn
+ctyecx.cn
+ctyiptv.cn
+ctyun.cn
+ctyun.com.cn
+ctyuncdn.cn
+ctyuncs.cn
+ctyuninner.com
+ctyunmds.cn
+ctyunwaf.cn
+ctyunwaf.com
+ctyunwaf1.com
+ctyunwaf3.cn
+ctyunxs.cn
+ctyunzos.cn
+ctzcdn.cn
+ctzcdn.com
+edgecloudx.cn
+faasapp.cn
+faasdev.cn
+fjctyiptv.cn
+gdctyiptv.cn
+gsctyiptv.cn
+gsjtyiptv.cn
+gzctyiptv.cn
+jsctyiptv.cn
+modelers.cn
+scctyiptv.cn
+snctyiptv.cn
+ynctyiptv.cn
+ynjtyiptv.cn

data/geolocation-!cn

@@ -31,6 +31,8 @@ include:typenetwork
 include:typography
 include:webtype
 
+bunka.go.jp
+
 # Automobile
 include:bmw
 include:bridgestone

data/geolocation-cn

@@ -51,8 +51,10 @@ lufaxcdn.com
 
 # CDN or SDWAN
 include:aws-cn
+include:baishancloud
 include:bootcdn
 include:cloudflare-cn
+include:ctyun
 include:dwion
 include:maocloud
 include:qingcloud
@@ -509,12 +511,10 @@ ycrx360.com
 1637.com
 16888.com
 16rd.com
-17173.com
 178linux.com
 17989.com
 17cdn.com
 17house.com
-18183.com
 1905.com
 19yxw.com
 1nongjing.com
@@ -595,7 +595,6 @@ ycrx360.com
 7230.com
 72byte.com
 7down.com
-7k7k.com
 7po.com
 7x24cc.com
 7xdown.com
@@ -1101,7 +1100,6 @@ mumayi.com
 muomou.com
 muzhiwan.com
 mydown.com
-mydrivers.com
 myshow360.net
 myyx618.com
 myzaker.com

data/google-ads

@@ -49,6 +49,7 @@ partnerad.l.google.com @ads
 urchin.com @ads
 
 full:analytics.google.com @ads
-full:fundingchoicesmessages.google.com @ads
+# https://github.com/AdguardTeam/FiltersRegistry/pull/1154
+# full:fundingchoicesmessages.google.com @ads
 
 regexp:^adservice\.google\.([a-z]{2}|com?)(\.[a-z]{2})?$ @ads

data/microsoft

@@ -1,6 +1,7 @@
 include:azure
 include:bing
 include:github
+include:microsoft-ads
 include:microsoft-dev
 include:microsoft-pki
 include:msn
@@ -83,7 +84,6 @@ c-msedge.net
 centralvalidation.com
 ch9.ms
 charticulator.com
-clarity.ms @ads
 cloudapp.net
 cloudappsecurity.com
 cortanaanalytics.com
@@ -147,7 +147,6 @@ microsoft-sbs-domains.com
 microsoft-smb.cn @cn
 microsoft365.com
 microsoftaccountguard.com
-microsoftads.com @ads
 microsoftadvertising.com
 microsoftadvertisingregionalawards.com
 microsoftaffiliates.com
@@ -189,7 +188,6 @@ morphcharts.com
 mpnevolution.com
 ms-studiosmedia.com
 ms365surfaceoffer.com
-msads.net @ads
 msauth.cn @cn
 msauth.net
 msauthimages.cn @cn
@@ -216,7 +214,6 @@ msra.cn @cn
 msturing.org
 msudalosti.com
 mymicrosoft.com
-nelreports.net @ads
 nextechafrica.net
 nxta.org
 o365cn.com @cn

data/microsoft-ads

@@ -0,0 +1,6 @@
+browser.events.data.msn.cn @ads
+browser.events.data.msn.com @ads
+clarity.ms @ads
+microsoftads.com @ads
+msads.net @ads
+nelreports.net @ads

data/mihoyo

@@ -1,2 +1,4 @@
 include:hoyoverse
 include:mihoyo-cn
+
+include:mihoyo-ads

data/mihoyo-ads

@@ -0,0 +1,2 @@
+log-upload-os.hoyoverse.com @ads
+log-upload.mihoyo.com @ads

data/ookla-speedtest

@@ -13,3 +13,6 @@ webtest.net
 full:www.speedtest.net.cdn.cloudflare.net
 
 include:ookla-speedtest-ads
+
+# SpeedTest Node
+ookla-speedtest-central.hgconair.hgc.com.hk # HGC Global Communications 香港環電

data/picacg

@@ -1,3 +1,5 @@
+include:picacg-ads
+
 bikaa.xyz
 bikac.xyz
 bikaios.xyz
@@ -8,3 +10,7 @@ picacn.xyz
 picacomic.com
 picacomic.xyz
 wikawika.xyz
+
+# Image Resource Domain like `img.diwodiwo.xyz` `s3.diwodiwo.xyz` `storage.diwodiwo.xyz` `storage-b.diwodiwo.xyz`
+diwodiwo.xyz
+

data/picacg-ads

@@ -0,0 +1,2 @@
+full:ad-channel.diwodiwo.xyz @ads
+full:ad-display.diwodiwo.xyz @ads

data/tencent-ads

@@ -19,6 +19,7 @@ adsview.qq.com @ads
 adsview2.qq.com @ads
 adv.app.qq.com @ads
 adver.qq.com @ads
+badjs.weixinbridge.com @ads
 beacon.cdn.qq.com @ads
 beacon.qq.com @ads
 beaconcdn.qq.com @ads

data/umeng-ads

@@ -1,5 +1,25 @@
+aaid.umeng.com @ads
 alog.umeng.com @ads
 alog.umengcloud.com @ads
+alogs.umeng.com @ads
+alogus.umeng.com @ads
+ar.umeng.com @ads
+aspect-upush.umeng.com @ads
+audid.umeng.com @ads
+ccs.umeng.com @ads
+cnlogs.umeng.com @ads
+cnlogs.umengcloud.com @ads
+# https://github.com/TG-Twilight/AWAvenue-Ads-Rule/issues/185
+# errlog.umeng.com @ads
+# errnewlog.umeng.com @ads
+new-aaid.umeng.com @ads
+new-aaid.umeng.com.gds.alibabadns.com @ads
+oc.umeng.com @ads
+plbslog.umeng.com @ads
+resolve.umeng.com @ads
+ulogs.umeng.com @ads
+ulogs.umengcloud.com @ads
+utoken.umeng.com @ads
 
 # CNZZ
 w.cnzz.com @ads

data/wink

@@ -0,0 +1,3 @@
+ngenix.net
+restream-media.net
+wink.ru

data/zhihu

@@ -1,2 +1,4 @@
+include:zhihu-ads
+
 zhihu.com
 zhimg.com

data/zhihu-ads

@@ -0,0 +1,2 @@
+crash2.zhihu.com @ads
+zhihu-web-analytics.zhihu.com @ads

main.go

@@ -2,11 +2,11 @@ package main
 
 import (
 	"bufio"
-	"errors"
 	"flag"
 	"fmt"
 	"os"
 	"path/filepath"
+	"regexp"
 	"sort"
 	"strconv"
 	"strings"
@@ -22,6 +22,14 @@ var (
 	exportLists = flag.String("exportlists", "", "Lists to be flattened and exported in plaintext format, separated by ',' comma")
 )
 
+const (
+	RuleTypeDomain     string = "domain"
+	RuleTypeFullDomain string = "full"
+	RuleTypeKeyword    string = "keyword"
+	RuleTypeRegexp     string = "regexp"
+	RuleTypeInclude    string = "include"
+)
+
 type Entry struct {
 	Type  string
 	Value string
@@ -64,32 +72,41 @@ func (l *ParsedList) toProto() (*router.GeoSite, error) {
 	}
 	for _, entry := range l.Entry {
 		switch entry.Type {
-		case "domain":
+		case RuleTypeDomain:
 			site.Domain = append(site.Domain, &router.Domain{
 				Type:      router.Domain_RootDomain,
 				Value:     entry.Value,
 				Attribute: entry.Attrs,
 			})
-		case "regexp":
+
+		case RuleTypeRegexp:
+			// check regexp validity to avoid runtime error
+			_, err := regexp.Compile(entry.Value)
+			if err != nil {
+				return nil, fmt.Errorf("invalid regexp in list %s: %s", l.Name, entry.Value)
+			}
 			site.Domain = append(site.Domain, &router.Domain{
 				Type:      router.Domain_Regex,
 				Value:     entry.Value,
 				Attribute: entry.Attrs,
 			})
-		case "keyword":
+
+		case RuleTypeKeyword:
 			site.Domain = append(site.Domain, &router.Domain{
 				Type:      router.Domain_Plain,
 				Value:     entry.Value,
 				Attribute: entry.Attrs,
 			})
-		case "full":
+
+		case RuleTypeFullDomain:
 			site.Domain = append(site.Domain, &router.Domain{
 				Type:      router.Domain_Full,
 				Value:     entry.Value,
 				Attribute: entry.Attrs,
 			})
+
 		default:
-			return nil, errors.New("unknown domain type: " + entry.Type)
+			return nil, fmt.Errorf("unknown domain type: %s", entry.Type)
 		}
 	}
 	return site, nil
@@ -118,24 +135,30 @@ func removeComment(line string) string {
 func parseDomain(domain string, entry *Entry) error {
 	kv := strings.Split(domain, ":")
 	if len(kv) == 1 {
-		entry.Type = "domain"
+		entry.Type = RuleTypeDomain
 		entry.Value = strings.ToLower(kv[0])
 		return nil
 	}
 
 	if len(kv) == 2 {
 		entry.Type = strings.ToLower(kv[0])
+
+		if strings.EqualFold(entry.Type, RuleTypeRegexp) {
+			entry.Value = kv[1]
+		} else {
 			entry.Value = strings.ToLower(kv[1])
+		}
+
 		return nil
 	}
 
-	return errors.New("Invalid format: " + domain)
+	return fmt.Errorf("invalid format: %s", domain)
 }
 
 func parseAttribute(attr string) (*router.Domain_Attribute, error) {
 	var attribute router.Domain_Attribute
 	if len(attr) == 0 || attr[0] != '@' {
-		return &attribute, errors.New("invalid attribute: " + attr)
+		return &attribute, fmt.Errorf("invalid attribute: %s", attr)
 	}
 
 	// Trim attribute prefix `@` character
@@ -148,7 +171,7 @@ func parseAttribute(attr string) (*router.Domain_Attribute, error) {
 		attribute.Key = strings.ToLower(parts[0])
 		intv, err := strconv.Atoi(parts[1])
 		if err != nil {
-			return &attribute, errors.New("invalid attribute: " + attr + ": " + err.Error())
+			return &attribute, fmt.Errorf("invalid attribute: %s: %v", attr, err)
 		}
 		attribute.TypedValue = &router.Domain_Attribute_IntValue{IntValue: int64(intv)}
 	}
@@ -161,7 +184,7 @@ func parseEntry(line string) (Entry, error) {
 
 	var entry Entry
 	if len(parts) == 0 {
-		return entry, errors.New("empty entry")
+		return entry, fmt.Errorf("empty entry")
 	}
 
 	if err := parseDomain(parts[0], &entry); err != nil {
@@ -255,7 +278,7 @@ func ParseList(list *List, ref map[string]*List) (*ParsedList, error) {
 		newEntryList := make([]Entry, 0, len(entryList))
 		hasInclude := false
 		for _, entry := range entryList {
-			if entry.Type == "include" {
+			if entry.Type == RuleTypeInclude {
 				refName := strings.ToUpper(entry.Value)
 				if entry.Attrs != nil {
 					for _, attr := range entry.Attrs {
@@ -267,7 +290,7 @@ func ParseList(list *List, ref map[string]*List) (*ParsedList, error) {
 
 						refList := ref[refName]
 						if refList == nil {
-							return nil, errors.New(entry.Value + " not found.")
+							return nil, fmt.Errorf("list not found: %s", entry.Value)
 						}
 						attrEntrys := createIncludeAttrEntrys(refList, attr)
 						if len(attrEntrys) != 0 {
@@ -282,7 +305,7 @@ func ParseList(list *List, ref map[string]*List) (*ParsedList, error) {
 					pl.Inclusion[InclusionName] = true
 					refList := ref[refName]
 					if refList == nil {
-						return nil, errors.New(entry.Value + " not found.")
+						return nil, fmt.Errorf("list not found: %s", entry.Value)
 					}
 					newEntryList = append(newEntryList, refList.Entry...)
 				}