yandex: add yandexwebcache.net (#3308)

Co-authored-by: Ivan Plekhanov <iaplekh@gmail.com>

.github/workflows/test-pr.yml

@@ -34,7 +34,7 @@ jobs:
           mv dlc.dat TEST-${{ github.run_number }}-dlc.dat
 
       - name: Upload TEST-${{ github.run_number }}-dlc.dat
-        uses: actions/upload-artifact@v6
+        uses: actions/upload-artifact@v7
         with:
           name: TEST-${{ github.run_number }}-dlc.dat
           path: TEST-${{ github.run_number }}-dlc.dat

data/amazon

@@ -33,6 +33,7 @@ aboutamazon.in
 aboutamazon.it
 aboutamazon.jp
 aboutamazon.pl
+aboutamazon.sg
 alexafund.cn @cn
 alexafund.com.cn @cn
 amaaozn.com
@@ -42,24 +43,35 @@ amazon-lantern.com
 amazon-launchpad.com
 amazon.ae
 amazon.ca
+amazon.cl
 amazon.cn @cn
 amazon.co.jp
 amazon.co.uk
+amazon.co.za
 amazon.com
 amazon.com.au
+amazon.com.be
 amazon.com.br
+amazon.com.co
 amazon.com.mx
+amazon.com.ng
 amazon.com.tr
 amazon.de
 amazon.dev
+amazon.eg
 amazon.es
 amazon.fr
+amazon.ie
 amazon.in
 amazon.it
 amazon.jobs
 amazon.jp
+amazon.lu
 amazon.nl
+amazon.pl
 amazon.red
+amazon.sa
+amazon.se
 amazon.sg
 amazonalexavoxcon.com
 amazonauthorinsights.com
@@ -103,6 +115,7 @@ media-amazon.com
 primeday.cn @cn
 primeday.com.cn @cn
 primeday.info
+seattlespheres.com
 siege-amazon.com
 ssl-images-amazon.com
 ueberamazon.de

data/aws

@@ -40,6 +40,7 @@ awssecworkshops.com
 awsstatic.com
 awsthinkbox.com
 awstrack.me
+awswaf.com
 cdkworkshop.com
 cloudfront.com
 cloudfront.net

data/category-bank-ru

@@ -5,6 +5,10 @@ alfabank.ru
 gazprombank.ru
 gpb.ru
 
+# Mts dengi
+dbo-dengi.online
+mtsdengi.ru
+
 # PSB Bank
 psbank.ru
 
@@ -28,5 +32,9 @@ cdn-tinkoff.ru
 tbank-online.com
 tbank.ru
 
+# Tochka bank
+tochka-tech.com
+tochka.com
+
 # VTB Bank
 vtb.ru

data/category-cryptocurrency

@@ -35,6 +35,7 @@ bitmex.com
 bitquick.co
 bitstamp.net
 bittrex.com
+blockchain.com
 blockfrost.io
 btcbox.co.jp
 cex.io

data/category-dev

@@ -123,6 +123,7 @@ notepad-plus-plus.org
 openresty.org
 openssl.org
 opensuse.org
+packagecloud.io
 packagist.org
 pcre.org
 phantomjs.org

data/category-gov-ru

@@ -18,6 +18,7 @@ cbr.ru          # Central Bank of Russia
 cikrf.ru        # Central Electoral Commission of the Russian Federation
 ebs.ru          # Unified Biometric System
 goskey.ru       # GosKey - an electronic signature on a smartphone
+grfc.ru         # General radio frequency center
 izbirkom.ru     # Information on ongoing elections and referendums
 kremlin.ru      # Online representation of the President of Russia
 nalog.ru        # Federal Tax Service

data/category-non-domestic-cn

@@ -1,31 +0,0 @@
-# Part of Chinese entities but exclusively serving for non-cn area
-
-include:alibaba @!cn
-include:anker @!cn
-include:bilibili @!cn
-include:boc @!cn
-include:bytedance @!cn
-include:ccb @!cn
-include:chinamobile @!cn
-include:chinatelecom @!cn
-include:chinaunicom @!cn
-include:citic @!cn
-include:cmb @!cn
-include:ctrip @!cn
-include:deepin @!cn
-include:dewu @!cn
-include:didi @!cn
-include:eastmoney @!cn
-include:huawei @!cn
-include:icbc @!cn
-include:ipip @!cn
-include:iqiyi @!cn
-include:jd @!cn
-include:oppo @!cn
-include:pingan @!cn
-include:sina @!cn
-include:tencent @!cn
-include:vivo @!cn
-include:xd @!cn
-include:xiaohongshu @!cn
-include:xiaomi @!cn

data/category-porn

@@ -137,6 +137,7 @@ porn
 18hmanga.com
 18insta.com
 18j.tv
+18jav.tv
 18jms.com
 18mh.co
 18mh.me
@@ -2223,6 +2224,8 @@ heavyfetish.com
 hegre.com
 heiguab.top
 heijidi.life
+heiliao.com
+heiliao88.com
 heise360181.buzz
 heise360182.buzz
 helixstudios.net
@@ -2690,6 +2693,7 @@ jasmin.com
 jav-angel.net
 jav-subtitles.com
 jav-vr.net
+jav.com.co
 jav.dog
 jav.gallery
 jav.guru
@@ -2792,6 +2796,7 @@ javpub.me
 javpush.com
 javqd.com
 javrank.com
+javrate.com
 javrave.club
 javroot.com
 javscat.net
@@ -2816,6 +2821,7 @@ javtrust.com
 javtube.cc
 javtube.com
 javtube.net
+javvideoporn.com
 javvids.com
 javxspot.com
 javxxx.me
@@ -3587,6 +3593,7 @@ ninpu.cyou
 niuc2.com
 niziero.info
 njav.tv
+njavtv.com
 nlsexfilmpjes.com
 nlt-media.com
 noc.syosetu.com
@@ -5566,9 +5573,6 @@ xgtd3.com
 xgtdr.buzz
 xh-porn.com
 xh.video
-xhamster.com
-xhamster.desi
-xhamster2.com
 xhot.pro
 xhub.tv
 xiangrikui-app.com
@@ -6157,6 +6161,7 @@ regexp:(^|\.)tqav[1-9][0-9]\.com$
 regexp:(^|\.)tt[1-2][0-9]\.tv$
 regexp:(^|\.)ttghg[1-9][0-9]\.xyz$
 regexp:(^|\.)tttv([1-9][0-9]?|100)\.com$
+regexp:(^|\.)twav[1-9]\.xyz$
 regexp:(^|\.)twseb([1-9][0-9]?)?\.com$
 regexp:(^|\.)uu[a-z][1-9][0-9]?\.com$
 regexp:(^|\.)whtdh0[1-3]\.cc$

data/category-ru

@@ -24,10 +24,15 @@ spvb.ru    # Saint-Petersburg Stock Exchange
 # Financial marketplace
 banki.ru
 finuslugi.ru
+# Investment
+sistema-capital.com
 # Mir payment system
 mirpayonline.ru
 # National Payment Card System
 nspk.ru
+# Tipping service
+netmonet.co
+tips.tips
 
 # Telecom operators
 beeline.ru
@@ -49,4 +54,6 @@ ngenix.net     # NGENIX is a Russian provider of acceleration and security servi
 pochta.ru      # Russian post
 qms.ru         # Russian internet speed testing service
 rustore.ru     # RuStore is a Russian mobile app store for Android
+t1.cloud       # Russian cloud storage provider
 taxsee.com     # Taxi for business (self-employed drivers)
+yclients.com   # Russian SaaS platform for online booking and business automation for service companies

data/category-social-media-!cn

@@ -6,6 +6,7 @@ include:instagram
 include:linkedin
 include:mailru
 include:misskey
+include:mixi
 include:ok
 include:threads
 include:twitter

data/cn

@@ -2,4 +2,3 @@
 
 include:tld-cn
 include:geolocation-cn
-#include:category-non-domestic-cn

data/geolocation-!cn

@@ -1,8 +1,5 @@
 # This list contains domains that don't have access point in China mainland. This is opposite to geolocation-cn.
 
-# Part of Chinese entities but exclusively serving for non-cn area
-include:category-non-domestic-cn
-
 # AI Chat
 include:category-ai-!cn
 

data/kinopub

@@ -11,5 +11,6 @@ cdn-service.space
 cdn2cdn.com
 cdn2site.com
 pushbr.com # poster images CDN
+smarttvcdn.online
 
 regexp:(\w+)-static-[0-9]+\.cdntogo\.net$

data/missav

@@ -1,5 +1,7 @@
 missav.ai
 missav.com
+missav.live
 missav.uno
 missav.vip
 missav.ws
+missav123.com

data/mixi

@@ -0,0 +1,3 @@
+mixi.co.jp
+mixi.jp
+mixi.net

data/perplexity

@@ -2,4 +2,5 @@ perplexity.ai
 perplexity.com
 pplx.ai
 
+full:ppl-ai-file-upload.s3.amazonaws.com
 full:pplx-res.cloudinary.com

data/salesforce

@@ -25,6 +25,7 @@ quotable.com
 radian6.com
 relateiq.com
 salesforce-setup.com
+salesforce-sites.com
 salesforce.com
 salesforce.org
 salesforceiq.com

data/xhamster

@@ -3,6 +3,8 @@ xhamster.desi
 xhamster.xxx
 xhamster18.com
 xhamster18.desi
+xhamster19.com
+xhamster3.com
 xhamsterlive.com
 xhcdn.com
 

data/yandex

@@ -53,8 +53,10 @@ yandexadexchange.net
 yandexcloud.net
 yandexcom.net
 yandexmetrica.com
+yandexwebcache.net
 yandexwebcache.org
 yastat.net
+yastatic-net.ru
 yastatic.net
 
 # Watching movies, included in the Yandex subscription

main.go

@@ -22,18 +22,11 @@ var (
 	exportLists = flag.String("exportlists", "", "Lists to be flattened and exported in plaintext format, separated by ',' comma")
 )
 
-var (
-	plMap     = make(map[string]*ParsedList)
-	finalMap  = make(map[string][]*Entry)
-	cirIncMap = make(map[string]bool) // Used for circular inclusion detection
-)
-
 type Entry struct {
 	Type  string
 	Value string
 	Attrs []string
 	Plain string
-	Affs  []string
 }
 
 type Inclusion struct {
@@ -48,6 +41,12 @@ type ParsedList struct {
 	Entries    []*Entry
 }
 
+type Processor struct {
+	plMap     map[string]*ParsedList
+	finalMap  map[string][]*Entry
+	cirIncMap map[string]bool
+}
+
 func makeProtoList(listName string, entries []*Entry) (*router.GeoSite, error) {
 	site := &router.GeoSite{
 		CountryCode: listName,
@@ -90,29 +89,27 @@ func writePlainList(listname string, entries []*Entry) error {
 	return w.Flush()
 }
 
-func parseEntry(line string) (Entry, error) {
+func parseEntry(line string) (*Entry, []string, error) {
-	var entry Entry
+	entry := new(Entry)
 	parts := strings.Fields(line)
 	if len(parts) == 0 {
-		return entry, fmt.Errorf("empty line")
+		return entry, nil, fmt.Errorf("empty line")
 	}
 
 	// Parse type and value
-	v := parts[0]
+	typ, val, isTypeSpecified := strings.Cut(parts[0], ":")
-	colonIndex := strings.Index(v, ":")
+	typ = strings.ToLower(typ)
-	if colonIndex == -1 {
+	if !isTypeSpecified { // Default RuleType
-		entry.Type = dlc.RuleTypeDomain // Default type
+		if !validateDomainChars(typ) {
-		entry.Value = strings.ToLower(v)
+			return entry, nil, fmt.Errorf("invalid domain: %q", typ)
-		if !validateDomainChars(entry.Value) {
-			return entry, fmt.Errorf("invalid domain: %q", entry.Value)
 		}
+		entry.Type = dlc.RuleTypeDomain
+		entry.Value = typ
 	} else {
-		typ := strings.ToLower(v[:colonIndex])
-		val := v[colonIndex+1:]
 		switch typ {
 		case dlc.RuleTypeRegexp:
 			if _, err := regexp.Compile(val); err != nil {
-				return entry, fmt.Errorf("invalid regexp %q: %w", val, err)
+				return entry, nil, fmt.Errorf("invalid regexp %q: %w", val, err)
 			}
 			entry.Type = dlc.RuleTypeRegexp
 			entry.Value = val
@@ -120,57 +117,60 @@ func parseEntry(line string) (Entry, error) {
 			entry.Type = dlc.RuleTypeInclude
 			entry.Value = strings.ToUpper(val)
 			if !validateSiteName(entry.Value) {
-				return entry, fmt.Errorf("invalid include list name: %q", entry.Value)
+				return entry, nil, fmt.Errorf("invalid included list name: %q", entry.Value)
 			}
 		case dlc.RuleTypeDomain, dlc.RuleTypeFullDomain, dlc.RuleTypeKeyword:
 			entry.Type = typ
 			entry.Value = strings.ToLower(val)
 			if !validateDomainChars(entry.Value) {
-				return entry, fmt.Errorf("invalid domain: %q", entry.Value)
+				return entry, nil, fmt.Errorf("invalid domain: %q", entry.Value)
 			}
 		default:
-			return entry, fmt.Errorf("invalid type: %q", typ)
+			return entry, nil, fmt.Errorf("invalid type: %q", typ)
 		}
 	}
 
 	// Parse attributes and affiliations
+	var affs []string
 	for _, part := range parts[1:] {
-		if strings.HasPrefix(part, "@") {
+		switch part[0] {
-			attr := strings.ToLower(part[1:]) // Trim attribute prefix `@` character
+		case '@':
+			attr := strings.ToLower(part[1:])
 			if !validateAttrChars(attr) {
-				return entry, fmt.Errorf("invalid attribute: %q", attr)
+				return entry, affs, fmt.Errorf("invalid attribute: %q", attr)
 			}
 			entry.Attrs = append(entry.Attrs, attr)
-		} else if strings.HasPrefix(part, "&") {
+		case '&':
-			aff := strings.ToUpper(part[1:]) // Trim affiliation prefix `&` character
+			aff := strings.ToUpper(part[1:])
 			if !validateSiteName(aff) {
-				return entry, fmt.Errorf("invalid affiliation: %q", aff)
+				return entry, affs, fmt.Errorf("invalid affiliation: %q", aff)
 			}
-			entry.Affs = append(entry.Affs, aff)
+			affs = append(affs, aff)
-		} else {
+		default:
-			return entry, fmt.Errorf("invalid attribute/affiliation: %q", part)
+			return entry, affs, fmt.Errorf("invalid attribute/affiliation: %q", part)
 		}
 	}
-	// Sort attributes
-	slices.Sort(entry.Attrs)
-	// Formated plain entry: type:domain.tld:@attr1,@attr2
-	var plain strings.Builder
-	plain.Grow(len(entry.Type) + len(entry.Value) + 10)
-	plain.WriteString(entry.Type)
-	plain.WriteByte(':')
-	plain.WriteString(entry.Value)
-	for i, attr := range entry.Attrs {
-		if i == 0 {
-			plain.WriteByte(':')
-		} else {
-			plain.WriteByte(',')
-		}
-		plain.WriteByte('@')
-		plain.WriteString(attr)
-	}
-	entry.Plain = plain.String()
 
-	return entry, nil
+	if entry.Type != dlc.RuleTypeInclude {
+		slices.Sort(entry.Attrs) // Sort attributes
+		// Formated plain entry: type:domain.tld:@attr1,@attr2
+		var plain strings.Builder
+		plain.Grow(len(entry.Type) + len(entry.Value) + 10)
+		plain.WriteString(entry.Type)
+		plain.WriteByte(':')
+		plain.WriteString(entry.Value)
+		for i, attr := range entry.Attrs {
+			if i == 0 {
+				plain.WriteByte(':')
+			} else {
+				plain.WriteByte(',')
+			}
+			plain.WriteByte('@')
+			plain.WriteString(attr)
+		}
+		entry.Plain = plain.String()
+	}
+	return entry, affs, nil
 }
 
 func validateDomainChars(domain string) bool {
@@ -206,62 +206,54 @@ func validateSiteName(name string) bool {
 	return true
 }
 
-func loadData(path string) ([]*Entry, error) {
+func (p *Processor) getOrCreateParsedList(name string) *ParsedList {
+	pl, exist := p.plMap[name]
+	if !exist {
+		pl = &ParsedList{Name: name}
+		p.plMap[name] = pl
+	}
+	return pl
+}
+
+func (p *Processor) loadData(listName string, path string) error {
 	file, err := os.Open(path)
 	if err != nil {
-		return nil, err
+		return err
 	}
 	defer file.Close()
 
-	var entries []*Entry
+	pl := p.getOrCreateParsedList(listName)
 	scanner := bufio.NewScanner(file)
 	lineIdx := 0
 	for scanner.Scan() {
-		line := scanner.Text()
 		lineIdx++
-		if idx := strings.Index(line, "#"); idx != -1 {
+		line, _, _ := strings.Cut(scanner.Text(), "#") // Remove comments
-			line = line[:idx] // Remove comments
-		}
 		line = strings.TrimSpace(line)
 		if line == "" {
 			continue
 		}
-		entry, err := parseEntry(line)
+		entry, affs, err := parseEntry(line)
 		if err != nil {
-			return entries, fmt.Errorf("error in %q at line %d: %w", path, lineIdx, err)
+			return fmt.Errorf("error in %q at line %d: %w", path, lineIdx, err)
 		}
-		entries = append(entries, &entry)
-	}
-	return entries, nil
-}
 
-func parseList(refName string, refList []*Entry) error {
-	pl, _ := plMap[refName]
-	if pl == nil {
-		pl = &ParsedList{Name: refName}
-		plMap[refName] = pl
-	}
-	for _, entry := range refList {
 		if entry.Type == dlc.RuleTypeInclude {
-			if len(entry.Affs) != 0 {
-				return fmt.Errorf("affiliation is not allowed for include:%q", entry.Value)
-			}
 			inc := &Inclusion{Source: entry.Value}
 			for _, attr := range entry.Attrs {
-				if strings.HasPrefix(attr, "-") {
+				if attr[0] == '-' {
-					inc.BanAttrs = append(inc.BanAttrs, attr[1:]) // Trim attribute prefix `-` character
+					inc.BanAttrs = append(inc.BanAttrs, attr[1:])
 				} else {
 					inc.MustAttrs = append(inc.MustAttrs, attr)
 				}
 			}
+			for _, aff := range affs {
+				apl := p.getOrCreateParsedList(aff)
+				apl.Inclusions = append(apl.Inclusions, inc)
+			}
 			pl.Inclusions = append(pl.Inclusions, inc)
 		} else {
-			for _, aff := range entry.Affs {
+			for _, aff := range affs {
-				apl, _ := plMap[aff]
+				apl := p.getOrCreateParsedList(aff)
-				if apl == nil {
-					apl = &ParsedList{Name: aff}
-					plMap[aff] = apl
-				}
 				apl.Entries = append(apl.Entries, entry)
 			}
 			pl.Entries = append(pl.Entries, entry)
@@ -296,9 +288,7 @@ func polishList(roughMap map[string]*Entry) []*Entry {
 	domainsMap := make(map[string]bool)
 	for _, entry := range roughMap {
 		switch entry.Type { // Bypass regexp, keyword and "full/domain with attr"
-		case dlc.RuleTypeRegexp:
+		case dlc.RuleTypeRegexp, dlc.RuleTypeKeyword:
-			finalList = append(finalList, entry)
-		case dlc.RuleTypeKeyword:
 			finalList = append(finalList, entry)
 		case dlc.RuleTypeDomain:
 			domainsMap[entry.Value] = true
@@ -323,11 +313,11 @@ func polishList(roughMap map[string]*Entry) []*Entry {
 			pd = "." + pd // So that `domain:example.org` overrides `full:example.org`
 		}
 		for {
-			idx := strings.Index(pd, ".")
+			var hasParent bool
-			if idx == -1 {
+			_, pd, hasParent = strings.Cut(pd, ".") // Go for next parent
+			if !hasParent {
 				break
 			}
-			pd = pd[idx+1:] // Go for next parent
 			if domainsMap[pd] {
 				isRedundant = true
 				break
@@ -344,36 +334,38 @@ func polishList(roughMap map[string]*Entry) []*Entry {
 	return finalList
 }
 
-func resolveList(pl *ParsedList) error {
+func (p *Processor) resolveList(plname string) error {
-	if _, pldone := finalMap[pl.Name]; pldone {
+	if _, pldone := p.finalMap[plname]; pldone {
 		return nil
 	}
-
+	pl, plexist := p.plMap[plname]
-	if cirIncMap[pl.Name] {
+	if !plexist {
-		return fmt.Errorf("circular inclusion in: %q", pl.Name)
+		return fmt.Errorf("list %q not found", plname)
 	}
-	cirIncMap[pl.Name] = true
+	if p.cirIncMap[plname] {
-	defer delete(cirIncMap, pl.Name)
+		return fmt.Errorf("circular inclusion in: %q", plname)
+	}
+	p.cirIncMap[plname] = true
+	defer delete(p.cirIncMap, plname)
 
 	roughMap := make(map[string]*Entry) // Avoid basic duplicates
 	for _, dentry := range pl.Entries { // Add direct entries
 		roughMap[dentry.Plain] = dentry
 	}
 	for _, inc := range pl.Inclusions {
-		incPl, exist := plMap[inc.Source]
+		if _, exist := p.plMap[inc.Source]; !exist {
-		if !exist {
+			return fmt.Errorf("list %q includes a non-existent list: %q", plname, inc.Source)
-			return fmt.Errorf("list %q includes a non-existent list: %q", pl.Name, inc.Source)
 		}
-		if err := resolveList(incPl); err != nil {
+		if err := p.resolveList(inc.Source); err != nil {
 			return err
 		}
-		for _, ientry := range finalMap[inc.Source] {
+		for _, ientry := range p.finalMap[inc.Source] {
 			if isMatchAttrFilters(ientry, inc) { // Add included entries
 				roughMap[ientry.Plain] = ientry
 			}
 		}
 	}
-	finalMap[pl.Name] = polishList(roughMap)
+	p.finalMap[plname] = polishList(roughMap)
 	return nil
 }
 
@@ -381,8 +373,8 @@ func run() error {
 	dir := *dataPath
 	fmt.Printf("using domain lists data in %q\n", dir)
 
-	// Generate refMap
+	// Generate plMap
-	refMap := make(map[string][]*Entry)
+	processor := &Processor{plMap: make(map[string]*ParsedList)}
 	err := filepath.WalkDir(dir, func(path string, d os.DirEntry, err error) error {
 		if err != nil {
 			return err
@@ -394,23 +386,16 @@ func run() error {
 		if !validateSiteName(listName) {
 			return fmt.Errorf("invalid list name: %q", listName)
 		}
-		refMap[listName], err = loadData(path)
+		return processor.loadData(listName, path)
-		return err
 	})
 	if err != nil {
 		return fmt.Errorf("failed to loadData: %w", err)
 	}
-
-	// Generate plMap
-	for refName, refList := range refMap {
-		if err := parseList(refName, refList); err != nil {
-			return fmt.Errorf("failed to parseList %q: %w", refName, err)
-		}
-	}
-
 	// Generate finalMap
-	for plname, pl := range plMap {
+	processor.finalMap = make(map[string][]*Entry, len(processor.plMap))
-		if err := resolveList(pl); err != nil {
+	processor.cirIncMap = make(map[string]bool)
+	for plname := range processor.plMap {
+		if err := processor.resolveList(plname); err != nil {
 			return fmt.Errorf("failed to resolveList %q: %w", plname, err)
 		}
 	}
@@ -419,11 +404,10 @@ func run() error {
 	if err := os.MkdirAll(*outputDir, 0755); err != nil {
 		return fmt.Errorf("failed to create output directory: %w", err)
 	}
-
+	// Export plaintext lists
-	// Export plaintext list
 	for rawEpList := range strings.SplitSeq(*exportLists, ",") {
 		if epList := strings.TrimSpace(rawEpList); epList != "" {
-			entries, exist := finalMap[strings.ToUpper(epList)]
+			entries, exist := processor.finalMap[strings.ToUpper(epList)]
 			if !exist || len(entries) == 0 {
 				fmt.Printf("list %q does not exist or is empty\n", epList)
 				continue
@@ -438,7 +422,7 @@ func run() error {
 
 	// Generate dat file
 	protoList := new(router.GeoSiteList)
-	for siteName, siteEntries := range finalMap {
+	for siteName, siteEntries := range processor.finalMap {
 		site, err := makeProtoList(siteName, siteEntries)
 		if err != nil {
 			return fmt.Errorf("failed to makeProtoList %q: %w", siteName, err)