Support to export all lists to a plain yml (#3211)

* Refactor: improve deduplicate

* Feat: support to export all lists to a plain yml

use: `--exportlists=_all_`

* Docs: add links for dlc plain yml

[skip ci]

.github/workflows/build.yml

@@ -33,15 +33,16 @@ jobs:
           echo "TAG_NAME=$(date +%Y%m%d%H%M%S)" >> $GITHUB_ENV
         shell: bash
 
-      - name: Build dlc.dat file
+      - name: Build dlc.dat and plain lists
         run: |
           cd code || exit 1
-          go run ./ --outputdir=../ --exportlists=category-ads-all,tld-cn,cn,tld-\!cn,geolocation-\!cn,apple,icloud
+          go run ./ --outputdir=../ --exportlists=_all_,category-ads-all,tld-cn,cn,tld-\!cn,geolocation-\!cn,apple,icloud
           cd ../ && rm -rf code
 
       - name: Generate dlc.dat sha256 hash
         run: |
           sha256sum dlc.dat > dlc.dat.sha256sum
+          sha256sum dlc.dat_plain.yml > dlc.dat_plain.yml.sha256sum
 
       - name: Generate Zip
         run: |
@@ -66,6 +67,6 @@ jobs:
 
       - name: Release and upload assets
         run: |
-          gh release create ${{ env.TAG_NAME }} --generate-notes --latest --title ${{ env.RELEASE_NAME }} ./dlc.dat ./dlc.dat.*
+          gh release create ${{ env.TAG_NAME }} --generate-notes --latest --title ${{ env.RELEASE_NAME }} ./dlc.dat ./dlc.dat.* ./dlc.dat_plain.yml ./dlc.dat_plain.yml.*
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.gitignore

@@ -8,4 +8,5 @@
 dlc.dat
 
 # Exported plaintext lists.
+dlc.dat_plain.yml
 /*.txt

README.md

@@ -10,6 +10,8 @@ This project is not opinionated. In other words, it does NOT endorse, claim or i
 
 - **dlc.dat**：[https://github.com/v2fly/domain-list-community/releases/latest/download/dlc.dat](https://github.com/v2fly/domain-list-community/releases/latest/download/dlc.dat)
 - **dlc.dat.sha256sum**：[https://github.com/v2fly/domain-list-community/releases/latest/download/dlc.dat.sha256sum](https://github.com/v2fly/domain-list-community/releases/latest/download/dlc.dat.sha256sum)
+- **dlc.dat_plain.yml**：[https://github.com/v2fly/domain-list-community/releases/latest/download/dlc.dat_plain.yml](https://github.com/v2fly/domain-list-community/releases/latest/download/dlc.dat_plain.yml)
+- **dlc.dat_plain.yml.sha256sum**：[https://github.com/v2fly/domain-list-community/releases/latest/download/dlc.dat_plain.yml.sha256sum](https://github.com/v2fly/domain-list-community/releases/latest/download/dlc.dat_plain.yml.sha256sum)
 
 ## Notice
 

main.go

@@ -92,6 +92,25 @@ func makeProtoList(listName string, entries []*Entry) (*router.GeoSite, error) {
 	return site, nil
 }
 
+func writePlainAll(siteList *[]string) error {
+	file, err := os.Create(filepath.Join(*outputDir, *outputName + "_plain.yml"))
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+	w := bufio.NewWriter(file)
+	w.WriteString("lists:\n")
+	for _, site := range *siteList {
+		fmt.Fprintf(w, "  - name: %s\n", strings.ToLower(site))
+		fmt.Fprintf(w, "    length: %d\n", len(finalMap[site]))
+		w.WriteString("    rules:\n")
+		for _, entry := range finalMap[site] {
+			fmt.Fprintf(w, "      - %s\n", entry.Plain)
+		}
+	}
+	return w.Flush()
+}
+
 func writePlainList(exportedName string) error {
 	targetList, exist := finalMap[strings.ToUpper(exportedName)]
 	if !exist || len(targetList) == 0 {
@@ -275,7 +294,10 @@ func polishList(roughMap *map[string]*Entry) []*Entry {
 	// Remove redundant subdomains for full/domain without attr
 	for _, qentry := range queuingList {
 		isRedundant := false
-		pd := qentry.Value // Parent domain
+		pd := qentry.Value // To be parent domain
+		if qentry.Type == RuleTypeFullDomain {
+			pd = "." + pd // So that `domain:example.org` overrides `full:example.org`
+		}
 		for {
 			idx := strings.Index(pd, ".")
 			if idx == -1 { break }
@@ -373,13 +395,16 @@ func main() {
 		}
 	}
 
-	// Generate finalMap
+	// Generate finalMap and sorted list of site names
+	siteList := make([]string, 0 ,len(plMap))
 	for _, pl := range plMap {
+		siteList = append(siteList, pl.Name)
 		if err := resolveList(pl); err != nil {
 			fmt.Println("Failed to resolveList:", err)
 			os.Exit(1)
 		}
 	}
+	slices.Sort(siteList)
 
 	// Create output directory if not exist
 	if _, err := os.Stat(*outputDir); os.IsNotExist(err) {
@@ -393,28 +418,31 @@ func main() {
 	if *exportLists != "" {
 		exportedListSlice := strings.Split(*exportLists, ",")
 		for _, exportedList := range exportedListSlice {
+			if exportedList == "_all_" {
+				if err := writePlainAll(&siteList); err != nil {
+					fmt.Println("Failed to writePlainAll:", err)
+					continue
+				}
+			} else {
 				if err := writePlainList(exportedList); err != nil {
 					fmt.Println("Failed to write list:", err)
 					continue
 				}
+			}
 			fmt.Printf("list: '%s' has been generated successfully.\n", exportedList)
 		}
 	}
 
 	// Generate dat file
 	protoList := new(router.GeoSiteList)
-	for siteName, siteEntries := range finalMap {
+	for _, siteName := range siteList { // So that protoList.Entry is sorted
-		site, err := makeProtoList(siteName, siteEntries)
+		site, err := makeProtoList(siteName, finalMap[siteName])
 		if err != nil {
-			fmt.Println("Failed:", err)
+			fmt.Println("Failed to makeProtoList:", err)
 			os.Exit(1)
 		}
 		protoList.Entry = append(protoList.Entry, site)
 	}
-	// Sort protoList so the marshaled list is reproducible
-	slices.SortFunc(protoList.Entry, func(a, b *router.GeoSite) int {
-		return strings.Compare(a.CountryCode, b.CountryCode)
-	})
 
 	protoBytes, err := proto.Marshal(protoList)
 	if err != nil {