From bd122478fc4f3a9ccd02f807a16fbbfb3c72fc5c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=96=E7=95=8C?= Date: Wed, 17 Jul 2024 17:57:35 +0800 Subject: [PATCH] Bump rule-set version --- cmd/sing-box/cmd_geoip_export.go | 2 +- cmd/sing-box/cmd_geosite_export.go | 2 +- cmd/sing-box/cmd_rule_set_compile.go | 6 +- cmd/sing-box/cmd_rule_set_upgrade.go | 91 ++++++++++++++++++++ common/srs/binary.go | 38 ++++---- constant/rule.go | 6 +- docs/configuration/rule-set/source-format.md | 21 ++++- option/rule_set.go | 6 +- route/rule_item_domain.go | 2 +- 9 files changed, 145 insertions(+), 29 deletions(-) create mode 100644 cmd/sing-box/cmd_rule_set_upgrade.go diff --git a/cmd/sing-box/cmd_geoip_export.go b/cmd/sing-box/cmd_geoip_export.go index 5787d2e5..b80e5cd3 100644 --- a/cmd/sing-box/cmd_geoip_export.go +++ b/cmd/sing-box/cmd_geoip_export.go @@ -87,7 +87,7 @@ func geoipExport(countryCode string) error { headlessRule.IPCIDR = append(headlessRule.IPCIDR, cidr.String()) } var plainRuleSet option.PlainRuleSetCompat - plainRuleSet.Version = C.RuleSetVersion1 + plainRuleSet.Version = C.RuleSetVersion2 plainRuleSet.Options.Rules = []option.HeadlessRule{ { Type: C.RuleTypeDefault, diff --git a/cmd/sing-box/cmd_geosite_export.go b/cmd/sing-box/cmd_geosite_export.go index 2a6c27a0..90a7955b 100644 --- a/cmd/sing-box/cmd_geosite_export.go +++ b/cmd/sing-box/cmd_geosite_export.go @@ -70,7 +70,7 @@ func geositeExport(category string) error { headlessRule.DomainKeyword = defaultRule.DomainKeyword headlessRule.DomainRegex = defaultRule.DomainRegex var plainRuleSet option.PlainRuleSetCompat - plainRuleSet.Version = C.RuleSetVersion1 + plainRuleSet.Version = C.RuleSetVersion2 plainRuleSet.Options.Rules = []option.HeadlessRule{ { Type: C.RuleTypeDefault, diff --git a/cmd/sing-box/cmd_rule_set_compile.go b/cmd/sing-box/cmd_rule_set_compile.go index 6e065101..7e3753c9 100644 --- a/cmd/sing-box/cmd_rule_set_compile.go +++ b/cmd/sing-box/cmd_rule_set_compile.go @@ -6,6 +6,7 @@ import ( "strings" "github.com/sagernet/sing-box/common/srs" + C "github.com/sagernet/sing-box/constant" "github.com/sagernet/sing-box/log" "github.com/sagernet/sing-box/option" "github.com/sagernet/sing/common/json" @@ -55,9 +56,6 @@ func compileRuleSet(sourcePath string) error { if err != nil { return err } - if err != nil { - return err - } ruleSet := plainRuleSet.Upgrade() var outputPath string if flagRuleSetCompileOutput == flagRuleSetCompileDefaultOutput { @@ -73,7 +71,7 @@ func compileRuleSet(sourcePath string) error { if err != nil { return err } - err = srs.Write(outputFile, ruleSet) + err = srs.Write(outputFile, ruleSet, plainRuleSet.Version == C.RuleSetVersion2) if err != nil { outputFile.Close() os.Remove(outputPath) diff --git a/cmd/sing-box/cmd_rule_set_upgrade.go b/cmd/sing-box/cmd_rule_set_upgrade.go new file mode 100644 index 00000000..0ec039fd --- /dev/null +++ b/cmd/sing-box/cmd_rule_set_upgrade.go @@ -0,0 +1,91 @@ +package main + +import ( + "bytes" + "io" + "os" + "path/filepath" + + C "github.com/sagernet/sing-box/constant" + "github.com/sagernet/sing-box/log" + "github.com/sagernet/sing-box/option" + E "github.com/sagernet/sing/common/exceptions" + "github.com/sagernet/sing/common/json" + + "github.com/spf13/cobra" +) + +var commandRuleSetUpgradeFlagWrite bool + +var commandRuleSetUpgrade = &cobra.Command{ + Use: "upgrade ", + Short: "Upgrade rule-set json", + Args: cobra.ExactArgs(1), + Run: func(cmd *cobra.Command, args []string) { + err := upgradeRuleSet(args[0]) + if err != nil { + log.Fatal(err) + } + }, +} + +func init() { + commandRuleSetUpgrade.Flags().BoolVarP(&commandRuleSetUpgradeFlagWrite, "write", "w", false, "write result to (source) file instead of stdout") + commandRuleSet.AddCommand(commandRuleSetUpgrade) +} + +func upgradeRuleSet(sourcePath string) error { + var ( + reader io.Reader + err error + ) + if sourcePath == "stdin" { + reader = os.Stdin + } else { + reader, err = os.Open(sourcePath) + if err != nil { + return err + } + } + content, err := io.ReadAll(reader) + if err != nil { + return err + } + plainRuleSetCompat, err := json.UnmarshalExtended[option.PlainRuleSetCompat](content) + if err != nil { + return err + } + switch plainRuleSetCompat.Version { + case C.RuleSetVersion1: + default: + log.Info("already up-to-date") + return nil + } + plainRuleSet := plainRuleSetCompat.Upgrade() + buffer := new(bytes.Buffer) + encoder := json.NewEncoder(buffer) + encoder.SetIndent("", " ") + err = encoder.Encode(plainRuleSet) + if err != nil { + return E.Cause(err, "encode config") + } + outputPath, _ := filepath.Abs(sourcePath) + if !commandRuleSetUpgradeFlagWrite || sourcePath == "stdin" { + os.Stdout.WriteString(buffer.String() + "\n") + return nil + } + if bytes.Equal(content, buffer.Bytes()) { + return nil + } + output, err := os.Create(sourcePath) + if err != nil { + return E.Cause(err, "open output") + } + _, err = output.Write(buffer.Bytes()) + output.Close() + if err != nil { + return E.Cause(err, "write output") + } + os.Stderr.WriteString(outputPath + "\n") + return nil +} diff --git a/common/srs/binary.go b/common/srs/binary.go index c7c55e08..0bd5a909 100644 --- a/common/srs/binary.go +++ b/common/srs/binary.go @@ -54,14 +54,14 @@ func Read(reader io.Reader, recover bool) (ruleSet option.PlainRuleSet, err erro if err != nil { return ruleSet, err } - if version != 1 { + if version > C.RuleSetVersion2 { return ruleSet, E.New("unsupported version: ", version) } - zReader, err := zlib.NewReader(reader) + compressReader, err := zlib.NewReader(reader) if err != nil { return } - bReader := bufio.NewReader(zReader) + bReader := bufio.NewReader(compressReader) length, err := binary.ReadUvarint(bReader) if err != nil { return @@ -77,26 +77,32 @@ func Read(reader io.Reader, recover bool) (ruleSet option.PlainRuleSet, err erro return } -func Write(writer io.Writer, ruleSet option.PlainRuleSet) error { +func Write(writer io.Writer, ruleSet option.PlainRuleSet, generateUnstable bool) error { _, err := writer.Write(MagicBytes[:]) if err != nil { return err } - err = binary.Write(writer, binary.BigEndian, uint8(1)) + var version uint8 + if generateUnstable { + version = C.RuleSetVersion2 + } else { + version = C.RuleSetVersion1 + } + err = binary.Write(writer, binary.BigEndian, version) if err != nil { return err } - zWriter, err := zlib.NewWriterLevel(writer, zlib.BestCompression) + compressWriter, err := zlib.NewWriterLevel(writer, zlib.BestCompression) if err != nil { return err } - bWriter := bufio.NewWriter(zWriter) + bWriter := bufio.NewWriter(compressWriter) _, err = varbin.WriteUvarint(bWriter, uint64(len(ruleSet.Rules))) if err != nil { return err } for _, rule := range ruleSet.Rules { - err = writeRule(bWriter, rule) + err = writeRule(bWriter, rule, generateUnstable) if err != nil { return err } @@ -105,7 +111,7 @@ func Write(writer io.Writer, ruleSet option.PlainRuleSet) error { if err != nil { return err } - return zWriter.Close() + return compressWriter.Close() } func readRule(reader varbin.Reader, recover bool) (rule option.HeadlessRule, err error) { @@ -127,12 +133,12 @@ func readRule(reader varbin.Reader, recover bool) (rule option.HeadlessRule, err return } -func writeRule(writer varbin.Writer, rule option.HeadlessRule) error { +func writeRule(writer varbin.Writer, rule option.HeadlessRule, generateUnstable bool) error { switch rule.Type { case C.RuleTypeDefault: - return writeDefaultRule(writer, rule.DefaultOptions) + return writeDefaultRule(writer, rule.DefaultOptions, generateUnstable) case C.RuleTypeLogical: - return writeLogicalRule(writer, rule.LogicalOptions) + return writeLogicalRule(writer, rule.LogicalOptions, generateUnstable) default: panic("unknown rule type: " + rule.Type) } @@ -219,7 +225,7 @@ func readDefaultRule(reader varbin.Reader, recover bool) (rule option.DefaultHea } } -func writeDefaultRule(writer varbin.Writer, rule option.DefaultHeadlessRule) error { +func writeDefaultRule(writer varbin.Writer, rule option.DefaultHeadlessRule, generateUnstable bool) error { err := binary.Write(writer, binary.BigEndian, uint8(0)) if err != nil { return err @@ -243,7 +249,7 @@ func writeDefaultRule(writer varbin.Writer, rule option.DefaultHeadlessRule) err if err != nil { return err } - err = domain.NewMatcher(rule.Domain, rule.DomainSuffix).Write(writer) + err = domain.NewMatcher(rule.Domain, rule.DomainSuffix, !generateUnstable).Write(writer) if err != nil { return err } @@ -420,7 +426,7 @@ func readLogicalRule(reader varbin.Reader, recovery bool) (logicalRule option.Lo return } -func writeLogicalRule(writer varbin.Writer, logicalRule option.LogicalHeadlessRule) error { +func writeLogicalRule(writer varbin.Writer, logicalRule option.LogicalHeadlessRule, generateUnstable bool) error { err := binary.Write(writer, binary.BigEndian, uint8(1)) if err != nil { return err @@ -441,7 +447,7 @@ func writeLogicalRule(writer varbin.Writer, logicalRule option.LogicalHeadlessRu return err } for _, rule := range logicalRule.Rules { - err = writeRule(writer, rule) + err = writeRule(writer, rule, generateUnstable) if err != nil { return err } diff --git a/constant/rule.go b/constant/rule.go index 5a8eaf12..fb0f39e2 100644 --- a/constant/rule.go +++ b/constant/rule.go @@ -13,7 +13,11 @@ const ( const ( RuleSetTypeLocal = "local" RuleSetTypeRemote = "remote" - RuleSetVersion1 = 1 RuleSetFormatSource = "source" RuleSetFormatBinary = "binary" ) + +const ( + RuleSetVersion1 = 1 + iota + RuleSetVersion2 +) diff --git a/docs/configuration/rule-set/source-format.md b/docs/configuration/rule-set/source-format.md index ee5e48e0..8c46cbf0 100644 --- a/docs/configuration/rule-set/source-format.md +++ b/docs/configuration/rule-set/source-format.md @@ -1,12 +1,20 @@ +--- +icon: material/new-box +--- + # Source Format +!!! quote "Changes in sing-box 1.10.0" + + :material-plus: version `2` + !!! question "Since sing-box 1.8.0" ### Structure ```json { - "version": 1, + "version": 2, "rules": [] } ``` @@ -21,7 +29,16 @@ Use `sing-box rule-set compile [--output .srs] .json` to c ==Required== -Version of Rule Set, must be `1`. +Version of rule-set, one of `1` or `2`. + +* 1: Initial rule-set version, since sing-box 1.8.0. +* 2: Optimized memory usages of `domain_suffix` rules. + +The new rule-set version `2` does not make any changes to the format, only affecting `binary` rule-sets compiled by command `rule-set compile` + +Since 1.10.0, the optimization is always applied to `source` rule-sets even if version is set to `1`. + +It is recommended to upgrade to `2` after sing-box 1.10.0 becomes a stable version. #### rules diff --git a/option/rule_set.go b/option/rule_set.go index 8e367e69..5498400f 100644 --- a/option/rule_set.go +++ b/option/rule_set.go @@ -185,7 +185,7 @@ type PlainRuleSetCompat _PlainRuleSetCompat func (r PlainRuleSetCompat) MarshalJSON() ([]byte, error) { var v any switch r.Version { - case C.RuleSetVersion1: + case C.RuleSetVersion1, C.RuleSetVersion2: v = r.Options default: return nil, E.New("unknown rule set version: ", r.Version) @@ -200,7 +200,7 @@ func (r *PlainRuleSetCompat) UnmarshalJSON(bytes []byte) error { } var v any switch r.Version { - case C.RuleSetVersion1: + case C.RuleSetVersion1, C.RuleSetVersion2: v = &r.Options case 0: return E.New("missing rule set version") @@ -217,7 +217,7 @@ func (r *PlainRuleSetCompat) UnmarshalJSON(bytes []byte) error { func (r PlainRuleSetCompat) Upgrade() PlainRuleSet { var result PlainRuleSet switch r.Version { - case C.RuleSetVersion1: + case C.RuleSetVersion1, C.RuleSetVersion2: result = r.Options default: panic("unknown rule set version: " + F.ToString(r.Version)) diff --git a/route/rule_item_domain.go b/route/rule_item_domain.go index 36839a55..c77890df 100644 --- a/route/rule_item_domain.go +++ b/route/rule_item_domain.go @@ -38,7 +38,7 @@ func NewDomainItem(domains []string, domainSuffixes []string) *DomainItem { } } return &DomainItem{ - domain.NewMatcher(domains, domainSuffixes), + domain.NewMatcher(domains, domainSuffixes, false), description, } }