refacter project

.gitignore

@@ -171,5 +171,5 @@ poetry.toml
 # End of https://www.toptal.com/developers/gitignore/api/python
 
 /.idea
-/custom_proxy_hosts.json
+/cache
-
+/custom_proxy_hosts.json

dnscrypt_to_smartdns.py

@@ -0,0 +1,84 @@
+import subprocess
+import logging
+from urllib.request import urlopen
+
+from utils.dnsstamps import parse, DNSoverHTTPS
+
+SMARTDNS_GFW_CONF_FILE = '/etc/smartdns/conf.d/gfw.conf'
+
+# https://github.com/DNSCrypt/dnscrypt-resolvers
+PUBLIC_RESOLVER_URL_LIST = [
+    "https://download.dnscrypt.info/resolvers-list/v3/public-resolvers.md",
+    "https://raw.githubusercontent.com/DNSCrypt/dnscrypt-resolvers/master/v3/public-resolvers.md"
+    "https://dnsr.evilvibes.com/v3/public-resolvers.md"
+]
+
+
+def get_public_resolver_md() -> str:
+    for url in PUBLIC_RESOLVER_URL_LIST:
+        try:
+            logging.info('request {url}'.format(url=url))
+            with urlopen(url, timeout=15) as responsee:
+                return responsee.read().decode('utf-8')
+        except:
+            pass
+    raise IOError("can't download public-resolvers.md")
+
+
+def get_stamps():
+    resolver_md: str = get_public_resolver_md()
+    lines = resolver_md.splitlines()
+    stamps = list(
+        map(
+            parse,
+            filter(lambda x: x.startswith('sdns://'), lines)
+        )
+    )
+    return stamps
+
+
+def get_not_china_doh_list():
+    stamps = get_stamps()
+
+    def is_match(s) -> bool:
+        if isinstance(s, DNSoverHTTPS) is False:
+            return False
+        if s.nofilter is False:
+            return False
+        if s.dnssec is False:
+            return False
+        return True
+
+    return list(filter(
+        is_match,
+        stamps
+    ))
+
+
+def get_smartdns_config():
+    stamps = get_not_china_doh_list()
+    lines = map(
+        lambda x: 'server-https https://' + x.hostname + x.path + ' -group GFW -exclude-default-group',
+        stamps
+    )
+    return '\n'.join(lines)
+
+
+def write_smartdns_config():
+    conf_txt = get_smartdns_config()
+    with open(SMARTDNS_GFW_CONF_FILE, 'w') as f:
+        f.write(conf_txt)
+
+
+def reload_smartdns():
+    subprocess.run(["/etc/init.d/smartdns", "reload"])
+
+
+def main():
+    write_smartdns_config()
+    reload_smartdns()
+
+
+if __name__ == '__main__':
+    s = get_smartdns_config()
+    print(s)

dnsmasq-china-list.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+PROXY="socks5h://127.0.0.1:1080"
+
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+CACHE_FOLDER="${SCRIPT_DIR}/cache"
+
+mkdir -p "${CACHE_FOLDER}"
+
+curl --proxy "${PROXY}" https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/refs/heads/master/accelerated-domains.china.conf -o "${CACHE_FOLDER}/accelerated-domains.china.conf"
+curl --proxy "${PROXY}" https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/refs/heads/master/google.china.conf -o "${CACHE_FOLDER}/google.china.conf"
+curl --proxy "${PROXY}" https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/refs/heads/master/apple.china.conf -o "${CACHE_FOLDER}/apple.china.conf"
+curl --proxy "${PROXY}" https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/refs/heads/master/bogus-nxdomain.china.conf -o "${CACHE_FOLDER}/bogus-nxdomain.china.conf"
+
+cp "${CACHE_FOLDER}/*.conf" /tmp/dnsmasq.d/
+grep -v '^#' "${CACHE_FOLDER}/bogus-nxdomain.china.conf" | grep -v '^$' | sed -e 's/=/ /g' > /etc/smartdns/conf.d/bogus-nxdomain.conf

gfwlist_to_dns.py

@@ -10,7 +10,8 @@ from urllib.request import urlopen
 PROXY_DNS_IP = '127.0.0.1'
 PROXY_DNS_PORT = '5353'
 
-DNSMASQ_RULES_FILE = '/tmp/dnsmasq.d/gfwlist'
+DNSMASQ_RULES_FILE = '/tmp/dnsmasq.d/gfwlist.conf'
+SMARTDNS_DOMAIN_SET_FILE = '/etc/smartdns/domain-set/gfwlist.conf'
 
 # https://github.com/gfwlist/gfwlist
 GFWLIST_URL_LIST = [
@@ -214,11 +215,33 @@ def get_dnsmasq_text() -> str:
     return '\n'.join(rule_list)
 
 
-def main():
+def write_dnsmasq():
     dnsmasq_text = get_dnsmasq_text()
     with open(DNSMASQ_RULES_FILE, 'w') as f:
         f.write(dnsmasq_text)
-    subprocess.run(["/etc/init.d/dnsmasq", "restart"])
+
+
+def reload_dnsmasq():
+    subprocess.run(["/etc/init.d/dnsmasq", "reload"])
+
+
+def get_smartdns_domain_set() -> str:
+    return '\n'.join(get_proxy_hosts())
+
+
+def write_smartdns_domain_set():
+    domain_set_text = get_smartdns_domain_set()
+    with open(SMARTDNS_DOMAIN_SET_FILE, 'w') as f:
+        f.write(domain_set_text)
+
+
+def reload_smartdns():
+    subprocess.run(["/etc/init.d/smartdns", "reload"])
+
+
+def main():
+    write_smartdns_domain_set()
+    reload_smartdns()
 
 
 if __name__ == '__main__':

update.sh

@@ -1,8 +0,0 @@
-#!/bin/bash
-
-cd /srv/app/gfw/
-
-yq "." pac/config/custom.yaml > gfwlist-to-dnsmasq-rule/custom_proxy_hosts.json
-
-rsync --verbose gfwlist-to-dnsmasq-rule/{main.py,custom_proxy_hosts.json} root-openwrt.rel.bgme.org:/root/gfwlist-to-dnsmasq-rule
-ssh root-openwrt.rel.bgme.org "/usr/bin/python /root/gfwlist-to-dnsmasq-rule/main.py"

utils/dnsstamps.py

@@ -0,0 +1,537 @@
+import base64
+
+
+def parse(stamp: str):
+    b = base64.urlsafe_b64decode(stamp.removeprefix("sdns://") + "==")
+    if b[0] == 0x01:
+        return DNSCrypt.parse(stamp)
+    elif b[0] == 0x02:
+        return DNSoverHTTPS.parse(stamp)
+    elif b[0] == 0x03:
+        return DNSoverTLS.parse(stamp)
+    elif b[0] == 0x04:
+        return DNSoverQUIC.parse(stamp)
+    elif b[0] == 0x05:
+        return ObliviousDoH.parse(stamp)
+    elif b[0] == 0x81:
+        return DNSCryptRelay.parse(stamp)
+    elif b[0] == 0x85:
+        return ObliviousDoHrelay.parse(stamp)
+    elif b[0] == 0x00:
+        return PlainDNS.parse(stamp)
+
+
+# Code from https://github.com/DNSCrypt/dnscrypt-resolvers/blob/21fcbaf858112c63fed2a504714cc829bd654483/utils/format.py#L101-L141
+class DNSCrypt:
+    dnssec: bool = False
+    nolog: bool = False
+    nofilter: bool = False
+    addr: str = None
+    pk: str = None
+    provider: str = None
+
+    @staticmethod
+    def parse(stamp: str):
+        b = base64.urlsafe_b64decode(stamp.removeprefix("sdns://") + "==")
+
+        # 0x01
+        i = 0
+        if b[i] != 0x01:
+            raise ValueError()
+        i = i + 1
+
+        parsed = DNSCrypt()
+
+        # props
+        props = b[i]
+        parsed.dnssec = not not ((props >> 0) & 1)
+        parsed.nolog = not not ((props >> 1) & 1)
+        parsed.nofilter = not not ((props >> 2) & 1)
+        i = i + 8
+
+        # LP(addr [:port])
+        addr_len = b[i]
+        i = i + 1
+        parsed.addr = b[i:i + addr_len].decode("utf-8")
+        i = i + addr_len
+
+        # LP(pk)
+        pk_len = b[i]
+        i = i + 1
+        if pk_len != 32:
+            raise ValueError()
+        hpk = b[i:i + pk_len].hex().upper()
+        hpks = []
+        for j in range(0, 16):
+            hpks.append(hpk[j * 4: j * 4 + 4])
+        parsed.pk = ":".join(hpks)
+        i = i + pk_len
+
+        # LP(providerName)
+        provider_len = b[i]
+        i = i + 1
+        parsed.provider = b[i:i + provider_len].decode("utf-8")
+        i = i + provider_len
+
+        return parsed
+
+
+class DNSoverHTTPS:
+    dnssec: bool = False
+    nolog: bool = False
+    nofilter: bool = False
+    addr: str = None
+    hashs: list[str] = []
+    hostname: str = None
+    path: str = None
+    bootstrap_ipi: list[str] = []
+
+    def parse(stamp: str):
+        b = base64.urlsafe_b64decode(stamp.removeprefix("sdns://") + "==")
+
+        # 0x02
+        i = 0
+        if b[i] != 0x02:
+            raise ValueError()
+        i = i + 1
+
+        parsed = DNSoverHTTPS()
+
+        # props
+        props = b[i]
+        parsed.dnssec = not not ((props >> 0) & 1)
+        parsed.nolog = not not ((props >> 1) & 1)
+        parsed.nofilter = not not ((props >> 2) & 1)
+        i = i + 8
+
+        # LP(addr)
+        addr_len = b[i]
+        i = i + 1
+        parsed.addr = b[i:i + addr_len].decode("utf-8")
+        i = i + addr_len
+
+        # VLP(hash1, hash2, ...hashn)
+        last_element = False
+        while True:
+            if b[i] & 0x80 == 0:
+                last_element = True
+                hashx_len = b[i]
+            else:
+                hashx_len = b[i] ^ 0x80
+            if hashx_len != 0 and hashx_len != 32:
+                raise ValueError()
+            i = i + 1
+            if hashx_len > 0:
+                hashx = b[i:i + hashx_len].hex()
+                parsed.hashs.append(hashx)
+                i = i + hashx_len
+            if last_element:
+                break
+
+        # LP(hostname [:port])
+        hostname_len = b[i]
+        i = i + 1
+        parsed.hostname = b[i:i + hostname_len].decode("utf-8")
+        i = i + hostname_len
+
+        # LP(path)
+        path_len = b[i]
+        i = i + 1
+        parsed.path = b[i:i + path_len].decode("utf-8")
+        i = i + path_len
+
+        # VLP(bootstrap_ip1, bootstrap_ip2, ...bootstrap_ipn) (optional)
+        if i < len(b):
+            last_element = False
+            while True:
+                if b[i] & 0x80 == 0:
+                    last_element = True
+                    bootstrap_ipx_len = b[i]
+                else:
+                    bootstrap_ipx_len = b[i] ^ 0x80
+                i = i + 1
+                if bootstrap_ipx_len > 0:
+                    bootstrap_ipx = b[i:i + bootstrap_ipx_len].decode("utf-8")
+                    parsed.bootstrap_ipi.append(bootstrap_ipx)
+                    i = i + bootstrap_ipx_len
+                if last_element:
+                    break
+
+        return parsed
+
+
+class DNSoverTLS:
+    dnssec: bool = False
+    nolog: bool = False
+    nofilter: bool = False
+    addr: str = None
+    hashs: list[str] = []
+    hostname: str = None
+    bootstrap_ipi: list[str] = []
+
+    @staticmethod
+    def parse(stamp: str):
+        b = base64.urlsafe_b64decode(stamp.removeprefix("sdns://") + "==")
+
+        # 0x03
+        i = 0
+        if b[i] != 0x03:
+            raise ValueError()
+        i = i + 1
+
+        parsed = DNSoverTLS()
+
+        # props
+        props = b[i]
+        parsed.dnssec = not not ((props >> 0) & 1)
+        parsed.nolog = not not ((props >> 1) & 1)
+        parsed.nofilter = not not ((props >> 2) & 1)
+        i = i + 8
+
+        # LP(addr)
+        addr_len = b[i]
+        i = i + 1
+        parsed.addr = b[i:i + addr_len].decode("utf-8")
+        i = i + addr_len
+
+        # VLP(hash1, hash2, ...hashn)
+        last_element = False
+        while True:
+            if b[i] & 0x80 == 0:
+                last_element = True
+                hashx_len = b[i]
+            else:
+                hashx_len = b[i] ^ 0x80
+            if hashx_len != 0 and hashx_len != 32:
+                raise ValueError()
+            i = i + 1
+            if hashx_len > 0:
+                hashx = b[i:i + hashx_len].hex()
+                parsed.hashs.append(hashx)
+                i = i + hashx_len
+            if last_element:
+                break
+
+        # LP(hostname[:port])
+        hostname_len = b[i]
+        i = i + 1
+        parsed.hostname = b[i:i + hostname_len].decode("utf-8")
+        i = i + hostname_len
+
+        # VLP(bootstrap_ip1, bootstrap_ip2, ...bootstrap_ipn) (optional)
+        if i < len(b):
+            last_element = False
+            while True:
+                if b[i] & 0x80 == 0:
+                    last_element = True
+                    bootstrap_ipx_len = b[i]
+                else:
+                    bootstrap_ipx_len = b[i] ^ 0x80
+                i = i + 1
+                if bootstrap_ipx_len > 0:
+                    bootstrap_ipx = b[i:i + bootstrap_ipx_len].decode("utf-8")
+                    parsed.bootstrap_ipi.append(bootstrap_ipx)
+                    i = i + bootstrap_ipx_len
+                if last_element:
+                    break
+
+        return parsed
+
+
+class DNSoverQUIC:
+    dnssec: bool = False
+    nolog: bool = False
+    nofilter: bool = False
+    addr: str = None
+    hashs: list[str] = []
+    hostname: str = None
+    bootstrap_ipi: list[str] = []
+
+    @staticmethod
+    def parse(stamp: str):
+        b = base64.urlsafe_b64decode(stamp.removeprefix("sdns://") + "==")
+
+        # 0x04
+        i = 0
+        if b[i] != 0x04:
+            raise ValueError()
+        i = i + 1
+
+        parsed = DNSoverQUIC()
+
+        # props
+        props = b[i]
+        parsed.dnssec = not not ((props >> 0) & 1)
+        parsed.nolog = not not ((props >> 1) & 1)
+        parsed.nofilter = not not ((props >> 2) & 1)
+        i = i + 8
+
+        # LP(addr)
+        addr_len = b[i]
+        i = i + 1
+        parsed.addr = b[i:i + addr_len].decode("utf-8")
+        i = i + addr_len
+
+        # VLP(hash1, hash2, ...hashn)
+        last_element = False
+        while True:
+            if b[i] & 0x80 == 0:
+                last_element = True
+                hashx_len = b[i]
+            else:
+                hashx_len = b[i] ^ 0x80
+            if hashx_len != 0 and hashx_len != 32:
+                raise ValueError()
+            i = i + 1
+            if hashx_len > 0:
+                hashx = b[i:i + hashx_len].hex()
+                parsed.hashs.append(hashx)
+                i = i + hashx_len
+            if last_element:
+                break
+
+        # LP(hostname[:port])
+        hostname_len = b[i]
+        i = i + 1
+        parsed.hostname = b[i:i + hostname_len].decode("utf-8")
+        i = i + hostname_len
+
+        # VLP(bootstrap_ip1, bootstrap_ip2, ...bootstrap_ipn) (optional)
+        if i < len(b):
+            last_element = False
+            while True:
+                if b[i] & 0x80 == 0:
+                    last_element = True
+                    bootstrap_ipx_len = b[i]
+                else:
+                    bootstrap_ipx_len = b[i] ^ 0x80
+                i = i + 1
+                if bootstrap_ipx_len > 0:
+                    bootstrap_ipx = b[i:i + bootstrap_ipx_len].decode("utf-8")
+                    parsed.bootstrap_ipi.append(bootstrap_ipx)
+                    i = i + bootstrap_ipx_len
+                if last_element:
+                    break
+
+        return parsed
+
+
+class ObliviousDoH:
+    dnssec: bool = False
+    nolog: bool = False
+    nofilter: bool = False
+    hostname: str = None
+    path: str = None
+
+    @staticmethod
+    def parse(stamp: str):
+        b = base64.urlsafe_b64decode(stamp.removeprefix("sdns://") + "==")
+
+        # 0x05
+        i = 0
+        if b[i] != 0x05:
+            raise ValueError()
+        i = i + 1
+
+        parsed = ObliviousDoH()
+
+        # props
+        props = b[i]
+        parsed.dnssec = not not ((props >> 0) & 1)
+        parsed.nolog = not not ((props >> 1) & 1)
+        parsed.nofilter = not not ((props >> 2) & 1)
+        i = i + 8
+
+        # LP(hostname [:port])
+        hostname_len = b[i]
+        i = i + 1
+        parsed.hostname = b[i:i + hostname_len].decode("utf-8")
+        i = i + hostname_len
+
+        # LP(path)
+        path_len = b[i]
+        i = i + 1
+        parsed.path = b[i:i + path_len].decode("utf-8")
+        i = i + path_len
+
+        return parsed
+
+
+class DNSCryptRelay:
+    addr: str = None
+
+    @staticmethod
+    def parse(stamp):
+        b = base64.urlsafe_b64decode(stamp.removeprefix("sdns://") + "==")
+
+        # 0x81
+        i = 0
+        if b[i] != 0x81:
+            raise ValueError()
+        i = i + 1
+
+        parsed = DNSCryptRelay()
+
+        # LP(addr)
+        addr_len = b[i]
+        i = i + 1
+        parsed.addr = b[i:i + addr_len].decode("utf-8")
+        i = i + addr_len
+
+        return parsed
+
+
+class ObliviousDoHrelay:
+    dnssec: bool = False
+    nolog: bool = False
+    nofilter: bool = False
+    addr: str = None
+    hashs: list[str] = []
+    hostname: str = None
+    path: str = None
+    bootstrap_ipi: list[str] = []
+
+    @staticmethod
+    def parse(stamp: str):
+        b = base64.urlsafe_b64decode(stamp.removeprefix("sdns://") + "==")
+
+        # 0x85
+        i = 0
+        if b[i] != 0x85:
+            raise ValueError()
+        i = i + 1
+
+        parsed = ObliviousDoHrelay()
+
+        # props
+        props = b[i]
+        parsed.dnssec = not not ((props >> 0) & 1)
+        parsed.nolog = not not ((props >> 1) & 1)
+        parsed.nofilter = not not ((props >> 2) & 1)
+        i = i + 8
+
+        # LP(addr)
+        addr_len = b[i]
+        i = i + 1
+        parsed.addr = b[i:i + addr_len].decode("utf-8")
+        i = i + addr_len
+
+        # VLP(hash1, hash2, ...hashn)
+        last_element = False
+        while True:
+            if b[i] & 0x80 == 0:
+                last_element = True
+                hashx_len = b[i]
+            else:
+                hashx_len = b[i] ^ 0x80
+            if hashx_len != 0 and hashx_len != 32:
+                raise ValueError()
+            i = i + 1
+            if hashx_len > 0:
+                hashx = b[i:i + hashx_len].hex()
+                parsed.hashs.append(hashx)
+                i = i + hashx_len
+            if last_element:
+                break
+
+        # LP(hostname [:port])
+        hostname_len = b[i]
+        i = i + 1
+        parsed.hostname = b[i:i + hostname_len].decode("utf-8")
+        i = i + hostname_len
+
+        # LP(path)
+        path_len = b[i]
+        i = i + 1
+        parsed.path = b[i:i + path_len].decode("utf-8")
+        i = i + path_len
+
+        # VLP(bootstrap_ip1, bootstrap_ip2, ...bootstrap_ipn) (optional)
+        if i < len(b):
+            last_element = False
+            while True:
+                if b[i] & 0x80 == 0:
+                    last_element = True
+                    bootstrap_ipx_len = b[i]
+                else:
+                    bootstrap_ipx_len = b[i] ^ 0x80
+                i = i + 1
+                if bootstrap_ipx_len > 0:
+                    bootstrap_ipx = b[i:i + bootstrap_ipx_len].decode("utf-8")
+                    parsed.bootstrap_ipi.append(bootstrap_ipx)
+                    i = i + bootstrap_ipx_len
+                if last_element:
+                    break
+
+        return parsed
+
+
+class PlainDNS:
+    dnssec: bool = False
+    nolog: bool = False
+    nofilter: bool = False
+    addr: str = None
+
+    @staticmethod
+    def parse(stamp: str):
+        b = base64.urlsafe_b64decode(stamp.removeprefix("sdns://") + "==")
+
+        # 0x00
+        i = 0
+        if b[i] != 0x00:
+            raise ValueError()
+        i = i + 1
+
+        parsed = PlainDNS()
+
+        # props
+        props = b[i]
+        parsed.dnssec = not not ((props >> 0) & 1)
+        parsed.nolog = not not ((props >> 1) & 1)
+        parsed.nofilter = not not ((props >> 2) & 1)
+        i = i + 8
+
+        # LP(addr)
+        addr_len = b[i]
+        i = i + 1
+        parsed.addr = b[i:i + addr_len].decode("utf-8")
+        i = i + addr_len
+
+        return parsed
+
+
+if __name__ == '__main__':
+    # DNSCrypt
+    t = parse(
+        "sdns://AQMAAAAAAAAAETk0LjE0MC4xNC4xNTo1NDQzILgxXdexS27jIKRw3C7Wsao5jMnlhvhdRUXWuMm1AFq6ITIuZG5zY3J5cHQuZmFtaWx5Lm5zMS5hZGd1YXJkLmNvbQ")
+    print(t)
+
+    # DoH
+    t = parse("sdns://AgcAAAAAAAAADTIxNy4xNjkuMjAuMjIADWRucy5hYS5uZXQudWsKL2Rucy1xdWVyeQ")
+    t = parse(
+        "sdns://AgMAAAAAAAAADjE2My40Ny4xMTcuMTc2oMwQYNOcgym2K2-8fQ1t-TCYabmB5-Y5LVzY-kCPTYDmoPf1ryiAHod9ffOivij-FJ8ydKftKfE2_VA845jLqAsNoLNeBZUM-9gln5N1uhAYcLjDxMDsWlKXV-YxZ-neJqnooEROvWe7g_iAezkh6TiskXi4gr1QqtsRIx8ETPXwjffOoOZEumlj4zX-dly5l2sSsQ61QpS0JHd2TMs6OsyjrLL8ICquP7e_BeTIHEGU3KRFEdT5rzBHhuwa5yGECc9ioINVEGFkbC5hZGZpbHRlci5uZXQKL2Rucy1xdWVyeQ")
+    print(t)
+
+    # DoT
+    t = parse("sdns://AwcAAAAAAAAABzEuMS4xLjEAD29uZS5vbmUub25lLm9uZQ")
+    print(t)
+
+    # DoQ
+    t = parse("sdns://BAcAAAAAAAAABzEuMS4xLjEAD29uZS5vbmUub25lLm9uZQ")
+    print(t)
+
+    # oDoH
+    t = parse("sdns://BQcAAAAAAAAADWpwLnRpYXJhcC5vcmcFL29kb2g")
+    print(t)
+
+    #  DNSCrypt relay
+    t = parse("sdns://gQ04Ni4xMDYuNzQuMjE5")
+    print(t)
+
+    #  oDoH relay
+    t = parse("sdns://hQcAAAAAAAAADDg5LjM4LjEzMS4zOAAYb2RvaC1ubC5hbGVrYmVyZy5uZXQ6NDQzBi9wcm94eQ")
+    print(t)
+
+    # Plain DNS
+    t = parse("sdns://AAUAAAAAAAAABzEuMS4xLjE")
+    print(t)