feat: unicode domain names and entries

Author: martinheidegger

dnslink.go

@@ -6,10 +6,11 @@ import (
 	"math/rand"
 	"net"
 	"net/url"
+	"regexp"
 	"sort"
+	"strconv"
 	"strings"
 
-	isd "github.com/jbenet/go-is-domain"
 	dns "github.com/miekg/dns"
 )
 
@@ -112,7 +113,28 @@ func (s ByValue) Less(i, j int) bool { return s.LookupEntries[i].Value < s.Looku
 
 type LookupTXTFunc func(name string) (txt []LookupEntry, err error)
 
-func NewUDPLookup(servers []string) LookupTXTFunc {
+var utf8Replace = regexp.MustCompile(`\\\d{3}`)
+
+func utf8ReplaceFunc(input []byte) (result []byte) {
+	result = make([]byte, 1)
+	num, _ := strconv.ParseUint(string(input[1:]), 10, 9)
+	result[0] = byte(num)
+	return result
+}
+
+func utf8Value(input []string) string {
+	str := strings.Join(input, "")
+	return string(utf8Replace.ReplaceAllFunc([]byte(str), utf8ReplaceFunc))
+}
+
+func NewUDPLookup(servers []string, udpSize uint16) LookupTXTFunc {
+	client := new(dns.Client)
+	if udpSize == 0 {
+		// Running into issues with too small buffer size of dns library in some cases
+		client.UDPSize = 4096
+	} else {
+		client.UDPSize = udpSize
+	}
 	return func(domain string) (entries []LookupEntry, err error) {
 		if !strings.HasSuffix(domain, ".") {
 			domain += "."
@@ -127,7 +149,7 @@ func NewUDPLookup(servers []string) LookupTXTFunc {
 			Qclass: dns.ClassINET,
 		}
 		server := servers[rand.Intn(len(servers))]
-		res, err := dns.Exchange(req, server)
+		res, _, err := client.Exchange(req, server)
 		if err != nil {
 			return nil, err
 		}
@@ -136,7 +158,7 @@ func NewUDPLookup(servers []string) LookupTXTFunc {
 			if answer.Header().Rrtype == dns.TypeTXT {
 				txtAnswer := answer.(*dns.TXT)
 				entries[index] = LookupEntry{
-					Value: strings.Join(txtAnswer.Txt, ""),
+					Value: utf8Value(txtAnswer.Txt),
 					Ttl:   txtAnswer.Header().Ttl,
 				}
 			}
@@ -185,11 +207,11 @@ func resolve(r *Resolver, domain string, recursive bool) (result Result, err err
 	if lookupTXT == nil {
 		lookupTXT = defaultLookupTXT
 	}
-	lookup, error := validateDomain(domain)
+	lookup, error := validateDomain(domain, "")
 	result.Links = map[string][]LookupEntry{}
 	result.Path = []PathEntry{}
 	result.Log = []LogStatement{}[:]
-	if lookup == nil {
+	if error != nil {
 		result.Log = append(result.Log, *error)
 		return result, nil
 	}
@@ -225,7 +247,7 @@ func resolve(r *Resolver, domain string, recursive bool) (result Result, err err
 	}
 }
 
-func validateDomain(input string) (*URLParts, *LogStatement) {
+func validateDomain(input string, entry string) (*URLParts, *LogStatement) {
 	urlParts := relevantURLParts(input)
 	domain := urlParts.Domain
 	if strings.HasPrefix(domain, dnsPrefix) {
@@ -241,21 +263,85 @@ func validateDomain(input string) (*URLParts, *LogStatement) {
 			}
 		}
 	}
-	if !isd.IsDomain(domain) {
+	if !isFqdn(domain) {
 		return nil, &LogStatement{
-			Code:     "INVALID_REDIRECT",
-			Domain:   urlParts.Domain,
-			Pathname: urlParts.Pathname,
-			Search:   urlParts.Search,
+			Code:  "INVALID_REDIRECT",
+			Entry: entry,
 		}
 	}
+	domain = strings.TrimSuffix(domain, ".")
 	return &URLParts{
 		Domain:   dnsPrefix + domain,
 		Pathname: urlParts.Pathname,
 		Search:   urlParts.Search,
 	}, nil
 }
 
+var intlDomainCharset = regexp.MustCompile("^([a-z\u00a1-\uffff]{2,}|xn[a-z0-9-]{2,})$")
+var spacesAndSpecialChars = regexp.MustCompile("[\\s\u2002-\u200B\u202F\u205F\u3000��\u00A9\uFFFD\uFEFF]")
+var domainCharset = regexp.MustCompile("^[a-z\u00a1-\u00ff0-9-]+$")
+
+func isFqdn(str string) bool {
+	str = strings.TrimSuffix(str, ".")
+	if str == "" {
+		return false
+	}
+	parts := strings.Split(str, ".")
+	tld := parts[len(parts)-1]
+
+	// disallow fqdns without tld
+	if len(parts) < 2 {
+		return false
+	}
+
+	if !intlDomainCharset.MatchString(tld) {
+		return false
+	}
+
+	// disallow spaces && special characers
+	if spacesAndSpecialChars.MatchString(tld) {
+		return false
+	}
+
+	// disallow all numbers
+	if every(parts, isNumber) {
+		return false
+	}
+
+	return every(parts, isDomainPart)
+}
+
+func isDomainPart(part string) bool {
+	if len(part) > 63 {
+		return false
+	}
+
+	if !domainCharset.MatchString(part) {
+		return false
+	}
+
+	// disallow parts starting or ending with hyphen
+	if strings.HasPrefix(part, "-") || strings.HasSuffix(part, "-") {
+		return false
+	}
+
+	return true
+}
+
+func isNumber(str string) bool {
+	_, err := strconv.Atoi(str)
+	return err == nil
+}
+
+func every(strings []string, test func(string) bool) bool {
+	for _, str := range strings {
+		if !test(str) {
+			return false
+		}
+	}
+	return true
+}
+
 type processedEntry struct {
 	value string
 	entry string
@@ -331,9 +417,8 @@ func resolveTxtEntries(domain string, recursive bool, txtEntries []LookupEntry)
 		hasRedirect := false
 		var redirect *URLParts
 		for _, dns := range dnsLinks {
-			validated, error := validateDomain(dns.value)
+			validated, error := validateDomain(dns.value, dns.entry)
 			if error != nil {
-				delete(found, "dns")
 				log = append(log, *error)
 			} else if !hasRedirect {
 				hasRedirect = true
@@ -345,11 +430,9 @@ func resolveTxtEntries(domain string, recursive bool, txtEntries []LookupEntry)
 				})
 			}
 		}
+		delete(found, "dns")
 		if hasRedirect {
-			for key, foundEntries := range found {
-				if key == "dns" {
-					continue
-				}
+			for _, foundEntries := range found {
 				for _, foundEntry := range foundEntries {
 					log = append(log, LogStatement{
 						Code:  "UNUSED_ENTRY",

dnslink/main.go

@@ -321,7 +321,7 @@ func main() {
 	}
 	resolver := dnslink.Resolver{}
 	if options.has("dns") {
-		resolver.LookupTXT = dnslink.NewUDPLookup(getServers(options.get("dns")))
+		resolver.LookupTXT = dnslink.NewUDPLookup(getServers(options.get("dns")), 0)
 	}
 	for _, lookup := range lookups {
 		var result dnslink.Result

dnslink_test.go

@@ -73,19 +73,19 @@ func TestRelevantURLParts(t *testing.T) {
 func TestValidateDomain(t *testing.T) {
 	var logNil *LogStatement = nil
 	var partsNil *URLParts = nil
-	assertResult(t, arr(validateDomain("hello.com")),
+	assertResult(t, arr(validateDomain("hello.com", "")),
 		&URLParts{Domain: "_dnslink.hello.com", Pathname: "", Search: make(map[string][]string)},
 		logNil,
 	)
-	assertResult(t, arr(validateDomain("_dnslink.hello.com/foo?bar=baz")),
+	assertResult(t, arr(validateDomain("_dnslink.hello.com/foo?bar=baz", "")),
 		&URLParts{Domain: "_dnslink.hello.com", Pathname: "/foo", Search: map[string][]string{"bar": {"baz"}}},
 		logNil,
 	)
-	assertResult(t, arr(validateDomain("hello .com")),
+	assertResult(t, arr(validateDomain("hello .com", "dnslink=/dns/hello .com/foo")),
 		partsNil,
-		&LogStatement{Code: "INVALID_REDIRECT", Domain: "hello .com", Pathname: "", Search: make(map[string][]string)},
+		&LogStatement{Code: "INVALID_REDIRECT", Entry: "dnslink=/dns/hello .com/foo"},
 	)
-	assertResult(t, arr(validateDomain("_dnslink._dnslink.hello.com")),
+	assertResult(t, arr(validateDomain("_dnslink._dnslink.hello.com", "")),
 		partsNil,
 		&LogStatement{Code: "RECURSIVE_DNSLINK_PREFIX", Domain: "_dnslink._dnslink.hello.com", Pathname: "", Search: make(map[string][]string)},
 	)
@@ -222,7 +222,7 @@ func TestDnsLinkN(t *testing.T) {
 }
 
 func TestUDPLookup(t *testing.T) {
-	lookup := NewUDPLookup([]string{"1.1.1.1:53"})
+	lookup := NewUDPLookup([]string{"1.1.1.1:53"}, 0)
 	txt, error := lookup("_dnslink.t05.dnslink.dev")
 	assert.NoError(t, error)
 	assert.Equal(t, len(txt), 2)

integration/main.go

@@ -20,18 +20,31 @@ func main() {
 	options := Options{}
 	json.Unmarshal([]byte(os.Args[2]), &options)
 	r := &dnslink.Resolver{
-		LookupTXT: dnslink.NewUDPLookup([]string{"127.0.0.1:" + fmt.Sprint(options.Udp)}),
+		LookupTXT: dnslink.NewUDPLookup([]string{"127.0.0.1:" + fmt.Sprint(options.Udp)}, 0),
 	}
 
 	resolved, error := r.ResolveN(domain)
 	if error != nil {
-		panic(error)
+		exitWithError(error)
 	}
 
-	result, err := json.Marshal(resolved)
+	result, err := json.MarshalIndent(resolved, "", "  ")
 	if err != nil {
 		panic(err)
 	} else {
 		fmt.Print(string(result))
 	}
 }
+
+func exitWithError(input error) {
+	result, err := json.MarshalIndent(map[string]map[string]string{
+		"error": {
+			"code": input.Error(),
+		},
+	}, "", "  ")
+	if err != nil {
+		panic(err)
+	}
+	fmt.Print(string(result))
+	os.Exit(1)
+}