diff --git a/src/rfc3986/parseresult.py b/src/rfc3986/parseresult.py
index f7ce624..b69432d 100644
--- a/src/rfc3986/parseresult.py
+++ b/src/rfc3986/parseresult.py
@@ -467,8 +467,8 @@ def authority_from(reference, strict):
         )
 
     if port:
-        try:
+        if port.isascii() and port.isdigit():
             port = int(port)
-        except ValueError:
+        else:
             raise exceptions.InvalidPort(port)
     return userinfo, host, port
diff --git a/tests/test_parseresult.py b/tests/test_parseresult.py
index e35a586..78c97f5 100644
--- a/tests/test_parseresult.py
+++ b/tests/test_parseresult.py
@@ -18,7 +18,7 @@
 from rfc3986 import exceptions
 from rfc3986 import parseresult as pr
 
-INVALID_PORTS = ["443:80", "443:80:443", "abcdef", "port", "43port"]
+INVALID_PORTS = ["443:80", "443:80:443", "abcdef", "port", "43port", " 1", "1 ", "𖭖", "-1", "　1", "1_1"]
 
 SNOWMAN = b"\xe2\x98\x83"
 SNOWMAN_IDNA_HOST = "http://xn--n3h.com"