Unicode.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 225,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "import pandas as pd\n",
    "import unicodedata\n",
    "from collections import defaultdict\n",
    "\n",
    "unicode_category = defaultdict(list)\n",
    "for c in map(chr, range(sys.maxunicode + 1)):\n",
    "    unicode_category[unicodedata.category(c)].append(ord(c))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 226,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_ranges(l):\n",
    "    ranges = []\n",
    "    prev_x = 0\n",
    "    \n",
    "    for x in sorted(l):\n",
    "        if not ranges:\n",
    "            ranges.append([x])\n",
    "        elif x - prev_x == 1:\n",
    "            ranges[-1] = [ranges[-1][0], x]\n",
    "        else:\n",
    "            ranges.append([x])\n",
    "        prev_x = x\n",
    "        \n",
    "    print(f\"reduced {len(l)} ints to {len(ranges)} ranges\")\n",
    "    return ranges\n",
    "\n",
    "def get_unicode_categories(*categories):\n",
    "    output = []\n",
    "    \n",
    "    for category in categories:\n",
    "        results = unicode_category[category]\n",
    "        output.extend(results)\n",
    "        print(f\"{len(results)} for ${category}\")\n",
    "    \n",
    "    return output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 227,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1788 for $Lu\n",
      "2151 for $Ll\n",
      "31 for $Lt\n",
      "259 for $Lm\n",
      "121414 for $Lo\n",
      "236 for $Nl\n",
      "1826 for $Mn\n",
      "429 for $Mc\n",
      "630 for $Nd\n",
      "10 for $Pc\n"
     ]
    }
   ],
   "source": [
    "UnicodeLetter = get_unicode_categories('Lu', 'Ll', 'Lt', 'Lm', 'Lo', 'Nl')\n",
    "UnicodeCombiningMark = get_unicode_categories('Mn', 'Mc')\n",
    "UnicodeDigit = get_unicode_categories('Nd')\n",
    "UnicodeConnectorPunctuation = get_unicode_categories('Pc')\n",
    "\n",
    "ZWNJ = 0x200C\n",
    "ZWJ = 0x200D\n",
    "\n",
    "\"\"\"\n",
    "\n",
    "    This implementation can safely ignore the \n",
    "    UnicodeEscapeSequence requirement as escape\n",
    "    sequences are preprocessed out before the \n",
    "    Identifier checks.\n",
    "\n",
    "    GRAMMAR:\n",
    "    IdentifierStart ::\n",
    "        UnicodeLetter\n",
    "        $\n",
    "        _\n",
    "        \\ UnicodeEscapeSequence\n",
    "        \n",
    "\"\"\"\n",
    "IdentifierStart = [\n",
    "    36, # $\n",
    "    95, # _\n",
    "    *UnicodeLetters\n",
    "]\n",
    "\n",
    "\"\"\"\n",
    "\n",
    "    GRAMMAR:\n",
    "    IdentifierPart ::\n",
    "        IdentifierStart\n",
    "        UnicodeCombiningMark\n",
    "        UnicodeDigit\n",
    "        UnicodeConnectorPunctuation\n",
    "        <ZWNJ>\n",
    "        <ZWJ>\n",
    "        \n",
    "\"\"\"\n",
    "IdentifierPart = [\n",
    "    *IdentifierStart,\n",
    "    *UnicodeCombiningMark,\n",
    "    *UnicodeDigit,\n",
    "    *UnicodeConnectorPunctuation,\n",
    "    ZWNJ,\n",
    "    ZWJ\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 228,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "reduced 128778 ints to 718 ranges\n",
      "\n",
      "((ch == 0x24)|(0x30 <= ch & ch <= 0x39)|(0x41 <= ch & ch <= 0x5a)|(ch == 0x5f)|(ch == 0x5f)|(0x61 <= ch & ch <= 0x7a)|(ch == 0xaa)|(ch == 0xb5)|(ch == 0xba)|(0xc0 <= ch & ch <= 0xd6)|(0xd8 <= ch & ch <= 0xf6)|(0xf8 <= ch & ch <= 0x2c1)|(0x2c6 <= ch & ch <= 0x2d1)|(0x2e0 <= ch & ch <= 0x2e4)|(ch == 0x2ec)|(ch == 0x2ee)|(0x300 <= ch & ch <= 0x374)|(0x376 <= ch & ch <= 0x377)|(0x37a <= ch & ch <= 0x37d)|(ch == 0x37f)|(ch == 0x386)|(0x388 <= ch & ch <= 0x38a)|(ch == 0x38c)|(0x38e <= ch & ch <= 0x3a1)|(0x3a3 <= ch & ch <= 0x3f5)|(0x3f7 <= ch & ch <= 0x481)|(0x483 <= ch & ch <= 0x487)|(0x48a <= ch & ch <= 0x52f)|(0x531 <= ch & ch <= 0x556)|(ch == 0x559)|(0x560 <= ch & ch <= 0x588)|(0x591 <= ch & ch <= 0x5bd)|(ch == 0x5bf)|(0x5c1 <= ch & ch <= 0x5c2)|(0x5c4 <= ch & ch <= 0x5c5)|(ch == 0x5c7)|(0x5d0 <= ch & ch <= 0x5ea)|(0x5ef <= ch & ch <= 0x5f2)|(0x610 <= ch & ch <= 0x61a)|(0x620 <= ch & ch <= 0x669)|(0x66e <= ch & ch <= 0x6d3)|(0x6d5 <= ch & ch <= 0x6dc)|(0x6df <= ch & ch <= 0x6e8)|(0x6ea <= ch & ch <= 0x6fc)|(ch == 0x6ff)|(0x710 <= ch & ch <= 0x74a)|(0x74d <= ch & ch <= 0x7b1)|(0x7c0 <= ch & ch <= 0x7f5)|(ch == 0x7fa)|(ch == 0x7fd)|(0x800 <= ch & ch <= 0x82d)|(0x840 <= ch & ch <= 0x85b)|(0x860 <= ch & ch <= 0x86a)|(0x8a0 <= ch & ch <= 0x8b4)|(0x8b6 <= ch & ch <= 0x8bd)|(0x8d3 <= ch & ch <= 0x8e1)|(0x8e3 <= ch & ch <= 0x963)|(0x966 <= ch & ch <= 0x96f)|(0x971 <= ch & ch <= 0x983)|(0x985 <= ch & ch <= 0x98c)|(0x98f <= ch & ch <= 0x990)|(0x993 <= ch & ch <= 0x9a8)|(0x9aa <= ch & ch <= 0x9b0)|(ch == 0x9b2)|(0x9b6 <= ch & ch <= 0x9b9)|(0x9bc <= ch & ch <= 0x9c4)|(0x9c7 <= ch & ch <= 0x9c8)|(0x9cb <= ch & ch <= 0x9ce)|(ch == 0x9d7)|(0x9dc <= ch & ch <= 0x9dd)|(0x9df <= ch & ch <= 0x9e3)|(0x9e6 <= ch & ch <= 0x9f1)|(ch == 0x9fc)|(ch == 0x9fe)|(0xa01 <= ch & ch <= 0xa03)|(0xa05 <= ch & ch <= 0xa0a)|(0xa0f <= ch & ch <= 0xa10)|(0xa13 <= ch & ch <= 0xa28)|(0xa2a <= ch & ch <= 0xa30)|(0xa32 <= ch & ch <= 0xa33)|(0xa35 <= ch & ch <= 0xa36)|(0xa38 <= ch & ch <= 0xa39)|(ch == 0xa3c)|(0xa3e <= ch & ch <= 0xa42)|(0xa47 <= ch & ch <= 0xa48)|(0xa4b <= ch & ch <= 0xa4d)|(ch == 0xa51)|(0xa59 <= ch & ch <= 0xa5c)|(ch == 0xa5e)|(0xa66 <= ch & ch <= 0xa75)|(0xa81 <= ch & ch <= 0xa83)|(0xa85 <= ch & ch <= 0xa8d)|(0xa8f <= ch & ch <= 0xa91)|(0xa93 <= ch & ch <= 0xaa8)|(0xaaa <= ch & ch <= 0xab0)|(0xab2 <= ch & ch <= 0xab3)|(0xab5 <= ch & ch <= 0xab9)|(0xabc <= ch & ch <= 0xac5)|(0xac7 <= ch & ch <= 0xac9)|(0xacb <= ch & ch <= 0xacd)|(ch == 0xad0)|(0xae0 <= ch & ch <= 0xae3)|(0xae6 <= ch & ch <= 0xaef)|(0xaf9 <= ch & ch <= 0xaff)|(0xb01 <= ch & ch <= 0xb03)|(0xb05 <= ch & ch <= 0xb0c)|(0xb0f <= ch & ch <= 0xb10)|(0xb13 <= ch & ch <= 0xb28)|(0xb2a <= ch & ch <= 0xb30)|(0xb32 <= ch & ch <= 0xb33)|(0xb35 <= ch & ch <= 0xb39)|(0xb3c <= ch & ch <= 0xb44)|(0xb47 <= ch & ch <= 0xb48)|(0xb4b <= ch & ch <= 0xb4d)|(0xb56 <= ch & ch <= 0xb57)|(0xb5c <= ch & ch <= 0xb5d)|(0xb5f <= ch & ch <= 0xb63)|(0xb66 <= ch & ch <= 0xb6f)|(ch == 0xb71)|(0xb82 <= ch & ch <= 0xb83)|(0xb85 <= ch & ch <= 0xb8a)|(0xb8e <= ch & ch <= 0xb90)|(0xb92 <= ch & ch <= 0xb95)|(0xb99 <= ch & ch <= 0xb9a)|(ch == 0xb9c)|(0xb9e <= ch & ch <= 0xb9f)|(0xba3 <= ch & ch <= 0xba4)|(0xba8 <= ch & ch <= 0xbaa)|(0xbae <= ch & ch <= 0xbb9)|(0xbbe <= ch & ch <= 0xbc2)|(0xbc6 <= ch & ch <= 0xbc8)|(0xbca <= ch & ch <= 0xbcd)|(ch == 0xbd0)|(ch == 0xbd7)|(0xbe6 <= ch & ch <= 0xbef)|(0xc00 <= ch & ch <= 0xc0c)|(0xc0e <= ch & ch <= 0xc10)|(0xc12 <= ch & ch <= 0xc28)|(0xc2a <= ch & ch <= 0xc39)|(0xc3d <= ch & ch <= 0xc44)|(0xc46 <= ch & ch <= 0xc48)|(0xc4a <= ch & ch <= 0xc4d)|(0xc55 <= ch & ch <= 0xc56)|(0xc58 <= ch & ch <= 0xc5a)|(0xc60 <= ch & ch <= 0xc63)|(0xc66 <= ch & ch <= 0xc6f)|(0xc80 <= ch & ch <= 0xc83)|(0xc85 <= ch & ch <= 0xc8c)|(0xc8e <= ch & ch <= 0xc90)|(0xc92 <= ch & ch <= 0xca8)|(0xcaa <= ch & ch <= 0xcb3)|(0xcb5 <= ch & ch <= 0xcb9)|(0xcbc <= ch & ch <= 0xcc4)|(0xcc6 <= ch & ch <= 0xcc8)|(0xcca <= ch & ch <= 0xccd)|(0xcd5 <= ch & ch <= 0xcd6)|(ch == 0xcde)|(0xce0 <= ch & ch <= 0xce3)|(0xce6 <= ch & ch <= 0xcef)|(0xcf1 <= ch & ch <= 0xcf2)|(0xd00 <= ch & ch <= 0xd03)|(0xd05 <= ch & ch <= 0xd0c)|(0xd0e <= ch & ch <= 0xd10)|(0xd12 <= ch & ch <= 0xd44)|(0xd46 <= ch & ch <= 0xd48)|(0xd4a <= ch & ch <= 0xd4e)|(0xd54 <= ch & ch <= 0xd57)|(0xd5f <= ch & ch <= 0xd63)|(0xd66 <= ch & ch <= 0xd6f)|(0xd7a <= ch & ch <= 0xd7f)|(0xd82 <= ch & ch <= 0xd83)|(0xd85 <= ch & ch <= 0xd96)|(0xd9a <= ch & ch <= 0xdb1)|(0xdb3 <= ch & ch <= 0xdbb)|(ch == 0xdbd)|(0xdc0 <= ch & ch <= 0xdc6)|(ch == 0xdca)|(0xdcf <= ch & ch <= 0xdd4)|(ch == 0xdd6)|(0xdd8 <= ch & ch <= 0xddf)|(0xde6 <= ch & ch <= 0xdef)|(0xdf2 <= ch & ch <= 0xdf3)|(0xe01 <= ch & ch <= 0xe3a)|(0xe40 <= ch & ch <= 0xe4e)|(0xe50 <= ch & ch <= 0xe59)|(0xe81 <= ch & ch <= 0xe82)|(ch == 0xe84)|(0xe86 <= ch & ch <= 0xe8a)|(0xe8c <= ch & ch <= 0xea3)|(ch == 0xea5)|(0xea7 <= ch & ch <= 0xebd)|(0xec0 <= ch & ch <= 0xec4)|(ch == 0xec6)|(0xec8 <= ch & ch <= 0xecd)|(0xed0 <= ch & ch <= 0xed9)|(0xedc <= ch & ch <= 0xedf)|(ch == 0xf00)|(0xf18 <= ch & ch <= 0xf19)|(0xf20 <= ch & ch <= 0xf29)|(ch == 0xf35)|(ch == 0xf37)|(ch == 0xf39)|(0xf3e <= ch & ch <= 0xf47)|(0xf49 <= ch & ch <= 0xf6c)|(0xf71 <= ch & ch <= 0xf84)|(0xf86 <= ch & ch <= 0xf97)|(0xf99 <= ch & ch <= 0xfbc)|(ch == 0xfc6)|(0x1000 <= ch & ch <= 0x1049)|(0x1050 <= ch & ch <= 0x109d)|(0x10a0 <= ch & ch <= 0x10c5)|(ch == 0x10c7)|(ch == 0x10cd)|(0x10d0 <= ch & ch <= 0x10fa)|(0x10fc <= ch & ch <= 0x1248)|(0x124a <= ch & ch <= 0x124d)|(0x1250 <= ch & ch <= 0x1256)|(ch == 0x1258)|(0x125a <= ch & ch <= 0x125d)|(0x1260 <= ch & ch <= 0x1288)|(0x128a <= ch & ch <= 0x128d)|(0x1290 <= ch & ch <= 0x12b0)|(0x12b2 <= ch & ch <= 0x12b5)|(0x12b8 <= ch & ch <= 0x12be)|(ch == 0x12c0)|(0x12c2 <= ch & ch <= 0x12c5)|(0x12c8 <= ch & ch <= 0x12d6)|(0x12d8 <= ch & ch <= 0x1310)|(0x1312 <= ch & ch <= 0x1315)|(0x1318 <= ch & ch <= 0x135a)|(0x135d <= ch & ch <= 0x135f)|(0x1380 <= ch & ch <= 0x138f)|(0x13a0 <= ch & ch <= 0x13f5)|(0x13f8 <= ch & ch <= 0x13fd)|(0x1401 <= ch & ch <= 0x166c)|(0x166f <= ch & ch <= 0x167f)|(0x1681 <= ch & ch <= 0x169a)|(0x16a0 <= ch & ch <= 0x16ea)|(0x16ee <= ch & ch <= 0x16f8)|(0x1700 <= ch & ch <= 0x170c)|(0x170e <= ch & ch <= 0x1714)|(0x1720 <= ch & ch <= 0x1734)|(0x1740 <= ch & ch <= 0x1753)|(0x1760 <= ch & ch <= 0x176c)|(0x176e <= ch & ch <= 0x1770)|(0x1772 <= ch & ch <= 0x1773)|(0x1780 <= ch & ch <= 0x17d3)|(ch == 0x17d7)|(0x17dc <= ch & ch <= 0x17dd)|(0x17e0 <= ch & ch <= 0x17e9)|(0x180b <= ch & ch <= 0x180d)|(0x1810 <= ch & ch <= 0x1819)|(0x1820 <= ch & ch <= 0x1878)|(0x1880 <= ch & ch <= 0x18aa)|(0x18b0 <= ch & ch <= 0x18f5)|(0x1900 <= ch & ch <= 0x191e)|(0x1920 <= ch & ch <= 0x192b)|(0x1930 <= ch & ch <= 0x193b)|(0x1946 <= ch & ch <= 0x196d)|(0x1970 <= ch & ch <= 0x1974)|(0x1980 <= ch & ch <= 0x19ab)|(0x19b0 <= ch & ch <= 0x19c9)|(0x19d0 <= ch & ch <= 0x19d9)|(0x1a00 <= ch & ch <= 0x1a1b)|(0x1a20 <= ch & ch <= 0x1a5e)|(0x1a60 <= ch & ch <= 0x1a7c)|(0x1a7f <= ch & ch <= 0x1a89)|(0x1a90 <= ch & ch <= 0x1a99)|(ch == 0x1aa7)|(0x1ab0 <= ch & ch <= 0x1abd)|(0x1b00 <= ch & ch <= 0x1b4b)|(0x1b50 <= ch & ch <= 0x1b59)|(0x1b6b <= ch & ch <= 0x1b73)|(0x1b80 <= ch & ch <= 0x1bf3)|(0x1c00 <= ch & ch <= 0x1c37)|(0x1c40 <= ch & ch <= 0x1c49)|(0x1c4d <= ch & ch <= 0x1c7d)|(0x1c80 <= ch & ch <= 0x1c88)|(0x1c90 <= ch & ch <= 0x1cba)|(0x1cbd <= ch & ch <= 0x1cbf)|(0x1cd0 <= ch & ch <= 0x1cd2)|(0x1cd4 <= ch & ch <= 0x1cfa)|(0x1d00 <= ch & ch <= 0x1df9)|(0x1dfb <= ch & ch <= 0x1f15)|(0x1f18 <= ch & ch <= 0x1f1d)|(0x1f20 <= ch & ch <= 0x1f45)|(0x1f48 <= ch & ch <= 0x1f4d)|(0x1f50 <= ch & ch <= 0x1f57)|(ch == 0x1f59)|(ch == 0x1f5b)|(ch == 0x1f5d)|(0x1f5f <= ch & ch <= 0x1f7d)|(0x1f80 <= ch & ch <= 0x1fb4)|(0x1fb6 <= ch & ch <= 0x1fbc)|(ch == 0x1fbe)|(0x1fc2 <= ch & ch <= 0x1fc4)|(0x1fc6 <= ch & ch <= 0x1fcc)|(0x1fd0 <= ch & ch <= 0x1fd3)|(0x1fd6 <= ch & ch <= 0x1fdb)|(0x1fe0 <= ch & ch <= 0x1fec)|(0x1ff2 <= ch & ch <= 0x1ff4)|(0x1ff6 <= ch & ch <= 0x1ffc)|(0x200c <= ch & ch <= 0x200d)|(0x203f <= ch & ch <= 0x2040)|(ch == 0x2054)|(ch == 0x2071)|(ch == 0x207f)|(0x2090 <= ch & ch <= 0x209c)|(0x20d0 <= ch & ch <= 0x20dc)|(ch == 0x20e1)|(0x20e5 <= ch & ch <= 0x20f0)|(ch == 0x2102)|(ch == 0x2107)|(0x210a <= ch & ch <= 0x2113)|(ch == 0x2115)|(0x2119 <= ch & ch <= 0x211d)|(ch == 0x2124)|(ch == 0x2126)|(ch == 0x2128)|(0x212a <= ch & ch <= 0x212d)|(0x212f <= ch & ch <= 0x2139)|(0x213c <= ch & ch <= 0x213f)|(0x2145 <= ch & ch <= 0x2149)|(ch == 0x214e)|(0x2160 <= ch & ch <= 0x2188)|(0x2c00 <= ch & ch <= 0x2c2e)|(0x2c30 <= ch & ch <= 0x2c5e)|(0x2c60 <= ch & ch <= 0x2ce4)|(0x2ceb <= ch & ch <= 0x2cf3)|(0x2d00 <= ch & ch <= 0x2d25)|(ch == 0x2d27)|(ch == 0x2d2d)|(0x2d30 <= ch & ch <= 0x2d67)|(ch == 0x2d6f)|(0x2d7f <= ch & ch <= 0x2d96)|(0x2da0 <= ch & ch <= 0x2da6)|(0x2da8 <= ch & ch <= 0x2dae)|(0x2db0 <= ch & ch <= 0x2db6)|(0x2db8 <= ch & ch <= 0x2dbe)|(0x2dc0 <= ch & ch <= 0x2dc6)|(0x2dc8 <= ch & ch <= 0x2dce)|(0x2dd0 <= ch & ch <= 0x2dd6)|(0x2dd8 <= ch & ch <= 0x2dde)|(0x2de0 <= ch & ch <= 0x2dff)|(ch == 0x2e2f)|(0x3005 <= ch & ch <= 0x3007)|(0x3021 <= ch & ch <= 0x302f)|(0x3031 <= ch & ch <= 0x3035)|(0x3038 <= ch & ch <= 0x303c)|(0x3041 <= ch & ch <= 0x3096)|(0x3099 <= ch & ch <= 0x309a)|(0x309d <= ch & ch <= 0x309f)|(0x30a1 <= ch & ch <= 0x30fa)|(0x30fc <= ch & ch <= 0x30ff)|(0x3105 <= ch & ch <= 0x312f)|(0x3131 <= ch & ch <= 0x318e)|(0x31a0 <= ch & ch <= 0x31ba)|(0x31f0 <= ch & ch <= 0x31ff)|(0x3400 <= ch & ch <= 0x4db5)|(0x4e00 <= ch & ch <= 0x9fef)|(0xa000 <= ch & ch <= 0xa48c)|(0xa4d0 <= ch & ch <= 0xa4fd)|(0xa500 <= ch & ch <= 0xa60c)|(0xa610 <= ch & ch <= 0xa62b)|(0xa640 <= ch & ch <= 0xa66f)|(0xa674 <= ch & ch <= 0xa67d)|(0xa67f <= ch & ch <= 0xa6f1)|(0xa717 <= ch & ch <= 0xa71f)|(0xa722 <= ch & ch <= 0xa788)|(0xa78b <= ch & ch <= 0xa7bf)|(0xa7c2 <= ch & ch <= 0xa7c6)|(0xa7f7 <= ch & ch <= 0xa827)|(0xa840 <= ch & ch <= 0xa873)|(0xa880 <= ch & ch <= 0xa8c5)|(0xa8d0 <= ch & ch <= 0xa8d9)|(0xa8e0 <= ch & ch <= 0xa8f7)|(ch == 0xa8fb)|(0xa8fd <= ch & ch <= 0xa92d)|(0xa930 <= ch & ch <= 0xa953)|(0xa960 <= ch & ch <= 0xa97c)|(0xa980 <= ch & ch <= 0xa9c0)|(0xa9cf <= ch & ch <= 0xa9d9)|(0xa9e0 <= ch & ch <= 0xa9fe)|(0xaa00 <= ch & ch <= 0xaa36)|(0xaa40 <= ch & ch <= 0xaa4d)|(0xaa50 <= ch & ch <= 0xaa59)|(0xaa60 <= ch & ch <= 0xaa76)|(0xaa7a <= ch & ch <= 0xaac2)|(0xaadb <= ch & ch <= 0xaadd)|(0xaae0 <= ch & ch <= 0xaaef)|(0xaaf2 <= ch & ch <= 0xaaf6)|(0xab01 <= ch & ch <= 0xab06)|(0xab09 <= ch & ch <= 0xab0e)|(0xab11 <= ch & ch <= 0xab16)|(0xab20 <= ch & ch <= 0xab26)|(0xab28 <= ch & ch <= 0xab2e)|(0xab30 <= ch & ch <= 0xab5a)|(0xab5c <= ch & ch <= 0xab67)|(0xab70 <= ch & ch <= 0xabea)|(0xabec <= ch & ch <= 0xabed)|(0xabf0 <= ch & ch <= 0xabf9)|(0xac00 <= ch & ch <= 0xd7a3)|(0xd7b0 <= ch & ch <= 0xd7c6)|(0xd7cb <= ch & ch <= 0xd7fb)|(0xf900 <= ch & ch <= 0xfa6d)|(0xfa70 <= ch & ch <= 0xfad9)|(0xfb00 <= ch & ch <= 0xfb06)|(0xfb13 <= ch & ch <= 0xfb17)|(0xfb1d <= ch & ch <= 0xfb28)|(0xfb2a <= ch & ch <= 0xfb36)|(0xfb38 <= ch & ch <= 0xfb3c)|(ch == 0xfb3e)|(0xfb40 <= ch & ch <= 0xfb41)|(0xfb43 <= ch & ch <= 0xfb44)|(0xfb46 <= ch & ch <= 0xfbb1)|(0xfbd3 <= ch & ch <= 0xfd3d)|(0xfd50 <= ch & ch <= 0xfd8f)|(0xfd92 <= ch & ch <= 0xfdc7)|(0xfdf0 <= ch & ch <= 0xfdfb)|(0xfe00 <= ch & ch <= 0xfe0f)|(0xfe20 <= ch & ch <= 0xfe2f)|(0xfe33 <= ch & ch <= 0xfe34)|(0xfe4d <= ch & ch <= 0xfe4f)|(0xfe70 <= ch & ch <= 0xfe74)|(0xfe76 <= ch & ch <= 0xfefc)|(0xff10 <= ch & ch <= 0xff19)|(0xff21 <= ch & ch <= 0xff3a)|(ch == 0xff3f)|(0xff41 <= ch & ch <= 0xff5a)|(0xff66 <= ch & ch <= 0xffbe)|(0xffc2 <= ch & ch <= 0xffc7)|(0xffca <= ch & ch <= 0xffcf)|(0xffd2 <= ch & ch <= 0xffd7)|(0xffda <= ch & ch <= 0xffdc)|(0x10000 <= ch & ch <= 0x1000b)|(0x1000d <= ch & ch <= 0x10026)|(0x10028 <= ch & ch <= 0x1003a)|(0x1003c <= ch & ch <= 0x1003d)|(0x1003f <= ch & ch <= 0x1004d)|(0x10050 <= ch & ch <= 0x1005d)|(0x10080 <= ch & ch <= 0x100fa)|(0x10140 <= ch & ch <= 0x10174)|(ch == 0x101fd)|(0x10280 <= ch & ch <= 0x1029c)|(0x102a0 <= ch & ch <= 0x102d0)|(ch == 0x102e0)|(0x10300 <= ch & ch <= 0x1031f)|(0x1032d <= ch & ch <= 0x1034a)|(0x10350 <= ch & ch <= 0x1037a)|(0x10380 <= ch & ch <= 0x1039d)|(0x103a0 <= ch & ch <= 0x103c3)|(0x103c8 <= ch & ch <= 0x103cf)|(0x103d1 <= ch & ch <= 0x103d5)|(0x10400 <= ch & ch <= 0x1049d)|(0x104a0 <= ch & ch <= 0x104a9)|(0x104b0 <= ch & ch <= 0x104d3)|(0x104d8 <= ch & ch <= 0x104fb)|(0x10500 <= ch & ch <= 0x10527)|(0x10530 <= ch & ch <= 0x10563)|(0x10600 <= ch & ch <= 0x10736)|(0x10740 <= ch & ch <= 0x10755)|(0x10760 <= ch & ch <= 0x10767)|(0x10800 <= ch & ch <= 0x10805)|(ch == 0x10808)|(0x1080a <= ch & ch <= 0x10835)|(0x10837 <= ch & ch <= 0x10838)|(ch == 0x1083c)|(0x1083f <= ch & ch <= 0x10855)|(0x10860 <= ch & ch <= 0x10876)|(0x10880 <= ch & ch <= 0x1089e)|(0x108e0 <= ch & ch <= 0x108f2)|(0x108f4 <= ch & ch <= 0x108f5)|(0x10900 <= ch & ch <= 0x10915)|(0x10920 <= ch & ch <= 0x10939)|(0x10980 <= ch & ch <= 0x109b7)|(0x109be <= ch & ch <= 0x109bf)|(0x10a00 <= ch & ch <= 0x10a03)|(0x10a05 <= ch & ch <= 0x10a06)|(0x10a0c <= ch & ch <= 0x10a13)|(0x10a15 <= ch & ch <= 0x10a17)|(0x10a19 <= ch & ch <= 0x10a35)|(0x10a38 <= ch & ch <= 0x10a3a)|(ch == 0x10a3f)|(0x10a60 <= ch & ch <= 0x10a7c)|(0x10a80 <= ch & ch <= 0x10a9c)|(0x10ac0 <= ch & ch <= 0x10ac7)|(0x10ac9 <= ch & ch <= 0x10ae6)|(0x10b00 <= ch & ch <= 0x10b35)|(0x10b40 <= ch & ch <= 0x10b55)|(0x10b60 <= ch & ch <= 0x10b72)|(0x10b80 <= ch & ch <= 0x10b91)|(0x10c00 <= ch & ch <= 0x10c48)|(0x10c80 <= ch & ch <= 0x10cb2)|(0x10cc0 <= ch & ch <= 0x10cf2)|(0x10d00 <= ch & ch <= 0x10d27)|(0x10d30 <= ch & ch <= 0x10d39)|(0x10f00 <= ch & ch <= 0x10f1c)|(ch == 0x10f27)|(0x10f30 <= ch & ch <= 0x10f50)|(0x10fe0 <= ch & ch <= 0x10ff6)|(0x11000 <= ch & ch <= 0x11046)|(0x11066 <= ch & ch <= 0x1106f)|(0x1107f <= ch & ch <= 0x110ba)|(0x110d0 <= ch & ch <= 0x110e8)|(0x110f0 <= ch & ch <= 0x110f9)|(0x11100 <= ch & ch <= 0x11134)|(0x11136 <= ch & ch <= 0x1113f)|(0x11144 <= ch & ch <= 0x11146)|(0x11150 <= ch & ch <= 0x11173)|(ch == 0x11176)|(0x11180 <= ch & ch <= 0x111c4)|(0x111c9 <= ch & ch <= 0x111cc)|(0x111d0 <= ch & ch <= 0x111da)|(ch == 0x111dc)|(0x11200 <= ch & ch <= 0x11211)|(0x11213 <= ch & ch <= 0x11237)|(ch == 0x1123e)|(0x11280 <= ch & ch <= 0x11286)|(ch == 0x11288)|(0x1128a <= ch & ch <= 0x1128d)|(0x1128f <= ch & ch <= 0x1129d)|(0x1129f <= ch & ch <= 0x112a8)|(0x112b0 <= ch & ch <= 0x112ea)|(0x112f0 <= ch & ch <= 0x112f9)|(0x11300 <= ch & ch <= 0x11303)|(0x11305 <= ch & ch <= 0x1130c)|(0x1130f <= ch & ch <= 0x11310)|(0x11313 <= ch & ch <= 0x11328)|(0x1132a <= ch & ch <= 0x11330)|(0x11332 <= ch & ch <= 0x11333)|(0x11335 <= ch & ch <= 0x11339)|(0x1133b <= ch & ch <= 0x11344)|(0x11347 <= ch & ch <= 0x11348)|(0x1134b <= ch & ch <= 0x1134d)|(ch == 0x11350)|(ch == 0x11357)|(0x1135d <= ch & ch <= 0x11363)|(0x11366 <= ch & ch <= 0x1136c)|(0x11370 <= ch & ch <= 0x11374)|(0x11400 <= ch & ch <= 0x1144a)|(0x11450 <= ch & ch <= 0x11459)|(0x1145e <= ch & ch <= 0x1145f)|(0x11480 <= ch & ch <= 0x114c5)|(ch == 0x114c7)|(0x114d0 <= ch & ch <= 0x114d9)|(0x11580 <= ch & ch <= 0x115b5)|(0x115b8 <= ch & ch <= 0x115c0)|(0x115d8 <= ch & ch <= 0x115dd)|(0x11600 <= ch & ch <= 0x11640)|(ch == 0x11644)|(0x11650 <= ch & ch <= 0x11659)|(0x11680 <= ch & ch <= 0x116b8)|(0x116c0 <= ch & ch <= 0x116c9)|(0x11700 <= ch & ch <= 0x1171a)|(0x1171d <= ch & ch <= 0x1172b)|(0x11730 <= ch & ch <= 0x11739)|(0x11800 <= ch & ch <= 0x1183a)|(0x118a0 <= ch & ch <= 0x118e9)|(ch == 0x118ff)|(0x119a0 <= ch & ch <= 0x119a7)|(0x119aa <= ch & ch <= 0x119d7)|(0x119da <= ch & ch <= 0x119e1)|(0x119e3 <= ch & ch <= 0x119e4)|(0x11a00 <= ch & ch <= 0x11a3e)|(ch == 0x11a47)|(0x11a50 <= ch & ch <= 0x11a99)|(ch == 0x11a9d)|(0x11ac0 <= ch & ch <= 0x11af8)|(0x11c00 <= ch & ch <= 0x11c08)|(0x11c0a <= ch & ch <= 0x11c36)|(0x11c38 <= ch & ch <= 0x11c40)|(0x11c50 <= ch & ch <= 0x11c59)|(0x11c72 <= ch & ch <= 0x11c8f)|(0x11c92 <= ch & ch <= 0x11ca7)|(0x11ca9 <= ch & ch <= 0x11cb6)|(0x11d00 <= ch & ch <= 0x11d06)|(0x11d08 <= ch & ch <= 0x11d09)|(0x11d0b <= ch & ch <= 0x11d36)|(ch == 0x11d3a)|(0x11d3c <= ch & ch <= 0x11d3d)|(0x11d3f <= ch & ch <= 0x11d47)|(0x11d50 <= ch & ch <= 0x11d59)|(0x11d60 <= ch & ch <= 0x11d65)|(0x11d67 <= ch & ch <= 0x11d68)|(0x11d6a <= ch & ch <= 0x11d8e)|(0x11d90 <= ch & ch <= 0x11d91)|(0x11d93 <= ch & ch <= 0x11d98)|(0x11da0 <= ch & ch <= 0x11da9)|(0x11ee0 <= ch & ch <= 0x11ef6)|(0x12000 <= ch & ch <= 0x12399)|(0x12400 <= ch & ch <= 0x1246e)|(0x12480 <= ch & ch <= 0x12543)|(0x13000 <= ch & ch <= 0x1342e)|(0x14400 <= ch & ch <= 0x14646)|(0x16800 <= ch & ch <= 0x16a38)|(0x16a40 <= ch & ch <= 0x16a5e)|(0x16a60 <= ch & ch <= 0x16a69)|(0x16ad0 <= ch & ch <= 0x16aed)|(0x16af0 <= ch & ch <= 0x16af4)|(0x16b00 <= ch & ch <= 0x16b36)|(0x16b40 <= ch & ch <= 0x16b43)|(0x16b50 <= ch & ch <= 0x16b59)|(0x16b63 <= ch & ch <= 0x16b77)|(0x16b7d <= ch & ch <= 0x16b8f)|(0x16e40 <= ch & ch <= 0x16e7f)|(0x16f00 <= ch & ch <= 0x16f4a)|(0x16f4f <= ch & ch <= 0x16f87)|(0x16f8f <= ch & ch <= 0x16f9f)|(0x16fe0 <= ch & ch <= 0x16fe1)|(ch == 0x16fe3)|(0x17000 <= ch & ch <= 0x187f7)|(0x18800 <= ch & ch <= 0x18af2)|(0x1b000 <= ch & ch <= 0x1b11e)|(0x1b150 <= ch & ch <= 0x1b152)|(0x1b164 <= ch & ch <= 0x1b167)|(0x1b170 <= ch & ch <= 0x1b2fb)|(0x1bc00 <= ch & ch <= 0x1bc6a)|(0x1bc70 <= ch & ch <= 0x1bc7c)|(0x1bc80 <= ch & ch <= 0x1bc88)|(0x1bc90 <= ch & ch <= 0x1bc99)|(0x1bc9d <= ch & ch <= 0x1bc9e)|(0x1d165 <= ch & ch <= 0x1d169)|(0x1d16d <= ch & ch <= 0x1d172)|(0x1d17b <= ch & ch <= 0x1d182)|(0x1d185 <= ch & ch <= 0x1d18b)|(0x1d1aa <= ch & ch <= 0x1d1ad)|(0x1d242 <= ch & ch <= 0x1d244)|(0x1d400 <= ch & ch <= 0x1d454)|(0x1d456 <= ch & ch <= 0x1d49c)|(0x1d49e <= ch & ch <= 0x1d49f)|(ch == 0x1d4a2)|(0x1d4a5 <= ch & ch <= 0x1d4a6)|(0x1d4a9 <= ch & ch <= 0x1d4ac)|(0x1d4ae <= ch & ch <= 0x1d4b9)|(ch == 0x1d4bb)|(0x1d4bd <= ch & ch <= 0x1d4c3)|(0x1d4c5 <= ch & ch <= 0x1d505)|(0x1d507 <= ch & ch <= 0x1d50a)|(0x1d50d <= ch & ch <= 0x1d514)|(0x1d516 <= ch & ch <= 0x1d51c)|(0x1d51e <= ch & ch <= 0x1d539)|(0x1d53b <= ch & ch <= 0x1d53e)|(0x1d540 <= ch & ch <= 0x1d544)|(ch == 0x1d546)|(0x1d54a <= ch & ch <= 0x1d550)|(0x1d552 <= ch & ch <= 0x1d6a5)|(0x1d6a8 <= ch & ch <= 0x1d6c0)|(0x1d6c2 <= ch & ch <= 0x1d6da)|(0x1d6dc <= ch & ch <= 0x1d6fa)|(0x1d6fc <= ch & ch <= 0x1d714)|(0x1d716 <= ch & ch <= 0x1d734)|(0x1d736 <= ch & ch <= 0x1d74e)|(0x1d750 <= ch & ch <= 0x1d76e)|(0x1d770 <= ch & ch <= 0x1d788)|(0x1d78a <= ch & ch <= 0x1d7a8)|(0x1d7aa <= ch & ch <= 0x1d7c2)|(0x1d7c4 <= ch & ch <= 0x1d7cb)|(0x1d7ce <= ch & ch <= 0x1d7ff)|(0x1da00 <= ch & ch <= 0x1da36)|(0x1da3b <= ch & ch <= 0x1da6c)|(ch == 0x1da75)|(ch == 0x1da84)|(0x1da9b <= ch & ch <= 0x1da9f)|(0x1daa1 <= ch & ch <= 0x1daaf)|(0x1e000 <= ch & ch <= 0x1e006)|(0x1e008 <= ch & ch <= 0x1e018)|(0x1e01b <= ch & ch <= 0x1e021)|(0x1e023 <= ch & ch <= 0x1e024)|(0x1e026 <= ch & ch <= 0x1e02a)|(0x1e100 <= ch & ch <= 0x1e12c)|(0x1e130 <= ch & ch <= 0x1e13d)|(0x1e140 <= ch & ch <= 0x1e149)|(ch == 0x1e14e)|(0x1e2c0 <= ch & ch <= 0x1e2f9)|(0x1e800 <= ch & ch <= 0x1e8c4)|(0x1e8d0 <= ch & ch <= 0x1e8d6)|(0x1e900 <= ch & ch <= 0x1e94b)|(0x1e950 <= ch & ch <= 0x1e959)|(0x1ee00 <= ch & ch <= 0x1ee03)|(0x1ee05 <= ch & ch <= 0x1ee1f)|(0x1ee21 <= ch & ch <= 0x1ee22)|(ch == 0x1ee24)|(ch == 0x1ee27)|(0x1ee29 <= ch & ch <= 0x1ee32)|(0x1ee34 <= ch & ch <= 0x1ee37)|(ch == 0x1ee39)|(ch == 0x1ee3b)|(ch == 0x1ee42)|(ch == 0x1ee47)|(ch == 0x1ee49)|(ch == 0x1ee4b)|(0x1ee4d <= ch & ch <= 0x1ee4f)|(0x1ee51 <= ch & ch <= 0x1ee52)|(ch == 0x1ee54)|(ch == 0x1ee57)|(ch == 0x1ee59)|(ch == 0x1ee5b)|(ch == 0x1ee5d)|(ch == 0x1ee5f)|(0x1ee61 <= ch & ch <= 0x1ee62)|(ch == 0x1ee64)|(0x1ee67 <= ch & ch <= 0x1ee6a)|(0x1ee6c <= ch & ch <= 0x1ee72)|(0x1ee74 <= ch & ch <= 0x1ee77)|(0x1ee79 <= ch & ch <= 0x1ee7c)|(ch == 0x1ee7e)|(0x1ee80 <= ch & ch <= 0x1ee89)|(0x1ee8b <= ch & ch <= 0x1ee9b)|(0x1eea1 <= ch & ch <= 0x1eea3)|(0x1eea5 <= ch & ch <= 0x1eea9)|(0x1eeab <= ch & ch <= 0x1eebb)|(0x20000 <= ch & ch <= 0x2a6d6)|(0x2a700 <= ch & ch <= 0x2b734)|(0x2b740 <= ch & ch <= 0x2b81d)|(0x2b820 <= ch & ch <= 0x2cea1)|(0x2ceb0 <= ch & ch <= 0x2ebe0)|(0x2f800 <= ch & ch <= 0x2fa1d)|(0xe0100 <= ch & ch <= 0xe01ef))\n"
     ]
    }
   ],
   "source": [
    "IdentifierPartRanges = get_ranges(IdentifierPart)\n",
    "nRanges = len(IdentifierPartRanges)\n",
    "\n",
    "output = \"\"\n",
    "i = 0\n",
    "for charRange in IdentifierPartRanges:\n",
    "    charRange = [hex(char) for char in charRange]\n",
    "    if len(charRange) == 1:\n",
    "        output += f\"(ch == {charRange[0]})\"\n",
    "    else:\n",
    "        output += f\"({charRange[0]} <= ch & ch <= {charRange[1]})\"\n",
    "        \n",
    "    i += 1\n",
    "    if (i < nRanges):\n",
    "        output += \"|\"\n",
    "        \n",
    "\n",
    "print(f\"\\n({output})\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}