feat: Use Babel for numbers and dates to support many locales (#67)

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

Author: DenverCoder1

Diff for: api/index.py

@@ -43,11 +43,7 @@ def render():
         thumbnail = data_uri_from_url(f"https://i.ytimg.com/vi/{video_id}/mqdefault.jpg")
         views = fetch_views(video_id)
         views = fetch_views(video_id, lang)
-        diff = (
-            format_relative_time(datetime.fromtimestamp(int(publish_timestamp)), lang)
-            if publish_timestamp
-            else ""
-        )
+        diff = format_relative_time(publish_timestamp, lang) if publish_timestamp else ""
         stats = f"{views} • {diff}" if views and diff else (views or diff)
         duration = seconds_to_duration(duration_seconds)
         duration_width = estimate_duration_width(duration)

Diff for: api/locale/bn.yml

@@ -1,20 +1,3 @@
 bn:
+  view: "1 বার দেখা হয়েছে"
   views: "%{number} বার দেখা হয়েছে"
-  seconds-ago:
-    one: "1 সেকেন্ড আগে"
-    many: "%{count} সেকেন্ড আগে"
-  minutes-ago:
-    one: "1 মিনিট আগে"
-    many: "%{count} মিনিট আগে"
-  hours-ago:
-    one: "1 ঘন্টা আগে"
-    many: "%{count} ঘন্টা আগে"
-  days-ago:
-    one: "1 দিন আগে"
-    many: "%{count} দিন আগে"
-  months-ago:
-    one: "1 মাস আগে"
-    many: "%{count} মাস আগে"
-  years-ago:
-    one: "1 বছর পূর্বে"
-    many: "%{count} বছর পূর্বে"

Diff for: api/locale/en.yml

@@ -1,20 +1,3 @@
 en:
+  view: "1 view"
   views: "%{number} views"
-  seconds-ago:
-    one: "1 second ago"
-    many: "%{count} seconds ago"
-  minutes-ago:
-    one: "1 minute ago"
-    many: "%{count} minutes ago"
-  hours-ago:
-    one: "1 hour ago"
-    many: "%{count} hours ago"
-  days-ago:
-    one: "1 day ago"
-    many: "%{count} days ago"
-  months-ago:
-    one: "1 month ago"
-    many: "%{count} months ago"
-  years-ago:
-    one: "1 year ago"
-    many: "%{count} years ago"

Diff for: api/locale/fa.yml

@@ -1,21 +1,4 @@
 fa:
   direction: rtl
+  view: "1 بازدید"
   views: "%{number} بازدید"
-  seconds-ago:
-    one: "یک ثانیه پیش"
-    many: "%{count} ثانیه پیش"
-  minutes-ago:
-    one: "یک دقیقه پیش"
-    many: "%{count} دقیقه پیش"
-  hours-ago:
-    one: "یک ساعت پیش"
-    many: "%{count} ساعت پیش"
-  days-ago:
-    one: "یک روز پیش"
-    many: "%{count} روز پیش"
-  months-ago:
-    one: "یک ماه پیش"
-    many: "%{count} ماه پیش"
-  years-ago:
-    one: "یک سال پیش"
-    many: "%{count} سال پیش"

Diff for: api/locale/fr.yml

@@ -1,20 +1,3 @@
 fr:
+  view: "1 vue"
   views: "%{number} vues"
-  seconds-ago:
-    one: "il y a 1 seconde"
-    many: "il y a %{count} secondes"
-  minutes-ago:
-    one: "il y a 1 minute"
-    many: "il y a %{count} minutes"
-  hours-ago:
-    one: "il y a 1 heure"
-    many: "il y a %{count} heures"
-  days-ago:
-    one: "il y a 1 jour"
-    many: "il y a %{count} jours"
-  months-ago:
-    one: "il y a 1 mois"
-    many: "il y a %{count} mois"
-  years-ago:
-    one: "il y a 1 an"
-    many: "il y a %{count} ans"

Diff for: api/locale/he.yml

@@ -1,21 +1,4 @@
 he:
   direction: rtl
+  view: "1 צפייה"
   views: "%{number} צפיות"
-  seconds-ago:
-    one: "לפני שניה"
-    many: "לפני %{count} שניות"
-  minutes-ago:
-    one: "לפני דקה"
-    many: "לפני %{count} דקות"
-  hours-ago:
-    one: "לפני שעה"
-    many: "לפני %{count} שעות"
-  days-ago:
-    one: "אתמול"
-    many: "לפני %{count} ימים"
-  months-ago:
-    one: "לפני חודש"
-    many: "לפני %{count} חודשים"
-  years-ago:
-    one: "לפני שנה"
-    many: "לפני %{count} שנים"

Diff for: api/locale/hi.yml

@@ -1,20 +1,3 @@
 hi:
+  view: "1 बार देखा गया"
   views: "%{number} बार देखा गया"
-  seconds-ago:
-    one: "1 सेकंड पहले"
-    many: "%{count} सेकंड पहले"
-  minutes-ago:
-    one: "1 मिनट पहले"
-    many: "%{count} मिनट पहले"
-  hours-ago:
-    one: "1 घंटे पहले"
-    many: "%{count} घंटे पहले"
-  days-ago:
-    one: "1 दिन पहले"
-    many: "%{count} दिन पहले"
-  months-ago:
-    one: "1 माह पहले"
-    many: "%{count} माह पहले"
-  years-ago:
-    one: "1 वर्ष पहले"
-    many: "%{count} वर्ष पहले"

Diff for: api/locale/id.yml

@@ -1,20 +1,3 @@
 id:
+  view: "1 ditonton"
   views: "%{number} ditonton"
-  seconds-ago:
-    one: "1 detik yang lalu"
-    many: "%{count} detik yang lalu"
-  minutes-ago:
-    one: "1 menit yang lalu"
-    many: "%{count} menit yang lalu"
-  hours-ago:
-    one: "1 jam yang lalu"
-    many: "%{count} jam yang lalu"
-  days-ago:
-    one: "1 hari yang lalu"
-    many: "%{count} hari yang lalu"
-  months-ago:
-    one: "1 bulan yang lalu"
-    many: "%{count} bulan yang lalu"
-  years-ago:
-    one: "1 tahun yang lalu"
-    many: "%{count} tahun yang lalu"

Diff for: api/locale/ur.yml

@@ -1,21 +1,4 @@
 ur:
   direction: rtl
+  view: "1 ملاحظة"
   views: "%{number} ملاحظات"
-  seconds-ago:
-    one: "1 سیکنڈ پہلے"
-    many: "%{count} سیکنڈ پہلے"
-  minutes-ago:
-    one: "1 منٹ پہلے"
-    many: "%{count} منٹ پہلے"
-  hours-ago:
-    one: "1 گھنٹہ پہلے"
-    many: "%{count} گھنٹے پہلے"
-  days-ago:
-    one: "1 دن پہلے"
-    many: "%{count} دن پہلے"
-  months-ago:
-    one: "1 مہینہ پہلے"
-    many: "%{count} مہینے پہلے"
-  years-ago:
-    one: "1 سال پہلے"
-    many: "%{count} سال پہلے"

Diff for: api/utils.py

@@ -1,52 +1,21 @@
 import codecs
-from datetime import datetime
+from datetime import datetime, timedelta
 from typing import Optional
 from urllib.request import Request, urlopen
 
 import i18n
 import orjson
+from babel import Locale, dates, numbers
 
 i18n.set("filename_format", "{locale}.{format}")
 i18n.set("enable_memoization", True)
 i18n.load_path.append("./api/locale")
 
 
-def format_relative_time(date: datetime, lang: str = "en") -> str:
-    """Get relative time from datetime (ex. "3 hours ago")"""
-    # find time difference in seconds
-    seconds_diff = int((datetime.now() - date).total_seconds())
-    # number of days in difference
-    days_diff = seconds_diff // 86400
-    # less than 50 seconds ago
-    if seconds_diff < 50:
-        return i18n.t("seconds-ago", count=seconds_diff, locale=lang)
-    # less than 2 minutes ago
-    if seconds_diff < 120:
-        return i18n.t("minutes-ago", count=1, locale=lang)
-    # less than an hour ago
-    if seconds_diff < 3600:
-        return i18n.t("minutes-ago", count=seconds_diff // 60, locale=lang)
-    # less than 2 hours ago
-    if seconds_diff < 7200:
-        return i18n.t("hours-ago", count=1, locale=lang)
-    # less than 24 hours ago
-    if seconds_diff < 86400:
-        return i18n.t("hours-ago", count=seconds_diff // 3600, locale=lang)
-    # 1 day ago
-    if days_diff == 1:
-        return i18n.t("days-ago", count=1, locale=lang)
-    # less than a month ago
-    if days_diff < 30:
-        return i18n.t("days-ago", count=days_diff, locale=lang)
-    # less than 12 months ago
-    if days_diff < 336:
-        if round(days_diff / 30.5) == 1:
-            return i18n.t("months-ago", count=1, locale=lang)
-        return i18n.t("months-ago", count=round(days_diff / 30.5), locale=lang)
-    # more than a year ago
-    if round(days_diff / 365) == 1:
-        return i18n.t("years-ago", count=1, locale=lang)
-    return i18n.t("years-ago", count=round(days_diff / 365), locale=lang)
+def format_relative_time(timestamp: float, lang: str = "en") -> str:
+    """Get relative time from unix timestamp (ex. "3 hours ago")"""
+    delta = timedelta(seconds=timestamp - datetime.now().timestamp())
+    return dates.format_timedelta(delta=delta, add_direction=True, locale=lang)
 
 
 def data_uri_from_bytes(*, data: bytes, mime_type: str) -> str:
@@ -93,19 +62,67 @@ def trim_text(text: str, max_length: int) -> str:
     return text[: max_length - 1].strip() + "…"
 
 
+def format_decimal_compact(number: float, lang: str = "en") -> str:
+    """Format number with compact notation (ex. "1.2K")
+
+    TODO: This can be refactored once it is supported by Babel
+    Sees https://github.com/python-babel/babel/pull/909
+    """
+    locale = Locale.parse(lang)
+    compact_format = locale._data["compact_decimal_formats"]["short"]
+    number_format = None
+    for magnitude in sorted([int(m) for m in compact_format["other"]], reverse=True):
+        if abs(number) >= magnitude:
+            # check the pattern using "other" as the amount
+            number_format = compact_format["other"][str(magnitude)]
+            pattern = numbers.parse_pattern(number_format).pattern
+            # if the pattern is "0", we do not divide the number
+            if pattern == "0":
+                break
+            # otherwise, we need to divide the number by the magnitude but remove zeros
+            # equal to the number of 0's in the pattern minus 1
+            number = number / (magnitude / (10 ** (pattern.count("0") - 1)))
+            # round to the number of fraction digits requested
+            number = round(number, 1)
+            # if the remaining number is like "one", use the singular format
+            plural_form = locale.plural_form(abs(number))
+            plural_form = plural_form if plural_form in compact_format else "other"
+            number_format = compact_format[plural_form][str(magnitude)]
+            break
+    return numbers.format_decimal(
+        number=number, format=number_format, locale=lang, decimal_quantization=False
+    )
+
+
+def parse_metric_value(value: str) -> int:
+    """Parse a metric value (ex. "1.2K" => 1200)
+
+    See https://github.com/badges/shields/blob/master/services/text-formatters.js#L56
+    for the reverse of this function.
+    """
+    suffixes = ["k", "M", "G", "T", "P", "E", "Z", "Y"]
+    if value[-1] in suffixes:
+        return int(float(value[:-1]) * 1000 ** (suffixes.index(value[-1]) + 1))
+    return int(value)
+
+
+def format_views_value(value: str, lang: str = "en") -> str:
+    """Format view count, for example "1.2M" => "1.2M views", translations included"""
+    int_value = parse_metric_value(value)
+    if int_value == 1:
+        return i18n.t("view", locale=lang)
+    formatted_value = format_decimal_compact(int_value, lang=lang)
+    return i18n.t("views", number=formatted_value, locale=lang)
+
+
 def fetch_views(video_id: str, lang: str = "en") -> str:
     """Get number of views for a YouTube video as a formatted metric"""
     try:
         req = Request(f"https://img.shields.io/youtube/views/{video_id}.json")
         req.add_header("User-Agent", "GitHub Readme YouTube Cards")
         with urlopen(req) as response:
             value = orjson.loads(response.read()).get("value", "")
-            # replace G with B for billion and convert to uppercase
-            return (
-                i18n.t("views", number=value.replace("G", "B").upper(), locale=lang)
-                if value
-                else ""
-            )
+            return format_views_value(value, lang)
     except Exception:
         return ""
 

Diff for: api/validate.py

@@ -1,6 +1,6 @@
-import os
 import re
 
+from babel import Locale, UnknownLocaleError
 from flask.wrappers import Request
 
 
@@ -40,12 +40,12 @@ def validate_string(req: Request, field: str, default: str = "") -> str:
 
 
 def validate_lang(req: Request, field: str, *, default: str = "en") -> str:
-    """Validate a string with a locale lang, returns the string if translations
-    are present in the locale directory, otherwise the default.
+    """Validate a string with a locale lang, returns the string if the locale
+    is known by Babel, otherwise the default.
     """
-    value = req.args.get(field, "")
-    # if there is no yaml file for the language, use the default
-    if not os.path.isfile(f"./api/locale/{value}.yml"):
-        return default
-
+    value = req.args.get(field, default)
+    try:
+        Locale.parse(value)
+    except UnknownLocaleError:
+        value = default
     return value

Diff for: requirements.txt

@@ -1,4 +1,5 @@
 Flask==2.2.2
 gunicorn>=20.1.0
 orjson==3.8.0
-python-i18n[yaml]==0.3.9
+python-i18n[yaml]==0.3.9
+Babel==2.10.3

Diff for: tests/test_app.py

@@ -93,5 +93,5 @@ def test_request_right_to_left(client):
     assert 'style="direction: rtl"' in data
 
     # test views
-    views_regex = re.compile(r"\d+(?:\.\d)?[KMBT]? צפיות")
+    views_regex = re.compile(r"\d+(?:\.\d)?[KMBT]?\u200f צפיות")
     assert views_regex.search(data) is not None