lib/url.py: Strengthen URL escaping.

Use similar implementation of how zulip sanitizes its stream name and
URL into a safe URL.
The only difference is the function now append a non-dot char as the
prefix so that it is safe to use as the filename especially for jekyll
usecase where it ignores file that has dot prefix in its filename.

Closes zulip#35
Closes zulip#51

lib/html.py

@@ -17,7 +17,7 @@
 
 from .url import (
     sanitize_stream,
-    sanitize_topic,
+    sanitize,
 )
 
 from .url import (
@@ -87,7 +87,7 @@ def format_message_html(
         site_url,
         html_root,
         sanitize_stream(stream_name, stream_id),
-        sanitize_topic(topic_name),
+        sanitize(topic_name),
         msg_id,
     )
     anchor_html = '<a name="{0}"></a>'.format(html.escape(msg_id))
@@ -184,7 +184,7 @@ def topic_list_html(topic_data):
     """
 
     def item_html(topic_name, message_data):
-        link_html = f'<a href="topic/{html.escape(sanitize_topic(topic_name))}.html">{html.escape(topic_name)}</a>'
+        link_html = f'<a href="topic/{html.escape(sanitize(topic_name))}.html">{html.escape(topic_name)}</a>'
         topic_info = topic_info_string(message_data)
         return f"<li> {link_html} ({html.escape(topic_info)}) </li>"
 

lib/populate.py

@@ -50,7 +50,7 @@
 )
 from .url import (
     sanitize_stream,
-    sanitize_topic,
+    sanitize,
 )
 
 
@@ -222,7 +222,7 @@ def populate_incremental(
             p = (
                 json_root
                 / Path(sanitize_stream(s["name"], s["stream_id"]))
-                / Path(sanitize_topic(topic_name) + ".json")
+                / Path(sanitize(topic_name) + ".json")
             )
             topic_exists = p.exists()
             old = []
@@ -257,7 +257,7 @@ def dump_topic_messages(json_root, stream_data, topic_name, message_data):
     sanitized_stream_name = sanitize_stream(stream_name, stream_id)
     stream_dir = json_root / Path(sanitized_stream_name)
 
-    sanitized_topic_name = sanitize_topic(topic_name)
+    sanitized_topic_name = sanitize(topic_name)
     topic_fn = sanitized_topic_name + ".json"
 
     out = open_outfile(stream_dir, topic_fn, "w")

lib/url.py

@@ -66,24 +66,15 @@ def archive_message_url(
 
 ## String cleaning functions
 
-# remove non-alnum ascii symbols from string
 def sanitize(s):
+    # Sanitize the string to a safe string that can be used as the URL
+    # It copies zulip's way of encoding stream and topic name to URL
     return (
-        "".join(
-            filter(
-                lambda x: x.isalnum or x == " ",
-                s.encode("ascii", "ignore").decode("utf-8"),
-            )
-        )
-        .replace(" ", "-")
-        .replace("?", "%3F")
-    )
-
-
-# create a unique sanitized identifier for a topic
-def sanitize_topic(topic_name):
-    return (
-        urllib.parse.quote(topic_name, safe="~()*!.'")
+        # We add a prefix of non-dot character so that the resulting string
+        # is safe to use as the filename especially for jekyll usecase
+        # where it ignores file that has dot prefix in its filename
+        "A-" +
+        urllib.parse.quote(s, safe=b"")
         .replace(".", "%2E")
         .replace("%", ".")
     )

lib/website.py

@@ -21,7 +21,7 @@
 
 from .url import (
     sanitize_stream,
-    sanitize_topic,
+    sanitize,
 )
 
 from .files import (
@@ -219,7 +219,7 @@ def write_topic_messages(
     stream_id = stream["id"]
 
     sanitized_stream_name = sanitize_stream(stream_name, stream_id)
-    sanitized_topic_name = sanitize_topic(topic_name)
+    sanitized_topic_name = sanitize(topic_name)
 
     messages = read_zulip_messages_for_topic(
         json_root, sanitized_stream_name, sanitized_topic_name

tests/testCommon.py

@@ -24,11 +24,24 @@ def assert_equal(v1, v2):
 
 
 def test_sanitize():
-    assert_equal(url.sanitize_stream(stream_name="foo bar", stream_id=7), "7-foo-bar")
+    assert_equal(
+        url.sanitize_stream(stream_name="foo bar", stream_id=7),
+        "7-A-foo.20bar")
+    assert_equal(
+        url.sanitize_stream(stream_name="foo/bar/turtle[🐢]", stream_id=7),
+        "7-A-foo.2Fbar.2Fturtle.5B.F0.9F.90.A2.5D")
 
     assert_equal(
-        url.sanitize_topic(topic_name="pick a place for lunch"),
-        "pick.20a.20place.20for.20lunch",
+        url.sanitize("pick a place for lunch"),
+        "A-pick.20a.20place.20for.20lunch",
+    )
+    assert_equal(
+        url.sanitize("!!cute-turlte/tortoise (🐢)?"),
+        "A-.21.21cute-turlte.2Ftortoise.20.28.F0.9F.90.A2.29.3F",
+    )
+    assert_equal(
+        url.sanitize('"the mighty turtle 🐢"'),
+        "A-.22the.20mighty.20turtle.20.F0.9F.90.A2.22",
     )