retronews

commit f8144103cab4bfb8a67d964f71b016b716e48206
parent 6110e268cf603a82ce7b37c0337a9fe976c46b2c
Author: luke8086 <55237178+luke8086@users.noreply.github.com>
Date:   Tue,  2 Aug 2022 14:13:09 +0000

Minor refactoring and basic tests for html parser

Diffstat:
M
Makefile
 | 
5
++++-
M
retronews.py
 | 
24
++++++++++++------------
A
tests.py
 | 
14
++++++++++++++

3 files changed, 30 insertions(+), 13 deletions(-)
diff --git a/Makefile b/Makefile
@@ -1,4 +1,4 @@
-SCRIPTS = retronews.py
+SCRIPTS = retronews.py tests.py
 
 .PHONY: venv-check
 venv-check:
@@ -22,3 +22,6 @@ lint: venv-check
 	isort $(SCRIPTS)
 	flake8 $(SCRIPTS)
 	mypy $(SCRIPTS)
+
+test:
+	python3 tests.py
diff --git a/retronews.py b/retronews.py
@@ -20,7 +20,7 @@ import sqlite3
 import sys
 import urllib.request
 from datetime import datetime
-from functools import partial
+from functools import partial, reduce
 from textwrap import wrap
 from typing import (
     Any,
@@ -293,13 +293,6 @@ class HTMLParser(html.parser.HTMLParser):
         self.after_pre = tag == "pre"
 
 
-def parse_html(html: str) -> str:
-    parser = HTMLParser()
-    parser.feed(html)
-    parser.close()
-    return parser.text.strip("\n")
-
-
 def wrap_paragraph(text: str) -> list[str]:
     if len(text) == 0:
         # Preserve empty lines
@@ -318,6 +311,16 @@ def wrap_paragraph(text: str) -> list[str]:
     return wrap(text, subsequent_indent=indent, break_on_hyphens=False, break_long_words=False)
 
 
+def parse_html(html: str) -> list[str]:
+    parser = HTMLParser()
+    parser.feed(html)
+    parser.close()
+
+    raw_lines = parser.text.strip("\n").split("\n")
+
+    return reduce(lambda acc, p: acc + wrap_paragraph(p), raw_lines, [])
+
+
 def fetch(url: str) -> str:
     logging.debug(f"Fetching '{url}'...")
 
@@ -570,10 +573,7 @@ def msg_build_lines(msg: Message) -> list[str]:
         "",
     ]
 
-    text = parse_html(msg.body or "")
-
-    for p in text.split("\n"):
-        lines += wrap_paragraph(p)
+    lines += parse_html(msg.body or "")
 
     return lines
 
diff --git a/tests.py b/tests.py
@@ -0,0 +1,14 @@
+import unittest
+
+import retronews
+
+
+class TestHtmlParser(unittest.TestCase):
+    def test_expanding_links(self):
+        html = '<a href="https://example.com/foo/bar">https://example.com/foo...</a>'
+        lines = retronews.parse_html(html)
+        self.assertListEqual(lines, ["https://example.com/foo/bar"])
+
+
+if __name__ == "__main__":
+    unittest.main()