retronews.py (52938B)
1 #!/usr/bin/env python3 2 # 3 # Copyright (c) luke8086 4 # 5 # This program is free software: you can redistribute it and/or modify 6 # it under the terms of the GNU General Public License version 2 as published by 7 # the Free Software Foundation. 8 # 9 10 import sys 11 12 # if sys.version_info < (3, 9): 13 # sys.stderr.write("Python 3.9 or newer is required.\n") 14 # sys.exit(1) 15 16 import argparse 17 import curses 18 import curses.textpad 19 import html.parser 20 import json 21 import logging 22 import os 23 import re 24 import sqlite3 25 import traceback 26 import unicodedata 27 import urllib.request 28 import webbrowser 29 import time 30 from collections import defaultdict 31 from datetime import datetime 32 from functools import partial, reduce 33 from textwrap import wrap 34 35 USER_AGENT = "retronews" 36 37 KEY_BINDINGS = { 38 ord("q"): lambda app: cmd_quit(app), 39 ord("?"): lambda app: cmd_help(app), 40 ord("\n"): lambda app: cmd_open(app), 41 ord(" "): lambda app: cmd_open(app), 42 ord("o"): lambda app: cmd_show_links(app), 43 ord("x"): lambda app: cmd_close(app), 44 ord("s"): lambda app: cmd_star(app), 45 ord("S"): lambda app: cmd_star_thread(app), 46 ord("u"): lambda app: cmd_set_unread(app), 47 ord("D"): lambda app: cmd_dump(app), 48 ord("r"): lambda app: cmd_toggle_raw_mode(app), 49 ord("k"): lambda app: cmd_up(app), 50 ord("j"): lambda app: cmd_down(app), 51 ord("p"): lambda app: cmd_prev(app), 52 ord("n"): lambda app: cmd_next(app), 53 ord("N"): lambda app: cmd_next_unread(app), 54 ord("P"): lambda app: cmd_parent(app), 55 ord(";"): lambda app: cmd_mark_set(app), 56 ord(","): lambda app: cmd_mark_jump(app), 57 ord("R"): lambda app: cmd_reload_page(app), 58 ord("<"): lambda app: cmd_load_prev_page(app), 59 ord(">"): lambda app: cmd_load_next_page(app), 60 ord("g"): lambda app: cmd_load_page(app), 61 curses.KEY_UP: lambda app: cmd_prev(app), 62 curses.KEY_DOWN: lambda app: cmd_next(app), 63 curses.KEY_PPAGE: lambda app: cmd_page_up(app), 64 curses.KEY_NPAGE: lambda app: cmd_page_down(app), 65 curses.KEY_RESIZE: lambda app: cmd_resize(app), 66 } 67 KEY_BINDINGS.update({ord(str(i)): lambda app, i=i: cmd_load_tab(app, i) for i in range(1, 10)}) 68 69 HELP_MENU = "q:Quit ?:Help p:Prev n:Next N:Next-Unread j:Down k:Up x:Close s:Star" 70 71 HELP_SCREEN = """\ 72 q Quit retronews 73 UP, DOWN Go up / down by one message / pager line 74 PG UP, PG DOWN Gp up / down by one page of messages / pager lines 75 p, n Go to previous / next message 76 N Go to next unread message 77 P Go to parent message 78 ; , Set mark, jump to mark & swap (valid within thread) 79 RETURN, SPACE Open selected message 80 x Close current message / thread 81 o Select link and open in browser 82 1 - 9 Change group 83 R Refresh current page 84 < > Go to previous / next page 85 g Go to specific page 86 k j Scroll pager up / down by one line 87 s Star / unstar current message 88 S Star / unstar current thread 89 u Mark current message as unread 90 r Toggle raw HTML mode 91 92 See https://github.com/luke8086/retronews for more information.""" 93 94 COLORS = { 95 "author": (curses.COLOR_YELLOW, -1), 96 "code": (curses.COLOR_GREEN, -1), 97 "cursor": (curses.COLOR_BLACK, curses.COLOR_CYAN), 98 "date": (curses.COLOR_CYAN, -1), 99 "default": (curses.COLOR_WHITE, -1), 100 "empty_pager_line": (curses.COLOR_GREEN, -1), 101 "deleted_message_pager_line": (curses.COLOR_RED, -1), 102 "menu": (curses.COLOR_GREEN, curses.COLOR_BLUE), 103 "menu_active": (curses.COLOR_YELLOW, curses.COLOR_BLUE), 104 "nested_quote": (curses.COLOR_CYAN, -1), 105 "quote": (curses.COLOR_YELLOW, -1), 106 "starred_subject": (curses.COLOR_CYAN, -1), 107 "header_subject": (curses.COLOR_GREEN, -1), 108 "tree": (curses.COLOR_RED, -1), 109 "unread_comments": (curses.COLOR_GREEN, -1), 110 "url": (curses.COLOR_MAGENTA, -1), 111 } 112 113 PREFERRED_PAGE_SIZE = 30 114 UNREAD_SIZE = 3 115 UNREAD_MANY_CHAR = '!' 116 AUTHOR_SIZE = 10 117 COLUMN_SPACING = 2 118 DATECOL_SIZE = 16 119 DATECOL_FUNC = lambda date: date.strftime("%Y-%m-%d %H:%M") 120 121 REQUEST_TIMEOUT = 10 122 123 # Recognize ">text", "> text", ">>text", ">> text", etc. 124 QUOTE_REX = re.compile(r"^(> ?)+") 125 126 # Recognize "[n] link", "[n]: link", "[n] - link", etc. 127 REFERENCE_REX = re.compile(r"^\[\d+\][ :-]*https?://[^ ]*$") 128 129 # Recognize http/https URLs 130 URL_REX = re.compile(r"(https?://[^\s\)\"<,]+[^\s\)\"<,\.])") 131 132 # Recognize HN message URLs 133 HN_URL_REX = re.compile(r"^https://news\.ycombinator\.com/item\?id=(\d+)$") 134 LB_URL_REX = re.compile(r"^https://lobste\.rs/s/([a-z0-9]{6}).*$") 135 136 TRUNCATE = lambda s,l: s[:l] 137 138 139 class ExitException(Exception): 140 def __init__(self, code = 0, message = ""): 141 self.code = code 142 self.message = message 143 144 super().__init__(message) 145 146 147 class Provider: 148 def __init__(self, fetch_thread, fetch_threads_by_id): 149 self.fetch_thread = fetch_thread 150 self.fetch_threads_by_id = fetch_threads_by_id 151 152 PROVIDERS = { 153 "hn": Provider( 154 fetch_thread=lambda msg_id: hn_fetch_thread(msg_id), 155 fetch_threads_by_id=lambda msg_ids: hn_fetch_threads_by_id(msg_ids), 156 ), 157 "lb": Provider( 158 fetch_thread=lambda msg_id: lb_fetch_thread(msg_id), 159 fetch_threads_by_id=lambda msg_ids: [lb_fetch_thread(x) for x in msg_ids], 160 ), 161 } 162 163 164 class Group: 165 def __init__(self, label, fetch, page=0): 166 self.label = label 167 self.fetch = fetch 168 self.page = page 169 170 171 GROUP_TABS = [ 172 Group(label="Front HN", fetch=lambda db, page: hn_fetch_threads("news", page)), 173 Group(label="New HN", fetch=lambda db, page: hn_fetch_new_threads(page)), 174 Group(label="Ask HN", fetch=lambda db, page: hn_fetch_threads("ask", page)), 175 Group(label="Show HN", fetch=lambda db, page: hn_fetch_threads("show", page)), 176 Group(label="Active HN", fetch=lambda db, page: hn_fetch_threads("active", page)), 177 Group(label="Front LB", fetch=lambda db, page: lb_fetch_threads("", page)), 178 Group(label="New LB", fetch=lambda db, page: lb_fetch_threads("newest", page)), 179 Group(label="Starred", fetch=lambda db, page: group_fetch_starred_threads(db, page)), 180 ] 181 182 183 class MessageFlags: 184 def __init__(self, read=False, starred=False): 185 self.read = False 186 self.starred = False 187 188 189 class Message: 190 def __init__(self, msg_id, thread_id, content_location, date, author, title, 191 body=None, children=None, total_comments=0, parent=None, url=None): 192 self.msg_id = msg_id 193 self.thread_id = thread_id 194 self.content_location = content_location 195 self.date = date 196 self.author = author 197 self.title = title 198 self.body = body 199 self.children = children 200 self.total_comments = total_comments 201 self.parent = parent 202 self.lines = [] 203 self.flags = MessageFlags() 204 self.read_comments = 0 205 self.index_position = 0 206 self.index_tree = "" 207 self.url = url 208 209 @property 210 def is_read(self): 211 return self.flags.read 212 213 @property 214 def is_shown_as_read(self): 215 # If the message is an unloaded thread, check if all comments are read 216 return self.read_comments >= self.total_comments if self.is_thread and self.children is None else self.is_read 217 218 @property 219 def is_thread(self): 220 return self.msg_id == self.thread_id 221 222 @property 223 def is_deleted(self): 224 return self.author is None 225 226 227 class Layout: 228 def __init__(self): 229 self.lines = 0 230 self.cols = 0 231 self.top_menu_row = 0 232 self.index_start = 1 233 self.index_height = 0 234 self.middle_menu_row = None 235 self.pager_start = None 236 self.pager_height = None 237 self.bottom_menu_row = 0 238 self.flash_menu_row = 0 239 240 241 class AppState: 242 def __init__(self, screen, db, group, ascii=False, monochrome=False): 243 self.screen = screen 244 self.db = db 245 self.group = group 246 self.ascii = ascii 247 self.monochrome = monochrome 248 self.colors = {} 249 self.messages = [] 250 self.messages_by_id = {} 251 self.selected_message = None 252 self.marked_message_id = "" 253 self.layout = Layout() 254 self.pager_visible = False 255 self.pager_offset = 0 256 self.raw_mode = False 257 self.flash = None 258 259 260 HTML_BLOCK_TAGS = set(("root", "p", "pre", "blockquote", "ul", "ol", "li", "hr")) 261 HTML_INLINE_TAGS = set(("code", "a", "em", "strong", "b", "br")) 262 HTML_KNOWN_TAGS = HTML_BLOCK_TAGS.union(HTML_INLINE_TAGS) 263 HTML_AUTOCLOSE_TAGS = set(("hr", "br")) 264 265 266 class HTMLNode: 267 def __init__(self, tag, attrs=None, text="", pre=False): 268 self.tag = tag 269 self.text = text 270 self.pre = pre 271 self.attrs = {} if attrs is None else attrs 272 self.parent = None 273 self.prev_sibling = None 274 self.next_sibling = None 275 self.first_child = None 276 self.last_child = None 277 278 279 class HTMLParser(html.parser.HTMLParser): 280 def __init__(self): 281 super().__init__() 282 self.pre_level = 0 283 self.root_node = self.current_node = HTMLNode(tag="root") 284 285 def handle_data(self, data): 286 if self.current_node.tag in HTML_AUTOCLOSE_TAGS: 287 self.handle_endtag(self.current_node.tag) 288 289 node = HTMLNode(tag="text", text=data, pre=self.pre_level > 0) 290 html_node_append(self.current_node, node) 291 292 def handle_starttag(self, tag, attrs): 293 if self.current_node.tag in HTML_AUTOCLOSE_TAGS: 294 self.handle_endtag(self.current_node.tag) 295 296 if tag not in HTML_KNOWN_TAGS: 297 return 298 299 if tag == "pre": 300 self.pre_level += 1 301 302 node = HTMLNode(tag=tag, attrs=dict(attrs), pre=self.pre_level > 0) 303 html_node_append(self.current_node, node) 304 self.current_node = node 305 306 def handle_endtag(self, tag): 307 if tag not in HTML_KNOWN_TAGS: 308 return 309 310 if tag == "pre": 311 self.pre_level = max(0, self.pre_level - 1) 312 313 while True: 314 node = self.current_node 315 316 if node.parent is None: 317 break 318 319 self.current_node = node.parent 320 321 if node.tag == tag: 322 break 323 324 325 def text_wrap(text, width=70): 326 if len(text) == 0: 327 # Preserve empty lines 328 return "" 329 330 if text.startswith(" "): 331 # Preserve code indentation 332 return text 333 334 if REFERENCE_REX.match(text): 335 # Keep reference numbers with long links in the same line 336 return text 337 338 indent = "" 339 340 match = QUOTE_REX.match(text) 341 if match is not None: 342 # Preserve quotation symbols in subsequent lines 343 indent = match[0] 344 345 lines = wrap(text, width, subsequent_indent=indent, break_on_hyphens=False, break_long_words=False) 346 lines = [line.rstrip() for line in lines] 347 348 return "\n".join(lines) 349 350 351 def text_clean(text, ascii = False): 352 """Cleanup text for rendering, currently only removes non-ascii characters in ascii mode""" 353 354 if ascii: 355 text = text.encode("ascii", "replace").decode("ascii") 356 357 return text 358 359 360 def text_unindent(text): 361 lines = text.split("\n") 362 363 while all(line.startswith(" ") or line == "" for line in lines): 364 lines = [line[1:] for line in lines] 365 366 return "\n".join(lines) 367 368 369 def text_sanitize(text): 370 # For safety, remove any control characters except for \n and \t 371 # At least on HN some messages contain \x00 characters 372 373 text = text or "" 374 allowed_cc = set(("\n", "\t")) 375 chars = (c for c in text if c in allowed_cc or unicodedata.category(c) != "Cc") 376 text = "".join(chars) 377 378 return text 379 380 381 def text_split_urls(text): 382 return [p for p in URL_REX.split(text) if p != ""] 383 384 385 def html_node_children(parent): 386 ret = [] 387 node = parent.first_child 388 389 while node is not None: 390 ret.append(node) 391 node = node.next_sibling 392 393 return ret 394 395 396 def html_node_append(parent, child): 397 if parent.first_child is None: 398 parent.first_child = child 399 400 if parent.last_child is not None: 401 parent.last_child.next_sibling = child 402 child.prev_sibling = parent.last_child 403 404 parent.last_child = child 405 child.parent = parent 406 407 408 def html_node_unlink(node): 409 if node.prev_sibling: 410 node.prev_sibling.next_sibling = node.next_sibling 411 412 if node.next_sibling: 413 node.next_sibling.prev_sibling = node.prev_sibling 414 415 if node.parent and node.parent.first_child is node: 416 node.parent.first_child = node.next_sibling 417 418 if node.parent and node.parent.last_child is node: 419 node.parent.last_child = node.prev_sibling 420 421 node.parent = node.prev_sibling = node.next_sibling = None 422 423 424 def html_node_dump(node): 425 lines = [] 426 lines.append("{} {}".format(node.tag, repr(node.attrs))) 427 428 for child in html_node_children(node): 429 if child.tag == "text": 430 lines.append(" text " + repr(child.text)) 431 else: 432 lines += [" " + line for line in html_node_dump(child).split("\n")] 433 434 return "\n".join(lines) 435 436 437 def html_node_trim_whitespace(node): 438 if node.pre: 439 node.text = node.text.rstrip("\r\n\t ") 440 return 441 442 text = node.text.strip("\r\n\t ") 443 text = re.sub(r"[\r\n\t ]+", " ", text) 444 text = text.replace("\x00", "\n") 445 text = re.sub(r" *\n *", "\n", text) 446 447 node.text = text 448 449 450 def html_node_process_inline(node, inline=False): 451 """Traverse tree flattening all inline nodes into text nodes""" 452 453 if node.tag not in HTML_BLOCK_TAGS: 454 # Never disable inline if already enabled 455 inline = True 456 457 text = "".join(html_node_process_inline(c, inline) for c in html_node_children(node)) 458 459 if not inline: 460 return "" 461 462 if node.tag == "text": 463 text = node.text 464 elif node.tag == "br": 465 text = "\x00" 466 elif node.tag == "em" or node.tag == "i": 467 text = "/{}/".format(text) 468 elif node.tag == "strong" or node.tag == "b": 469 text = "*{}*".format(text) 470 elif node.tag == "code" and not node.pre: 471 text = "`{}`".format(text) 472 elif node.tag == "a": 473 href = node.attrs.get("href", "") or "" 474 if text.endswith("...") and href.startswith(text[:-3]): 475 # Workaround for link formatting on HN 476 text = href 477 elif text != href: 478 text = "{} {}".format(text, href) 479 480 node.tag = "text" 481 node.text = text 482 node.first_child = node.last_child = None 483 484 return text 485 486 487 def html_node_process_text(node): 488 """Traverse tree merging, trimming and pruning text nodes""" 489 490 for child in html_node_children(node): 491 html_node_process_text(child) 492 493 # Merge adjacent text nodes 494 for child in html_node_children(node): 495 prev = child.prev_sibling 496 if child.tag == "text" and prev is not None and prev.tag == "text" and child.pre == prev.pre: 497 child.text = prev.text + child.text 498 html_node_unlink(prev) 499 500 # Trim whitespace from text nodes 501 for child in html_node_children(node): 502 if child.tag == "text": 503 html_node_trim_whitespace(child) 504 505 # Remove empty text nodes 506 for child in html_node_children(node): 507 if child.tag == "text" and child.text == "": 508 html_node_unlink(child) 509 510 511 def html_node_render_block(node, width=70): 512 if node.tag == "blockquote" or node.tag == "li" or node.tag == "pre": 513 width -= 2 514 515 parts = [] 516 517 if node.tag == "hr": 518 parts.append("-" * width) 519 520 for child in html_node_children(node): 521 if child.tag == "text" and child.pre: 522 parts.append(child.text) 523 524 elif child.tag == "text" and not child.pre: 525 subparts = [text_wrap(p, width) for p in child.text.split("\n")] 526 parts.append("\n".join(subparts)) 527 528 else: 529 parts.append(html_node_render_block(child, width)) 530 531 if child.next_sibling is not None and child.tag != "li": 532 parts.append("") 533 534 text = "\n".join(parts) 535 536 if node.tag == "blockquote": 537 text = "\n".join((">" if line.startswith("> ") else "> ") + line for line in text.split("\n")) 538 elif node.tag == "pre": 539 text = text_unindent(text) 540 text = "\n".join("| " + line for line in text.split("\n")) 541 elif node.tag == "li": 542 lines = text.split("\n") 543 lines = [("- " + lines[0]).rstrip()] + [(" " + line).rstrip() for line in lines[1:]] 544 text = "\n".join(lines) 545 546 return text 547 548 549 def html_render(html): 550 # This renderer works well for HN messages because their markup is simple, and it can do 551 # some custom optimizations, like expanding ellipsis-shortened links, preserving quote 552 # symbols in wrapped lines, and preventing references with long urls from being broken 553 # into separate lines. For other backends it may make more sense to use an external app 554 # (links, w3m, etc) 555 556 html = text_sanitize(html) 557 558 parser = HTMLParser() 559 parser.feed(html) 560 parser.close() 561 562 node = parser.root_node 563 564 log_sep = "\n" + "-" * 80 + "\n" 565 logging.debug("Initial HTML tree{}{}{}".format(log_sep, html_node_dump(node), log_sep)) 566 567 html_node_process_inline(node) 568 html_node_process_text(node) 569 570 logging.debug("Processed HTML tree{}{}{}".format(log_sep, html_node_dump(node), log_sep)) 571 572 return html_node_render_block(node) 573 574 575 def fetch(url): 576 logging.debug("Fetching '{}'...".format(url)) 577 578 headers = {} 579 if USER_AGENT is not None: 580 headers["User-Agent"] = USER_AGENT 581 582 req = urllib.request.Request(url, headers=headers) 583 resp = urllib.request.urlopen(req, timeout=REQUEST_TIMEOUT).read().decode() 584 585 return resp 586 587 588 def list_get(lst, index, default=None): 589 return lst[index] if 0 <= index < len(lst) else default 590 591 592 def list_chunk(lst, size): 593 # Flake8 conflicts with Black here - https://github.com/PyCQA/pycodestyle/issues/373 594 return [lst[i : i + size] for i in range(0, len(lst), size)] # noqa: E203 595 596 597 def cmd_quit(_): 598 raise ExitException() 599 600 601 def cmd_help(app): 602 app_show_help_screen(app) 603 604 605 def cmd_show_links(app): 606 app_show_links_screen(app) 607 608 609 def cmd_up(app): 610 cmd_pager_up(app) if app.pager_visible else cmd_prev(app) 611 612 613 def cmd_down(app): 614 cmd_pager_down(app) if app.pager_visible else cmd_next(app) 615 616 617 def cmd_prev(app): 618 pos = app.selected_message.index_position - 1 if app.selected_message else 0 619 app_select_message(app, list_get(app.messages, pos, app.selected_message)) 620 621 622 def cmd_next(app): 623 pos = app.selected_message.index_position + 1 if app.selected_message else 0 624 app_select_message(app, list_get(app.messages, pos, app.selected_message)) 625 626 627 def cmd_next_unread(app): 628 pos = app.selected_message.index_position + 1 if app.selected_message else 0 629 message = next((msg for msg in app.messages[pos:] if not msg.is_shown_as_read), None) 630 if message is not None: 631 app_select_message(app, message) 632 633 634 def cmd_next_sibling(app): 635 msg = app.selected_message 636 if msg is None: 637 return 638 parent_msg = msg.parent 639 if parent_msg and parent_msg.children: 640 # find the next sibling 641 try: 642 idx = parent_msg.children.index(msg) 643 if idx < len(parent_msg.children): 644 app_select_message(app, parent_msg.children[idx + 1]) 645 except IndexError: 646 pass 647 648 649 def cmd_prev_sibling(app): 650 msg = app.selected_message 651 if msg is None: 652 return 653 parent_msg = msg.parent 654 if parent_msg and parent_msg.children: 655 # find the previous sibling 656 try: 657 idx = parent_msg.children.index(msg) 658 if idx > 0: 659 app_select_message(app, parent_msg.children[idx - 1]) 660 except IndexError: 661 pass 662 663 664 def cmd_mark_thread_as_read(app): 665 # recursively mark us and all children as read 666 # then jump to the next sibling 667 def iterate(message): 668 if message.children: 669 for child in message.children: 670 child.flags.read = True 671 db_save_message(app.db, child) 672 iterate(child) 673 674 msg = app.selected_message 675 if msg is not None: 676 iterate(msg) 677 # jump to the next sibling 678 cmd_next_sibling(app) 679 680 681 def cmd_parent(app): 682 if app.selected_message is not None and app.selected_message.parent is not None: 683 app_select_message(app, app.selected_message.parent) 684 685 686 def cmd_mark_set(app): 687 if app.selected_message is not None: 688 app.marked_message_id = app.selected_message.msg_id 689 app_show_flash(app, "Mark set") 690 691 692 def cmd_mark_jump(app): 693 marked_msg = app.messages_by_id.get(app.marked_message_id) if app.marked_message_id else None 694 cmd_mark_set(app) 695 if marked_msg is not None: 696 app_select_message(app, marked_msg) 697 app_show_flash(app, "Mark swapped") 698 699 700 def cmd_pager_up(app): 701 app.pager_offset = max(0, app.pager_offset - 1) 702 703 704 def cmd_pager_down(app): 705 if app.selected_message is not None and app.layout.pager_height is not None: 706 app.pager_offset = min(app.pager_offset + 1, max(0, len(app.selected_message.lines) - app.layout.pager_height)) 707 708 709 def cmd_page_up(app): 710 cmd_pager_page_up(app) if app.pager_visible else cmd_index_page_up(app) 711 712 713 def cmd_page_down(app): 714 cmd_pager_page_down(app) if app.pager_visible else cmd_index_page_down(app) 715 716 717 def cmd_index_page_up(app): 718 pos = app.selected_message.index_position - app.layout.index_height if app.selected_message else 0 719 pos = max(pos, 0) 720 app_select_message(app, list_get(app.messages, pos, app.selected_message)) 721 722 723 def cmd_index_page_down(app): 724 pos = app.selected_message.index_position + app.layout.index_height if app.selected_message else 0 725 pos = min(pos, len(app.messages) - 1) 726 app_select_message(app, list_get(app.messages, pos, app.selected_message)) 727 728 729 def cmd_pager_page_up(app): 730 if app.layout.pager_height is not None: 731 app.pager_offset = max(0, app.pager_offset - app.layout.pager_height) 732 733 734 def cmd_pager_page_down(app): 735 message = app.selected_message 736 pager_height = app.layout.pager_height 737 if message is not None and pager_height is not None: 738 app.pager_offset = min(app.pager_offset + pager_height, max(0, len(message.lines) - pager_height)) 739 740 741 def cmd_load_tab(app, tab): 742 group = list_get(GROUP_TABS, tab - 1) 743 if group: 744 app_load_group(app, group) 745 746 747 def cmd_reload_page(app): 748 app_load_group(app, app.group) 749 750 751 def cmd_load_prev_page(app): 752 app_load_group(app, group_advance_page(app.group, -1)) 753 754 755 def cmd_load_next_page(app): 756 app_load_group(app, group_advance_page(app.group, 1)) 757 758 759 def cmd_load_page(app): 760 user_input = app_prompt(app, "Go to page (empty to cancel): ") 761 762 if not user_input.isnumeric() or int(user_input) < 1: 763 app_show_flash(app, "Invalid page number") 764 else: 765 app_load_group(app, group_set_page(app.group, int(user_input))) 766 767 def make_group(name, f): 768 return Group(label=name, fetch=lambda db, page: f(page)) 769 770 def cmd_lb_see_tags(app): 771 user_input = app_prompt(app, "Stories for tag(s) (comma to combine): ") 772 app_load_group(app, make_group(TRUNCATE(user_input, 25), 773 lambda page: lb_fetch_threads("t/"+user_input, page))) 774 775 def cmd_lb_see_user(app): 776 user_input = app_prompt(app, "Stories posted by user: ") 777 app_load_group(app, make_group(TRUNCATE(user_input, 25), 778 lambda page: lb_fetch_threads("~{}/stories".format(user_input), page))) 779 780 def cmd_hn_see_user(app): 781 user_input = app_prompt(app, "Stories posted by user: ") 782 app_load_group(app, make_group(TRUNCATE(user_input, 25), 783 lambda page: hn_fetch_user_threads(user_input, page))) 784 785 def cmd_open(app): 786 if app.selected_message is None: 787 return 788 789 if app.selected_message.is_thread: 790 app_open_thread(app, app.selected_message) 791 else: 792 app_select_message(app, app.selected_message, show_pager=True) 793 794 795 def cmd_close(app): 796 if app.pager_visible: 797 app.pager_visible = False 798 else: 799 app_close_thread(app) 800 801 802 def cmd_star(app): 803 msg = app.selected_message 804 if msg is not None: 805 msg.flags.starred = not msg.flags.starred 806 db_save_message(app.db, msg) 807 cmd_next(app) 808 809 810 def cmd_star_thread(app): 811 msg = app.selected_message 812 if msg is None: 813 return 814 815 thread_msg = app.messages_by_id.get(msg.thread_id) 816 if thread_msg is None: 817 return 818 819 thread_msg.flags.starred = not thread_msg.flags.starred 820 db_save_message(app.db, thread_msg) 821 cmd_next(app) 822 823 824 def cmd_set_unread(app): 825 if app.selected_message is not None: 826 app.selected_message.flags.read = False 827 db_save_message(app.db, app.selected_message) 828 cmd_next(app) 829 830 831 def cmd_dump(app): 832 if app.selected_message is None: 833 return 834 835 filename = "{}.html".format(app.selected_message.msg_id) 836 837 with open(filename, "w") as fp: 838 fp.write(app.selected_message.body or "") 839 840 app_show_flash(app, "Message body dumped to {}".format(filename)) 841 842 843 def cmd_toggle_raw_mode(app): 844 app.raw_mode = not app.raw_mode 845 app_select_message(app, app.selected_message) 846 847 848 def cmd_resize(app): 849 app_refresh_message(app) 850 851 852 def cmd_unknown(app): 853 app.flash = "Unknown key" 854 855 856 def db_init(path): 857 path = os.path.expanduser(path) 858 create_table_sql = """ 859 CREATE TABLE IF NOT EXISTS messages ( 860 msg_id TEXT NOT NULL PRIMARY KEY, 861 thread_id TEXT NOT NULL, 862 date INTEGER NOT NULL, 863 starred BOOLEAN NOT NULL, 864 read BOOLEAN NOT NULL 865 ); 866 867 CREATE INDEX IF NOT EXISTS messages_starred_date ON messages (starred, date); 868 """ 869 870 db = sqlite3.connect(path) 871 db.row_factory = sqlite3.Row 872 db.executescript(create_table_sql) 873 db.commit() 874 875 return db 876 877 878 def db_save_message(db, message): 879 sql = """INSERT OR REPLACE INTO messages (msg_id, thread_id, date, starred, read) VALUES (?, ?, ?, ?, ?)""" 880 date = int(time.mktime(message.date.timetuple())) 881 db.execute(sql, (message.msg_id, message.thread_id, date, message.flags.starred, message.flags.read)) 882 db.commit() 883 884 885 def db_load_message_flags(db, messages_by_id): 886 message_ids = list(messages_by_id.keys()) 887 sql = "SELECT * FROM messages WHERE msg_id IN ({})".format(','.join('?' for _ in message_ids)) 888 889 for row in db.execute(sql, message_ids): 890 flags = MessageFlags() 891 flags.starred = row["starred"] 892 flags.read = row["read"] 893 messages_by_id[row["msg_id"]].flags = flags 894 895 896 def db_load_read_comments(db, messages_by_id): 897 threads_by_id = {msg.msg_id: msg for msg in messages_by_id.values() if msg.is_thread} 898 thread_ids = list(threads_by_id.keys()) 899 900 sql = """ 901 SELECT thread_id, COUNT(*) AS count 902 FROM messages 903 WHERE thread_id IN ({}) AND read 904 GROUP BY thread_id 905 """.format(','.join('?' for _ in thread_ids)) 906 907 for row in db.execute(sql, thread_ids): 908 threads_by_id[row["thread_id"]].read_comments = row["count"] 909 910 911 def db_load_starred_thread_ids(db, page = 1): 912 page_size = 30 913 offset = (page - 1) * page_size 914 sql = """ 915 SELECT thread_id 916 FROM messages 917 WHERE starred 918 GROUP BY thread_id 919 ORDER BY date DESC 920 LIMIT ? 921 OFFSET ? 922 """ 923 924 return [row["thread_id"] for row in db.execute(sql, (page_size, offset))] 925 926 927 def msg_populate_total_count(msg): 928 children = msg.children or [] 929 msg.total_count = 0 930 931 if len(children) == 0: 932 return 933 934 for child in children: 935 msg_populate_total_count(child) 936 msg.total_count += 1 + child.total_count 937 938 def msg_flatten_thread( 939 msg, prefix = "", is_last_child = False, ascii = False 940 ): 941 blcorner = "'-" if ascii else "└─" 942 ltee = "|-" if ascii else "├─" 943 vline = "| " if ascii else "│ " 944 945 msg.index_tree = "" if msg.is_thread else "{}{}> ".format(prefix, blcorner if is_last_child else ltee) 946 yield msg 947 948 children = msg.children or [] 949 950 child_count = len(children) 951 child_prefix = "" if msg.is_thread else "{}{}".format(prefix, ' ' if is_last_child else vline) 952 953 for i, child_node in enumerate(children): 954 child_is_last = i == child_count - 1 955 for child in msg_flatten_thread(child_node, prefix=child_prefix, is_last_child=child_is_last, ascii=ascii): 956 yield child 957 958 959 def msg_build_raw_lines(msg): 960 text = text_sanitize(msg.body) 961 962 # Unescape selected entities for better readability 963 repl = {"/": "/", "'": "'", """: '"'} 964 for k, v in repl.items(): 965 text = text.replace(k, v) 966 967 return reduce(lambda acc, line: acc + wrap(line, width=120, replace_whitespace=False), text.split("\n"), []) 968 969 970 def msg_build_lines(msg): 971 lines = [ 972 "Content-Location: {}".format(msg.content_location), 973 "Date: {}".format(msg.date.strftime('%Y-%m-%d %H:%M')), 974 "From: {}".format(msg.author or '<unknown>'), 975 "Subject: {}".format(msg.title), 976 "", 977 ] 978 979 lines += html_render(msg.body or "").split("\n") if not msg.is_deleted else ["<deleted>"] 980 981 return lines 982 983 984 def msg_unload(msg): 985 msg.children = None 986 msg.body = None 987 return msg 988 989 990 def hn_parse_search_hit(hit): 991 return Message( 992 msg_id="{}@hn".format(hit['objectID']), 993 thread_id="{}@hn".format(hit['objectID']), 994 content_location="https://news.ycombinator.com/item?id={}".format(hit['objectID']), 995 date=datetime.fromtimestamp(hit["created_at_i"]), 996 author=hit["author"], 997 title=html_unescape(hit["title"]), 998 total_comments=(hit["num_comments"] or 0) + 1, 999 url=hit["url"] if "url" in hit else None, 1000 ) 1001 1002 1003 def hn_parse_entry(entry, thread_id="", parent=None, parent_title=""): 1004 thread_id = thread_id or str(entry["id"]) 1005 1006 my_title = html_unescape(entry["title"]) if entry["title"] else None 1007 1008 parent_title = parent.title if parent else parent_title 1009 parent_title = parent_title if parent_title.startswith("Re: ") else "Re: {}".format(parent_title) 1010 1011 body = "<p>{}</p>".format(entry['url']) if entry["url"] else "" 1012 body = "{}{}".format(body, entry['text']) if entry["text"] else body 1013 1014 msg = Message( 1015 msg_id="{}@hn".format(entry['id']), 1016 thread_id="{}@hn".format(thread_id), 1017 content_location="https://news.ycombinator.com/item?id={}".format(entry['id']), 1018 date=datetime.fromtimestamp(entry["created_at_i"]), 1019 author=entry["author"], 1020 title=my_title or parent_title, 1021 body=body, 1022 parent=parent, 1023 url=entry["url"] if "url" in entry else None, 1024 ) 1025 1026 msg.children = [hn_parse_entry(child, thread_id, msg) for child in entry["children"]] 1027 1028 return msg 1029 1030 1031 def hn_fetch_threads_by_id(thread_ids): 1032 story_tags = ",".join("story_{}".format(x) for x in thread_ids) 1033 url = "https://hn.algolia.com/api/v1/search_by_date?hitsPerPage={}&tags=story,({})".format(len(thread_ids), story_tags) 1034 hits = json.loads(fetch(url))["hits"] 1035 hits_by_id = {hit["objectID"]: hit for hit in hits} 1036 threads = [hn_parse_search_hit(hits_by_id[tid]) for tid in thread_ids if tid in hits_by_id] 1037 1038 return threads 1039 1040 def hn_fetch_user_threads(username, page = 1): 1041 url = "https://hn.algolia.com/api/v1/search?hitsPerPage={}&page={}&tags=story,author_{}".format(PREFERRED_PAGE_SIZE, page-1, username) 1042 print(url) 1043 hits = json.loads(fetch(url))["hits"] 1044 return [hn_parse_search_hit(hit) for hit in hits] 1045 1046 def hn_fetch_threads(group = "news", page = 1): 1047 rex = re.compile(r'href="item\?id=(\d+)"') 1048 1049 url = "https://news.ycombinator.com/{}".format(group) 1050 # HN seems to be trigger-happy about sending 429s to some paginated requests with weird user agents 1051 if page != 1: 1052 url += "?p={}".format(page) 1053 1054 html = fetch(url) 1055 thread_ids = list(dict.fromkeys(match.group(1) for match in rex.finditer(html))) 1056 1057 return hn_fetch_threads_by_id(thread_ids) 1058 1059 1060 def hn_fetch_new_threads(page = 1): 1061 url = "https://hn.algolia.com/api/v1/search_by_date?tags=story&hitsPerPage=30&page={}".format(page-1) 1062 hits = json.loads(fetch(url))["hits"] 1063 1064 return [hn_parse_search_hit(hit) for hit in hits] 1065 1066 1067 def hn_fetch_thread(entry_id): 1068 resp = fetch("http://hn.algolia.com/api/v1/items/{}".format(entry_id)) 1069 entry = json.loads(resp) 1070 1071 parent_title = "" 1072 if not entry["title"] and "story_id" in entry: 1073 resp = fetch("http://hn.algolia.com/api/v1/items/{}".format(entry["story_id"])) 1074 parent_title = json.loads(resp)["title"] 1075 1076 return hn_parse_entry(entry, parent_title=parent_title) 1077 1078 1079 def datetime_from_iso(iso): 1080 # strptime()'s supported UTC offset format string requires it be 1081 # HHMM not HH:MM before python 3.12... 1082 iso = iso.replace(":", "") 1083 return datetime.strptime(iso, "%Y-%m-%dT%H%M%S.%f%z") 1084 1085 def html_unescape(string, mapping={}): 1086 # gross hack exploiting how python only evaluates the default argument once 1087 if len(mapping) == 0: 1088 print("requesting...") 1089 req = urllib.request.Request("https://html.spec.whatwg.org/entities.json") 1090 resp = urllib.request.urlopen(req).read().decode() 1091 mapping.update(json.loads(resp)) 1092 for replacement in mapping: 1093 string = string.replace(replacement, mapping[replacement]["characters"]) 1094 return string 1095 1096 def lb_parse_thread(thread): 1097 comments = {} 1098 1099 thread_body = thread['description'] 1100 thread_body = "<p>{}</p>{}".format(thread['url'], thread_body) if thread["url"] else thread_body 1101 1102 ret = Message( 1103 msg_id="{}@lb".format(thread['short_id']), 1104 thread_id="{}@lb".format(thread['short_id']), 1105 content_location=thread["short_id_url"], 1106 date=datetime_from_iso(thread["created_at"]), 1107 author=thread["submitter_user"], 1108 title=thread["title"], 1109 body=thread_body, 1110 children=None if thread.get("comments") is None else [], 1111 total_comments=thread["comment_count"] + 1, 1112 url=thread["url"] if "url" in thread else None, 1113 ) 1114 1115 for comment in thread.get("comments", []) or []: 1116 comments[comment["short_id"]] = Message( 1117 msg_id="{}@lb".format(comment['short_id']), 1118 thread_id="{}@lb".format(thread['short_id']), 1119 content_location=comment["url"], 1120 date=datetime_from_iso(comment["created_at"]), 1121 author=comment["commenting_user"], 1122 title="Re: {}".format(thread['title']), 1123 body=comment["comment"], 1124 children=[] 1125 ) 1126 1127 for comment in thread.get("comments", []) or []: 1128 msg = comments[comment["short_id"]] 1129 parent_msg = comments[comment["parent_comment"]] if comment["parent_comment"] else ret 1130 msg.parent = parent_msg 1131 if parent_msg.children is not None: 1132 parent_msg.children.append(msg) 1133 1134 return ret 1135 1136 1137 def lb_fetch_threads(group = "", page = 1): 1138 group_path = group + "/" if group else "" 1139 resp = fetch("https://lobste.rs/{}page/{}.json".format(group_path, page)) 1140 threads = json.loads(resp) 1141 1142 return [lb_parse_thread(thread) for thread in threads] 1143 1144 1145 def lb_fetch_thread(entry_id): 1146 resp = fetch("https://lobste.rs/s/{}.json".format(entry_id)) 1147 thread = json.loads(resp) 1148 1149 return lb_parse_thread(thread) 1150 1151 1152 def group_set_page(group, page): 1153 return Group(group.label, group.fetch, page=page) 1154 1155 1156 def group_advance_page(group, offset = 1): 1157 return group_set_page(group, page=max(1, group.page + offset)) 1158 1159 1160 def group_fetch_starred_threads(db, page = 1): 1161 thread_ids = db_load_starred_thread_ids(db, page) 1162 threads_by_provider_id = {} 1163 threads = [] 1164 1165 for source_id, provider_id in (t.split("@") for t in thread_ids): 1166 threads_by_provider_id.setdefault(provider_id, list()).append(source_id) 1167 1168 for provider_id, thread_ids in threads_by_provider_id.items(): 1169 provider = PROVIDERS[provider_id] 1170 threads += provider.fetch_threads_by_id(thread_ids) 1171 1172 threads.sort(key=lambda x: x.date, reverse=True) 1173 1174 return threads 1175 1176 1177 def group_fetch_thread(thread_id): 1178 (msg_id, provider_id) = thread_id.split("@") 1179 provider = PROVIDERS[provider_id] 1180 1181 return provider.fetch_thread(msg_id) 1182 1183 1184 def group_for_msg_url(url): 1185 match = HN_URL_REX.match(url) 1186 if match is not None: 1187 msg_id = match[1] 1188 return Group(label=msg_id, fetch=lambda *x: [hn_fetch_thread(msg_id)]) 1189 1190 match = LB_URL_REX.match(url) 1191 if match is not None: 1192 msg_id = match[1] 1193 return Group(label=msg_id, fetch=lambda *x: [lb_fetch_thread(msg_id)]) 1194 return None 1195 1196 1197 def app_safe_run(app, fn, flash): 1198 if flash is not None: 1199 app_show_flash(app, flash) 1200 1201 ret = None 1202 1203 try: 1204 ret = fn() 1205 except Exception as e: 1206 logging.debug("\n".join(traceback.format_exception(type(e), e, e.__traceback__))) 1207 app_show_flash(app, "Error: " + str(e)) 1208 else: 1209 if flash is not None: 1210 app_show_flash(app, None) 1211 1212 return ret 1213 1214 1215 def app_refresh_message(app): 1216 app.pager_offset = 0 1217 1218 # Converting html to lines lazily on render for easier debugging 1219 msg = app.selected_message 1220 if msg is not None: 1221 msg.lines = msg_build_raw_lines(msg) if app.raw_mode else msg_build_lines(msg) 1222 1223 1224 def app_select_message(app, message, show_pager = False): 1225 app.selected_message = message 1226 1227 app_refresh_message(app) 1228 1229 if message is None or message.body is None: 1230 app.pager_visible = False 1231 return 1232 1233 if show_pager: 1234 app.pager_visible = True 1235 1236 if app.pager_visible: 1237 message.flags.read = True 1238 db_save_message(app.db, message) 1239 db_load_read_comments(app.db, {message.thread_id: app.messages_by_id[message.thread_id]}) 1240 1241 1242 def app_load_messages( 1243 app, messages, selected_message_id = None, show_pager = False 1244 ): 1245 if selected_message_id is None and app.selected_message is not None: 1246 selected_message_id = app.selected_message.msg_id 1247 1248 selected_message = None 1249 1250 for i, message in enumerate(messages): 1251 message.index_position = i 1252 1253 if message.msg_id == selected_message_id: 1254 selected_message = message 1255 1256 if selected_message is None and len(messages) > 0: 1257 selected_message = messages[0] 1258 1259 app.messages = messages 1260 app.messages_by_id = {msg.msg_id: msg for msg in messages} 1261 1262 db_load_message_flags(app.db, app.messages_by_id) 1263 db_load_read_comments(app.db, app.messages_by_id) 1264 1265 app_select_message(app, selected_message, show_pager) 1266 1267 1268 def app_load_group(app, group): 1269 fn = partial(group.fetch, app.db, group.page) 1270 flash = "Fetching stories from '{}' (page {})...".format(group.label, group.page) 1271 1272 messages = app_safe_run(app, fn, flash=flash) 1273 if messages is None: 1274 return 1275 1276 app_load_messages(app, messages) 1277 app.group = group 1278 1279 1280 def app_close_thread(app): 1281 selected_thread_id = app.selected_message.thread_id if app.selected_message else None 1282 filtered_messages = [msg_unload(msg) for msg in app.messages if msg.is_thread] 1283 1284 app_load_messages(app, filtered_messages, selected_message_id=selected_thread_id) 1285 1286 1287 def app_open_thread(app, thread_message): 1288 fn = partial(group_fetch_thread, thread_message.thread_id) 1289 flash = "Fetching thread '{}'...".format(thread_message.thread_id) 1290 1291 new_thread_message = app_safe_run(app, fn, flash=flash) 1292 if new_thread_message is None: 1293 return 1294 1295 app_close_thread(app) 1296 1297 index_pos = thread_message.index_position 1298 thread_messages = list(msg_flatten_thread(new_thread_message, ascii=app.ascii)) 1299 new_thread_message.total_comments = len(thread_messages) 1300 messages = app.messages[:index_pos] + thread_messages + app.messages[index_pos + 1 :] # noqa: E203 1301 1302 app_load_messages(app, messages, selected_message_id=thread_message.msg_id, show_pager=True) 1303 1304 1305 def app_update_layout(app): 1306 lt = app.layout 1307 1308 (lt.lines, lt.cols) = app.screen.getmaxyx() 1309 1310 if lt.lines < 12 or lt.cols < 80: 1311 raise ExitException(1, "At least 80x12 terminal is required") 1312 1313 max_index_height = lt.lines - 3 1314 lt.index_height = (max_index_height // 3) if app.pager_visible else max_index_height 1315 1316 lt.middle_menu_row = lt.index_start + lt.index_height if app.pager_visible else None 1317 lt.pager_start = lt.index_start + lt.index_height + 1 if app.pager_visible else None 1318 lt.pager_height = lt.lines - lt.pager_start - 2 if lt.pager_start is not None else None 1319 1320 lt.bottom_menu_row = lt.lines - 2 1321 lt.flash_menu_row = lt.lines - 1 1322 1323 1324 def app_show_help_screen(app): 1325 max_lines = app.layout.lines - 4 1326 1327 help_lines = HELP_SCREEN.split("\n") 1328 help_pages = ["\n".join(lines) for lines in list_chunk(help_lines, max_lines)] 1329 1330 for page in help_pages: 1331 app.screen.erase() 1332 app.screen.addstr(0, 0, "Available commands:\n\n") 1333 app.screen.addstr(page) 1334 app.screen.addstr("\n\nPress any key to continue...") 1335 app.screen.refresh() 1336 app.screen.getch() 1337 1338 1339 def app_show_links_screen(app): 1340 lines = app.selected_message.lines if app.selected_message is not None else [] 1341 1342 urls = set(URL_REX.findall(" ".join(lines[1:]))) 1343 if app.selected_message is not None and app.selected_message.url is not None: 1344 urls.add(app.selected_message.url) 1345 1346 if len(urls) == 0: 1347 return app_show_flash(app, "No links available for opening") 1348 elif len(urls) == 1: 1349 url = urls.pop() 1350 else: 1351 # Max amount of keys is 21 to fit on 25-line terminals 1352 keys = "1234567890abcdefghijk" 1353 items = dict(zip((ord(k) for k in keys), urls)) 1354 1355 app.screen.erase() 1356 app.screen.addstr(0, 0, "Select link to open:") 1357 1358 for i, (key, url) in enumerate(items.items()): 1359 app.screen.addstr(i + 2, 0, "{} - {}".format(chr(key), url)) 1360 1361 app.screen.addstr(i + 4, 0, "To change browser run: BROWSER='firefox %s' ./retronews.py") 1362 app.screen.refresh() 1363 1364 key = app.screen.getch() 1365 1366 if key not in items.keys(): 1367 return app_show_flash(app, "Unknown key") 1368 1369 url = items[key] 1370 1371 group = group_for_msg_url(url) 1372 if group is not None: 1373 app_load_group(app, group) 1374 else: 1375 app_show_flash(app, "Opening " + url) 1376 webbrowser.open(url) 1377 1378 # Refresh window in case a terminal browser was used 1379 app.screen.clearok(True) 1380 1381 1382 def app_show_flash(app, flash): 1383 app.flash = flash 1384 app_render(app) 1385 1386 1387 def app_prompt(app, prompt): 1388 lt = app.layout 1389 1390 app.screen.insstr(lt.flash_menu_row, 0, prompt.ljust(lt.cols)) 1391 app.screen.refresh() 1392 1393 curses.curs_set(1) 1394 win = curses.newwin(1, lt.cols - len(prompt), lt.flash_menu_row, len(prompt)) 1395 1396 textbox = curses.textpad.Textbox(win) 1397 textbox.stripspaces = True 1398 ret = textbox.edit().strip() 1399 1400 del win 1401 curses.curs_set(0) 1402 1403 return ret 1404 1405 def app_chgat(app, row, start, size, color): 1406 app.screen.chgat(row, start, size, color) 1407 return start + size 1408 1409 def app_render_index_row(app, row, message): 1410 cols = app.layout.cols 1411 date = TRUNCATE(DATECOL_FUNC(message.date), DATECOL_SIZE).rjust(DATECOL_SIZE) 1412 author = TRUNCATE(message.author or "<unknown>", AUTHOR_SIZE).ljust(AUTHOR_SIZE) 1413 1414 is_response = message.title.startswith("Re:") and not message.is_thread 1415 is_selected = message == app.selected_message 1416 hide_title = is_response and row > app.layout.index_start and not message.flags.starred and not is_selected 1417 title = "" if hide_title else text_clean(message.title, ascii=app.ascii) 1418 1419 unread_count = max(message.total_comments - message.read_comments, 0) 1420 unread_repr = str(unread_count) if unread_count < (10**UNREAD_SIZE - 1) else (UNREAD_MANY_CHAR * UNREAD_SIZE) 1421 unread = unread_repr.rjust(UNREAD_SIZE) if message.is_thread else (' ' * UNREAD_SIZE) 1422 1423 spacing = ' ' * COLUMN_SPACING 1424 app.screen.insstr(row, 0, "[{}]{}[{}]{}[{}]{}{}{}".format(date, spacing, author, spacing, unread, spacing, message.index_tree, title)) 1425 1426 if is_selected: 1427 cursor_attr = curses.A_REVERSE if app.monochrome else 0 1428 app.screen.chgat(row, 0, cols, app.colors["cursor"] | cursor_attr) 1429 else: 1430 read_attr = 0 if message.is_shown_as_read else curses.A_BOLD 1431 subject_attr = app.colors["starred_subject"] if message.flags.starred else app.colors["default"] 1432 subject_attr = subject_attr | read_attr 1433 1434 start = 1 1435 start = app_chgat(app, row, start, len(date), app.colors["date"] | read_attr) 1436 start = app_chgat(app, row, start + 2 + COLUMN_SPACING, AUTHOR_SIZE, app.colors["author"] | read_attr) 1437 start = app_chgat(app, row, start + 2 + COLUMN_SPACING, UNREAD_SIZE, app.colors["unread_comments"] | read_attr) 1438 start = app_chgat(app, row, start + 1 + COLUMN_SPACING, len(message.index_tree), app.colors["tree"]) 1439 app_chgat(app, row, start, cols - start, subject_attr) 1440 1441 1442 def app_render_index(app): 1443 height = app.layout.index_height 1444 1445 offset = app.selected_message.index_position - height // 2 if app.selected_message else 0 1446 offset = min(offset, len(app.messages) - height) 1447 offset = max(offset, 0) 1448 1449 rows_to_render = min(height, len(app.messages) - offset) 1450 1451 for i in range(rows_to_render): 1452 app_render_index_row(app, app.layout.index_start + i, app.messages[i + offset]) 1453 1454 1455 def app_get_pager_line_attr(app, line): 1456 if line.startswith("Content-Location: "): 1457 return app.colors["tree"] 1458 elif line.startswith("Date: "): 1459 return app.colors["date"] 1460 elif line.startswith("From: "): 1461 return app.colors["author"] 1462 elif line.startswith("Subject: "): 1463 return app.colors["header_subject"] 1464 elif line.startswith(">>") or line.startswith("> >"): 1465 return app.colors["nested_quote"] 1466 elif line.startswith(">"): 1467 return app.colors["quote"] 1468 elif line.startswith("| "): 1469 return app.colors["code"] 1470 elif line == "~": 1471 return app.colors["empty_pager_line"] 1472 elif line == "<deleted>" or line == "[dead]": 1473 return app.colors["deleted_message_pager_line"] 1474 else: 1475 return 0 1476 1477 1478 def app_render_pager_line(app, row, line): 1479 hl_lines = not app.raw_mode 1480 line_attr = app_get_pager_line_attr(app, line) if hl_lines else 0 1481 hl_urls = line_attr == 0 and hl_lines 1482 1483 line = text_clean(line, ascii=app.ascii) 1484 1485 app.screen.move(row, 0) 1486 app.screen.clrtoeol() 1487 app.screen.move(row, 0) 1488 1489 for part in text_split_urls(line): 1490 re_match = URL_REX.match(part) 1491 is_url = re_match.end() == len(part) if re_match is not None else False 1492 part_attr = app.colors["url"] if is_url and hl_urls else line_attr 1493 app.screen.addstr(part, part_attr) 1494 1495 1496 def app_render_pager(app): 1497 message = app.selected_message 1498 start = app.layout.pager_start 1499 height = app.layout.pager_height 1500 1501 if message is None or start is None or height is None: 1502 return 1503 1504 for i in range(height): 1505 line = list_get(message.lines, i + app.pager_offset, "~") 1506 app_render_pager_line(app, i + start, line) 1507 1508 1509 def app_render_top_menu(app): 1510 lt = app.layout 1511 cols = lt.cols 1512 base_attr = curses.A_REVERSE if app.monochrome else curses.A_BOLD 1513 app.screen.insstr(lt.top_menu_row, 0, HELP_MENU[:cols].ljust(cols), app.colors["menu"] | base_attr) 1514 1515 1516 def app_render_middle_menu(app): 1517 row = app.layout.middle_menu_row 1518 message = app.selected_message 1519 if row is None or message is None: 1520 return 1521 1522 thread_message = app.messages_by_id.get(message.thread_id) 1523 if thread_message is None: 1524 return 1525 1526 cols = app.layout.cols 1527 total = thread_message.total_comments 1528 unread = total - thread_message.read_comments 1529 1530 text = "--({}/{} unread)".format(unread, total) 1531 if thread_message.flags.starred: 1532 text += "--(starred thread)" 1533 if app.raw_mode: 1534 text += "--(raw mode on)" 1535 text = text[:cols].ljust(cols, "-") 1536 1537 base_attr = curses.A_REVERSE if app.monochrome else curses.A_BOLD 1538 app.screen.insstr(row, 0, text, app.colors["menu"] | base_attr) 1539 1540 1541 def app_render_bottom_menu(app): 1542 lt = app.layout 1543 base_attr = curses.A_REVERSE if app.monochrome else curses.A_BOLD 1544 1545 app.screen.chgat(lt.bottom_menu_row, 0, lt.cols, app.colors["menu"] | base_attr) 1546 app.screen.move(lt.bottom_menu_row, 0) 1547 1548 for i, group in enumerate(GROUP_TABS): 1549 is_active = group.label == app.group.label 1550 item_attr = app.colors["menu_active"] | curses.A_BOLD if is_active else app.colors["menu"] 1551 item_attr = item_attr | base_attr 1552 app.screen.addstr("{}:{}".format(i+1, group.label), item_attr) 1553 app.screen.addstr(" ", app.colors["menu"] | base_attr) 1554 1555 page_text = "page: {}".format(app.group.page) 1556 app.screen.insstr(lt.bottom_menu_row, lt.cols - len(page_text), page_text, app.colors["menu"] | base_attr) 1557 1558 1559 def app_render(app): 1560 app_update_layout(app) 1561 app.screen.erase() 1562 app_render_index(app) 1563 app_render_pager(app) 1564 app_render_top_menu(app) 1565 app_render_middle_menu(app) 1566 app_render_bottom_menu(app) 1567 app.screen.insstr(app.layout.flash_menu_row, 0, app.flash or "") 1568 app.screen.refresh() 1569 1570 1571 def app_init_colors(app): 1572 if app.monochrome: 1573 app.colors = defaultdict(lambda: 0) 1574 return 1575 1576 try: 1577 curses.use_default_colors() 1578 for i, (name, (fg, bg)) in enumerate(COLORS.items()): 1579 curses.init_pair(i + 1, fg, bg) 1580 app.colors[name] = curses.color_pair(i + 1) 1581 except curses.error: 1582 app.colors = defaultdict(lambda: 0) 1583 app.monochrome = True 1584 1585 1586 def app_main(screen, db, group, ascii, monochrome, default_open=False): 1587 curses.curs_set(0) 1588 1589 app = AppState(screen=screen, db=db, group=group, ascii=ascii, monochrome=monochrome) 1590 1591 app_init_colors(app) 1592 app_load_group(app, app.group) 1593 1594 if default_open and app.selected_message is not None: 1595 app_open_thread(app, app.selected_message) 1596 1597 while True: 1598 app_render(app) 1599 app.flash = "" 1600 c = app.screen.getch() 1601 KEY_BINDINGS.get(c, cmd_unknown)(app) 1602 1603 1604 def setup_logging(path): 1605 if path is None: 1606 return logging.disable(logging.CRITICAL) 1607 1608 format = "%(asctime)s %(levelname)s: %(message)s" 1609 stream = sys.stderr if path == "-" else open(path, "a") 1610 logging.basicConfig(format=format, level="DEBUG", stream=stream) 1611 logging.debug("Session started") 1612 1613 1614 def run_rcfile(path): 1615 path = os.path.expanduser(path) 1616 1617 if not os.path.isfile(path): 1618 return 1619 1620 code = compile(open(path).read(), path, "exec") 1621 exec(code, {"retronews": sys.modules[__name__]}) 1622 1623 1624 if __name__ == "__main__": 1625 tab_choices = range(1, len(GROUP_TABS) + 1) 1626 1627 ap = argparse.ArgumentParser( 1628 formatter_class=lambda prog: argparse.ArgumentDefaultsHelpFormatter(prog, max_help_position=32) 1629 ) 1630 ap.add_argument("--ascii", action="store_true", help="show only ascii characters") 1631 ap.add_argument("--monochrome", action="store_true", help="disable colors") 1632 ap.add_argument("-c", "--rcfile", metavar="PATH", default="~/.retronewsrc.py", help="optional startup code path") 1633 ap.add_argument("-d", "--db", metavar="PATH", default="~/.retronews.db", help="database path") 1634 ap.add_argument("-l", "--logfile", metavar="PATH", default=None, help="debug logfile path") 1635 ap.add_argument("-t", "--tab", metavar="TAB", type=int, default=1, choices=tab_choices, help="initial tab") 1636 ap.add_argument("-r", "--render", metavar="PATH", default=None, help="render raw html message and quit") 1637 ap.add_argument("-m", "--msg", metavar="URL", default=None, help="render message from URL") 1638 args = ap.parse_args() 1639 1640 setup_logging(args.logfile) 1641 run_rcfile(args.rcfile) 1642 1643 if args.render is not None: 1644 with open(args.render) as fp: 1645 print(html_render(fp.read())) 1646 sys.exit(0) 1647 1648 try: 1649 db = db_init(args.db) 1650 1651 if args.msg: 1652 group = group_for_msg_url(args.msg) 1653 if group is None: 1654 msg = "Unknown URL, available patterns: \n" + "\n".join( 1655 "- {}".format(r.pattern) for r in [HN_URL_REX, LB_URL_REX] 1656 ) 1657 raise ExitException(1, msg) 1658 else: 1659 group = GROUP_TABS[args.tab - 1] 1660 1661 ascii = args.ascii 1662 monochrome = args.monochrome or "NO_COLOR" in os.environ 1663 1664 ret = curses.wrapper(app_main, db=db, group=group, ascii=ascii, 1665 monochrome=monochrome, default_open=(args.msg is not None)) 1666 except ExitException as e: 1667 if e.message: 1668 sys.stderr.write(e.message + "\n") 1669 ret = e.code 1670 except BaseException as e: 1671 sys.stderr.write("\n".join(traceback.format_exception(type(e), e, e.__traceback__))) 1672 ret = 1 1673 finally: 1674 db.close() 1675 sys.exit(ret)