plutopluto

commit

2f6a3ccfac37290a969cd2b6814d4be0bb3ff80e

parent

901783e18e632ab75845c27f79ffa35fb3c2fc5c

Author

Tobias Bengfort <tobias.bengfort@posteo.de>

Date

2023-12-21 09:32

async fetch

Diffstat

2 files changed, 25 insertions, 15 deletions

diff --git a/plutopluto/__init__.py b/plutopluto/__init__.py

@@ -2,15 +2,15 @@
    2     2 
    3     3 import argparse
    4     4 import datetime
    5    -1 import functools
    6     5 import os
    7     6 import sys
    8     7 from time import mktime
    9     8 from time import time
   10     9 from xml.sax.saxutils import escape
   11    10 
   -1    11 import aiohttp
   12    12 import feedparser
   13    -1 import requests
   -1    13 from aiohttp import web
   14    14 from feedparser.sanitizer import _sanitize_html
   15    15 from flask import Flask
   16    16 from flask import request
@@ -22,6 +22,20 @@ __version__ = '1.2.0'
   22    22 app = Flask(__name__)
   23    23 
   24    24 
   -1    25 async def fetch(url, *, raw=False, **kwargs):
   -1    26     async with aiohttp.ClientSession() as session:
   -1    27         async with session.get(url, **kwargs) as response:
   -1    28             if response.status == 404:
   -1    29                 raise web.HTTPNotFound
   -1    30             if response.status == 429:
   -1    31                 raise web.HTTPServiceUnavailable
   -1    32             response.raise_for_status()
   -1    33             if raw:
   -1    34                 return await response.read()
   -1    35             else:
   -1    36                 return await response.json()
   -1    37 
   -1    38 
   25    39 def linebreaks(text):
   26    40     html = (
   27    41         text
@@ -31,11 +45,10 @@ def linebreaks(text):
   31    45     return '<p>' + html + '</p>'
   32    46 
   33    47 
   34    -1 @functools.lru_cache
   35    -1 def parse_feed(url):
   -1    48 async def parse_feed(url):
   36    49     """Get feed and convert to JSON."""
   37    50 
   38    -1     feed = feedparser.parse(url)
   -1    51     feed = feedparser.parse(await fetch(url, raw=True))
   39    52 
   40    53     def _parse_item(i, item):
   41    54         d = dict()
@@ -67,11 +80,8 @@ def parse_feed(url):
   67    80     }
   68    81 
   69    82 
   70    -1 @functools.lru_cache
   71    -1 def parse_activity_stream(url):
   72    -1     r = requests.get(url, headers={'Accept': 'application/activity+json'})
   73    -1     r.raise_for_status()
   74    -1     data = r.json()
   -1    83 async def parse_activity_stream(url):
   -1    84     data = await fetch(url, headers={'Accept': 'application/activity+json'})
   75    85     entries = []
   76    86 
   77    87     def _parse_item(obj):
@@ -118,7 +128,7 @@ def parse_activity_stream(url):
  118   128 
  119   129 
  120   130 @app.route('/parse', methods=['GET'])
  121    -1 def _parse():
   -1   131 async def _parse():
  122   132     if 'url' not in request.values:
  123   133         abort(400)
  124   134 
@@ -126,9 +136,9 @@ def _parse():
  126   136 
  127   137     try:
  128   138         if 'outbox' in url:
  129    -1             data = parse_activity_stream(url)
   -1   139             data = await parse_activity_stream(url)
  130   140         else:
  131    -1             data = parse_feed(url)
   -1   141             data = await parse_feed(url)
  132   142     except Exception as err:
  133   143         app.logger.warning('%s: %s' % (url, err))
  134   144         abort(500)

diff --git a/setup.py b/setup.py

@@ -23,8 +23,8 @@ setup(
   23    23     packages=['plutopluto'],
   24    24     include_package_data=True,
   25    25     install_requires=[
   26    -1         'requests',
   27    -1         'flask',
   -1    26         'aiohttp',
   -1    27         'flask[async]',
   28    28         'feedparser',
   29    29     ],
   30    30     entry_points={'console_scripts': [