PyJSONProxy

simple proxy and scraper
git clone https://git.ce9e.org/PyJSONProxy.git

commit
96b71a7601860fcb1e981e2aa00d22a2a38f593d
parent
11f97180fe1bfaad3ede84e64b3caf744c9232d1
Author
Tobias Bengfort <tobias.bengfort@gmx.net>
Date
2015-12-13 00:55
move web.py to separate project "Fakes"

Diffstat

M jsonproxy/__init__.py 17 ++++++++---------
D jsonproxy/web.py 97 ------------------------------------------------------------
M setup.py 3 +--

3 files changed, 9 insertions, 108 deletions


diff --git a/jsonproxy/__init__.py b/jsonproxy/__init__.py

@@ -4,11 +4,10 @@ import sys
    4     4 
    5     5 import aiohttp
    6     6 
    7    -1 from .web import Application
    8    -1 from .web import jsonify
    9    -1 from .web import render_template
   10    -1 from .web import make_response
   11    -1 from .web import abort
   -1     7 from fakes import Fakes
   -1     8 from fakes import jsonify
   -1     9 from fakes import make_response
   -1    10 from fakes import abort
   12    11 
   13    12 from .lib import check_config
   14    13 from .lib import _doc
@@ -16,7 +15,7 @@ from .lib import ENDPOINTS
   16    15 from .lib import parse_args
   17    16 from .lib import scrape
   18    17 
   19    -1 app = Application(__name__)
   -1    18 app = Fakes(__name__)
   20    19 
   21    20 
   22    21 def get_config(endpoint):
@@ -67,7 +66,7 @@ def handle(request):
   67    66 	if 'fields' in config and request.method == 'GET':
   68    67 		response = jsonify(scrape(url, body, config), status=remote.status)
   69    68 	else:
   70    -1 		response = make_response(body, status=remote.status)
   -1    69 		response = make_response((body, remote.status, []))
   71    70 
   72    71 	if app.config.get('ALLOW_CORS', False):
   73    72 		response.headers['Access-Control-Allow-Origin'] = '*'
@@ -79,7 +78,7 @@ def handle(request):
   79    78 def index(request):
   80    79 	config = app.config[ENDPOINTS]
   81    80 	data = [_doc(config[endpoint], endpoint) for endpoint in config]
   82    -1 	return render_template('index.html', endpoints=data)
   -1    81 	return app.render_template('index.html', endpoints=data)
   83    82 
   84    83 
   85    84 @app.route('/{endpoint}/')
@@ -87,7 +86,7 @@ def doc(request):
   87    86 	endpoint = request.match_info['endpoint']
   88    87 	config = get_config(endpoint)
   89    88 	data = [_doc(config, endpoint)]
   90    -1 	return render_template('index.html', endpoints=data)
   -1    89 	return app.render_template('index.html', endpoints=data)
   91    90 
   92    91 
   93    92 def main():

diff --git a/jsonproxy/web.py b/jsonproxy/web.py

@@ -1,97 +0,0 @@
    1    -1 """Flask inspired wrapper around aiohttp."""
    2    -1 
    3    -1 from functools import lru_cache
    4    -1 from pkg_resources import resource_filename
    5    -1 import asyncio
    6    -1 import logging
    7    -1 import os
    8    -1 
    9    -1 from aiohttp import web
   10    -1 import jinja2
   11    -1 
   12    -1 
   13    -1 @lru_cache()
   14    -1 def get_template(name):
   15    -1 	path = resource_filename(__name__, os.path.join('templates', name))
   16    -1 	with open(path) as fh:
   17    -1 		return jinja2.Template(fh.read())
   18    -1 
   19    -1 
   20    -1 def render_template(name, **kwargs):
   21    -1 	template = get_template(name)
   22    -1 	text = template.render(**kwargs)
   23    -1 	return web.Response(body=text.encode('utf8'))
   24    -1 
   25    -1 
   26    -1 def jsonify(data, **kwargs):
   27    -1 	return web.json_response(data, **kwargs)
   28    -1 
   29    -1 
   30    -1 def abort(code):
   31    -1 	if code == 404:
   32    -1 		raise web.HTTPNotFound
   33    -1 	elif code >= 500:
   34    -1 		raise web.HTTPInternalServerError
   35    -1 	else:
   36    -1 		raise web.HTTPBadRequest
   37    -1 
   38    -1 
   39    -1 def make_response(data, **kwargs):
   40    -1 	if isinstance(data, web.StreamResponse):
   41    -1 		return data
   42    -1 	elif isinstance(data, str):
   43    -1 		return web.Response(body=data.encode('utf8'), **kwargs)
   44    -1 	elif isinstance(data, bytes):
   45    -1 		return web.Response(body=data, **kwargs)
   46    -1 	else:
   47    -1 		raise TypeError('cannot make response from {}'.format(data))
   48    -1 
   49    -1 
   50    -1 class Application:
   51    -1 	def __init__(self, name):
   52    -1 		self.name = name
   53    -1 		self.loop = asyncio.get_event_loop()
   54    -1 		self.app = web.Application(loop=self.loop)
   55    -1 		self.config = {}
   56    -1 		self.debug = False
   57    -1 
   58    -1 		self.logger = logging.getLogger(self.name)
   59    -1 		self.logger.setLevel(logging.INFO)
   60    -1 
   61    -1 		consoleHandler = logging.StreamHandler()
   62    -1 		formatter = logging.Formatter('%(asctime)s - %(message)s')
   63    -1 		consoleHandler.setFormatter(formatter)
   64    -1 		self.logger.addHandler(consoleHandler)
   65    -1 
   66    -1 	def config_from_file(self, path):
   67    -1 		with open(path) as fh:
   68    -1 			exec(compile(fh.read(), path, 'exec'), self.config)
   69    -1 
   70    -1 	def add_route(self, path, fn, methods=('GET',)):
   71    -1 		@asyncio.coroutine
   72    -1 		def wrapped(*args, **kwargs):
   73    -1 			if asyncio.iscoroutinefunction(fn):
   74    -1 				data = yield from fn(*args, **kwargs)
   75    -1 			else:
   76    -1 				data = fn(*args, **kwargs)
   77    -1 			return make_response(data)
   78    -1 
   79    -1 		for method in methods:
   80    -1 			self.app.router.add_route(method, path, wrapped)
   81    -1 
   82    -1 	def route(self, path, methods=('GET',)):
   83    -1 		def decorator(fn):
   84    -1 			self.add_route(path, fn, methods=methods)
   85    -1 			return fn
   86    -1 		return decorator
   87    -1 
   88    -1 	def run(self, host='localhost', port=5000):
   89    -1 		if self.debug:
   90    -1 			self.logger.setLevel(logging.DEBUG)
   91    -1 		server = self.loop.create_server(self.app.make_handler(), host, port)
   92    -1 		self.loop.run_until_complete(server)
   93    -1 		self.logger.info("Server started at http://{}:{}".format(host, port))
   94    -1 		try:
   95    -1 			self.loop.run_forever()
   96    -1 		except KeyboardInterrupt:
   97    -1 			pass

diff --git a/setup.py b/setup.py

@@ -13,9 +13,8 @@ setup(
   13    13     author_email='tobias.bengfort@posteo.de',
   14    14     packages=['jsonproxy'],
   15    15     install_requires=[
   16    -1         'aiohttp',
   -1    16         'Fakes',
   17    17         'beautifulsoup4',
   18    -1         'jinja2',
   19    18     ],
   20    19     entry_points={'console_scripts': [
   21    20         'jsonproxy=jsonproxy:main',