PyJSONProxy

simple proxy and scraper
git clone https://git.ce9e.org/PyJSONProxy.git

commit
75dfc6834307e3989467112711318ba07ad9b0af
parent
eba025c9e0ef97bd71014c5469c8fec80f9b64de
Author
Tobias Bengfort <tobias.bengfort@gmx.net>
Date
2015-02-05 17:30
Fix KeyError

Diffstat

M jsonproxy/api.py 10 ++++++----

1 files changed, 6 insertions, 4 deletions


diff --git a/jsonproxy/api.py b/jsonproxy/api.py

@@ -41,14 +41,14 @@ def _doc(endpoint):
   41    41 		'fields': [],
   42    42 	}
   43    43 
   44    -1 	if config['type'] == 'scrape_item':
   -1    44 	if data['type'] == 'scrape_item':
   45    45 		fields_doc = config.get('fields_doc', {})
   46    46 		data['fields'].append(('url', url_doc))
   47    47 		for key in config['fields']:
   48    48 			doc = fields_doc.get(key, '')
   49    49 			data['fields'].append((key, doc))
   50    50 
   51    -1 	if config['type'] == 'scrape_list':
   -1    51 	if data['type'] == 'scrape_list':
   52    52 		data['fields'] = [
   53    53 			('url', url_doc),
   54    54 			('l', 'list of results'),
@@ -88,7 +88,9 @@ def proxy(endpoint, path):
   88    88 	code = original.getcode()
   89    89 	headers = original.headers.items()
   90    90 
   91    -1 	if config['type'] == 'scrape_item':
   -1    91 	type = config.get('type', 'proxy')
   -1    92 
   -1    93 	if type == 'scrape_item':
   92    94 		html = BeautifulSoup(body)
   93    95 		data = {
   94    96 			'url': url
@@ -96,7 +98,7 @@ def proxy(endpoint, path):
   96    98 		for key, selector in config['fields'].items():
   97    99 			data[key] = get_attribute(html, selector)
   98   100 		response = jsonify(data)
   99    -1 	elif config['type'] == 'scrape_list':
   -1   101 	elif type == 'scrape_list':
  100   102 		html = BeautifulSoup(body)
  101   103 		response = jsonify({
  102   104 			'url': url,