- commit
- e842762d49a680756617e647bb544d34b31f1386
- parent
- c10db7590749b3807dc4bda38b58fda51eb87560
- Author
- Tobias Bengfort <tobias.bengfort@gmx.net>
- Date
- 2015-12-18 15:37
Merge branch 'feature-asyncio'
Diffstat
| M | project_stats.py | 154 | ++++++++++++++++++++++++++++++++++++++----------------------- |
| M | setup.py | 3 | +-- |
2 files changed, 96 insertions, 61 deletions
diff --git a/project_stats.py b/project_stats.py
@@ -1,19 +1,15 @@1 -1 from __future__ import absolute_import2 -1 from __future__ import print_function3 -1 from __future__ import unicode_literals4 -15 1 from functools import total_ordering 6 2 import argparse -1 3 import asyncio 7 4 import json 8 5 import logging9 -1 import multiprocessing10 6 import os 11 7 import re 12 8 import subprocess 13 9 import sys 14 10 15 11 from dateutil import parser as dt16 -1 import requests-1 12 import aiohttp 17 13 import yaml 18 14 19 15 try: @@ -52,6 +48,21 @@ KEYS = [ 52 48 ] 53 49 54 50 -1 51 def aiorun(future): -1 52 """Return value of a future synchronously.""" -1 53 container = [] -1 54 -1 55 @asyncio.coroutine -1 56 def wrapper(): -1 57 result = yield from future -1 58 container.append(result) -1 59 -1 60 loop = asyncio.get_event_loop() -1 61 loop.run_until_complete(wrapper()) -1 62 -1 63 return container[0] -1 64 -1 65 55 66 def r_get(d, *keys): 56 67 """Recursively get key from dict or return None.""" 57 68 if len(keys) == 0: @@ -151,11 +162,16 @@ def cheesecake_index(name): 151 162 return None 152 163 153 164 -1 165 @asyncio.coroutine 154 166 def get_bower_info(name):155 -1 try:156 -1 s = subprocess.check_output(['bower', 'info', name]).decode('utf8')157 -1 except OSError:158 -1 return None-1 167 process = yield from asyncio.create_subprocess_exec( -1 168 'bower', 'info', name, -1 169 stdout=asyncio.subprocess.PIPE, -1 170 stderr=asyncio.subprocess.PIPE) -1 171 stdout, stderr = yield from process.communicate() -1 172 if process.returncode != 0: -1 173 return -1 174 s = stdout.decode('utf8') 159 175 160 176 # re handles \n specially, so it is replaced by \t 161 177 s = '\t'.join(s.splitlines()) @@ -171,40 +187,50 @@ def get_bower_info(name): 171 187 return json.loads(s) 172 188 173 189 -1 190 @asyncio.coroutine 174 191 def get_json(url, user=None, password=None): 175 192 assert not (user is None) ^ (password is None) 176 193 177 194 if user is None:178 -1 req = requests.get(url)-1 195 req = yield from aiohttp.get(url) 179 196 else:180 -1 req = requests.get(181 -1 url, auth=requests.auth.HTTPBasicAuth(user, password))-1 197 req = yield from aiohttp.get( -1 198 url, auth=aiohttp.BasicAuth(user, password)) 182 199183 -1 req.raise_for_status()184 -1 return req.json()-1 200 data = yield from req.json() -1 201 return data 185 202 186 203 -1 204 @asyncio.coroutine 187 205 def get_github(url, user=None, password=None): -1 206 api_url = re.sub( -1 207 'https?://github.com', 'https://api.github.com/repos', url) -1 208 -1 209 @asyncio.coroutine 188 210 def _get_json(url):189 -1 data = get_json(url, user=user, password=password)-1 211 data = yield from get_json(url, user=user, password=password) 190 212 if 'documentation_url' in data: 191 213 raise requests.HTTPError(data['documentation_url']) 192 214 return data 193 215194 -1 api_url = re.sub(195 -1 'https?://github.com', 'https://api.github.com/repos', url)196 -1 data = _get_json(api_url)197 -1-1 216 @asyncio.coroutine 198 217 def get_latest_tag():199 -1 tags = _get_json(data['tags_url'] + '?per_page=100')200 -1 tags = [tag['name'] for tag in tags]-1 218 data = yield from _get_json(api_url + '/tags?per_page=100') -1 219 tags = [tag['name'] for tag in data] 201 220 if len(tags) > 0: 202 221 return max(tags, key=lambda tag: tag.lstrip('v')) -1 222 else: -1 223 return 203 224 -1 225 @asyncio.coroutine 204 226 def get_open_pull_requests():205 -1 url = data['pulls_url'].replace('{/number}', '')206 -1 pulls = _get_json(url)207 -1 return len(pulls)-1 227 data = yield from _get_json(api_url + '/pulls') -1 228 return len(data) -1 229 -1 230 data, version, pulls = yield from asyncio.gather( -1 231 _get_json(api_url), -1 232 get_latest_tag(), -1 233 get_open_pull_requests()) 208 234 209 235 return { 210 236 'name': data['name'], @@ -218,12 +244,14 @@ def get_github(url, user=None, password=None): 218 244 'subscribers_count': data['subscribers_count'], 219 245 'forks_count': data['forks_count'], 220 246 'open_issues': data['open_issues'],221 -1 'open_pull_requests': get_open_pull_requests(),222 -1 'version': get_latest_tag(),-1 247 'open_pull_requests': pulls, -1 248 'version': version, 223 249 } 224 250 225 251 -1 252 @asyncio.coroutine 226 253 def get_gitlab(_id, token=None): -1 254 @asyncio.coroutine 227 255 def _get_json(path): 228 256 api_url = 'https://gitlab.com/api/v3/projects/' + _id + path 229 257 if token is not None: @@ -233,9 +261,10 @@ def get_gitlab(_id, token=None): 233 261 api_url += '?private_token=' + token 234 262 return get_json(api_url) 235 263236 -1 data = _get_json('')237 -1 issues = _get_json('/issues?state=opened')238 -1 pulls = _get_json('/merge_requests?state=opened')-1 264 data, issues, pulls = yield from asyncio.gather( -1 265 _get_json(''), -1 266 _get_json('/issues?state=opened'), -1 267 _get_json('/merge_requests?state=opened')) 239 268 240 269 return { 241 270 'name': data['name'], @@ -250,6 +279,7 @@ def get_gitlab(_id, token=None): 250 279 } 251 280 252 281 -1 282 @asyncio.coroutine 253 283 def get_local(path): 254 284 def git(cmd, *args): 255 285 _cmd = ['git', '-C', path, cmd] + list(args) @@ -279,9 +309,9 @@ def get_local(path): 279 309 } 280 310 281 311 -1 312 @asyncio.coroutine 282 313 def get_pypi(url):283 -1 data = get_json(url + '/json')284 -1-1 314 data = yield from get_json(url + '/json') 285 315 return { 286 316 'version': data['info']['version'], 287 317 'description': data['info']['summary'], @@ -293,8 +323,9 @@ def get_pypi(url): 293 323 } 294 324 295 325 -1 326 @asyncio.coroutine 296 327 def get_bower(name):297 -1 data = get_bower_info(name)-1 328 data = yield from get_bower_info(name) 298 329 if data is None: 299 330 return {} 300 331 else: @@ -307,48 +338,53 @@ def get_bower(name): 307 338 } 308 339 309 340 -1 341 @asyncio.coroutine 310 342 def get_travis(url): 311 343 api_url = re.sub( 312 344 'https?://travis-ci.org', 'https://api.travis-ci.org/repos', url)313 -1 data = get_json(api_url)-1 345 data = yield from get_json(api_url) 314 346 return { 315 347 'description': data['description'], 316 348 'tests': data['last_build_result'] == 0, 317 349 } 318 350 319 351320 -1 def get_project(args):321 -1 key, project, config = args-1 352 @asyncio.coroutine -1 353 def get_source(key, source, config, claims): -1 354 fn = globals()['get_' + key] -1 355 if key == 'github': -1 356 future = fn( -1 357 source, -1 358 user=r_get(config, 'github', 'user'), -1 359 password=r_get(config, 'github', 'password')) -1 360 elif key == 'gitlab': -1 361 future = fn(source, token=r_get(config, 'gitlab', 'token')) -1 362 else: -1 363 future = fn(source) -1 364 -1 365 try: -1 366 data = yield from future -1 367 claims.update(data, key) -1 368 except Exception as e: -1 369 message = 'Error while gathering stats for %s from %s: %s', -1 370 logging.error(message, key, source, e) -1 371 -1 372 -1 373 @asyncio.coroutine -1 374 def get_project(key, project, config): 322 375 claims = ClaimsDict(KEYS) -1 376 futures = [] 323 377 for source in SOURCES: 324 378 if source in project:325 -1 try:326 -1 fn = globals()['get_' + source]327 -1 if source == 'github':328 -1 data = fn(329 -1 project[source],330 -1 user=r_get(config, 'github', 'user'),331 -1 password=r_get(config, 'github', 'password'))332 -1 elif source == 'gitlab':333 -1 data = fn(334 -1 project[source],335 -1 token=r_get(config, 'gitlab', 'token'))336 -1 else:337 -1 data = fn(project[source])338 -1 claims.update(data, source)339 -1 except Exception as e:340 -1 message = 'Error while gathering stats for %s from %s: %s',341 -1 logging.error(message, key, source, e)-1 379 futures.append(get_source(source, project[source], config, claims)) -1 380 yield from asyncio.gather(*futures) 342 381 return claims 343 382 344 383 345 384 def get_projects(projects_config, config):346 -1 pool = multiprocessing.Pool()347 -1 # HACK to get KeyboardInterrupt to work.348 -1 # See https://stackoverflow.com/questions/1408356349 -1 pool_map = lambda a, b: pool.map_async(a, b).get(99999)350 -1 args = ((key, project, config) for key, project in projects_config.items())351 -1 projects_list = pool_map(get_project, args)-1 385 projects_list = aiorun(asyncio.gather(*[ -1 386 get_project(key, project, config) -1 387 for key, project in projects_config.items()])) 352 388 353 389 projects = {} 354 390 for key, project in zip(projects_config.keys(), projects_list):
diff --git a/setup.py b/setup.py
@@ -21,7 +21,6 @@ setup( 21 21 py_modules=['project_stats'], 22 22 install_requires=[ 23 23 'python-dateutil',24 -1 'requests',25 24 'pyyaml', 26 25 ], 27 26 extras_require={ @@ -36,7 +35,7 @@ setup( 36 35 'Environment :: Console', 37 36 'Intended Audience :: Developers', 38 37 'Operating System :: OS Independent',39 -1 'Programming Language :: Python',-1 38 'Programming Language :: Python :: 3', 40 39 'License :: OSI Approved :: GNU General Public License v2 or later ' 41 40 '(GPLv2+)', 42 41 'Topic :: Utilities',