- commit
- c7a1994738a11bb3b5c886ef72f6cf6bf7276f61
- parent
- c10db7590749b3807dc4bda38b58fda51eb87560
- Author
- Tobias Bengfort <tobias.bengfort@gmx.net>
- Date
- 2015-12-18 10:08
use asyncio
Diffstat
| M | project_stats.py | 150 | ++++++++++++++++++++++++++++++++++++++---------------------- |
| M | setup.py | 1 | - |
2 files changed, 95 insertions, 56 deletions
diff --git a/project_stats.py b/project_stats.py
@@ -4,16 +4,16 @@ from __future__ import unicode_literals 4 4 5 5 from functools import total_ordering 6 6 import argparse -1 7 import asyncio 7 8 import json 8 9 import logging9 -1 import multiprocessing10 10 import os 11 11 import re 12 12 import subprocess 13 13 import sys 14 14 15 15 from dateutil import parser as dt16 -1 import requests-1 16 import aiohttp 17 17 import yaml 18 18 19 19 try: @@ -52,6 +52,21 @@ KEYS = [ 52 52 ] 53 53 54 54 -1 55 def aiorun(future): -1 56 """Return value of a future synchronously.""" -1 57 container = [] -1 58 -1 59 @asyncio.coroutine -1 60 def wrapper(): -1 61 result = yield from future -1 62 container.append(result) -1 63 -1 64 loop = asyncio.get_event_loop() -1 65 loop.run_until_complete(wrapper()) -1 66 -1 67 return container[0] -1 68 -1 69 55 70 def r_get(d, *keys): 56 71 """Recursively get key from dict or return None.""" 57 72 if len(keys) == 0: @@ -151,11 +166,16 @@ def cheesecake_index(name): 151 166 return None 152 167 153 168 -1 169 @asyncio.coroutine 154 170 def get_bower_info(name):155 -1 try:156 -1 s = subprocess.check_output(['bower', 'info', name]).decode('utf8')157 -1 except OSError:158 -1 return None-1 171 process = yield from asyncio.create_subprocess_exec( -1 172 'bower', 'info', name, -1 173 stdout=asyncio.subprocess.PIPE, -1 174 stderr=asyncio.subprocess.PIPE) -1 175 stdout, stderr = yield from process.communicate() -1 176 if process.returncode != 0: -1 177 return -1 178 s = stdout.decode('utf8') 159 179 160 180 # re handles \n specially, so it is replaced by \t 161 181 s = '\t'.join(s.splitlines()) @@ -171,40 +191,50 @@ def get_bower_info(name): 171 191 return json.loads(s) 172 192 173 193 -1 194 @asyncio.coroutine 174 195 def get_json(url, user=None, password=None): 175 196 assert not (user is None) ^ (password is None) 176 197 177 198 if user is None:178 -1 req = requests.get(url)-1 199 req = yield from aiohttp.get(url) 179 200 else:180 -1 req = requests.get(181 -1 url, auth=requests.auth.HTTPBasicAuth(user, password))-1 201 req = yield from aiohttp.get( -1 202 url, auth=aiohttp.BasicAuth(user, password)) 182 203183 -1 req.raise_for_status()184 -1 return req.json()-1 204 data = yield from req.json() -1 205 return data 185 206 186 207 -1 208 @asyncio.coroutine 187 209 def get_github(url, user=None, password=None): -1 210 api_url = re.sub( -1 211 'https?://github.com', 'https://api.github.com/repos', url) -1 212 -1 213 @asyncio.coroutine 188 214 def _get_json(url):189 -1 data = get_json(url, user=user, password=password)-1 215 data = yield from get_json(url, user=user, password=password) 190 216 if 'documentation_url' in data: 191 217 raise requests.HTTPError(data['documentation_url']) 192 218 return data 193 219194 -1 api_url = re.sub(195 -1 'https?://github.com', 'https://api.github.com/repos', url)196 -1 data = _get_json(api_url)197 -1-1 220 @asyncio.coroutine 198 221 def get_latest_tag():199 -1 tags = _get_json(data['tags_url'] + '?per_page=100')200 -1 tags = [tag['name'] for tag in tags]-1 222 data = yield from _get_json(api_url + '/tags?per_page=100') -1 223 tags = [tag['name'] for tag in data] 201 224 if len(tags) > 0: 202 225 return max(tags, key=lambda tag: tag.lstrip('v')) -1 226 else: -1 227 return 203 228 -1 229 @asyncio.coroutine 204 230 def get_open_pull_requests():205 -1 url = data['pulls_url'].replace('{/number}', '')206 -1 pulls = _get_json(url)207 -1 return len(pulls)-1 231 data = yield from _get_json(api_url + '/pulls') -1 232 return len(data) -1 233 -1 234 data, version, pulls = yield from asyncio.gather( -1 235 _get_json(api_url), -1 236 get_latest_tag(), -1 237 get_open_pull_requests()) 208 238 209 239 return { 210 240 'name': data['name'], @@ -218,12 +248,14 @@ def get_github(url, user=None, password=None): 218 248 'subscribers_count': data['subscribers_count'], 219 249 'forks_count': data['forks_count'], 220 250 'open_issues': data['open_issues'],221 -1 'open_pull_requests': get_open_pull_requests(),222 -1 'version': get_latest_tag(),-1 251 'open_pull_requests': pulls, -1 252 'version': version, 223 253 } 224 254 225 255 -1 256 @asyncio.coroutine 226 257 def get_gitlab(_id, token=None): -1 258 @asyncio.coroutine 227 259 def _get_json(path): 228 260 api_url = 'https://gitlab.com/api/v3/projects/' + _id + path 229 261 if token is not None: @@ -233,9 +265,10 @@ def get_gitlab(_id, token=None): 233 265 api_url += '?private_token=' + token 234 266 return get_json(api_url) 235 267236 -1 data = _get_json('')237 -1 issues = _get_json('/issues?state=opened')238 -1 pulls = _get_json('/merge_requests?state=opened')-1 268 data, issues, pulls = yield from asyncio.gather( -1 269 _get_json(''), -1 270 _get_json('/issues?state=opened'), -1 271 _get_json('/merge_requests?state=opened')) 239 272 240 273 return { 241 274 'name': data['name'], @@ -250,6 +283,7 @@ def get_gitlab(_id, token=None): 250 283 } 251 284 252 285 -1 286 @asyncio.coroutine 253 287 def get_local(path): 254 288 def git(cmd, *args): 255 289 _cmd = ['git', '-C', path, cmd] + list(args) @@ -279,9 +313,9 @@ def get_local(path): 279 313 } 280 314 281 315 -1 316 @asyncio.coroutine 282 317 def get_pypi(url):283 -1 data = get_json(url + '/json')284 -1-1 318 data = yield from get_json(url + '/json') 285 319 return { 286 320 'version': data['info']['version'], 287 321 'description': data['info']['summary'], @@ -293,8 +327,9 @@ def get_pypi(url): 293 327 } 294 328 295 329 -1 330 @asyncio.coroutine 296 331 def get_bower(name):297 -1 data = get_bower_info(name)-1 332 data = yield from get_bower_info(name) 298 333 if data is None: 299 334 return {} 300 335 else: @@ -307,48 +342,53 @@ def get_bower(name): 307 342 } 308 343 309 344 -1 345 @asyncio.coroutine 310 346 def get_travis(url): 311 347 api_url = re.sub( 312 348 'https?://travis-ci.org', 'https://api.travis-ci.org/repos', url)313 -1 data = get_json(api_url)-1 349 data = yield from get_json(api_url) 314 350 return { 315 351 'description': data['description'], 316 352 'tests': data['last_build_result'] == 0, 317 353 } 318 354 319 355320 -1 def get_project(args):321 -1 key, project, config = args-1 356 @asyncio.coroutine -1 357 def get_source(key, source, config, claims): -1 358 fn = globals()['get_' + key] -1 359 if key == 'github': -1 360 future = fn( -1 361 source, -1 362 user=r_get(config, 'github', 'user'), -1 363 password=r_get(config, 'github', 'password')) -1 364 elif key == 'gitlab': -1 365 future = fn(source, token=r_get(config, 'gitlab', 'token')) -1 366 else: -1 367 future = fn(source) -1 368 -1 369 try: -1 370 data = yield from future -1 371 claims.update(data, key) -1 372 except Exception as e: -1 373 message = 'Error while gathering stats for %s from %s: %s', -1 374 logging.error(message, key, source, e) -1 375 -1 376 -1 377 @asyncio.coroutine -1 378 def get_project(key, project, config): 322 379 claims = ClaimsDict(KEYS) -1 380 futures = [] 323 381 for source in SOURCES: 324 382 if source in project:325 -1 try:326 -1 fn = globals()['get_' + source]327 -1 if source == 'github':328 -1 data = fn(329 -1 project[source],330 -1 user=r_get(config, 'github', 'user'),331 -1 password=r_get(config, 'github', 'password'))332 -1 elif source == 'gitlab':333 -1 data = fn(334 -1 project[source],335 -1 token=r_get(config, 'gitlab', 'token'))336 -1 else:337 -1 data = fn(project[source])338 -1 claims.update(data, source)339 -1 except Exception as e:340 -1 message = 'Error while gathering stats for %s from %s: %s',341 -1 logging.error(message, key, source, e)-1 383 futures.append(get_source(source, project[source], config, claims)) -1 384 yield from asyncio.gather(*futures) 342 385 return claims 343 386 344 387 345 388 def get_projects(projects_config, config):346 -1 pool = multiprocessing.Pool()347 -1 # HACK to get KeyboardInterrupt to work.348 -1 # See https://stackoverflow.com/questions/1408356349 -1 pool_map = lambda a, b: pool.map_async(a, b).get(99999)350 -1 args = ((key, project, config) for key, project in projects_config.items())351 -1 projects_list = pool_map(get_project, args)-1 389 projects_list = aiorun(asyncio.gather(*[ -1 390 get_project(key, project, config) -1 391 for key, project in projects_config.items()])) 352 392 353 393 projects = {} 354 394 for key, project in zip(projects_config.keys(), projects_list):
diff --git a/setup.py b/setup.py
@@ -21,7 +21,6 @@ setup( 21 21 py_modules=['project_stats'], 22 22 install_requires=[ 23 23 'python-dateutil',24 -1 'requests',25 24 'pyyaml', 26 25 ], 27 26 extras_require={