- commit
- eeeb3ab77e11ab13f6dc0c8e33d57f231a32a098
- parent
- 79502ba501fa81eab5abdb5d09c6daeb1ae610ee
- Author
- Tobias Bengfort <tobias.bengfort@posteo.de>
- Date
- 2017-07-23 19:38
port to async/await
Diffstat
| M | project_stats.py | 76 | +++++++++++++++++++++++++------------------------------------ |
| M | setup.py | 2 | +- |
2 files changed, 32 insertions, 46 deletions
diff --git a/project_stats.py b/project_stats.py
@@ -162,47 +162,41 @@ def cheesecake_index(name): 162 162 return None 163 163 164 164165 -1 @asyncio.coroutine166 -1 def get_json(url, user=None, password=None):-1 165 async def get_json(url, user=None, password=None): 167 166 assert not (user is None) ^ (password is None) 168 167169 -1 if user is None:170 -1 req = yield from aiohttp.get(url)171 -1 else:172 -1 req = yield from aiohttp.get(173 -1 url, auth=aiohttp.BasicAuth(user, password))-1 168 auth = None -1 169 if user is not None: -1 170 auth=aiohttp.BasicAuth(user, password) 174 171175 -1 data = yield from req.json()176 -1 return data-1 172 async with aiohttp.ClientSession(auth=auth) as session: -1 173 async with session.get(url) as resp: -1 174 return await resp.json() 177 175 178 176179 -1 @asyncio.coroutine180 -1 def get_github(url, user=None, password=None):-1 177 async def get_github(url, user=None, password=None): 181 178 api_url = re.sub( 182 179 'https?://github.com', 'https://api.github.com/repos', url) 183 180184 -1 @asyncio.coroutine185 -1 def _get_json(url):186 -1 data = yield from get_json(url, user=user, password=password)-1 181 async def _get_json(url): -1 182 data = await get_json(url, user=user, password=password) 187 183 if 'documentation_url' in data: 188 184 raise aiohttp.ClientError(data['documentation_url']) 189 185 return data 190 186191 -1 @asyncio.coroutine192 -1 def get_latest_tag():193 -1 data = yield from _get_json(api_url + '/tags?per_page=100')-1 187 async def get_latest_tag(): -1 188 data = await _get_json(api_url + '/tags?per_page=100') 194 189 tags = [tag['name'] for tag in data] 195 190 if len(tags) > 0: 196 191 return max(tags, key=lambda tag: tag.lstrip('v')) 197 192 else: 198 193 return 199 194200 -1 @asyncio.coroutine201 -1 def get_open_pull_requests():202 -1 data = yield from _get_json(api_url + '/pulls')-1 195 async def get_open_pull_requests(): -1 196 data = await _get_json(api_url + '/pulls') 203 197 return len(data) 204 198205 -1 data, version, pulls = yield from asyncio.gather(-1 199 data, version, pulls = await asyncio.gather( 206 200 _get_json(api_url), 207 201 get_latest_tag(), 208 202 get_open_pull_requests()) @@ -224,19 +218,17 @@ def get_github(url, user=None, password=None): 224 218 } 225 219 226 220227 -1 @asyncio.coroutine228 -1 def get_gitlab(_id, token=None):229 -1 @asyncio.coroutine230 -1 def _get_json(path):-1 221 async def get_gitlab(_id, token=None): -1 222 async def _get_json(path): 231 223 api_url = 'https://gitlab.com/api/v3/projects/' + _id + path 232 224 if token is not None: 233 225 if '?' in api_url: 234 226 api_url += '&private_token=' + token 235 227 else: 236 228 api_url += '?private_token=' + token237 -1 return get_json(api_url)-1 229 return await get_json(api_url) 238 230239 -1 data, issues, pulls = yield from asyncio.gather(-1 231 data, issues, pulls = await asyncio.gather( 240 232 _get_json(''), 241 233 _get_json('/issues?state=opened'), 242 234 _get_json('/merge_requests?state=opened')) @@ -254,8 +246,7 @@ def get_gitlab(_id, token=None): 254 246 } 255 247 256 248257 -1 @asyncio.coroutine258 -1 def get_local(path):-1 249 async def get_local(path): 259 250 def git(cmd, *args): 260 251 _cmd = ['git', '-C', path, cmd] + list(args) 261 252 return subprocess.check_output(_cmd).decode('utf8') @@ -284,9 +275,8 @@ def get_local(path): 284 275 } 285 276 286 277287 -1 @asyncio.coroutine288 -1 def get_pypi(url):289 -1 data = yield from get_json(url + '/json')-1 278 async def get_pypi(url): -1 279 data = await get_json(url + '/json') 290 280 return { 291 281 'version': data['info']['version'], 292 282 'description': data['info']['summary'], @@ -298,9 +288,8 @@ def get_pypi(url): 298 288 } 299 289 300 290301 -1 @asyncio.coroutine302 -1 def get_npm(name):303 -1 process = yield from asyncio.create_subprocess_exec(-1 291 async def get_npm(name): -1 292 process = await asyncio.create_subprocess_exec( 304 293 'npm', 'view', name, 305 294 'name', 306 295 'version', @@ -311,7 +300,7 @@ def get_npm(name): 311 300 'time.modified', 312 301 stdout=asyncio.subprocess.PIPE, 313 302 stderr=asyncio.subprocess.PIPE)314 -1 stdout, stderr = yield from process.communicate()-1 303 stdout, stderr = await process.communicate() 315 304 if process.returncode != 0: 316 305 return 317 306 s = stdout.decode('utf8') @@ -333,19 +322,17 @@ def get_npm(name): 333 322 return data 334 323 335 324336 -1 @asyncio.coroutine337 -1 def get_travis(url):-1 325 async def get_travis(url): 338 326 api_url = re.sub( 339 327 'https?://travis-ci.org', 'https://api.travis-ci.org/repos', url)340 -1 data = yield from get_json(api_url)-1 328 data = await get_json(api_url) 341 329 return { 342 330 'description': data['description'], 343 331 'tests': data['last_build_result'] == 0, 344 332 } 345 333 346 334347 -1 @asyncio.coroutine348 -1 def get_source(key, source, config, claims):-1 335 async def get_source(key, source, config, claims): 349 336 fn = globals()['get_' + key] 350 337 if key == 'github': 351 338 future = fn( @@ -358,21 +345,20 @@ def get_source(key, source, config, claims): 358 345 future = fn(source) 359 346 360 347 try:361 -1 data = yield from future-1 348 data = await future 362 349 claims.update(data, key) 363 350 except Exception as e: 364 351 message = 'Error while gathering stats for %s from %s: %s', 365 352 logging.error(message, key, source, e) 366 353 367 354368 -1 @asyncio.coroutine369 -1 def get_project(key, project, config):-1 355 async def get_project(key, project, config): 370 356 claims = ClaimsDict(KEYS) 371 357 futures = [] 372 358 for source in SOURCES: 373 359 if source in project: 374 360 futures.append(get_source(source, project[source], config, claims))375 -1 yield from asyncio.gather(*futures)-1 361 await asyncio.gather(*futures) 376 362 return claims 377 363 378 364
diff --git a/setup.py b/setup.py
@@ -20,7 +20,7 @@ setup( 20 20 author_email='tobias.bengfort@posteo.de', 21 21 py_modules=['project_stats'], 22 22 install_requires=[23 -1 'aiohttp<2',-1 23 'aiohttp', 24 24 'python-dateutil', 25 25 'pyyaml', 26 26 ],