project-stats

commit

14d1f0de50560811e110d787a7d3e05a0c0e76b3

parent

7b83b0b245b1b262c2102232ce9f73cd91a64053

Author

Tobias Bengfort <tobias.bengfort@gmx.net>

Date

2015-04-16 07:32

Merge branch 'feature-parallel'

Diffstat

1 files changed, 27 insertions, 18 deletions

diff --git a/project_stats.py b/project_stats.py

@@ -7,6 +7,7 @@ from xml.etree import ElementTree
    7     7 import argparse
    8     8 import json
    9     9 import logging
   -1    10 import multiprocessing
   10    11 import os
   11    12 import re
   12    13 import subprocess
@@ -367,26 +368,34 @@ def parse_args():
  367   368     return parser.parse_args()
  368   369 
  369   370 
   -1   371 def get_project(args):
   -1   372     key, config = args
   -1   373     try:
   -1   374         claims = ClaimsDict(KEYS)
   -1   375         for source in SOURCES:
   -1   376             if source in config:
   -1   377                 fn = globals()['get_' + source]
   -1   378                 if source == 'github':
   -1   379                     data = fn(
   -1   380                         config[source],
   -1   381                         user=r_get(config, 'github', 'user'),
   -1   382                         password=r_get(config, 'github', 'password'))
   -1   383                 else:
   -1   384                     data = fn(config[source])
   -1   385                 claims.update(data, source)
   -1   386         return claims
   -1   387     except Exception as e:
   -1   388         logging.error('Error while gathering stats for %s: %s', key, e)
   -1   389 
   -1   390 
  370   391 def get_projects(projects_config):
   -1   392     pool = multiprocessing.Pool()
   -1   393     projects_list = pool.map(get_project, projects_config.items())
   -1   394 
  371   395     projects = {}
  372    -1     for key, config in projects_config.items():
  373    -1         try:
  374    -1             claims = ClaimsDict(KEYS)
  375    -1             for source in SOURCES:
  376    -1                 if source in config:
  377    -1                     fn = globals()['get_' + source]
  378    -1                     if source == 'github':
  379    -1                         data = fn(
  380    -1                             config[source],
  381    -1                             user=r_get(config, 'github', 'user'),
  382    -1                             password=r_get(config, 'github', 'password'))
  383    -1                     else:
  384    -1                         data = fn(config[source])
  385    -1                     claims.update(data, source)
  386    -1 
  387    -1             projects[key] = claims
  388    -1         except Exception as e:
  389    -1             logging.error('Error while gathering stats for %s: %s', key, e)
   -1   396     for key, project in zip(projects_config.keys(), projects_list):
   -1   397         if project is not None:
   -1   398             projects[key] = project
  390   399     return projects
  391   400 
  392   401