From 590ca8a5df17c166ec814e6002b14d23c6a6051d Mon Sep 17 00:00:00 2001 From: Israel Blancas Date: Sun, 20 Dec 2015 19:17:55 +0100 Subject: [PATCH 01/17] Solving #31 --- LICENSE => LICENSE.txt | 0 GitHubCity.py => githubcity/GitHubCity.py | 0 GitHubUser.py => githubcity/GitHubUser.py | 0 githubcity/__init__.py | 2 ++ setup.cfg | 2 ++ setup.py | 42 +++++++++++++++++++++++ testGitHubCity.py | 3 +- testGitHubUser.py | 4 +-- 8 files changed, 50 insertions(+), 3 deletions(-) rename LICENSE => LICENSE.txt (100%) rename GitHubCity.py => githubcity/GitHubCity.py (100%) rename GitHubUser.py => githubcity/GitHubUser.py (100%) create mode 100644 githubcity/__init__.py create mode 100644 setup.cfg create mode 100644 setup.py diff --git a/LICENSE b/LICENSE.txt similarity index 100% rename from LICENSE rename to LICENSE.txt diff --git a/GitHubCity.py b/githubcity/GitHubCity.py similarity index 100% rename from GitHubCity.py rename to githubcity/GitHubCity.py diff --git a/GitHubUser.py b/githubcity/GitHubUser.py similarity index 100% rename from GitHubUser.py rename to githubcity/GitHubUser.py diff --git a/githubcity/__init__.py b/githubcity/__init__.py new file mode 100644 index 0000000..644dfda --- /dev/null +++ b/githubcity/__init__.py @@ -0,0 +1,2 @@ +from GitHubCity import GitHubCity +from GitHubUser import GitHubUser diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..b88034e --- /dev/null +++ b/setup.cfg @@ -0,0 +1,2 @@ +[metadata] +description-file = README.md diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..8523412 --- /dev/null +++ b/setup.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +"""This module is used to install githubcity package + +Author: Israel Blancas @iblancasa +License: + +The MIT License (MIT) + Copyright (c) 2015 Israel Blancas @iblancasa (http://iblancasa.com/) + + Permission is hereby granted, free of charge, to any person obtaining a copy of this software + and associated documentation files (the “Software”), to deal in the Software without + restriction, including without limitation the rights to use, copy, modify, merge, publish, + distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom + the Software is furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all copies or + substantial portions of the Software. + + THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, + INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR + PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE + FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + IN THE SOFTWARE. + +""" + +from distutils.core import setup +setup( + name = 'githubcity', + packages = ['githubcity'], + version = '0.01', + description = 'GitHub city ranking creator', + author = 'Israel Blancas @iblancasa', + author_email = 'iblancasa@gmail.com', + url='https://github.com/iblancasa/GitHubCity', + download_url = 'https://github.com/iblancasa/GitHubCity/0.01', + keywords = ['github', 'ranking', 'data', 'api'], # arbitrary keywords + classifiers = [], +) diff --git a/testGitHubCity.py b/testGitHubCity.py index ef1a732..68e342c 100644 --- a/testGitHubCity.py +++ b/testGitHubCity.py @@ -34,8 +34,9 @@ import coloredlogs import sys import os +import datetime import json -sys.path.append(os.getcwd()) +sys.path.append(os.getcwd()+"/githubcity") from GitHubCity import * ########################################################### diff --git a/testGitHubUser.py b/testGitHubUser.py index 644935c..fb4b1c8 100644 --- a/testGitHubUser.py +++ b/testGitHubUser.py @@ -35,8 +35,8 @@ import sys import os import json -sys.path.append(os.getcwd()) -from GitHubUser import * +sys.path.append(os.getcwd()+"/githubcity") +from GitHubUser import GitHubUser import datetime ########################################################### From 3657959db10fe801abec1b47d77fe6e1d26cff6a Mon Sep 17 00:00:00 2001 From: Israel Blancas Date: Tue, 22 Dec 2015 17:28:26 +0100 Subject: [PATCH 02/17] Reading and exporting configuration files #11 --- githubcity/GitHubCity.py | 121 ++++++++++++++++++++++++++++++++------- 1 file changed, 100 insertions(+), 21 deletions(-) diff --git a/githubcity/GitHubCity.py b/githubcity/GitHubCity.py index f5c84dd..8c2955d 100755 --- a/githubcity/GitHubCity.py +++ b/githubcity/GitHubCity.py @@ -60,13 +60,15 @@ class GitHubCity: _githubSecret (str): secretGH of your GitHub application. _dataUsers (List[GitHubUser]): the list of GitHub users. _excluded (set): list of names of excluded users. + _lastDay (str): day of last interval _names (Queue): Queue with all users that we still have to process. _threads (set): Set of active Threads. _logger (logger): Logger. _l (Lock): lock to solve problems with threads. """ - def __init__(self, city, githubID, githubSecret, excludedJSON=None, debug=False): + def __init__(self, githubID, githubSecret, config=None, city=None, locations=None, + excludedJSON=None, debug=False): """Constructor of the class. Note: @@ -85,11 +87,6 @@ def __init__(self, city, githubID, githubSecret, excludedJSON=None, debug=False) """ - if city==None: - raise Exception("City is not defined") - - self._city = city - if githubID==None: raise Exception("No GitHub ID inserted") self._githubID = githubID @@ -98,27 +95,47 @@ def __init__(self, city, githubID, githubSecret, excludedJSON=None, debug=False) raise Exception("No GitHub Secret inserted") self._githubSecret = githubSecret + if config: + self.readConfig(config) + else: + self._city = city + self._locations = locations + + if excludedJSON: + for e in excludedJSON: + self._excluded.add(e) + + self._names = queue.Queue() self._myusers = set() self._excluded = set() self._dataUsers = [] self._threads = set() - - if excludedJSON: - for e in excludedJSON: - self._excluded.add(e["name"]) - self._logger = logging.getLogger("GitHubCity") if debug: coloredlogs.install(level='DEBUG') self._fin = False - self._l = Lock() + + def readConfig(self, config): + self._city = config["name"] + self._intervals = config["intervals"] + self._lastDay = config["last_date"] + excluded = config["excludedUsers"] + for e in excluded: + self._excluded.add(e) + + + def readConfigFromJSON(self, fileName): + with open(fileName) as data_file: + data = json.load(data_file) + self.readConfig(data) + def _addUser(self, new_user): """Add new users to the list (private). @@ -204,8 +221,8 @@ def _getURL(self, page=1, start_date=None, final_date=None,order="asc"): else: url = "https://api.github.com/search/users?client_id=" + self._githubID + "&client_secret=" + self._githubSecret + \ "&order=desc&q=sort:joined+type:user+location:" + urllib.parse.quote(self._city) + \ - "+created:" + start_date.strftime("%Y-%m-%d") +\ - ".." + final_date.strftime("%Y-%m-%d") +\ + "+created:" + start_date +\ + ".." + final_date +\ "&sort=joined&order="+order+"&per_page=100&page=" + str(page) return url @@ -265,8 +282,7 @@ def _getPeriodUsers(self, start_date, final_date): final_date (datetime.date): final date of the range to search users. """ - self._logger.info("Getting users from "+start_date.strftime("%Y-%m-%d")+" to "+\ - final_date.strftime("%Y-%m-%d")) + self._logger.info("Getting users from " + start_date + " to " + final_date) url = self._getURL(1, start_date, final_date) data = self._readAPI(url) @@ -321,13 +337,13 @@ def _validInterval(self, start, finish): Valid periods are added to the private _intervals attribute. """ - data = self._readAPI(self._getURL(1,start,finish)) + data = self._readAPI(self._getURL(1,start.strftime("%Y-%m-%d"),finish.strftime("%Y-%m-%d"))) if data["total_count"]>=1000: middle = start + (finish - start)/2 self._validInterval(start,middle) self._validInterval(middle,finish) else: - self._intervals.add((start,finish)) + self._intervals.append([start.strftime("%Y-%m-%d"),finish.strftime("%Y-%m-%d")]) self._logger.debug("Valid interval: "+start.strftime("%Y-%m-%d")+" to "+\ finish.strftime("%Y-%m-%d")) @@ -336,13 +352,12 @@ def _validInterval(self, start, finish): def calculateBestIntervals(self): """Calcules valid intervals of a city (with less than 1000 users) """ - self._intervals = None comprobation = self._readAPI(self._getURL()) - self._intervals = set() + self._intervals = [] self._bigCity = True self._validInterval(datetime.date(2008, 1, 1), datetime.datetime.now().date()) self._logger.info("Total number of intervals: "+ str(len(self._intervals))) - + self._lastDay = datetime.datetime.now().date().strftime("%Y-%m-%d") def getTotalUsers(self): @@ -351,3 +366,67 @@ def getTotalUsers(self): Number (int) of calculated users """ return len(self._dataUsers) + + + + def getSortUsers(self, order): + """Returns a list with sorted users. + + Args: + order (str): a str with one of these values (field to sort by). + - contributions + - name + - lstreak + - cstreak + - language + - followers + - join: + - organizations + - repositories + - stars + + Returns: + str with a list of GitHubUsers by the field indicate. If + an invalid field is given, the result will be None + + """ + if order == "contributions": + self._dataUsers.sort(key=lambda u: u.getContributions(), reverse=True) + elif order == "name": + self._dataUsers.sort(key=lambda u: u.getName(), reverse=True) + elif order == "lstreak": + self._dataUsers.sort(key=lambda u: u.getLongestStreak(), reverse=True) + elif order == "cstreak": + self._dataUsers.sort(key=lambda u: u.getCurrentStreak(), reverse=True) + elif order == "language": + self._dataUsers.sort(key=lambda u: u.getLanguage(), reverse=True) + elif order == "followers": + self._dataUsers.sort(key=lambda u: u.getFollowers(), reverse=True) + elif order == "join": + self._dataUsers.sort(key=lambda u: u.getJoin(), reverse=True) + elif order == "organizations": + self._dataUsers.sort(key=lambda u: u.getOrganizations(), reverse=True) + elif order == "respositories": + self._dataUsers.sort(key=lambda u: u.getNumberOfRepositories(), reverse=True) + elif order == "stars": + self._dataUsers.sort(key=lambda u: u.getStars(), reverse=True) + else: + return None + return self._dataUsers + + + def getConfig(self): + config = {} + config["name"] = self._city + config["intervals"] = self._intervals + config["last_date"] = self._lastDay + config["excludedUsers"]=[] + + for e in self._excluded: + config["excludedUsers"].append(e) + return config + + def configToJson(self, fileName): + config = self.getConfig() + with open(fileName, "w") as outfile: + json.dump(config, outfile, indent=4, sort_keys=True) From 976387b0e5548f03114080bb84bf4baebaf553a5 Mon Sep 17 00:00:00 2001 From: Israel Blancas Date: Tue, 22 Dec 2015 17:33:45 +0100 Subject: [PATCH 03/17] Stringfy class #54 --- githubcity/GitHubCity.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/githubcity/GitHubCity.py b/githubcity/GitHubCity.py index 8c2955d..482863c 100755 --- a/githubcity/GitHubCity.py +++ b/githubcity/GitHubCity.py @@ -120,7 +120,8 @@ def __init__(self, githubID, githubSecret, config=None, city=None, locations=Non self._l = Lock() - + def __str__(self): + return str(self.getConfig()) def readConfig(self, config): self._city = config["name"] From 7d3587fb42ba87bed8adbbb82cf2cc75dc0dfa10 Mon Sep 17 00:00:00 2001 From: Israel Blancas Date: Tue, 22 Dec 2015 18:01:24 +0100 Subject: [PATCH 04/17] Solving #53 and #56 --- githubcity/GitHubCity.py | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/githubcity/GitHubCity.py b/githubcity/GitHubCity.py index 482863c..e2baea8 100755 --- a/githubcity/GitHubCity.py +++ b/githubcity/GitHubCity.py @@ -97,20 +97,26 @@ def __init__(self, githubID, githubSecret, config=None, city=None, locations=Non if config: self.readConfig(config) + print(config) else: self._city = city self._locations = locations + if not self._locations: + self._locations = [] + if self._city: + self._locations.append(self._city) + + self._excluded = set() if excludedJSON: for e in excludedJSON: self._excluded.add(e) - self._names = queue.Queue() self._myusers = set() - self._excluded = set() self._dataUsers = [] self._threads = set() + self._addLocationsToURL(self._locations) self._logger = logging.getLogger("GitHubCity") if debug: @@ -123,14 +129,25 @@ def __init__(self, githubID, githubSecret, config=None, city=None, locations=Non def __str__(self): return str(self.getConfig()) + def _addLocationsToURL(self, locations): + self._urlLocations = "" + + for l in self._locations: + print(type(l)) + self._urlLocations += "+location:" + str(urllib.parse.quote(l)) + + def readConfig(self, config): self._city = config["name"] self._intervals = config["intervals"] self._lastDay = config["last_date"] + self._locations = config["locations"] excluded = config["excludedUsers"] for e in excluded: self._excluded.add(e) + self._addLocationsToURL(self._locations) + def readConfigFromJSON(self, fileName): with open(fileName) as data_file: @@ -217,11 +234,11 @@ def _getURL(self, page=1, start_date=None, final_date=None,order="asc"): """ if not start_date or not final_date: url = "https://api.github.com/search/users?client_id=" + self._githubID + "&client_secret=" + self._githubSecret + \ - "&order=desc&q=sort:joined+type:user+location:" + urllib.parse.quote(self._city) + \ + "&order=desc&q=sort:joined+type:user" + self._urlLocations + \ "&sort=joined&order=asc&per_page=100&page=" + str(page) else: url = "https://api.github.com/search/users?client_id=" + self._githubID + "&client_secret=" + self._githubSecret + \ - "&order=desc&q=sort:joined+type:user+location:" + urllib.parse.quote(self._city) + \ + "&order=desc&q=sort:joined+type:user" + self._urlLocations + \ "+created:" + start_date +\ ".." + final_date +\ "&sort=joined&order="+order+"&per_page=100&page=" + str(page) @@ -425,6 +442,8 @@ def getConfig(self): for e in self._excluded: config["excludedUsers"].append(e) + + config["locations"]=self._locations return config def configToJson(self, fileName): From c182052b55bae793b2b34b44e6b9007b10113caa Mon Sep 17 00:00:00 2001 From: Israel Blancas Date: Thu, 24 Dec 2015 18:11:57 +0100 Subject: [PATCH 05/17] Adding support to exlude locations --- githubcity/GitHubCity.py | 34 +++++++++++++++++++++++++--------- 1 file changed, 25 insertions(+), 9 deletions(-) diff --git a/githubcity/GitHubCity.py b/githubcity/GitHubCity.py index e2baea8..01ce54b 100755 --- a/githubcity/GitHubCity.py +++ b/githubcity/GitHubCity.py @@ -68,7 +68,7 @@ class GitHubCity: """ def __init__(self, githubID, githubSecret, config=None, city=None, locations=None, - excludedJSON=None, debug=False): + excludedUsers=None, excludedLocations=None, debug=False): """Constructor of the class. Note: @@ -79,7 +79,7 @@ def __init__(self, githubID, githubSecret, config=None, city=None, locations=Non city (str): Name of the city you want to search about. githubID (str): ID of your GitHub application. githubSecret (str): Secret of your GitHub application. - excludedJSON (dir): Excluded users of the ranking (see schemaExcluded.json) + excludedUsers (dir): Excluded users of the ranking (see schemaExcluded.json) debug (bool): Show a log in your terminal. Default: False Returns: @@ -97,7 +97,6 @@ def __init__(self, githubID, githubSecret, config=None, city=None, locations=Non if config: self.readConfig(config) - print(config) else: self._city = city self._locations = locations @@ -108,10 +107,16 @@ def __init__(self, githubID, githubSecret, config=None, city=None, locations=Non self._locations.append(self._city) self._excluded = set() - if excludedJSON: - for e in excludedJSON: + if excludedUsers: + for e in excludedUsers: self._excluded.add(e) + + self._excludedLocations = set() + if excludedLocations: + for e in excludedLocations: + self._excludedLocations.add(e) + self._names = queue.Queue() self._myusers = set() self._dataUsers = [] @@ -133,7 +138,6 @@ def _addLocationsToURL(self, locations): self._urlLocations = "" for l in self._locations: - print(type(l)) self._urlLocations += "+location:" + str(urllib.parse.quote(l)) @@ -142,10 +146,15 @@ def readConfig(self, config): self._intervals = config["intervals"] self._lastDay = config["last_date"] self._locations = config["locations"] + excluded = config["excludedUsers"] for e in excluded: self._excluded.add(e) + excluded = config["excludedLocations"] + for e in excluded: + self._excludedLocations.add(e) + self._addLocationsToURL(self._locations) @@ -169,9 +178,12 @@ def _addUser(self, new_user): self._myusers.add(new_user) myNewUser = GitHubUser(new_user) myNewUser.getData() - self._dataUsers.append(myNewUser) - self._logger.debug("NEW USER "+new_user+" "+str(len(self._dataUsers)+1)+" "+\ - threading.current_thread().name) + + userLoc = myNewUser.getLocation() + if not any(s in userLoc for s in self._excludedLocations): + self._dataUsers.append(myNewUser) + self._logger.debug("NEW USER "+new_user+" "+str(len(self._dataUsers)+1)+" "+\ + threading.current_thread().name) @@ -439,10 +451,14 @@ def getConfig(self): config["intervals"] = self._intervals config["last_date"] = self._lastDay config["excludedUsers"]=[] + config["excludedLocations"]=[] for e in self._excluded: config["excludedUsers"].append(e) + for e in self._excludedLocations: + config["excludedLocations"].append(e) + config["locations"]=self._locations return config From eef548578b0a8273616501d523aec22ab7bd326c Mon Sep 17 00:00:00 2001 From: Israel Blancas Date: Fri, 25 Dec 2015 13:20:09 +0100 Subject: [PATCH 06/17] Removing unused files --- schemaExcluded.json | 16 ---------------- testExclude.json | 14 -------------- 2 files changed, 30 deletions(-) delete mode 100644 schemaExcluded.json delete mode 100644 testExclude.json diff --git a/schemaExcluded.json b/schemaExcluded.json deleted file mode 100644 index bc5796c..0000000 --- a/schemaExcluded.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "user": { - "id": "user", - "type": "object", - "properties": { - "name": { - "id": "name", - "type": "string" - }, - "reason": { - "id": "reason", - "type": "string" - } - } - } -} \ No newline at end of file diff --git a/testExclude.json b/testExclude.json deleted file mode 100644 index 9802070..0000000 --- a/testExclude.json +++ /dev/null @@ -1,14 +0,0 @@ -[ - { - "name": "asdpokjdf", - "reason": "It is only a test" - }, - { - "name": "asdfasdf", - "reason": "It is only a test" - }, - { - "name": "iblancasa", - "reason": "It is only a test" - } -] From 8028220d25dcae93f112f558cf2ff4082eaf6778 Mon Sep 17 00:00:00 2001 From: Israel Blancas Date: Fri, 25 Dec 2015 13:22:02 +0100 Subject: [PATCH 07/17] Creating schema #62 --- githubcity/cityconfschema.json | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 githubcity/cityconfschema.json diff --git a/githubcity/cityconfschema.json b/githubcity/cityconfschema.json new file mode 100644 index 0000000..7899f46 --- /dev/null +++ b/githubcity/cityconfschema.json @@ -0,0 +1,21 @@ +{ + "description": "Configuration of a city", + "type": "object", + "properties": { + "excludedLocations": { + "type": "array" + }, + "excludedUsers": { + "type": "array" + }, + "locations": { + "type": "array" + }, + "name": { + "type": "string" + }, + "intervals": { + "type": "array" + } + } +} From 851860281462a04a22d7c5b81db552aaefad33fb Mon Sep 17 00:00:00 2001 From: Israel Blancas Date: Fri, 25 Dec 2015 13:29:37 +0100 Subject: [PATCH 08/17] Solving #60 --- githubcity/GitHubCity.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/githubcity/GitHubCity.py b/githubcity/GitHubCity.py index 01ce54b..5bb5afe 100755 --- a/githubcity/GitHubCity.py +++ b/githubcity/GitHubCity.py @@ -128,7 +128,9 @@ def __init__(self, githubID, githubSecret, config=None, city=None, locations=Non coloredlogs.install(level='DEBUG') self._fin = False - self._l = Lock() + self._lockGetUser = Lock() + self._lockReadAddUser = Lock() + def __str__(self): @@ -174,7 +176,9 @@ def _addUser(self, new_user): new_user (str): name of a GitHub user to include in the ranking """ + self._lockReadAddUser.acquire() if not new_user in self._myusers and not new_user in self._excluded: + self._lockReadAddUser.release() self._myusers.add(new_user) myNewUser = GitHubUser(new_user) myNewUser.getData() @@ -184,6 +188,8 @@ def _addUser(self, new_user): self._dataUsers.append(myNewUser) self._logger.debug("NEW USER "+new_user+" "+str(len(self._dataUsers)+1)+" "+\ threading.current_thread().name) + else: + self._lockReadAddUser.release() @@ -270,14 +276,14 @@ def _processUsers(self): pass while not self._fin or not self._names.empty(): - self._l.acquire() + self._lockGetUser.acquire() try: new_user = self._names.get(False) except queue.Empty: - self._l.release() + self._lockGetUser.release() return else: - self._l.release() + self._lockGetUser.release() self._addUser(new_user) self._logger.debug(str(self._names.qsize())+" users to process") From 852c4917373c70971a820e9847787b5c04d74d23 Mon Sep 17 00:00:00 2001 From: Israel Blancas Date: Fri, 25 Dec 2015 13:58:02 +0100 Subject: [PATCH 09/17] Solving #61 --- githubcity/GitHubCity.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/githubcity/GitHubCity.py b/githubcity/GitHubCity.py index 5bb5afe..3bf6174 100755 --- a/githubcity/GitHubCity.py +++ b/githubcity/GitHubCity.py @@ -100,6 +100,7 @@ def __init__(self, githubID, githubSecret, config=None, city=None, locations=Non else: self._city = city self._locations = locations + self._intervals=[] if not self._locations: self._locations = [] @@ -340,6 +341,9 @@ def _getPeriodUsers(self, start_date, final_date): def getCityUsers(self): """Get all the users from the city. """ + if len(self._intervals)==0: + self.calculateBestIntervals() + self._fin = False self._threads = set() @@ -389,7 +393,6 @@ def calculateBestIntervals(self): """Calcules valid intervals of a city (with less than 1000 users) """ comprobation = self._readAPI(self._getURL()) - self._intervals = [] self._bigCity = True self._validInterval(datetime.date(2008, 1, 1), datetime.datetime.now().date()) self._logger.info("Total number of intervals: "+ str(len(self._intervals))) From 77107a70ed0c8c588054862d77483122d77ccd09 Mon Sep 17 00:00:00 2001 From: Israel Blancas Date: Fri, 25 Dec 2015 14:50:29 +0100 Subject: [PATCH 10/17] Solving #63 --- githubcity/GitHubCity.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/githubcity/GitHubCity.py b/githubcity/GitHubCity.py index 3bf6174..4e8db65 100755 --- a/githubcity/GitHubCity.py +++ b/githubcity/GitHubCity.py @@ -160,6 +160,10 @@ def readConfig(self, config): self._addLocationsToURL(self._locations) + self._validInterval(self._lastDay, datetime.datetime.now().date().strftime("%Y-%m-%d")) + + + def readConfigFromJSON(self, fileName): with open(fileName) as data_file: From 80a779da285e2c7d654337cc7a62523c0de1a6d2 Mon Sep 17 00:00:00 2001 From: Israel Blancas Date: Fri, 25 Dec 2015 17:21:02 +0100 Subject: [PATCH 11/17] Fixing small problem with get new intervals in configuration --- githubcity/GitHubCity.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/githubcity/GitHubCity.py b/githubcity/GitHubCity.py index 4e8db65..2ed1bc7 100755 --- a/githubcity/GitHubCity.py +++ b/githubcity/GitHubCity.py @@ -37,6 +37,7 @@ import calendar import time import queue +import pystache import json from urllib.error import HTTPError from dateutil.relativedelta import relativedelta @@ -159,8 +160,10 @@ def readConfig(self, config): self._excludedLocations.add(e) self._addLocationsToURL(self._locations) + last = datetime.datetime.strptime(self._lastDay, "%Y-%m-%d") + today = datetime.datetime.now().date() - self._validInterval(self._lastDay, datetime.datetime.now().date().strftime("%Y-%m-%d")) + self._validInterval(last, today) From f965415659f3a213213ff1a357599f8f05288a89 Mon Sep 17 00:00:00 2001 From: Israel Blancas Date: Fri, 25 Dec 2015 17:46:50 +0100 Subject: [PATCH 12/17] Perfect in a template --- githubcity/GitHubCity.py | 18 ++++++++++++++++++ githubcity/GitHubUser.py | 18 ++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/githubcity/GitHubCity.py b/githubcity/GitHubCity.py index 2ed1bc7..ed3f9ba 100755 --- a/githubcity/GitHubCity.py +++ b/githubcity/GitHubCity.py @@ -482,3 +482,21 @@ def configToJson(self, fileName): config = self.getConfig() with open(fileName, "w") as outfile: json.dump(config, outfile, indent=4, sort_keys=True) + + def export(self): + exportedData = {} + + dataUsers = self.getSortUsers("contributions") + exportedUsers = [] + + for u in dataUsers: + exportedUsers.append(u.export()) + + exportedData["users"] = exportedUsers + + with open("githubcity/template") as template_file: + template_raw = template_file.read() + + template = pystache.parse(template_raw) + renderer = pystache.Renderer() + print(renderer.render(template, exportedData)) diff --git a/githubcity/GitHubUser.py b/githubcity/GitHubUser.py index 11c8704..623056b 100755 --- a/githubcity/GitHubUser.py +++ b/githubcity/GitHubUser.py @@ -56,6 +56,24 @@ def __init__(self, name): """ self._name = name + def export(self): + data = {} + data["name"] = self.getName() + data["contributions"] = self.getContributions() + data["longestStreak"] = self.getLongestStreak() + data["currentStreak"] = self.getCurrentStreak() + data["language"] = self.getLanguage() + data["avatar"] = self.getAvatar() + data["followers"] = self.getFollowers() + data["join"] = self.getJoin() + data["organizations"] = self.getOrganizations() + data["repositories"] = self.getNumberOfRepositories() + data["stars"] = self.getStars() + + return data + + + def getName(self): return self._name From 7fed238974c79d3692294459347a7858944f12c7 Mon Sep 17 00:00:00 2001 From: Israel Blancas Date: Sat, 26 Dec 2015 13:04:07 +0100 Subject: [PATCH 13/17] Adding parameters #11 --- githubcity/GitHubCity.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/githubcity/GitHubCity.py b/githubcity/GitHubCity.py index ed3f9ba..78b6cad 100755 --- a/githubcity/GitHubCity.py +++ b/githubcity/GitHubCity.py @@ -478,15 +478,17 @@ def getConfig(self): config["locations"]=self._locations return config + def configToJson(self, fileName): config = self.getConfig() with open(fileName, "w") as outfile: json.dump(config, outfile, indent=4, sort_keys=True) - def export(self): + + def export(self, template_file_name, output_file_name, sort): exportedData = {} - dataUsers = self.getSortUsers("contributions") + dataUsers = self.getSortUsers(sort) exportedUsers = [] for u in dataUsers: @@ -494,9 +496,13 @@ def export(self): exportedData["users"] = exportedUsers - with open("githubcity/template") as template_file: + with open(template_file_name) as template_file: template_raw = template_file.read() template = pystache.parse(template_raw) renderer = pystache.Renderer() - print(renderer.render(template, exportedData)) + + output = renderer.render(template, exportedData) + + with open(output_file_name, "w") as text_file: + text_file.write(output) From 44514569e4f08a888e7a78899c49f8ec5099eb1b Mon Sep 17 00:00:00 2001 From: Israel Blancas Date: Sat, 26 Dec 2015 18:58:45 +0100 Subject: [PATCH 14/17] Changing test to the new implementation of GitHubCity#52 #11 --- githubcity/GitHubCity.py | 39 +++---- testConfig.json | 19 ++++ testGitHubCity.py | 213 +++++++++++++++++++++++++-------------- testTemplate | 3 + 4 files changed, 182 insertions(+), 92 deletions(-) create mode 100644 testConfig.json create mode 100644 testTemplate diff --git a/githubcity/GitHubCity.py b/githubcity/GitHubCity.py index 78b6cad..00ee1b4 100755 --- a/githubcity/GitHubCity.py +++ b/githubcity/GitHubCity.py @@ -96,8 +96,22 @@ def __init__(self, githubID, githubSecret, config=None, city=None, locations=Non raise Exception("No GitHub Secret inserted") self._githubSecret = githubSecret + self._names = queue.Queue() + self._myusers = set() + self._dataUsers = [] + self._threads = set() + self._logger = logging.getLogger("GitHubCity") + + if debug: + coloredlogs.install(level='DEBUG') + + self._fin = False + self._lockGetUser = Lock() + self._lockReadAddUser = Lock() + if config: self.readConfig(config) + self._addLocationsToURL(self._locations) else: self._city = city self._locations = locations @@ -108,6 +122,8 @@ def __init__(self, githubID, githubSecret, config=None, city=None, locations=Non if self._city: self._locations.append(self._city) + self._addLocationsToURL(self._locations) + self._excluded = set() if excludedUsers: for e in excludedUsers: @@ -119,20 +135,6 @@ def __init__(self, githubID, githubSecret, config=None, city=None, locations=Non for e in excludedLocations: self._excludedLocations.add(e) - self._names = queue.Queue() - self._myusers = set() - self._dataUsers = [] - self._threads = set() - self._addLocationsToURL(self._locations) - self._logger = logging.getLogger("GitHubCity") - - if debug: - coloredlogs.install(level='DEBUG') - - self._fin = False - self._lockGetUser = Lock() - self._lockReadAddUser = Lock() - def __str__(self): @@ -150,6 +152,8 @@ def readConfig(self, config): self._intervals = config["intervals"] self._lastDay = config["last_date"] self._locations = config["locations"] + self._excluded = set() + self._excludedLocations = set() excluded = config["excludedUsers"] for e in excluded: @@ -415,7 +419,7 @@ def getTotalUsers(self): - def getSortUsers(self, order): + def getSortedUsers(self, order="contributions"): """Returns a list with sorted users. Args: @@ -456,8 +460,6 @@ def getSortUsers(self, order): self._dataUsers.sort(key=lambda u: u.getNumberOfRepositories(), reverse=True) elif order == "stars": self._dataUsers.sort(key=lambda u: u.getStars(), reverse=True) - else: - return None return self._dataUsers @@ -487,8 +489,7 @@ def configToJson(self, fileName): def export(self, template_file_name, output_file_name, sort): exportedData = {} - - dataUsers = self.getSortUsers(sort) + dataUsers = self.getSortedUsers(sort) exportedUsers = [] for u in dataUsers: diff --git a/testConfig.json b/testConfig.json new file mode 100644 index 0000000..8661e11 --- /dev/null +++ b/testConfig.json @@ -0,0 +1,19 @@ +{ + "excludedLocations": [ + "Granada" + ], + "excludedUsers": [ + "vrivas" + ], + "intervals": [ + [ + "2008-01-01", + "2015-12-24" + ] + ], + "last_date": "2015-12-24", + "locations": [ + "Jaén" + ], + "name": "Jaen" +} diff --git a/testGitHubCity.py b/testGitHubCity.py index 68e342c..d174ccd 100644 --- a/testGitHubCity.py +++ b/testGitHubCity.py @@ -44,18 +44,32 @@ ########################################################### idGH = None secretGH = None - -cityA = None - +city = None url = None -excluded = None +config = None + def setup(): - global dataExclude, excluded - os.environ['TZ'] = 'Europe/London' - fileJSON = open('testExclude.json') - excluded = json.loads(fileJSON.read()) - fileJSON.close() + global config + config = { + "excludedLocations": [ + "Granada" + ], + "excludedUsers": [ + "vrivas" + ], + "intervals": [ + [ + "2008-01-01", + "2015-12-24" + ] + ], + "last_date": "2015-12-24", + "locations": [ + "Jaén" + ], + "name": "Jaen" + } def test_githubKeys(): """GitHub keys are inserted correctly @@ -75,65 +89,70 @@ def test_githubKeys(): def test_classCreation(): """GitHubCity instance is created correctly """ - global idGH, secretGH + global idGH, secretGH, city, config - assert_raises(Exception,GitHubCity, None, None, None) - assert_raises(Exception,GitHubCity,"Granada", None, None) - assert_raises(Exception,GitHubCity,"Granada", "asdad78asdd48ad4", None,debug=False) - - cityA = GitHubCity("Granada", idGH, secretGH, debug=True) - ok_(cityA!=None, "City was not created") - ok_(cityA._city!=None, "City name was not setting") - ok_(cityA._githubID!=None, "GitHub ID was not setting") - ok_(cityA._githubSecret!=None, "GitHub Secret was not setting") - eq_(cityA._githubID, os.environ.get('GH_ID'), "GitHub ID was not setting correctly") - eq_(cityA._githubSecret, os.environ.get('GH_SECRET'), "GitHub Secret was not setting correctly") - ok_(cityA._city!="", "City name is an empy str") + assert_raises(Exception,GitHubCity, None, None) + assert_raises(Exception,GitHubCity, "asdad78asdd48ad4",None) + assert_raises(Exception,GitHubCity, None, "asdad78asdd48ad4") + + city = GitHubCity(idGH, secretGH, config=config) + ok_(city._intervals!=None, "City was not created correctly from config") + + city = GitHubCity(idGH, secretGH, config=None,locations=["Granada"], city="Granada", + excludedUsers=["iblancasa"], excludedLocations=["Europe"], debug=True) + ok_(city._city!=None, "City was not created correctly") coloredlogs.install(level='CRITICAL') +def test_loadConfiguration(): + """Loading configuration from file + """ + city = GitHubCity(idGH, secretGH) + city.readConfigFromJSON("testConfig.json") + ok_(city._city=="Jaen", "Configuration was not load correctly from JSON") + def test_urlComposition(): """URL's are composed correctly """ - global url, cityA - cityA = GitHubCity("Granada", idGH, secretGH, debug=False) - url = cityA._getURL() + global url, city + url = city._getURL() expected_url = "https://api.github.com/search/users?client_id="+\ - cityA._githubID + "&client_secret=" + cityA._githubSecret +\ - "&order=desc&q=sort:joined+type:user+location:" + cityA._city + "&sort=joined&order=asc&per_page=100&page=1" + city._githubID + "&client_secret=" + city._githubSecret +\ + "&order=desc&q=sort:joined+type:user+location:" + city._city + "&sort=joined&order=asc&per_page=100&page=1" eq_(url, expected_url, "URL is not well formed when there are not params " + url) - url = cityA._getURL(2) + url = city._getURL(2) expected_url = "https://api.github.com/search/users?client_id="+\ - cityA._githubID + "&client_secret=" + cityA._githubSecret +\ - "&order=desc&q=sort:joined+type:user+location:" + cityA._city + "&sort=joined&order=asc&per_page=100&page=2" + city._githubID + "&client_secret=" + city._githubSecret +\ + "&order=desc&q=sort:joined+type:user+location:" + city._city + "&sort=joined&order=asc&per_page=100&page=2" eq_(url, expected_url, "URL is not well formed when there are 1 param (page) " + url) expected_url = "https://api.github.com/search/users?client_id="+\ - cityA._githubID + "&client_secret=" + cityA._githubSecret +\ - "&order=desc&q=sort:joined+type:user+location:"+ cityA._city +"+created:2008-01-01..2015-12-18"+\ + city._githubID + "&client_secret=" + city._githubSecret +\ + "&order=desc&q=sort:joined+type:user+location:"+ city._city +"+created:2008-01-01..2015-12-18"+\ "&sort=joined&order=asc&per_page=100&page=2" - url = cityA._getURL(2,datetime.date(2008, 1, 1),datetime.date(2015, 12, 18)) + url = city._getURL(2,"2008-01-01", "2015-12-18") eq_(url, expected_url, "URL is not well formed when there are 3 params (page and dates) " + url) + expected_url = "https://api.github.com/search/users?client_id="+\ - cityA._githubID + "&client_secret=" + cityA._githubSecret +\ - "&order=desc&q=sort:joined+type:user+location:"+ cityA._city +"+created:2008-01-01..2015-12-18"+\ + city._githubID + "&client_secret=" + city._githubSecret +\ + "&order=desc&q=sort:joined+type:user+location:"+ city._city +"+created:2008-01-01..2015-12-18"+\ "&sort=joined&order=desc&per_page=100&page=2" - url = cityA._getURL(2,datetime.date(2008, 1, 1),datetime.date(2015, 12, 18),"desc") + url = city._getURL(2,"2008-01-01", "2015-12-18","desc") eq_(url, expected_url, "URL is not well formed when there are 4 params (page, dates and sort) " + url) def test_readAPI(): """Reading API""" - global cityA, url, data + global city, url, data - data = cityA._readAPI(url) + data = city._readAPI(url) ok_(data!=None, "Data received from API is None") ok_("total_count" in data, "Total_count is not correct") ok_("items" in data, "Items are not correct") @@ -141,69 +160,117 @@ def test_readAPI(): def test_addUser(): """Add new users to the list""" - global cityA + global city + + city._addUser("nitehack") + eq_(len(city._myusers), 1, "User was not added to the names list") + eq_(len(city._dataUsers), 1, "User was not added to the dataUsers list") + + city._addUser("nitehack") + eq_(len(city._myusers), 1, "User was added two times to the names list") + eq_(len(city._dataUsers), 1, "User was added two times to the dataUsers list") - cityA._addUser("iblancasa") - eq_(len(cityA._myusers), 1, "User was not added to the names list") - eq_(len(cityA._dataUsers), 1, "User was not added to the dataUsers list") + ok_("nitehack" in city._myusers, + "Add new user was no completed correctly when there is an excluded list") + + city._addUser("iblancasa") + ok_(not "iblancasa" in city._myusers, + "User was added to the users list and he is in excluded list") - cityA._addUser("iblancasa") - eq_(len(cityA._myusers), 1, "User was added two times to the names list") - eq_(len(cityA._dataUsers), 1, "User was added two times to the dataUsers list") + city._addUser("JJ") + + for i in city._dataUsers: + ok_(i.getName() !="JJ", "User was added to the list when his location was excluded") def test_getBestIntervals(): """Get best intervals to query""" - global cityA - cityA = GitHubCity("Barcelona", idGH, secretGH, debug=True) - cityA.calculateBestIntervals() + global city + city = GitHubCity(idGH, secretGH,city="Barcelona", debug=True) + city.calculateBestIntervals() - for i in cityA._intervals: + for i in city._intervals: ok_(i[0]!="" and i[0]!=None, "First part of interval is not correct") - ok_(i[1]!="" and i[0]!=None, "First part of interval is not correct") +def test_strCity(): + """Checking if str is correct + """ + global city + ok_(isinstance(str(city),str), "Str of city is not correct") + def test_getAllUsers(): """Get all users from a city """ - global cityA - cityA.getCityUsers() - ok_(len(cityA._myusers)>=len(cityA._dataUsers),"Get all users is not ok") - - -def test_excludeUsers(): - """Excluded users that are excluded from the list - """ - global idGH, secretGH, excluded, dataExclude, cityA - - cityA = GitHubCity("Granada", idGH, secretGH, excluded) - cityA._addUser("nitehack") - cityA._addUser("iblancasa") - - ok_("nitehack" in cityA._myusers, - "Add new user was no completed correctly when there is an excluded list") - ok_(not "iblancasa" in cityA._myusers, - "User was added to the users list and he is in excluded list") + global idGH, secretGH, city + city.getCityUsers() + ok_(len(city._myusers)>=len(city._dataUsers), "Get all users is not ok") + smallCity = GitHubCity(idGH, secretGH, config=None,locations=["Ceuta"], city="Ceuta", + excludedUsers=[], excludedLocations=[], debug=True) + smallCity.getCityUsers() + ok_(len(smallCity._myusers)>=len(smallCity._dataUsers), "Get all users without calcule intervals\ + before is correct") def test_getTotalUsers(): """Total users number is correct """ - global cityA + global city - users = cityA.getTotalUsers() - eq_(users,len(cityA._dataUsers), "Get users is not correct when there are users") + users = city.getTotalUsers() + eq_(users,len(city._dataUsers), "Get users is not correct when there are users") + + +def test_getSortUsers(): + """Getting sort users + """ + global city + users = city.getSortedUsers("name") + ok_(users[0].getName() >= users[1].getName(), "Users are not sorted correctly -name") + users = city.getSortedUsers("lstreak") + ok_(users[0].getLongestStreak() >= users[1].getLongestStreak(), "Users are not sorted correctly -lstreak") + users = city.getSortedUsers("cstreak") + ok_(users[0].getCurrentStreak() >= users[1].getCurrentStreak(), "Users are not sorted correctly -cstreak") + users = city.getSortedUsers("language") + ok_(users[0].getLanguage() >= users[1].getLanguage(), "Users are not sorted correctly -language") + users = city.getSortedUsers("followers") + ok_(users[0].getFollowers() >= users[1].getFollowers(), "Users are not sorted correctly -followers") + users = city.getSortedUsers("join") + ok_(users[0].getJoin() >= users[1].getJoin(), "Users are not sorted correctly -join") + users = city.getSortedUsers("organizations") + ok_(users[0].getOrganizations() >= users[1].getOrganizations(), "Users are not sorted correctly -organizations") + users = city.getSortedUsers("repositories") + ok_(users[0].getNumberOfRepositories() >= users[1].getNumberOfRepositories(), "Users are not sorted correctly -repositories") + users = city.getSortedUsers("stars") + ok_(users[0].getStars() >= users[1].getStars(), "Users are not sorted correctly -stars") + users = city.getSortedUsers("contributions") + ok_(users[0].getContributions() >= users[1].getContributions(), "Users are not sorted correctly -contributions") + + +def test_export(): + """Exporting users""" + global city + city.export("testTemplate", "out", "contributions") + + +def test_getConfig(): + city = GitHubCity(idGH, secretGH) + city.readConfigFromJSON("testConfig.json") + city.configToJson("asd.json") + city2 = GitHubCity(idGH, secretGH) + city2.readConfigFromJSON("asd.json") + eq_(city._city,city2._city, "Configuration was not saved correctly") def test_checkWhenApiLimit(): """Checking when the API limit is reached """ - global url, cityA + global url, city i = 0 while i<50: - result = cityA._readAPI(url) + result = city._readAPI(url) i+=1 ok_(result!=None, "Problem checking when API limit is reached") diff --git a/testTemplate b/testTemplate new file mode 100644 index 0000000..4cbd585 --- /dev/null +++ b/testTemplate @@ -0,0 +1,3 @@ +{{#users}} +{{name}} +{{/users}} From 688fcaa2c2a8dca9cb73a6910e34102bfd38cccf Mon Sep 17 00:00:00 2001 From: Israel Blancas Date: Sat, 26 Dec 2015 19:47:54 +0100 Subject: [PATCH 15/17] Changing test to the new implementation of GitHubUser #52 #11 --- testGitHubUser.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/testGitHubUser.py b/testGitHubUser.py index fb4b1c8..47ce1d8 100644 --- a/testGitHubUser.py +++ b/testGitHubUser.py @@ -69,6 +69,10 @@ def testGettingData(): ok_(user._numRepos!=None,"numRepos is not correctly") ok_(user._stars!=None,"Stars is not correctly") + user2 = GitHubUser("ManuelPeinado") + user2.getData() + ok_(user2._followers!=None,"Followers is not correctly when an user has Xk followers") + def testGetters(): """Testing getters""" global user @@ -91,15 +95,20 @@ def testNoUser(): falseUser = GitHubUser("shurmanicop") assert_raises(Exception,falseUser.getData()) -''' +def testExport(): + """Testing export users""" + global user + data = user.export() + ok_("name" in data, "Export is correct") + + #This test is deactivated to improve perfomance. Run only local def testLotOfRequest(): """Test if request are completed when server says: 'no more' """ global user i = 0 - while i<40: + while i<20: user._contributions = 0 user.getData() i+=1 ok_(user._contributions!=0, "Lot of request fail") -''' From 6e132beec396a4aa834943dad3ced6ce67038720 Mon Sep 17 00:00:00 2001 From: Israel Blancas Date: Sat, 26 Dec 2015 21:11:04 +0100 Subject: [PATCH 16/17] Fixing GitHubUser documentation --- githubcity/GitHubUser.py | 75 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 72 insertions(+), 3 deletions(-) diff --git a/githubcity/GitHubUser.py b/githubcity/GitHubUser.py index 623056b..eb66f55 100755 --- a/githubcity/GitHubUser.py +++ b/githubcity/GitHubUser.py @@ -41,9 +41,14 @@ class GitHubUser: _name (str): Name of the user (private). _contributions (int): total contributions of a user in the last year (private). _followers (int): total number of followers of an user (private). - _gists (int): total number of gists of an user (private). _longestStreak (int): maximum number of consecutive days with activity (private). _numRepos (int): number of repositories of an user (private). + _stars (int): number of total stars given to the user (private). + _organizations (int): number of public organizations where the user is (private). + _join (str): when the user joined to GitHub. Format: %Y-%M-%DT%H:%i:%sZ (private). + _avatar (str): URL where the user's avatar is (private). + _language (str): language most user by the user (private). + _currentStreak (int): actual number of consecutive days making contributions (private). """ def __init__(self, name): @@ -57,6 +62,10 @@ def __init__(self, name): self._name = name def export(self): + """Export all attributes of the user to a dict + Returns: + dict with all attributes of the user + """ data = {} data["name"] = self.getName() data["contributions"] = self.getContributions() @@ -69,45 +78,105 @@ def export(self): data["organizations"] = self.getOrganizations() data["repositories"] = self.getNumberOfRepositories() data["stars"] = self.getStars() - return data - def getName(self): + """Get the name of the user + Returns: + str with the name of the user + """ return self._name def getContributions(self): + """Get the number of public contributions of the user + Returns: + int with the number of public contributions of the user + """ + return self._contributions def getLongestStreak(self): + """Get the longest streak of the user + + Returns: + int with the longest streak of the user + """ return self._longestStreak def getCurrentStreak(self): + """Get the current streak of the user + + Returns: + int with the current streak of the user + """ return self._currentStreak def getLanguage(self): + """Get the most used language by the user + + Returns: + str with the language most used + + """ return self._language def getAvatar(self): + """Get the URL where the avatar is + + Returns: + str with an URL where the avatar is + """ return self._avatar def getFollowers(self): + """Get the number of followers of this user + + Returns: + int with the number of followers + """ return self._followers def getLocation(self): + """Get the location of the user + + Returns: + str with location of the user + """ return self._location def getJoin(self): + """Get when an user joined to GitHub + + Returns: + a str with this time format %Y-%M-%DT%H:%i:%sZ + """ + return self._join def getOrganizations(self): + """Get the number of public organizations where the user is + + Returns: + int with the number of organizations + + """ return self._organizations def getNumberOfRepositories(self): + """Get the number of repositories of this user + + Returns: + int with the number of repositories + """ return self._numRepos def getStars(self): + """Get number of stars given from GitHub users to repositories created by this user + + Returns: + int with the number of stars + """ return self._stars From d87c68653d71fe065fbb979323a369f7868c5d33 Mon Sep 17 00:00:00 2001 From: Israel Blancas Date: Sat, 26 Dec 2015 23:51:58 +0100 Subject: [PATCH 17/17] Fixing documentation of GitHubCity --- githubcity/GitHubCity.py | 75 ++++++++++++++++++++++++++++++++-------- 1 file changed, 61 insertions(+), 14 deletions(-) diff --git a/githubcity/GitHubCity.py b/githubcity/GitHubCity.py index 00ee1b4..66b13c7 100755 --- a/githubcity/GitHubCity.py +++ b/githubcity/GitHubCity.py @@ -57,17 +57,24 @@ class GitHubCity: Attributes: _city (str): Name of the city (private). _myusers (set): Name of all users in a city (private). - _githubID (str): ID of your GitHub application. - _githubSecret (str): secretGH of your GitHub application. - _dataUsers (List[GitHubUser]): the list of GitHub users. - _excluded (set): list of names of excluded users. - _lastDay (str): day of last interval - _names (Queue): Queue with all users that we still have to process. - _threads (set): Set of active Threads. - _logger (logger): Logger. - _l (Lock): lock to solve problems with threads. + _githubID (str): ID of your GitHub application (private). + _githubSecret (str): secretGH of your GitHub application (private). + _dataUsers (List[GitHubUser]): the list of GitHub users (private). + _excluded (set): set of names of excluded users (private). + _excludedLocations (set): set of excluded locations (private). + _lastDay (str): day of last interval (private). + _names (Queue): Queue with all users that we still have to process (private). + _threads (set): Set of active Threads (private). + _logger (logger): Logger (private). + _lockGetUser (Lock): lock to solve problems with threads in get user (private). + _lockReadAddUser (Lock): lock to solve problems with threads in check if an user is repeated(private). + _fin (bool): check if there are more users to process (private). + _locations (list): list of locations where search users (private). + __urlLocations (str): string with the locations formatted to GitHub API URL (private). + """ + def __init__(self, githubID, githubSecret, config=None, city=None, locations=None, excludedUsers=None, excludedLocations=None, debug=False): """Constructor of the class. @@ -77,11 +84,14 @@ def __init__(self, githubID, githubSecret, config=None, city=None, locations=Non https://github.com/settings/applications/new . Args: - city (str): Name of the city you want to search about. githubID (str): ID of your GitHub application. githubSecret (str): Secret of your GitHub application. - excludedUsers (dir): Excluded users of the ranking (see schemaExcluded.json) - debug (bool): Show a log in your terminal. Default: False + city (str): Name of the city you want to search about (optional). + config (dict): set with configuration (see cityconfigschema.json) + locations (list): locations where search users (optional). + excludedUsers (dir): excluded users of the ranking (optional). + excludedLocations (list): excluded locations (optional). + debug (bool): show a log in your terminal (optional). Returns: a new instance of GithubCity class @@ -138,16 +148,31 @@ def __init__(self, githubID, githubSecret, config=None, city=None, locations=Non def __str__(self): + """str the class""" return str(self.getConfig()) def _addLocationsToURL(self, locations): + """Format all locations to GitHub's URL API + + Note: + This method is private. + + Args: + locations (list): list of str where each str is a location where search + """ self._urlLocations = "" for l in self._locations: self._urlLocations += "+location:" + str(urllib.parse.quote(l)) + def readConfig(self, config): + """Read config from a dict + + Args: + config: config to read (see cityconfigschema.json) + """ self._city = config["name"] self._intervals = config["intervals"] self._lastDay = config["last_date"] @@ -173,6 +198,11 @@ def readConfig(self, config): def readConfigFromJSON(self, fileName): + """Read configuration from a file + + Args: + fileName (str): name of a config file (see cityconfigschema.json) + """ with open(fileName) as data_file: data = json.load(data_file) self.readConfig(data) @@ -254,8 +284,8 @@ def _getURL(self, page=1, start_date=None, final_date=None,order="asc"): Args: page (int): number of the page. - start_date (datetime.date): start date of the range to search users. - final_date (datetime.date): final date of the range to search users. + start_date (str): start date of the range to search users (Y-m-d). + final_date (str): final date of the range to search users (Y-m-d). order (str): order of the query. Valid values are 'asc' or 'desc'. Default: asc Returns: @@ -464,6 +494,11 @@ def getSortedUsers(self, order="contributions"): def getConfig(self): + """Returns the configuration of the city + + Returns: + configuration of the city (dict) + """ config = {} config["name"] = self._city config["intervals"] = self._intervals @@ -482,12 +517,24 @@ def getConfig(self): def configToJson(self, fileName): + """Saves the configuration of the city in a json + + Args: + fileName (str): where save the configuration + """ config = self.getConfig() with open(fileName, "w") as outfile: json.dump(config, outfile, indent=4, sort_keys=True) def export(self, template_file_name, output_file_name, sort): + """Export ranking to a file + + Args: + template_file_name (str): where is the template (moustache template) + output_file_name (str): where create the file with the ranking + sort (str): field to sort the users + """ exportedData = {} dataUsers = self.getSortedUsers(sort) exportedUsers = []