adding token support for bitbucket, adding create/mod date for bitbucket

README.md

@@ -105,6 +105,7 @@ The basic structure is:
         "url": "https://bitbucket.internal",  // Base URL for a Bitbucket Server instance
         "username": "",                       // Username to authenticate with
         "password": "",                       // Password to authenticate with
+        "token": "",                          // Token to authenticate with, if supplied username and password are ignored
 
         "exclude": [ ... ]  // List of projects / repositories to exclude from inventory
     }

scraper/bitbucket/__init__.py

@@ -1,25 +1,52 @@
 import logging
 import stashy
 
+import datetime
+
+from stashy.client import Stash
+
 logger = logging.getLogger(__name__)
 
 
-def connect(url, username, password):
+def connect(url, username=None, password=None, token=None):
     """
     Return a connected Bitbucket session
     """
-
-    bb_session = stashy.connect(url, username, password)
-
-    logger.info("Connected to: %s as %s", url, username)
+    if token is not None:
+        bb_session = Stash(url, token=token)
+        logger.info("Connected to: %s with token", url)
+    else:
+        bb_session = stashy.connect(url, username, password)
+        logger.info("Connected to: %s as username %s", url, username)
 
     return bb_session
 
 
 def all_repos(bb_session):
     """
-    Yields Stashy repo objects for all projects in Bitbucket
+    Yields Stashy repo dictionary objects for all repos in Bitbucket
     """
 
     for repo in bb_session.repos.all():
+        all_commits = sorted(
+            bb_session.projects[repo["project"]["key"]]
+            .repos[repo["name"]]
+            .commits(None),
+            key=lambda x: x["authorTimestamp"],
+        )
+        if all_commits:
+            repo["created"] = (
+                datetime.datetime.fromtimestamp(
+                    all_commits[0]["authorTimestamp"] / 1000
+                )
+                .date()
+                .isoformat()
+            )
+            repo["lastModified"] = (
+                datetime.datetime.fromtimestamp(
+                    all_commits[-1]["authorTimestamp"] / 1000
+                )
+                .date()
+                .isoformat()
+            )
         yield repo

scraper/code_gov/__init__.py

@@ -88,12 +88,12 @@ def process_config(config):
         url = instance.get("url")
         # orgs = instance.get('orgs', None)
         # public_only = instance.get('public_only', True)
-        # token = instance.get('token', None)
-        username = instance.get("username")
-        password = instance.get("password")
+        username = instance.get("username", None)
+        password = instance.get("password", None)
+        token = instance.get("token", None)
         excluded = instance.get("exclude", [])
 
-        bb_session = bitbucket.connect(url, username, password)
+        bb_session = bitbucket.connect(url, username, password, token)
 
         for repo in bitbucket.all_repos(bb_session):
             project = repo["project"]["key"]

scraper/code_gov/models.py

@@ -429,15 +429,14 @@ def from_stashy(klass, repository, labor_hours=True):
 
         # project['reusedCode'] = []
 
-        # date: [object] A date object describing the release.
+        # date: [object] A date object describing the release. Empty if repo has no commits.
         #   created: [string] The date the release was originally created, in YYYY-MM-DD or ISO 8601 format.
         #   lastModified: [string] The date the release was modified, in YYYY-MM-DD or ISO 8601 format.
-        #   metadataLastUpdated: [string] The date the metadata of the release was last updated, in YYYY-MM-DD or ISO 8601 format.
-        # project['date'] = {
-        #     'created': repository.pushed_at.isoformat(),
-        #     'lastModified': repository.updated_at.isoformat(),
-        #     'metadataLastUpdated': '',
-        # }
+        if repository.get("created", None):
+            project["date"] = {
+                "created": repository["created"],
+                "lastModified": repository["lastModified"],
+            }
 
         _prune_dict_null_str(project)