gl-list-projects: add script to list projects in a namespaceHEADmaster

This script allows you to traverse a list of all projects
in a namespace by looking at the top-level group and subgroups.

Settings from the groups and projects are saved to a yaml file
under ./data/<servername>/<path to object>

Change-Id: I465435eaa6b2c020cc94932128340cac3a61fd0e

4 files changed, 127 insertions, 13 deletions

diff --git a/.gitignore b/.gitignore
index 77e2c56..a4d3019 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,2 +1,4 @@
 .*swp
 __pycache__
+data/
+*.pyc
diff --git a/download-repo.py b/download-repo.py
index 3b295d6..e19fa7e 100755
--- a/download-repo.py
+++ b/download-repo.py
@@ -7,7 +7,7 @@ import sys
 import yaml
 import json
 from requests.utils import quote
-from gitlab_lib import gl_get, valid_project_settings
+from gitlab_lib import gl_get, valid_project_settings, get_env_var
 
 
 # Let's do the CLI setup
@@ -27,17 +27,10 @@ arguments=parser.parse_args()
 
 PROJECT = quote(arguments.project, safe="")
 
-gitlab_server=os.getenv("GITLAB_SERVER")
-if gitlab_server is None:
-    print("Missing enviroment variable: GITLAB_SERVER")
-    sys.exit(1)
+gitlab_server=get_env_var("GITLAB_SERVER")
+gitlab_token=get_env_var("GITLAB_TOKEN")
 
-gitlab_token=os.getenv("GITLAB_TOKEN")
-if gitlab_token is None:
-    print("Missing enviroment variable: GITLAB_TOKEN")
-    sys.exit(1)
-
-base_url = "{0}/api/v4/".format(gitlab_server)
+base_url = "{0}/api/v4".format(gitlab_server)
 
 settings_endpoint="{0}/projects/{1}".format(base_url,
                     quote(arguments.project, safe=""))
diff --git a/gitlab_lib.py b/gitlab_lib.py
index 1a9045d..326f01b 100755
--- a/gitlab_lib.py
+++ b/gitlab_lib.py
@@ -1,6 +1,21 @@
 # Commonly used code for gitlab tools
 
+import json
 import requests
+import os
+import sys
+import yaml
+
+from simplejson import errors
+
+
+def get_env_var(varname, required=True):
+    varvalue = os.getenv(varname)
+    if required:
+        if varvalue is None:
+            print("ERROR: '{0}' not defined".format(varname))
+            sys.exit(1)
+    return varvalue
 
 # fetches a url from the gitlab server and returns the json data.
 # If the result is paginated, it will make multiple requests and
@@ -11,14 +26,42 @@ def gl_get(url, token=None):
 
     response = requests.get(url, headers={'PRIVATE-TOKEN': token})
 
-    json_data = response.json()
+    try: 
+        json_data = response.json()
+    except errors.JSONDecodeError as e:
+        print(e.msg)
+        return {}
     links = response.links
 
     if links is not None and 'next' in links:
-        json_data += get(links['next']['url'])
+        json_data += gl_get(links['next']['url'], token)
 
     return json_data
 
+def dump_yaml(path, filename, data):
+    os.makedirs(path, exist_ok=True)
+    with open('/'.join([path,filename]), "w") as outfile:
+        yaml.dump(data, outfile, default_flow_style=None)
+
+def get_group_info(glconn, group_id):
+    group = {}
+    url = '{0}/groups/{1}'.format(glconn.api_url, group_id)
+
+    group["settings"] = gl_get(url, glconn.private_token)
+
+    return group
+
+def get_project_info(glconn, project_id):
+    project = {}
+    url = '{0}/projects/{1}'.format(glconn.api_url, project_id)
+
+    project["settings"] = gl_get(url, glconn.private_token)
+    project["variables"] = gl_get('/'.join([url,"variables"]), glconn.private_token)
+    project["deploy_keys"] = gl_get('/'.join([url,"deploy_keys"]), glconn.private_token)
+    project["deploy_tokens"] = gl_get('/'.join([url,"deploy_tokens"]), glconn.private_token)
+
+    return project
+
 # should match items at
 # https://docs.gitlab.com/ee/api/projects.html#edit-project
 valid_project_settings = ['allow_merge_on_skipped_pipeline',
diff --git a/gl-list-projects.py b/gl-list-projects.py
new file mode 100755
index 0000000..881a8e2
--- /dev/null
+++ b/gl-list-projects.py
@@ -0,0 +1,76 @@
+#!/usr/bin/python3
+
+# This script will list all projects and backup group and project
+# configurations for a given namespace.
+#
+# Usage:  gl-list-projects.py Linaro/lkft
+#
+# Set the following environment vars:
+# export GITLAB_SERVER=https://gitlab.com
+# export GITLAB_TOKEN=<your API token>
+#
+# The namespace is optional.  If left off, the script will attempt to
+# download all projects from all namespaces.
+#
+# Copies of the group and project settings are dumped into a directory
+# corresponding to the object's path in gitlab starting in ./data/<servername>
+#
+# Group vars are put into group.yaml, project settings, variables, deploy_keys,
+# and deploy tokens are put into their own sections in the project.yaml file
+# 
+# Finally, the script will out a list of projects it has found with the
+# project and 'path_with_namespace' attribute for the project:
+#
+# (27819928) Linaro/lkft/lkft-private
+# (24255949) Linaro/lkft/bisect
+# (18855345) Linaro/lkft/lkft-bisection
+# (16895870) Linaro/lkft/lkft-tuxpub-deploy
+
+
+from gitlab_lib import *
+import json
+import sys
+from gitlab import *
+
+DATA_DIR = "./data"
+
+def data_path(path):
+    return "%s/%s/%s" % (DATA_DIR, server.replace('https://',''), path)
+
+try:
+    NAMESPACE=sys.argv[1]
+except IndexError:
+    NAMESPACE=None
+
+
+server=get_env_var("GITLAB_SERVER")
+token=get_env_var("GITLAB_TOKEN")
+gl = Gitlab(server, private_token=token)
+
+# get a list of namespaces
+namespaces = gl.namespaces.list()
+if NAMESPACE is not None:
+    namespaces = [x for x in namespaces if x.full_path == NAMESPACE]
+
+for ns in namespaces:
+    groups = []
+    # get the top level group
+    groups.append(gl.groups.get(ns.id))
+    # now grab the subgroups
+    for subgroup in groups[0].subgroups.list():
+        groups.append(gl.groups.get(subgroup.id))
+
+    projects = []
+    for group in groups:
+        group_info = get_group_info(gl, group.id)
+        dump_yaml(data_path(group.full_path), "group.yaml", group_info)
+
+        for group_project in group.projects.list():
+            project = gl.projects.get(group_project.id)
+            project_info = get_project_info(gl, project.id)
+
+            dump_yaml(data_path(project.path_with_namespace), "project.yaml", project_info)
+            projects.append(project)
+
+for p in projects:
+    print("(%s) %s" % (p.id, p.path_with_namespace))