From a3ecb02e75cdcff7229dcd0c7623c16efc2e05b4 Mon Sep 17 00:00:00 2001 From: Kelley Spoon Date: Wed, 17 Nov 2021 07:34:33 -0600 Subject: gl-list-projects: add script to list projects in a namespace This script allows you to traverse a list of all projects in a namespace by looking at the top-level group and subgroups. Settings from the groups and projects are saved to a yaml file under ./data// Change-Id: I465435eaa6b2c020cc94932128340cac3a61fd0e --- .gitignore | 2 ++ download-repo.py | 15 +++-------- gitlab_lib.py | 47 +++++++++++++++++++++++++++++++-- gl-list-projects.py | 76 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 127 insertions(+), 13 deletions(-) create mode 100755 gl-list-projects.py diff --git a/.gitignore b/.gitignore index 77e2c56..a4d3019 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,4 @@ .*swp __pycache__ +data/ +*.pyc diff --git a/download-repo.py b/download-repo.py index 3b295d6..e19fa7e 100755 --- a/download-repo.py +++ b/download-repo.py @@ -7,7 +7,7 @@ import sys import yaml import json from requests.utils import quote -from gitlab_lib import gl_get, valid_project_settings +from gitlab_lib import gl_get, valid_project_settings, get_env_var # Let's do the CLI setup @@ -27,17 +27,10 @@ arguments=parser.parse_args() PROJECT = quote(arguments.project, safe="") -gitlab_server=os.getenv("GITLAB_SERVER") -if gitlab_server is None: - print("Missing enviroment variable: GITLAB_SERVER") - sys.exit(1) +gitlab_server=get_env_var("GITLAB_SERVER") +gitlab_token=get_env_var("GITLAB_TOKEN") -gitlab_token=os.getenv("GITLAB_TOKEN") -if gitlab_token is None: - print("Missing enviroment variable: GITLAB_TOKEN") - sys.exit(1) - -base_url = "{0}/api/v4/".format(gitlab_server) +base_url = "{0}/api/v4".format(gitlab_server) settings_endpoint="{0}/projects/{1}".format(base_url, quote(arguments.project, safe="")) diff --git a/gitlab_lib.py b/gitlab_lib.py index 1a9045d..326f01b 100755 --- a/gitlab_lib.py +++ b/gitlab_lib.py @@ -1,6 +1,21 @@ # Commonly used code for gitlab tools +import json import requests +import os +import sys +import yaml + +from simplejson import errors + + +def get_env_var(varname, required=True): + varvalue = os.getenv(varname) + if required: + if varvalue is None: + print("ERROR: '{0}' not defined".format(varname)) + sys.exit(1) + return varvalue # fetches a url from the gitlab server and returns the json data. # If the result is paginated, it will make multiple requests and @@ -11,14 +26,42 @@ def gl_get(url, token=None): response = requests.get(url, headers={'PRIVATE-TOKEN': token}) - json_data = response.json() + try: + json_data = response.json() + except errors.JSONDecodeError as e: + print(e.msg) + return {} links = response.links if links is not None and 'next' in links: - json_data += get(links['next']['url']) + json_data += gl_get(links['next']['url'], token) return json_data +def dump_yaml(path, filename, data): + os.makedirs(path, exist_ok=True) + with open('/'.join([path,filename]), "w") as outfile: + yaml.dump(data, outfile, default_flow_style=None) + +def get_group_info(glconn, group_id): + group = {} + url = '{0}/groups/{1}'.format(glconn.api_url, group_id) + + group["settings"] = gl_get(url, glconn.private_token) + + return group + +def get_project_info(glconn, project_id): + project = {} + url = '{0}/projects/{1}'.format(glconn.api_url, project_id) + + project["settings"] = gl_get(url, glconn.private_token) + project["variables"] = gl_get('/'.join([url,"variables"]), glconn.private_token) + project["deploy_keys"] = gl_get('/'.join([url,"deploy_keys"]), glconn.private_token) + project["deploy_tokens"] = gl_get('/'.join([url,"deploy_tokens"]), glconn.private_token) + + return project + # should match items at # https://docs.gitlab.com/ee/api/projects.html#edit-project valid_project_settings = ['allow_merge_on_skipped_pipeline', diff --git a/gl-list-projects.py b/gl-list-projects.py new file mode 100755 index 0000000..881a8e2 --- /dev/null +++ b/gl-list-projects.py @@ -0,0 +1,76 @@ +#!/usr/bin/python3 + +# This script will list all projects and backup group and project +# configurations for a given namespace. +# +# Usage: gl-list-projects.py Linaro/lkft +# +# Set the following environment vars: +# export GITLAB_SERVER=https://gitlab.com +# export GITLAB_TOKEN= +# +# The namespace is optional. If left off, the script will attempt to +# download all projects from all namespaces. +# +# Copies of the group and project settings are dumped into a directory +# corresponding to the object's path in gitlab starting in ./data/ +# +# Group vars are put into group.yaml, project settings, variables, deploy_keys, +# and deploy tokens are put into their own sections in the project.yaml file +# +# Finally, the script will out a list of projects it has found with the +# project and 'path_with_namespace' attribute for the project: +# +# (27819928) Linaro/lkft/lkft-private +# (24255949) Linaro/lkft/bisect +# (18855345) Linaro/lkft/lkft-bisection +# (16895870) Linaro/lkft/lkft-tuxpub-deploy + + +from gitlab_lib import * +import json +import sys +from gitlab import * + +DATA_DIR = "./data" + +def data_path(path): + return "%s/%s/%s" % (DATA_DIR, server.replace('https://',''), path) + +try: + NAMESPACE=sys.argv[1] +except IndexError: + NAMESPACE=None + + +server=get_env_var("GITLAB_SERVER") +token=get_env_var("GITLAB_TOKEN") +gl = Gitlab(server, private_token=token) + +# get a list of namespaces +namespaces = gl.namespaces.list() +if NAMESPACE is not None: + namespaces = [x for x in namespaces if x.full_path == NAMESPACE] + +for ns in namespaces: + groups = [] + # get the top level group + groups.append(gl.groups.get(ns.id)) + # now grab the subgroups + for subgroup in groups[0].subgroups.list(): + groups.append(gl.groups.get(subgroup.id)) + + projects = [] + for group in groups: + group_info = get_group_info(gl, group.id) + dump_yaml(data_path(group.full_path), "group.yaml", group_info) + + for group_project in group.projects.list(): + project = gl.projects.get(group_project.id) + project_info = get_project_info(gl, project.id) + + dump_yaml(data_path(project.path_with_namespace), "project.yaml", project_info) + projects.append(project) + +for p in projects: + print("(%s) %s" % (p.id, p.path_with_namespace)) -- cgit v1.2.3