From 5e6ada58ae47b84881a9307204ed3d98a175685c Mon Sep 17 00:00:00 2001 From: Matthew Dillon Date: Sat, 30 Jan 2021 22:56:21 -0700 Subject: [PATCH] pygit2 git workspace sync (way faster) --- README.md | 2 + bin/bootstrap_git_repos.py | 54 ------------ bin/generate_vscode_workspaces.py | 57 ------------- bin/mds_projects.py | 10 --- bin/personal_projects.py | 5 -- bin/qiime2_projects.py | 79 ------------------ bin/sync_git_repos.py | 75 +++++++++++++++++ install.conf.yaml | 8 +- repos.ini | 132 ++++++++++++++++++++++++++++++ 9 files changed, 210 insertions(+), 212 deletions(-) delete mode 100644 bin/bootstrap_git_repos.py delete mode 100644 bin/generate_vscode_workspaces.py delete mode 100644 bin/mds_projects.py delete mode 100644 bin/personal_projects.py delete mode 100644 bin/qiime2_projects.py create mode 100644 bin/sync_git_repos.py create mode 100644 repos.ini diff --git a/README.md b/README.md index 0b1ad55..4c646bb 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,8 @@ $ sudo sh Miniconda3-latest-MacOSX-x86_64.sh -p /opt/miniconda3 -b $ sudo conda update conda $ wget https://raw.githubusercontent.com/qiime2/environment-files/master/latest/staging/qiime2-latest-py36-osx-conda.yml $ conda env create -n q2dev --file qiime2-latest-py36-osx-conda.yml +$ conda env create -n dotfiles -c conda-forge pygit2 +$ conda run -n dotfiles python ~/.dotfiles/bin/sync_git_repos.py ~/.dotfiles/repos.ini ``` ## optional vscode extensions diff --git a/bin/bootstrap_git_repos.py b/bin/bootstrap_git_repos.py deleted file mode 100644 index 06a356a..0000000 --- a/bin/bootstrap_git_repos.py +++ /dev/null @@ -1,54 +0,0 @@ -import json -import os -import subprocess - -from qiime2_projects import PROJECTS as Q2_PROJECTS -from mds_projects import PROJECTS as MDS_PROJECTS -from personal_projects import PROJECTS as PERSONAL_PROJECTS - - -def fetch_projects(projects, base_fp, remotes): - for org, repos in projects.items(): - base_fp = os.path.join('~', 'projects', base_fp) - base_fp = os.path.expanduser(base_fp) - - for repo in repos: - print('repo: %s/%s' % (org, repo)) - - repo_fp = os.path.join(base_fp, repo) - repo_fp = os.path.expanduser(repo_fp) - if not os.path.isdir(repo_fp): - url = 'https://github.com/%s/%s' % (org, repo) - subprocess.run(['git', 'clone', url, repo_fp]) - subprocess.run(['git', 'remote', 'rename', 'origin', org], - cwd=repo_fp) - for remote in remotes: - url = 'https://github.com/%s/%s' % (remote, repo) - try: - subprocess.run(['git', 'remote', 'add', remote, url], - cwd=repo_fp, check=True, - capture_output=True) - except subprocess.CalledProcessError as e: - msg = 'remote %s already exists' % (remote, ) - if msg not in str(e.stderr): - raise - - try: - subprocess.run(['git', 'fetch', remote], - cwd=repo_fp, check=True, - capture_output=True) - except subprocess.CalledProcessError as e: - if url not in str(e.stderr): - raise - else: - subprocess.run(['git', 'remote', 'remove', remote], - cwd=repo_fp) - - -if __name__ == '__main__': - remotes = ['thermokarst', 'ebolyen', 'gregcaporaso', 'ChrisKeefe', - 'Oddant1', 'nbokulich', 'andrewsanchez', 'David-Rod'] - - fetch_projects(Q2_PROJECTS, 'qiime2', remotes) - fetch_projects(MDS_PROJECTS, 'mds', []) - fetch_projects(PERSONAL_PROJECTS, 'personal', []) diff --git a/bin/generate_vscode_workspaces.py b/bin/generate_vscode_workspaces.py deleted file mode 100644 index e7e7515..0000000 --- a/bin/generate_vscode_workspaces.py +++ /dev/null @@ -1,57 +0,0 @@ -import json -import os - -from qiime2_projects import PROJECTS as Q2_PROJECTS -from mds_projects import PROJECTS as MDS_PROJECTS -from personal_projects import PROJECTS as PERSONAL_PROJECTS - - -def render_vscode_workspace(projects, project_name, output_fp, - include_dotfiles=True, extra_dirs=None): - folders = [] - for org, repos in projects.items(): - for repo in repos: - fp = os.path.join('~', 'projects', project_name, repo) - fp = os.path.expanduser(fp) - folders.append({'name': repo.lower(), - 'path': fp}) - - if include_dotfiles: - dotfile_fp = os.path.join(os.sep, 'Users', 'matthew', '.dotfiles') - folders.append({'name': 'dotfiles', - 'path': dotfile_fp}) - - if extra_dirs is not None: - for name, path in extra_dirs.items(): - fp = os.path.join('~', 'projects', project_name, path) - fp = os.path.expanduser(fp) - folders.append({'name': name, - 'path': fp}) - - with open(output_fp, 'w') as fh: - json.dump({'folders': sorted(folders, key=lambda x: x['name'])}, - fh, sort_keys=True, indent=4) - - -if __name__ == '__main__': - def qws(ws_number: int): - return render_vscode_workspace( - Q2_PROJECTS, - 'qiime2', - os.path.join('vscode', 'qiime2-%d.code-workspace' % (ws_number, )), - extra_dirs={'data': 'data'}, - ) - - [qws(i) for i in range(1, 4)] - - render_vscode_workspace( - MDS_PROJECTS, - 'mds', - os.path.join('vscode', 'mds.code-workspace'), - ) - - render_vscode_workspace( - PERSONAL_PROJECTS, - 'personal', - os.path.join('vscode', 'personal.code-workspace'), - ) diff --git a/bin/mds_projects.py b/bin/mds_projects.py deleted file mode 100644 index 94825b4..0000000 --- a/bin/mds_projects.py +++ /dev/null @@ -1,10 +0,0 @@ -PROJECTS = { - 'thermokarst': [ - 'ccdb-api', - 'ccdb-web', - 'tucotuco', - 'fathm', - 'hibernators', - 'hibernators-web', - ], -} diff --git a/bin/personal_projects.py b/bin/personal_projects.py deleted file mode 100644 index b557888..0000000 --- a/bin/personal_projects.py +++ /dev/null @@ -1,5 +0,0 @@ -PROJECTS = { - 'thermokarst': [ - 'elixir-class', - ], -} diff --git a/bin/qiime2_projects.py b/bin/qiime2_projects.py deleted file mode 100644 index 0a6e095..0000000 --- a/bin/qiime2_projects.py +++ /dev/null @@ -1,79 +0,0 @@ -PROJECTS = { - 'qiime2': [ - 'Keemei', - 'action-library-packaging', - 'busywork', - 'data302', - 'dev-docs', - 'discourse-unhandled-tagger', - 'docs', - 'environment-files', - 'library', - 'logos', - 'paper2', - 'q2-alignment', - 'q2-composition', - 'q2-cutadapt', - 'q2-dada2', - 'q2-deblur', - 'q2-demux', - 'q2-diversity', - 'q2-diversity-lib', - 'q2-emperor', - 'q2-feature-classifier', - 'q2-feature-table', - 'q2-fragment-insertion', - 'q2-gneiss', - 'q2-longitudinal', - 'q2-metadata', - 'q2-mystery-stew', - 'q2-phylogeny', - 'q2-quality-control', - 'q2-quality-filter', - 'q2-sample-classifier', - 'q2-shogun', - 'q2-taxa', - 'q2-types', - 'q2-vsearch', - 'q2cli', - 'q2cwl', - 'q2galaxy', - 'q2studio', - 'q2templates', - 'q2view', - 'qiime2', - 'qiime2.github.io', - 'static-site-infrastructure', - 'template-repo', - 'view.qiime2.org', - 'vm-playbooks', - 'workshop-playbooks', - 'workshops.qiime2.org', - ], - - 'caporaso-lab': [ - 'caporaso-lab.github.io', - 'pretrained-feature-classifiers', - 'q2-phylogenomics', - 'genome-sampler', - ], - - 'gregcaporaso': [ - 'caporaso-lab-secrets', - 'qiime2-meta-figures', - ], - - 'biocore': [ - 'scikit-bio', - 'deblur', - ], - - 'thermokarst': [ - 'q2-no-op', - 'busywork2_action_playground', - ], - - 'bioconda': [ - 'bioconda-recipes', - ] -} diff --git a/bin/sync_git_repos.py b/bin/sync_git_repos.py new file mode 100644 index 0000000..73c8802 --- /dev/null +++ b/bin/sync_git_repos.py @@ -0,0 +1,75 @@ +import configparser +import os +import sys + +import pygit2 + + +def init_repo(repo_name, repo_fp, remote, remote_name, callbacks, + github_peers): + repo = pygit2.discover_repository(repo_fp) + if repo is None: + print('cloning %s' % remote) + + init_remote = lambda r, n, u: r.remotes.create(remote_name, u) + repo = pygit2.clone_repository(remote, repo_fp, remote=init_remote, + callbacks=callbacks) + else: + print('already cloned %s' % remote) + + repo = pygit2.Repository(repo) + + for peer in github_peers: + url = 'ssh://git@github.com/%s/%s' % (peer, repo_name) + + try: + repo.remotes[peer] + except KeyError: + repo.remotes.create(peer, url) + + if repo.remotes[peer].url != url: + repo.remotes.set_url(peer, url) + + +def sync_workspace(workspace_fp, repos, remote_host, remote_name, callbacks, + github_peers): + if not os.path.exists(workspace_fp): + os.makedirs(workspace_fp) + + for repo in repos: + repo_fp = os.path.join(workspace_fp, repo) + remote = 'ssh://%s/%s' % (remote_host, repo) + + init_repo(repo, repo_fp, remote, remote_name, callbacks, github_peers) + + +def setup_callbacks(): + pub_fp = os.path.expanduser('~/.ssh/id_ecdsa.pub') + priv_fp = os.path.expanduser('~/.ssh/id_ecdsa') + keypair = pygit2.Keypair('git', pub_fp, priv_fp, '') + callbacks = pygit2.RemoteCallbacks(credentials=keypair) + return callbacks + + +if __name__ == '__main__': + ini_fp = sys.argv[1] + cfg = configparser.ConfigParser() + cfg.read(ini_fp) + + callbacks = setup_callbacks() + + for section in cfg.sections(): + workspace_fp = cfg[section]['workspace'] + workspace_fp = os.path.expanduser(workspace_fp) + + repos = cfg[section]['repos'].split(',') + repos = [r.strip() for r in repos] + remote_host = cfg[section]['remote_host'] + remote_name = cfg[section]['remote_name'] + + github_peers = cfg[section]['github_peers'].split(',') + if github_peers == ['']: + github_peers = [] + + sync_workspace(workspace_fp, repos, remote_host, + remote_name, callbacks, github_peers) diff --git a/install.conf.yaml b/install.conf.yaml index 705598a..aca7a7e 100644 --- a/install.conf.yaml +++ b/install.conf.yaml @@ -3,12 +3,11 @@ - create: - ~/projects/qiime2 - - ~/projects/qiime2/data/moving-pictures + - ~/projects/qiime2/data/ - ~/projects/mds - ~/projects/personal - shell: - - python3 bin/generate_vscode_workspaces.py - python3 bin/bootstrap_git_repos.py - link: @@ -33,11 +32,6 @@ ~/Library/Application Support/Code/User/tasks.json: create: true path: vscode/tasks.json - ~/qiime2-1.code-workspace: vscode/qiime2-1.code-workspace - ~/qiime2-2.code-workspace: vscode/qiime2-2.code-workspace - ~/qiime2-3.code-workspace: vscode/qiime2-3.code-workspace - ~/mds.code-workspace: vscode/mds.code-workspace - ~/personal.code-workspace: vscode/personal.code-workspace ~/.config/kak/kakrc: create: true path: kakrc diff --git a/repos.ini b/repos.ini new file mode 100644 index 0000000..d2af60e --- /dev/null +++ b/repos.ini @@ -0,0 +1,132 @@ +[personal] +workspace=~/projects/personal2 +remote_host=git@pingo.thermokar.st +remote_name=pingo +github_peers= +repos= + 3dmodels, + advent-of-code-2015, + advent-of-code-2016, + akdillon, + akextract, + akindices, + aoc2020, + arctic_hibernators_schema, + bactdb, + bactdb_data, + ccdb-api, + ccdb-old, + ccdb-web, + cs425_anch_tax_data_loader, + cs425_anch_tax_map_app, + cs680_aes, + cs680_aes_report, + cs685, + dot_ssh, + dotfiles, + drf_ember_pagination, + elixir-class, + fpjs, + gitolite-admin, + gpx-web-utils, + hibernators, + hibernators-web, + hymenobacterdotinfo, + jwt, + pingo, + planner, + qzv-view, + stem2017, + thermokar.st, + zettel + +[github_thermokast_personal] +workspace=~/projects/personal2 +remote_host=git@github.com/thermokarst +remote_name=thermokarst +github_peers= +repos= + thermokarst + +[github_thermokarst_qiime2] +workspace=~/projects/qiime22 +remote_host=git@github.com/thermokarst +remote_name=thermokarst +github_peers= +repos= + workflows-playground, + q2-no-op + +[github_thermokarst_forks_qiime2] +workspace=~/projects/qiime22 +remote_host=git@github.com/thermokarst-forks +remote_name=thermokarst +github_peers=ebolyen,gregcaporaso,ChrisKeefe,Oddant1,nbokulich,andrewsanchez +repos= + An-Introduction-To-Applied-Bioinformatics, + Keemei, + RESCRIPt, + action-library-packaging, + biom-format-feedstock, + busywork, + caporaso-lab.github.io, + code-of-conduct, + conda-channel-resource, + data302, + deblur, + dev-docs, + docs, + emperor, + environment-files, + genome-sampler, + gneiss, + ijson-feedstock, + library, + logos, + paper1, + paper2, + pretrained-feature-classifiers, + q2-alignment, + q2-composition, + q2-cutadapt, + q2-dada2, + q2-deblur, + q2-demux, + q2-diversity, + q2-diversity-lib, + q2-emperor, + q2-feature-classifier, + q2-feature-table, + q2-fragment-insertion, + q2-gneiss, + q2-longitudinal, + q2-metadata, + q2-perc-norm, + q2-phylogenomics, + q2-phylogeny, + q2-quality-control, + q2-quality-filter, + q2-sample-classifier, + q2-taxa, + q2-types, + q2-vsearch, + q2_itsxpress, + q2cli, + q2cwl, + q2galaxy, + q2lint, + q2studio, + q2templates, + q2view, + qiime2, + qiime2.github.io, + scikit-bio, + scikit-bio-feedstock, + spcss, + sphinx-qiime2-theme, + static-site-infrastructure, + template-repo, + view.qiime2.org, + vm-playbooks, + workshop-playbooks, + workshops