pygit2 git workspace sync (way faster)
This commit is contained in:
parent
214101899e
commit
5e6ada58ae
9 changed files with 210 additions and 212 deletions
|
|
@ -1,54 +0,0 @@
|
|||
import json
|
||||
import os
|
||||
import subprocess
|
||||
|
||||
from qiime2_projects import PROJECTS as Q2_PROJECTS
|
||||
from mds_projects import PROJECTS as MDS_PROJECTS
|
||||
from personal_projects import PROJECTS as PERSONAL_PROJECTS
|
||||
|
||||
|
||||
def fetch_projects(projects, base_fp, remotes):
|
||||
for org, repos in projects.items():
|
||||
base_fp = os.path.join('~', 'projects', base_fp)
|
||||
base_fp = os.path.expanduser(base_fp)
|
||||
|
||||
for repo in repos:
|
||||
print('repo: %s/%s' % (org, repo))
|
||||
|
||||
repo_fp = os.path.join(base_fp, repo)
|
||||
repo_fp = os.path.expanduser(repo_fp)
|
||||
if not os.path.isdir(repo_fp):
|
||||
url = 'https://github.com/%s/%s' % (org, repo)
|
||||
subprocess.run(['git', 'clone', url, repo_fp])
|
||||
subprocess.run(['git', 'remote', 'rename', 'origin', org],
|
||||
cwd=repo_fp)
|
||||
for remote in remotes:
|
||||
url = 'https://github.com/%s/%s' % (remote, repo)
|
||||
try:
|
||||
subprocess.run(['git', 'remote', 'add', remote, url],
|
||||
cwd=repo_fp, check=True,
|
||||
capture_output=True)
|
||||
except subprocess.CalledProcessError as e:
|
||||
msg = 'remote %s already exists' % (remote, )
|
||||
if msg not in str(e.stderr):
|
||||
raise
|
||||
|
||||
try:
|
||||
subprocess.run(['git', 'fetch', remote],
|
||||
cwd=repo_fp, check=True,
|
||||
capture_output=True)
|
||||
except subprocess.CalledProcessError as e:
|
||||
if url not in str(e.stderr):
|
||||
raise
|
||||
else:
|
||||
subprocess.run(['git', 'remote', 'remove', remote],
|
||||
cwd=repo_fp)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
remotes = ['thermokarst', 'ebolyen', 'gregcaporaso', 'ChrisKeefe',
|
||||
'Oddant1', 'nbokulich', 'andrewsanchez', 'David-Rod']
|
||||
|
||||
fetch_projects(Q2_PROJECTS, 'qiime2', remotes)
|
||||
fetch_projects(MDS_PROJECTS, 'mds', [])
|
||||
fetch_projects(PERSONAL_PROJECTS, 'personal', [])
|
||||
|
|
@ -1,57 +0,0 @@
|
|||
import json
|
||||
import os
|
||||
|
||||
from qiime2_projects import PROJECTS as Q2_PROJECTS
|
||||
from mds_projects import PROJECTS as MDS_PROJECTS
|
||||
from personal_projects import PROJECTS as PERSONAL_PROJECTS
|
||||
|
||||
|
||||
def render_vscode_workspace(projects, project_name, output_fp,
|
||||
include_dotfiles=True, extra_dirs=None):
|
||||
folders = []
|
||||
for org, repos in projects.items():
|
||||
for repo in repos:
|
||||
fp = os.path.join('~', 'projects', project_name, repo)
|
||||
fp = os.path.expanduser(fp)
|
||||
folders.append({'name': repo.lower(),
|
||||
'path': fp})
|
||||
|
||||
if include_dotfiles:
|
||||
dotfile_fp = os.path.join(os.sep, 'Users', 'matthew', '.dotfiles')
|
||||
folders.append({'name': 'dotfiles',
|
||||
'path': dotfile_fp})
|
||||
|
||||
if extra_dirs is not None:
|
||||
for name, path in extra_dirs.items():
|
||||
fp = os.path.join('~', 'projects', project_name, path)
|
||||
fp = os.path.expanduser(fp)
|
||||
folders.append({'name': name,
|
||||
'path': fp})
|
||||
|
||||
with open(output_fp, 'w') as fh:
|
||||
json.dump({'folders': sorted(folders, key=lambda x: x['name'])},
|
||||
fh, sort_keys=True, indent=4)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
def qws(ws_number: int):
|
||||
return render_vscode_workspace(
|
||||
Q2_PROJECTS,
|
||||
'qiime2',
|
||||
os.path.join('vscode', 'qiime2-%d.code-workspace' % (ws_number, )),
|
||||
extra_dirs={'data': 'data'},
|
||||
)
|
||||
|
||||
[qws(i) for i in range(1, 4)]
|
||||
|
||||
render_vscode_workspace(
|
||||
MDS_PROJECTS,
|
||||
'mds',
|
||||
os.path.join('vscode', 'mds.code-workspace'),
|
||||
)
|
||||
|
||||
render_vscode_workspace(
|
||||
PERSONAL_PROJECTS,
|
||||
'personal',
|
||||
os.path.join('vscode', 'personal.code-workspace'),
|
||||
)
|
||||
|
|
@ -1,10 +0,0 @@
|
|||
PROJECTS = {
|
||||
'thermokarst': [
|
||||
'ccdb-api',
|
||||
'ccdb-web',
|
||||
'tucotuco',
|
||||
'fathm',
|
||||
'hibernators',
|
||||
'hibernators-web',
|
||||
],
|
||||
}
|
||||
|
|
@ -1,5 +0,0 @@
|
|||
PROJECTS = {
|
||||
'thermokarst': [
|
||||
'elixir-class',
|
||||
],
|
||||
}
|
||||
|
|
@ -1,79 +0,0 @@
|
|||
PROJECTS = {
|
||||
'qiime2': [
|
||||
'Keemei',
|
||||
'action-library-packaging',
|
||||
'busywork',
|
||||
'data302',
|
||||
'dev-docs',
|
||||
'discourse-unhandled-tagger',
|
||||
'docs',
|
||||
'environment-files',
|
||||
'library',
|
||||
'logos',
|
||||
'paper2',
|
||||
'q2-alignment',
|
||||
'q2-composition',
|
||||
'q2-cutadapt',
|
||||
'q2-dada2',
|
||||
'q2-deblur',
|
||||
'q2-demux',
|
||||
'q2-diversity',
|
||||
'q2-diversity-lib',
|
||||
'q2-emperor',
|
||||
'q2-feature-classifier',
|
||||
'q2-feature-table',
|
||||
'q2-fragment-insertion',
|
||||
'q2-gneiss',
|
||||
'q2-longitudinal',
|
||||
'q2-metadata',
|
||||
'q2-mystery-stew',
|
||||
'q2-phylogeny',
|
||||
'q2-quality-control',
|
||||
'q2-quality-filter',
|
||||
'q2-sample-classifier',
|
||||
'q2-shogun',
|
||||
'q2-taxa',
|
||||
'q2-types',
|
||||
'q2-vsearch',
|
||||
'q2cli',
|
||||
'q2cwl',
|
||||
'q2galaxy',
|
||||
'q2studio',
|
||||
'q2templates',
|
||||
'q2view',
|
||||
'qiime2',
|
||||
'qiime2.github.io',
|
||||
'static-site-infrastructure',
|
||||
'template-repo',
|
||||
'view.qiime2.org',
|
||||
'vm-playbooks',
|
||||
'workshop-playbooks',
|
||||
'workshops.qiime2.org',
|
||||
],
|
||||
|
||||
'caporaso-lab': [
|
||||
'caporaso-lab.github.io',
|
||||
'pretrained-feature-classifiers',
|
||||
'q2-phylogenomics',
|
||||
'genome-sampler',
|
||||
],
|
||||
|
||||
'gregcaporaso': [
|
||||
'caporaso-lab-secrets',
|
||||
'qiime2-meta-figures',
|
||||
],
|
||||
|
||||
'biocore': [
|
||||
'scikit-bio',
|
||||
'deblur',
|
||||
],
|
||||
|
||||
'thermokarst': [
|
||||
'q2-no-op',
|
||||
'busywork2_action_playground',
|
||||
],
|
||||
|
||||
'bioconda': [
|
||||
'bioconda-recipes',
|
||||
]
|
||||
}
|
||||
75
bin/sync_git_repos.py
Normal file
75
bin/sync_git_repos.py
Normal file
|
|
@ -0,0 +1,75 @@
|
|||
import configparser
|
||||
import os
|
||||
import sys
|
||||
|
||||
import pygit2
|
||||
|
||||
|
||||
def init_repo(repo_name, repo_fp, remote, remote_name, callbacks,
|
||||
github_peers):
|
||||
repo = pygit2.discover_repository(repo_fp)
|
||||
if repo is None:
|
||||
print('cloning %s' % remote)
|
||||
|
||||
init_remote = lambda r, n, u: r.remotes.create(remote_name, u)
|
||||
repo = pygit2.clone_repository(remote, repo_fp, remote=init_remote,
|
||||
callbacks=callbacks)
|
||||
else:
|
||||
print('already cloned %s' % remote)
|
||||
|
||||
repo = pygit2.Repository(repo)
|
||||
|
||||
for peer in github_peers:
|
||||
url = 'ssh://git@github.com/%s/%s' % (peer, repo_name)
|
||||
|
||||
try:
|
||||
repo.remotes[peer]
|
||||
except KeyError:
|
||||
repo.remotes.create(peer, url)
|
||||
|
||||
if repo.remotes[peer].url != url:
|
||||
repo.remotes.set_url(peer, url)
|
||||
|
||||
|
||||
def sync_workspace(workspace_fp, repos, remote_host, remote_name, callbacks,
|
||||
github_peers):
|
||||
if not os.path.exists(workspace_fp):
|
||||
os.makedirs(workspace_fp)
|
||||
|
||||
for repo in repos:
|
||||
repo_fp = os.path.join(workspace_fp, repo)
|
||||
remote = 'ssh://%s/%s' % (remote_host, repo)
|
||||
|
||||
init_repo(repo, repo_fp, remote, remote_name, callbacks, github_peers)
|
||||
|
||||
|
||||
def setup_callbacks():
|
||||
pub_fp = os.path.expanduser('~/.ssh/id_ecdsa.pub')
|
||||
priv_fp = os.path.expanduser('~/.ssh/id_ecdsa')
|
||||
keypair = pygit2.Keypair('git', pub_fp, priv_fp, '')
|
||||
callbacks = pygit2.RemoteCallbacks(credentials=keypair)
|
||||
return callbacks
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
ini_fp = sys.argv[1]
|
||||
cfg = configparser.ConfigParser()
|
||||
cfg.read(ini_fp)
|
||||
|
||||
callbacks = setup_callbacks()
|
||||
|
||||
for section in cfg.sections():
|
||||
workspace_fp = cfg[section]['workspace']
|
||||
workspace_fp = os.path.expanduser(workspace_fp)
|
||||
|
||||
repos = cfg[section]['repos'].split(',')
|
||||
repos = [r.strip() for r in repos]
|
||||
remote_host = cfg[section]['remote_host']
|
||||
remote_name = cfg[section]['remote_name']
|
||||
|
||||
github_peers = cfg[section]['github_peers'].split(',')
|
||||
if github_peers == ['']:
|
||||
github_peers = []
|
||||
|
||||
sync_workspace(workspace_fp, repos, remote_host,
|
||||
remote_name, callbacks, github_peers)
|
||||
Loading…
Add table
Add a link
Reference in a new issue