thermokarst/dotfiles
1
0
Fork 0
Code Issues 4 Pull requests Projects Releases Packages Wiki Activity Actions

pygit2 git workspace sync (way faster)

Browse source
This commit is contained in:
Matthew Ryan Dillon 2021-01-30 22:56:21 -07:00
parent 214101899e
commit 5e6ada58ae
9 changed files with 210 additions and 212 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
README.md
View file

@ -34,6 +34,8 @@ $ sudo sh Miniconda3-latest-MacOSX-x86_64.sh -p /opt/miniconda3 -b
$ sudo conda update conda
$ wget https://raw.githubusercontent.com/qiime2/environment-files/master/latest/staging/qiime2-latest-py36-osx-conda.yml
$ conda env create -n q2dev --file qiime2-latest-py36-osx-conda.yml
$ conda env create -n dotfiles -c conda-forge pygit2
$ conda run -n dotfiles python ~/.dotfiles/bin/sync_git_repos.py ~/.dotfiles/repos.ini
```
## optional vscode extensions

54
bin/bootstrap_git_repos.py
View file

@ -1,54 +0,0 @@
import json
import os
import subprocess
from qiime2_projects import PROJECTS as Q2_PROJECTS
from mds_projects import PROJECTS as MDS_PROJECTS
from personal_projects import PROJECTS as PERSONAL_PROJECTS
def fetch_projects(projects, base_fp, remotes):
for org, repos in projects.items():
base_fp = os.path.join('~', 'projects', base_fp)
base_fp = os.path.expanduser(base_fp)
for repo in repos:
print('repo: %s/%s' % (org, repo))
repo_fp = os.path.join(base_fp, repo)
repo_fp = os.path.expanduser(repo_fp)
if not os.path.isdir(repo_fp):
url = 'https://github.com/%s/%s' % (org, repo)
subprocess.run(['git', 'clone', url, repo_fp])
subprocess.run(['git', 'remote', 'rename', 'origin', org],
cwd=repo_fp)
for remote in remotes:
url = 'https://github.com/%s/%s' % (remote, repo)
try:
subprocess.run(['git', 'remote', 'add', remote, url],
cwd=repo_fp, check=True,
capture_output=True)
except subprocess.CalledProcessError as e:
msg = 'remote %s already exists' % (remote, )
if msg not in str(e.stderr):
raise
try:
subprocess.run(['git', 'fetch', remote],
cwd=repo_fp, check=True,
capture_output=True)
except subprocess.CalledProcessError as e:
if url not in str(e.stderr):
raise
else:
subprocess.run(['git', 'remote', 'remove', remote],
cwd=repo_fp)
if __name__ == '__main__':
remotes = ['thermokarst', 'ebolyen', 'gregcaporaso', 'ChrisKeefe',
'Oddant1', 'nbokulich', 'andrewsanchez', 'David-Rod']
fetch_projects(Q2_PROJECTS, 'qiime2', remotes)
fetch_projects(MDS_PROJECTS, 'mds', [])
fetch_projects(PERSONAL_PROJECTS, 'personal', [])

57
bin/generate_vscode_workspaces.py
View file

@ -1,57 +0,0 @@
import json
import os
from qiime2_projects import PROJECTS as Q2_PROJECTS
from mds_projects import PROJECTS as MDS_PROJECTS
from personal_projects import PROJECTS as PERSONAL_PROJECTS
def render_vscode_workspace(projects, project_name, output_fp,
include_dotfiles=True, extra_dirs=None):
folders = []
for org, repos in projects.items():
for repo in repos:
fp = os.path.join('~', 'projects', project_name, repo)
fp = os.path.expanduser(fp)
folders.append({'name': repo.lower(),
'path': fp})
if include_dotfiles:
dotfile_fp = os.path.join(os.sep, 'Users', 'matthew', '.dotfiles')
folders.append({'name': 'dotfiles',
'path': dotfile_fp})
if extra_dirs is not None:
for name, path in extra_dirs.items():
fp = os.path.join('~', 'projects', project_name, path)
fp = os.path.expanduser(fp)
folders.append({'name': name,
'path': fp})
with open(output_fp, 'w') as fh:
json.dump({'folders': sorted(folders, key=lambda x: x['name'])},
fh, sort_keys=True, indent=4)
if __name__ == '__main__':
def qws(ws_number: int):
return render_vscode_workspace(
Q2_PROJECTS,
'qiime2',
os.path.join('vscode', 'qiime2-%d.code-workspace' % (ws_number, )),
extra_dirs={'data': 'data'},
)
[qws(i) for i in range(1, 4)]
render_vscode_workspace(
MDS_PROJECTS,
'mds',
os.path.join('vscode', 'mds.code-workspace'),
)
render_vscode_workspace(
PERSONAL_PROJECTS,
'personal',
os.path.join('vscode', 'personal.code-workspace'),
)

10
bin/mds_projects.py
View file

@ -1,10 +0,0 @@
PROJECTS = {
'thermokarst': [
'ccdb-api',
'ccdb-web',
'tucotuco',
'fathm',
'hibernators',
'hibernators-web',
],
}

5
bin/personal_projects.py
View file

@ -1,5 +0,0 @@
PROJECTS = {
'thermokarst': [
'elixir-class',
],
}

79
bin/qiime2_projects.py
View file

@ -1,79 +0,0 @@
PROJECTS = {
'qiime2': [
'Keemei',
'action-library-packaging',
'busywork',
'data302',
'dev-docs',
'discourse-unhandled-tagger',
'docs',
'environment-files',
'library',
'logos',
'paper2',
'q2-alignment',
'q2-composition',
'q2-cutadapt',
'q2-dada2',
'q2-deblur',
'q2-demux',
'q2-diversity',
'q2-diversity-lib',
'q2-emperor',
'q2-feature-classifier',
'q2-feature-table',
'q2-fragment-insertion',
'q2-gneiss',
'q2-longitudinal',
'q2-metadata',
'q2-mystery-stew',
'q2-phylogeny',
'q2-quality-control',
'q2-quality-filter',
'q2-sample-classifier',
'q2-shogun',
'q2-taxa',
'q2-types',
'q2-vsearch',
'q2cli',
'q2cwl',
'q2galaxy',
'q2studio',
'q2templates',
'q2view',
'qiime2',
'qiime2.github.io',
'static-site-infrastructure',
'template-repo',
'view.qiime2.org',
'vm-playbooks',
'workshop-playbooks',
'workshops.qiime2.org',
],
'caporaso-lab': [
'caporaso-lab.github.io',
'pretrained-feature-classifiers',
'q2-phylogenomics',
'genome-sampler',
],
'gregcaporaso': [
'caporaso-lab-secrets',
'qiime2-meta-figures',
],
'biocore': [
'scikit-bio',
'deblur',
],
'thermokarst': [
'q2-no-op',
'busywork2_action_playground',
],
'bioconda': [
'bioconda-recipes',
]
}

75
bin/sync_git_repos.py Normal file
View file

@ -0,0 +1,75 @@
import configparser
import os
import sys
import pygit2
def init_repo(repo_name, repo_fp, remote, remote_name, callbacks,
github_peers):
repo = pygit2.discover_repository(repo_fp)
if repo is None:
print('cloning %s' % remote)
init_remote = lambda r, n, u: r.remotes.create(remote_name, u)
repo = pygit2.clone_repository(remote, repo_fp, remote=init_remote,
callbacks=callbacks)
else:
print('already cloned %s' % remote)
repo = pygit2.Repository(repo)
for peer in github_peers:
url = 'ssh://git@github.com/%s/%s' % (peer, repo_name)
try:
repo.remotes[peer]
except KeyError:
repo.remotes.create(peer, url)
if repo.remotes[peer].url != url:
repo.remotes.set_url(peer, url)
def sync_workspace(workspace_fp, repos, remote_host, remote_name, callbacks,
github_peers):
if not os.path.exists(workspace_fp):
os.makedirs(workspace_fp)
for repo in repos:
repo_fp = os.path.join(workspace_fp, repo)
remote = 'ssh://%s/%s' % (remote_host, repo)
init_repo(repo, repo_fp, remote, remote_name, callbacks, github_peers)
def setup_callbacks():
pub_fp = os.path.expanduser('~/.ssh/id_ecdsa.pub')
priv_fp = os.path.expanduser('~/.ssh/id_ecdsa')
keypair = pygit2.Keypair('git', pub_fp, priv_fp, '')
callbacks = pygit2.RemoteCallbacks(credentials=keypair)
return callbacks
if __name__ == '__main__':
ini_fp = sys.argv[1]
cfg = configparser.ConfigParser()
cfg.read(ini_fp)
callbacks = setup_callbacks()
for section in cfg.sections():
workspace_fp = cfg[section]['workspace']
workspace_fp = os.path.expanduser(workspace_fp)
repos = cfg[section]['repos'].split(',')
repos = [r.strip() for r in repos]
remote_host = cfg[section]['remote_host']
remote_name = cfg[section]['remote_name']
github_peers = cfg[section]['github_peers'].split(',')
if github_peers == ['']:
github_peers = []
sync_workspace(workspace_fp, repos, remote_host,
remote_name, callbacks, github_peers)

8
install.conf.yaml
View file

@ -3,12 +3,11 @@
- create:
- ~/projects/qiime2
- ~/projects/qiime2/data/moving-pictures
- ~/projects/qiime2/data/
- ~/projects/mds
- ~/projects/personal
- shell:
- python3 bin/generate_vscode_workspaces.py
- python3 bin/bootstrap_git_repos.py
- link:
@ -33,11 +32,6 @@
~/Library/Application Support/Code/User/tasks.json:
create: true
path: vscode/tasks.json
~/qiime2-1.code-workspace: vscode/qiime2-1.code-workspace
~/qiime2-2.code-workspace: vscode/qiime2-2.code-workspace
~/qiime2-3.code-workspace: vscode/qiime2-3.code-workspace
~/mds.code-workspace: vscode/mds.code-workspace
~/personal.code-workspace: vscode/personal.code-workspace
~/.config/kak/kakrc:
create: true
path: kakrc

132
repos.ini Normal file
View file

@ -0,0 +1,132 @@
[personal]
workspace=~/projects/personal2
remote_host=git@pingo.thermokar.st
remote_name=pingo
github_peers=
repos=
3dmodels,
advent-of-code-2015,
advent-of-code-2016,
akdillon,
akextract,
akindices,
aoc2020,
arctic_hibernators_schema,
bactdb,
bactdb_data,
ccdb-api,
ccdb-old,
ccdb-web,
cs425_anch_tax_data_loader,
cs425_anch_tax_map_app,
cs680_aes,
cs680_aes_report,
cs685,
dot_ssh,
dotfiles,
drf_ember_pagination,
elixir-class,
fpjs,
gitolite-admin,
gpx-web-utils,
hibernators,
hibernators-web,
hymenobacterdotinfo,
jwt,
pingo,
planner,
qzv-view,
stem2017,
thermokar.st,
zettel
[github_thermokast_personal]
workspace=~/projects/personal2
remote_host=git@github.com/thermokarst
remote_name=thermokarst
github_peers=
repos=
thermokarst
[github_thermokarst_qiime2]
workspace=~/projects/qiime22
remote_host=git@github.com/thermokarst
remote_name=thermokarst
github_peers=
repos=
workflows-playground,
q2-no-op
[github_thermokarst_forks_qiime2]
workspace=~/projects/qiime22
remote_host=git@github.com/thermokarst-forks
remote_name=thermokarst
github_peers=ebolyen,gregcaporaso,ChrisKeefe,Oddant1,nbokulich,andrewsanchez
repos=
An-Introduction-To-Applied-Bioinformatics,
Keemei,
RESCRIPt,
action-library-packaging,
biom-format-feedstock,
busywork,
caporaso-lab.github.io,
code-of-conduct,
conda-channel-resource,
data302,
deblur,
dev-docs,
docs,
emperor,
environment-files,
genome-sampler,
gneiss,
ijson-feedstock,
library,
logos,
paper1,
paper2,
pretrained-feature-classifiers,
q2-alignment,
q2-composition,
q2-cutadapt,
q2-dada2,
q2-deblur,
q2-demux,
q2-diversity,
q2-diversity-lib,
q2-emperor,
q2-feature-classifier,
q2-feature-table,
q2-fragment-insertion,
q2-gneiss,
q2-longitudinal,
q2-metadata,
q2-perc-norm,
q2-phylogenomics,
q2-phylogeny,
q2-quality-control,
q2-quality-filter,
q2-sample-classifier,
q2-taxa,
q2-types,
q2-vsearch,
q2_itsxpress,
q2cli,
q2cwl,
q2galaxy,
q2lint,
q2studio,
q2templates,
q2view,
qiime2,
qiime2.github.io,
scikit-bio,
scikit-bio-feedstock,
spcss,
sphinx-qiime2-theme,
static-site-infrastructure,
template-repo,
view.qiime2.org,
vm-playbooks,
workshop-playbooks,
workshops