Skip to content

Commit

Permalink
Merge pull request #1 from ecmwf/develop
Browse files Browse the repository at this point in the history 
Develop
  • Loading branch information
@corentincarton
corentincarton authored Feb 28, 2023
2 parents 5ac59ad + 73d6110 commit 473b28d
Show file tree
Hide file tree
Showing 4 changed files with 226 additions and 79 deletions.
7 changes: 6 additions & 1 deletion README.md
View file
Original file line number Diff line number Diff line change
@@ -1,2 +1,7 @@
# trackploy
Track and Deploy workflows and suites
Track and Deploy workflows and suites through git

**DISCLAIMER**:
This project is **BETA** and will be **Experimental** for the foreseeable future.
Interfaces and functionality are likely to change, and the project itself may be scrapped.
**DO NOT** use this software in any project/software that is operational.
2 changes: 2 additions & 0 deletions trackploy/__init__.py
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
# flake8: noqa

from .deploy import GitDeployment
from .init import setup_remote
from .version import __version__
229 changes: 166 additions & 63 deletions trackploy/deploy.py
View file
Original file line number Diff line number Diff line change
@@ -1,21 +1,34 @@
import os
import git
import argparse
import os
import subprocess
from filecmp import dircmp

import git


class GitDeployment:
def __init__(self,
host=None,
user=None,
suite_dir=None,
local_repo=None,
target_repo=None,
backup_repo=None,
):

print('Creating deployer:')
def __init__(
self,
host=None,
user=None,
suite_dir=None,
local_repo=None,
target_repo=None,
backup_repo=None,
):
"""
Class used to deploy suites through git.
Parameters:
host(str): The target host.
user(str): The deploying user.
suite_dir(str): The source suite directory.
local_repo(str): Path to the local repository.
target_repo(str): Path to the target repository on the target host.
backup_repo(str): URL of the backup repository.
"""

print("Creating deployer:")
deploy_user = os.getenv("USER")
deploy_host = os.getenv("HOSTNAME")
self.user = deploy_user if user is None else user
Expand All @@ -32,30 +45,58 @@ def __init__(self,
print(f" -> Loading local repo {local_repo}")
self.repo = git.Repo(local_repo)
except (git.exc.NoSuchPathError, git.exc.InvalidGitRepositoryError):
print(f" -> Could not find git repo in {local_repo}, cloning from {self.target_repo}")
print(
f" -> Could not find git repo in {local_repo}, cloning from {self.target_repo}"
)
self.repo = git.Repo.clone_from(self.target_repo, local_repo, depth=1)

# link with backup repo
self.backup_repo = backup_repo
if backup_repo and 'backup' not in self.repo.remotes:
if backup_repo and "backup" not in self.repo.remotes:
print(f" -> Creating backup remote {backup_repo}")
self.repo.create_remote('backup', url=backup_repo)
self.check_sync_remotes('origin', 'backup')
self.repo.create_remote("backup", url=backup_repo)
self.check_sync_remotes("origin", "backup")

def get_hash_remote(self, remote):
return self.repo.git.ls_remote("--heads", remote, "master").split('\t')[0]
"""
Get the git hash of a remote repository on the master branch.
Parameters:
remote(str): Name of the remote repository (typically "origin").
Returns:
The git hash of the master branch.
"""
return self.repo.git.ls_remote("--heads", remote, "master").split("\t")[0]

def check_sync_local_remote(self, remote):
"""
Check that the local repository git hash is the same as the remote.
Raise exception if the git hashes don't match.
Parameters:
remote(str): Name of the remote repository (typically "origin").
"""
remote_repo = self.repo.remotes[remote]
remote_repo.fetch()
hash_target = self.get_hash_remote(remote)
hash_local = self.repo.git.rev_parse('master')
hash_local = self.repo.git.rev_parse("master")
if hash_target != hash_local:
print(f"Local hash {hash_local}")
print(f"Target hash {hash_target}")
raise Exception(f"Local ({self.local_dir}) and remote ({remote}) git repositories not in sync!")

raise Exception(
f"Local ({self.local_dir}) and remote ({remote}) git repositories not in sync!"
)

def check_sync_remotes(self, remote1, remote2):
"""
Check that two remote repositories have the same git hash.
Raise exception if the git hashes don't match.
Parameters:
remote1(str): Name of the first remote repository (typically "origin").
remote2(str): Name of the second remote repository (typically "backup").
"""
remote_repo1 = self.repo.remotes[remote1]
remote_repo2 = self.repo.remotes[remote2]
remote_repo1.fetch()
Expand All @@ -65,40 +106,70 @@ def check_sync_remotes(self, remote1, remote2):
if hash1 != hash2:
print(f"Remote {remote1} hash {hash1}")
print(f"Remote {remote2} hash {hash2}")
raise Exception(f"Remote git repositories ({remote1} and {remote2}) not in sync!")
raise Exception(
f"Remote git repositories ({remote1} and {remote2}) not in sync!"
)

def commit(self, message=None):
"""
Commits the current stage of the local repository.
Throws exception if there is nothing to commit.
Default commit message will be:
"deployed by {user} from {host}:{suite_dir}"
def commit(self, message):
Parameters:
message(str): optional git commit message to append to default message
"""
try:
commit_message = (
f"deployed by {self.user} from {self.host}:{self.suite_dir}\n"
)
if message:
commit_message += message
self.repo.git.add('--all')
self.repo.git.add("--all")
diff = self.repo.index.diff(self.repo.commit())
if diff:
self.repo.index.commit(commit_message)
else:
raise Exception('Nothing to commit')
raise Exception("Nothing to commit")
except Exception as e:
print('Commit failed!')
print("Commit failed!")
raise e

def push(self, remote):
"""
Pushes the local state to the remote repository
Parameters:
remote(str): Name of the remote repository (typically "origin").
"""
remote_repo = self.repo.remotes[remote]
try:
remote_repo.push().raise_if_error()
except git.exc.GitCommandError:
raise git.exc.GitCommandError(f"Could not push changes to remote repository {remote}. Check configuration and states of remote repository!")
raise git.exc.GitCommandError(
f"Could not push changes to remote repository {remote}.\n"
+ "Check configuration and states of remote repository!"
)

def pull_target(self, remote):
def pull_remote(self, remote):
"""
Git pull the remote repository to the local repository
Parameters:
remote(str): Name of the remote repository (typically "origin").
"""
remote_repo = self.repo.remotes[remote]
remote_repo.pull()

def diff_staging(self):
"""
Prints the difference between the staged suite and the current suite
"""
modified = []
removed = []
added = []

def get_diff_files(dcmp, root=""):
for name in dcmp.diff_files:
path = os.path.join(root, name)
Expand All @@ -109,7 +180,9 @@ def get_diff_files(dcmp, root=""):
if os.path.isdir(fullpath):
for root_dir, dirs, files in os.walk(fullpath):
for file in files:
filepath = os.path.relpath(os.path.join(root, root_dir, file), self.suite_dir)
filepath = os.path.relpath(
os.path.join(root, root_dir, file), self.suite_dir
)
added.append(filepath)
else:
added.append(path)
Expand All @@ -119,51 +192,66 @@ def get_diff_files(dcmp, root=""):
if os.path.isdir(fullpath):
for root_dir, dirs, files in os.walk(fullpath):
for file in files:
filepath = os.path.relpath(os.path.join(root, root_dir, file), self.target_dir)
filepath = os.path.relpath(
os.path.join(root, root_dir, file), self.target_dir
)
removed.append(filepath)
else:
removed.append(path)
for dir, sub_dcmp in dcmp.subdirs.items():
get_diff_files(sub_dcmp, root=os.path.join(root, dir))

diff = dircmp(self.suite_dir, self.target_dir)
print('Changes in staged suite:')
print("Changes in staged suite:")
get_diff_files(diff)
changes = [
('Removed', removed),
('Added', added),
('Modified', modified),
("Removed", removed),
("Added", added),
("Modified", modified),
]
for name, files in changes:
if files:
print(f" - {name}:")
for path in files:
print(f" - {path}")

def deploy(self, message):
def deploy(self, message=None):
"""
Deploy the staged suite to the target repository.
Steps:
- git fetch remote repositories and check they are in sync
- rsync the staged folder to the local repository
- git add all the suite files and commit
- git push to remotes
Default commit message will be:
"deployed by {user} from {host}:{suite_dir}"
Parameters:
message(str): optional git commit message to append to default message.
"""
print("Deploying suite to remote locations:")
# check if repos are in sync
print(' -> Checking that git repos are in sync')
self.check_sync_local_remote('origin')
print(" -> Checking that git repos are in sync")
self.check_sync_local_remote("origin")
if self.backup_repo:
self.check_sync_local_remote('backup')
self.check_sync_remotes('origin', 'backup')
self.check_sync_local_remote("backup")
self.check_sync_remotes("origin", "backup")

# rsync staging folder to current repo
print(' -> Staging suite')
# TO DO: check if rsync fails
print(" -> Staging suite")
# TODO: check if rsync fails
cmd = f"rsync -avz --delete {self.suite_dir}/ {self.local_dir}/ --exclude .git"
run_cmd(cmd)
# POSSIBLE TO DO: lock others for change
# POSSIBLE TODO: lock others for change

# git commit and push to remotes
print(' -> Git commit')
print(" -> Git commit")
self.commit(message)
print(f" -> Git push to target {self.target_repo} on host {self.host}")
self.push('origin')
self.push("origin")
if self.backup_repo:
print(f" -> Git push to backup repository {self.backup_repo}")
self.push('backup')
self.push("backup")

# TODO: add function to sync remotes
def sync_remotes(self, source, target):
Expand All @@ -172,30 +260,43 @@ def sync_remotes(self, source, target):

class FakeOuput:
returncode = 1
stderr = 'Timeout! It took more than 300 seconds'
stderr = "Timeout! It took more than 300 seconds"


def run_cmd(cmd, capture_output=True, timeout=1000, **kwargs):
"""
Runs a shell command.
Parameters:
cmd(str): command to run.
Returns:
Exit code.
"""
try:
value = subprocess.run(cmd, shell=True, capture_output=capture_output, timeout=timeout, **kwargs)
value = subprocess.run(
cmd, shell=True, capture_output=capture_output, timeout=timeout, **kwargs
)
except subprocess.TimeoutExpired:
value = FakeOuput()
if value.returncode > 0:
raise Exception(f'ERROR! Command failed!\n{value}')
raise Exception(f"ERROR! Command failed!\n{value}")
return value


def main(args=None):
description = "Suite deployment tool"
parser = argparse.ArgumentParser(description=description)
parser.add_argument('--suite', required=True, help='Suite to deploy')
parser.add_argument('--staging', required=True, help='Staging suite directory')
parser.add_argument('--target', required=True, help='Target directory')
parser.add_argument('--backup', help='Backup git repository')
parser.add_argument('--host', default=os.getenv("HOSTNAME"), help='Target host')
parser.add_argument('--user', default=os.getenv("USER"), help='Deploy user')
parser.add_argument('--push', action="store_true", help='Push staged suite to target')
parser.add_argument('--message', help='Git message')
parser.add_argument("--suite", required=True, help="Suite to deploy")
parser.add_argument("--staging", required=True, help="Staging suite directory")
parser.add_argument("--target", required=True, help="Target directory")
parser.add_argument("--backup", help="Backup git repository")
parser.add_argument("--host", default=os.getenv("HOSTNAME"), help="Target host")
parser.add_argument("--user", default=os.getenv("USER"), help="Deploy user")
parser.add_argument(
"--push", action="store_true", help="Push staged suite to target"
)
parser.add_argument("--message", help="Git message")

args = parser.parse_args()

Expand All @@ -214,15 +315,17 @@ def main(args=None):
suite_dir=args.suite,
local_repo=args.staging,
target_repo=args.target,
backup_repo=args.backup
)
deployer.pull_target('origin')
backup_repo=args.backup,
)

deployer.pull_remote("origin")
deployer.diff_staging()

if args.push:
check = input('You are about to push the staged suite to the target directory. Are you sure? (Y/n)')
if check != 'Y':
check = input(
"You are about to push the staged suite to the target directory. Are you sure? (Y/n)"
)
if check != "Y":
exit(1)
deployer.deploy(args.message)

Expand Down
Loading

0 comments on commit 473b28d

Please sign in to comment.