Skip to content

Commit

Permalink
Merge pull request #2 from ecmwf/develop
Browse files Browse the repository at this point in the history 
Develop
  • Loading branch information
@corentincarton
corentincarton authored Mar 1, 2023
2 parents 473b28d + 70015c3 commit efd20c7
Show file tree
Hide file tree
Showing 3 changed files with 97 additions and 31 deletions.
42 changes: 42 additions & 0 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,3 +5,45 @@ Track and Deploy workflows and suites through git
This project is **BETA** and will be **Experimental** for the foreseeable future.
Interfaces and functionality are likely to change, and the project itself may be scrapped.
**DO NOT** use this software in any project/software that is operational.

## Overview

![](workflow.png)

## Installation
To install trackploy using pip (requires python, ecflow and pip):

python -m pip install .

## Usage
To initialise the remote target git repository:

usage: trackploy-init [-h] --target TARGET [--backup BACKUP] [--host HOST] [--user USER] [--force]

Remote suite folder initialisation tool

optional arguments:
-h, --help show this help message and exit
--target TARGET Target directory
--backup BACKUP Backup git repository
--host HOST Target host
--user USER Deploy user
--force Force push to remote

To stage and deploy a suite:

usage: trackploy-deploy [-h] --stage STAGE --local LOCAL --target TARGET [--backup BACKUP] [--host HOST] [--user USER]
[--push] [--message MESSAGE]

Suite deployment tool

optional arguments:
-h, --help show this help message and exit
--stage STAGE Staged suite
--local LOCAL Path to local git repository (will be created if doesn't exist)
--target TARGET Path to target git repository on host
--backup BACKUP URL to backup git repository
--host HOST Target host
--user USER Deploy user
--push Push staged suite to target
--message MESSAGE Git message
86 changes: 55 additions & 31 deletions trackploy/deploy.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ def __init__(
self,
host=None,
user=None,
suite_dir=None,
staging_dir=None,
local_repo=None,
target_repo=None,
backup_repo=None,
Expand All @@ -22,7 +22,7 @@ def __init__(
Parameters:
host(str): The target host.
user(str): The deploying user.
suite_dir(str): The source suite directory.
staging_dir(str): The source suite directory.
local_repo(str): Path to the local repository.
target_repo(str): Path to the target repository on the target host.
backup_repo(str): URL of the backup repository.
Expand All @@ -34,7 +34,7 @@ def __init__(
self.user = deploy_user if user is None else user
self.host = deploy_host if host is None else host

self.suite_dir = suite_dir
self.staging_dir = staging_dir

self.local_dir = local_repo
self.target_dir = target_repo
Expand All @@ -49,20 +49,21 @@ def __init__(
f" -> Could not find git repo in {local_repo}, cloning from {self.target_repo}"
)
self.repo = git.Repo.clone_from(self.target_repo, local_repo, depth=1)
self.repo.remotes['origin'].rename('target')

# link with backup repo
self.backup_repo = backup_repo
if backup_repo and "backup" not in self.repo.remotes:
print(f" -> Creating backup remote {backup_repo}")
self.repo.create_remote("backup", url=backup_repo)
self.check_sync_remotes("origin", "backup")
self.check_sync_remotes("target", "backup")

def get_hash_remote(self, remote):
"""
Get the git hash of a remote repository on the master branch.
Parameters:
remote(str): Name of the remote repository (typically "origin").
remote(str): Name of the remote repository (typically "target").
Returns:
The git hash of the master branch.
Expand All @@ -75,7 +76,10 @@ def check_sync_local_remote(self, remote):
Raise exception if the git hashes don't match.
Parameters:
remote(str): Name of the remote repository (typically "origin").
remote(str): Name of the remote repository (typically "target").
Returns:
The matching git hash.
"""
remote_repo = self.repo.remotes[remote]
remote_repo.fetch()
Expand All @@ -87,15 +91,19 @@ def check_sync_local_remote(self, remote):
raise Exception(
f"Local ({self.local_dir}) and remote ({remote}) git repositories not in sync!"
)
return hash_local

def check_sync_remotes(self, remote1, remote2):
"""
Check that two remote repositories have the same git hash.
Raise exception if the git hashes don't match.
Parameters:
remote1(str): Name of the first remote repository (typically "origin").
remote1(str): Name of the first remote repository (typically "target").
remote2(str): Name of the second remote repository (typically "backup").
Returns:
The matching git hash.
"""
remote_repo1 = self.repo.remotes[remote1]
remote_repo2 = self.repo.remotes[remote2]
Expand All @@ -109,20 +117,21 @@ def check_sync_remotes(self, remote1, remote2):
raise Exception(
f"Remote git repositories ({remote1} and {remote2}) not in sync!"
)
return hash1

def commit(self, message=None):
"""
Commits the current stage of the local repository.
Throws exception if there is nothing to commit.
Default commit message will be:
"deployed by {user} from {host}:{suite_dir}"
"deployed by {user} from {host}:{staging_dir}"
Parameters:
message(str): optional git commit message to append to default message
"""
try:
commit_message = (
f"deployed by {self.user} from {self.host}:{self.suite_dir}\n"
f"deployed by {self.user} from {self.host}:{self.staging_dir}\n"
)
if message:
commit_message += message
Expand All @@ -141,7 +150,7 @@ def push(self, remote):
Pushes the local state to the remote repository
Parameters:
remote(str): Name of the remote repository (typically "origin").
remote(str): Name of the remote repository (typically "target").
"""
remote_repo = self.repo.remotes[remote]
try:
Expand All @@ -157,10 +166,11 @@ def pull_remote(self, remote):
Git pull the remote repository to the local repository
Parameters:
remote(str): Name of the remote repository (typically "origin").
remote(str): Name of the remote repository (typically "target").
"""
remote_repo = self.repo.remotes[remote]
remote_repo.pull()
self.check_sync_local_remote(remote)

def diff_staging(self):
"""
Expand All @@ -176,12 +186,12 @@ def get_diff_files(dcmp, root=""):
modified.append(path)
for name in dcmp.left_only:
path = os.path.join(root, name)
fullpath = os.path.join(self.suite_dir, path)
fullpath = os.path.join(self.staging_dir, path)
if os.path.isdir(fullpath):
for root_dir, dirs, files in os.walk(fullpath):
for file in files:
filepath = os.path.relpath(
os.path.join(root, root_dir, file), self.suite_dir
os.path.join(root, root_dir, file), self.staging_dir
)
added.append(filepath)
else:
Expand All @@ -201,7 +211,7 @@ def get_diff_files(dcmp, root=""):
for dir, sub_dcmp in dcmp.subdirs.items():
get_diff_files(sub_dcmp, root=os.path.join(root, dir))

diff = dircmp(self.suite_dir, self.target_dir)
diff = dircmp(self.staging_dir, self.target_dir)
print("Changes in staged suite:")
get_diff_files(diff)
changes = [
Expand All @@ -224,31 +234,39 @@ def deploy(self, message=None):
- git add all the suite files and commit
- git push to remotes
Default commit message will be:
"deployed by {user} from {host}:{suite_dir}"
"deployed by {user} from {host}:{staging_dir}"
Parameters:
message(str): optional git commit message to append to default message.
"""
print("Deploying suite to remote locations:")
print("Deploying suite to remote locations")
# check if repos are in sync
print(" -> Checking that git repos are in sync")
self.check_sync_local_remote("origin")
hash_init = self.check_sync_local_remote("target")
if self.backup_repo:
self.check_sync_local_remote("backup")
self.check_sync_remotes("origin", "backup")
self.check_sync_remotes("target", "backup")

# rsync staging folder to current repo
print(" -> Staging suite")
# TODO: check if rsync fails
cmd = f"rsync -avz --delete {self.suite_dir}/ {self.local_dir}/ --exclude .git"
cmd = f"rsync -avz --delete {self.staging_dir}/ {self.local_dir}/ --exclude .git"
run_cmd(cmd)
# POSSIBLE TODO: lock others for change

# git commit and push to remotes
print(" -> Git commit")
self.commit(message)
print(f" -> Git push to target {self.target_repo} on host {self.host}")
self.push("origin")

hash_check = self.get_hash_remote("target")
if hash_check != hash_init:
raise Exception(
"Remote repositories have changed during deployment!\n \
Please check the state of the remote repositories"
)

self.push("target")
if self.backup_repo:
print(f" -> Git push to backup repository {self.backup_repo}")
self.push("backup")
Expand Down Expand Up @@ -287,10 +305,16 @@ def run_cmd(cmd, capture_output=True, timeout=1000, **kwargs):
def main(args=None):
description = "Suite deployment tool"
parser = argparse.ArgumentParser(description=description)
parser.add_argument("--suite", required=True, help="Suite to deploy")
parser.add_argument("--staging", required=True, help="Staging suite directory")
parser.add_argument("--target", required=True, help="Target directory")
parser.add_argument("--backup", help="Backup git repository")
parser.add_argument("--stage", required=True, help="Staged suite")
parser.add_argument(
"--local",
required=True,
help="Path to local git repository (will be created if doesn't exist)",
)
parser.add_argument(
"--target", required=True, help="Path to target git repository on host"
)
parser.add_argument("--backup", help="URL to backup git repository")
parser.add_argument("--host", default=os.getenv("HOSTNAME"), help="Target host")
parser.add_argument("--user", default=os.getenv("USER"), help="Deploy user")
parser.add_argument(
Expand All @@ -303,22 +327,22 @@ def main(args=None):
print("Initialisation options:")
print(f" - host: {args.host}")
print(f" - user: {args.user}")
print(f" - suite: {args.staging}")
print(f" - staging: {args.staging}")
print(f" - target: {args.target}")
print(f" - backup: {args.backup}")
print(f" - staged suite: {args.stage}")
print(f" - local repo: {args.local}")
print(f" - target repo: {args.target}")
print(f" - backup repo: {args.backup}")
print(f" - git message: {args.message}")

deployer = GitDeployment(
host=args.host,
user=args.user,
suite_dir=args.suite,
local_repo=args.staging,
staging_dir=args.stage,
local_repo=args.local,
target_repo=args.target,
backup_repo=args.backup,
)

deployer.pull_remote("origin")
deployer.pull_remote("target")
deployer.diff_staging()

if args.push:
Expand Down
Binary file added workflow.png
View file
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.

0 comments on commit efd20c7

Please sign in to comment.