-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
17 changed files
with
314 additions
and
158 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,42 @@ | ||
| import base64 | ||
| from urllib.parse import urlparse | ||
|
|
||
| import boto3 | ||
| import docker | ||
|
|
||
| from babysage.utils import find_project_root | ||
| from babysage.config import get_config | ||
|
|
||
|
|
||
| def deploy(dockerfile): | ||
| aws_account_id = boto3.client('sts').get_caller_identity()['Account'] | ||
| ecr = boto3.client(service_name='ecr') | ||
| resp = ecr.get_authorization_token( | ||
| registryIds=[aws_account_id]) | ||
| token = resp['authorizationData'][0]['authorizationToken'] | ||
| proxy_endpoint = resp['authorizationData'][0]['proxyEndpoint'] | ||
| tag = f"babysage-{get_config('experiment_name')}" | ||
| repository_tag = urlparse(proxy_endpoint).netloc + '/' + tag | ||
| user, password = base64.b64decode(token).decode().split(':') | ||
|
|
||
| client = docker.from_env() | ||
| client.login( | ||
| username=user, | ||
| password=password, | ||
| registry=proxy_endpoint) | ||
|
|
||
| project_root = find_project_root() | ||
|
|
||
| if project_root: | ||
| dockerfile = dockerfile or 'Dockerfile' | ||
| client.images.build( | ||
| path=project_root, | ||
| quiet=False, | ||
| rm=True, | ||
| tag=repository_tag, | ||
| dockerfile=dockerfile) | ||
| for l in client.images.push(repository_tag, stream=True, decode=True): | ||
| print(l) | ||
| else: | ||
| raise Exception(''' | ||
| Current directry is not a babysage project directry (could not find `babysage.yml` file''') |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,90 @@ | ||
| import os | ||
| import pkg_resources | ||
|
|
||
| import boto3 | ||
| from jinja2 import Template | ||
|
|
||
|
|
||
| def init(name, ecr_repo_name, s3_bucket_name, execution_role_arn, region): | ||
| aws_account_id = boto3.client('sts').get_caller_identity()['Account'] | ||
|
|
||
| print(f'Creating babysage experiment {name}.') | ||
|
|
||
| session = boto3.session.Session() | ||
| region_name = region or session.region_name | ||
| ecr = session.client('ecr') | ||
| ecr_repo_name = ecr_repo_name or f'babysage-{name}' | ||
| ecr_repo_arn = f'{aws_account_id}.dkr.ecr.{region_name}.amazonaws.com/{ecr_repo_name}' | ||
|
|
||
| try: | ||
| resp = ecr.describe_repositories(repositoryNames=[ecr_repo_name]) | ||
| ecr_repo_arn = resp['repositories'][0]['repositoryArn'] | ||
| except Exception as e: | ||
| print(e.__dict__) | ||
| if e.response['Error']['Code'] == 'RepositoryNotFoundException': | ||
| print(f'ECR repositroy {ecr_repo_arn} doesn\'t seem to be exist') | ||
| if input('Can I create one? [y/N]') in 'yY': | ||
| resp = ecr.create_repository(repositoryName=ecr_repo_name) | ||
|
|
||
| s3 = session.client('s3') | ||
| s3_bucket_name = s3_bucket_name or f'babysage-{aws_account_id}-{region_name}' | ||
|
|
||
| try: | ||
| s3.get_bucket_location(Bucket=s3_bucket_name) | ||
| except Exception as e: | ||
| if e.response['Error']['Code'] == 'NoSuchBucket': | ||
| # bucket doesn't exist | ||
| print(f'S3 bucket {s3_bucket_name} doesn\'t seem to be exist') | ||
| if input('Can I create one? [y/N]') in 'yY': | ||
| s3.create_bucket( | ||
| Bucket=s3_bucket_name, | ||
| CreateBucketConfiguration={'LocationConstraint': region_name}) | ||
|
|
||
| ''' | ||
| iam = session.client('iam') | ||
| try: | ||
| iam.get_bucket_location(Bucket=s3_bucket_name) | ||
| except Exception as e: | ||
| if e.response['Error']['Code'] == 'NoSuchBucket': | ||
| # bucket doesn't exist | ||
| click.echo(f'S3 bucket {s3_bucket_name} doesn\'t seem to be exist') | ||
| if input('Can I create one? [Y/n]') in 'yY ': | ||
| s3.create_bucket( | ||
| Bucket=s3_bucket_name, | ||
| CreateBucketConfiguration={'LocationConstraint': region_name}) | ||
| ''' | ||
|
|
||
| template_dir = 'templates/default' | ||
|
|
||
| ctx = { | ||
| 'babysage': | ||
| { | ||
| 'experiment_name': name, | ||
| 'ecr_repo_name': ecr_repo_name, | ||
| 's3_bucket_name': s3_bucket_name}} | ||
|
|
||
| for t in _expand_template(template_dir): | ||
| out_path = Template(os.path.join(t[len(template_dir) + 1:])).render(ctx) | ||
| if not os.path.isdir(os.path.dirname(out_path)): | ||
| os.makedirs(os.path.dirname(out_path)) | ||
|
|
||
| with open(pkg_resources.resource_filename('babysage', t), 'r') as f_in: | ||
| with open(out_path, 'w') as f_out: | ||
| f_out.write( | ||
| Template(f_in.read()).render(ctx)) | ||
| if os.path.basename(out_path) == 'train': | ||
| import stat | ||
| os.chmod(out_path, stat.S_IEXEC) | ||
|
|
||
|
|
||
| def _expand_template(root): | ||
| if pkg_resources.resource_isdir('babysage', root): | ||
| resources = [] | ||
| for r in pkg_resources.resource_listdir('babysage', root): | ||
| resources += _expand_template(os.path.join(root, r)) | ||
| return resources | ||
| else: | ||
| if pkg_resources.resource_exists('babysage', root): | ||
| return [root] | ||
| else: | ||
| raise Exception(f'Couldn\'t find the resource {root} on package `babysage`') |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,33 @@ | ||
| import os | ||
|
|
||
| import docker | ||
|
|
||
| from babysage.utils import find_project_root, get_input_dir, get_output_dir | ||
| from babysage.config import get_config | ||
|
|
||
|
|
||
| def run_local(dockerfile): | ||
| client = docker.from_env() | ||
|
|
||
| project_root = find_project_root() | ||
|
|
||
| if project_root: | ||
| dockerfile = dockerfile or 'Dockerfile' | ||
| tag = f"babysage-{get_config('experiment_name')}", | ||
| client.images.build( | ||
| path=project_root, | ||
| quiet=False, | ||
| rm=True, | ||
| tag=tag, | ||
| dockerfile=dockerfile) | ||
| client.containers.run( | ||
| tag, | ||
| 'train', | ||
| volumes={ | ||
| os.path.join(project_root, 'input'): { | ||
| 'bind': get_input_dir(), 'mode': 'rw'}, | ||
| os.path.join(project_root, 'output'): { | ||
| 'bind': get_output_dir(), 'mode': 'rw'}}) | ||
| else: | ||
| raise Exception(''' | ||
| Current directry is not a babysage project directry (could not find `babysage.yml` file''') |
Oops, something went wrong.