generateTaskQueue

#!/usr/bin/env python

import jinja2
import sys
import os
import os, shutil
import sysconfig

site_package_data=os.path.join(sysconfig.get_paths()["purelib"],"generateTaskQueues", "data")

# Per-stack Mongo replica-set hosts. Prod drops the stack segment per the go.apis convention.
STACK_MONGO_HOSTS = {
    "dev":   "mongo-rs0.dev.apis.huit.harvard.edu:27017,mongo-rs1.dev.apis.huit.harvard.edu:27017,mongo-rs2.dev.apis.huit.harvard.edu:27017",
    "stage": "mongo-rs0.stage.apis.huit.harvard.edu:27017,mongo-rs1.stage.apis.huit.harvard.edu:27017,mongo-rs2.stage.apis.huit.harvard.edu:27017",
    "prod":  "mongo-rs0.apis.huit.harvard.edu:27017,mongo-rs1.apis.huit.harvard.edu:27017,mongo-rs2.apis.huit.harvard.edu:27017",
}

def copy_and_metadata(from_path, to_path):
    if os.path.exists(to_path):
        raise Exception("Directory already exists")
    shutil.copytree(from_path, to_path)

def templateRender(tmpl, data):
    template = jinja2.Template(tmpl)
    return template.render(data)

def saveOutput(render, filename):
    with open(filename, 'w') as f1:
        f1.write(render)

def generateTemplate(name, directory):
    # set up the directory structure
    copy_and_metadata(site_package_data,os.path.join(directory, name) )
    # create task directory
    os.makedirs(os.path.join(directory, name, name, "tasks"))
    # templates
    saveOutput(templateRender(templ_initpy, {"init_item": "from .tasks import __init__"}), os.path.join(
        directory, name, name, "__init__.py"))
    saveOutput(templateRender(templ_initpy, {
               "init_item": "from .tasks import *"}), os.path.join(directory, name, name, "tasks", "__init__.py"))
    saveOutput(templateRender(tmpl_readme, {"queue_name": name}), os.path.join(
        directory, name, "README.md"))
    saveOutput(templateRender(tmpl_setup, {"queue_name": name}), os.path.join(
        directory, name, "setup.py"))
    saveOutput(templateRender(tmpl_gitignore, {}),
               os.path.join(directory, name, ".gitignore"))
    saveOutput(templateRender(tmpl_tasks, {}), os.path.join(
        directory, name, name, "tasks", "tasks.py"))
    # K8s Vars: shared config in common_k8s_vars.yml; each <stack>_k8s_vars.yml holds only its
    # deltas (MONGO_HOSTS) and is deep-merged over common by harvard-huit/hapi-action-k8svars.
    os.makedirs(os.path.join(directory, name, "k8s_vars"))
    saveOutput(templateRender(tmpl_common_k8s_vars, {"queue_name": name}), os.path.join(
        directory, name, "k8s_vars", "common_k8s_vars.yml"))
    for stack in ["dev", "stage", "prod"]:
        saveOutput(templateRender(tmpl_env_k8s_vars, {"stack": stack, "mongo_hosts": STACK_MONGO_HOSTS[stack]}),
                   os.path.join(directory, name, "k8s_vars", f"{stack}_k8s_vars.yml"))

templ_initpy = "{{ init_item }}"
tmpl_readme = """{{ queue_name }} Queue
======================

Please add Queue description.

Requirements
------------

Any pre-requisites that needs to covered. Environmental Variable required by Queue.

Dependencies
------------

A list of other dependencies need to run queue tasks.


License
-------


Author Information
------------------
"""
tmpl_setup = """from setuptools import setup, find_packages

setup(name='{{ queue_name }}',
      version='0.0',
      packages= find_packages(),
      install_requires=[],
)
"""
tmpl_tasks = """from celery import Celery
import celeryconfig

app = Celery()
app.config_from_object(celeryconfig)

#Example task
@app.task()
def add(x, y):
    \"\"\" Example task that adds two numbers or strings
        args: x and y
        return addition or concatination of strings
    \"\"\"
    result = x + y
    return result
"""
tmpl_gitignore = """*.py[cod]
.idea
# C extensions
*.so

# Packages
*.egg
*.egg-info
dist
build
eggs
parts
bin
var
sdist
develop-eggs
.installed.cfg
lib
lib64

# Installer logs
pip-log.txt

# Unit test / coverage reports
.coverage
.tox
nosetests.xml

# Translations
*.mo

# Mr Developer
.mr.developer.cfg
.project
.pydevproject
"""
# Shared config for ALL environments. Environment-specific values live in <stack>_k8s_vars.yml
# and are DEEP-MERGED over these by harvard-huit/hapi-action-k8svars (list values merge by
# key/name; scalars override). Keep only per-env deltas in the <stack> files.
tmpl_common_k8s_vars = """target_app_secrets_ref:
  - HAPI_CELERY_CONFIG: hapi-celery-config
    CLIENT_KEY_PEM: hapi-rabbitmq-key
    CLIENT_CERT_PEM: hapi-rabbitmq-cert
    TESTCA_CACERT_PEM: hapi-cybercom-ca-cert
    CLIENT_MONGO_PEM: hapi-mongodb-rs-cert
    CLIENT_MONGO_CA_PEM: hapi-mongodb-rs-ca-cert
    # Apigee X service-account keys — UNCOMMENT (with the apigee-auth volume mount below) if this
    # queue calls the Apigee management API. The task reads the file matching its environment.
    # APIGEE_X_NONPROD: apigee-service-account-apigee-x-nonprod-406719
    # APIGEE_X_PREPROD: apigee-service-account-apigee-x-preprod
    # APIGEE_X_PROD: apigee-service-account-apigee-x-prod-406719

target_app_env:
  - name: BROKER_USE_SSL
    value: "True"
  - name: LOG_LEVEL
    value: INFO
  - name: MONGO_DB
    value: cybercom
  - name: MONGO_LOG_COLLECTION
    value: task_log
  - name: MONGO_TOMBSTONE_COLLECTION
    value: tombstone
  - name: RABBITMQ_HOST
    value: rabbitmq.cybercom.svc.cluster.local
  - name: RABBITMQ_PORT
    value: "5671"
  - name: RABBITMQ_SSL_CA_FILE
    value: /ssl/testca/cacert.pem
  - name: RABBITMQ_SSL_CERT_FILE
    value: /ssl/client/cert.pem
  - name: RABBITMQ_SSL_KEY_FILE
    value: /ssl/client/key.pem
  - name: SSL_PATH
    value: /ssl
  - name: MONGO_USERNAME
    value: muser
  - name: RABBITMQ_DEFAULT_USER
    value: ruser
  - name: RABBITMQ_DEFAULT_VHOST
    value: rhost
  - name: CELERY_IMPORTS
    value: {{ queue_name }}
  - name: CELERY_QUEUE
    value: {{ queue_name }}

target_volume_mount:
  - name: ssl-keys
    mount_path: /ssl
    type: secret
    data_items:
      - key: CLIENT_CERT_PEM
        path: client/cert.pem
      - key: CLIENT_KEY_PEM
        path: client/key.pem
      - key: TESTCA_CACERT_PEM
        path: testca/cacert.pem
      - key: CLIENT_MONGO_PEM
        path: client/mongodb.pem
      - key: CLIENT_MONGO_CA_PEM
        path: client/mongodb-ca.pem
  # Apigee X SA keys — UNCOMMENT (with the secret refs above) if this queue calls the Apigee
  # management API. Mounted at /xkeys; the task reads the file for its environment.
  # - name: apigee-auth
  #   mount_path: /xkeys
  #   type: secret
  #   data_items:
  #     - key: APIGEE_X_NONPROD
  #       path: apigee-x-nonprod.json
  #     - key: APIGEE_X_PREPROD
  #       path: apigee-x-preprod.json
  #     - key: APIGEE_X_PROD
  #       path: apigee-x-prod.json

target_namespace: celery-worker
target_app_name: {{ queue_name }}
target_replica_count: 1
create_service: ''
"""
tmpl_env_k8s_vars = """# {{ stack }}-only overrides, deep-merged over common_k8s_vars.yml.
target_app_env:
  - name: MONGO_HOSTS
    value: {{ mongo_hosts }}
"""

if __name__ == '__main__':
    try:
        directory = sys.argv[2]
    except:
        directory = os.getcwd()  # set default as current working directory
    if len(sys.argv) > 1:
        name = sys.argv[1]
    else:
        raise Exception("Template name is required as an argument")
    generateTemplate(name, directory)