first commit

.dockerignore

+playbooks/*

.env

+#!/bin/bash
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
+
+export ANSIBLE_ROLES_PATH="$DIR/roles"

.gitignore

+*.retry
+.vaultpassword
+*.log
+inventories/*.gcharbon.yml

.ssh/ssh.cfg

+Host 192.168.20.*
+  ProxyCommand ssh -W %h:%p LFR028762
+  User ansible_user
+  IdentityFile ~/.ssh/id_rsa
+  GSSAPIAuthentication no
+
+Host 192.168.100.*
+  ProxyCommand ssh -W %h:%p LF028762
+  User ansible_user
+  IdentityFile ~/.ssh/id_rsa
+  GSSAPIAuthentication no
+
+Host 192.168.10.*
+  ProxyCommand ssh -W %h:%p LF028762
+  User ansible_user
+  IdentityFile ~/.ssh/id_rsa
+  GSSAPIAuthentication no
+
+Host LFR028762
+  Hostname 10.68.150.240
+  User gcharbon
+  ControlMaster auto
+  ControlPath ~/.ssh/ansible-%r@%h:%p
+  ControlPersist 5m
+  GSSAPIAuthentication no

LICENSE

+MIT License
+
+Copyright (c) 2018 Guillaume Charbonnier
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

Makefile

+.DEFAULT_GOAL := prepare_demo
+
+VERSION	:= 1.0
+ROOT_DIR := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
+CUR_DATE := $(shell date +"%s")
+
+PLAYBOOK_FOLDER := playbooks
+
+INVENTORY_FOLDER := inventories
+
+HYPERVISOR_INVENTORY := LFR028762.yml
+GCHARBON_INVENTORY := demo.gcharbon.yml
+
+VIRTUAL_INVENTORY := dynamic_inventories/demo_datalab.py
+
+HYPERVISOR_INVENTORY_PATH := "$(INVENTORY_FOLDER)/$(HYPERVISOR_INVENTORY)"
+VIRTUAL_INVENTORY_PATH := "$(INVENTORY_FOLDER)/$(VIRTUAL_INVENTORY)"
+GCHARBON_INVENTORY_PATH := "$(INVENTORY_FOLDER)/$(GCHARBON_INVENTORY)"
+
+VAULT_ID := --vault-id @prompt
+
+
+setup_demo:
+	ANSIBLE_ROLES_PATH=$(ROOT_DIR)/roles ansible-playbook -i $(HYPERVISOR_INVENTORY_PATH) \
+	  $(PLAYBOOK_FOLDER)/demo_init_virt_setup.yml
+
+demo:
+	ANSIBLE_ROLES_PATH=$(ROOT_DIR)/roles ansible-playbook -i $(GCHARBON_INVENTORY_PATH) \
+          $(PLAYBOOK_FOLDER)/demo_init_virt_setup.yml
+
+update_demo:
+	ansible-playbook -i $(HYPERVISOR_INVENTORY_PATH) \
+	  $(PLAYBOOK_FOLDER)/demo_update_virt_setup.yml
+
+provision_demo:
+	ansible-playbook -i $(VIRTUAL_INVENTORY_PATH) \
+	  $(PLAYBOOK_FOLDER)/demo_vm_provisioning.yml

README.md

+# Setup your Python environment with Ansible
+
+Table of Content:
+
+* [Getting started](#getting_started)
+  * [Available ansible roles](#ansible_roles)
+  * [Available ansible playbooks](#ansible_playbooks)
+    * [install_pip.yml](#install_pip)
+    * [prepare_rh-python36](#demo_nodes)
+    * [prepare_rh-python36_centos](#centos_nodes)
+* [Authentication](#authentication)
+  * [How to generate new value for `ansible_become_pass` variable](#ansible_become_pass)
+  * [Providing Ansible vault secrets](#vault_secrets)
+
+
+## Getting started
+<a name="getting_started"></a>
+
+This project uses Ansible to configure remote or local python environment.
+You must use `ansible>=2.6`.
+
+You can install it directly with `pip install -r requirements.txt` after cloning the repository.
+
+
+#### Available ansible roles
+<a name="ansible_roles"></a>
+
+This repository is composed of 3 [ansible roles](https://docs.ansible.com/ansible/2.6/user_guide/playbooks_reuse_roles.html):
+
+- **get-pip**: Can be used to install pip and virtualenv
+-
+- **pip**: Can be use to perform operations with pip
+-
+- **yum**: Can be used to perform action with yum
+
+
+#### Available ansible playbooks
+<a name="ansible_playbooks"></a>
+
+Three [major playbooks](https://docs.ansible.com/ansible/devel/user_guide/playbooks.html) are available for direct usage:
+
+> Note: You must be inside the playbooks directory to run example commands. 
+
+
+- _**install_pip.yml**_:
+<a name="install_pip"></a>
+
+This playbook install `pip` and `virtualenv` with `ansible_python_interpreter` by default. Any other interpreter can be used if `python_interpreter` variable is defined.
+
+###### Workflow:
+
+1) Check if pip is installed
+  If it is not instaled:
+  1.1) Download get-pip installer
+  1.2) Execute get-pip installer
+  1.3) Remove get-pip installer
+2) Install virtualenv if it is not installed yet
+
+###### Example usage:
+
+- `python_interpreter` is not defined in inventory neither in any variable.
+  Default interpreter (`ansible_python_interpreter`) is used:
+
+  ```
+  ansible-playbook -i inventories/test_centos.yml \
+    --vault-id @prompt \
+    install_pip.yml
+  ```
+
+- `python_interpreter` is set to `/opt/rh/rh-python36/root/bin/python`
+
+  ```
+  ansible-playbook -i inventories/test_centos.yml \
+    --vault-id @prompt \
+    --extra-vars "python_interpreter=/opt/rh/rh-python36/root/bin/python"
+    install_pip.yml
+  ```
+
+
+- _**prepare_rh-python36.yml**_
+<a name="demo_nodes"></a>
+
+
+This playbook assume that rhel repositories has already been enabled and `rh-pyhton36` package is available for download. It installs `rh-python36` and dependencies based on requirements file.
+
+> Note: The playbook installs `rh-python36` because `ansible_python_interpreter` is set to `rh_python_interpreter` and `rh_python_interpreter` is set to `rh-python36` in group demo_nodes in inventory. You can change it to anyother package.
+
+> Warning: If you want to install a package non managed by scl, set `scn_enable_python` to `false`.
+
+###### Workflow:
+
+1) Ensure `rh-pyhton-36` is installed
+2) Optionally add line in .bashrc to enable rh-python36 with scl at startup
+3) Copy python requirements to rmeote host
+4) Install python requirements with pip from `rh-python36`
+
+###### Example usages:
+
+- `rh-python-interpreter` is set to `rh-python36` by default
+
+  ```
+  ansible-playbook -i inventories/test_centos.yml \
+    --vault-id @prompt \
+    prepare_rh-python36.yml
+  ```
+
+- Install another version of Python:
+
+```
+ansible-playbook -i inventories/test_centos.yml \
+  --ask-vault-pass \
+  --extra-vars "rh_python_package=rh-python35 python_interpreter=/opt/rh/rh-python35/root/bin/python" \
+  prepare_rh-python36.yml
+```
+
+- Install with another requirements file:
+```
+ansible-playbook -i inventories/test_centos.yml \
+  --vault-id @prompt \
+  --extra-vars "pip_requirements_path=~/some_folder/requirements.txt"
+  prepare_rh-python36.yml
+```
+
+- _**prepare_rh-python36_centos.yml**_:
+<a name="centos_nodes"></a>
+
+This playbook enables `centos-sclo-rh-testing` repository and
+download `rh-python36` before installing python dependencies with this
+interpreter.
+
+###### Playbook workflow:
+
+1) Ensure centos-release-scl is installed
+2) Enable centos-sclo-rh-testing repository
+3) Install `rh-python36`
+2) Add line to `.bashrc` to enable rh-python36 by default at startup
+3) Copy python requirements to remote host
+4) Install python requirements
+
+It ban be used the same wah `prepare_rh-python36` is used.
+
+
+## Authentication
+<a name="authentification"></a>
+
+- Logging to remote host is realized with SSH Key-Based Authentication.
+
+> Use `ssh-copy-id <ansible_user@ansible_remote_host>`  to ensure your own ssh key is authorized by remote agent.
+
+- [Privilege escalation method](https://docs.ansible.com/ansible/latest/user_guide/become.html) used in playbook can be configured with `ansible_become_method` variable.
+  Default value is `su`. Password of root user is expected to be present as vault encrypted variable named `ansible_become_pass`. Is can be accessed as any other vault secrets once the vault password is given to playbooks.
+
+> List of available values: https://docs.ansible.com/ansible/latest/user_guide/become.html#command-line-options
+
+## How to generate new value for `ansible_become_pass` variable ?
+<a name="ansible_become_pass"></a>
+
+- Run the following command
+
+  ```
+  ansible-vault encrypt_string "PASSWORD"
+  ```
+
+- Before returning the encrypted string it will ask you for a pasword (you will provide this password at runtime to decrypt secret). Store it into a file like the following:
+
+You shoud get something like:
+
+```
+ansible_become_pass: !vault |
+          $ANSIBLE_VAULT;1.1;AES256
+          34306464383862303338666336306239306335393366656136313362643334383264326530333136
+          3831326639343639643063643664666331356236346239640a346531326465333330363761373831
+          61353139323635333461313732386538366361326163613865333462353161623039356433643032
+          3962303266363532330a616432653534333431363938386531373864616635393462356337336334
+          3834
+```
+
+> See [official documentation](https://docs.ansible.com/ansible/2.4/vault.html#use-encrypt-string-to-create-encrypted-variables-to-embed-in-yaml) for more information.
+
+
+## Providing vault secrets
+<a name="vault_secrets"></a>
+
+You can choose several options to [provide vault password](https://docs.ansible.com/ansible/2.4/vault.html#providing-vault-passwords) to playbooks at runtime:
+
+- Using `--vault-id @prompt`.
+
+  Example:
+
+  ```
+  ansible-playbook -i inventories/test_centos.yml \
+    --vault-id @prompt \
+    prepare_rh-python36.yml
+  ```
+
+- Using a file or an executable :
+
+  Examples:
+
+  - Assuming `get_vault_id.py` is an existing python script:
+    ```
+    ansible-playbook -i inventories/test_centos.yml \
+      --vault-id get-vault-password.py \
+      prepare_rh-python36.yml
+    ```
+
+  - Assuming `.vaultpassword` is an existing file:
+  ```
+  ansible-playbook -i inventories/test_centos.yml \
+    --vault-id .vaultpassword \
+    prepare_rh-python36.yml
+  ```

ansible.cfg

+[defaults]
+roles_path = $ANSIBLE_ROLES_PATH
+
+[ssh_connection]
+ssh_args = -F .ssh/ssh.cfg -o ControlMaster=auto -o ControlPersist=30m
+control_path = ~/.ssh/ansible-%%r@%%h:%%p

deploy/portainer/deploy-stack.yml

+version: '3.2'
+
+services:
+  agent:
+    image: portainer/agent
+    environment:
+      # REQUIRED: Should be equal to the service name prefixed by "tasks." when
+      # deployed inside an overlay network
+      AGENT_CLUSTER_ADDR: tasks.agent
+      # AGENT_PORT: 9001
+      # LOG_LEVEL: debug
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+      - /var/lib/docker/volumes:/var/lib/docker/volumes
+    networks:
+      - agent_network
+    deploy:
+      mode: global
+      placement:
+        constraints: [node.platform.os == linux]
+
+  portainer:
+    image: portainer/portainer
+    command: -H tcp://tasks.agent:9001 --tlsskipverify
+    ports:
+      - "9000:9000"
+    volumes:
+      - portainer_data:/data
+    networks:
+      - agent_network
+    deploy:
+      mode: replicated
+      replicas: 1
+      placement:
+        constraints: [node.role == manager]
+
+networks:
+  agent_network:
+    driver: overlay
+    attachable: true
+
+volumes:
+  portainer_data:

get-vault-password.py

+#!/usr/bin/env python
+
+from __future__ import print_function
+import getpass
+
+if __name__ == "__main__":
+    pswd = getpass.getpass('Vault Password:')
+    print(pswd)

inventories/LFR028762.yml

+---
+
+all:
+
+  vars:
+    ansible_user: root
+    ansible_python_interpreter: /usr/bin/python
+
+  children:
+
+    centos_hosts:
+      hosts:
+        LFR028762:
+          ansible_host: 10.68.150.240
+
+    datalab:
+      hosts:
+        LFR028762:
+
+    hypervisors:
+      hosts:
+        LFR028762:

inventories/demo_digital_ocean.yml

+---
+
+all:
+
+  vars:
+    ansible_user: root
+
+  children:
+
+    centos_hosts:
+      hosts:
+        demo-centos-01:
+          ansible_host: 159.65.207.209
+          local_ipv4: 10.133.55.212
+        demo-centos-02:
+          ansible_host: 188.166.122.69
+          local_ipv4: 10.133.49.170
+        demo-centos-03:
+          ansible_host: 174.138.11.9
+          local_ipv4: 10.133.53.14
+
+    debian_hosts:
+      hosts:
+        demo-ubuntu-01:
+          ansible_host: 159.65.196.145
+          local_ipv4: 10.133.67.137
+        demo-ubuntu-02:
+          ansible_host: 188.166.45.196
+          local_ipv4: 10.133.41.182
+        demo-ubuntu-03:
+          ansible_host: 174.138.13.10
+          local_ipv4: 10.133.48.63
+    datalab:
+      hosts:
+        demo-centos-01:
+        demo-centos-02:
+        demo-centos-03:
+        demo-ubuntu-01:
+        demo-ubuntu-02:
+        demo-ubuntu-03:
+    docker:
+      hosts:
+        demo-centos-01:
+        demo-centos-02:
+        demo-centos-03:
+        demo-ubuntu-01:
+        demo-ubuntu-02:
+        demo-ubuntu-03:
+    swarm_managers:
+      hosts:
+        demo-centos-01:
+        demo-ubuntu-01:
+
+    swarm_workers:
+      hosts:
+        demo-centos-02:
+        demo-centos-03:
+        demo-ubuntu-02:
+        demo-ubuntu-03:  # hypervisors:
+  #   hosts:
+  #     # demo_node-01:
+  #     # demo_node-02:

inventories/dynamic_inventories/demo_datalab.py

+#!/usr/bin/env python
+from __future__ import print_function
+
+import argparse
+import subprocess
+
+import os
+
+dir_path = os.path.dirname(os.path.realpath(__file__))
+
+LIBVIRT_URI="qemu+tcp://root@10.68.150.240:16509/system"
+
+VIRT_NETWORK="demo_datalab"
+
+COMMON_GROUPS="demo,virtual_machines,centos_hosts,docker"
+
+INVENTORY_SCRIPT_RELATIVE_PATH="scripts/libvirt_inventory.py"
+
+cmd =  "python {0} --libvirt_uri '{1}' --virt_network {2} --common_groups {3}"
+
+inventory_script = os.path.join(dir_path,
+                                INVENTORY_SCRIPT_RELATIVE_PATH)
+
+
+cmd = cmd.format(inventory_script,
+                 LIBVIRT_URI,
+                 VIRT_NETWORK,
+                 COMMON_GROUPS)
+
+if __name__ == "__main__"    :
+
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument("--list",
+                        "-l",
+                        action='store_true',
+                        required=False)
+
+    p = subprocess.Popen(cmd.split(" "), stdout=subprocess.PIPE)
+
+    output = p.stdout.read()
+
+    print(output.decode())
+    
+    exit(0)

inventories/dynamic_inventories/production_datalab.py

+#!/usr/bin/env python
+from __future__ import print_function
+
+import argparse
+import subprocess
+
+import os
+dir_path = os.path.dirname(os.path.realpath(__file__))
+
+inventory_script = dir_path + "/scripts/libvirt_inventory.py"
+
+cmd =  'python {0}'.format(inventory_script)
+
+if __name__ == "__main__"    :
+
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument("--list",
+                        "-l",
+                        action='store_true',
+                        required=False)
+
+    p = subprocess.Popen(cmd.split(" "), stdout=subprocess.PIPE)
+
+    output = p.stdout.read()
+
+    print(output)
+
+    exit(0)

inventories/dynamic_inventories/scripts/libvirt_inventory.py

+#!/usr/bin/env python
+
+from __future__ import print_function
+
+import argparse
+import libvirt
+import json
+import logging
+
+logger = logging.getLogger(__name__)
+
+LIBVIRT_DEFAULT_URI = 'qemu:///system'
+
+LIBVIRT_DEFAULT_NETWORK = 'testing_datalab'
+
+DEFAULT_SPLIT_HOSTNAME = '_'
+
+DEFAULT_SPLIT_GROUP_POS = 1
+
+DEFAULT_COMMON_GROUP = 'docker_hosts,centos_hosts,virtual_machines'
+
+class LibvirtInventory(object):
+
+    def __init__(self,
+                 target_network=LIBVIRT_DEFAULT_NETWORK,
+                 uri=LIBVIRT_DEFAULT_URI,
+                 group_split=DEFAULT_SPLIT_HOSTNAME,
+                 group_pos=DEFAULT_SPLIT_GROUP_POS,
+                 additional_group=DEFAULT_COMMON_GROUP):
+
+        self.uri = uri.replace("'","").replace('"','')
+        self.open_libvirt_connection(self.uri)
+        self.parser = {
+                                "split": group_split,
+                                "pos": group_pos,
+        }
+        self.common_groups = additional_group.split(",")
+        self.set_target_network(target_network)
+        self.inventory = {}
+        self.set_all_hosts_and_meta()
+        self.add_all_hosts_to_groups()
+        self.add_all_common_groups()
+        self.close_libvirt_connection()
+
+
+    def open_libvirt_connection(self, uri):
+        self.conn = libvirt.openReadOnly(uri)
+
+        if self.conn == None:
+            logger.error('Failed to open connection to {0}'.format(uri))
+            exit(1)
+        else:
+            logger.info("Connected to libvirt on remote host: {0}".format(uri))
+
+    def close_libvirt_connection(self):
+        self.conn.close()
+
+
+    def set_target_network(self, target_network):
+        self._networks = {network.name(): network for network in self.conn.listAllNetworks()}
+        try:
+            self._network = self._networks[target_network]
+        except KeyError:
+            available_networks = ", ".join(self._networks.keys())
+            raise ValueError("Target network ({0}) does not exist. Available networks: {1}".format(target_network, available_networks))
+
+
+    @property
+    def network(self):
+        return self._network
+
+    def set_all_hosts_and_meta(self):
+        self._hosts = { lease["hostname"]: { "ansible_host": lease["ipaddr"] } for lease in self.network.DHCPLeases() }
+
+        self.add_group("all")
+
+        self.inventory["all"]["hosts"] = self.hosts = list(self._hosts.keys())
+
+        self.inventory["_meta"]= {"hostvars": self._hosts}
+
+    def add_group(self, group):
+        """ Add a group to inventory """
+        if group not in self.inventory:
+            self.inventory[group] = dict()
+
+    def get_group(self, group):
+        """ Return a group as dictionary from inventory """
+        return self.inventory[group]
+
+
+    def add_child(self, child, group):
+        """ Add a child group (string) to group in inventory"""
+        if not "children" in self.get_group(group):
+            self.inventory[group]["children"] = [child]
+        else:
+            self.inventory[group]["children"] = list(set([child] + self.inventory[group]["children"]))
+
+
+    def add_childgroup(self, child_group, parent_group="all"):
+        """ Add a group and mark it as child of another group in inventory """
+        self.add_group(child_group)
+        self.add_child(child_group, parent_group)
+
+
+    def add_children(self, children, group):
+        """ Add list of children to group in inventory"""
+        if not "children" in self.get_group(group):
+            self.inventory[group]["children"] = list(children)
+        else:
+            self.inventory[group]["children"] = list(set(children + self.inventory[group]["children"]))
+
+
+    def add_groupvars(self, vars_dict, group):
+        """ Takes a dictionary as argument and add the keys and values as group variables"""
+        if "vars" not in self.get_group(group):
+            self.inventory[group]["vars"] = vars_dict
+        else:
+            self.inventory[group]["vars"].update(vars_dict)