Commit ee21491a authored by 's avatar

first commit

parents
playbooks/*
#!/bin/bash
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
export ANSIBLE_ROLES_PATH="$DIR/roles"
*.retry
.vaultpassword
*.log
inventories/*.gcharbon.yml
Host 192.168.20.*
ProxyCommand ssh -W %h:%p LFR028762
User ansible_user
IdentityFile ~/.ssh/id_rsa
GSSAPIAuthentication no
Host 192.168.100.*
ProxyCommand ssh -W %h:%p LF028762
User ansible_user
IdentityFile ~/.ssh/id_rsa
GSSAPIAuthentication no
Host 192.168.10.*
ProxyCommand ssh -W %h:%p LF028762
User ansible_user
IdentityFile ~/.ssh/id_rsa
GSSAPIAuthentication no
Host LFR028762
Hostname 10.68.150.240
User gcharbon
ControlMaster auto
ControlPath ~/.ssh/ansible-%r@%h:%p
ControlPersist 5m
GSSAPIAuthentication no
MIT License
Copyright (c) 2018 Guillaume Charbonnier
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
.DEFAULT_GOAL := prepare_demo
VERSION := 1.0
ROOT_DIR := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
CUR_DATE := $(shell date +"%s")
PLAYBOOK_FOLDER := playbooks
INVENTORY_FOLDER := inventories
HYPERVISOR_INVENTORY := LFR028762.yml
GCHARBON_INVENTORY := demo.gcharbon.yml
VIRTUAL_INVENTORY := dynamic_inventories/demo_datalab.py
HYPERVISOR_INVENTORY_PATH := "$(INVENTORY_FOLDER)/$(HYPERVISOR_INVENTORY)"
VIRTUAL_INVENTORY_PATH := "$(INVENTORY_FOLDER)/$(VIRTUAL_INVENTORY)"
GCHARBON_INVENTORY_PATH := "$(INVENTORY_FOLDER)/$(GCHARBON_INVENTORY)"
VAULT_ID := --vault-id @prompt
setup_demo:
ANSIBLE_ROLES_PATH=$(ROOT_DIR)/roles ansible-playbook -i $(HYPERVISOR_INVENTORY_PATH) \
$(PLAYBOOK_FOLDER)/demo_init_virt_setup.yml
demo:
ANSIBLE_ROLES_PATH=$(ROOT_DIR)/roles ansible-playbook -i $(GCHARBON_INVENTORY_PATH) \
$(PLAYBOOK_FOLDER)/demo_init_virt_setup.yml
update_demo:
ansible-playbook -i $(HYPERVISOR_INVENTORY_PATH) \
$(PLAYBOOK_FOLDER)/demo_update_virt_setup.yml
provision_demo:
ansible-playbook -i $(VIRTUAL_INVENTORY_PATH) \
$(PLAYBOOK_FOLDER)/demo_vm_provisioning.yml
# Setup your Python environment with Ansible
Table of Content:
* [Getting started](#getting_started)
* [Available ansible roles](#ansible_roles)
* [Available ansible playbooks](#ansible_playbooks)
* [install_pip.yml](#install_pip)
* [prepare_rh-python36](#demo_nodes)
* [prepare_rh-python36_centos](#centos_nodes)
* [Authentication](#authentication)
* [How to generate new value for `ansible_become_pass` variable](#ansible_become_pass)
* [Providing Ansible vault secrets](#vault_secrets)
## Getting started
<a name="getting_started"></a>
This project uses Ansible to configure remote or local python environment.
You must use `ansible>=2.6`.
You can install it directly with `pip install -r requirements.txt` after cloning the repository.
#### Available ansible roles
<a name="ansible_roles"></a>
This repository is composed of 3 [ansible roles](https://docs.ansible.com/ansible/2.6/user_guide/playbooks_reuse_roles.html):
- **get-pip**: Can be used to install pip and virtualenv
-
- **pip**: Can be use to perform operations with pip
-
- **yum**: Can be used to perform action with yum
#### Available ansible playbooks
<a name="ansible_playbooks"></a>
Three [major playbooks](https://docs.ansible.com/ansible/devel/user_guide/playbooks.html) are available for direct usage:
> Note: You must be inside the playbooks directory to run example commands.
- _**install_pip.yml**_:
<a name="install_pip"></a>
This playbook install `pip` and `virtualenv` with `ansible_python_interpreter` by default. Any other interpreter can be used if `python_interpreter` variable is defined.
###### Workflow:
1) Check if pip is installed
If it is not instaled:
1.1) Download get-pip installer
1.2) Execute get-pip installer
1.3) Remove get-pip installer
2) Install virtualenv if it is not installed yet
###### Example usage:
- `python_interpreter` is not defined in inventory neither in any variable.
Default interpreter (`ansible_python_interpreter`) is used:
```
ansible-playbook -i inventories/test_centos.yml \
--vault-id @prompt \
install_pip.yml
```
- `python_interpreter` is set to `/opt/rh/rh-python36/root/bin/python`
```
ansible-playbook -i inventories/test_centos.yml \
--vault-id @prompt \
--extra-vars "python_interpreter=/opt/rh/rh-python36/root/bin/python"
install_pip.yml
```
- _**prepare_rh-python36.yml**_
<a name="demo_nodes"></a>
This playbook assume that rhel repositories has already been enabled and `rh-pyhton36` package is available for download. It installs `rh-python36` and dependencies based on requirements file.
> Note: The playbook installs `rh-python36` because `ansible_python_interpreter` is set to `rh_python_interpreter` and `rh_python_interpreter` is set to `rh-python36` in group demo_nodes in inventory. You can change it to anyother package.
> Warning: If you want to install a package non managed by scl, set `scn_enable_python` to `false`.
###### Workflow:
1) Ensure `rh-pyhton-36` is installed
2) Optionally add line in .bashrc to enable rh-python36 with scl at startup
3) Copy python requirements to rmeote host
4) Install python requirements with pip from `rh-python36`
###### Example usages:
- `rh-python-interpreter` is set to `rh-python36` by default
```
ansible-playbook -i inventories/test_centos.yml \
--vault-id @prompt \
prepare_rh-python36.yml
```
- Install another version of Python:
```
ansible-playbook -i inventories/test_centos.yml \
--ask-vault-pass \
--extra-vars "rh_python_package=rh-python35 python_interpreter=/opt/rh/rh-python35/root/bin/python" \
prepare_rh-python36.yml
```
- Install with another requirements file:
```
ansible-playbook -i inventories/test_centos.yml \
--vault-id @prompt \
--extra-vars "pip_requirements_path=~/some_folder/requirements.txt"
prepare_rh-python36.yml
```
- _**prepare_rh-python36_centos.yml**_:
<a name="centos_nodes"></a>
This playbook enables `centos-sclo-rh-testing` repository and
download `rh-python36` before installing python dependencies with this
interpreter.
###### Playbook workflow:
1) Ensure centos-release-scl is installed
2) Enable centos-sclo-rh-testing repository
3) Install `rh-python36`
2) Add line to `.bashrc` to enable rh-python36 by default at startup
3) Copy python requirements to remote host
4) Install python requirements
It ban be used the same wah `prepare_rh-python36` is used.
## Authentication
<a name="authentification"></a>
- Logging to remote host is realized with SSH Key-Based Authentication.
> Use `ssh-copy-id <ansible_user@ansible_remote_host>` to ensure your own ssh key is authorized by remote agent.
- [Privilege escalation method](https://docs.ansible.com/ansible/latest/user_guide/become.html) used in playbook can be configured with `ansible_become_method` variable.
Default value is `su`. Password of root user is expected to be present as vault encrypted variable named `ansible_become_pass`. Is can be accessed as any other vault secrets once the vault password is given to playbooks.
> List of available values: https://docs.ansible.com/ansible/latest/user_guide/become.html#command-line-options
## How to generate new value for `ansible_become_pass` variable ?
<a name="ansible_become_pass"></a>
- Run the following command
```
ansible-vault encrypt_string "PASSWORD"
```
- Before returning the encrypted string it will ask you for a pasword (you will provide this password at runtime to decrypt secret). Store it into a file like the following:
You shoud get something like:
```
ansible_become_pass: !vault |
$ANSIBLE_VAULT;1.1;AES256
34306464383862303338666336306239306335393366656136313362643334383264326530333136
3831326639343639643063643664666331356236346239640a346531326465333330363761373831
61353139323635333461313732386538366361326163613865333462353161623039356433643032
3962303266363532330a616432653534333431363938386531373864616635393462356337336334
3834
```
> See [official documentation](https://docs.ansible.com/ansible/2.4/vault.html#use-encrypt-string-to-create-encrypted-variables-to-embed-in-yaml) for more information.
## Providing vault secrets
<a name="vault_secrets"></a>
You can choose several options to [provide vault password](https://docs.ansible.com/ansible/2.4/vault.html#providing-vault-passwords) to playbooks at runtime:
- Using `--vault-id @prompt`.
Example:
```
ansible-playbook -i inventories/test_centos.yml \
--vault-id @prompt \
prepare_rh-python36.yml
```
- Using a file or an executable :
Examples:
- Assuming `get_vault_id.py` is an existing python script:
```
ansible-playbook -i inventories/test_centos.yml \
--vault-id get-vault-password.py \
prepare_rh-python36.yml
```
- Assuming `.vaultpassword` is an existing file:
```
ansible-playbook -i inventories/test_centos.yml \
--vault-id .vaultpassword \
prepare_rh-python36.yml
```
[defaults]
roles_path = $ANSIBLE_ROLES_PATH
[ssh_connection]
ssh_args = -F .ssh/ssh.cfg -o ControlMaster=auto -o ControlPersist=30m
control_path = ~/.ssh/ansible-%%r@%%h:%%p
version: '3.2'
services:
agent:
image: portainer/agent
environment:
# REQUIRED: Should be equal to the service name prefixed by "tasks." when
# deployed inside an overlay network
AGENT_CLUSTER_ADDR: tasks.agent
# AGENT_PORT: 9001
# LOG_LEVEL: debug
volumes:
- /var/run/docker.sock:/var/run/docker.sock
- /var/lib/docker/volumes:/var/lib/docker/volumes
networks:
- agent_network
deploy:
mode: global
placement:
constraints: [node.platform.os == linux]
portainer:
image: portainer/portainer
command: -H tcp://tasks.agent:9001 --tlsskipverify
ports:
- "9000:9000"
volumes:
- portainer_data:/data
networks:
- agent_network
deploy:
mode: replicated
replicas: 1
placement:
constraints: [node.role == manager]
networks:
agent_network:
driver: overlay
attachable: true
volumes:
portainer_data:
#!/usr/bin/env python
from __future__ import print_function
import getpass
if __name__ == "__main__":
pswd = getpass.getpass('Vault Password:')
print(pswd)
---
all:
vars:
ansible_user: root
ansible_python_interpreter: /usr/bin/python
children:
centos_hosts:
hosts:
LFR028762:
ansible_host: 10.68.150.240
datalab:
hosts:
LFR028762:
hypervisors:
hosts:
LFR028762:
---
all:
vars:
ansible_user: root
children:
centos_hosts:
hosts:
demo-centos-01:
ansible_host: 159.65.207.209
local_ipv4: 10.133.55.212
demo-centos-02:
ansible_host: 188.166.122.69
local_ipv4: 10.133.49.170
demo-centos-03:
ansible_host: 174.138.11.9
local_ipv4: 10.133.53.14
debian_hosts:
hosts:
demo-ubuntu-01:
ansible_host: 159.65.196.145
local_ipv4: 10.133.67.137
demo-ubuntu-02:
ansible_host: 188.166.45.196
local_ipv4: 10.133.41.182
demo-ubuntu-03:
ansible_host: 174.138.13.10
local_ipv4: 10.133.48.63
datalab:
hosts:
demo-centos-01:
demo-centos-02:
demo-centos-03:
demo-ubuntu-01:
demo-ubuntu-02:
demo-ubuntu-03:
docker:
hosts:
demo-centos-01:
demo-centos-02:
demo-centos-03:
demo-ubuntu-01:
demo-ubuntu-02:
demo-ubuntu-03:
swarm_managers:
hosts:
demo-centos-01:
demo-ubuntu-01:
swarm_workers:
hosts:
demo-centos-02:
demo-centos-03:
demo-ubuntu-02:
demo-ubuntu-03: # hypervisors:
# hosts:
# # demo_node-01:
# # demo_node-02:
#!/usr/bin/env python
from __future__ import print_function
import argparse
import subprocess
import os
dir_path = os.path.dirname(os.path.realpath(__file__))
LIBVIRT_URI="qemu+tcp://root@10.68.150.240:16509/system"
VIRT_NETWORK="demo_datalab"
COMMON_GROUPS="demo,virtual_machines,centos_hosts,docker"
INVENTORY_SCRIPT_RELATIVE_PATH="scripts/libvirt_inventory.py"
cmd = "python {0} --libvirt_uri '{1}' --virt_network {2} --common_groups {3}"
inventory_script = os.path.join(dir_path,
INVENTORY_SCRIPT_RELATIVE_PATH)
cmd = cmd.format(inventory_script,
LIBVIRT_URI,
VIRT_NETWORK,
COMMON_GROUPS)
if __name__ == "__main__" :
parser = argparse.ArgumentParser()
parser.add_argument("--list",
"-l",
action='store_true',
required=False)
p = subprocess.Popen(cmd.split(" "), stdout=subprocess.PIPE)
output = p.stdout.read()
print(output.decode())
exit(0)
#!/usr/bin/env python
from __future__ import print_function
import argparse
import subprocess
import os
dir_path = os.path.dirname(os.path.realpath(__file__))
inventory_script = dir_path + "/scripts/libvirt_inventory.py"
cmd = 'python {0}'.format(inventory_script)
if __name__ == "__main__" :
parser = argparse.ArgumentParser()
parser.add_argument("--list",
"-l",
action='store_true',
required=False)
p = subprocess.Popen(cmd.split(" "), stdout=subprocess.PIPE)
output = p.stdout.read()
print(output)
exit(0)
#!/usr/bin/env python
from __future__ import print_function
import argparse
import libvirt
import json
import logging
logger = logging.getLogger(__name__)
LIBVIRT_DEFAULT_URI = 'qemu:///system'
LIBVIRT_DEFAULT_NETWORK = 'testing_datalab'
DEFAULT_SPLIT_HOSTNAME = '_'
DEFAULT_SPLIT_GROUP_POS = 1
DEFAULT_COMMON_GROUP = 'docker_hosts,centos_hosts,virtual_machines'
class LibvirtInventory(object):
def __init__(self,
target_network=LIBVIRT_DEFAULT_NETWORK,
uri=LIBVIRT_DEFAULT_URI,
group_split=DEFAULT_SPLIT_HOSTNAME,
group_pos=DEFAULT_SPLIT_GROUP_POS,
additional_group=DEFAULT_COMMON_GROUP):
self.uri = uri.replace("'","").replace('"','')
self.open_libvirt_connection(self.uri)
self.parser = {
"split": group_split,
"pos": group_pos,
}
self.common_groups = additional_group.split(",")
self.set_target_network(target_network)
self.inventory = {}
self.set_all_hosts_and_meta()
self.add_all_hosts_to_groups()
self.add_all_common_groups()
self.close_libvirt_connection()
def open_libvirt_connection(self, uri):
self.conn = libvirt.openReadOnly(uri)
if self.conn == None:
logger.error('Failed to open connection to {0}'.format(uri))
exit(1)
else:
logger.info("Connected to libvirt on remote host: {0}".format(uri))
def close_libvirt_connection(self):
self.conn.close()
def set_target_network(self, target_network):
self._networks = {network.name(): network for network in self.conn.listAllNetworks()}
try:
self._network = self._networks[target_network]
except KeyError:
available_networks = ", ".join(self._networks.keys())
raise ValueError("Target network ({0}) does not exist. Available networks: {1}".format(target_network, available_networks))
@property
def network(self):
return self._network
def set_all_hosts_and_meta(self):
self._hosts = { lease["hostname"]: { "ansible_host": lease["ipaddr"] } for lease in self.network.DHCPLeases() }
self.add_group("all")
self.inventory["all"]["hosts"] = self.hosts = list(self._hosts.keys())
self.inventory["_meta"]= {"hostvars": self._hosts}
def add_group(self, group):
""" Add a group to inventory """
if group not in self.inventory:
self.inventory[group] = dict()
def get_group(self, group):
""" Return a group as dictionary from inventory """
return self.inventory[group]
def add_child(self, child, group):
""" Add a child group (string) to group in inventory"""
if not "children" in self.get_group(group):
self.inventory[group]["children"] = [child]
else:
self.inventory[group]["children"] = list(set([child] + self.inventory[group]["children"]))
def add_childgroup(self, child_group, parent_group="all"):
""" Add a group and mark it as child of another group in inventory """
self.add_group(child_group)
self.add_child(child_group, parent_group)
def add_children(self, children, group):
""" Add list of children to group in inventory"""
if not "children" in self.get_group(group):
self.inventory[group]["children"] = list(children)
else:
self.inventory[group]["children"] = list(set(children + self.inventory[group]["children"]))
def add_groupvars(self, vars_dict, group):
""" Takes a dictionary as argument and add the keys and values as group variables"""
if "vars" not in self.get_group(group):
self.inventory[group]["vars"] = vars_dict
else:
self.inventory[group]["vars"].update(vars_dict)
def add_host(self, hostname, group, create=True):
""" Add one host (string) to a group.
If create is True (default) then the group will be automatically created if it does not exist yet.
Else it will fail on KeyError
"""
if create:
self.add_group(group)