Compare commits

..

24 Commits

Author SHA1 Message Date
4f3de27656 Removed README.md 2022-03-15 08:41:04 +01:00
7b00e6d893 Prevent accidental .ssh overwrite fuck up 2022-03-15 08:35:34 +01:00
63b959c8cd Better indication no postgres instance detected 2022-03-15 08:34:58 +01:00
04a7ff551c Fixed broken ansible package install_deps.sh 2022-03-15 08:30:07 +01:00
0b574e24bb Added annotations import to service.py 2022-03-14 22:53:20 +01:00
52b3dad70c Fixed install_deps.sh for ubuntu 20.04 2022-03-14 22:53:08 +01:00
b9d844d405 Updated install_deps.sh 2022-03-14 22:29:46 +01:00
e7f26fbf6f Replaced self_service.sh with service.py 2022-03-14 21:58:25 +01:00
cdca807aa0 Added php7.4-pgsql dep to nginx-webserver 2022-03-14 15:20:10 +01:00
ba93280fd6 Updated index.php 2022-03-14 15:19:44 +01:00
790e83b8fb Added copying of sample-data to db vm 2022-03-14 14:39:23 +01:00
b465413d42 Added update step to update pg_hba.conf postgresql/tasks 2022-03-14 11:47:55 +01:00
58f2a973d9 Moved sample.csv 2022-03-14 11:47:20 +01:00
c23898f2d0 WIP self-service.sh 2022-03-14 11:47:07 +01:00
400806fd29 WIP postgresql role 2022-03-14 11:33:22 +01:00
6e55c7ef47 Added postgres section to Vagrantfile.template 2022-03-14 11:32:58 +01:00
b71d98dcea install_deps.sh installs ansible requirements 2022-03-14 11:32:32 +01:00
e8d86d41ea Added postgres to site.yml 2022-03-10 21:27:31 +01:00
909ef2b5c8 Added README.md 2022-03-10 21:27:13 +01:00
4fb0c5c43b Added postgresql role 2022-03-10 21:26:46 +01:00
ec0a1adc01 Added php to nginx-webserver 2022-03-10 09:47:12 +01:00
5108efea23 Converted nginx-webserver index.html to php 2022-03-10 08:55:57 +01:00
b80bd3e913 Added shebang to install_deps.sh 2022-03-10 08:55:36 +01:00
fa939a9e90 Removed scripts (included in inv-alias branch) 2022-03-10 08:50:11 +01:00
20 changed files with 411 additions and 289 deletions

View File

@ -1 +0,0 @@
export PATH="$PATH:$PWD/scripts/bin"

View File

@ -1,2 +0,0 @@
cd ./scripts
go build -o ./bin/inv-alias ./inv-alias.go

View File

@ -1 +1,4 @@
sudo apt-get -y install virtualbox vagrant ansible
#!/usr/bin/env bash
sudo apt update --yes && sudo apt upgrade --yes
sudo apt install --yes virtualbox vagrant ansible
ansible-galaxy install -r ./requirements.yml

4
requirements.yml Normal file
View File

@ -0,0 +1,4 @@
---
# ansible-galaxy requirements for this repository
collections:
- 'community.postgresql'

View File

@ -1,4 +0,0 @@
#!/usr/bin/env bash
USER_DIR=./customers/$1
(cd $USER_DIR && vagrant destroy -f)
rm -rf $USER_DIR

View File

@ -1,6 +1,6 @@
---
# handlers file for nginx-webserver
- name: restart nginx
- name: reload nginx
ansible.builtin.service:
name: nginx
state: restarted

View File

@ -1,14 +1,36 @@
---
# tasks file for nginx-webserver
- name: Install nginx
- name: Install nginx and php
package:
name: nginx
name:
- nginx
- php7.4
- php7.4-fpm
- php7.4-cli
- php7.4-pgsql
state: present
update_cache: yes
become: true
notify: restart nginx
- name: Copy over index.html
- name: Copy over nginx.conf
ansible.builtin.template:
src: ./templates/index.html.j2
dest: /var/www/html/index.html
src: ./templates/nginx.cfg.j2
dest: /etc/nginx/sites-available/nginx.cfg
become: true
notify: reload nginx
- name: Enable nginx.conf
file:
src: /etc/nginx/sites-available/nginx.cfg
dest: /etc/nginx/sites-enabled/default
state: link
become: true
notify: reload nginx
- name: Remove nginx default crap
file:
state: absent
path: /var/www/html/*
become: true
- name: Copy over index.php
ansible.builtin.template:
src: ./templates/index.php.j2
dest: /var/www/html/index.php
become: true

View File

@ -1,17 +0,0 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8" />
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>Hello</title>
</head>
<body>
<p>Hostname: {{ ansible_facts.nodename }}</p>
<p>OS: {{ ansible_facts.distribution }} {{ ansible_facts.distribution_version }}</p>
<p>Kernel: {{ ansible_facts.kernel }}</p>
<p>Memory usage: {{ ansible_facts.memfree_mb }}/{{ ansible_facts.memtotal_mb }}MB</p>
<p>Python version: {{ ansible_facts.python_version }}</p>
</body>
</html>

View File

@ -0,0 +1,35 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8" />
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>Hello</title>
</head>
<body>
<h1><?php echo 'Hello, World!'; ?></h1>
<p>Hostname: {{ ansible_facts.nodename }}</p>
<p>OS: {{ ansible_facts.distribution }} {{ ansible_facts.distribution_version }}</p>
<p>Kernel: {{ ansible_facts.kernel }}</p>
<p>Memory usage: {{ ansible_facts.memfree_mb }}/{{ ansible_facts.memtotal_mb }}MB</p>
<p>Python version: {{ ansible_facts.python_version }}</p>
{% if groups['postgresql'] is defined and groups['postgresql']|length > 0 %}
<p>
<?php
$db_handle = pg_connect("host={{ groups['postgresql'][0] }} dbname=test user=postgres password={{ hostvars[groups['postgresql'][0]]['psql_pass'] }}");
if ($db_handle) {
echo "Connection attempt succeeded.\n";
} else {
echo "Connection attempt failed.\n";
}
$query = "SELECT m.* FROM test.public.message m";
$result = pg_exec($db_handle, $query);
var_dump(pg_fetch_all($result));
?>
</p>
{% else %}
<p>No postgres instance detected.</p>
{% endif %}
</body>
</html>

View File

@ -0,0 +1,15 @@
server {
listen 80;
root /var/www/html;
index index.php;
location / {
try_files $uri $uri/ =404;
}
location ~ \.php$ {
include snippets/fastcgi-php.conf;
fastcgi_pass unix:/run/php/php7.4-fpm.sock;
}
}

View File

@ -0,0 +1,2 @@
hello
world
1 hello
2 world

View File

@ -0,0 +1,2 @@
---
# handlers file for postgresql

View File

@ -0,0 +1,10 @@
galaxy_info:
author: strNophix
description: Postgresql Role
license: MIT
min_ansible_version: 2.1
galaxy_tags: []
dependencies: []

View File

@ -0,0 +1,72 @@
---
# tasks file for postgresql
- name: Install package dependencies
package:
name:
- postgresql
- postgresql-contrib
- python3-pip
- acl
state: present
update_cache: yes
become: true
- name: Install `psycopg2` driver for postgresql
pip:
name: psycopg2-binary
state: present
become: true
- name: Update `listen_address` in `/etc/postgresql/12/main/postgresql.conf`
lineinfile:
path: /etc/postgresql/12/main/postgresql.conf
regexp: ^#listen_addresses = 'localhost'
line: listen_addresses='*'
become: true
- name: Update `pg_hba.conf`
community.postgresql.postgresql_pg_hba:
dest: /etc/postgresql/12/main/pg_hba.conf
contype: host
users: postgres
source: 192.168.56.0/24
method: md5
create: true
become: true
- name: Create new test-database
become_user: postgres
become: yes
community.postgresql.postgresql_db:
name: test
- name: Create table `test`.`message`
become_user: postgres
become: yes
community.postgresql.postgresql_table:
db: test
name: message
columns:
- id bigserial primary key
- content text
- name: Copy over dummy-data for `test`.`message`
copy:
src: "{{ role_path }}/files/sample.csv"
dest: /tmp/sample.csv
- name: Insert sample data into `test`.`message`
become_user: postgres
become: yes
community.postgresql.postgresql_copy:
copy_from: /tmp/sample.csv
db: test
dst: message
columns: content
options:
format: csv
- name: Update password of postgres user
become_user: postgres
become: yes
community.postgresql.postgresql_user:
name: postgres
password: "{{ psql_pass }}"
- name: Restart postgresql service
service:
name: postgresql
state: restarted
enabled: yes
become: true

View File

@ -1,54 +0,0 @@
#!/usr/bin/env bash
VERSION="0.1.0"
function help() {
echo -e \
"Usage: $(basename $0) [OPTIONS] [COMMAND]\n\n" \
"Options:\n" \
" -i, --inv-file <path> Specify the Ansible inventory to add.\n" \
" -h, --help Show help.\n" \
" -v, --version Show version."
}
if [[ $# -eq 0 ]]; then
help
exit 1
fi
INVENTORY_FILE="$(pwd)/inventory"
while [[ $# -gt 0 ]]; do
case $1 in
-i|--inv-file)
INVENTORY_FILE="$2"
shift
shift
;;
-h|--help)
help
exit 1
;;
-v|--version)
echo $VERSION
exit 1
;;
-*|--*)
echo "hosto: unrecognized option '$1'"
help
exit 1
;;
*)
break
;;
esac
done
if [ -f $INVENTORY_FILE ]; then
sudo inv-alias add $INVENTORY_FILE
eval $@
sudo inv-alias rm $INVENTORY_FILE
else
echo "hosto: Could not find inventory file at $INVENTORY_FILE"
eval $@
fi

View File

@ -1,150 +0,0 @@
package main
import (
"bufio"
"fmt"
"io"
"io/fs"
"io/ioutil"
"log"
"os"
"path/filepath"
"regexp"
"strings"
)
type AliasMap map[string]string
const (
HostsFile string = "/etc/hosts"
)
func FixedSplit(s, sep string, parts int) []string {
n := make([]string, parts)
p := strings.SplitN(s, sep, parts)
copy(n, p)
return n
}
func IsLegalLine(line string) bool {
c := line[0]
return c != '[' && c != '#'
}
func BuildRegionString(regionName string, aliases AliasMap) string {
b := strings.Builder{}
b.WriteString("#region ")
b.WriteString(regionName)
b.WriteString("\n")
for ip, alias := range aliases {
b.WriteString(ip)
b.WriteString("\t")
b.WriteString(alias)
b.WriteString("\n")
}
b.WriteString("#endregion")
return b.String()
}
func BuildRegionRegexp(regionName string) *regexp.Regexp {
b := strings.Builder{}
b.WriteString("(?s)\n#region ")
b.WriteString(regexp.QuoteMeta(regionName))
b.WriteString(".*#endregion")
r := regexp.MustCompile(b.String())
return r
}
func ScanAliases(fileReader io.Reader) (AliasMap, error) {
aliasMap := AliasMap{}
scanner := bufio.NewScanner(fileReader)
for scanner.Scan() {
line := scanner.Text()
if IsLegalLine(line) {
s := FixedSplit(line, "#", 2)
ip, alias := strings.TrimSpace(s[0]), strings.TrimSpace(s[1])
if _, ok := aliasMap[ip]; !ok && alias != "" {
aliasMap[ip] = alias
}
}
}
if err := scanner.Err(); err != nil {
return nil, err
}
return aliasMap, nil
}
func AddAliases(fileName string) {
file, err := os.Open(fileName)
if err != nil {
log.Fatal(err)
}
h, err := ScanAliases(file)
if err != nil {
log.Fatal(err)
}
file.Close()
r := BuildRegionRegexp(fileName)
s := BuildRegionString(fileName, h)
content, err := ioutil.ReadFile(HostsFile)
if err != nil {
log.Fatal(err)
}
c := r.ReplaceAllString(string(content), s)
if !r.MatchString(c) {
c += ("\n" + s)
}
err = os.WriteFile(HostsFile, []byte(c[:]), fs.FileMode(os.O_WRONLY|os.O_TRUNC))
if err != nil {
log.Fatal(err)
}
}
func RemoveAliases(fileName string) {
regionReg := BuildRegionRegexp(fileName)
content, err := ioutil.ReadFile(HostsFile)
if err != nil {
log.Fatal(err)
}
c := regionReg.ReplaceAll(content, []byte(""))
err = os.WriteFile(HostsFile, c, fs.FileMode(os.O_WRONLY|os.O_TRUNC))
if err != nil {
log.Fatal(err)
}
}
func main() {
u := fmt.Sprintf("Please use: %s <add|rm> <file:path>\n", os.Args[0])
if len(os.Args) < 3 {
fmt.Println(u)
os.Exit(1)
}
p, err := filepath.Abs(os.Args[2])
if err != nil {
fmt.Println(err)
os.Exit(1)
}
switch os.Args[1] {
case "add":
AddAliases(p)
case "rm":
RemoveAliases(p)
default:
fmt.Println(u)
os.Exit(1)
}
}

View File

@ -1,47 +0,0 @@
#!/usr/bin/env bash
copy_template() {
cp -f ../../templates/$1 ./$2
}
write_inventory_group() {
local fmt="[$1]\n"
for ((i=$2; i<$2+$3; i++)) do
fmt+="192.168.56.$i\n"
done
echo -e $fmt >> ./inventory
}
# Take customer inputs
read -p "Klantnaam: " customerName
read -p "IpInt: " ipAddr
read -p "Number of webservers: " numWebserver
read -p "Number of loadbalancers: " numLoadbalancers
# Create customer directory and cd
mkdir -p ./customers/$customerName && cd $_
# Copy and fill-in necessary templates
copy_template ./Vagrantfile.template ./Vagrantfile
sed -i "s/#{customerName}/$customerName/" ./Vagrantfile
sed -i "s/#{ipAddr}/$ipAddr/" ./Vagrantfile
sed -i "s/#{numWebserver}/$numWebserver/" ./Vagrantfile
sed -i "s/#{numLoadbalancers}/$numLoadbalancers/" ./Vagrantfile
copy_template ./ansible.cfg.template ./ansible.cfg
# Generate ansible inventory file.
ipOffset=$ipAddr
write_inventory_group "webserver" $ipOffset $numWebserver
((ipOffset+=numWebserver))
write_inventory_group "loadbalancer" $ipOffset $numLoadbalancers
((ipOffset+=numLoadbalancers))
# Generate a new seperate ssh key for the customer
mkdir -p ./.ssh/
ssh-keygen -t rsa -b 2048 -f ./.ssh/id_rsa
# Provision and configure machines
vagrant up
ansible-playbook ../../site.yml

208
service.py Executable file
View File

@ -0,0 +1,208 @@
#!/usr/bin/env python3
from __future__ import annotations
import argparse
import sys
from os import path
from pathlib import Path
import os
import time
from typing import Any, Callable, DefaultDict, Iterable, Mapping
from jinja2 import Template
import itertools
import subprocess as sub
import shutil
import string
import secrets
TEMPLATE_DIR = "./templates"
def gen_pass() -> str:
alphabet = string.ascii_letters + string.digits
password = "".join(secrets.choice(alphabet) for _ in range(20))
return password
def check_positive(value: str):
ivalue = int(value)
if ivalue <= 0:
raise Exception("Supplied number must be >= 0")
return ivalue
def encode_member(member: str, mapping: Mapping[str, Any]) -> str:
return member + " " + " ".join([f"{k}={v}" for k, v in mapping.items()])
def iter_ips(ip_format: str, start_octet: int):
ip_int = start_octet
ip_fmt = ip_format
while ip_int < 255:
yield ip_fmt.format(ip_int)
ip_int += 1
class InventoryWriter:
def __init__(self, location: str) -> None:
self._file_handle = Path(location)
self._groups: dict[str, set[str]] = DefaultDict(set)
def add(self, name: str, members: Iterable[str]):
self._groups[name] |= set(members)
def _build_group(self, name: str, members: set[str]):
fmt = f"[{name}]\n" + "\n".join(members)
return fmt
def flush(self):
txt = ""
for name, members in self._groups.items():
txt += self._build_group(name, members) + "\n\n"
self._file_handle.write_text(txt, encoding="utf8")
def copy_template(src: str, dest: str, mapping: Mapping[str, Any] = {}):
c = Path(src).read_text()
t: Template = Template(c)
r = t.render(mapping)
Path(dest).write_text(r)
def list_envs(args: argparse.Namespace):
try:
customer_path = path.join("customers", args.customer_name, "envs")
print(" ".join(os.listdir(customer_path)))
except FileNotFoundError:
raise Exception(f"Customer `{args.customer_name}` does not exist.")
def delete_env(args: argparse.Namespace):
for env in args.env_names:
env_path = path.join("customers", args.customer_name, "envs", env)
sub.call(["vagrant", "destroy", "-f"], cwd=env_path)
shutil.rmtree(env_path)
print(f"Deleted `{env}` from customer `{ args.customer_name}`")
def create_env(args: argparse.Namespace):
if (args.num_nginx_web + args.num_nginx_lb + args.num_postgres) == 0:
raise Exception("At least one item should be deployed")
env_path = path.join("customers", args.customer_name, "envs", args.env_name)
Path(env_path).mkdir(exist_ok=True, parents=True)
# Template `ansible.cfg`
src = path.join(TEMPLATE_DIR, "ansible.cfg.template")
dest = path.join(env_path, "ansible.cfg")
copy_template(src=src, dest=dest)
# Create inventory file
inv_path = path.join(env_path, "inventory")
iw = InventoryWriter(inv_path)
ip_generator = iter_ips(args.ip_format, args.ip_int)
web_ips = itertools.islice(ip_generator, args.num_nginx_web)
iw.add("webserver", web_ips)
lb_ips = itertools.islice(ip_generator, args.num_nginx_lb)
iw.add("loadbalancer", lb_ips)
psql_gen_pass: Callable[[str], str] = lambda x: encode_member(
x, {"psql_pass": gen_pass()}
)
psql_ips = list(itertools.islice(ip_generator, args.num_postgres))
psql_ips = map(psql_gen_pass, psql_ips)
iw.add("postgresql", psql_ips)
iw.flush()
# Template `Vagrantfile`
src = path.join(TEMPLATE_DIR, "Vagrantfile.template")
dest = path.join(env_path, "Vagrantfile")
mapping = {
"env": args.env_name,
"customer_name": args.customer_name,
"ip_int": args.ip_int,
"ip_format": args.ip_format.replace("{}", "%d"),
"num_webserver": args.num_nginx_web,
"num_loadbalancers": args.num_nginx_lb,
"num_postgres": args.num_postgres,
}
copy_template(src=src, dest=dest, mapping=mapping)
# Generate .ssh
ssh_dir = path.join(env_path, ".ssh")
Path(ssh_dir).mkdir(exist_ok=True)
rsa_path = path.join(ssh_dir, "id_rsa")
if not Path(rsa_path).exists():
ssh_key_cmd = [
"ssh-keygen",
"-t",
"rsa",
"-b",
"2048",
"-f",
rsa_path,
]
sub.call(ssh_key_cmd)
# Provision and configure machines
sub.call(["vagrant", "up"], cwd=env_path)
time.sleep(1)
sub.call(["ansible-playbook", "../../../../site.yml"], cwd=env_path)
def main() -> int:
parser = argparse.ArgumentParser()
sub_parser = parser.add_subparsers()
list_parser = sub_parser.add_parser("list", help="list customer-owned environments")
list_parser.add_argument("customer_name", type=str, help="name of the customer")
list_parser.set_defaults(func=list_envs)
cenv_parser = sub_parser.add_parser("create", help="create a new environment")
cenv_parser.add_argument("customer_name", type=str, help="name of the customer")
cenv_parser.add_argument("env_name", type=str, help="name of the environment")
cenv_parser.add_argument(
"--num-postgres",
type=check_positive,
help="number of postgres databases",
default=0,
)
cenv_parser.add_argument(
"--num-nginx-web",
type=check_positive,
help="number of nginx webservers",
default=0,
)
cenv_parser.add_argument(
"--num-nginx-lb",
type=check_positive,
help="number of nginx loadbalancers",
default=0,
)
cenv_parser.add_argument(
"--ip-format", type=str, help="format of ip", default="192.168.56.{}"
)
cenv_parser.add_argument(
"--ip-int", type=check_positive, help="4th octet to start at", default=10
)
cenv_parser.set_defaults(func=create_env)
denv_parser = sub_parser.add_parser("delete", help="delete an environment")
denv_parser.add_argument("customer_name", type=str, help="name of the customer")
denv_parser.add_argument(
"env_names", type=str, nargs="+", help="name of one or more environments"
)
denv_parser.set_defaults(func=delete_env)
args = parser.parse_args(sys.argv[1:])
args.func(args)
return 0
if __name__ == "__main__":
raise SystemExit(main())

View File

@ -6,3 +6,7 @@
- hosts: loadbalancer
roles:
- nginx-loadbalancer
- hosts: postgresql
roles:
- postgresql

View File

@ -1,7 +1,7 @@
$ip_int = #{ipAddr}
$ip_int = {{ ip_int }}
def increment_ip()
ip = "192.168.56.%d" % [$ip_int]
ip = "{{ ip_format }}" % [$ip_int]
$ip_int += 1
return ip
end
@ -10,11 +10,12 @@ Vagrant.configure("2") do |config|
config.ssh.insert_key = false
config.ssh.private_key_path = ["./.ssh/id_rsa","~/.vagrant.d/insecure_private_key"]
num_webserver = #{numWebserver}
num_loadbalancer = #{numLoadbalancers}
num_webserver = {{ num_webserver }}
num_loadbalancer = {{ num_loadbalancers }}
num_postgresql = {{ num_postgres }}
(1..num_webserver).each do |nth|
machine_id = "#{customerName}-bloated-debian-web%d" % [nth]
machine_id = "{{ customer_name }}-{{ env }}-web%d" % [nth]
machine_ip = increment_ip()
config.vm.define machine_id do |web|
@ -33,7 +34,26 @@ Vagrant.configure("2") do |config|
end
(1..num_loadbalancer).each do |nth|
machine_id = "#{customerName}-bloated-debian-lb%d" % [nth]
machine_id = "{{ customer_name }}-{{ env }}-lb%d" % [nth]
machine_ip = increment_ip()
config.vm.define machine_id do |web|
web.vm.box = "ubuntu/focal64"
web.vm.hostname = machine_id
web.vm.network "private_network", ip: machine_ip
web.vm.provision "file", source: "./.ssh/id_rsa.pub", destination: "~/.ssh/authorized_keys"
web.vm.provider "virtualbox" do |vb|
vb.memory = "1024"
vb.gui = false
vb.name = machine_id
end
end
end
(1..num_postgresql).each do |nth|
machine_id = "{{ customer_name }}-{{ env }}-db%d" % [nth]
machine_ip = increment_ip()
config.vm.define machine_id do |web|