Commit 06294a05 authored by Daniel Dehennin's avatar Daniel Dehennin
Browse files

Import from all-in-one zephir repository

parent 8095dc9c
####
#### Temporary layer to prepare installation
####
FROM ubuntu:bionic AS build
# Packages required to build
ENV DEBIAN_FRONTEND=noninteractive
RUN apt-get update -y && apt-get install -y \
curl \
git \
gnupg
ARG CONTAINERPILOT_VERSION=3.4.3
ARG CONTAINERPILOT_CHECKSUM=e8258ed166bcb3de3e06638936dcc2cae32c7c58
RUN curl -Lso /tmp/containerpilot.tar.gz \
"https://github.com/joyent/containerpilot/releases/download/${CONTAINERPILOT_VERSION}/containerpilot-${CONTAINERPILOT_VERSION}.tar.gz" \
&& echo "${CONTAINERPILOT_CHECKSUM} /tmp/containerpilot.tar.gz" | sha1sum -c \
&& tar zxf /tmp/containerpilot.tar.gz -C /tmp
ARG CONSULTEMPLATE_VERSION=0.19.3
ARG CONSULTEMPLATE_CHECKSUM=6467b442d8fd68ca0a1eba35f42edb6f3b9941e10b6d58688edc3506fca0bb19
RUN curl -Lso /tmp/consul-template.tar.gz \
"https://releases.hashicorp.com/consul-template/${CONSULTEMPLATE_VERSION}/consul-template_${CONSULTEMPLATE_VERSION}_linux_amd64.tgz" \
&& echo "${CONSULTEMPLATE_CHECKSUM} /tmp/consul-template.tar.gz" | sha256sum -c \
&& tar zxf /tmp/consul-template.tar.gz -C /tmp
# Orcherstration using containerpilot
# Configuration using consul-template
# Use dev-eole repository since gitlab is not public
ARG CONTAINERPILOT_REPO_URL=https://dev-eole.ac-dijon.fr/git/zephir-orchestrate-containerpilot.git
ARG CONTAINERPILOT_REPO_REF=223dafd3d093702c8717292247e65d0c55ffb513
RUN git clone "${CONTAINERPILOT_REPO_URL}" "/tmp/orchestrate" \
&& cd /tmp/orchestrate \
&& git checkout "${CONTAINERPILOT_REPO_REF}"
# API messages description
# Use dev-eole repository since gitlab is not public
ARG MESSAGES_API_REPO_URL=https://dev-eole.ac-dijon.fr/git/zephir-messages-api.git
ARG MESSAGES_API_REPO_REF=e10c743b3a7d1e84ed13bd49a46e7361fd403ccc
RUN git clone "${MESSAGES_API_REPO_URL}" "/tmp/messages-api" \
&& cd /tmp/messages-api \
&& git checkout "${MESSAGES_API_REPO_REF}"
# Common python Zéphir library
# Use dev-eole repository since gitlab is not public
ARG PYTHON_ZEPHIR_REPO_URL=https://dev-eole.ac-dijon.fr/git/python-zephir.git
ARG PYTHON_ZEPHIR_REPO_REF=bc098b33494c7ef7ae010e130bb773507e196b72
RUN git clone "${PYTHON_ZEPHIR_REPO_URL}" "/tmp/python-zephir" \
&& cd /tmp/python-zephir \
&& git checkout "${PYTHON_ZEPHIR_REPO_REF}"
##
## Service specific
##
# Configure package manager
RUN echo "deb http://repo.saltstack.com/apt/ubuntu/18.04/amd64/latest bionic main" > /etc/apt/sources.list.d/saltstack.list
RUN curl https://repo.saltstack.com/apt/ubuntu/18.04/amd64/latest/SALTSTACK-GPG-KEY.pub | apt-key --keyring /etc/apt/trusted.gpg.d/saltstack.gpg add -
# SaltStack formulas parameters
ENV EOLE_FORMULA_DIR=/srv/formulas
ENV EOLE_CONFIGURATION_FORMULA_URL="https://dev-eole.ac-dijon.fr/git/eole-configuration-formula.git"
ENV EOLE_CONFIGURATION_FORMULA_REF=8c08f4fb902408e320ea1fa2fd74e12fdab60fa3
RUN git clone "${EOLE_CONFIGURATION_FORMULA_URL}" "${EOLE_FORMULA_DIR}/eole-configuration-formula" \
&& cd "${EOLE_FORMULA_DIR}/eole-configuration-formula" \
&& git checkout -b deployed "${EOLE_CONFIGURATION_FORMULA_REF}"
####
#### Target layer
####
FROM ubuntu:bionic
MAINTAINER Pôle EOLE <eole@ac-dijon.fr>
# Packages required for working service
ENV DEBIAN_FRONTEND=noninteractive
RUN apt-get update -y && apt-get install -y \
gnupg \
jq \
locales \
openssl \
python3 \
python3-aiohttp \
python3-autobahn \
python3-dev \
python3-jwt \
python3-psycopg2 \
python3-requests \
python3-yaml \
sqitch \
tzdata
# For debug only
RUN apt-get update -y && apt-get install -y \
curl \
iproute2 \
iputils-ping \
vim
# Force french locale
RUN locale-gen fr_FR.UTF-8
ENV LANG fr_FR.UTF-8
ENV LC_ALL fr_FR.UTF-8
# Force timezone
RUN ln -fs /usr/share/zoneinfo/Europe/Paris /etc/localtime
RUN dpkg-reconfigure --frontend noninteractive tzdata
# Sqitch
RUN sqitch config --user user.name 'Equipe EOLE'\
&& sqitch config --user user.email 'eole@ac-dijon.fr'
# Install tools from build layer
COPY --from=build /tmp/containerpilot /usr/local/bin
COPY --from=build /tmp/consul-template /usr/local/bin
# consul-template wrapper to generate /etc/zephir-services.conf
COPY --from=build /tmp/orchestrate/update-service-conf /usr/local/bin/
RUN chmod +x /usr/local/bin/*
# Service template for consul-template
ENV services_conf_filename=zephir-services.conf
ARG services_conf=/etc/zephir-services.conf
ENV services_conf=$services_conf
COPY --from=build /tmp/orchestrate/${services_conf_filename}.ctmpl ${services_conf}.ctmpl
# Install libraries required by service
COPY --from=build /tmp/python-zephir/zephir /usr/lib/python3/dist-packages/zephir
COPY --from=build /tmp/messages-api/messages /srv/messages
# Manage container with ContainerPilot
CMD ["/usr/local/bin/containerpilot", "-config", "/etc/containerpilot.json5"]
COPY containerpilot.json5 /etc/containerpilot.json5
##
## Service specific
##
COPY --from=build /etc/apt/sources.list.d/saltstack.list /etc/apt/sources.list.d/saltstack.list
COPY --from=build /etc/apt/trusted.gpg.d/saltstack.gpg /etc/apt/trusted.gpg.d/saltstack.gpg
RUN apt-get update -y && apt-get install -y \
netcat \
python3-crypto \
salt-api \
salt-master \
screen
# Library for SaltMaster manager
COPY src/python/zephir/saltmaster /usr/lib/python3/dist-packages/zephir/saltmaster
# Configuration
COPY master.d/*.conf /etc/salt/master.d/
RUN mkdir /srv/salt
RUN mkdir /srv/pillar
RUN mkdir /keys
# Install salt files
COPY --from=build /srv/formulas/eole-configuration-formula/eole/ /srv/salt/eole/
RUN mkdir /srv/salt/_modules
COPY environment/* /srv/salt/_modules/
COPY pillars/* /srv/pillar
# Controller
RUN mkdir -p /srv/bin
COPY scripts/* /srv/bin/
COPY migrations /migrations
RUN chmod +x /srv/bin/*
# Create the password
ENV SALT_PASSWORD=eole
RUN /srv/bin/create-user.sh
FROM zephir_saltmaster-manager:latest
RUN apt-get update -y && apt-get install -y \
postgresql \
postgresql-contrib \
python3-coverage \
python3-pytest \
python3-pytest-cov
COPY saltmaster-manager/test /srv/test
COPY saltmaster-manager/scripts/test /usr/local/bin/test-saltmaster
RUN chmod +x /usr/local/bin/test-saltmaster
CMD /bin/bash /usr/local/bin/test-saltmaster
{
consul: {
address: "{{ .CONSUL_HOST }}"
},
logging: {
level: '{{ .CONTAINERPILOT_LOGLEVEL | default "debug" }}'
},
jobs: [
{
name: "update-service-conf",
exec: ["/usr/local/bin/update-service-conf"]
},
{
name: "controller",
exec: ["python3", "-u", "/srv/bin/controller"],
health: {
exec: "pkill -0 -f /srv/bin/controller",
interval: 10,
ttl: 20
},
when: {
source: "update-service-conf",
once: "exitSuccess"
},
restarts: "unlimited",
port: -1
},
{
name: "oncrossbarchange-update-service-conf",
exec: ["/usr/local/bin/update-service-conf", "reload"],
when: {
source: "watch.crossbar",
each: "changed"
}
},
{
name: "wait-4-db",
exec: ["python3", "-u", "/srv/bin/persistent-database"],
when: {
source: "watch.crossbar",
once: "healthy",
},
},
{
name: "salt-master",
exec: ["salt-master", "-l", '{{ .SALTMASTER_LOG_LEVEL | default "debug" }}'],
health: {
exec: "/bin/nc -z 127.0.0.1 30009",
interval: 10,
ttl: 20
},
when: {
source: "wait-4-db",
once: "exitSuccess",
},
port: 30009,
restarts: "unlimited"
},
{
name: "salt-api",
exec: ["salt-api", "-l", '{{ .SALT_API_LOG_LEVEL | default "debug" }}'],
health: {
exec: "/bin/nc -z 127.0.0.1 8000",
interval: 10,
ttl: 20
},
port: 8000,
when: {
source: "salt-master",
once: "healthy"
},
restarts: "unlimited"
}
],
watches: [
{
name: "crossbar",
interval: 5
}
]
}
from json import load
from creole import eosfunc
from os.path import isfile
FILE_PATH = '/var/lib/eole/zephir/config.env'
def environment():
grains = {}
with open(FILE_PATH, "r") as fhr:
probes = load(fhr)
if isfile(FILE_PATH):
for name, probe in probes.items():
grains[name] = getattr(eosfunc, probe["function"])(*probe["args"], **probe["kwargs"])
return grains
rest_cherrypy:
port: 8000
disable_ssl: true
#ssl_crt: /etc/pki/tls/certs/localhost.crt
#ssl_key: /etc/pki/tls/certs/localhost.key
external_auth:
pam:
eole:
- .*
- '@wheel' # to allow access to all wheel modules
# A file glob (or list of file globs) that will be matched against the file
# path before syncing the modules and states to the minions. This is similar
# to file_ignore_regex above, but works on globs instead of regex. By default
# nothing is ignored.
file_ignore_glob:
- '*.pyc'
# The tcp port used by the publisher
publish_port: 30009
# Store jobs cache in PostgreSQL
master_job_cache: pgjsonb
returner.pgjsonb.host: 'postgres'
returner.pgjsonb.port: 5432
returner.pgjsonb.db: '{{DBNAME}}'
returner.pgjsonb.user: '{{USER}}'
returner.pgjsonb.pass: '{{PASSWORD}}'
-- Deploy saltmaster:salt-runner-db to pg
BEGIN;
SET ROLE "saltmaster_admin";
--
-- Table structure for table `jids`
--
CREATE TABLE jids (
jid varchar(255) NOT NULL primary key,
load jsonb NOT NULL
);
CREATE INDEX idx_jids_jsonb on jids
USING gin (load)
WITH (fastupdate=on);
--
-- Table structure for table `salt_returns`
--
CREATE TABLE salt_returns (
fun varchar(50) NOT NULL,
jid varchar(255) NOT NULL,
return jsonb NOT NULL,
id varchar(255) NOT NULL,
success varchar(10) NOT NULL,
full_ret jsonb NOT NULL,
alter_time TIMESTAMP WITH TIME ZONE DEFAULT NOW());
CREATE INDEX idx_salt_returns_id ON salt_returns (id);
CREATE INDEX idx_salt_returns_jid ON salt_returns (jid);
CREATE INDEX idx_salt_returns_fun ON salt_returns (fun);
CREATE INDEX idx_salt_returns_return ON salt_returns
USING gin (return) with (fastupdate=on);
CREATE INDEX idx_salt_returns_full_ret ON salt_returns
USING gin (full_ret) with (fastupdate=on);
--
-- Table structure for table `salt_events`
--
CREATE SEQUENCE seq_salt_events_id;
CREATE TABLE salt_events (
id BIGINT NOT NULL UNIQUE DEFAULT nextval('seq_salt_events_id'),
tag varchar(255) NOT NULL,
data jsonb NOT NULL,
alter_time TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
master_id varchar(255) NOT NULL);
CREATE INDEX idx_salt_events_tag on
salt_events (tag);
CREATE INDEX idx_salt_events_data ON salt_events
USING gin (data) with (fastupdate=on);
COMMIT;
-- Revert saltmaster:salt-runner-db from pg
BEGIN;
SET ROLE "saltmaster_admin";
-- XXX Add DDLs here.
--
-- Table structure for table `jids`
--
DROP TABLE IF EXISTS jids;
--
-- Table structure for table `salt_returns`
--
DROP TABLE IF EXISTS salt_returns;
COMMIT;
[core]
engine = pg
%syntax-version=1.0.0
%project=saltmaster
salt-runner-db 2018-04-10T08:02:27Z Equipe EOLE <eole@ac-dijon.fr> # Add salt-runner
-- Verify saltmaster:salt-runner-db on pg
BEGIN;
SET ROLE "saltmaster_admin";
-- XXX Add verifications here.
-- Limit verification to existence of tables
SELECT jid from jids LIMIT 1;
SELECT jid from salt_returns LIMIT 1;
SELECT id from salt_events LIMIT 1;
ROLLBACK;
base:
'*':
- pillars_{{ grains['id'] }}
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import asyncio
import signal
import time
import json, yaml
from os import path, makedirs, getpid, unlink
from autobahn.wamp.exception import ApplicationError
from zephir.controller import ZephirCommonController, run
from zephir.http import register as register_http
from zephir.wamp import register as register_wamp
from zephir.coroutine import register as register_coroutine
from zephir.saltmaster.execute import SaltExec
from zephir.saltmaster.execute import SaltExecErrorMissingMinionPattern
from zephir.saltmaster.execute import SaltExecErrorMissingCommand
from zephir.saltmaster.execute import SaltExecErrorLoginError
from zephir.saltmaster.execute import SaltExecErrorExecuteError
from zephir.saltmaster.jobs import SaltJobs
from zephir.saltmaster.jobs import SaltJobsErrorDbConnectionError
from zephir.saltmaster.jobs import SaltJobsErrorListError
from zephir.saltmaster.jobs import SaltJobsErrorMissingJobId
from zephir.saltmaster.jobs import SaltJobsErrorJobIdTypeError
from zephir.saltmaster.jobs import SaltJobsErrorUnknownJobId
from zephir.saltmaster.peering import SaltPeering
from zephir.saltmaster.events import SaltEvents
MINION_DIR = '/srv/salt/eole/configuration/files/'
class SaltRunner(ZephirCommonController):
"""Salt controller based on salt-api
"""
def __init__(self, *args, **kwargs):
# Initialise self.config
super().__init__(*args, **kwargs)
# Create instance of working code
self.executor = SaltExec(config=self.config.extra)
self.jobs = SaltJobs(config=self.config.extra)
self.peers = SaltPeering(config=self.config.extra)
self.events = SaltEvents(config=self.config.extra, wamp=self)
@register_wamp('v1.execution.salt.exec', notification_uri='v1.execution.salt.exec.command-executed')
async def exec_command(self, minion_pattern, command, arg, client_mode):
try:
ret = self.executor.exec_command(minion_pattern, command, arg, client_mode)
return ret
except SaltExecErrorMissingMinionPattern as err:
raise ApplicationError('execution.salt.exec.error.missing-minion_pattern', reason=str(err))
except SaltExecErrorMissingCommand as err:
raise ApplicationError(u'execution.salt.exec.error.missing-command', reason=str(err))
except SaltExecErrorLoginError as err:
raise ApplicationError('execution.salt.exec.error.login', reason=str(err))
except SaltExecErrorExecuteError as err:
raise ApplicationError('execution.salt.exec.error.execute', reason=str(err))
@register_wamp('v1.execution.salt.job.list', notification_uri=None, database=True)
async def list_jobs(self, cursor, minion_pattern):
try:
return list(self.jobs.list_jobs(cursor, minion_pattern))
except SaltJobsErrorDbConnectionError as err:
raise ApplicationError('execution.salt.job.error.db-connection', reason=str(err))
except SaltJobsErrorListError as err:
raise ApplicationError('execution.salt.job.error.list', reason=str(err))
@register_wamp('v1.execution.salt.job.describe', notification_uri=None, database=True)
async def describe_job(self, cursor, jid):
try:
return self.jobs.describe_job(cursor, jid)
except SaltJobsErrorDbConnectionError as err:
raise ApplicationError('execution.salt.job.error.db-connection', reason=str(err))
except SaltJobsErrorMissingJobId as err:
raise ApplicationError('execution.salt.job.error.missing-jid', reason=str(err))
except SaltJobsErrorJobIdTypeError as err:
raise ApplicationError('execution.salt.job.error.jid-type', reason=str(err))
except SaltJobsErrorUnknownJobId as err:
raise ApplicationError('execution.salt.job.error.unknown-jid', reason=str(err))
@register_wamp('v1.execution.salt.configuration.deploy',
notification_uri='v1.execution.salt.configuration.deploy-scheduled')
async def configuration_deploy(self, minion_pattern):
await self.download_configuration_files(int(minion_pattern))
try:
deploy_command = 'state.apply'
deploy_arg = 'eole.configuration.deploy'
return self.executor.exec_command(minion_pattern=minion_pattern,
command=deploy_command,
arg=deploy_arg)
except SaltExecErrorMissingMinionPattern as err:
raise ApplicationError('execution.salt.exec.error.missing-minion_pattern', reason=str(err))
except SaltExecErrorLoginError as err:
raise ApplicationError('execution.salt.exec.error.login', reason=str(err))
except SaltExecErrorExecuteError as err:
raise ApplicationError('execution.salt.exec.error.execute', reason=str(err))
async def server_started(self, server_id):
await self.download_env(server_id)
self.executor.exec_command(minion_pattern=str(server_id),
command='state.apply',
arg='eole.configuration.deploy')
# FIXME refresh = true
self.executor.exec_command(minion_pattern=str(server_id),
command='saltutil.sync_all',
client_mode="local")
@register_wamp('v1.execution.salt.master.event.start', notification_uri='v1.execution.salt.master.event.ready')
async def minion_start(self, server_id):
await self.server_started(server_id)
await self.call('v1.server.peer-connection.update',
serverid=server_id)
return {'server_id': server_id}
@register_wamp('v1.execution.salt.environment.get', notification_uri=None)
async def get_environment(self, server_id):
server_id = str(server_id)
return self.executor.exec_command(server_id,
'zephir.environment',
None,
'local')[server_id]
async def download_configuration_files(self,
server_id):
minion_dir = '{0}/{1}'.format(MINION_DIR, server_id)
schema_file_path = '{0}/config.schema'.format(minion_dir)
config_file_path = '{0}/config.values'.format(minion_dir)
pillar_file_path = '/srv/pillar/pillars_{0}.sls'.format(server_id)
makedirs(minion_dir, exist_ok=True)
server = await self.call('v1.server.describe',
serverid=server_id,
configuration=True)
servermodel_id = server['servermodelid']
with open(config_file_path, 'w') as fh:
fh.write(server['configuration'])
with open(pillar_file_path, 'w') as fh:
if server['configuration']:
yaml.dump(json.loads(server['configuration']), fh, allow_unicode=True)
else:
yaml.dump({}, fh)
del server
servermodel = await self.call('v1.servermodel.describe',
servermodelid=servermodel_id,
inheritance=False,
resolvdepends=False,
schema=True,
probes=True)
with open(schema_file_path, 'w') as fh:
fh.write(servermodel['schema'])