Skip to content

GitLab

Commit 0c523d41 authored by Daniel Dehennin's avatar Daniel Dehennin
Browse files

Import from all-in-one zephir repository

parent d0119eba
Hide whitespace changes
Inline Side-by-side
Showing with 959 additions and 0 deletions
Dockerfile 0 → 100644
View file @ 0c523d41
####
#### Temporary layer to prepare installation
####
FROM ubuntu:bionic AS build
# Packages required to build
ENV DEBIAN_FRONTEND=noninteractive
RUN apt-get update -y && apt-get install -y \
curl \
git
ARG CONTAINERPILOT_VERSION=3.4.3
ARG CONTAINERPILOT_CHECKSUM=e8258ed166bcb3de3e06638936dcc2cae32c7c58
RUN curl -Lso /tmp/containerpilot.tar.gz \
"https://github.com/joyent/containerpilot/releases/download/${CONTAINERPILOT_VERSION}/containerpilot-${CONTAINERPILOT_VERSION}.tar.gz" \
&& echo "${CONTAINERPILOT_CHECKSUM} /tmp/containerpilot.tar.gz" | sha1sum -c \
&& tar zxf /tmp/containerpilot.tar.gz -C /tmp
ARG CONSULTEMPLATE_VERSION=0.19.3
ARG CONSULTEMPLATE_CHECKSUM=6467b442d8fd68ca0a1eba35f42edb6f3b9941e10b6d58688edc3506fca0bb19
RUN curl -Lso /tmp/consul-template.tar.gz \
"https://releases.hashicorp.com/consul-template/${CONSULTEMPLATE_VERSION}/consul-template_${CONSULTEMPLATE_VERSION}_linux_amd64.tgz" \
&& echo "${CONSULTEMPLATE_CHECKSUM} /tmp/consul-template.tar.gz" | sha256sum -c \
&& tar zxf /tmp/consul-template.tar.gz -C /tmp
# Orcherstration using containerpilot
# Configuration using consul-template
# Use dev-eole repository since gitlab is not public
ARG CONTAINERPILOT_REPO_URL=https://dev-eole.ac-dijon.fr/git/zephir-orchestrate-containerpilot.git
ARG CONTAINERPILOT_REPO_REF=223dafd3d093702c8717292247e65d0c55ffb513
RUN git clone "${CONTAINERPILOT_REPO_URL}" "/tmp/orchestrate" \
&& cd /tmp/orchestrate \
&& git checkout "${CONTAINERPILOT_REPO_REF}"
# API messages description
# Use dev-eole repository since gitlab is not public
ARG MESSAGES_API_REPO_URL=https://dev-eole.ac-dijon.fr/git/zephir-messages-api.git
ARG MESSAGES_API_REPO_REF=e10c743b3a7d1e84ed13bd49a46e7361fd403ccc
RUN git clone "${MESSAGES_API_REPO_URL}" "/tmp/messages-api" \
&& cd /tmp/messages-api \
&& git checkout "${MESSAGES_API_REPO_REF}"
# Common python Zéphir library
# Use dev-eole repository since gitlab is not public
ARG PYTHON_ZEPHIR_REPO_URL=https://dev-eole.ac-dijon.fr/git/python-zephir.git
ARG PYTHON_ZEPHIR_REPO_REF=bc098b33494c7ef7ae010e130bb773507e196b72
RUN git clone "${PYTHON_ZEPHIR_REPO_URL}" "/tmp/python-zephir" \
&& cd /tmp/python-zephir \
&& git checkout "${PYTHON_ZEPHIR_REPO_REF}"
####
#### Target layer
####
FROM ubuntu:bionic
MAINTAINER Pôle EOLE <eole@ac-dijon.fr>
# Packages required for working service
ENV DEBIAN_FRONTEND=noninteractive
RUN apt-get update -y && apt-get install -y \
gnupg \
jq \
locales \
openssl \
python3 \
python3-aiohttp \
python3-autobahn \
python3-dev \
python3-jwt \
python3-psycopg2 \
python3-requests \
python3-yaml \
sqitch \
tzdata
# For debug only
RUN apt-get update -y && apt-get install -y \
curl \
iproute2 \
iputils-ping \
vim
# Force french locale
RUN locale-gen fr_FR.UTF-8
ENV LANG fr_FR.UTF-8
ENV LC_ALL fr_FR.UTF-8
# Force timezone
RUN ln -fs /usr/share/zoneinfo/Europe/Paris /etc/localtime
RUN dpkg-reconfigure --frontend noninteractive tzdata
# Sqitch
RUN sqitch config --user user.name 'Equipe EOLE'\
&& sqitch config --user user.email 'eole@ac-dijon.fr'
# Install tools from build layer
COPY --from=build /tmp/containerpilot /usr/local/bin
COPY --from=build /tmp/consul-template /usr/local/bin
# consul-template wrapper to generate /etc/zephir-services.conf
COPY --from=build /tmp/orchestrate/update-service-conf /usr/local/bin/
RUN chmod +x /usr/local/bin/*
# Service template for consul-template
ENV services_conf_filename=zephir-services.conf
ARG services_conf=/etc/zephir-services.conf
ENV services_conf=$services_conf
COPY --from=build /tmp/orchestrate/${services_conf_filename}.ctmpl ${services_conf}.ctmpl
# Install libraries required by service
COPY --from=build /tmp/python-zephir/zephir /usr/lib/python3/dist-packages/zephir
COPY --from=build /tmp/messages-api/messages /srv/messages
# Manage container with ContainerPilot
CMD ["/usr/local/bin/containerpilot", "-config", "/etc/containerpilot.json5"]
COPY --from=build /tmp/orchestrate/containerpilot.json5 /etc/containerpilot.json5
##
## Service specific
##
# Library
COPY src/python/server /usr/lib/python3/dist-packages/server
# Controller
RUN mkdir -p /srv/bin
COPY scripts/* /srv/bin/
COPY migrations /migrations
Dockerfile.Test 0 → 100644
View file @ 0c523d41
FROM zephir_server-manager:latest
RUN apt-get update -y && apt-get install -y \
postgresql \
postgresql-contrib \
python3-coverage \
python3-pytest \
python3-pytest-cov
COPY server-manager/test /srv/test
COPY server-manager/scripts/test /usr/local/bin/test-server
RUN chmod +x /usr/local/bin/test-server
CMD /bin/bash /usr/local/bin/test-server
migrations/deploy/server_schema.sql 0 → 100644
View file @ 0c523d41
-- Deploy server:server_schema to pg
BEGIN;
-- Server table creation
CREATE TABLE Server (
ServerId SERIAL PRIMARY KEY,
ServerName VARCHAR(255) NOT NULL UNIQUE,
ServerDescription VARCHAR(255) NOT NULL,
ServerModelId INTEGER NOT NULL,
ZoneId INTEGER,
MachineId INTEGER,
ServerConfiguration JSON,
Automation VARCHAR(25),
ServerEnvironment JSON,
LastPeerConnection TIMESTAMP DEFAULT NULL
);
COMMIT;
migrations/revert/server_schema.sql 0 → 100644
View file @ 0c523d41
-- Revert server:server_schema from pg
BEGIN;
DROP SCHEMA server CASCADE;
COMMIT;
migrations/sqitch.conf 0 → 100644
View file @ 0c523d41
[core]
engine = pg
# plan_file = sqitch.plan
# top_dir = .
[engine "pg"]
# target = db:pg:
registry = sqitch_server
# client = psql
migrations/sqitch.plan 0 → 100644
View file @ 0c523d41
%syntax-version=1.0.0
%project=server
server_schema 2017-10-09T09:31:30Z Equipe EOLE <eole@ac-dijon.fr> # Definition du schema de base pour le domaine Server
migrations/verify/server_schema.sql 0 → 100644
View file @ 0c523d41
-- Verify server:server_schema on pg
BEGIN;
SELECT 1/COUNT(*) FROM information_schema.schemata WHERE schema_name = 'server';
ROLLBACK;
scripts/controller 0 → 100755
View file @ 0c523d41
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import json
import datetime
from autobahn.wamp.exception import ApplicationError
from zephir.i18n import _
from zephir.controller import ZephirCommonController, run
from zephir.http import register as register_http
from zephir.wamp import register as register_wamp
from server.server.lib import Server
from server.server.error import (ServerError,
ServerErrorDatabaseNotAvailable,
ServerErrorPeeringConfNotAvailable,
ServerErrorDbConnection,
ServerErrorInvalidServerId,
ServerErrorInvalidServerModelId,
ServerErrorServerNameNotProvided,
ServerErrorUnknownServerId,
ServerErrorUnknownServerModelId)
from zephir.config import ServiceConfig
class ServerRunner(ZephirCommonController):
"""Server controller
"""
def __init__(self, *args, **kwargs):
# Create instance of working code
super().__init__(*args, **kwargs)
self.server = Server()
self.conn = None
@register_wamp('v1.server.list', notification_uri=None, database=True)
async def list_servers(self, cursor):
try:
return self.server.list_servers(cursor)
except ServerErrorDatabaseNotAvailable as err:
raise ApplicationError('server.error.database-not-available', reason=str(err))
except ServerErrorDbConnection as err:
raise ApplicationError('server.error.db-connection', reason=str(err))
except ServerError as err:
raise ApplicationError('server.error', reason=str(err))
@register_wamp('v1.server.describe', notification_uri=None, database=True)
async def describe_server(self, cursor, serverid, configuration):
try:
server = self.server.describe_server(cursor, serverid)
if configuration:
server['configuration'] = serverid
return server
except ServerErrorDatabaseNotAvailable as err:
raise ApplicationError('server.error.database-not-available', reason=str(err))
except ServerErrorDbConnection as err:
raise ApplicationError('server.error.db-connection', reason=str(err))
except ServerErrorInvalidServerId as err:
raise ApplicationError('server.error.invalid-server-id', reason=str(err))
except ServerErrorUnknownServerId as err:
raise ApplicationError('server.error.unknown-server-id', reason=str(err))
except ServerError as err:
raise ApplicationError('server.error', reason=str(err))
@register_http('v1.server.describe', param='configuration', database=True)
async def describe_configuration(self, cursor, secret):
values = self.server.fetch_configuration(cursor, secret)
if values:
return json.dumps(values).encode()
else:
return b'{}'
@register_wamp('v1.server.create', notification_uri='v1.server.created', database=True)
async def create_server(self, cursor, servername, serverdescription, servermodelid, serverpassphrase):
try:
result = self.server.create_server(cursor, servername, serverdescription, servermodelid)
return_code = await self.call('v1.vault.secret.set',
secretkey="{}_passphrase".format(result['serverid']),
secret={"passphrase" : serverpassphrase})
if return_code:
return result
else:
raise ServerError('put passphrase return code status not available')
except ServerErrorDatabaseNotAvailable as err:
raise ApplicationError('server.error.database-not-available', reason=str(err))
except ServerErrorDbConnection as err:
raise ApplicationError('server.error.db-connection', reason=str(err))
except ServerErrorInvalidServerModelId as err:
raise ApplicationError('server.error.invalid-servermodel-id', reason=str(err))
except ServerErrorUnknownServerModelId as err:
raise ApplicationError('server.error.unknown-servermodel-id', reason=str(err))
except ServerErrorServerNameNotProvided as err:
raise ApplicationError('server.error.servername-not-provided', reason=str(err))
except ServerError as err:
raise ApplicationError('server.error', reason=str(err))
@register_wamp('v1.server.update', notification_uri='v1.server.updated', database=True)
async def update_server(self, cursor, serverid, servername, serverdescription):
try:
return self.server.update_server(cursor, serverid, servername, serverdescription)
except ServerErrorDatabaseNotAvailable as err:
raise ApplicationError('server.error.database-not-available', reason=str(err))
except ServerErrorDbConnection as err:
raise ApplicationError('server.error.db-connection', reason=str(err))
except ServerErrorInvalidServerId as err:
raise ApplicationError('server.error.invalid-server-id', reason=str(err))
except ServerErrorUnknownServerId as err:
raise ApplicationError('server.error.unknown-server-id', reason=str(err))
except ServerErrorServerNameNotProvided as err:
raise ApplicationError('server.error.servername-not-provided', reason=str(err))
except ServerError as err:
raise ApplicationError('server.error', reason=str(err))
@register_wamp('v1.server.delete', notification_uri='v1.server.deleted', database=True)
async def delete_server(self, cursor, serverid):
try:
return self.server.delete_server(cursor, serverid)
except ServerErrorDatabaseNotAvailable as err:
raise ApplicationError('server.error.database-not-available', reason=str(err))
except ServerErrorDbConnection as err:
raise ApplicationError('server.error.db-connection', reason=str(err))
except ServerErrorInvalidServerId as err:
raise ApplicationError('server.error.invalid-server-id', reason=str(err))
except ServerErrorUnknownServerId as err:
raise ApplicationError('server.error.unknown-server-id', reason=str(err))
except ServerError as err:
raise ApplicationError('server.error', reason=str(err))
@register_wamp('v1.server.peering-conf.get', notification_uri='v1.server.peering-conf.sent')
async def get_peering_conf(self, serverid):
try:
secret = await self.call('v1.vault.secret.get', secretkey="{}_peeringconf".format(serverid))
return secret['secret']
except ServerErrorPeeringConfNotAvailable as err:
raise ApplicationError('server.error.peering-conf-not-available', reason=str(err))
except ServerError as err:
raise ApplicationError('server.error', reason=str(err))
@register_wamp('v1.execution.salt.master.event.ready', notification_uri='v1.server.environment.updated', database=True)
async def minion_ready(self,
cursor,
server_id):
environ = await self.call('v1.execution.salt.environment.get',
server_id=server_id)
self.server.update_environment(cursor,
server_id,
json.dumps(environ))
return {'server_id': server_id}
@register_wamp('v1.execution.salt.peer.registered', notification_uri='v1.server.salt.registered', database=True)
async def salt_register(self, cursor, serverid, automation):
try:
self.server.register_server_for_automation(cursor, serverid, automation)
return {'serverid': serverid}
except Exception as err:
raise ApplicationError('server.registering.error',
reason=str(err))
@register_wamp('v1.server.exec.command', notification_uri='v1.server.executed', database=True)
async def exec_cmd_on_server(self, cursor, server_id, command):
"""
Transfer command transmitted to automation (salt, ...)
"""
automation, automation_command = self.server.get_automation_command(cursor, server_id)
if automation == 'salt':
result = await self.call('v1.execution.salt.exec',
minion_pattern=str(server_id),
command=automation_command,
arg=command,
client_mode='local_async')
else:
raise Exception(_('Automation engine not supported: {}').format(automation))
if result['minions'] != [str(server_id)]:
raise Exception(_('Job ({}) not executed only in selected server, all client affected : {}').format(result['jid'],
result['minions']))
return {'job_id': result['jid'],
'server_id': server_id,
'command': command,
'automation': automation,
'executed': False}
def result_to_dict(self, result, automation):
dico = {'job_id': result['jid'],
'server_id': int(result['minion']),
'automation': automation,
'executed': result['executed']}
if result['command'] == 'deploy':
dico['command'] = 'v1.server.exec.deploy'
else:
dico['command'] = result['arg']
if dico['executed']:
dico['success'] = result['success']
dico['retcode'] = result['retcode']
dico['return'] = result['return']
return dico
@register_wamp('v1.server.exec.list', notification_uri=None, database=True)
async def exec_job_on_server(self, cursor, server_id):
automation, automation_command = self.server.get_automation_command(cursor, server_id)
if automation == 'salt':
results = await self.call('v1.execution.salt.job.list',
minion_pattern=str(server_id))
else:
raise Exception(_('Automation engine not supported: {}').format(automation))
ret = []
for result in results:
ret.append(self.result_to_dict(result, automation))
return ret
@register_wamp('v1.server.exec.deploy', notification_uri=None, database=True)
async def exec_deploy(self, cursor, server_id):
automation, automation_command = self.server.get_automation_command(cursor, server_id)
if automation == 'salt':
result = await self.call('v1.execution.salt.configuration.deploy',
minion_pattern=str(server_id))
else:
raise Exception(_('Automation engine not supported: {}').format(automation))
dico = {'job_id': result['jid'],
'command': 'v1.server.exec.deploy',
'automation': automation,
'server_id': server_id,
'executed': False}
return dico
@register_wamp('v1.server.exec.describe', notification_uri=None)
async def exec_describe(self, job_id, automation):
if automation == 'salt':
results = await self.call('v1.execution.salt.job.describe',
jid=str(job_id))
else:
raise Exception(_('Automation engine not supported: {}').format(automation))
ret = []
for result in results:
ret.append(self.result_to_dict(result, automation))
return ret
@register_wamp('v1.config.configuration.server.saved', None, database=True)
async def update_configuration(self, cursor, server_id):
configuration = await self.call('v1.config.configuration.server.get', server_id=server_id)
self.server.update_configuration(cursor, server_id, configuration['configuration'])
@register_wamp('v1.server.peer-connection.update', notification_uri=None, database=True)
async def update_peerconnection(self, cursor, serverid):
try:
lastpeerconnection = datetime.datetime.now()
return self.server.update_peerconnection(cursor, serverid, lastpeerconnection)
except ServerErrorDatabaseNotAvailable as err:
raise ApplicationError('server.error.database-not-available', reason=str(err))
except ServerErrorDbConnection as err:
raise ApplicationError('server.error.db-connection', reason=str(err))
except ServerErrorInvalidServerId as err:
raise ApplicationError('server.error.invalid-server-id', reason=str(err))
except ServerErrorUnknownServerId as err:
raise ApplicationError('server.error.unknown-server-id', reason=str(err))
except ServerErrorServerNameNotProvided as err:
raise ApplicationError('server.error.servername-not-provided', reason=str(err))
except ServerError as err:
raise ApplicationError('server.error', reason=str(err))
if __name__ == '__main__':
run(ServerRunner)
scripts/test 0 → 100644
View file @ 0c523d41
#!/bin/bash
set -eo pipefail
if [ -z $ZEPHIR_TEST_TYPE ] && [ ! -z $1 ]; then
ZEPHIR_TEST_TYPE=$1
fi
case $ZEPHIR_TEST_TYPE in
"unit")
/etc/init.d/postgresql start
cd /srv/test/unit
su - postgres -c "psql -c \"ALTER USER postgres PASSWORD 'mynewpassword';\""
PYTHONPATH='.' /usr/bin/python3-coverage run /usr/bin/py.test-3 -s -x
if [ ! -z "$ZEPHIR_TEST_COVERAGE_PATTERN" ]; then
/usr/bin/python3-coverage html -d /srv/report --include=$ZEPHIR_TEST_COVERAGE_PATTERN
/usr/bin/python3-coverage report --include=$ZEPHIR_TEST_COVERAGE_PATTERN
fi
;;
"integration")
py.test /srv/test/test_server.py -x
exit $?
;;
"e2e")
echo "No e2e tests yet."
;;
*)
1>&2 echo "Unknown test type: '$ZEPHIR_TEST_TYPE'"
exit 1
esac
src/python/server/server/__init__.py 0 → 100644
View file @ 0c523d41
from .lib import Server
__all__ = ['Server']
src/python/server/server/error.py 0 → 100644
View file @ 0c523d41
class ServerError(Exception):
"""Base class of :class:`Server` exceptions
"""
class ServerErrorPeeringConfNotAvailable(ServerError):
"""No available peering configuration
"""
class ServerErrorDatabaseNotAvailable(ServerError):
"""No available database service
"""
class ServerErrorDbConnection(ServerError):
"""Database connection error
"""
class ServerErrorInvalidServerId(ServerError):
"""Invalid server ID
"""
class ServerErrorInvalidServerModelId(ServerError):
"""Invalid servermodel ID
"""
class ServerErrorServerNameNotProvided(ServerError):
"""Servername expected
"""
class ServerErrorUnknownServerId(ServerError):
"""Unknown server ID
"""
class ServerErrorUnknownServerModelId(ServerError):
"""Unknown servermodel ID
"""
src/python/server/server/lib.py 0 → 100644
View file @ 0c523d41
from .error import ServerError
from .query import (list_all_servers,
fetch_server,
fetch_server_dict,
insert_server,
update_server,
update_configuration,
update_peerconnection,
update_environment,
fetch_configuration,
delete_server,
set_automation_value,
erase_server
)
class Server():
"""Server Manage API
"""
#def get_columns_index_by_name(self, cursor) -> dict:
# """convenience method to retrieve the columns by name
# """
# cols = {}
# for index, col in enumerate(cursor.description):
# cols[col.name] = index
# return cols
def list_servers(self, cursor):
"""Fetch Servers from database
:return: list of server object
"""
return list_all_servers(cursor)
def describe_server(self, cursor, serverid):
"""Get server information asynchronously from database
:param `int` serverid: server identifier
"""
return fetch_server_dict(cursor, serverid)
def create_server(self, cursor, servername, serverdescription, servermodelid):
"""Creates a server in database
:param str serverame: server name
:param str serverdescription: servermodel identifier
:param int servermodelid: servermodel identifier
:return: newly created server identifier
:rtype: int
"""
return insert_server(cursor, servername, serverdescription, servermodelid)
def update_server(self, cursor, serverid, servername, serverdescription):