Import from all-in-one zephir repository

Dockerfile

+####
+#### Temporary layer to prepare installation
+####
+FROM ubuntu:bionic AS build
+
+# Packages required to build
+ENV DEBIAN_FRONTEND=noninteractive
+RUN apt-get update -y && apt-get install -y \
+    curl \
+    git
+
+ARG CONTAINERPILOT_VERSION=3.4.3
+ARG CONTAINERPILOT_CHECKSUM=e8258ed166bcb3de3e06638936dcc2cae32c7c58
+
+RUN curl -Lso /tmp/containerpilot.tar.gz \
+         "https://github.com/joyent/containerpilot/releases/download/${CONTAINERPILOT_VERSION}/containerpilot-${CONTAINERPILOT_VERSION}.tar.gz" \
+    && echo "${CONTAINERPILOT_CHECKSUM}  /tmp/containerpilot.tar.gz" | sha1sum -c \
+    && tar zxf /tmp/containerpilot.tar.gz -C /tmp
+
+ARG CONSULTEMPLATE_VERSION=0.19.3
+ARG CONSULTEMPLATE_CHECKSUM=6467b442d8fd68ca0a1eba35f42edb6f3b9941e10b6d58688edc3506fca0bb19
+
+RUN curl -Lso /tmp/consul-template.tar.gz \
+         "https://releases.hashicorp.com/consul-template/${CONSULTEMPLATE_VERSION}/consul-template_${CONSULTEMPLATE_VERSION}_linux_amd64.tgz" \
+    && echo "${CONSULTEMPLATE_CHECKSUM}  /tmp/consul-template.tar.gz" | sha256sum -c \
+    && tar zxf /tmp/consul-template.tar.gz -C /tmp
+
+
+# Orcherstration using containerpilot
+# Configuration using consul-template
+# Use dev-eole repository since gitlab is not public
+ARG CONTAINERPILOT_REPO_URL=https://dev-eole.ac-dijon.fr/git/zephir-orchestrate-containerpilot.git
+ARG CONTAINERPILOT_REPO_REF=223dafd3d093702c8717292247e65d0c55ffb513
+
+RUN git clone "${CONTAINERPILOT_REPO_URL}" "/tmp/orchestrate" \
+    && cd /tmp/orchestrate \
+    && git checkout "${CONTAINERPILOT_REPO_REF}"
+
+
+# API messages description
+# Use dev-eole repository since gitlab is not public
+ARG MESSAGES_API_REPO_URL=https://dev-eole.ac-dijon.fr/git/zephir-messages-api.git
+ARG MESSAGES_API_REPO_REF=e10c743b3a7d1e84ed13bd49a46e7361fd403ccc
+
+RUN git clone "${MESSAGES_API_REPO_URL}" "/tmp/messages-api" \
+    && cd /tmp/messages-api \
+    && git checkout "${MESSAGES_API_REPO_REF}"
+
+
+# Common python Zéphir library
+# Use dev-eole repository since gitlab is not public
+ARG PYTHON_ZEPHIR_REPO_URL=https://dev-eole.ac-dijon.fr/git/python-zephir.git
+ARG PYTHON_ZEPHIR_REPO_REF=bc098b33494c7ef7ae010e130bb773507e196b72
+
+RUN git clone "${PYTHON_ZEPHIR_REPO_URL}" "/tmp/python-zephir" \
+    && cd /tmp/python-zephir \
+    && git checkout "${PYTHON_ZEPHIR_REPO_REF}"
+
+
+####
+#### Target layer
+####
+FROM ubuntu:bionic
+
+MAINTAINER Pôle EOLE <eole@ac-dijon.fr>
+
+# Packages required for working service
+ENV DEBIAN_FRONTEND=noninteractive
+RUN apt-get update -y && apt-get install -y \
+    gnupg \
+    jq \
+    locales \
+    openssl \
+    python3 \
+    python3-aiohttp \
+    python3-autobahn \
+    python3-dev \
+    python3-jwt \
+    python3-psycopg2 \
+    python3-requests \
+    python3-yaml \
+    sqitch \
+    tzdata
+
+# For debug only
+RUN apt-get update -y && apt-get install -y \
+    curl \
+    iproute2 \
+    iputils-ping \
+    vim
+
+# Force french locale
+RUN locale-gen fr_FR.UTF-8
+ENV LANG fr_FR.UTF-8
+ENV LC_ALL fr_FR.UTF-8
+
+# Force timezone
+RUN ln -fs /usr/share/zoneinfo/Europe/Paris /etc/localtime
+RUN dpkg-reconfigure --frontend noninteractive tzdata
+
+# Sqitch
+RUN sqitch config --user user.name 'Equipe EOLE'\
+    && sqitch config --user user.email 'eole@ac-dijon.fr'
+
+# Install tools from build layer
+COPY --from=build /tmp/containerpilot /usr/local/bin
+COPY --from=build /tmp/consul-template /usr/local/bin
+
+# consul-template wrapper to generate /etc/zephir-services.conf
+COPY --from=build /tmp/orchestrate/update-service-conf /usr/local/bin/
+RUN chmod +x /usr/local/bin/*
+
+# Service template for consul-template
+ENV services_conf_filename=zephir-services.conf
+ARG services_conf=/etc/zephir-services.conf
+ENV services_conf=$services_conf
+COPY --from=build /tmp/orchestrate/${services_conf_filename}.ctmpl ${services_conf}.ctmpl
+
+# Install libraries required by service
+COPY --from=build /tmp/python-zephir/zephir /usr/lib/python3/dist-packages/zephir
+COPY --from=build /tmp/messages-api/messages /srv/messages
+
+# Manage container with ContainerPilot
+CMD ["/usr/local/bin/containerpilot", "-config", "/etc/containerpilot.json5"]
+COPY --from=build /tmp/orchestrate/containerpilot.json5 /etc/containerpilot.json5
+
+
+##
+## Service specific
+##
+
+# Library
+COPY src/python/server /usr/lib/python3/dist-packages/server
+
+# Controller
+RUN mkdir -p /srv/bin
+COPY scripts/* /srv/bin/
+COPY migrations /migrations

Dockerfile.Test

+FROM zephir_server-manager:latest
+
+RUN apt-get update -y && apt-get install -y \
+    postgresql \
+    postgresql-contrib \
+    python3-coverage \
+    python3-pytest \
+    python3-pytest-cov
+
+COPY server-manager/test /srv/test
+COPY server-manager/scripts/test /usr/local/bin/test-server
+
+RUN chmod +x /usr/local/bin/test-server
+
+CMD /bin/bash /usr/local/bin/test-server

migrations/deploy/server_schema.sql

+-- Deploy server:server_schema to pg
+
+BEGIN;
+
+
+-- Server table creation
+CREATE TABLE Server (
+  ServerId SERIAL PRIMARY KEY,
+  ServerName VARCHAR(255) NOT NULL UNIQUE,
+  ServerDescription VARCHAR(255) NOT NULL,
+  ServerModelId INTEGER NOT NULL,
+  ZoneId INTEGER,
+  MachineId INTEGER,
+  ServerConfiguration JSON,
+  Automation VARCHAR(25),
+  ServerEnvironment JSON,
+  LastPeerConnection TIMESTAMP DEFAULT NULL
+);
+
+COMMIT;

migrations/revert/server_schema.sql

+-- Revert server:server_schema from pg
+
+BEGIN;
+
+DROP SCHEMA server CASCADE;
+
+COMMIT;

migrations/sqitch.conf

+[core]
+	engine = pg
+	# plan_file = sqitch.plan
+	# top_dir = .
+[engine "pg"]
+	# target = db:pg:
+	registry = sqitch_server
+	# client = psql

migrations/sqitch.plan

+%syntax-version=1.0.0
+%project=server
+
+server_schema 2017-10-09T09:31:30Z Equipe EOLE <eole@ac-dijon.fr> # Definition du schema de base pour le domaine Server

migrations/verify/server_schema.sql

+-- Verify server:server_schema on pg
+
+BEGIN;
+
+SELECT 1/COUNT(*) FROM information_schema.schemata WHERE schema_name = 'server';
+
+ROLLBACK;

scripts/controller

+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+import json
+import datetime
+from autobahn.wamp.exception import ApplicationError
+
+from zephir.i18n import _
+from zephir.controller import ZephirCommonController, run
+from zephir.http import register as register_http
+from zephir.wamp import register as register_wamp
+from server.server.lib import Server
+from server.server.error import (ServerError,
+                                 ServerErrorDatabaseNotAvailable,
+                                 ServerErrorPeeringConfNotAvailable,
+                                 ServerErrorDbConnection,
+                                 ServerErrorInvalidServerId,
+                                 ServerErrorInvalidServerModelId,
+                                 ServerErrorServerNameNotProvided,
+                                 ServerErrorUnknownServerId,
+                                 ServerErrorUnknownServerModelId)
+
+from zephir.config import ServiceConfig
+
+
+class ServerRunner(ZephirCommonController):
+    """Server controller
+
+    """
+    def __init__(self, *args, **kwargs):
+        # Create instance of working code
+        super().__init__(*args, **kwargs)
+        self.server = Server()
+        self.conn = None
+
+    @register_wamp('v1.server.list', notification_uri=None, database=True)
+    async def list_servers(self, cursor):
+        try:
+            return self.server.list_servers(cursor)
+
+        except ServerErrorDatabaseNotAvailable as err:
+            raise ApplicationError('server.error.database-not-available', reason=str(err))
+
+        except ServerErrorDbConnection as err:
+            raise ApplicationError('server.error.db-connection', reason=str(err))
+
+        except ServerError as err:
+            raise ApplicationError('server.error', reason=str(err))
+
+    @register_wamp('v1.server.describe', notification_uri=None, database=True)
+    async def describe_server(self, cursor, serverid, configuration):
+        try:
+            server = self.server.describe_server(cursor, serverid)
+            if configuration:
+                server['configuration'] = serverid
+            return server
+
+        except ServerErrorDatabaseNotAvailable as err:
+            raise ApplicationError('server.error.database-not-available', reason=str(err))
+        except ServerErrorDbConnection as err:
+            raise ApplicationError('server.error.db-connection', reason=str(err))
+        except ServerErrorInvalidServerId as err:
+            raise ApplicationError('server.error.invalid-server-id', reason=str(err))
+        except ServerErrorUnknownServerId as err:
+            raise ApplicationError('server.error.unknown-server-id', reason=str(err))
+        except ServerError as err:
+            raise ApplicationError('server.error', reason=str(err))
+
+    @register_http('v1.server.describe', param='configuration', database=True)
+    async def describe_configuration(self, cursor, secret):
+        values = self.server.fetch_configuration(cursor, secret)
+        if values:
+            return json.dumps(values).encode()
+        else:
+            return b'{}'
+
+    @register_wamp('v1.server.create', notification_uri='v1.server.created', database=True)
+    async def create_server(self, cursor, servername, serverdescription, servermodelid, serverpassphrase):
+        try:
+            result = self.server.create_server(cursor, servername, serverdescription, servermodelid)
+
+            return_code = await self.call('v1.vault.secret.set',
+                                          secretkey="{}_passphrase".format(result['serverid']),
+                                          secret={"passphrase" : serverpassphrase})
+            if return_code:
+                return result
+            else:
+                raise ServerError('put passphrase return code status not available')
+
+        except ServerErrorDatabaseNotAvailable as err:
+            raise ApplicationError('server.error.database-not-available', reason=str(err))
+        except ServerErrorDbConnection as err:
+            raise ApplicationError('server.error.db-connection', reason=str(err))
+        except ServerErrorInvalidServerModelId as err:
+            raise ApplicationError('server.error.invalid-servermodel-id', reason=str(err))
+        except ServerErrorUnknownServerModelId as err:
+            raise ApplicationError('server.error.unknown-servermodel-id', reason=str(err))
+        except ServerErrorServerNameNotProvided as err:
+            raise ApplicationError('server.error.servername-not-provided', reason=str(err))
+        except ServerError as err:
+            raise ApplicationError('server.error', reason=str(err))
+
+    @register_wamp('v1.server.update', notification_uri='v1.server.updated', database=True)
+    async def update_server(self, cursor, serverid, servername, serverdescription):
+        try:
+            return self.server.update_server(cursor, serverid, servername, serverdescription)
+
+        except ServerErrorDatabaseNotAvailable as err:
+            raise ApplicationError('server.error.database-not-available', reason=str(err))
+        except ServerErrorDbConnection as err:
+            raise ApplicationError('server.error.db-connection', reason=str(err))
+        except ServerErrorInvalidServerId as err:
+            raise ApplicationError('server.error.invalid-server-id', reason=str(err))
+        except ServerErrorUnknownServerId as err:
+            raise ApplicationError('server.error.unknown-server-id', reason=str(err))
+        except ServerErrorServerNameNotProvided as err:
+            raise ApplicationError('server.error.servername-not-provided', reason=str(err))
+        except ServerError as err:
+            raise ApplicationError('server.error', reason=str(err))
+
+    @register_wamp('v1.server.delete', notification_uri='v1.server.deleted', database=True)
+    async def delete_server(self, cursor, serverid):
+        try:
+            return self.server.delete_server(cursor, serverid)
+        except ServerErrorDatabaseNotAvailable as err:
+            raise ApplicationError('server.error.database-not-available', reason=str(err))
+        except ServerErrorDbConnection as err:
+            raise ApplicationError('server.error.db-connection', reason=str(err))
+        except ServerErrorInvalidServerId as err:
+            raise ApplicationError('server.error.invalid-server-id', reason=str(err))
+        except ServerErrorUnknownServerId as err:
+            raise ApplicationError('server.error.unknown-server-id', reason=str(err))
+        except ServerError as err:
+            raise ApplicationError('server.error', reason=str(err))
+
+    @register_wamp('v1.server.peering-conf.get', notification_uri='v1.server.peering-conf.sent')
+    async def get_peering_conf(self, serverid):
+        try:
+            secret = await self.call('v1.vault.secret.get', secretkey="{}_peeringconf".format(serverid))
+            return secret['secret']
+        except ServerErrorPeeringConfNotAvailable as err:
+            raise ApplicationError('server.error.peering-conf-not-available', reason=str(err))
+        except ServerError as err:
+            raise ApplicationError('server.error', reason=str(err))
+
+    @register_wamp('v1.execution.salt.master.event.ready', notification_uri='v1.server.environment.updated', database=True)
+    async def minion_ready(self,
+                           cursor,
+                           server_id):
+        environ = await self.call('v1.execution.salt.environment.get',
+                                  server_id=server_id)
+        self.server.update_environment(cursor,
+                                       server_id,
+                                       json.dumps(environ))
+        return {'server_id': server_id}
+
+    @register_wamp('v1.execution.salt.peer.registered', notification_uri='v1.server.salt.registered', database=True)
+    async def salt_register(self, cursor, serverid, automation):
+        try:
+            self.server.register_server_for_automation(cursor, serverid, automation)
+            return {'serverid': serverid}
+        except Exception as err:
+            raise ApplicationError('server.registering.error',
+                                   reason=str(err))
+
+    @register_wamp('v1.server.exec.command', notification_uri='v1.server.executed', database=True)
+    async def exec_cmd_on_server(self, cursor, server_id, command):
+        """
+        Transfer command transmitted to automation (salt, ...)
+        """
+        automation, automation_command = self.server.get_automation_command(cursor, server_id)
+        if automation == 'salt':
+            result = await self.call('v1.execution.salt.exec',
+                                     minion_pattern=str(server_id),
+                                     command=automation_command,
+                                     arg=command,
+                                     client_mode='local_async')
+        else:
+            raise Exception(_('Automation engine not supported: {}').format(automation))
+        if result['minions'] != [str(server_id)]:
+            raise Exception(_('Job ({}) not executed only in selected server, all client affected : {}').format(result['jid'],
+                                                                                                                result['minions']))
+        return {'job_id': result['jid'],
+                'server_id': server_id,
+                'command': command,
+                'automation': automation,
+                'executed': False}
+
+    def result_to_dict(self, result, automation):
+        dico = {'job_id': result['jid'],
+                'server_id': int(result['minion']),
+                'automation': automation,
+                'executed': result['executed']}
+        if result['command'] == 'deploy':
+            dico['command'] = 'v1.server.exec.deploy'
+        else:
+            dico['command'] = result['arg']
+        if dico['executed']:
+            dico['success'] = result['success']
+            dico['retcode'] = result['retcode']
+            dico['return'] = result['return']
+        return dico
+
+    @register_wamp('v1.server.exec.list', notification_uri=None, database=True)
+    async def exec_job_on_server(self, cursor, server_id):
+        automation, automation_command = self.server.get_automation_command(cursor, server_id)
+        if automation == 'salt':
+            results = await self.call('v1.execution.salt.job.list',
+                                      minion_pattern=str(server_id))
+        else:
+            raise Exception(_('Automation engine not supported: {}').format(automation))
+        ret = []
+        for result in results:
+            ret.append(self.result_to_dict(result, automation))
+        return ret
+
+    @register_wamp('v1.server.exec.deploy', notification_uri=None, database=True)
+    async def exec_deploy(self, cursor, server_id):
+        automation, automation_command = self.server.get_automation_command(cursor, server_id)
+        if automation == 'salt':
+            result = await self.call('v1.execution.salt.configuration.deploy',
+                                     minion_pattern=str(server_id))
+        else:
+            raise Exception(_('Automation engine not supported: {}').format(automation))
+        dico = {'job_id': result['jid'],
+                'command': 'v1.server.exec.deploy',
+                'automation': automation,
+                'server_id': server_id,
+                'executed': False}
+        return dico
+
+    @register_wamp('v1.server.exec.describe', notification_uri=None)
+    async def exec_describe(self, job_id, automation):
+        if automation == 'salt':
+            results = await self.call('v1.execution.salt.job.describe',
+                                     jid=str(job_id))
+
+        else:
+            raise Exception(_('Automation engine not supported: {}').format(automation))
+        ret = []
+        for result in results:
+            ret.append(self.result_to_dict(result, automation))
+        return ret
+
+    @register_wamp('v1.config.configuration.server.saved', None, database=True)
+    async def update_configuration(self, cursor, server_id):
+        configuration = await self.call('v1.config.configuration.server.get', server_id=server_id)
+        self.server.update_configuration(cursor, server_id, configuration['configuration'])
+
+    @register_wamp('v1.server.peer-connection.update', notification_uri=None, database=True)
+    async def update_peerconnection(self, cursor, serverid):
+        try:
+            lastpeerconnection = datetime.datetime.now()
+            return self.server.update_peerconnection(cursor, serverid, lastpeerconnection)
+
+        except ServerErrorDatabaseNotAvailable as err:
+            raise ApplicationError('server.error.database-not-available', reason=str(err))
+        except ServerErrorDbConnection as err:
+            raise ApplicationError('server.error.db-connection', reason=str(err))
+        except ServerErrorInvalidServerId as err:
+            raise ApplicationError('server.error.invalid-server-id', reason=str(err))
+        except ServerErrorUnknownServerId as err:
+            raise ApplicationError('server.error.unknown-server-id', reason=str(err))
+        except ServerErrorServerNameNotProvided as err:
+            raise ApplicationError('server.error.servername-not-provided', reason=str(err))
+        except ServerError as err:
+            raise ApplicationError('server.error', reason=str(err))
+
+
+if __name__ == '__main__':
+    run(ServerRunner)

scripts/test

+#!/bin/bash
+set -eo pipefail
+
+if [ -z $ZEPHIR_TEST_TYPE ] && [ ! -z $1 ]; then
+    ZEPHIR_TEST_TYPE=$1
+fi
+
+case $ZEPHIR_TEST_TYPE in
+  "unit")
+    /etc/init.d/postgresql start
+    cd /srv/test/unit
+    su - postgres -c "psql -c \"ALTER USER postgres PASSWORD 'mynewpassword';\""
+    PYTHONPATH='.' /usr/bin/python3-coverage run /usr/bin/py.test-3 -s -x
+    if [ ! -z "$ZEPHIR_TEST_COVERAGE_PATTERN" ]; then
+        /usr/bin/python3-coverage html -d /srv/report --include=$ZEPHIR_TEST_COVERAGE_PATTERN
+        /usr/bin/python3-coverage report --include=$ZEPHIR_TEST_COVERAGE_PATTERN
+    fi
+    ;;
+  "integration")
+    py.test /srv/test/test_server.py -x
+    exit $?
+    ;;
+  "e2e")
+    echo "No e2e tests yet."
+    ;;
+  *)
+    1>&2 echo "Unknown test type: '$ZEPHIR_TEST_TYPE'"
+    exit 1
+esac

src/python/server/server/__init__.py

+from .lib import Server
+
+__all__ = ['Server']

src/python/server/server/error.py

+class ServerError(Exception):
+    """Base class of :class:`Server` exceptions
+    """
+
+
+class ServerErrorPeeringConfNotAvailable(ServerError):
+    """No available peering configuration
+    """
+
+
+class ServerErrorDatabaseNotAvailable(ServerError):
+    """No available database service
+    """
+
+
+class ServerErrorDbConnection(ServerError):
+    """Database connection error
+    """
+
+
+class ServerErrorInvalidServerId(ServerError):
+    """Invalid server ID
+    """
+
+
+class ServerErrorInvalidServerModelId(ServerError):
+    """Invalid servermodel ID
+    """
+
+
+class ServerErrorServerNameNotProvided(ServerError):
+    """Servername expected
+    """
+
+
+class ServerErrorUnknownServerId(ServerError):
+    """Unknown server ID
+    """
+
+
+class ServerErrorUnknownServerModelId(ServerError):
+    """Unknown servermodel ID
+    """

src/python/server/server/lib.py

+from .error import ServerError
+from .query import (list_all_servers,
+                   fetch_server,
+                   fetch_server_dict,
+                   insert_server,
+                   update_server,
+                   update_configuration,
+                   update_peerconnection,
+                   update_environment,
+                   fetch_configuration,
+                   delete_server,
+                   set_automation_value,
+                   erase_server
+                   )
+
+class Server():
+    """Server Manage API
+    """
+    #def get_columns_index_by_name(self, cursor) -> dict:
+    #    """convenience method to retrieve the columns by name
+    #    """
+    #    cols = {}
+    #    for index, col in enumerate(cursor.description):
+    #        cols[col.name] = index
+    #    return cols
+
+    def list_servers(self, cursor):
+        """Fetch Servers from database
+
+        :return: list of server object
+        """
+        return list_all_servers(cursor)
+
+    def describe_server(self, cursor, serverid):
+        """Get server information asynchronously from database
+
+        :param `int` serverid: server identifier
+        """
+        return fetch_server_dict(cursor, serverid)
+
+    def create_server(self, cursor, servername, serverdescription, servermodelid):
+        """Creates a server in database
+
+        :param str serverame: server name
+        :param str serverdescription: servermodel identifier
+        :param int servermodelid: servermodel identifier
+        :return: newly created server identifier
+        :rtype: int
+        """
+        return insert_server(cursor, servername, serverdescription, servermodelid)
+
+    def update_server(self, cursor, serverid, servername, serverdescription):
+        """Updates a server in database