Import from all-in-one zephir repository

Dockerfile

+####
+#### Temporary layer to prepare installation
+####
+FROM ubuntu:bionic AS build
+
+# Packages required to build
+ENV DEBIAN_FRONTEND=noninteractive
+RUN apt-get update -y && apt-get install -y \
+    curl \
+    git \
+    gnupg
+
+ARG CONTAINERPILOT_VERSION=3.4.3
+ARG CONTAINERPILOT_CHECKSUM=e8258ed166bcb3de3e06638936dcc2cae32c7c58
+
+RUN curl -Lso /tmp/containerpilot.tar.gz \
+         "https://github.com/joyent/containerpilot/releases/download/${CONTAINERPILOT_VERSION}/containerpilot-${CONTAINERPILOT_VERSION}.tar.gz" \
+    && echo "${CONTAINERPILOT_CHECKSUM}  /tmp/containerpilot.tar.gz" | sha1sum -c \
+    && tar zxf /tmp/containerpilot.tar.gz -C /tmp
+
+ARG CONSULTEMPLATE_VERSION=0.19.3
+ARG CONSULTEMPLATE_CHECKSUM=6467b442d8fd68ca0a1eba35f42edb6f3b9941e10b6d58688edc3506fca0bb19
+
+RUN curl -Lso /tmp/consul-template.tar.gz \
+         "https://releases.hashicorp.com/consul-template/${CONSULTEMPLATE_VERSION}/consul-template_${CONSULTEMPLATE_VERSION}_linux_amd64.tgz" \
+    && echo "${CONSULTEMPLATE_CHECKSUM}  /tmp/consul-template.tar.gz" | sha256sum -c \
+    && tar zxf /tmp/consul-template.tar.gz -C /tmp
+
+
+# Orcherstration using containerpilot
+# Configuration using consul-template
+# Use dev-eole repository since gitlab is not public
+ARG CONTAINERPILOT_REPO_URL=https://dev-eole.ac-dijon.fr/git/zephir-orchestrate-containerpilot.git
+ARG CONTAINERPILOT_REPO_REF=223dafd3d093702c8717292247e65d0c55ffb513
+
+RUN git clone "${CONTAINERPILOT_REPO_URL}" "/tmp/orchestrate" \
+    && cd /tmp/orchestrate \
+    && git checkout "${CONTAINERPILOT_REPO_REF}"
+
+
+# API messages description
+# Use dev-eole repository since gitlab is not public
+ARG MESSAGES_API_REPO_URL=https://dev-eole.ac-dijon.fr/git/zephir-messages-api.git
+ARG MESSAGES_API_REPO_REF=e10c743b3a7d1e84ed13bd49a46e7361fd403ccc
+
+RUN git clone "${MESSAGES_API_REPO_URL}" "/tmp/messages-api" \
+    && cd /tmp/messages-api \
+    && git checkout "${MESSAGES_API_REPO_REF}"
+
+
+# Common python Zéphir library
+# Use dev-eole repository since gitlab is not public
+ARG PYTHON_ZEPHIR_REPO_URL=https://dev-eole.ac-dijon.fr/git/python-zephir.git
+ARG PYTHON_ZEPHIR_REPO_REF=bc098b33494c7ef7ae010e130bb773507e196b72
+
+RUN git clone "${PYTHON_ZEPHIR_REPO_URL}" "/tmp/python-zephir" \
+    && cd /tmp/python-zephir \
+    && git checkout "${PYTHON_ZEPHIR_REPO_REF}"
+
+
+##
+## Service specific
+##
+
+# Configure package manager
+RUN echo "deb http://repo.saltstack.com/apt/ubuntu/18.04/amd64/latest bionic main" > /etc/apt/sources.list.d/saltstack.list
+RUN curl https://repo.saltstack.com/apt/ubuntu/18.04/amd64/latest/SALTSTACK-GPG-KEY.pub | apt-key --keyring /etc/apt/trusted.gpg.d/saltstack.gpg add -
+
+# SaltStack formulas parameters
+ENV EOLE_FORMULA_DIR=/srv/formulas
+
+ENV EOLE_CONFIGURATION_FORMULA_URL="https://dev-eole.ac-dijon.fr/git/eole-configuration-formula.git"
+ENV EOLE_CONFIGURATION_FORMULA_REF=8c08f4fb902408e320ea1fa2fd74e12fdab60fa3
+
+RUN git clone "${EOLE_CONFIGURATION_FORMULA_URL}" "${EOLE_FORMULA_DIR}/eole-configuration-formula" \
+    && cd "${EOLE_FORMULA_DIR}/eole-configuration-formula" \
+    && git checkout -b deployed "${EOLE_CONFIGURATION_FORMULA_REF}"
+
+
+####
+#### Target layer
+####
+FROM ubuntu:bionic
+
+MAINTAINER Pôle EOLE <eole@ac-dijon.fr>
+
+# Packages required for working service
+ENV DEBIAN_FRONTEND=noninteractive
+RUN apt-get update -y && apt-get install -y \
+    gnupg \
+    jq \
+    locales \
+    openssl \
+    python3 \
+    python3-aiohttp \
+    python3-autobahn \
+    python3-dev \
+    python3-jwt \
+    python3-psycopg2 \
+    python3-requests \
+    python3-yaml \
+    sqitch \
+    tzdata
+
+# For debug only
+RUN apt-get update -y && apt-get install -y \
+    curl \
+    iproute2 \
+    iputils-ping \
+    vim
+
+# Force french locale
+RUN locale-gen fr_FR.UTF-8
+ENV LANG fr_FR.UTF-8
+ENV LC_ALL fr_FR.UTF-8
+
+# Force timezone
+RUN ln -fs /usr/share/zoneinfo/Europe/Paris /etc/localtime
+RUN dpkg-reconfigure --frontend noninteractive tzdata
+
+# Sqitch
+RUN sqitch config --user user.name 'Equipe EOLE'\
+    && sqitch config --user user.email 'eole@ac-dijon.fr'
+
+# Install tools from build layer
+COPY --from=build /tmp/containerpilot /usr/local/bin
+COPY --from=build /tmp/consul-template /usr/local/bin
+
+# consul-template wrapper to generate /etc/zephir-services.conf
+COPY --from=build /tmp/orchestrate/update-service-conf /usr/local/bin/
+RUN chmod +x /usr/local/bin/*
+
+# Service template for consul-template
+ENV services_conf_filename=zephir-services.conf
+ARG services_conf=/etc/zephir-services.conf
+ENV services_conf=$services_conf
+COPY --from=build /tmp/orchestrate/${services_conf_filename}.ctmpl ${services_conf}.ctmpl
+
+# Install libraries required by service
+COPY --from=build /tmp/python-zephir/zephir /usr/lib/python3/dist-packages/zephir
+COPY --from=build /tmp/messages-api/messages /srv/messages
+
+# Manage container with ContainerPilot
+CMD ["/usr/local/bin/containerpilot", "-config", "/etc/containerpilot.json5"]
+COPY containerpilot.json5 /etc/containerpilot.json5
+
+
+##
+## Service specific
+##
+
+COPY --from=build /etc/apt/sources.list.d/saltstack.list /etc/apt/sources.list.d/saltstack.list
+COPY --from=build /etc/apt/trusted.gpg.d/saltstack.gpg /etc/apt/trusted.gpg.d/saltstack.gpg
+
+RUN apt-get update -y && apt-get install -y \
+    netcat \
+    python3-crypto \
+    salt-api \
+    salt-master \
+    screen
+
+# Library for SaltMaster manager
+COPY src/python/zephir/saltmaster /usr/lib/python3/dist-packages/zephir/saltmaster
+
+# Configuration
+COPY master.d/*.conf /etc/salt/master.d/
+
+RUN mkdir /srv/salt
+RUN mkdir /srv/pillar
+RUN mkdir /keys
+
+# Install salt files
+COPY --from=build /srv/formulas/eole-configuration-formula/eole/ /srv/salt/eole/
+RUN mkdir /srv/salt/_modules
+COPY environment/* /srv/salt/_modules/
+COPY pillars/* /srv/pillar
+
+# Controller
+RUN mkdir -p /srv/bin
+COPY scripts/* /srv/bin/
+COPY migrations /migrations
+RUN chmod +x /srv/bin/*
+
+# Create the password
+ENV SALT_PASSWORD=eole
+RUN /srv/bin/create-user.sh

Dockerfile.Test

+FROM zephir_saltmaster-manager:latest
+
+RUN apt-get update -y && apt-get install -y \
+    postgresql \
+    postgresql-contrib \
+    python3-coverage \
+    python3-pytest \
+    python3-pytest-cov
+
+COPY saltmaster-manager/test /srv/test
+COPY saltmaster-manager/scripts/test /usr/local/bin/test-saltmaster
+
+RUN chmod +x /usr/local/bin/test-saltmaster
+
+CMD /bin/bash /usr/local/bin/test-saltmaster

containerpilot.json5

+{
+    consul: {
+        address: "{{ .CONSUL_HOST }}"
+    },
+    logging: {
+      level: '{{ .CONTAINERPILOT_LOGLEVEL | default "debug" }}'
+    },
+    jobs: [
+        {
+            name: "update-service-conf",
+            exec: ["/usr/local/bin/update-service-conf"]
+        },
+        {
+            name: "controller",
+            exec: ["python3", "-u", "/srv/bin/controller"],
+            health: {
+                exec: "pkill -0 -f /srv/bin/controller",
+                interval: 10,
+                ttl: 20
+            },
+            when: {
+                source: "update-service-conf",
+                once: "exitSuccess"
+            },
+            restarts: "unlimited",
+            port: -1
+        },
+        {
+            name: "oncrossbarchange-update-service-conf",
+            exec: ["/usr/local/bin/update-service-conf", "reload"],
+            when: {
+                source: "watch.crossbar",
+                each: "changed"
+            }
+        },
+        {
+            name: "wait-4-db",
+            exec: ["python3", "-u", "/srv/bin/persistent-database"],
+            when: {
+               source: "watch.crossbar",
+               once: "healthy",
+           },
+        },
+        {
+            name: "salt-master",
+            exec: ["salt-master", "-l", '{{ .SALTMASTER_LOG_LEVEL | default "debug" }}'],
+            health: {
+                exec: "/bin/nc -z 127.0.0.1 30009",
+                interval: 10,
+                ttl: 20
+            },
+            when: {
+                source: "wait-4-db",
+                once: "exitSuccess",
+            },
+            port: 30009,
+            restarts: "unlimited"
+        },
+        {
+          name: "salt-api",
+          exec: ["salt-api", "-l", '{{ .SALT_API_LOG_LEVEL | default "debug" }}'],
+          health: {
+            exec: "/bin/nc -z 127.0.0.1 8000",
+            interval: 10,
+            ttl: 20
+          },
+          port: 8000,
+          when: {
+                  source: "salt-master",
+            once: "healthy"
+          },
+          restarts: "unlimited"
+        }
+    ],
+    watches: [
+        {
+            name: "crossbar",
+            interval: 5
+        }
+    ]
+}

environment/zephir.py

+from json import load
+from creole import eosfunc
+from os.path import isfile
+
+FILE_PATH = '/var/lib/eole/zephir/config.env'
+
+def environment():
+    grains = {}
+
+    with open(FILE_PATH, "r") as fhr:
+        probes = load(fhr)
+
+    if isfile(FILE_PATH):
+        for name, probe in probes.items():
+            grains[name] = getattr(eosfunc, probe["function"])(*probe["args"], **probe["kwargs"])
+
+    return grains

master.d/cherrypy.conf

+rest_cherrypy:
+  port: 8000
+  disable_ssl: true
+  #ssl_crt: /etc/pki/tls/certs/localhost.crt
+  #ssl_key: /etc/pki/tls/certs/localhost.key

master.d/ext_auth.conf

+external_auth:
+  pam:
+    eole:
+      - .*
+      - '@wheel'   # to allow access to all wheel modules

master.d/ignore-pyc-files.conf

+# A file glob (or list of file globs) that will be matched against the file
+# path before syncing the modules and states to the minions. This is similar
+# to file_ignore_regex above, but works on globs instead of regex. By default
+# nothing is ignored.
+file_ignore_glob:
+ - '*.pyc'

master.d/publish-port.conf

+# The tcp port used by the publisher
+publish_port: 30009

master.d/returner-pgjsonb.conf

+# Store jobs cache in PostgreSQL
+master_job_cache: pgjsonb
+
+returner.pgjsonb.host: 'postgres'
+returner.pgjsonb.port: 5432
+returner.pgjsonb.db: '{{DBNAME}}'
+returner.pgjsonb.user: '{{USER}}'
+returner.pgjsonb.pass: '{{PASSWORD}}'

migrations/deploy/salt-runner-db.sql

+-- Deploy saltmaster:salt-runner-db to pg
+
+BEGIN;
+
+SET ROLE "saltmaster_admin";
+
+--
+-- Table structure for table `jids`
+--
+CREATE TABLE jids (
+   jid varchar(255) NOT NULL primary key,
+   load jsonb NOT NULL
+);
+CREATE INDEX idx_jids_jsonb on jids
+       USING gin (load)
+       WITH (fastupdate=on);
+
+--
+-- Table structure for table `salt_returns`
+--
+CREATE TABLE salt_returns (
+  fun varchar(50) NOT NULL,
+  jid varchar(255) NOT NULL,
+  return jsonb NOT NULL,
+  id varchar(255) NOT NULL,
+  success varchar(10) NOT NULL,
+  full_ret jsonb NOT NULL,
+  alter_time TIMESTAMP WITH TIME ZONE DEFAULT NOW());
+
+CREATE INDEX idx_salt_returns_id ON salt_returns (id);
+CREATE INDEX idx_salt_returns_jid ON salt_returns (jid);
+CREATE INDEX idx_salt_returns_fun ON salt_returns (fun);
+CREATE INDEX idx_salt_returns_return ON salt_returns
+    USING gin (return) with (fastupdate=on);
+CREATE INDEX idx_salt_returns_full_ret ON salt_returns
+    USING gin (full_ret) with (fastupdate=on);
+
+--
+-- Table structure for table `salt_events`
+--
+CREATE SEQUENCE seq_salt_events_id;
+CREATE TABLE salt_events (
+    id BIGINT NOT NULL UNIQUE DEFAULT nextval('seq_salt_events_id'),
+    tag varchar(255) NOT NULL,
+    data jsonb NOT NULL,
+    alter_time TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
+    master_id varchar(255) NOT NULL);
+
+CREATE INDEX idx_salt_events_tag on
+    salt_events (tag);
+CREATE INDEX idx_salt_events_data ON salt_events
+    USING gin (data) with (fastupdate=on);
+
+COMMIT;

migrations/revert/salt-runner-db.sql

+-- Revert saltmaster:salt-runner-db from pg
+
+BEGIN;
+
+SET ROLE "saltmaster_admin";
+
+-- XXX Add DDLs here.
+
+--
+-- Table structure for table `jids`
+--
+DROP TABLE IF EXISTS jids;
+
+--
+-- Table structure for table `salt_returns`
+--
+DROP TABLE IF EXISTS salt_returns;
+
+COMMIT;

migrations/sqitch.conf

+[core]
+	engine = pg

migrations/sqitch.plan

+%syntax-version=1.0.0
+%project=saltmaster
+
+salt-runner-db 2018-04-10T08:02:27Z Equipe EOLE <eole@ac-dijon.fr> # Add salt-runner

migrations/verify/salt-runner-db.sql

+-- Verify saltmaster:salt-runner-db on pg
+
+BEGIN;
+
+SET ROLE "saltmaster_admin";
+
+-- XXX Add verifications here.
+
+-- Limit verification to existence of tables
+SELECT jid from jids LIMIT 1;
+SELECT jid from salt_returns LIMIT 1;
+SELECT id from salt_events LIMIT 1;
+
+ROLLBACK;

pillars/top.sls

+base:
+  '*':
+    - pillars_{{ grains['id'] }}

scripts/controller

+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+import asyncio
+import signal
+import time
+import json, yaml
+from os import path, makedirs, getpid, unlink
+
+from autobahn.wamp.exception import ApplicationError
+
+from zephir.controller import ZephirCommonController, run
+from zephir.http import register as register_http
+from zephir.wamp import register as register_wamp
+from zephir.coroutine import register as register_coroutine
+
+from zephir.saltmaster.execute import SaltExec
+from zephir.saltmaster.execute import SaltExecErrorMissingMinionPattern
+from zephir.saltmaster.execute import SaltExecErrorMissingCommand
+from zephir.saltmaster.execute import SaltExecErrorLoginError
+from zephir.saltmaster.execute import SaltExecErrorExecuteError
+
+from zephir.saltmaster.jobs import SaltJobs
+from zephir.saltmaster.jobs import SaltJobsErrorDbConnectionError
+from zephir.saltmaster.jobs import SaltJobsErrorListError
+from zephir.saltmaster.jobs import SaltJobsErrorMissingJobId
+from zephir.saltmaster.jobs import SaltJobsErrorJobIdTypeError
+from zephir.saltmaster.jobs import SaltJobsErrorUnknownJobId
+
+from zephir.saltmaster.peering import SaltPeering
+from zephir.saltmaster.events import SaltEvents
+
+
+MINION_DIR = '/srv/salt/eole/configuration/files/'
+
+
+class SaltRunner(ZephirCommonController):
+    """Salt controller based on salt-api
+
+    """
+
+    def __init__(self, *args, **kwargs):
+        # Initialise self.config
+        super().__init__(*args, **kwargs)
+
+        # Create instance of working code
+        self.executor = SaltExec(config=self.config.extra)
+        self.jobs = SaltJobs(config=self.config.extra)
+        self.peers = SaltPeering(config=self.config.extra)
+        self.events = SaltEvents(config=self.config.extra, wamp=self)
+
+
+    @register_wamp('v1.execution.salt.exec', notification_uri='v1.execution.salt.exec.command-executed')
+    async def exec_command(self, minion_pattern, command, arg, client_mode):
+        try:
+            ret = self.executor.exec_command(minion_pattern, command, arg, client_mode)
+            return ret
+
+        except SaltExecErrorMissingMinionPattern as err:
+            raise ApplicationError('execution.salt.exec.error.missing-minion_pattern', reason=str(err))
+
+        except SaltExecErrorMissingCommand as err:
+            raise ApplicationError(u'execution.salt.exec.error.missing-command', reason=str(err))
+
+        except SaltExecErrorLoginError as err:
+            raise ApplicationError('execution.salt.exec.error.login', reason=str(err))
+
+        except SaltExecErrorExecuteError as err:
+            raise ApplicationError('execution.salt.exec.error.execute', reason=str(err))
+
+
+    @register_wamp('v1.execution.salt.job.list', notification_uri=None, database=True)
+    async def list_jobs(self, cursor, minion_pattern):
+        try:
+            return list(self.jobs.list_jobs(cursor, minion_pattern))
+
+        except SaltJobsErrorDbConnectionError as err:
+            raise ApplicationError('execution.salt.job.error.db-connection', reason=str(err))
+
+        except SaltJobsErrorListError as err:
+            raise ApplicationError('execution.salt.job.error.list', reason=str(err))
+
+
+    @register_wamp('v1.execution.salt.job.describe', notification_uri=None, database=True)
+    async def describe_job(self, cursor, jid):
+        try:
+            return self.jobs.describe_job(cursor, jid)
+
+        except SaltJobsErrorDbConnectionError as err:
+            raise ApplicationError('execution.salt.job.error.db-connection', reason=str(err))
+
+        except SaltJobsErrorMissingJobId as err:
+            raise ApplicationError('execution.salt.job.error.missing-jid', reason=str(err))
+
+        except SaltJobsErrorJobIdTypeError as err:
+            raise ApplicationError('execution.salt.job.error.jid-type', reason=str(err))
+
+        except SaltJobsErrorUnknownJobId as err:
+            raise ApplicationError('execution.salt.job.error.unknown-jid', reason=str(err))
+
+    @register_wamp('v1.execution.salt.configuration.deploy',
+                   notification_uri='v1.execution.salt.configuration.deploy-scheduled')
+    async def configuration_deploy(self, minion_pattern):
+        await self.download_configuration_files(int(minion_pattern))
+
+        try:
+            deploy_command = 'state.apply'
+            deploy_arg = 'eole.configuration.deploy'
+
+            return self.executor.exec_command(minion_pattern=minion_pattern,
+                                              command=deploy_command,
+                                              arg=deploy_arg)
+
+        except SaltExecErrorMissingMinionPattern as err:
+            raise ApplicationError('execution.salt.exec.error.missing-minion_pattern', reason=str(err))
+
+        except SaltExecErrorLoginError as err:
+            raise ApplicationError('execution.salt.exec.error.login', reason=str(err))
+
+        except SaltExecErrorExecuteError as err:
+            raise ApplicationError('execution.salt.exec.error.execute', reason=str(err))
+
+    async def server_started(self, server_id):
+        await self.download_env(server_id)
+        self.executor.exec_command(minion_pattern=str(server_id),
+                                   command='state.apply',
+                                   arg='eole.configuration.deploy')
+        # FIXME refresh = true
+        self.executor.exec_command(minion_pattern=str(server_id),
+                                   command='saltutil.sync_all',
+                                   client_mode="local")
+
+    @register_wamp('v1.execution.salt.master.event.start', notification_uri='v1.execution.salt.master.event.ready')
+    async def minion_start(self, server_id):
+        await self.server_started(server_id)
+        await self.call('v1.server.peer-connection.update',
+                        serverid=server_id)
+        return {'server_id': server_id}
+
+    @register_wamp('v1.execution.salt.environment.get', notification_uri=None)
+    async def get_environment(self, server_id):
+        server_id = str(server_id)
+        return self.executor.exec_command(server_id,
+                                          'zephir.environment',
+                                          None,
+                                          'local')[server_id]
+
+    async def download_configuration_files(self,
+                                           server_id):
+        minion_dir = '{0}/{1}'.format(MINION_DIR, server_id)
+        schema_file_path = '{0}/config.schema'.format(minion_dir)
+        config_file_path = '{0}/config.values'.format(minion_dir)
+        pillar_file_path = '/srv/pillar/pillars_{0}.sls'.format(server_id)
+        makedirs(minion_dir, exist_ok=True)
+
+        server = await self.call('v1.server.describe',
+                                 serverid=server_id,
+                                 configuration=True)
+
+        servermodel_id = server['servermodelid']
+
+        with open(config_file_path, 'w') as fh:
+            fh.write(server['configuration'])
+
+        with open(pillar_file_path, 'w') as fh:
+            if server['configuration']:
+                yaml.dump(json.loads(server['configuration']), fh, allow_unicode=True)
+            else:
+                yaml.dump({}, fh)
+
+        del server
+
+        servermodel = await self.call('v1.servermodel.describe',
+                                      servermodelid=servermodel_id,
+                                      inheritance=False,
+                                      resolvdepends=False,
+                                      schema=True,
+                                      probes=True)
+
+        with open(schema_file_path, 'w') as fh:
+            fh.write(servermodel['schema'])
+        self._download_env(server_id, servermodel)
+
+    def _download_env(self, server_id, servermodel):
+        minion_dir = '{0}/{1}'.format(MINION_DIR, server_id)
+        makedirs(minion_dir, exist_ok=True)
+        env_file_path = '{0}/config.env'.format(minion_dir)
+
+        with open(env_file_path, 'w') as fh: