From bf1b6482ce3535ef7e7b3f77879def12ff454c0c Mon Sep 17 00:00:00 2001
From: makefu <github@syntax-fehler.de>
Date: Tue, 22 Dec 2015 19:36:19 +0100
Subject: mv makefu->krebs 3 buildbot

---
 krebs/3modules/buildbot/master.nix | 263 +++++++++++++++++++++++++++++++++++++
 krebs/3modules/buildbot/slave.nix  | 185 ++++++++++++++++++++++++++
 krebs/3modules/default.nix         |   2 +
 3 files changed, 450 insertions(+)
 create mode 100644 krebs/3modules/buildbot/master.nix
 create mode 100644 krebs/3modules/buildbot/slave.nix

(limited to 'krebs/3modules')

diff --git a/krebs/3modules/buildbot/master.nix b/krebs/3modules/buildbot/master.nix
new file mode 100644
index 000000000..2f73e44bc
--- /dev/null
+++ b/krebs/3modules/buildbot/master.nix
@@ -0,0 +1,263 @@
+{ config, pkgs, lib, ... }:
+
+with lib;
+let
+  buildbot = pkgs.buildbot;
+  buildbot-master-config = pkgs.writeText "buildbot-master.cfg" ''
+    # -*- python -*-
+    from buildbot.plugins import *
+    import re
+
+    c = BuildmasterConfig = {}
+
+    c['slaves'] = []
+    # TODO: template potential buildslaves
+    # TODO: set password?
+    slavenames= [ 'testslave' ]
+    for i in slavenames:
+      c['slaves'].append(buildslave.BuildSlave(i, "krebspass"))
+
+    c['protocols'] = {'pb': {'port': 9989}}
+
+    ####### Build Inputs
+    stockholm_repo = 'http://cgit.gum/stockholm'
+    c['change_source'] = []
+    c['change_source'].append(changes.GitPoller(
+            stockholm_repo,
+            workdir='stockholm-poller', branch='master',
+            project='stockholm',
+            pollinterval=120))
+
+    ####### Build Scheduler
+    # TODO: configure scheduler
+    c['schedulers'] = []
+
+    # test the master real quick
+    fast = schedulers.SingleBranchScheduler(
+                                change_filter=util.ChangeFilter(branch="master"),
+                                name="fast-master-test",
+                                builderNames=["fast-tests"])
+
+    force = schedulers.ForceScheduler(
+                                name="force",
+                                builderNames=["full-tests"])
+
+    # files everyone depends on or are part of the share branch
+    def shared_files(change):
+      r =re.compile("^((krebs|share)/.*|Makefile|default.nix)")
+      for file in change.files:
+        if r.match(file):
+          return True
+      return False
+
+    full = schedulers.SingleBranchScheduler(
+                                change_filter=util.ChangeFilter(branch="master"),
+                                fileIsImportant=shared_files,
+                                name="full-master-test",
+                                builderNames=["full-tests"])
+    c['schedulers'] = [ fast, force, full ]
+    ###### The actual build
+    # couple of fast steps:
+    f = util.BuildFactory()
+    ## fetch repo
+    grab_repo = steps.Git(repourl=stockholm_repo, mode='incremental')
+    f.addStep(grab_repo)
+
+    # the dependencies which are used by the test script
+    deps = [ "gnumake", "jq" ]
+    nixshell = ["nix-shell", "-p" ] + deps + [ "--run" ]
+    def addShell(f,**kwargs):
+      f.addStep(steps.ShellCommand(**kwargs))
+
+    addShell(f,name="centos7-eval",env={"LOGNAME": "shared",
+                  "get" : "krebs.deploy",
+                  "filter" : "json"
+                 },
+             command=nixshell + ["make -s eval system=test-centos7"])
+
+    addShell(f,name="wolf-eval",env={"LOGNAME": "shared",
+                  "get" : "krebs.deploy",
+                  "filter" : "json"
+                 },
+             command=nixshell + ["make -s eval system=wolf"])
+
+    c['builders'] = []
+    c['builders'].append(
+        util.BuilderConfig(name="fast-tests",
+          slavenames=slavenames,
+          factory=f))
+
+    # TODO slow build
+    c['builders'].append(
+        util.BuilderConfig(name="full-tests",
+          slavenames=slavenames,
+          factory=f))
+
+    ####### Status of Builds
+    c['status'] = []
+
+    from buildbot.status import html
+    from buildbot.status.web import authz, auth
+    # TODO: configure if http is wanted
+    authz_cfg=authz.Authz(
+        # TODO: configure user/pw
+        auth=auth.BasicAuth([("krebs","bob")]),
+        gracefulShutdown = False,
+        forceBuild = 'auth',
+        forceAllBuilds = 'auth',
+        pingBuilder = False,
+        stopBuild = False,
+        stopAllBuilds = False,
+        cancelPendingBuild = False,
+    )
+    # TODO: configure nginx
+    c['status'].append(html.WebStatus(http_port=8010, authz=authz_cfg))
+
+    from buildbot.status import words
+    ${optionalString (cfg.irc.enable) ''
+      irc = words.IRC("${cfg.irc.server}", "krebsbuild",
+                      # TODO: multiple channels
+                      channels=["${cfg.irc.channel}"],
+                      notify_events={
+                        #'success': 1,
+                        #'failure': 1,
+                        'exception': 1,
+                        'successToFailure': 1,
+                        'failureToSuccess': 1,
+                      }${optionalString cfg.irc.allowForce ",allowForce=True"})
+      c['status'].append(irc)
+      ''}
+
+    ####### PROJECT IDENTITY
+    c['title'] = "Stockholm"
+    c['titleURL'] = "http://krebsco.de"
+
+    #c['buildbotURL'] = "http://buildbot.krebsco.de/"
+    # TODO: configure url
+    c['buildbotURL'] = "http://vbob:8010/"
+
+    ####### DB URL
+    c['db'] = {
+        'db_url' : "sqlite:///state.sqlite",
+    }
+    ${cfg.extraConfig}
+    '';
+
+  cfg = config.krebs.buildbot.master;
+
+  api = {
+    enable = mkEnableOption "Buildbot Master";
+    workDir = mkOption {
+      default = "/var/lib/buildbot/master";
+      type = types.str;
+      description = ''
+        Path to build bot master directory.
+        Will be created on startup.
+      '';
+    };
+    irc = mkOption {
+      default = {};
+      type = types.submodule ({ config, ... }: {
+        options = {
+          enable = mkEnableOption "Buildbot Master IRC Status";
+          channel = mkOption {
+            default = "nix-buildbot-meetup";
+            type = types.str;
+            description = ''
+              irc channel the bot should connect to
+            '';
+          };
+          allowForce = mkOption {
+            default = false;
+            type = types.bool;
+            description = ''
+              Determines if builds can be forced via IRC
+            '';
+          };
+          nick = mkOption {
+            default = "nix-buildbot";
+            type = types.str;
+            description = ''
+              nickname for IRC
+            '';
+          };
+          server = mkOption {
+            default = "irc.freenode.net";
+            type = types.str;
+            description = ''
+              Buildbot Status IRC Server to connect to
+            '';
+          };
+        };
+      });
+    };
+
+    extraConfig = mkOption {
+      default = "";
+      type = types.lines;
+      description = ''
+        extra config appended to the generated master.cfg
+      '';
+    };
+  };
+
+  imp = {
+
+    users.extraUsers.buildbotMaster = {
+      uid = 672626386; #genid buildbotMaster
+      description = "Buildbot Master";
+      home = cfg.workDir;
+      createHome = false;
+    };
+
+    users.extraGroups.buildbotMaster = {
+      gid = 672626386;
+    };
+
+    systemd.services.buildbotMaster = {
+      description = "Buildbot Master";
+      after = [ "network.target" ];
+      wantedBy = [ "multi-user.target" ];
+      path = [ pkgs.git ];
+      serviceConfig = let
+        workdir="${lib.shell.escape cfg.workDir}";
+        # TODO: check if git is the only dep
+      in {
+        PermissionsStartOnly = true;
+        Type = "forking";
+        PIDFile = "${workdir}/twistd.pid";
+        # TODO: maybe also prepare buildbot.tac?
+        ExecStartPre = pkgs.writeScript "buildbot-master-init" ''
+          #!/bin/sh
+          set -efux
+          if [ ! -e ${workdir} ];then
+            mkdir -p ${workdir}
+            ${buildbot}/bin/buildbot create-master -r -l 10 -f ${workdir}
+          fi
+          # always override the master.cfg
+          cp ${buildbot-master-config} ${workdir}/master.cfg
+          # sanity
+          ${buildbot}/bin/buildbot checkconfig ${workdir}
+
+          # TODO: maybe upgrade? not sure about this
+          #       normally we should write buildbot.tac by our own
+          # ${buildbot}/bin/buildbot upgrade-master ${workdir}
+
+          chmod 700 -R ${workdir}
+          chown buildbotMaster:buildbotMaster -R ${workdir}
+        '';
+        ExecStart = "${buildbot}/bin/buildbot start ${workdir}";
+        ExecStop = "${buildbot}/bin/buildbot stop ${workdir}";
+        ExecReload = "${buildbot}/bin/buildbot reconfig ${workdir}";
+        PrivateTmp = "true";
+        User = "buildbotMaster";
+        Restart = "always";
+        RestartSec = "10";
+      };
+    };
+  };
+in
+{
+  options.krebs.buildbot.master = api;
+  config = mkIf cfg.enable imp;
+}
diff --git a/krebs/3modules/buildbot/slave.nix b/krebs/3modules/buildbot/slave.nix
new file mode 100644
index 000000000..65291f63e
--- /dev/null
+++ b/krebs/3modules/buildbot/slave.nix
@@ -0,0 +1,185 @@
+{ config, pkgs, lib, ... }:
+
+with lib;
+let
+  buildbot-slave-init = pkgs.writeText "buildbot-slave.tac" ''
+    import os
+
+    from buildslave.bot import BuildSlave
+    from twisted.application import service
+
+    basedir = '${cfg.workDir}'
+    rotateLength = 10000000
+    maxRotatedFiles = 10
+
+    application = service.Application('buildslave')
+
+    from twisted.python.logfile import LogFile
+    from twisted.python.log import ILogObserver, FileLogObserver
+    logfile = LogFile.fromFullPath(os.path.join(basedir, "twistd.log"), rotateLength=rotateLength,
+                                  maxRotatedFiles=maxRotatedFiles)
+    application.setComponent(ILogObserver, FileLogObserver(logfile).emit)
+
+    buildmaster_host = '${cfg.masterhost}'
+    # TODO: masterport?
+    port = 9989
+    slavename = '${cfg.username}'
+    passwd = '${cfg.password}'
+    keepalive = 600
+    usepty = 0
+    umask = None
+    maxdelay = 300
+    allow_shutdown = None
+
+    ${cfg.extraConfig}
+
+    s = BuildSlave(buildmaster_host, port, slavename, passwd, basedir,
+                  keepalive, usepty, umask=umask, maxdelay=maxdelay,
+                  allow_shutdown=allow_shutdown)
+    s.setServiceParent(application)
+    '';
+  default-packages = [ pkgs.git pkgs.bash ];
+  cfg = config.krebs.buildbot.slave;
+
+  api = {
+    enable = mkEnableOption "Buildbot Slave";
+
+    workDir = mkOption {
+      default = "/var/lib/buildbot/slave";
+      type = types.str;
+      description = ''
+        Path to build bot slave directory.
+        Will be created on startup.
+      '';
+    };
+
+    masterhost = mkOption {
+      default = "localhost";
+      type = types.str;
+      description = ''
+        Hostname/IP of the buildbot master
+      '';
+    };
+
+    username = mkOption {
+      type = types.str;
+      description = ''
+        slavename used to authenticate with master
+      '';
+    };
+
+    password = mkOption {
+      type = types.str;
+      description = ''
+        slave password used to authenticate with master
+      '';
+    };
+
+    contact = mkOption {
+      default = "nix slave <buildslave@${config.networking.hostName}>";
+      type = types.str;
+      description = ''
+        contact to be announced by buildslave
+      '';
+    };
+
+    description = mkOption {
+      default = "Nix Generated BuildSlave";
+      type = types.str;
+      description = ''
+        description for hostto be announced by buildslave
+      '';
+    };
+
+    packages = mkOption {
+      default = [ pkgs.git ];
+      type = with types; listOf package;
+      description = ''
+        packages which should be in path for buildslave
+      '';
+    };
+
+    extraEnviron = mkOption {
+      default = {};
+      example = {
+        NIX_PATH = "nixpkgs=/path/to/my/nixpkgs";
+      };
+      type = types.attrsOf types.str;
+      description = ''
+        extra environment variables to be provided to the buildslave service
+        if you need nixpkgs, e.g. for running nix-shell you can set NIX_PATH here.
+      '';
+    };
+
+    extraConfig = mkOption {
+      default = "";
+      type = types.lines;
+      example = ''
+        port = 443
+        keepalive = 600
+      '';
+      description = ''
+        extra config evaluated before calling BuildSlave init in .tac file
+      '';
+    };
+  };
+
+  imp = {
+
+    users.extraUsers.buildbotSlave = {
+      uid = 1408105834; #genid buildbotMaster
+      description = "Buildbot Slave";
+      home = cfg.workDir;
+      createHome = false;
+    };
+
+    users.extraGroups.buildbotSlave = {
+      gid = 1408105834;
+    };
+
+    systemd.services."buildbotSlave-${cfg.username}-${cfg.masterhost}" = {
+      description = "Buildbot Slave for ${cfg.username}@${cfg.masterhost}";
+      after = [ "network.target" ];
+      wantedBy = [ "multi-user.target" ];
+      path = default-packages ++ cfg.packages;
+
+      environment = {
+          NIX_REMOTE="daemon";
+      } // cfg.extraEnviron;
+
+      serviceConfig = let
+        workdir = "${lib.shell.escape cfg.workDir}";
+        contact = "${lib.shell.escape cfg.contact}";
+        description = "${lib.shell.escape cfg.description}";
+        buildbot = pkgs.buildbot-slave;
+        # TODO:make this
+      in {
+        PermissionsStartOnly = true;
+        Type = "forking";
+        PIDFile = "${workdir}/twistd.pid";
+        # TODO: maybe also prepare buildbot.tac?
+        ExecStartPre = pkgs.writeScript "buildbot-master-init" ''
+          #!/bin/sh
+          set -efux
+          mkdir -p ${workdir}/info
+          cp ${buildbot-slave-init} ${workdir}/buildbot.tac
+          echo ${contact} > ${workdir}/info/admin
+          echo ${description} > ${workdir}/info/host
+
+          chown buildbotSlave:buildbotSlave -R ${workdir}
+          chmod 700 -R ${workdir}
+        '';
+        ExecStart = "${buildbot}/bin/buildslave start ${workdir}";
+        ExecStop = "${buildbot}/bin/buildslave stop ${workdir}";
+        PrivateTmp = "true";
+        User = "buildbotSlave";
+        Restart = "always";
+        RestartSec = "10";
+      };
+    };
+  };
+in
+{
+  options.krebs.buildbot.slave = api;
+  config = mkIf cfg.enable imp;
+}
diff --git a/krebs/3modules/default.nix b/krebs/3modules/default.nix
index 740ba67b8..cbc1291fa 100644
--- a/krebs/3modules/default.nix
+++ b/krebs/3modules/default.nix
@@ -9,6 +9,8 @@ let
       ./apt-cacher-ng.nix
       ./bepasty-server.nix
       ./build.nix
+      ./buildbot/master.nix
+      ./buildbot/slave.nix
       ./current.nix
       ./exim-retiolum.nix
       ./exim-smarthost.nix
-- 
cgit v1.3.1


From 56e8346faa75fc42f65d11ea3569a3e5bdd252ec Mon Sep 17 00:00:00 2001
From: makefu <github@syntax-fehler.de>
Date: Tue, 22 Dec 2015 20:53:11 +0100
Subject: k 5 krebs-ci: remove obsolete trap rm

---
 krebs/3modules/buildbot/master.nix | 2 +-
 krebs/5pkgs/krebs-ci/notes         | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

(limited to 'krebs/3modules')

diff --git a/krebs/3modules/buildbot/master.nix b/krebs/3modules/buildbot/master.nix
index 2f73e44bc..e66e0d6b2 100644
--- a/krebs/3modules/buildbot/master.nix
+++ b/krebs/3modules/buildbot/master.nix
@@ -44,7 +44,7 @@ let
 
     # files everyone depends on or are part of the share branch
     def shared_files(change):
-      r =re.compile("^((krebs|share)/.*|Makefile|default.nix)")
+      r =re.compile("^((krebs|shared)/.*|Makefile|default.nix)")
       for file in change.files:
         if r.match(file):
           return True
diff --git a/krebs/5pkgs/krebs-ci/notes b/krebs/5pkgs/krebs-ci/notes
index 7e34d6a28..f6b193ddb 100755
--- a/krebs/5pkgs/krebs-ci/notes
+++ b/krebs/5pkgs/krebs-ci/notes
@@ -19,7 +19,7 @@ krebs_secrets=$(mktemp -d)
 sec_file=$krebs_secrets/cac_config
 krebs_ssh=$krebs_secrets/tempssh
 # we need to receive this key from buildmaster to speed up tinc bootstrap
-TRAP="rm $sec_file;rm -r $krebs_secrets"
+TRAP="rm -r $krebs_secrets"
 trap "$TRAP" INT TERM EXIT
 
 cat > $sec_file <<EOF
-- 
cgit v1.3.1


From f59080e76f950a5a8e33d1edd4314ffaa14187fc Mon Sep 17 00:00:00 2001
From: makefu <github@syntax-fehler.de>
Date: Wed, 23 Dec 2015 00:06:27 +0100
Subject: m 3 buildbot: add new slow factory to complete integration test

---
 krebs/3modules/buildbot/master.nix | 47 +++++++++++++++++++++++++-------------
 krebs/5pkgs/krebs-ci/notes         |  8 +++----
 2 files changed, 35 insertions(+), 20 deletions(-)

(limited to 'krebs/3modules')

diff --git a/krebs/3modules/buildbot/master.nix b/krebs/3modules/buildbot/master.nix
index e66e0d6b2..0d9c53977 100644
--- a/krebs/3modules/buildbot/master.nix
+++ b/krebs/3modules/buildbot/master.nix
@@ -59,27 +59,28 @@ let
     ###### The actual build
     # couple of fast steps:
     f = util.BuildFactory()
+    # some slow steps
+    s = util.BuildFactory()
     ## fetch repo
     grab_repo = steps.Git(repourl=stockholm_repo, mode='incremental')
     f.addStep(grab_repo)
+    s.addStep(grab_repo)
 
     # the dependencies which are used by the test script
-    deps = [ "gnumake", "jq" ]
-    nixshell = ["nix-shell", "-p" ] + deps + [ "--run" ]
+    deps = [ "gnumake", "jq", "(import <stockholm> {}).pkgs.krebs-ci" ]
+    nixshell = ["nix-shell", "-I", ".", "-p" ] + deps + [ "--run" ]
+
     def addShell(f,**kwargs):
       f.addStep(steps.ShellCommand(**kwargs))
 
-    addShell(f,name="centos7-eval",env={"LOGNAME": "shared",
-                  "get" : "krebs.deploy",
-                  "filter" : "json"
-                 },
-             command=nixshell + ["make -s eval system=test-centos7"])
+    addShell(f,name="centos7-eval",env={"LOGNAME": "shared"},
+             command=nixshell + ["make -s eval get=krebs.deploy filter=json system=test-centos7"])
+
+    addShell(f,name="wolf-eval",env={"LOGNAME": "shared"},
+             command=nixshell + ["make -s eval get=krebs.deploy filter=json system=wolf"])
 
-    addShell(f,name="wolf-eval",env={"LOGNAME": "shared",
-                  "get" : "krebs.deploy",
-                  "filter" : "json"
-                 },
-             command=nixshell + ["make -s eval system=wolf"])
+    addShell(f,name="eval-cross-check",env={"LOGNAME": "shared"},
+             command=nixshell + ["! make eval get=krebs.deploy filter=json system=test-failing"])
 
     c['builders'] = []
     c['builders'].append(
@@ -87,11 +88,20 @@ let
           slavenames=slavenames,
           factory=f))
 
-    # TODO slow build
+    # slave needs 2 files:
+    # * cac.json
+    # * retiolum
+    for file in ["cac.json", "retiolum.rsa_key.priv"]:
+      s.addStep(steps.FileDownload(mastersrc="${cfg.workDir}/{}".format(file),
+                              slavedest=file))
+
+    addShell(s,name="complete-build-centos7",env={"LOGNAME": "shared"},
+             command=nix-shell + ["krebs-ci"])
+
     c['builders'].append(
         util.BuilderConfig(name="full-tests",
           slavenames=slavenames,
-          factory=f))
+          factory=s))
 
     ####### Status of Builds
     c['status'] = []
@@ -119,8 +129,8 @@ let
                       # TODO: multiple channels
                       channels=["${cfg.irc.channel}"],
                       notify_events={
-                        #'success': 1,
-                        #'failure': 1,
+                        'success': 1,
+                        'failure': 1,
                         'exception': 1,
                         'successToFailure': 1,
                         'failureToSuccess': 1,
@@ -221,6 +231,7 @@ let
       path = [ pkgs.git ];
       serviceConfig = let
         workdir="${lib.shell.escape cfg.workDir}";
+        secretsdir="${lib.shell.escape (toString <secrets>)}";
         # TODO: check if git is the only dep
       in {
         PermissionsStartOnly = true;
@@ -236,6 +247,10 @@ let
           fi
           # always override the master.cfg
           cp ${buildbot-master-config} ${workdir}/master.cfg
+          # copy secrets
+          cp ${secretsdir}/cac.json ${workdir}
+          cp ${secretsdir}/retiolum-ci.rsa_key.priv \
+             ${workdir}/retiolum.rsa_key.priv
           # sanity
           ${buildbot}/bin/buildbot checkconfig ${workdir}
 
diff --git a/krebs/5pkgs/krebs-ci/notes b/krebs/5pkgs/krebs-ci/notes
index f6f3da8db..f162656f7 100755
--- a/krebs/5pkgs/krebs-ci/notes
+++ b/krebs/5pkgs/krebs-ci/notes
@@ -19,10 +19,10 @@ fi
 krebs_secrets=$(mktemp -d)
 sec_file=$krebs_secrets/cac_config
 krebs_ssh=$krebs_secrets/tempssh
-cac_resources_cache=$krebs_secrets/res_cache.json
-cac_servers_cache=$krebs_secrets/servers_cache.json
-cac_tasks_cache=$krebs_secrets/tasks_cache.json
-cac_templates_cache=$krebs_secrets/templates_cache.json
+export cac_resources_cache=$krebs_secrets/res_cache.json
+export cac_servers_cache=$krebs_secrets/servers_cache.json
+export cac_tasks_cache=$krebs_secrets/tasks_cache.json
+export cac_templates_cache=$krebs_secrets/templates_cache.json
 # we need to receive this key from buildmaster to speed up tinc bootstrap
 TRAP="rm -r $krebs_secrets;exit"
 trap "$TRAP" INT TERM EXIT
-- 
cgit v1.3.1


From dc8e270d2a5346e4316b7c2050b26fd428ec3fc3 Mon Sep 17 00:00:00 2001
From: makefu <github@syntax-fehler.de>
Date: Wed, 23 Dec 2015 00:06:27 +0100
Subject: m 3 buildbot: add new slow factory to complete integration test

---
 krebs/3modules/buildbot/master.nix | 52 +++++++++++++++++++++++++-------------
 krebs/3modules/buildbot/slave.nix  |  1 +
 krebs/5pkgs/krebs-ci/notes         |  8 +++---
 3 files changed, 40 insertions(+), 21 deletions(-)

(limited to 'krebs/3modules')

diff --git a/krebs/3modules/buildbot/master.nix b/krebs/3modules/buildbot/master.nix
index e66e0d6b2..b4fd6bb2f 100644
--- a/krebs/3modules/buildbot/master.nix
+++ b/krebs/3modules/buildbot/master.nix
@@ -59,27 +59,28 @@ let
     ###### The actual build
     # couple of fast steps:
     f = util.BuildFactory()
+    # some slow steps
+    s = util.BuildFactory()
     ## fetch repo
     grab_repo = steps.Git(repourl=stockholm_repo, mode='incremental')
     f.addStep(grab_repo)
+    s.addStep(grab_repo)
 
     # the dependencies which are used by the test script
-    deps = [ "gnumake", "jq" ]
-    nixshell = ["nix-shell", "-p" ] + deps + [ "--run" ]
+    deps = [ "gnumake", "jq", "(import <stockholm> {}).pkgs.krebs-ci" ]
+    nixshell = ["nix-shell", "-I", "stockholm=.", "-p" ] + deps + [ "--run" ]
+
     def addShell(f,**kwargs):
       f.addStep(steps.ShellCommand(**kwargs))
 
-    addShell(f,name="centos7-eval",env={"LOGNAME": "shared",
-                  "get" : "krebs.deploy",
-                  "filter" : "json"
-                 },
-             command=nixshell + ["make -s eval system=test-centos7"])
+    addShell(f,name="centos7-eval",env={"LOGNAME": "shared"},
+             command=nixshell + ["make -s eval get=krebs.deploy filter=json system=test-centos7"])
+
+    addShell(f,name="wolf-eval",env={"LOGNAME": "shared"},
+             command=nixshell + ["make -s eval get=krebs.deploy filter=json system=wolf"])
 
-    addShell(f,name="wolf-eval",env={"LOGNAME": "shared",
-                  "get" : "krebs.deploy",
-                  "filter" : "json"
-                 },
-             command=nixshell + ["make -s eval system=wolf"])
+    addShell(f,name="eval-cross-check",env={"LOGNAME": "shared"},
+             command=nixshell + ["! make eval get=krebs.deploy filter=json system=test-failing"])
 
     c['builders'] = []
     c['builders'].append(
@@ -87,11 +88,20 @@ let
           slavenames=slavenames,
           factory=f))
 
-    # TODO slow build
+    # slave needs 2 files:
+    # * cac.json
+    # * retiolum
+    for file in ["cac.json", "retiolum.rsa_key.priv"]:
+      s.addStep(steps.FileDownload(mastersrc="${cfg.workDir}/{}".format(file),
+                              slavedest=file))
+
+    addShell(s,name="complete-build-centos7",env={"LOGNAME": "shared"},
+             command=nixshell + ["krebs-ci"])
+
     c['builders'].append(
         util.BuilderConfig(name="full-tests",
           slavenames=slavenames,
-          factory=f))
+          factory=s))
 
     ####### Status of Builds
     c['status'] = []
@@ -106,7 +116,7 @@ let
         forceBuild = 'auth',
         forceAllBuilds = 'auth',
         pingBuilder = False,
-        stopBuild = False,
+        stopBuild = 'auth',
         stopAllBuilds = False,
         cancelPendingBuild = False,
     )
@@ -119,8 +129,8 @@ let
                       # TODO: multiple channels
                       channels=["${cfg.irc.channel}"],
                       notify_events={
-                        #'success': 1,
-                        #'failure': 1,
+                        'success': 1,
+                        'failure': 1,
                         'exception': 1,
                         'successToFailure': 1,
                         'failureToSuccess': 1,
@@ -219,8 +229,12 @@ let
       after = [ "network.target" ];
       wantedBy = [ "multi-user.target" ];
       path = [ pkgs.git ];
+      environment = {
+        SSL_CERT_FILE = "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt";
+      };
       serviceConfig = let
         workdir="${lib.shell.escape cfg.workDir}";
+        secretsdir="${lib.shell.escape (toString <secrets>)}";
         # TODO: check if git is the only dep
       in {
         PermissionsStartOnly = true;
@@ -236,6 +250,10 @@ let
           fi
           # always override the master.cfg
           cp ${buildbot-master-config} ${workdir}/master.cfg
+          # copy secrets
+          cp ${secretsdir}/cac.json ${workdir}
+          cp ${secretsdir}/retiolum-ci.rsa_key.priv \
+             ${workdir}/retiolum.rsa_key.priv
           # sanity
           ${buildbot}/bin/buildbot checkconfig ${workdir}
 
diff --git a/krebs/3modules/buildbot/slave.nix b/krebs/3modules/buildbot/slave.nix
index 65291f63e..8711a287a 100644
--- a/krebs/3modules/buildbot/slave.nix
+++ b/krebs/3modules/buildbot/slave.nix
@@ -144,6 +144,7 @@ let
       path = default-packages ++ cfg.packages;
 
       environment = {
+          SSL_CERT_FILE = "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt";
           NIX_REMOTE="daemon";
       } // cfg.extraEnviron;
 
diff --git a/krebs/5pkgs/krebs-ci/notes b/krebs/5pkgs/krebs-ci/notes
index f6f3da8db..f162656f7 100755
--- a/krebs/5pkgs/krebs-ci/notes
+++ b/krebs/5pkgs/krebs-ci/notes
@@ -19,10 +19,10 @@ fi
 krebs_secrets=$(mktemp -d)
 sec_file=$krebs_secrets/cac_config
 krebs_ssh=$krebs_secrets/tempssh
-cac_resources_cache=$krebs_secrets/res_cache.json
-cac_servers_cache=$krebs_secrets/servers_cache.json
-cac_tasks_cache=$krebs_secrets/tasks_cache.json
-cac_templates_cache=$krebs_secrets/templates_cache.json
+export cac_resources_cache=$krebs_secrets/res_cache.json
+export cac_servers_cache=$krebs_secrets/servers_cache.json
+export cac_tasks_cache=$krebs_secrets/tasks_cache.json
+export cac_templates_cache=$krebs_secrets/templates_cache.json
 # we need to receive this key from buildmaster to speed up tinc bootstrap
 TRAP="rm -r $krebs_secrets;exit"
 trap "$TRAP" INT TERM EXIT
-- 
cgit v1.3.1


From 14ddb767eb10dbe43d3112c4b4674f6c1d4ff32a Mon Sep 17 00:00:00 2001
From: makefu <github@syntax-fehler.de>
Date: Wed, 23 Dec 2015 11:18:00 +0100
Subject: k 5 mv krebs-ci test/infest-cac-centos7

---
 krebs/3modules/buildbot/master.nix              |   6 +-
 krebs/5pkgs/default.nix                         |   4 +
 krebs/5pkgs/krebs-ci/default.nix                |  37 --------
 krebs/5pkgs/krebs-ci/notes                      | 116 -----------------------
 krebs/5pkgs/test/infest-cac-centos7/default.nix |  39 ++++++++
 krebs/5pkgs/test/infest-cac-centos7/notes       | 117 ++++++++++++++++++++++++
 6 files changed, 163 insertions(+), 156 deletions(-)
 delete mode 100644 krebs/5pkgs/krebs-ci/default.nix
 delete mode 100755 krebs/5pkgs/krebs-ci/notes
 create mode 100644 krebs/5pkgs/test/infest-cac-centos7/default.nix
 create mode 100755 krebs/5pkgs/test/infest-cac-centos7/notes

(limited to 'krebs/3modules')

diff --git a/krebs/3modules/buildbot/master.nix b/krebs/3modules/buildbot/master.nix
index b4fd6bb2f..483ba18e7 100644
--- a/krebs/3modules/buildbot/master.nix
+++ b/krebs/3modules/buildbot/master.nix
@@ -67,7 +67,7 @@ let
     s.addStep(grab_repo)
 
     # the dependencies which are used by the test script
-    deps = [ "gnumake", "jq", "(import <stockholm> {}).pkgs.krebs-ci" ]
+    deps = [ "gnumake", "jq", "(import <stockholm> {}).pkgs.test.infest-cac-centos7" ]
     nixshell = ["nix-shell", "-I", "stockholm=.", "-p" ] + deps + [ "--run" ]
 
     def addShell(f,**kwargs):
@@ -95,8 +95,8 @@ let
       s.addStep(steps.FileDownload(mastersrc="${cfg.workDir}/{}".format(file),
                               slavedest=file))
 
-    addShell(s,name="complete-build-centos7",env={"LOGNAME": "shared"},
-             command=nixshell + ["krebs-ci"])
+    addShell(s,name="infest-cac-centos7",env={"LOGNAME": "shared"},
+             command=nixshell + ["infest-cac-centos7"])
 
     c['builders'].append(
         util.BuilderConfig(name="full-tests",
diff --git a/krebs/5pkgs/default.nix b/krebs/5pkgs/default.nix
index 7df7b7d3c..0562fe836 100644
--- a/krebs/5pkgs/default.nix
+++ b/krebs/5pkgs/default.nix
@@ -40,6 +40,10 @@ subdirs // rec {
       }
     '';
 
+  test = {
+    infest-cac-centos7 = pkgs.callPackage ./test/infest-cac-centos7 {};
+  };
+
   execveBin = name: cfg: execve name (cfg // { destination = "/bin/${name}"; });
 
   writeC = name: { destination ? "" }: src: pkgs.runCommand name {} ''
diff --git a/krebs/5pkgs/krebs-ci/default.nix b/krebs/5pkgs/krebs-ci/default.nix
deleted file mode 100644
index f5b302b52..000000000
--- a/krebs/5pkgs/krebs-ci/default.nix
+++ /dev/null
@@ -1,37 +0,0 @@
-{ stdenv, coreutils,makeWrapper, cac, cacpanel, gnumake, gnused, jq, openssh, ... }:
-
-stdenv.mkDerivation rec {
-  name = "krebs-ci-0.1.0";
-
-  src = ./notes;
-
-  phases = [
-    "installPhase"
-  ];
-  buildInputs = [ makeWrapper ];
-
-  path = stdenv.lib.makeSearchPath "bin" [
-    coreutils
-    cac
-    cacpanel
-    gnumake
-    gnused
-    jq
-    openssh
-  ];
-
-  installPhase =
-    ''
-      mkdir -p $out/bin
-      cp ${src} $out/bin/krebs-ci
-      chmod +x $out/bin/krebs-ci
-      wrapProgram $out/bin/krebs-ci \
-              --prefix PATH : ${path}
-    '';
-  meta = with stdenv.lib; {
-    homepage = http://krebsco.de;
-    description = "Krebs CI Scripts";
-    license = licenses.wtfpl;
-    maintainers = [ maintainers.makefu ];
-  };
-}
diff --git a/krebs/5pkgs/krebs-ci/notes b/krebs/5pkgs/krebs-ci/notes
deleted file mode 100755
index f162656f7..000000000
--- a/krebs/5pkgs/krebs-ci/notes
+++ /dev/null
@@ -1,116 +0,0 @@
-#! /bin/sh
-
-# nix-shell -p gnumake jq openssh cac cacpanel
-set -eufx
-
-# 2 secrets are required:
-
-krebs_cred=${krebs_cred-./cac.json}
-retiolum_key=${retiolum_key-./retiolum.rsa_key.priv}
-
-# Sanity
-if test ! -r "$krebs_cred";then
-  echo "\$krebs_cred=$krebs_cred must be readable"; exit 1
-fi
-if test ! -r "$retiolum_key";then
-  echo "\$retiolum_key=$retiolum_key must be readable"; exit 1
-fi
-
-krebs_secrets=$(mktemp -d)
-sec_file=$krebs_secrets/cac_config
-krebs_ssh=$krebs_secrets/tempssh
-export cac_resources_cache=$krebs_secrets/res_cache.json
-export cac_servers_cache=$krebs_secrets/servers_cache.json
-export cac_tasks_cache=$krebs_secrets/tasks_cache.json
-export cac_templates_cache=$krebs_secrets/templates_cache.json
-# we need to receive this key from buildmaster to speed up tinc bootstrap
-TRAP="rm -r $krebs_secrets;exit"
-trap "$TRAP" INT TERM EXIT
-
-cat > $sec_file <<EOF
-cac_login="$(jq -r .email $krebs_cred)"
-cac_key="$(cac-cli panel --config $krebs_cred settings | jq -r .apicode)"
-EOF
-
-export cac_secrets=$sec_file
-cac-cli panel --config $krebs_cred update-api-ip
-
-# test login:
-cac update
-cac servers
-
-# Template 26: CentOS7
-# TODO: use cac templates to determine the real Centos7 template in case it changes
-name=$( cac build cpu=1 ram=512 storage=10 os=26 2>&1\
-  | jq -r .servername)
-
-id=servername:$name
-trap "cac delete $id;$TRAP" INT TERM EXIT
-# TODO: timeout?
-# cac_always_update=true cac waitstatus $id "Powered On"
-
-wait_login_cac(){
-  # timeout
-  for t in `seq 180`;do
-    # now we have a working cac server
-    if cac ssh $1 cat /etc/redhat-release | \
-      grep CentOS ;then
-      return 0
-    fi
-    sleep 10
-  done
-  return 1
-}
-# die on timeout
-wait_login_cac $id
-
-mkdir -p shared/2configs/temp
-cac generatenetworking $id > \
-  shared/2configs/temp/networking.nix
-# new temporary ssh key we will use to log in after infest
-ssh-keygen -f $krebs_ssh -N ""
-cp $retiolum_key $krebs_secrets/retiolum.rsa_key.priv
-# we override the directories for secrets and stockholm
-# additionally we set the ssh key we generated
-ip=$(cac getserver $id | jq -r .ip)
-
-cat > shared/2configs/temp/dirs.nix <<EOF
-_: {
-  krebs.build.source.dir = {
-    secrets.path = "$krebs_secrets";
-    stockholm.path = "$(pwd)";
-  };
-  users.extraUsers.root.openssh.authorizedKeys.keys = [
-    "$(cat ${krebs_ssh}.pub)"
-  ];
-  krebs.build.target = "$ip";
-}
-EOF
-
-LOGNAME=shared make eval get=krebs.infest \
-  target=derp system=test-centos7 filter=json \
-  | sed -e "s#^ssh.*<<#cac ssh $id<<#" \
-        -e "/^rsync/a -e 'cac ssh $id' \\\\"  \
-        -e "s#root.derp:#:#" > $krebs_secrets/infest
-sh -x $krebs_secrets/infest
-
-# TODO: generate secrets directory $krebs_secrets for nix import
-cac powerop $id reset
-
-wait_login(){
-  # timeout
-  for t in `seq 20`;do
-    # now we have a working cac server
-    if ssh -o StrictHostKeyChecking=no \
-           -o UserKnownHostsFile=/dev/null \
-           -i $krebs_ssh \
-           -o ConnectTimeout=10 \
-           -o BatchMode=yes \
-           root@$1 nixos-version ;then
-      return 0
-    fi
-    sleep 10
-  done
-  return 1
-}
-wait_login $ip
diff --git a/krebs/5pkgs/test/infest-cac-centos7/default.nix b/krebs/5pkgs/test/infest-cac-centos7/default.nix
new file mode 100644
index 000000000..7f2e3f231
--- /dev/null
+++ b/krebs/5pkgs/test/infest-cac-centos7/default.nix
@@ -0,0 +1,39 @@
+{ stdenv, coreutils,makeWrapper, cac, cacpanel, gnumake, gnused, jq, openssh, ... }:
+
+stdenv.mkDerivation rec {
+  name = "${shortname}-${version}";
+  shortname = "infest-cac-centos7";
+  version = "0.2.0";
+
+  src = ./notes;
+
+  phases = [
+    "installPhase"
+  ];
+  buildInputs = [ makeWrapper ];
+
+  path = stdenv.lib.makeSearchPath "bin" [
+    coreutils
+    cac
+    cacpanel
+    gnumake
+    gnused
+    jq
+    openssh
+  ];
+
+  installPhase =
+    ''
+      mkdir -p $out/bin
+      cp ${src} $out/bin/${shortname}
+      chmod +x $out/bin/${shortname}
+      wrapProgram $out/bin/${shortname} \
+              --prefix PATH : ${path}
+    '';
+  meta = with stdenv.lib; {
+    homepage = http://krebsco.de;
+    description = "Krebs CI Scripts";
+    license = licenses.wtfpl;
+    maintainers = [ maintainers.makefu ];
+  };
+}
diff --git a/krebs/5pkgs/test/infest-cac-centos7/notes b/krebs/5pkgs/test/infest-cac-centos7/notes
new file mode 100755
index 000000000..1e350084c
--- /dev/null
+++ b/krebs/5pkgs/test/infest-cac-centos7/notes
@@ -0,0 +1,117 @@
+#! /bin/sh
+
+# nix-shell -p gnumake jq openssh cac cacpanel
+set -eufx
+
+# 2 secrets are required:
+
+krebs_cred=${krebs_cred-./cac.json}
+retiolum_key=${retiolum_key-./retiolum.rsa_key.priv}
+
+# Sanity
+if test ! -r "$krebs_cred";then
+  echo "\$krebs_cred=$krebs_cred must be readable"; exit 1
+fi
+if test ! -r "$retiolum_key";then
+  echo "\$retiolum_key=$retiolum_key must be readable"; exit 1
+fi
+
+krebs_secrets=$(mktemp -d)
+sec_file=$krebs_secrets/cac_config
+krebs_ssh=$krebs_secrets/tempssh
+export cac_resources_cache=$krebs_secrets/res_cache.json
+export cac_servers_cache=$krebs_secrets/servers_cache.json
+export cac_tasks_cache=$krebs_secrets/tasks_cache.json
+export cac_templates_cache=$krebs_secrets/templates_cache.json
+# we need to receive this key from buildmaster to speed up tinc bootstrap
+TRAP="rm -r $krebs_secrets;trap - INT TERM EXIT"
+trap "$TRAP" INT TERM EXIT
+
+cat > $sec_file <<EOF
+cac_login="$(jq -r .email $krebs_cred)"
+cac_key="$(cac-cli panel --config $krebs_cred settings | jq -r .apicode)"
+EOF
+
+export cac_secrets=$sec_file
+cac-cli panel --config $krebs_cred update-api-ip
+
+# test login:
+cac update
+cac servers
+
+# Template 26: CentOS7
+# TODO: use cac templates to determine the real Centos7 template in case it changes
+name=$( cac build cpu=1 ram=512 storage=10 os=26 2>&1\
+  | jq -r .servername)
+
+id=servername:$name
+trap "cac delete $id;$TRAP;exit" INT TERM EXIT
+# TODO: timeout?
+
+wait_login_cac(){
+  # timeout
+  for t in `seq 180`;do
+    # now we have a working cac server
+    if cac ssh $1 -o ConnectTimeout=10 \
+                  -o BatchMode=yes \
+                  cat /etc/redhat-release | \
+                    grep CentOS ;then
+      return 0
+    fi
+    sleep 10
+  done
+  return 1
+}
+# die on timeout
+wait_login_cac $id
+
+mkdir -p shared/2configs/temp
+cac generatenetworking $id > \
+  shared/2configs/temp/networking.nix
+# new temporary ssh key we will use to log in after infest
+ssh-keygen -f $krebs_ssh -N ""
+cp $retiolum_key $krebs_secrets/retiolum.rsa_key.priv
+# we override the directories for secrets and stockholm
+# additionally we set the ssh key we generated
+ip=$(cac getserver $id | jq -r .ip)
+
+cat > shared/2configs/temp/dirs.nix <<EOF
+_: {
+  krebs.build.source.dir = {
+    secrets.path = "$krebs_secrets";
+    stockholm.path = "$(pwd)";
+  };
+  users.extraUsers.root.openssh.authorizedKeys.keys = [
+    "$(cat ${krebs_ssh}.pub)"
+  ];
+  krebs.build.target = "$ip";
+}
+EOF
+
+LOGNAME=shared make eval get=krebs.infest \
+  target=derp system=test-centos7 filter=json \
+  | sed -e "s#^ssh.*<<#cac ssh $id<<#" \
+        -e "/^rsync/a -e 'cac ssh $id' \\\\"  \
+        -e "s#root.derp:#:#" > $krebs_secrets/infest
+sh -x $krebs_secrets/infest
+
+# TODO: generate secrets directory $krebs_secrets for nix import
+cac powerop $id reset
+
+wait_login(){
+  # timeout
+  for t in `seq 20`;do
+    # now we have a working cac server
+    if ssh -o StrictHostKeyChecking=no \
+           -o UserKnownHostsFile=/dev/null \
+           -i $krebs_ssh \
+           -o ConnectTimeout=10 \
+           -o BatchMode=yes \
+           root@$1 nixos-version ;then
+      return 0
+    fi
+    sleep 10
+  done
+  return 1
+}
+wait_login $ip
-- 
cgit v1.3.1


From 9a386718714d70f7100b5de297dfd0869d98e47b Mon Sep 17 00:00:00 2001
From: makefu <github@syntax-fehler.de>
Date: Wed, 23 Dec 2015 16:06:41 +0100
Subject: k 3 buildbot: fix merge fuckup

---
 krebs/3modules/buildbot/master.nix | 5 -----
 1 file changed, 5 deletions(-)

(limited to 'krebs/3modules')

diff --git a/krebs/3modules/buildbot/master.nix b/krebs/3modules/buildbot/master.nix
index 19aecead1..483ba18e7 100644
--- a/krebs/3modules/buildbot/master.nix
+++ b/krebs/3modules/buildbot/master.nix
@@ -95,13 +95,8 @@ let
       s.addStep(steps.FileDownload(mastersrc="${cfg.workDir}/{}".format(file),
                               slavedest=file))
 
-<<<<<<< HEAD
     addShell(s,name="infest-cac-centos7",env={"LOGNAME": "shared"},
              command=nixshell + ["infest-cac-centos7"])
-=======
-    addShell(s,name="complete-build-centos7",env={"LOGNAME": "shared"},
-             command=nix-shell + ["krebs-ci"])
->>>>>>> f59080e76f950a5a8e33d1edd4314ffaa14187fc
 
     c['builders'].append(
         util.BuilderConfig(name="full-tests",
-- 
cgit v1.3.1


From 1ef9af2c9a49490a2dda21884ad761675c520d1a Mon Sep 17 00:00:00 2001
From: makefu <github@syntax-fehler.de>
Date: Wed, 23 Dec 2015 16:20:27 +0100
Subject: k 3 buildbot/master: send sigterm before sigkill for cleanup

---
 krebs/3modules/buildbot/master.nix | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

(limited to 'krebs/3modules')

diff --git a/krebs/3modules/buildbot/master.nix b/krebs/3modules/buildbot/master.nix
index 483ba18e7..6ce708769 100644
--- a/krebs/3modules/buildbot/master.nix
+++ b/krebs/3modules/buildbot/master.nix
@@ -95,8 +95,10 @@ let
       s.addStep(steps.FileDownload(mastersrc="${cfg.workDir}/{}".format(file),
                               slavedest=file))
 
-    addShell(s,name="infest-cac-centos7",env={"LOGNAME": "shared"},
-             command=nixshell + ["infest-cac-centos7"])
+    addShell(s, name="infest-cac-centos7",env={"LOGNAME": "shared"},
+                sigtermTime=60, # SIGTERM 1 minute before SIGKILL
+                timeout=5400,   # 1.5h timeout
+                command=nixshell + ["infest-cac-centos7"])
 
     c['builders'].append(
         util.BuilderConfig(name="full-tests",
-- 
cgit v1.3.1


From 6b7506dc672b4bd658088bf37fad06fd64c777fe Mon Sep 17 00:00:00 2001
From: makefu <github@syntax-fehler.de>
Date: Wed, 23 Dec 2015 16:37:02 +0100
Subject: k 3 buildbot: add rsync as explicit dep

do not be pure yet
---
 krebs/3modules/buildbot/master.nix | 17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)

(limited to 'krebs/3modules')

diff --git a/krebs/3modules/buildbot/master.nix b/krebs/3modules/buildbot/master.nix
index 6ce708769..6bf3fda2c 100644
--- a/krebs/3modules/buildbot/master.nix
+++ b/krebs/3modules/buildbot/master.nix
@@ -67,19 +67,24 @@ let
     s.addStep(grab_repo)
 
     # the dependencies which are used by the test script
-    deps = [ "gnumake", "jq", "(import <stockholm> {}).pkgs.test.infest-cac-centos7" ]
+    deps = [ "gnumake", "jq","nix",
+             "(import <stockholm> {}).pkgs.test.infest-cac-centos7",
+             "rsync" ]
+    # TODO: --pure , prepare ENV in nix-shell command:
+    #                   SSL_CERT_FILE,LOGNAME,NIX_REMOTE
     nixshell = ["nix-shell", "-I", "stockholm=.", "-p" ] + deps + [ "--run" ]
-
+    env = {"LOGNAME": "shared",
+           "NIX_REMOTE": "daemon"}
     def addShell(f,**kwargs):
       f.addStep(steps.ShellCommand(**kwargs))
 
-    addShell(f,name="centos7-eval",env={"LOGNAME": "shared"},
+    addShell(f,name="centos7-eval",env=env,
              command=nixshell + ["make -s eval get=krebs.deploy filter=json system=test-centos7"])
 
-    addShell(f,name="wolf-eval",env={"LOGNAME": "shared"},
+    addShell(f,name="wolf-eval",env=env,
              command=nixshell + ["make -s eval get=krebs.deploy filter=json system=wolf"])
 
-    addShell(f,name="eval-cross-check",env={"LOGNAME": "shared"},
+    addShell(f,name="eval-cross-check",env=env,
              command=nixshell + ["! make eval get=krebs.deploy filter=json system=test-failing"])
 
     c['builders'] = []
@@ -95,7 +100,7 @@ let
       s.addStep(steps.FileDownload(mastersrc="${cfg.workDir}/{}".format(file),
                               slavedest=file))
 
-    addShell(s, name="infest-cac-centos7",env={"LOGNAME": "shared"},
+    addShell(s, name="infest-cac-centos7",env=env,
                 sigtermTime=60, # SIGTERM 1 minute before SIGKILL
                 timeout=5400,   # 1.5h timeout
                 command=nixshell + ["infest-cac-centos7"])
-- 
cgit v1.3.1


From 3adf78473d2deff0b991d7222e928fa2888529f6 Mon Sep 17 00:00:00 2001
From: makefu <github@syntax-fehler.de>
Date: Thu, 24 Dec 2015 00:02:59 +0100
Subject: k 3 buildbot.master: refactor

see buildbot-standalone.nix in shared/2configs for the current buildbot config
---
 krebs/3modules/buildbot/master.nix      | 341 ++++++++++++++++++++------------
 shared/2configs/buildbot-standalone.nix | 107 +++++++++-
 2 files changed, 317 insertions(+), 131 deletions(-)

(limited to 'krebs/3modules')

diff --git a/krebs/3modules/buildbot/master.nix b/krebs/3modules/buildbot/master.nix
index 6bf3fda2c..7078000fe 100644
--- a/krebs/3modules/buildbot/master.nix
+++ b/krebs/3modules/buildbot/master.nix
@@ -7,134 +7,81 @@ let
     # -*- python -*-
     from buildbot.plugins import *
     import re
-
+    import json
     c = BuildmasterConfig = {}
 
     c['slaves'] = []
-    # TODO: template potential buildslaves
-    # TODO: set password?
-    slavenames= [ 'testslave' ]
-    for i in slavenames:
-      c['slaves'].append(buildslave.BuildSlave(i, "krebspass"))
+    slaves = json.loads('${builtins.toJSON cfg.slaves}')
+    slavenames = [ s for s in slaves ]
+    for k,v in slaves.items():
+      c['slaves'].append(buildslave.BuildSlave(k, v))
 
+    # TODO: configure protocols?
     c['protocols'] = {'pb': {'port': 9989}}
 
     ####### Build Inputs
-    stockholm_repo = 'http://cgit.gum/stockholm'
-    c['change_source'] = []
-    c['change_source'].append(changes.GitPoller(
-            stockholm_repo,
-            workdir='stockholm-poller', branch='master',
-            project='stockholm',
-            pollinterval=120))
+    c['change_source'] = cs = []
+
+    ${ concatStringsSep "\n"
+    (mapAttrsToList (n: v: ''
+        #### Change_Source: Begin of ${n}
+        ${v}
+        #### Change_Source: End of ${n}
+      '') cfg.change_source )}
 
     ####### Build Scheduler
-    # TODO: configure scheduler
-    c['schedulers'] = []
-
-    # test the master real quick
-    fast = schedulers.SingleBranchScheduler(
-                                change_filter=util.ChangeFilter(branch="master"),
-                                name="fast-master-test",
-                                builderNames=["fast-tests"])
-
-    force = schedulers.ForceScheduler(
-                                name="force",
-                                builderNames=["full-tests"])
-
-    # files everyone depends on or are part of the share branch
-    def shared_files(change):
-      r =re.compile("^((krebs|shared)/.*|Makefile|default.nix)")
-      for file in change.files:
-        if r.match(file):
-          return True
-      return False
-
-    full = schedulers.SingleBranchScheduler(
-                                change_filter=util.ChangeFilter(branch="master"),
-                                fileIsImportant=shared_files,
-                                name="full-master-test",
-                                builderNames=["full-tests"])
-    c['schedulers'] = [ fast, force, full ]
-    ###### The actual build
-    # couple of fast steps:
-    f = util.BuildFactory()
-    # some slow steps
-    s = util.BuildFactory()
-    ## fetch repo
-    grab_repo = steps.Git(repourl=stockholm_repo, mode='incremental')
-    f.addStep(grab_repo)
-    s.addStep(grab_repo)
-
-    # the dependencies which are used by the test script
-    deps = [ "gnumake", "jq","nix",
-             "(import <stockholm> {}).pkgs.test.infest-cac-centos7",
-             "rsync" ]
-    # TODO: --pure , prepare ENV in nix-shell command:
-    #                   SSL_CERT_FILE,LOGNAME,NIX_REMOTE
-    nixshell = ["nix-shell", "-I", "stockholm=.", "-p" ] + deps + [ "--run" ]
-    env = {"LOGNAME": "shared",
-           "NIX_REMOTE": "daemon"}
-    def addShell(f,**kwargs):
-      f.addStep(steps.ShellCommand(**kwargs))
-
-    addShell(f,name="centos7-eval",env=env,
-             command=nixshell + ["make -s eval get=krebs.deploy filter=json system=test-centos7"])
-
-    addShell(f,name="wolf-eval",env=env,
-             command=nixshell + ["make -s eval get=krebs.deploy filter=json system=wolf"])
-
-    addShell(f,name="eval-cross-check",env=env,
-             command=nixshell + ["! make eval get=krebs.deploy filter=json system=test-failing"])
-
-    c['builders'] = []
-    c['builders'].append(
-        util.BuilderConfig(name="fast-tests",
-          slavenames=slavenames,
-          factory=f))
-
-    # slave needs 2 files:
-    # * cac.json
-    # * retiolum
-    for file in ["cac.json", "retiolum.rsa_key.priv"]:
-      s.addStep(steps.FileDownload(mastersrc="${cfg.workDir}/{}".format(file),
-                              slavedest=file))
-
-    addShell(s, name="infest-cac-centos7",env=env,
-                sigtermTime=60, # SIGTERM 1 minute before SIGKILL
-                timeout=5400,   # 1.5h timeout
-                command=nixshell + ["infest-cac-centos7"])
-
-    c['builders'].append(
-        util.BuilderConfig(name="full-tests",
-          slavenames=slavenames,
-          factory=s))
-
-    ####### Status of Builds
-    c['status'] = []
-
-    from buildbot.status import html
-    from buildbot.status.web import authz, auth
-    # TODO: configure if http is wanted
-    authz_cfg=authz.Authz(
-        # TODO: configure user/pw
-        auth=auth.BasicAuth([("krebs","bob")]),
-        gracefulShutdown = False,
-        forceBuild = 'auth',
-        forceAllBuilds = 'auth',
-        pingBuilder = False,
-        stopBuild = 'auth',
-        stopAllBuilds = False,
-        cancelPendingBuild = False,
-    )
-    # TODO: configure nginx
-    c['status'].append(html.WebStatus(http_port=8010, authz=authz_cfg))
-
-    from buildbot.status import words
+    c['schedulers'] = sched = []
+
+    ${ concatStringsSep "\n"
+    (mapAttrsToList (n: v: ''
+        #### Schedulers: Begin of ${n}
+        ${v}
+        #### Schedulers: End of ${n}
+      '') cfg.scheduler )}
+
+    ###### Builder
+    c['builders'] = bu = []
+    
+    # Builder Pre: Begin
+    ${cfg.builder_pre}
+    # Builder Pre: End
+
+    ${ concatStringsSep "\n"
+    (mapAttrsToList (n: v: ''
+        #### Builder: Begin of ${n}
+        ${v}
+        #### Builder: End of ${n}
+      '') cfg.builder )}
+
+
+    ####### Status
+    c['status'] = st = []
+
+    # If you want to configure this url, override with extraConfig
+    c['buildbotURL'] = "http://${config.networking.hostName}:${toString cfg.web.port}/"
+
+    ${optionalString (cfg.web.enable) ''
+      from buildbot.status import html
+      from buildbot.status.web import authz, auth
+      authz_cfg=authz.Authz(
+          auth=auth.BasicAuth([ ("${cfg.web.username}","${cfg.web.password}") ]),
+          # TODO: configure harder
+          gracefulShutdown = False,
+          forceBuild = 'auth',
+          forceAllBuilds = 'auth',
+          pingBuilder = False,
+          stopBuild = 'auth',
+          stopAllBuilds = 'auth',
+          cancelPendingBuild = 'auth'
+      )
+      # TODO: configure krebs.nginx
+      st.append(html.WebStatus(http_port=${toString cfg.web.port}, authz=authz_cfg))
+      ''}
+
     ${optionalString (cfg.irc.enable) ''
-      irc = words.IRC("${cfg.irc.server}", "krebsbuild",
-                      # TODO: multiple channels
-                      channels=["${cfg.irc.channel}"],
+      from buildbot.status import words
+      irc = words.IRC("${cfg.irc.server}", "${cfg.irc.nick}",
+                      channels=${builtins.toJSON cfg.irc.channels},
                       notify_events={
                         'success': 1,
                         'failure': 1,
@@ -145,15 +92,20 @@ let
       c['status'].append(irc)
       ''}
 
+    ${ concatStringsSep "\n"
+    (mapAttrsToList (n: v: ''
+        #### Status: Begin of ${n}
+        ${v}
+        #### Status: End of ${n}
+      '') cfg.status )}
+
     ####### PROJECT IDENTITY
-    c['title'] = "Stockholm"
+    c['title'] = "${cfg.title}"
     c['titleURL'] = "http://krebsco.de"
 
-    #c['buildbotURL'] = "http://buildbot.krebsco.de/"
-    # TODO: configure url
-    c['buildbotURL'] = "http://vbob:8010/"
 
     ####### DB URL
+    # TODO: configure
     c['db'] = {
         'db_url' : "sqlite:///state.sqlite",
     }
@@ -164,6 +116,13 @@ let
 
   api = {
     enable = mkEnableOption "Buildbot Master";
+    title = mkOption {
+      default = "Buildbot CI";
+      type = types.str;
+      description = ''
+        Title of the Buildbot Installation
+      '';
+    };
     workDir = mkOption {
       default = "/var/lib/buildbot/master";
       type = types.str;
@@ -172,16 +131,144 @@ let
         Will be created on startup.
       '';
     };
+
+    slaves = mkOption {
+      default = {};
+      type = types.attrsOf types.str;
+      description = ''
+        Attrset of slavenames with their passwords
+        slavename = slavepassword
+      '';
+    };
+
+    change_source = mkOption {
+      default = {};
+      type = types.attrsOf types.str;
+      example = {
+        stockholm = ''
+          cs.append(changes.GitPoller(
+                  'http://cgit.gum/stockholm',
+                  workdir='stockholm-poller', branch='master',
+                  project='stockholm',
+                  pollinterval=120))
+        '';
+      };
+      description = ''
+        Attrset of all the change_sources which should be configured.
+        It will be directly included into the master configuration.
+
+        At the end an change object should be appended to <literal>cs</literal>
+      '';
+    };
+
+    scheduler = mkOption {
+      default = {};
+      type = types.attrsOf types.str;
+      example = {
+        force-scheduler = ''
+          sched.append(schedulers.ForceScheduler(
+                                      name="force",
+                                      builderNames=["full-tests"]))
+        '';
+      };
+      description = ''
+        Attrset of all the schedulers which should be configured.
+        It will be directly included into the master configuration.
+
+        At the end an change object should be appended to <literal>sched</literal>
+      '';
+    };
+
+    builder_pre = mkOption {
+      default = "";
+      type = types.lines;
+      example = ''
+        grab_repo = steps.Git(repourl=stockholm_repo, mode='incremental')
+      '';
+      description = ''
+        some code before the builders are being assembled.
+        can be used to define functions used by multiple builders
+      '';
+    };
+
+    builder = mkOption {
+      default = {};
+      type = types.attrsOf types.str;
+      example = {
+        fast-test = ''
+        '';
+      };
+      description = ''
+        Attrset of all the builder which should be configured.
+        It will be directly included into the master configuration.
+
+        At the end an change object should be appended to <literal>bu</literal>
+      '';
+    };
+
+    status = mkOption {
+      default = {};
+      type = types.attrsOf types.str;
+      description = ''
+        Attrset of all the extra status which should be configured.
+        It will be directly included into the master configuration.
+
+        At the end an change object should be appended to <literal>st</literal>
+
+        Right now IRC and Web status can be configured by setting
+        <literal>buildbot.master.irc.enable</literal> and
+        <literal>buildbot.master.web.enable</literal>
+      '';
+    };
+
+    # Configurable Stati
+    web = mkOption {
+      default = {};
+      type = types.submodule ({ config2, ... }: {
+        options = {
+          enable = mkEnableOption "Buildbot Master Web Status";
+          username = mkOption {
+            default = "krebs";
+            type = types.str;
+            description = ''
+              username for web authentication
+            '';
+          };
+          hostname = mkOption {
+            default = config.networking.hostName;
+            type = types.str;
+            description = ''
+              web interface Hostname
+            '';
+          };
+          password = mkOption {
+            default = "bob";
+            type = types.str;
+            description = ''
+              password for web authentication
+            '';
+          };
+          port = mkOption {
+            default = 8010;
+            type = types.int;
+            description = ''
+              port for buildbot web status
+            '';
+          };
+        };
+      });
+    };
+
     irc = mkOption {
       default = {};
       type = types.submodule ({ config, ... }: {
         options = {
           enable = mkEnableOption "Buildbot Master IRC Status";
-          channel = mkOption {
-            default = "nix-buildbot-meetup";
-            type = types.str;
+          channels = mkOption {
+            default = [ "nix-buildbot-meetup" ];
+            type = with types; listOf str;
             description = ''
-              irc channel the bot should connect to
+              irc channels the bot should connect to
             '';
           };
           allowForce = mkOption {
@@ -235,6 +322,7 @@ let
       description = "Buildbot Master";
       after = [ "network.target" ];
       wantedBy = [ "multi-user.target" ];
+      # TODO: add extra dependencies to master like svn and cvs
       path = [ pkgs.git ];
       environment = {
         SSL_CERT_FILE = "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt";
@@ -242,7 +330,6 @@ let
       serviceConfig = let
         workdir="${lib.shell.escape cfg.workDir}";
         secretsdir="${lib.shell.escape (toString <secrets>)}";
-        # TODO: check if git is the only dep
       in {
         PermissionsStartOnly = true;
         Type = "forking";
diff --git a/shared/2configs/buildbot-standalone.nix b/shared/2configs/buildbot-standalone.nix
index adf44cada..baab059c9 100644
--- a/shared/2configs/buildbot-standalone.nix
+++ b/shared/2configs/buildbot-standalone.nix
@@ -8,16 +8,115 @@ in {
   };
   networking.firewall.allowedTCPPorts = [ 8010 ];
   krebs.buildbot.master = {
+    slaves = {
+      testslave =  "krebspass";
+      testslave2 = "krebspass";
+    };
+    change_source.stockholm = ''
+  stockholm_repo = 'http://cgit.gum/stockholm'
+  cs.append(changes.GitPoller(
+          stockholm_repo,
+          workdir='stockholm-poller', branch='master',
+          project='stockholm',
+          pollinterval=120))
+    '';
+    scheduler = {
+        force-scheduler = ''
+  sched.append(schedulers.ForceScheduler(
+                              name="force",
+                              builderNames=["full-tests"]))
+        '';
+        fast-tests-scheduler = ''
+  # test the master real quick
+  sched.append(schedulers.SingleBranchScheduler(
+                              change_filter=util.ChangeFilter(branch="master"),
+                              name="fast-master-test",
+                              builderNames=["fast-tests"]))
+        '';
+        full-master-scheduler = ''
+  # files everyone depends on or are part of the share branch
+  def shared_files(change):
+    r =re.compile("^((krebs|shared)/.*|Makefile|default.nix)")
+    for file in change.files:
+      if r.match(file):
+        return True
+    return False
+
+  sched.append(schedulers.SingleBranchScheduler(
+                              change_filter=util.ChangeFilter(branch="master"),
+                              fileIsImportant=shared_files,
+                              name="full-master-test",
+                              builderNames=["full-tests"]))
+        '';
+    };
+    builder_pre = ''
+  # prepare grab_repo step for stockholm
+  stockholm_repo = "http://cgit.gum.retiolum/stockholm"
+  grab_repo = steps.Git(repourl=stockholm_repo, mode='incremental')
+
+  env = {"LOGNAME": "shared", "NIX_REMOTE": "daemon"}
+
+  # prepare nix-shell
+  # the dependencies which are used by the test script
+  deps = [ "gnumake", "jq","nix","rsync",
+            "(import <stockholm> {}).pkgs.test.infest-cac-centos7" ]
+  # TODO: --pure , prepare ENV in nix-shell command:
+  #                   SSL_CERT_FILE,LOGNAME,NIX_REMOTE
+  nixshell = ["nix-shell", "-I", "stockholm=.", "-p" ] + deps + [ "--run" ]
+
+  # prepare addShell function
+  def addShell(factory,**kwargs):
+    factory.addStep(steps.ShellCommand(**kwargs))
+    '';
+    builder = {
+      fast-tests = ''
+  f = util.BuildFactory()
+  f.addStep(grab_repo)
+  addShell(f,name="centos7-eval",env=env,
+            command=nixshell + ["make -s eval get=krebs.deploy filter=json system=test-centos7"])
+
+  addShell(f,name="wolf-eval",env=env,
+            command=nixshell + ["make -s eval get=krebs.deploy filter=json system=wolf"])
+
+  addShell(f,name="eval-cross-check",env=env,
+            command=nixshell + ["! make eval get=krebs.deploy filter=json system=test-failing"])
+
+  bu.append(util.BuilderConfig(name="fast-tests",
+        slavenames=slavenames,
+        factory=f))
+      '';
+      slow-tests = ''
+  s = util.BuildFactory()
+  s.addStep(grab_repo)
+
+  # slave needs 2 files:
+  # * cac.json
+  # * retiolum
+  for file in ["cac.json", "retiolum.rsa_key.priv"]:
+    s.addStep(steps.FileDownload(mastersrc="${config.krebs.buildbot.master.workDir}/{}".format(file),
+                            slavedest=file))
+
+  addShell(s, name="infest-cac-centos7",env=env,
+              sigtermTime=60,           # SIGTERM 1 minute before SIGKILL
+              timeout=5400,             # 1.5h timeout
+              command=nixshell + ["infest-cac-centos7"])
+
+  bu.append(util.BuilderConfig(name="full-tests",
+        slavenames=slavenames,
+        factory=s))
+      '';
+    };
     enable = true;
+    web = {
+      enable = true;
+    };
     irc = {
       enable = true;
+      nick = "shared-buildbot";
       server = "cd.retiolum";
-      channel = "retiolum";
+      channels = [ "retiolum" ];
       allowForce = true;
     };
-    extraConfig = ''
-      c['buildbotURL']  = "http://${config.krebs.build.host.name}:8010/"
-    '';
   };
 
   krebs.buildbot.slave = {
-- 
cgit v1.3.1


From cef2be532b0cc76071b0b3515fc71214b37591f0 Mon Sep 17 00:00:00 2001
From: makefu <github@syntax-fehler.de>
Date: Sat, 26 Dec 2015 10:44:56 +0100
Subject: m 3 Reaktor: add workdir/state_dir

---
 krebs/3modules/Reaktor.nix | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

(limited to 'krebs/3modules')

diff --git a/krebs/3modules/Reaktor.nix b/krebs/3modules/Reaktor.nix
index 1ec49b81e..d219d1800 100644
--- a/krebs/3modules/Reaktor.nix
+++ b/krebs/3modules/Reaktor.nix
@@ -62,6 +62,14 @@ let
         configuration appended to the default or overridden configuration
       '';
     };
+
+    workdir = mkOption {
+      default = "/var/lib/Reaktor";
+      type = types.str;
+      description = ''
+        Reaktor working directory
+      '';
+    };
     extraEnviron = mkOption {
       default = {};
       type = types.attrsOf types.str;
@@ -91,7 +99,7 @@ let
       # uid = config.ids.uids.Reaktor;
       uid = 2066439104; #genid Reaktor
       description = "Reaktor user";
-      home = "/var/lib/Reaktor";
+      home = cfg.workdir;
       createHome = true;
     };
 
@@ -113,6 +121,7 @@ let
         GIT_SSL_CAINFO = "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt";
         REAKTOR_NICKNAME = cfg.nickname;
         REAKTOR_DEBUG = (if cfg.debug  then "True" else "False");
+        state_dir = cfg.workdir;
         } // cfg.extraEnviron;
       serviceConfig= {
         ExecStartPre = pkgs.writeScript "Reaktor-init" ''
-- 
cgit v1.3.1


From 8f98ae9842963d801945c850e9da1e450e098ce3 Mon Sep 17 00:00:00 2001
From: makefu <github@syntax-fehler.de>
Date: Sat, 26 Dec 2015 10:54:02 +0100
Subject: m 3 buildbot/master: use genid

---
 krebs/3modules/buildbot/master.nix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'krebs/3modules')

diff --git a/krebs/3modules/buildbot/master.nix b/krebs/3modules/buildbot/master.nix
index 7078000fe..5870c3145 100644
--- a/krebs/3modules/buildbot/master.nix
+++ b/krebs/3modules/buildbot/master.nix
@@ -308,7 +308,7 @@ let
   imp = {
 
     users.extraUsers.buildbotMaster = {
-      uid = 672626386; #genid buildbotMaster
+      uid = genid "buildbotMaster";
       description = "Buildbot Master";
       home = cfg.workDir;
       createHome = false;
-- 
cgit v1.3.1


From 7bed1761bdbfc3fc7e2df56dcf069511eec2a97d Mon Sep 17 00:00:00 2001
From: makefu <github@syntax-fehler.de>
Date: Sat, 26 Dec 2015 11:41:41 +0100
Subject: m 3 Reaktor: now supports plugin infra see m/1/pornocauster

---
 krebs/3modules/Reaktor.nix       |  5 ++++-
 krebs/5pkgs/Reaktor/plugins.nix  |  4 ++--
 makefu/1systems/pornocauster.nix | 13 +++++++------
 3 files changed, 13 insertions(+), 9 deletions(-)

(limited to 'krebs/3modules')

diff --git a/krebs/3modules/Reaktor.nix b/krebs/3modules/Reaktor.nix
index 59058bffc..607eb7cac 100644
--- a/krebs/3modules/Reaktor.nix
+++ b/krebs/3modules/Reaktor.nix
@@ -9,6 +9,7 @@ let
       ${cfg.overrideConfig}
       '' else ""}
       ## Extra Config
+      ${concatStringsSep "\n" (map (plug: plug.config) cfg.plugins)}
       ${cfg.extraConfig}
     '';
   cfg = config.krebs.Reaktor;
@@ -35,7 +36,6 @@ let
       '';
     };
 
-
     overrideConfig = mkOption {
       default = null;
       type = types.nullOr types.str;
@@ -44,6 +44,9 @@ let
         Reaktor default cfg can be retrieved via `reaktor get-config`
       '';
     };
+    plugins = mkOption {
+      default = [pkgs.ReaktorPlugins.nixos-version];
+    };
     extraConfig = mkOption {
       default = "";
       type = types.string;
diff --git a/krebs/5pkgs/Reaktor/plugins.nix b/krebs/5pkgs/Reaktor/plugins.nix
index b1a61d3fa..5c7b89f5c 100644
--- a/krebs/5pkgs/Reaktor/plugins.nix
+++ b/krebs/5pkgs/Reaktor/plugins.nix
@@ -32,7 +32,7 @@ rec {
                           ''indirect_pattern.format("${name}")'' else
                           ''"${pattern}"'' },
           'argv'    : ["${src_file}"],
-          'env'     : ${builtins.toJSON (path_env // env)})})
+          'env'     : ${builtins.toJSON (path_env // env)} })
       '';
       config_file = pkgs.writeText "plugin.py" config;
     in buildBaseReaktorPlugin (attrs // rec {
@@ -81,7 +81,7 @@ rec {
   stockholm-issue = buildSimpleReaktorPlugin "stockholm-issue" {
     script = ./scripts/random-issue.sh;
     path = with pkgs; [ git gnused lentil ];
-    env = { "origin"= "http://cgit.gum/stockholm"; };
+    env = { "origin" = "http://cgit.gum/stockholm"; };
   };
 
   titlebot =
diff --git a/makefu/1systems/pornocauster.nix b/makefu/1systems/pornocauster.nix
index 28b77d330..690e26b36 100644
--- a/makefu/1systems/pornocauster.nix
+++ b/makefu/1systems/pornocauster.nix
@@ -26,6 +26,7 @@
       # services
       ../2configs/git/brain-retiolum.nix
       ../2configs/tor.nix
+      # ../2configs/buildbot-standalone.nix
 
       # hardware specifics are in here
       ../2configs/hw/tp-x220.nix
@@ -36,14 +37,14 @@
     ];
   nixpkgs.config.packageOverrides = pkgs: {
     tinc = pkgs.tinc_pre;
-    buildbot = let
-      pkgs1509 = import (fetchTarball https://github.com/NixOS/nixpkgs-channels/archive/nixos-unstable.tar.gz) {};
-      in pkgs1509.buildbot;
   };
-  makefu.buildbot.master.enable = true;
 
-  #krebs.Reaktor.enable = true;
-  #krebs.Reaktor.nickname = "makefu|r";
+  krebs.Reaktor = {
+    enable = true;
+    nickname = "makefu|r";
+    plugins = with pkgs.ReaktorPlugins; [ nixos-version random-emoji ];
+  };
+
   # nix.binaryCaches = [ "http://acng.shack/nixos" "https://cache.nixos.org" ];
 
   environment.systemPackages = with pkgs;[
-- 
cgit v1.3.1


From 5a9ccbef0abe1f3acb16d716a2e1d7faa9bb0af1 Mon Sep 17 00:00:00 2001
From: tv <tv@krebsco.de>
Date: Mon, 28 Dec 2015 19:43:31 +0100
Subject: {tv 2 => krebs 3} backup

---
 krebs/3modules/backup.nix  | 286 +++++++++++++++++++++++++++++++++++++++++++++
 krebs/3modules/default.nix |   1 +
 krebs/4lib/types.nix       |  17 +++
 tv/1systems/cd.nix         |   1 -
 tv/1systems/xu.nix         |   1 -
 tv/2configs/backup.nix     | 268 +++++-------------------------------------
 tv/2configs/default.nix    |   1 +
 7 files changed, 333 insertions(+), 242 deletions(-)
 create mode 100644 krebs/3modules/backup.nix

(limited to 'krebs/3modules')

diff --git a/krebs/3modules/backup.nix b/krebs/3modules/backup.nix
new file mode 100644
index 000000000..01bb16a2b
--- /dev/null
+++ b/krebs/3modules/backup.nix
@@ -0,0 +1,286 @@
+{ config, lib, pkgs, ... }:
+with lib;
+let
+  out = {
+    options.krebs.backup = api;
+    config = mkIf cfg.enable imp;
+  };
+
+  cfg = config.krebs.backup;
+
+  api = {
+    enable = mkEnableOption "krebs.backup" // { default = true; };
+    plans = mkOption {
+      default = {};
+      type = types.attrsOf (types.submodule ({
+        # TODO enable = mkEnableOption "TODO" // { default = true; };
+        options = {
+          method = mkOption {
+            type = types.enum ["pull" "push"];
+          };
+          name = mkOption {
+            type = types.str;
+          };
+          src = mkOption {
+            type = types.krebs.file-location;
+          };
+          dst = mkOption {
+            type = types.krebs.file-location;
+          };
+          startAt = mkOption {
+            type = types.str;
+          };
+          snapshots = mkOption {
+            type = types.attrsOf (types.submodule {
+              options = {
+                format = mkOption {
+                  type = types.str; # TODO date's +FORMAT
+                };
+                retain = mkOption {
+                  type = types.nullOr types.int;
+                  default = null; # null = retain all snapshots
+                };
+              };
+            });
+          };
+        };
+      }));
+    };
+  };
+
+  imp = {
+    users.groups.backup.gid = genid "backup";
+    users.users = {}
+      // {
+        root.openssh.authorizedKeys.keys =
+          map (plan: plan.dst.host.ssh.pubkey)
+              (filter isPullSrc (attrValues cfg.plans))
+          ++
+          map (plan: plan.src.host.ssh.pubkey)
+              (filter isPushDst (attrValues cfg.plans))
+          ;
+      }
+      ;
+    systemd.services =
+      flip mapAttrs' (filterAttrs (_:isPullDst) cfg.plans) (name: plan: {
+        name = "backup.${name}.pull";
+        value = makePullService plan;
+      })
+      //
+      flip mapAttrs' (filterAttrs (_:isPushSrc) cfg.plans) (name: plan: {
+        name = "backup.${name}.push";
+        value = makePushService plan;
+      })
+      ;
+  };
+
+  isPushSrc = plan:
+    plan.method == "push" &&
+    plan.src.host.name == config.krebs.build.host.name;
+
+  isPullSrc = plan:
+    plan.method == "pull" &&
+    plan.src.host.name == config.krebs.build.host.name;
+
+  isPushDst = plan:
+    plan.method == "push" &&
+    plan.dst.host.name == config.krebs.build.host.name;
+
+  isPullDst = plan:
+    plan.method == "pull" &&
+    plan.dst.host.name == config.krebs.build.host.name;
+
+  # TODO push destination needs this in the dst.user's PATH
+  service-path = [
+    pkgs.coreutils
+    pkgs.gnused
+    pkgs.openssh
+    pkgs.rsync
+    pkgs.utillinux
+  ];
+
+  # TODO if there is plan.user, then use its privkey
+  makePushService = plan: assert isPushSrc plan; {
+    path = service-path;
+    serviceConfig = {
+      ExecStart = push plan;
+      Type = "oneshot";
+    };
+    startAt = plan.startAt;
+  };
+
+  makePullService = plan: assert isPullDst plan; {
+    path = service-path;
+    serviceConfig = {
+      ExecStart = pull plan;
+      Type = "oneshot";
+    };
+    startAt = plan.startAt;
+  };
+
+  push = plan: let
+    # We use writeDashBin and return the absolute path so systemd will produce
+    # nice names in the log, i.e. without the Nix store hash.
+    out = "${main}/bin/${main.name}";
+
+    main = writeDashBin "backup.${plan.name}.push" ''
+      set -efu
+      dst=${shell.escape plan.dst.path}
+
+      mkdir -m 0700 -p "$dst"
+      exec flock -n "$dst" ${critical-section}
+    '';
+
+    critical-section = writeDash "backup.${plan.name}.push.critical-section" ''
+      # TODO check if there is a previous
+      set -efu
+      identity=${shell.escape plan.src.host.ssh.privkey.path}
+      src=${shell.escape plan.src.path}
+      dst_target=${shell.escape "root@${getFQDN plan.dst.host}"}
+      dst_path=${shell.escape plan.dst.path}
+      dst=$dst_target:$dst_path
+
+      # Export NOW so runtime of rsync doesn't influence snapshot naming.
+      export NOW
+      NOW=$(date +%s)
+
+      echo >&2 "update snapshot: current; $src -> $dst"
+      rsync >&2 \
+          -aAXF --delete \
+          -e "ssh -F /dev/null -i $identity" \
+          --rsync-path ${shell.escape
+            "mkdir -m 0700 -p ${shell.escape plan.dst.path} && rsync"} \
+          --link-dest="$dst_path/current" \
+          "$src/" \
+          "$dst/.partial"
+
+      exec ssh -F /dev/null \
+          -i "$identity" \
+          "$dst_target" \
+          -T \
+          env NOW="$NOW" /bin/sh < ${remote-snapshot}
+      EOF
+    '';
+
+    remote-snapshot = writeDash "backup.${plan.name}.push.remote-snapshot" ''
+      set -efu
+      dst=${shell.escape plan.dst.path}
+
+      if test -e "$dst/current"; then
+        mv "$dst/current" "$dst/.previous"
+      fi
+      mv "$dst/.partial" "$dst/current"
+      rm -fR "$dst/.previous"
+      echo >&2
+
+      (${(take-snapshots plan).text})
+    '';
+
+  in out;
+
+  # TODO admit plan.dst.user and its ssh identity
+  pull = plan: let
+    # We use writeDashBin and return the absolute path so systemd will produce
+    # nice names in the log, i.e. without the Nix store hash.
+    out = "${main}/bin/${main.name}";
+
+    main = writeDashBin "backup.${plan.name}.pull" ''
+      set -efu
+      dst=${shell.escape plan.dst.path}
+
+      mkdir -m 0700 -p "$dst"
+      exec flock -n "$dst" ${critical-section}
+    '';
+
+    critical-section = writeDash "backup.${plan.name}.pull.critical-section" ''
+      # TODO check if there is a previous
+      set -efu
+      identity=${shell.escape plan.dst.host.ssh.privkey.path}
+      src=${shell.escape "root@${getFQDN plan.src.host}:${plan.src.path}"}
+      dst=${shell.escape plan.dst.path}
+
+      # Export NOW so runtime of rsync doesn't influence snapshot naming.
+      export NOW
+      NOW=$(date +%s)
+
+      echo >&2 "update snapshot: current; $dst <- $src"
+      mkdir -m 0700 -p ${shell.escape plan.dst.path}
+      rsync >&2 \
+          -aAXF --delete \
+          -e "ssh -F /dev/null -i $identity" \
+          --link-dest="$dst/current" \
+          "$src/" \
+          "$dst/.partial"
+      mv "$dst/current" "$dst/.previous"
+      mv "$dst/.partial" "$dst/current"
+      rm -fR "$dst/.previous"
+      echo >&2
+
+      exec ${take-snapshots plan}
+    '';
+  in out;
+
+  take-snapshots = plan: writeDash "backup.${plan.name}.take-snapshots" ''
+    set -efu
+    NOW=''${NOW-$(date +%s)}
+    dst=${shell.escape plan.dst.path}
+
+    snapshot() {(
+      : $ns $format $retain
+      name=$(date --date="@$NOW" +"$format")
+      if ! test -e "$dst/$ns/$name"; then
+        echo >&2 "create snapshot: $ns/$name"
+        mkdir -m 0700 -p "$dst/$ns"
+        rsync >&2 \
+            -aAXF --delete \
+            --link-dest="$dst/current" \
+            "$dst/current/" \
+            "$dst/$ns/.partial.$name"
+        mv "$dst/$ns/.partial.$name" "$dst/$ns/$name"
+        echo >&2
+      fi
+      case $retain in
+        ([0-9]*)
+          delete_from=$(($retain + 1))
+          ls -r "$dst/$ns" \
+            | sed -n "$delete_from,\$p" \
+            | while read old_name; do
+                echo >&2 "delete snapshot: $ns/$old_name"
+                rm -fR "$dst/$ns/$old_name"
+              done
+          ;;
+        (ALL)
+          :
+          ;;
+      esac
+    )}
+
+    ${concatStringsSep "\n" (mapAttrsToList (ns: { format, retain ? null, ... }:
+      toString (map shell.escape [
+        "ns=${ns}"
+        "format=${format}"
+        "retain=${if retain == null then "ALL" else toString retain}"
+        "snapshot"
+      ]))
+      plan.snapshots)}
+  '';
+
+  # TODO getFQDN: admit hosts in other domains
+  getFQDN = host: "${host.name}.${config.krebs.search-domain}";
+
+  writeDash = name: text: pkgs.writeScript name ''
+    #! ${pkgs.dash}/bin/dash
+    ${text}
+  '';
+
+  writeDashBin = name: text: pkgs.writeTextFile {
+    executable = true;
+    destination = "/bin/${name}";
+    name = name;
+    text = ''
+      #! ${pkgs.dash}/bin/dash
+      ${text}
+    '';
+  };
+
+in out
diff --git a/krebs/3modules/default.nix b/krebs/3modules/default.nix
index 740ba67b8..deccecc48 100644
--- a/krebs/3modules/default.nix
+++ b/krebs/3modules/default.nix
@@ -7,6 +7,7 @@ let
   out = {
     imports = [
       ./apt-cacher-ng.nix
+      ./backup.nix
       ./bepasty-server.nix
       ./build.nix
       ./current.nix
diff --git a/krebs/4lib/types.nix b/krebs/4lib/types.nix
index c52afa246..81ce659bd 100644
--- a/krebs/4lib/types.nix
+++ b/krebs/4lib/types.nix
@@ -177,4 +177,21 @@ types // rec {
   addr6 = str;
   hostname = str;
   label = str;
+
+  krebs.file-location = types.submodule {
+    options = {
+      # TODO user
+      host = mkOption {
+        type = host;
+      };
+      # TODO merge with ssl.privkey.path
+      path = mkOption {
+        type = types.either types.path types.str;
+        apply = x: {
+          path = toString x;
+          string = x;
+        }.${typeOf x};
+      };
+    };
+  };
 }
diff --git a/tv/1systems/cd.nix b/tv/1systems/cd.nix
index 7cb903a44..8c2a9ae43 100644
--- a/tv/1systems/cd.nix
+++ b/tv/1systems/cd.nix
@@ -7,7 +7,6 @@ with lib;
   krebs.build.target = "root@cd.internet";
 
   imports = [
-    ../2configs/backup.nix
     ../2configs/hw/CAC-Developer-2.nix
     ../2configs/fs/CAC-CentOS-7-64bit.nix
     #../2configs/consul-server.nix
diff --git a/tv/1systems/xu.nix b/tv/1systems/xu.nix
index e1a9076dc..1f3e010a4 100644
--- a/tv/1systems/xu.nix
+++ b/tv/1systems/xu.nix
@@ -9,7 +9,6 @@ with lib;
     "7ae05edcdd14f6ace83ead9bf0d114e97c89a83a";
 
   imports = [
-    ../2configs/backup.nix # TODO
     ../2configs/hw/x220.nix
     #../2configs/consul-client.nix
     ../2configs/git.nix
diff --git a/tv/2configs/backup.nix b/tv/2configs/backup.nix
index 1cef0a6dc..51d3bb8a7 100644
--- a/tv/2configs/backup.nix
+++ b/tv/2configs/backup.nix
@@ -1,38 +1,22 @@
-{ config, lib, pkgs, ... }:
+{ config, lib, ... }:
 with lib;
-let
-  # Users that are allowed to connect to the backup user.
-  # Note: the user must own a push plan destination otherwise no rsync.
-  backup-users = [
-    config.krebs.users.tv
-  ];
-
-  ## TODO parse.file-location admit user
-  ## loc has the form <host-name>:<abs-path>
-  #parse.file-location = loc: let
-  #  parts = splitString ":" loc;
-  #  host-name = head parts;
-  #  path = concatStringsSep ":" (tail parts);
-  #in {
-  #  type = "types.krebs.file-location";
-  #  host = config.krebs.hosts.${host-name};
-  #  path = path;
-  #};
-
-  # TODO assert plan.dst.path & co
-  plans = with config.krebs.users; with config.krebs.hosts; addNames {
+{
+  krebs.backup.plans = addNames {
     xu-test-cd = {
       method = "push";
-      #src = parse.file-location xu:/tmp/xu-test;
-      #dst = parse.file-location cd:/krebs/backup/xu-test;
-      src = { user = tv; host = xu; path = "/tmp/xu-test"; };
-      dst = { user = tv; host = cd; path = "/krebs/backup/xu-test"; };
-      startAt = "0,6,12,18:00";
-      retain = {
-        hourly = 4; # sneakily depends on startAt
-        daily = 7;
-        weekly = 4;
-        monthly = 3;
+
+      src = { host = config.krebs.hosts.xu; path = "/tmp/xu-test"; };
+      dst = { host = config.krebs.hosts.cd; path = "/tmp/backups/xu-test"; };
+
+      #startAt = "0,6,12,18:00";
+      startAt = "minutely";
+      snapshots = {
+        minutely = { format = "%Y-%m-%dT%H:%M"; retain =  5; };
+        hourly   = { format = "%Y-%m-%dT%H";    retain =  4; };
+        daily    = { format = "%Y-%m-%d";       retain =  7; };
+        weekly   = { format = "%YW%W";          retain =  4; };
+        monthly  = { format = "%Y-%m";          retain = 12; };
+        yearly   = { format = "%Y";                          };
       };
     };
     #xu-test-wu = {
@@ -41,214 +25,18 @@ let
     #};
     cd-test-xu = {
       method = "pull";
-      #src = parse.file-location cd:/tmp/cd-test;
-      #dst = parse.file-location xu:/bku/cd-test;
-      src = { user = tv; host = cd; path = "/tmp/cd-test"; };
-      dst = { user = tv; host = xu; path = "/bku/cd-test"; };
+      src = { host = config.krebs.hosts.cd; path = "/tmp/cd-test"; };
+      dst = { host = config.krebs.hosts.xu; path = "/tmp/backups/cd-test"; };
+      startAt = "minutely";
+      snapshots = {
+        minutely = { format = "%Y-%m-%dT%H:%M"; retain =  5; };
+        hourly   = { format = "%Y-%m-%dT%H";    retain =  4; };
+        daily    = { format = "%Y-%m-%d";       retain =  7; };
+        weekly   = { format = "%YW%W";          retain =  4; };
+        monthly  = { format = "%Y-%m";          retain = 12; };
+        yearly   = { format = "%Y";                          };
+      };
     };
 
   };
-
-  out = {
-    #options.krebs.backup = api;
-    config = imp;
-  };
-
-  imp = {
-    users.groups.backup.gid = genid "backup";
-    users.users = map makeUser (filter isPushDst (attrValues plans));
-    systemd.services =
-      flip mapAttrs' (filterAttrs (_:isPushSrc) plans) (name: plan: {
-        name = "backup.${name}";
-        value = makePushService plan;
-      });
-  };
-
-
-  # TODO getFQDN: admit hosts in other domains
-  getFQDN = host: "${host.name}.${config.krebs.search-domain}";
-
-  isPushSrc = plan:
-    plan.method == "push" &&
-    plan.src.host.name == config.krebs.build.host.name;
-
-  makePushService = plan: assert isPushSrc plan; {
-    startAt = plan.startAt;
-    serviceConfig.ExecStart = writeSh plan "rsync" ''
-      exec ${pkgs.rsync}/bin/rsync ${concatMapStringsSep " " shell.escape [
-        "-a"
-        "-e"
-        "${pkgs.openssh}/bin/ssh -F /dev/null -i ${plan.src.host.ssh.privkey.path}"
-        "${plan.src.path}"
-        "${plan.name}@${getFQDN plan.dst.host}::push"
-      ]}
-    '';
-  };
-
-  isPushDst = plan:
-    plan.method == "push" &&
-    plan.dst.host.name == config.krebs.build.host.name;
-
-  makeUser = plan: assert isPushDst plan; rec {
-    name = plan.name;
-    uid = genid name;
-    group = config.users.groups.backup.name;
-    home = plan.dst.path;
-    createHome = true;
-    shell = "${writeSh plan "shell" ''
-      case $2 in
-        'rsync --server --daemon .')
-          exec ${backup.rsync plan [ "--server" "--daemon" "." ]}
-          ;;
-        ''')
-          echo "ERROR: no command specified" >&2
-          exit 23
-          ;;
-        *)
-          echo "ERROR: no unknown command: $SSH_ORIGINAL_COMMAND" >&2
-          exit 23
-          ;;
-      esac
-    ''}";
-    openssh.authorizedKeys.keys = [ plan.src.host.ssh.pubkey ];
-  };
-
-  rsync = plan: args: writeSh plan "rsync" ''
-    install -v -m 0700 -d ${plan.dst.path}/push >&2
-    install -v -m 0700 -d ${plan.dst.path}/list >&2
-
-    ${pkgs.rsync}/bin/rsync \
-        --config=${backup.rsyncd-conf plan {
-          post-xfer = writeSh plan "rsyncd.post-xfer" ''
-            case $RSYNC_EXIT_STATUS in 0)
-              exec ${backup.rsnapshot plan {
-                preexec = writeSh plan "rsnapshot.preexec" ''
-                  touch ${plan.dst.path}/rsnapshot.$RSNAPSHOT_INTERVAL
-                '';
-                postexec = writeSh plan "rsnapshot.postexec" ''
-                  rm ${plan.dst.path}/rsnapshot.$RSNAPSHOT_INTERVAL
-                '';
-              }}
-            esac
-          '';
-        }} \
-        ${toString (map shell.escape args)}
-
-    fail=0
-    for i in monthly weekly daily hourly; do
-      if test -e ${plan.dst.path}/rsnapshot.$i; then
-        rm ${plan.dst.path}/rsnapshot.$i
-        echo "ERROR: $i snapshot failed" >&2
-        fail=1
-      fi
-    done
-    if test $fail != 0; then
-      exit -1
-    fi
-  '';
-
-  rsyncd-conf = plan: conf: pkgs.writeText "${plan.name}.rsyncd.conf" ''
-    fake super = yes
-    use chroot = no
-    lock file = ${plan.dst.path}/rsyncd.lock
-
-    [push]
-    max connections = 1
-    path = ${plan.dst.path}/push
-    write only = yes
-    read only = no
-    post-xfer exec = ${conf.post-xfer}
-
-    [list]
-    path = ${plan.dst.path}/list
-    read only = yes
-    write only = no
-  '';
-
-  rsnapshot = plan: conf: writeSh plan "rsnapshot" ''
-    rsnapshot() {
-      ${pkgs.proot}/bin/proot \
-          -b /bin \
-          -b /nix \
-          -b /run/current-system \
-          -b ${plan.dst.path} \
-          -r ${plan.dst.path} \
-          -w / \
-          ${pkgs.rsnapshot}/bin/rsnapshot \
-              -c ${pkgs.writeText "${plan.name}.rsnapshot.conf" ''
-                config_version	1.2
-                snapshot_root	${plan.dst.path}/list
-                cmd_cp	${pkgs.coreutils}/bin/cp
-                cmd_du	${pkgs.coreutils}/bin/du
-                #cmd_rm	${pkgs.coreutils}/bin/rm
-                cmd_rsync	${pkgs.rsync}/bin/rsync
-                cmd_rsnapshot_diff	${pkgs.rsnapshot}/bin/rsnapshot-diff
-                cmd_preexec	${conf.preexec}
-                cmd_postexec	${conf.postexec}
-                retain	hourly	4
-                retain	daily	7
-                retain	weekly	4
-                retain	monthly	3
-                lockfile	${plan.dst.path}/rsnapshot.pid
-                link_dest	1
-                backup	/push	./
-                verbose	4
-              ''} \
-          "$@"
-    }
-
-    cd ${plan.dst.path}/list/
-
-    now=$(date +%s)
-    is_older_than() {
-      test $(expr $now - $(date +%s -r $1 2>/dev/null || echo 0)) \
-       -ge $2
-    }
-
-    # TODO report stale snapshots
-    # i.e. there are $interval.$i > $interval.$max
-
-    hour_s=3600
-    day_s=86400
-    week_s=604800
-    month_s=2419200 # 4 weeks
-
-    set --
-
-    if test -e weekly.3 && is_older_than monthly.0 $month_s; then
-      set -- "$@" monthly
-    fi
-
-    if test -e daily.6 && is_older_than weekly.0 $week_s; then
-      set -- "$@" weekly
-    fi
-
-    if test -e hourly.3 && is_older_than daily.0 $day_s; then
-      set -- "$@" daily
-    fi
-
-    if is_older_than hourly.0 $hour_s; then
-      set -- "$@" hourly
-    fi
-
-
-    if test $# = 0; then
-      echo "taking no snapshots" >&2
-    else
-      echo "taking snapshots: $@" >&2
-    fi
-
-    export RSNAPSHOT_INTERVAL
-    for RSNAPSHOT_INTERVAL; do
-      rsnapshot "$RSNAPSHOT_INTERVAL"
-    done
-  '';
-
-  writeSh = plan: name: text: pkgs.writeScript "${plan.name}.${name}" ''
-    #! ${pkgs.dash}/bin/dash
-    set -efu
-    export PATH=${makeSearchPath "bin" (with pkgs; [ coreutils ])}
-    ${text}
-  '';
-
-in out
+}
diff --git a/tv/2configs/default.nix b/tv/2configs/default.nix
index 3400c13b6..c300633bb 100644
--- a/tv/2configs/default.nix
+++ b/tv/2configs/default.nix
@@ -28,6 +28,7 @@ with lib;
 
   imports = [
     <secrets>
+    ./backup.nix
     ./vim.nix
     {
       # stockholm dependencies
-- 
cgit v1.3.1


From 9abd00f6af48676f08b6afdddd03d12410a1d1cd Mon Sep 17 00:00:00 2001
From: makefu <github@syntax-fehler.de>
Date: Tue, 29 Dec 2015 21:20:11 +0100
Subject: k 3 makefu: add ssh pubkeys to hosts

---
 krebs/3modules/makefu/default.nix | 11 +++++++++++
 1 file changed, 11 insertions(+)

(limited to 'krebs/3modules')

diff --git a/krebs/3modules/makefu/default.nix b/krebs/3modules/makefu/default.nix
index 1970a0777..31516d591 100644
--- a/krebs/3modules/makefu/default.nix
+++ b/krebs/3modules/makefu/default.nix
@@ -83,6 +83,9 @@ with lib;
             '';
         };
       };
+      ssh.privkey.path = <secrets/ssh_host_ed25519_key>;
+      ssh.pubkey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHDM0E608d/6rGzXqGbNSuMb2RlCojCJSiiz6QcPOC2G root@pornocauster";
+
     };
 
     vbob = {
@@ -108,6 +111,8 @@ with lib;
             '';
         };
       };
+      ssh.privkey.path = <secrets/ssh_host_ed25519_key>;
+      ssh.pubkey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICPLTMl+thSq77cjYa2XF7lz5fA7JMftrLo8Dy/OBXSg root@nixos";
     };
     flap = rec {
       cores = 1;
@@ -238,6 +243,8 @@ with lib;
           '';
         };
       };
+      ssh.privkey.path = <secrets/ssh_host_ed25519_key>;
+      ssh.pubkey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIH4Tjx9qK6uWtxT1HCpeC0XvDZKO/kaPygyKatpAqU6I root@wry";
     };
     filepimp = rec {
       cores = 1;
@@ -287,6 +294,8 @@ with lib;
             '';
         };
       };
+      ssh.privkey.path = <secrets/ssh_host_ed25519_key>;
+      ssh.pubkey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIujMZ3ZFxKpWeB/cjfKfYRr77+VRZk0Eik+92t03NoA root@servarch";
     };
     gum = rec {
       cores = 1;
@@ -327,6 +336,8 @@ with lib;
             '';
         };
       };
+      ssh.privkey.path = <secrets/ssh_host_ed25519_key>;
+      ssh.pubkey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIcxWFEPzke/Sdd9qNX6rSJgXal8NmINYajpFCxXfYdj root@gum";
     };
   };
   users = addNames rec {
-- 
cgit v1.3.1


From d574c0ef78f7572aec88e484d3ff6256247e878c Mon Sep 17 00:00:00 2001
From: makefu <github@syntax-fehler.de>
Date: Wed, 30 Dec 2015 01:38:33 +0100
Subject: m 3 buildbot/master: add secrets

---
 krebs/3modules/buildbot/master.nix      | 16 +++++++++++++---
 shared/2configs/buildbot-standalone.nix |  8 +++++---
 2 files changed, 18 insertions(+), 6 deletions(-)

(limited to 'krebs/3modules')

diff --git a/krebs/3modules/buildbot/master.nix b/krebs/3modules/buildbot/master.nix
index 5870c3145..74385a433 100644
--- a/krebs/3modules/buildbot/master.nix
+++ b/krebs/3modules/buildbot/master.nix
@@ -132,6 +132,16 @@ let
       '';
     };
 
+    secrets = mkOption {
+      default = [];
+      type = types.listOf types.str;
+      example = [ "cac.json" ];
+      description = ''
+        List of all the secrets in <secrets> which should be copied into the
+        buildbot master directory.
+      '';
+    };
+
     slaves = mkOption {
       default = {};
       type = types.attrsOf types.str;
@@ -344,10 +354,10 @@ let
           fi
           # always override the master.cfg
           cp ${buildbot-master-config} ${workdir}/master.cfg
+
           # copy secrets
-          cp ${secretsdir}/cac.json ${workdir}
-          cp ${secretsdir}/retiolum-ci.rsa_key.priv \
-             ${workdir}/retiolum.rsa_key.priv
+          ${ concatMapStringsSep "\n"
+            (f: "cp ${secretsdir}/${f} ${workdir}/${f}" ) cfg.secrets }
           # sanity
           ${buildbot}/bin/buildbot checkconfig ${workdir}
 
diff --git a/shared/2configs/buildbot-standalone.nix b/shared/2configs/buildbot-standalone.nix
index 51c600329..28d1eef2e 100644
--- a/shared/2configs/buildbot-standalone.nix
+++ b/shared/2configs/buildbot-standalone.nix
@@ -8,6 +8,9 @@ in {
   };
   networking.firewall.allowedTCPPorts = [ 8010 9989 ];
   krebs.buildbot.master = {
+    secrets = [
+      "cac.json"
+    ];
     slaves = {
       testslave =  "krebspass";
       omo = "krebspass";
@@ -93,9 +96,8 @@ in {
   # slave needs 2 files:
   # * cac.json
   # * retiolum
-  for file in ["cac.json", "retiolum.rsa_key.priv"]:
-    s.addStep(steps.FileDownload(mastersrc="${config.krebs.buildbot.master.workDir}/{}".format(file),
-                            slavedest=file))
+  s.addStep(steps.FileDownload(mastersrc="${config.krebs.buildbot.master.workDir}/cac.json", slavedest="cac.json"))
+  s.addStep(steps.FileDownload(mastersrc="${config.krebs.buildbot.master.workDir}/retiolum-ci.rsa_key.priv", slavedest="retiolum.rsa_key.priv"))
 
   addShell(s, name="infest-cac-centos7",env=env,
               sigtermTime=60,           # SIGTERM 1 minute before SIGKILL
-- 
cgit v1.3.1


From a8cb9d41bdefe8a5dc72ca76b3ebc5b4047ddf65 Mon Sep 17 00:00:00 2001
From: makefu <github@syntax-fehler.de>
Date: Wed, 30 Dec 2015 02:45:47 +0100
Subject: s 1 test-all-krebs-modules: init

---
 krebs/3modules/shared/default.nix          |  1 +
 shared/1systems/test-all-krebs-modules.nix | 45 ++++++++++++++++++++++++++++++
 shared/2configs/buildbot-standalone.nix    | 17 ++++++++---
 3 files changed, 59 insertions(+), 4 deletions(-)
 create mode 100644 shared/1systems/test-all-krebs-modules.nix

(limited to 'krebs/3modules')

diff --git a/krebs/3modules/shared/default.nix b/krebs/3modules/shared/default.nix
index b332676c6..518e46587 100644
--- a/krebs/3modules/shared/default.nix
+++ b/krebs/3modules/shared/default.nix
@@ -7,6 +7,7 @@ let
     "test-arch"
     "test-centos6"
     "test-centos7"
+    "test-all-krebs-modules"
   ] (name: {
     inherit name;
     cores = 1;
diff --git a/shared/1systems/test-all-krebs-modules.nix b/shared/1systems/test-all-krebs-modules.nix
new file mode 100644
index 000000000..b98004dfe
--- /dev/null
+++ b/shared/1systems/test-all-krebs-modules.nix
@@ -0,0 +1,45 @@
+{ config, pkgs, lib, ... }:
+let
+  en = { enable = true;};
+in {
+  krebs = {
+    enable = true;
+    build.user = config.krebs.users.shared;
+    build.host = config.krebs.hosts.test-all-krebs-modules;
+    Reaktor.enable = true;
+    apt-cacher-ng.enable = true;
+    backup.enable = true;
+    bepasty.enable = true;
+    buildbot.master.enable = true;
+    buildbot.slave = {
+      enable = true;
+      username = "lol";
+      password = "wut";
+    };
+    exim-retiolum.enable = true;
+    exim-smarthost = {
+      enable = true;
+      system-aliases = [ { from = "dick"; to = "butt"; } ];
+    };
+    go.enable = true;
+    iptables = {
+      enable = true;
+      tables = {};
+    };
+    nginx.enable = true;
+    realwallpaper.enable = true;
+    retiolum.enable = true;
+    retiolum-bootstrap.enable = true;
+    tinc_graphs.enable = true;
+    urlwatch.enable = true;
+    fetchWallpaper = {
+      enable = true;
+      url ="localhost";
+    };
+  };
+  # just get the system running
+  boot.loader.grub.devices = ["/dev/sda"];
+  fileSystems."/" = {
+    device = "/dev/lol";
+  };
+}
diff --git a/shared/2configs/buildbot-standalone.nix b/shared/2configs/buildbot-standalone.nix
index 28d1eef2e..22e9861cc 100644
--- a/shared/2configs/buildbot-standalone.nix
+++ b/shared/2configs/buildbot-standalone.nix
@@ -8,12 +8,9 @@ in {
   };
   networking.firewall.allowedTCPPorts = [ 8010 9989 ];
   krebs.buildbot.master = {
-    secrets = [
-      "cac.json"
-    ];
+    secrets = [ "retiolum-ci.rsa_key.priv" "cac.json" ];
     slaves = {
       testslave =  "krebspass";
-      omo = "krebspass";
     };
     change_source.stockholm = ''
   stockholm_repo = 'http://cgit.gum/stockholm'
@@ -85,6 +82,18 @@ in {
   addShell(f,name="eval-cross-check",env=env,
             command=nixshell + ["! make eval get=krebs.deploy filter=json system=test-failing"])
 
+  addShell(f,name="instaniate-test-all-modules",env=env,
+            command=nixshell + \
+                      ["touch retiolum.rsa_key.priv; \
+                        nix-instantiate --eval -A \
+                            users.shared.test-all-krebs-modules.system \
+                            -I stockholm=. \
+                            -I secrets=. '<stockholm>' \
+                            --argstr current-date lol \
+                            --argstr current-user-name shared \
+                            --argstr current-host-name lol \
+                            --strict --json"])
+
   bu.append(util.BuilderConfig(name="fast-tests",
         slavenames=slavenames,
         factory=f))
-- 
cgit v1.3.1


From c962e8549e968fd15d4f15b4d184e86e1cd7ed04 Mon Sep 17 00:00:00 2001
From: makefu <github@syntax-fehler.de>
Date: Wed, 30 Dec 2015 11:29:28 +0100
Subject: k 3 Reaktor: add channels Option

---
 krebs/3modules/Reaktor.nix | 11 +++++++++--
 makefu/1systems/wry.nix    | 21 ++++++++++++++++-----
 2 files changed, 25 insertions(+), 7 deletions(-)

(limited to 'krebs/3modules')

diff --git a/krebs/3modules/Reaktor.nix b/krebs/3modules/Reaktor.nix
index 607eb7cac..92400139c 100644
--- a/krebs/3modules/Reaktor.nix
+++ b/krebs/3modules/Reaktor.nix
@@ -70,12 +70,17 @@ let
           REAKTOR_HOST
           REAKTOR_PORT
           REAKTOR_STATEDIR
-          REAKTOR_CHANNELS
 
           debug and nickname can be set separately via the Reaktor api
       '';
     };
-
+    channels = mkOption {
+      default = [ "#krebs" ];
+      type = types.listOf types.str;
+      description = ''
+        Channels the Reaktor should connect to at startup.
+      '';
+    };
     debug = mkOption {
       default = false;
       description = ''
@@ -112,7 +117,9 @@ let
         GIT_SSL_CAINFO = "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt";
         REAKTOR_NICKNAME = cfg.nickname;
         REAKTOR_DEBUG = (if cfg.debug  then "True" else "False");
+        REAKTOR_CHANNELS = lib.concatStringsSep "," cfg.channels;
         state_dir = cfg.workdir;
+
         } // cfg.extraEnviron;
       serviceConfig= {
         ExecStartPre = pkgs.writeScript "Reaktor-init" ''
diff --git a/makefu/1systems/wry.nix b/makefu/1systems/wry.nix
index cd2b3f657..3bdf053db 100644
--- a/makefu/1systems/wry.nix
+++ b/makefu/1systems/wry.nix
@@ -18,8 +18,6 @@ in {
 
       ../2configs/iodined.nix
 
-      # Reaktor
-      ../2configs/Reaktor/simpleExtend.nix
 
       # other nginx
       ../2configs/nginx/euer.wiki.nix
@@ -29,9 +27,22 @@ in {
       # collectd
       ../2configs/collectd/collectd-base.nix
   ];
+
   krebs.build.host = config.krebs.hosts.wry;
 
-  krebs.Reaktor.enable = true;
+  krebs.Reaktor = {
+    nickname = "Reaktor|bot";
+    channels = [ "#krebs_test" ];
+    enable = true;
+    debug = true;
+    plugins = with pkgs.ReaktorPlugins;[
+                               titlebot
+                               # stockholm-issue
+                               nixos-version
+                               shack-correct
+                               sed-plugin
+                               random-emoji ];
+  };
 
   # bepasty to listen only on the correct interfaces
   krebs.bepasty.servers.internal.nginx.listen  = [ "${internal-ip}:80" ];
@@ -59,11 +70,11 @@ in {
   };
 
   networking = {
-  firewall = {
+    firewall = {
       allowPing = true;
       logRefusedConnections = false;
       allowedTCPPorts = [ 53 80 443 ];
-      allowedUDPPorts = [ 655 ];
+      allowedUDPPorts = [ 655 53 ];
     };
     interfaces.enp2s1.ip4 = [{
       address = external-ip;
-- 
cgit v1.3.1


From b9c4a5e4a0800dbfb6f4cb1d20bbfd2c1d228d11 Mon Sep 17 00:00:00 2001
From: tv <tv@krebsco.de>
Date: Wed, 30 Dec 2015 14:54:04 +0100
Subject: exim-retiolum: move assert to proper location

---
 krebs/3modules/exim-retiolum.nix | 20 +++++++++-----------
 1 file changed, 9 insertions(+), 11 deletions(-)

(limited to 'krebs/3modules')

diff --git a/krebs/3modules/exim-retiolum.nix b/krebs/3modules/exim-retiolum.nix
index e1315d8c8..ea012c38c 100644
--- a/krebs/3modules/exim-retiolum.nix
+++ b/krebs/3modules/exim-retiolum.nix
@@ -1,14 +1,12 @@
 { config, pkgs, lib, ... }:
 
-with builtins;
 with lib;
 let
   cfg = config.krebs.exim-retiolum;
 
   out = {
     options.krebs.exim-retiolum = api;
-    config =
-      mkIf cfg.enable imp;
+    config = mkIf cfg.enable imp;
   };
 
   api = {
@@ -16,13 +14,13 @@ let
   };
 
   imp = {
-    services.exim =
-      # This configuration makes only sense for retiolum-enabled hosts.
-      # TODO modular configuration
-      assert config.krebs.retiolum.enable;
-      {
-        enable = true;
-        config = ''
+    services.exim = {
+      enable = true;
+      config =
+        # This configuration makes only sense for retiolum-enabled hosts.
+        # TODO modular configuration
+        assert config.krebs.retiolum.enable;
+        ''
           primary_hostname = ${retiolumHostname}
           domainlist local_domains    = @ : localhost
           domainlist relay_to_domains = *.retiolum
@@ -134,7 +132,7 @@ let
 
           begin authenticators
         '';
-      };
+    };
   };
 
   # TODO get the hostname from somewhere else.
-- 
cgit v1.3.1


From f16742895c26b0f3df71ca8503afc5f4cb97a9ae Mon Sep 17 00:00:00 2001
From: lassulus <lass@aidsballs.de>
Date: Wed, 30 Dec 2015 17:14:31 +0100
Subject: l: add new host dishfire

---
 krebs/3modules/lass/default.nix | 32 ++++++++++++++++++++++++++++++
 krebs/Zhosts/dishfire           | 12 ++++++++++++
 lass/1systems/dishfire.nix      | 43 +++++++++++++++++++++++++++++++++++++++++
 3 files changed, 87 insertions(+)
 create mode 100644 krebs/Zhosts/dishfire
 create mode 100644 lass/1systems/dishfire.nix

(limited to 'krebs/3modules')

diff --git a/krebs/3modules/lass/default.nix b/krebs/3modules/lass/default.nix
index 26b0947bb..592ed475d 100644
--- a/krebs/3modules/lass/default.nix
+++ b/krebs/3modules/lass/default.nix
@@ -4,6 +4,38 @@ with lib;
 
 {
   hosts = addNames {
+    dishfire = {
+      cores = 4;
+      dc = "lass"; #dc = "cac";
+      nets = rec {
+        internet = {
+          addrs4 = ["144.76.172.188"];
+          aliases = [
+            "dishfire.internet"
+          ];
+        };
+        retiolum = {
+          via = internet;
+          addrs4 = ["10.243.133.99"];
+          addrs6 = ["42:0000:0000:0000:0000:0000:d15f:1233"];
+          aliases = [
+            "dishfire.retiolum"
+          ];
+          tinc.pubkey = ''
+            -----BEGIN RSA PUBLIC KEY-----
+            MIIBCgKCAQEAwKi49fN+0s5Cze6JThM7f7lj4da27PSJ/3w3tDFPvtQco11ksNLs
+            Xd3qPaQIgmcNVCR06aexae3bBeTx9y3qHvKqZVE1nCtRlRyqy1LVKSj15J1D7yz7
+            uS6u/BSZiCzmdZwu3Fq5qqoK0nfzWe/NKEDWNa5l4Mz/BZQyI/hbOpn6UfFD0LpK
+            R4jzc9Dbk/IFNAvwb5yrgEYtwBzlXzeDvHW2JcPq3qQjK2byQYNiIyV3g0GHppEd
+            vDbIPDFhTn3Hv5zz/lX+/We8izzRge7MEd+Vn9Jwb5NAzwDsOHl6ExpqASv9H49U
+            HwgPw5pstabyrsDWXybSYUb+8LcZf+unGwIDAQAB
+            -----END RSA PUBLIC KEY-----
+          '';
+        };
+      };
+      #ssh.privkey.path = <secrets/ssh.id_ed25519>;
+      #ssh.pubkey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIL21QDOEFdODFh6WAfNp6odrXo15pEsDQuGJfMu/cKzK";
+    };
     echelon = {
       cores = 2;
       dc = "lass"; #dc = "cac";
diff --git a/krebs/Zhosts/dishfire b/krebs/Zhosts/dishfire
new file mode 100644
index 000000000..c4cf68b6b
--- /dev/null
+++ b/krebs/Zhosts/dishfire
@@ -0,0 +1,12 @@
+Address = 144.76.172.188
+Subnet = 10.243.133.99
+Subnet = 42:0000:0000:0000:0000:0000:d15f:1233
+
+-----BEGIN RSA PUBLIC KEY-----
+MIIBCgKCAQEAwKi49fN+0s5Cze6JThM7f7lj4da27PSJ/3w3tDFPvtQco11ksNLs
+Xd3qPaQIgmcNVCR06aexae3bBeTx9y3qHvKqZVE1nCtRlRyqy1LVKSj15J1D7yz7
+uS6u/BSZiCzmdZwu3Fq5qqoK0nfzWe/NKEDWNa5l4Mz/BZQyI/hbOpn6UfFD0LpK
+R4jzc9Dbk/IFNAvwb5yrgEYtwBzlXzeDvHW2JcPq3qQjK2byQYNiIyV3g0GHppEd
+vDbIPDFhTn3Hv5zz/lX+/We8izzRge7MEd+Vn9Jwb5NAzwDsOHl6ExpqASv9H49U
+HwgPw5pstabyrsDWXybSYUb+8LcZf+unGwIDAQAB
+-----END RSA PUBLIC KEY-----
diff --git a/lass/1systems/dishfire.nix b/lass/1systems/dishfire.nix
new file mode 100644
index 000000000..a1288d578
--- /dev/null
+++ b/lass/1systems/dishfire.nix
@@ -0,0 +1,43 @@
+{ config, lib, pkgs, ... }:
+
+{
+  imports = [
+    <nixpkgs/nixos/modules/profiles/qemu-guest.nix>
+    ../2configs/base.nix
+    {
+      boot.loader.grub = {
+        device = "/dev/vda";
+        splashImage = null;
+      };
+
+      boot.initrd.availableKernelModules = [
+        "ata_piix"
+        "ehci_pci"
+        "uhci_hcd"
+        "virtio_pci"
+        "virtio_blk"
+      ];
+
+      fileSystems."/" = {
+        device = "/dev/mapper/pool-nix";
+        fsType = "ext4";
+      };
+
+      fileSystems."/boot" = {
+        device = "/dev/vda1";
+        fsType = "ext4";
+      };
+    }
+    {
+      networking.dhcpcd.allowInterfaces = [
+        "enp*"
+        "eth*"
+      ];
+    }
+    {
+      sound.enable = false;
+    }
+  ];
+
+  krebs.build.host = config.krebs.hosts.dishfire;
+}
-- 
cgit v1.3.1