Author: ckauhaus
Date: Fri Dec  5 11:05:30 2008
New Revision: 7172

Log:
download a specified version of the portage tree from a Gentoo mirror and store
it at /home/puppet/snapshots


Added:
   gocept.infrastructure/feature_systemupgrade/puppet/manifests/defines.pp
  
gocept.infrastructure/feature_systemupgrade/puppet/modules/sys_portage/files/convergence.sh
  
gocept.infrastructure/feature_systemupgrade/puppet/modules/sys_portage/manifests/update.pp
Modified:
   gocept.infrastructure/feature_systemupgrade/puppet/manifests/classes/role.pp
   gocept.infrastructure/feature_systemupgrade/puppet/manifests/site.pp
  
gocept.infrastructure/feature_systemupgrade/puppet/modules/app_admin/manifests/puppet.pp
  
gocept.infrastructure/feature_systemupgrade/puppet/modules/net_misc/manifests/rsync.pp
  
gocept.infrastructure/feature_systemupgrade/puppet/modules/net_misc/templates/rsyncd.conf.erb
  
gocept.infrastructure/feature_systemupgrade/puppet/modules/sys_portage/files/cron.daily_emerge
  
gocept.infrastructure/feature_systemupgrade/puppet/modules/sys_portage/files/update-system.sh
  
gocept.infrastructure/feature_systemupgrade/puppet/modules/sys_portage/manifests/init.pp
  
gocept.infrastructure/feature_systemupgrade/puppet/modules/sys_portage/manifests/portage.pp
  
gocept.infrastructure/feature_systemupgrade/puppet/modules/sys_portage/templates/update-system.conf.erb
  
gocept.infrastructure/feature_systemupgrade/puppet/modules/www_apache/manifests/apache.pp

Modified:
gocept.infrastructure/feature_systemupgrade/puppet/manifests/classes/role.pp
==============================================================================
---
gocept.infrastructure/feature_systemupgrade/puppet/manifests/classes/role.pp	(original)
+++
gocept.infrastructure/feature_systemupgrade/puppet/manifests/classes/role.pp	Fri
Dec  5 11:05:30 2008
(at)(at) -28,6 +28,7 (at)(at)
     include sys_apps::baselayout
     include sys_fs::udev
     include sys_portage::portage
+    include sys_portage::portage::update
     include sys_process::vixie_cron
 
     net_analyzer::hostgroup { "role::generic": comment => "All nodes" }

Added: gocept.infrastructure/feature_systemupgrade/puppet/manifests/defines.pp
==============================================================================
--- (empty file)
+++ gocept.infrastructure/feature_systemupgrade/puppet/manifests/defines.pp	Fri
Dec  5 11:05:30 2008
(at)(at) -0,0 +1,24 (at)(at)
+# Copyright (c) 2008 gocept gmbh & co. kg
+# See also LICENSE.txt
+
+# Global utilities for portage manifests.
+
+define download($url, $check_md5=false) {
+    $file = $title
+
+    exec { "wget -q -c -O '$file' $url":
+        creates => $file,
+        refreshonly => false,
+        timeout => 900
+    }
+
+    if $check_md5 {
+        $md5 = "$file.md5sum"
+        exec {
+            "wget -q -O '$md5' $url.md5sum && \
+             cd `dirname '$md5'` && md5sum -c '$md5'":
+                creates => $md5,
+                refreshonly => false
+        }
+    }
+}

Modified: gocept.infrastructure/feature_systemupgrade/puppet/manifests/site.pp
==============================================================================
---
gocept.infrastructure/feature_systemupgrade/puppet/manifests/site.pp	(original)
+++ gocept.infrastructure/feature_systemupgrade/puppet/manifests/site.pp	Fri
Dec  5 11:05:30 2008
(at)(at) -1,5 +1,6 (at)(at)
 # Globals
 import "defaults.pp"
+import "defines.pp"
 import "classes/*.pp"
 import "users/*.pp"
 

Modified:
gocept.infrastructure/feature_systemupgrade/puppet/modules/app_admin/manifests/puppet.pp
==============================================================================
---
gocept.infrastructure/feature_systemupgrade/puppet/modules/app_admin/manifests/puppet.pp	(original)
+++
gocept.infrastructure/feature_systemupgrade/puppet/modules/app_admin/manifests/puppet.pp	Fri
Dec  5 11:05:30 2008
(at)(at) -76,6 +76,18 (at)(at)
             mode => 0755;
         "/etc/cron.hourly/puppetmaster":
             content => template("app_admin/puppetmaster.cron.erb"),
-            mode => 0711
+            mode => 0711;
+        "$puppet_root/snapshots":
+            ensure => directory,
+            owner => puppet,
+            group => puppet;
+    }
+
+    $snapshot = $sys_portage::portage::update::snapshot
+
+    download { "$puppet_root/snapshots/portage-$snapshot.tar.lzma":
+        url =>
"ftp://de-mirror.org/distro/gentoo/snapshots/portage-$snapshot.tar.lzma",
+        check_md5 => true,
+        require => File["$puppet_root/snapshots"]
     }
 }

Modified:
gocept.infrastructure/feature_systemupgrade/puppet/modules/net_misc/manifests/rsync.pp
==============================================================================
---
gocept.infrastructure/feature_systemupgrade/puppet/modules/net_misc/manifests/rsync.pp	(original)
+++
gocept.infrastructure/feature_systemupgrade/puppet/modules/net_misc/manifests/rsync.pp	Fri
Dec  5 11:05:30 2008
(at)(at) -22,6 +22,7 (at)(at)
         ensure => true
     }
 
+    $puppet_root = $app_admin::puppetclient::puppet_root
     $envpaths = $app_admin::puppetclient::envpaths
     $hosts_allow = "*.${location}.${location::suffix}"
 

Modified:
gocept.infrastructure/feature_systemupgrade/puppet/modules/net_misc/templates/rsyncd.conf.erb
==============================================================================
---
gocept.infrastructure/feature_systemupgrade/puppet/modules/net_misc/templates/rsyncd.conf.erb	(original)
+++
gocept.infrastructure/feature_systemupgrade/puppet/modules/net_misc/templates/rsyncd.conf.erb	Fri
Dec  5 11:05:30 2008
(at)(at) -7,16 +7,16 (at)(at)
 use chroot = yes
 read only = yes
 
-<%
-for path in envpaths.split
-    env = File.basename(path)
-%>
-[<%= env %>-portage]
-        path = <%= path %>/portage/
+[snapshots]
+        path = <%= puppet_root %>/portage/
         comment = gocept's current version of the portage tree
         exclude = .svn .cvs
         hosts allow = <%= hosts_allow %>
 
+<%
+for path in envpaths.split
+    env = File.basename(path)
+%>
 [<%= env %>-portage-gocept]
         path = <%= path %>/portage-gocept/
         comment = gocept's own packages

Added:
gocept.infrastructure/feature_systemupgrade/puppet/modules/sys_portage/files/convergence.sh
==============================================================================
--- (empty file)
+++
gocept.infrastructure/feature_systemupgrade/puppet/modules/sys_portage/files/convergence.sh	Fri
Dec  5 11:05:30 2008
(at)(at) -0,0 +1,79 (at)(at)
+#!/bin/bash
+# Simple tool to speed up the portage package convergence check. Once
+# convergence has been reached, it can only go away if one of the following
+# changes:
+# - /etc/make.* or /etc/portage/*
+# - /usr/portage/* or /usr/local/portage/*
+# - system or world file
+# We build a global MD5 sum over all of the file metadata in the mentioned
+# directory trees. If this sum did not change, most probably a reached state
of
+# convergence has not been left.
+#
+# Copyright (c) 2008 gocept gmbh & co. kg
+# See also LICENSE.txt
+# Managed by Puppet: do not edit this file directly. It will be overwritten!
+set -e
+
+usage() {
+    cat <<__EOT__
+Usage: $0 --set|-s|--query|-q
+    --set|-s    Indicate that convergence has been reached
+    --query|-q  Find out if convergence is still probable
+    --help|-h   This help screen
+__EOT__
+}
+
+MARKER="/var/state/portage_convergence"
+
+checksum() {
+    find /etc/make.* /usr/portage /usr/local/portage /var/lib/portage \
+      -type f | xargs ls -l | md5sum
+}
+
+set_convergence() {
+    checksum > ${MARKER}
+    return 0
+}
+
+query() {
+    if [[ ! -f ${MARKER} ]]; then
+        return 1
+    fi
+    if [[ $(checksum) == $(< ${MARKER}) ]]; then
+        return 0
+    else
+        rm -f ${MARKER}
+        return 1
+    fi
+}
+
+
+rc=0
+while [[ -n $1 ]]; do
+    case $1 in
+    -s|--set)
+        action="set_convergence"
+        break
+        ;;
+    -q|--query)
+        action="query"
+        break
+        ;;
+    -h|--help)
+        action="usage"
+        break
+        ;;
+    *)
+        echo "$0: unknown argument '$1'" >&2
+        usage >&2
+        exit 2
+        ;;
+    esac
+done
+
+if [[ -n ${action} ]]; then
+    eval ${action}
+else
+    usage
+fi
+exit ${rc}

Modified:
gocept.infrastructure/feature_systemupgrade/puppet/modules/sys_portage/files/cron.daily_emerge
==============================================================================
---
gocept.infrastructure/feature_systemupgrade/puppet/modules/sys_portage/files/cron.daily_emerge	(original)
+++
gocept.infrastructure/feature_systemupgrade/puppet/modules/sys_portage/files/cron.daily_emerge	Fri
Dec  5 11:05:30 2008
(at)(at) -1,6 +1,6 (at)(at)
 #!/bin/bash
 # We use the live update on the portage tree only for GLSAs - the rest is kept
-# in lock step.
+# in lock step via system update.
 #
 # Copyright (c) 2008 gocept gmbh & co. kg
 # See also LICENSE.txt

Modified:
gocept.infrastructure/feature_systemupgrade/puppet/modules/sys_portage/files/update-system.sh
==============================================================================
---
gocept.infrastructure/feature_systemupgrade/puppet/modules/sys_portage/files/update-system.sh	(original)
+++
gocept.infrastructure/feature_systemupgrade/puppet/modules/sys_portage/files/update-system.sh	Fri
Dec  5 11:05:30 2008
(at)(at) -13,13 +13,12 (at)(at)
 
 update_files() {
     local opts="-q $(portageq envvar PORTAGE_RSYNC_OPTS)"
-    rsync ${opts} --exclude=/metadata/glsa \
-                    ${PUPPET_SERVER}::${PUPPET_ENVIRONMENT}-portage \
-                    /usr/portage/
     rsync ${opts}   ${PUPPET_SERVER}::${PUPPET_ENVIRONMENT}-portage-gocept \
                     /usr/local/portage/
     rsync ${opts}   ${PUPPET_SERVER}::${PUPPET_ENVIRONMENT}-fixes \
                     /usr/local/fixes/
+    #rsync ${opts} --exclude=/metadata/glsa ${PUPPET_SERVER}::portage \
+    #                /usr/portage/
 }
 
 regen_metadata() {

Modified:
gocept.infrastructure/feature_systemupgrade/puppet/modules/sys_portage/manifests/init.pp
==============================================================================
---
gocept.infrastructure/feature_systemupgrade/puppet/modules/sys_portage/manifests/init.pp	(original)
+++
gocept.infrastructure/feature_systemupgrade/puppet/modules/sys_portage/manifests/init.pp	Fri
Dec  5 11:05:30 2008
(at)(at) -1,2 +1,5 (at)(at)
-import "portage.pp"
+# Copyright (c) 2008 gocept gmbh & co. kg
+# See also LICENSE.txt
+
 import "defines.pp"
+import "update.pp"

Modified:
gocept.infrastructure/feature_systemupgrade/puppet/modules/sys_portage/manifests/portage.pp
==============================================================================
---
gocept.infrastructure/feature_systemupgrade/puppet/modules/sys_portage/manifests/portage.pp	(original)
+++
gocept.infrastructure/feature_systemupgrade/puppet/modules/sys_portage/manifests/portage.pp	Fri
Dec  5 11:05:30 2008
(at)(at) -50,10 +50,5 (at)(at)
         "/etc/cron.daily/emerge":
             source => "puppet:///sys_portage/cron.daily_emerge",
             mode => 0755;
-        "/usr/local/sbin/update-system":
-            source => "puppet:///sys_portage/update-system.sh",
-            mode => 0755;
-        "/etc/update-system.conf":
-            content => template("sys_portage/update-system.conf.erb");
     }
 }

Added:
gocept.infrastructure/feature_systemupgrade/puppet/modules/sys_portage/manifests/update.pp
==============================================================================
--- (empty file)
+++
gocept.infrastructure/feature_systemupgrade/puppet/modules/sys_portage/manifests/update.pp	Fri
Dec  5 11:05:30 2008
(at)(at) -0,0 +1,20 (at)(at)
+# Copyright (c) 2008 gocept gmbh & co. kg
+# See also LICENSE.txt
+
+# Automatic system update
+class sys_portage::portage::update inherits sys_portage::portage {
+
+    $snapshot = "20081204"
+
+    file {
+        "/usr/local/sbin/update-system":
+            source => "puppet:///sys_portage/update-system.sh",
+            mode => 0755;
+        "/etc/update-system.conf":
+            content => template("sys_portage/update-system.conf.erb");
+        "/usr/local/sbin/convergence":
+            source => "puppet:///sys_portage/convergence.sh",
+            mode => 0755;
+    }
+
+}

Modified:
gocept.infrastructure/feature_systemupgrade/puppet/modules/sys_portage/templates/update-system.conf.erb
==============================================================================
---
gocept.infrastructure/feature_systemupgrade/puppet/modules/sys_portage/templates/update-system.conf.erb	(original)
+++
gocept.infrastructure/feature_systemupgrade/puppet/modules/sys_portage/templates/update-system.conf.erb	Fri
Dec  5 11:05:30 2008
(at)(at) -5,3 +5,4 (at)(at)
 
 PUPPET_SERVER="<%= puppet_server %>"
 PUPPET_ENVIRONMENT="<%= environment %>"
+SNAPSHOT="<%= snapshot %>"

Modified:
gocept.infrastructure/feature_systemupgrade/puppet/modules/www_apache/manifests/apache.pp
==============================================================================
---
gocept.infrastructure/feature_systemupgrade/puppet/modules/www_apache/manifests/apache.pp	(original)
+++
gocept.infrastructure/feature_systemupgrade/puppet/modules/www_apache/manifests/apache.pp	Fri
Dec  5 11:05:30 2008
(at)(at) -24,7 +24,7 (at)(at)
 
     net_analyzer::service { "proc apache":
         type => "nrpe",
-        command => "procs -w 3:12 -c 1:24 -C apache2 -u apache"
+        command => "procs -w 2:12 -c 1:24 -C apache2 -u apache"
     }
 
 }

Author: ckauhaus
Date: Mon Dec  8 15:32:55 2008
New Revision: 7196

Log:
wrote master script for (a) system update and (b) puppetd run


Added:
  
gocept.infrastructure/feature_systemupgrade/puppet/modules/app_admin/files/run-update-puppet.sh
Modified:
   gocept.infrastructure/feature_systemupgrade/puppet/manifests/classes/role.pp
  
gocept.infrastructure/feature_systemupgrade/puppet/modules/app_admin/manifests/puppet.pp
  
gocept.infrastructure/feature_systemupgrade/puppet/modules/app_admin/templates/cron.d_puppet.erb
  
gocept.infrastructure/feature_systemupgrade/puppet/modules/sys_portage/files/update-system.sh
  
gocept.infrastructure/feature_systemupgrade/puppet/modules/sys_portage/manifests/portage.pp

Modified:
gocept.infrastructure/feature_systemupgrade/puppet/manifests/classes/role.pp
==============================================================================
---
gocept.infrastructure/feature_systemupgrade/puppet/manifests/classes/role.pp	(original)
+++
gocept.infrastructure/feature_systemupgrade/puppet/manifests/classes/role.pp	Mon
Dec  8 15:32:55 2008
(at)(at) -14,6 +14,7 (at)(at)
 
 # generic functionality to be found on every node
 class role::generic inherits role::default {
+    include app_admin::logrotate
     include app_admin::puppetclient
     include app_admin::sudo
     include app_admin::tmpwatch

Added:
gocept.infrastructure/feature_systemupgrade/puppet/modules/app_admin/files/run-update-puppet.sh
==============================================================================
--- (empty file)
+++
gocept.infrastructure/feature_systemupgrade/puppet/modules/app_admin/files/run-update-puppet.sh	Mon
Dec  8 15:32:55 2008
(at)(at) -0,0 +1,20 (at)(at)
+#!/bin/bash
+# Master script for system update and puppetd run
+# Copyright (c) 2008 gocept gmbh & co. kg
+# See also LICENSE.txt
+# Managed by Puppet: do not edit this file directly. It will be overwritten!
+
+if [[ -x /usr/local/sbin/update-system ]]; then
+    log=$(tempfile -p log)
+    trap "logger -t run-update-puppet < ${log}; rm -f ${log}" 0 1 2 3 5 15
+    /usr/local/sbin/update-system > ${log} 2>&1
+    if [[ $? -gt 0 ]]; then
+        echo "$0: update-system failed"
+        echo "see also /var/log/portage/*.log"
+        echo
+        cat ${log}
+    fi
+else
+    # run just puppetd since update-systems seems not to be in place yet
+    puppetd --onetime
+fi

Modified:
gocept.infrastructure/feature_systemupgrade/puppet/modules/app_admin/manifests/puppet.pp
==============================================================================
---
gocept.infrastructure/feature_systemupgrade/puppet/modules/app_admin/manifests/puppet.pp	(original)
+++
gocept.infrastructure/feature_systemupgrade/puppet/modules/app_admin/manifests/puppet.pp	Mon
Dec  8 15:32:55 2008
(at)(at) -36,7 +36,10 (at)(at)
             mode => "0644";
         "/etc/cron.daily/puppet":
             source => "puppet:///app_admin/puppet.cron",
-            mode => 0755
+            mode => 0755;
+        "/usr/local/sbin/run-update-puppet":
+            source => "puppet:///app_admin/run-update-puppet.sh",
+            mode => 0755;
     }
 
     net_analyzer::service { "puppet_localconfig":

Modified:
gocept.infrastructure/feature_systemupgrade/puppet/modules/app_admin/templates/cron.d_puppet.erb
==============================================================================
---
gocept.infrastructure/feature_systemupgrade/puppet/modules/app_admin/templates/cron.d_puppet.erb	(original)
+++
gocept.infrastructure/feature_systemupgrade/puppet/modules/app_admin/templates/cron.d_puppet.erb	Mon
Dec  8 15:32:55 2008
(at)(at) -1,3 +1,3 (at)(at)
 HOME=/root
 PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/bin
-<%= cron_minute %>-59/30 * * * *    root    /usr/bin/puppetd --onetime
+<%= cron_minute %>-59/30 * * * *    root   
/usr/local/sbin/run-update-puppet

Modified:
gocept.infrastructure/feature_systemupgrade/puppet/modules/sys_portage/files/update-system.sh
==============================================================================
---
gocept.infrastructure/feature_systemupgrade/puppet/modules/sys_portage/files/update-system.sh	(original)
+++
gocept.infrastructure/feature_systemupgrade/puppet/modules/sys_portage/files/update-system.sh	Mon
Dec  8 15:32:55 2008
(at)(at) -7,6 +7,7 (at)(at)
 
 source /etc/update-system.conf
 fixdir="/usr/local/fixes"
+reboot_reason=""
 
 bootstrap() {
     puppetd --onetime --tags bootstrap
(at)(at) -71,40 +72,38 (at)(at)
 apply_fixes() {
     ${fixdir}/emergency
     ${fixdir}/check
-    local todo=$(${fixdir}/check)
+    local todo="$(${fixdir}/check)"
     local todo_old
-    while [[ -n ${todo} ]]; do
-        if [[ ${todo} == ${todo_old} ]]; then
+    while [[ -n "${todo}" ]]; do
+        if [[ "${todo}" == "${todo_old}" ]]; then
             return 1
         fi
-        todo_old=${todo}
+        todo_old="${todo}"
         ${fixdir}/fix
-        todo=$(${fixdir}/check)
+        todo="$(${fixdir}/check)"
     done
     return 0
 }
 
-# Perform a complete update. If a package fails, proceed by updating other
-# packages first. After each cycle, fixes are applied. If there are still
-# packages to update and no further progress can be made, exit unsuccessfully.
+# Perform a complete update. After each cycle, fixes are applied. If there are
+# still packages to update and no further progress can be made, exit
+# unsuccessfully.
 update_world() {
-    apply_fixes
+    apply_fixes || true
     emerge -pquDN world
-    local todo=$(emerge -pquDN world)
+    local todo="$(emerge -pquDN world)"
     local todo_old
     while [[ -n "${todo}" ]]; do
-        if [[ ${todo} == ${todo_old} ]]; then
+        if [[ "${todo}" == "${todo_old}" ]]; then
             return 1
         fi
-        todo_old=${todo}
+        reboot_reason="emerge world"
+        todo_old="${todo}"
         emerge -quDN world >>/var/log/portage/build.log || true
-        until emerge -q --resume --skipfirst
>>/var/log/portage/build.log
-        do : ; done
-        apply_fixes
-        todo=$(emerge -pquDN world)
+        apply_fixes || true
+        todo="$(emerge -pquDN world)"
     done
     apply_fixes
-    return $?
 }
 
 # main: perform update steps until convergence is reached
(at)(at) -118,7 +117,10 (at)(at)
 
 if ! convergence --query; then
     update_cycle
-    if [[ $? == 0 ]]; then
-        convergence --set
+    convergence --set
+    if [[ -n ${reboot_reason} ]]; then
+        echo ${reboot_reason} >> /reboot
     fi
 fi
+
+/usr/bin/puppetd --onetime

Modified:
gocept.infrastructure/feature_systemupgrade/puppet/modules/sys_portage/manifests/portage.pp
==============================================================================
---
gocept.infrastructure/feature_systemupgrade/puppet/modules/sys_portage/manifests/portage.pp	(original)
+++
gocept.infrastructure/feature_systemupgrade/puppet/modules/sys_portage/manifests/portage.pp	Mon
Dec  8 15:32:55 2008
(at)(at) -50,5 +50,7 (at)(at)
         "/etc/cron.daily/emerge":
             source => "puppet:///sys_portage/cron.daily_emerge",
             mode => 0755;
+        "/etc/logrotate.d/portage":
+            source => "puppet:///sys_portage/logrotate_portage";
     }
 }