snapshot of https://github.com/zulip/zulip.git @ 48c69911c7dd4e03b20552eb3bd77ad8151c1357 (2026-09-15)

puppet/kandra/manifests/apache.pp

@@ -0,0 +1,40 @@
+class kandra::apache {
+  package { 'apache2':
+    ensure => installed,
+  }
+  service { 'apache2':
+    require => Package['apache2'],
+  }
+
+  apache2mod { [ 'headers', 'proxy', 'proxy_http', 'rewrite', 'auth_digest', 'ssl' ]:
+    ensure  => present,
+    require => Package['apache2'],
+    notify  => Service['apache2'],
+  }
+
+  file { '/etc/apache2/certs/':
+    ensure  => directory,
+    require => Package['apache2'],
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0755',
+  }
+
+  file { '/etc/apache2/ports.conf':
+    ensure  => file,
+    require => Package[apache2],
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0640',
+    source  => 'puppet:///modules/kandra/apache/ports.conf',
+    notify  => Service['apache2'],
+  }
+
+  file { '/etc/apache2/sites-available/':
+    ensure  => directory,
+    require => Package[apache2],
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0750',
+  }
+}

puppet/kandra/manifests/app_frontend.pp

@@ -0,0 +1,82 @@
+class kandra::app_frontend {
+  include zulip::snakeoil
+  include zulip::app_frontend_base
+  include zulip::profile::memcached
+  include zulip::profile::rabbitmq
+  include zulip::local_mailserver
+  include zulip::hooks::sentry
+  include kandra::app_frontend_monitoring
+  include kandra::app_frontend_tmpfs
+
+  kandra::firewall_allow{ 'smtp': }
+  kandra::firewall_allow{ 'http': }
+  kandra::firewall_allow{ 'https': }
+
+  $redis_hostname = zulipconf('redis', 'hostname', undef)
+  group { 'redistunnel':
+    ensure => present,
+    gid    => '1080',
+  }
+  user { 'redistunnel':
+    ensure     => present,
+    uid        => '1080',
+    gid        => '1080',
+    groups     => ['zulip'],
+    shell      => '/bin/true',
+    home       => '/home/redistunnel',
+    managehome => true,
+  }
+  kandra::user_dotfiles { 'redistunnel':
+    keys        => true,
+    known_hosts => [$redis_hostname],
+  }
+  package { 'autossh': ensure => installed }
+  file { "${zulip::common::supervisor_conf_dir}/redis_tunnel.conf":
+    ensure  => file,
+    require => [
+      Package['supervisor', 'autossh'],
+      Kandra::User_Dotfiles['redistunnel'],
+    ],
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    content => template('kandra/supervisor/conf.d/redis_tunnel.conf.template.erb'),
+    notify  => Service['supervisor'],
+  }
+  # Need redis_password in its own file for Nagios
+  file { '/var/lib/nagios/redis_password':
+    ensure  => file,
+    mode    => '0600',
+    owner   => 'nagios',
+    group   => 'nagios',
+    content => zulipsecret('secrets', 'redis_password', ''),
+  }
+
+  # Mount /etc/zulip/well-known/ as /.well-known/
+  file { '/etc/nginx/zulip-include/app.d/well-known.conf':
+    require => File['/etc/nginx/zulip-include/app.d'],
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    source  => 'puppet:///modules/kandra/nginx/zulip-include-app.d/well-known.conf',
+    notify  => Service['nginx'],
+  }
+
+  # Serve /static/navigation-tour-video/
+  file { '/etc/nginx/zulip-include/app.d/navigation-tour-video.conf':
+    require => File['/etc/nginx/zulip-include/app.d'],
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    source  => 'puppet:///modules/kandra/nginx/zulip-include-app.d/navigation-tour-video.conf',
+    notify  => Service['nginx'],
+  }
+
+  # Each server does its own fetching of contributor data, since
+  # we don't have a way to synchronize that among several servers.
+  zulip::cron { 'fetch-contributor-data':
+    hour    => '8',
+    minute  => '0',
+    command => '/home/zulip/deployments/current/tools/fetch-contributor-data',
+  }
+}

puppet/kandra/manifests/app_frontend_monitoring.pp

@@ -0,0 +1,25 @@
+# @summary Prometheus monitoring of a Django frontend and RabbitMQ server.
+#
+class kandra::app_frontend_monitoring {
+  include kandra::prometheus::memcached
+  include kandra::prometheus::rabbitmq
+  include kandra::prometheus::uwsgi
+  include kandra::prometheus::process
+  include kandra::prometheus::grok
+  kandra::teleport::prometheus_app { 'tusd': port => '9900' }
+  kandra::teleport::prometheus_app { 'katex': port => '9700' }
+
+  file { '/etc/cron.d/rabbitmq-monitoring':
+    ensure => absent,
+  }
+  zulip::cron { 'check-rabbitmq-queue':
+    minute  => '*',
+    user    => 'root',
+    command => '/home/zulip/deployments/current/scripts/nagios/check-rabbitmq-queue',
+  }
+  zulip::cron { 'check-rabbitmq-consumers':
+    minute  => '*',
+    user    => 'root',
+    command => '/home/zulip/deployments/current/scripts/nagios/check-rabbitmq-consumers',
+  }
+}

puppet/kandra/manifests/app_frontend_tmpfs.pp

@@ -0,0 +1,43 @@
+class kandra::app_frontend_tmpfs {
+  $disk = '/dev/nvme1n1'
+  $partition = "${disk}p1"
+
+  exec { 'create partition table':
+    command => "/usr/sbin/parted -s ${disk} mklabel gpt",
+    unless  => "/usr/sbin/blkid -s PTUUID ${disk}",
+  }
+
+  exec { 'create partition':
+    command => "/usr/sbin/parted -a optimal -s ${disk} mkpart slack-tmp ext4 0% 100%",
+    unless  => "/usr/sbin/blkid ${partition}",
+    require => Exec['create partition table'],
+  }
+
+  exec { 'format partition':
+    command => "/sbin/mkfs.ext4 -L slack-tmp ${partition}",
+    unless  => "/sbin/blkid -t TYPE=ext4 ${partition}",
+    require => Exec['create partition'],
+  }
+
+  file { '/tmp/slack-conversions':
+    ensure => 'directory',
+    mode   => '0755',
+  }
+  mount { '/tmp/slack-conversions':
+    ensure  => 'mounted',
+    device  => $partition,
+    fstype  => 'ext4',
+    atboot  => true,
+    options => 'defaults,noatime,discard,commit=60,errors=remount-ro',
+    require => Exec['format partition'],
+  }
+
+  # We want ownership to be enforced after the mount, so the File
+  # directive above doesn't have user/group, and we manage it
+  # explicitly via chown.
+  exec { 'chown-slack-conversions':
+    command => '/bin/chown zulip:zulip /tmp/slack-conversions',
+    unless  => '/usr/bin/test "$(stat -c %U:%G /tmp/slack-conversions)" = "zulip:zulip"',
+    require => Mount['/tmp/slack-conversions'],
+  }
+}

puppet/kandra/manifests/aws_tools.pp

@@ -0,0 +1,90 @@
+# @summary Installs the AWS CLI
+#
+class kandra::aws_tools {
+  $is_ec2 = zulipconf('machine', 'hosting_provider', 'ec2') == 'ec2'
+
+  file { '/usr/local/bin/install-aws-cli':
+    ensure => file,
+    mode   => '0755',
+    source => 'puppet:///modules/kandra/install-aws-cli',
+  }
+  exec { 'install-aws-cli':
+    require => File['/usr/local/bin/install-aws-cli'],
+    command => '/usr/local/bin/install-aws-cli',
+    # When puppet is initially determining which resources need to be
+    # applied, it will call the unless -- but install-aws-cli may not
+    # exist yet.  Count this as needing to run.
+    unless  => '[ -f /usr/local/bin/install-aws-cli ] && /usr/local/bin/install-aws-cli check',
+  }
+
+  if ! $is_ec2 {
+    if $facts['os']['architecture'] == 'amd64' {
+      $archname = 'X86_64'
+    } else {
+      $archname = 'Aarch64'
+    }
+    $helper_version = $zulip::common::versions['aws_signing_helper']['version']
+    zulip::external_dep { 'aws_signing_helper':
+      version => $helper_version,
+      url     => "https://rolesanywhere.amazonaws.com/releases/${helper_version}/${archname}/Linux/Amzn2023/aws_signing_helper",
+      before  => File['/root/.aws/config'],
+    }
+    file { '/srv/zulip-aws-tools/bin/aws_signing_helper':
+      ensure  => link,
+      target  => "/srv/zulip-aws_signing_helper-${helper_version}",
+      require => [
+        File["/srv/zulip-aws_signing_helper-${helper_version}"],
+        Exec['install-aws-cli'],
+      ],
+      before  => Exec['Cleanup aws_signing_helper'],
+    }
+    file { '/usr/local/bin/teleport-aws-credentials':
+      ensure  => file,
+      require => [
+        Package['sqlite3'],
+        Service['teleport_node'],
+      ],
+      before  => [
+        File['/root/.aws/config'],
+      ],
+      mode    => '0755',
+      owner   => 'root',
+      group   => 'root',
+      source  => 'puppet:///modules/kandra/teleport-aws-credentials',
+    }
+  }
+  file { '/root/.aws':
+    ensure => directory,
+    mode   => '0755',
+    owner  => 'root',
+    group  => 'root',
+  }
+  $aws_trust_arn = zulipsecret('secrets','aws_trust_arn','')
+  $aws_profile_arn = zulipsecret('secrets','aws_profile_arn','')
+  $aws_role_arn = zulipsecret('secrets','aws_role_arn','')
+  file { '/root/.aws/config':
+    ensure  => file,
+    mode    => '0644',
+    owner   => 'root',
+    group   => 'root',
+    content => template('kandra/dotfiles/aws_config.erb'),
+  }
+
+  # Pull keys and authorized_keys from AWS secretsmanager
+  file { '/usr/local/bin/install-ssh-keys':
+    ensure  => file,
+    require => File['/root/.aws/config'],
+    mode    => '0755',
+    owner   => 'root',
+    group   => 'root',
+    source  => 'puppet:///modules/kandra/install-ssh-keys',
+  }
+  file { '/usr/local/bin/install-ssh-authorized-keys':
+    ensure  => file,
+    require => File['/root/.aws/config'],
+    mode    => '0755',
+    owner   => 'root',
+    group   => 'root',
+    source  => 'puppet:///modules/kandra/install-ssh-authorized-keys',
+  }
+}

puppet/kandra/manifests/camo.pp

@@ -0,0 +1,8 @@
+class kandra::camo {
+  class { 'zulip::camo':
+    listen_address => '0.0.0.0',
+  }
+
+  kandra::firewall_allow { 'camo': port => '9292' }
+  kandra::teleport::prometheus_app { 'camo': port => '9292' }
+}

puppet/kandra/manifests/firewall.pp

@@ -0,0 +1,56 @@
+class kandra::firewall {
+  package { 'iptables-persistent': }
+  concat { '/etc/iptables/rules.v4':
+    ensure  => present,
+    mode    => '0600',
+    require => Package['iptables-persistent'],
+  }
+  concat::fragment { 'iptables-header.v4':
+    target => '/etc/iptables/rules.v4',
+    source => 'puppet:///modules/kandra/iptables/header.v4',
+    order  => '01',
+  }
+  concat::fragment { 'iptables-trailer.v4':
+    target => '/etc/iptables/rules.v4',
+    source => 'puppet:///modules/kandra/iptables/trailer.v4',
+    order  => '99',
+  }
+
+  concat { '/etc/iptables/rules.v6':
+    ensure  => present,
+    mode    => '0600',
+    require => Package['iptables-persistent'],
+  }
+  concat::fragment { 'iptables-header.v6':
+    target => '/etc/iptables/rules.v6',
+    source => 'puppet:///modules/kandra/iptables/header.v6',
+    order  => '01',
+  }
+  concat::fragment { 'iptables-trailer.v6':
+    target => '/etc/iptables/rules.v6',
+    source => 'puppet:///modules/kandra/iptables/trailer.v6',
+    order  => '99',
+  }
+
+  service { 'netfilter-persistent':
+    ensure     => running,
+
+    # Because there is no running process for this service, the normal status
+    # checks fail.  Because Puppet then thinks the service has been manually
+    # stopped, it won't restart it.  This fake status command will trick Puppet
+    # into thinking the service is *always* running (which in a way it is, as
+    # iptables is part of the kernel.)
+    hasstatus  => true,
+    status     => '/bin/true',
+
+    # Under Debian, the "restart" parameter does not reload the rules, so tell
+    # Puppet to fall back to stop/start, which does work.
+    hasrestart => false,
+
+    require    => Package['iptables-persistent'],
+    subscribe  => [
+      Concat['/etc/iptables/rules.v4'],
+      Concat['/etc/iptables/rules.v6'],
+    ],
+  }
+}

puppet/kandra/manifests/firewall_allow.pp

@@ -0,0 +1,27 @@
+# @summary Adds an iptables "allow" rule for the host for a port.
+#
+# Rules with the same ordering are ordered by the rule name.
+#
+define kandra::firewall_allow (
+  $port = '',
+  $proto = 'tcp',
+  $order = '50',
+) {
+  if $port == '' {
+    $portname = $name
+  } else {
+    $portname = $port
+  }
+
+  concat::fragment { "iptables_v4_${portname}":
+    target  => '/etc/iptables/rules.v4',
+    order   => $order,
+    content => "-A INPUT -p ${proto} --dport ${portname} -j ACCEPT -m comment --comment \"${name}\"\n",
+  }
+
+  concat::fragment { "iptables_v6_${portname}":
+    target  => '/etc/iptables/rules.v6',
+    order   => $order,
+    content => "-A INPUT -p ${proto} --dport ${portname} -j ACCEPT -m comment --comment \"${name}\"\n",
+  }
+}

puppet/kandra/manifests/hooks/file.pp

@@ -0,0 +1,14 @@
+# @summary Install a static hook file
+#
+define kandra::hooks::file() {
+  include zulip::hooks::base
+
+  file { "/etc/zulip/hooks/${title}":
+    ensure => file,
+    mode   => '0755',
+    owner  => 'zulip',
+    group  => 'zulip',
+    source => "puppet:///modules/kandra/hooks/${title}",
+    tag    => ['hooks'],
+  }
+}

puppet/kandra/manifests/hooks/zulip_notify_schema_diff.pp

@@ -0,0 +1,8 @@
+# @summary Install hook that checks for schema drift from published ref
+#
+class kandra::hooks::zulip_notify_schema_diff {
+  include zulip::hooks::base
+  include zulip::hooks::zulip_common
+
+  kandra::hooks::file { 'post-deploy.d/zulip_notify_schema_diff.hook': }
+}

puppet/kandra/manifests/ksplice_uptrack.pp

@@ -0,0 +1,31 @@
+class kandra::ksplice_uptrack {
+  $ksplice_access_key = zulipsecret('secrets', 'ksplice_access_key', '')
+  if $ksplice_access_key != '' {
+    file { '/etc/uptrack':
+      ensure => directory,
+      owner  => 'root',
+      group  => 'adm',
+      mode   => '0750',
+    }
+    file { '/etc/uptrack/uptrack.conf':
+      ensure  => file,
+      owner   => 'root',
+      group   => 'adm',
+      mode    => '0640',
+      content => template('kandra/uptrack/uptrack.conf.erb'),
+    }
+    $setup_apt_repo_file = "${facts['zulip_scripts_path']}/lib/setup-apt-repo"
+    exec{ 'setup-apt-repo-ksplice':
+      command => "${setup_apt_repo_file} --list ksplice",
+      unless  => "${setup_apt_repo_file} --list ksplice --verify",
+    }
+    Package { 'uptrack':
+      require => [
+        Exec['setup-apt-repo-ksplice'],
+        File['/etc/uptrack/uptrack.conf'],
+      ],
+    }
+  } else {
+    warning('No ksplice uptrack key is configured; ksplice is not installed!')
+  }
+}

puppet/kandra/manifests/mirror_to_czo.pp

@@ -0,0 +1,34 @@
+class kandra::mirror_to_czo {
+  include zulip::hooks::base
+  include zulip::supervisor
+
+  # We embed the hash of the contents into the name of the process, so
+  # that `supervisorctl reread` knows that it has updated.
+  $full_script_hash = sha256(file('kandra/mirror_to_czo'))
+  $script_hash = $full_script_hash[0,8]
+
+  $bin = '/usr/local/bin/mirror_to_czo'
+  file { $bin:
+    ensure => file,
+    owner  => 'root',
+    group  => 'root',
+    mode   => '0755',
+    source => 'puppet:///modules/kandra/mirror_to_czo',
+  }
+
+  file { "${zulip::common::supervisor_conf_dir}/mirror_to_czo.conf":
+    ensure  => file,
+    require => [
+      User[zulip],
+      Package[supervisor],
+      File[$bin],
+    ],
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    content => template('kandra/supervisor/conf.d/mirror_to_czo.conf.template.erb'),
+    notify  => Service[supervisor],
+  }
+
+  kandra::hooks::file { 'post-deploy.d/restart_mirror_to_czo.hook': }
+}

puppet/kandra/manifests/prod_app_frontend_once.pp

@@ -0,0 +1,29 @@
+class kandra::prod_app_frontend_once {
+  include zulip::app_frontend_once
+  include zulip::hooks::push_git_ref
+  include zulip::hooks::zulip_notify
+  include kandra::hooks::zulip_notify_schema_diff
+  include kandra::mirror_to_czo
+
+  zulip::cron { 'update-first-visible-message-id':
+    hour   => '7',
+    minute => '0',
+    manage => 'calculate_first_visible_message_id --lookback-hours 30',
+  }
+
+  zulip::cron { 'invoice-plans':
+    hour   => '22',
+    minute => '0',
+  }
+  zulip::cron { 'downgrade-small-realms-behind-on-payments':
+    hour   => '17',
+    minute => '0',
+  }
+
+  zulip::cron { 'check_send_receive_time':
+    hour      => '*',
+    minute    => '*',
+    command   => '/usr/lib/nagios/plugins/zulip_app_frontend/check_send_receive_time',
+    use_proxy => false,
+  }
+}

puppet/kandra/manifests/profile/base.pp

@@ -0,0 +1,126 @@
+class kandra::profile::base {
+  include zulip::profile::base
+  include kandra::ksplice_uptrack
+  include kandra::firewall
+  include kandra::teleport::node
+  include kandra::prometheus::node
+
+  kandra::firewall_allow { 'ssh': order => '10'}
+  $is_ec2 = zulipconf('machine', 'hosting_provider', 'ec2') == 'ec2'
+
+  $org_base_packages = [
+    # Standard kernel, not AWS', so ksplice works
+    'linux-image-virtual',
+    # Management for our systems
+    'openssh-server',
+    # package management
+    'aptitude',
+    # SSL certificates
+    'certbot',
+    # For managing our current Debian packages
+    'debian-goodies',
+    # Popular editors
+    'vim',
+    'emacs-nox',
+    # Prevent accidental reboots
+    'molly-guard',
+    # For extracting Teleport certs
+    'sqlite3',
+    # Useful tools in a production environment
+    'screen',
+    'strace',
+    'bind9-host',
+    'git',
+    'nagios-plugins-contrib',
+    'ripgrep',
+    'bat',
+  ]
+  zulip::safepackage { $org_base_packages: ensure => installed }
+  file { '/usr/local/bin/bat':
+    ensure => link,
+    target => '/usr/bin/batcat',
+  }
+
+  # Uninstall the AWS kernel, but only after we install the usual one
+  package { ['linux-image-aws', 'linux-headers-aws', 'linux-aws-*', 'linux-image-*-aws', 'linux-modules-*-aws']:
+    ensure  => absent,
+    require => Package['linux-image-virtual'],
+  }
+
+  file { '/etc/apt/apt.conf.d/02periodic':
+    ensure => file,
+    mode   => '0644',
+    source => 'puppet:///modules/kandra/apt/apt.conf.d/02periodic',
+  }
+
+  file { '/etc/apt/apt.conf.d/50unattended-upgrades':
+    ensure => file,
+    mode   => '0644',
+    source => 'puppet:///modules/kandra/apt/apt.conf.d/50unattended-upgrades',
+  }
+  file { '/etc/needrestart/conf.d/zulip.conf':
+    ensure => file,
+    mode   => '0644',
+    source => 'puppet:///modules/kandra/needrestart/zulip.conf',
+  }
+
+  user { 'root': }
+  kandra::user_dotfiles { 'root':
+    home            => '/root',
+    keys            => 'internal-read-only-deploy-key',
+    authorized_keys => 'common',
+    known_hosts     => ['github.com'],
+  }
+
+  kandra::user_dotfiles { 'zulip':
+    keys            => 'internal-read-only-deploy-key',
+    authorized_keys => 'common',
+    known_hosts     => ['github.com'],
+  }
+
+  service { 'ssh':
+    ensure => running,
+  }
+
+  include kandra::aws_tools
+
+  if $is_ec2 {
+    # EC2 hosts can use the in-VPC timeserver
+    file { '/etc/chrony/chrony.conf':
+      ensure  => file,
+      mode    => '0644',
+      source  => 'puppet:///modules/kandra/chrony.conf',
+      require => Package['chrony'],
+      notify  => Service['chrony'],
+    }
+  }
+
+  group { 'nagios':
+    ensure => present,
+    gid    => '1050',
+  }
+  user { 'nagios':
+    ensure     => present,
+    uid        => '1050',
+    gid        => '1050',
+    shell      => '/bin/bash',
+    home       => '/var/lib/nagios',
+    managehome => true,
+  }
+  file { '/var/lib/nagios':
+    ensure  => directory,
+    require => User['nagios'],
+    owner   => 'nagios',
+    group   => 'nagios',
+    mode    => '0700',
+  }
+  kandra::user_dotfiles { 'nagios':
+    home            => '/var/lib/nagios',
+    authorized_keys => true,
+  }
+  file { '/home/nagios':
+    ensure  => absent,
+    force   => true,
+    recurse => true,
+  }
+}

puppet/kandra/manifests/profile/chat_zulip_org.pp

@@ -0,0 +1,23 @@
+class kandra::profile::chat_zulip_org inherits kandra::profile::base {
+  include zulip::profile::standalone
+  include zulip::local_mailserver
+  include zulip::hooks::sentry
+  include zulip::hooks::zulip_notify
+
+  include kandra::app_frontend_monitoring
+  include kandra::prometheus::redis
+  include kandra::prometheus::postgresql
+  kandra::teleport::prometheus_app { 'smokescreen': port => '4760' }
+  kandra::teleport::prometheus_app { 'camo': port => '9292' }
+
+  kandra::firewall_allow { 'http': }
+  kandra::firewall_allow { 'https': }
+  kandra::firewall_allow { 'smtp': }
+
+  Kandra::User_Dotfiles['root'] {
+    keys => false,
+  }
+  Kandra::User_Dotfiles['zulip'] {
+    keys => false,
+  }
+}

puppet/kandra/manifests/profile/grafana.pp

@@ -0,0 +1,78 @@
+# @summary Observability using Grafana
+#
+class kandra::profile::grafana inherits kandra::profile::base {
+
+  include zulip::supervisor
+
+  $version = $zulip::common::versions['grafana']['version']
+  $dir = "/srv/zulip-grafana-${version}"
+  $bin = "${dir}/bin/grafana-server"
+  $data_dir = '/var/lib/grafana'
+
+  zulip::external_dep { 'grafana':
+    version        => $version,
+    url            => "https://dl.grafana.com/oss/release/grafana-${version}.linux-${zulip::common::goarch}.tar.gz",
+    tarball_prefix => "grafana-${version}",
+    bin            => [$bin],
+    cleanup_after  => [Service[supervisor]],
+  }
+
+  group { 'grafana':
+    ensure => present,
+    gid    => '1070',
+  }
+  user { 'grafana':
+    ensure     => present,
+    uid        => '1070',
+    gid        => '1070',
+    shell      => '/bin/bash',
+    home       => $data_dir,
+    managehome => false,
+  }
+  file { $data_dir:
+    ensure  => directory,
+    owner   => 'grafana',
+    group   => 'grafana',
+    require => [ User[grafana], Group[grafana] ],
+  }
+  file { '/var/log/grafana':
+    ensure => directory,
+    owner  => 'grafana',
+    group  => 'grafana',
+  }
+
+  kandra::teleport::application { 'monitoring': port => '3000' }
+  file { "${zulip::common::supervisor_conf_dir}/grafana.conf":
+    ensure  => file,
+    require => [
+      Package[supervisor],
+      File[$bin],
+      File[$data_dir],
+      File['/var/log/grafana'],
+    ],
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    content => template('kandra/supervisor/conf.d/grafana.conf.erb'),
+    notify  => Service[supervisor],
+  }
+
+  $email_host = zulipconf('grafana', 'email_host', '')
+  $email_from = zulipconf('grafana', 'email_from', '')
+  $email_user = zulipsecret('secrets', 'grafana_email_user', '')
+  $email_password = zulipsecret('secrets', 'grafana_email_password', '')
+  file { '/etc/grafana':
+    ensure => directory,
+    owner  => 'root',
+    group  => 'root',
+    mode   => '0755',
+  }
+  file { '/etc/grafana/grafana.ini':
+    ensure  => file,
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    content => template('kandra/grafana.ini.template.erb'),
+    notify  => Service[supervisor],
+  }
+}

puppet/kandra/manifests/profile/nagios.pp

@@ -0,0 +1,178 @@
+class kandra::profile::nagios inherits kandra::profile::base {
+
+  include kandra::apache
+
+  package { ['nagios4', 'msmtp', 'autossh']: ensure => installed }
+  $nagios_alert_email = zulipconf('nagios', 'alert_email', undef)
+  $nagios_test_email = zulipconf('nagios', 'test_email', undef)
+  $nagios_pager_email = zulipconf('nagios', 'pager_email', undef)
+
+  $nagios_mail_domain = zulipconf('nagios', 'mail_domain', undef)
+  $nagios_mail_host = zulipconf('nagios', 'mail_host', undef)
+  $nagios_mail_password = zulipsecret('secrets', 'nagios_mail_password', '')
+  if zulipconf('nagios', 'camo_check_url', undef) =~ /^https:\/\/([^\/]*)(\/.*)$/ {
+    $nagios_camo_check_host = $1
+    $nagios_camo_check_path = $2
+  }
+
+  $default_host_domain = zulipconf('nagios', 'default_host_domain', undef)
+  $hosts_app_prod = split(zulipconf('nagios', 'hosts_app_prod', undef), ',')
+  $hosts_app_staging = split(zulipconf('nagios', 'hosts_app_staging', undef), ',')
+  $hosts_postgresql_primary = split(zulipconf('nagios', 'hosts_postgresql_primary', undef), ',')
+  $hosts_postgresql_replica = split(zulipconf('nagios', 'hosts_postgresql_replica', undef), ',')
+  $hosts_redis = split(zulipconf('nagios', 'hosts_redis', undef), ',')
+  $hosts_fullstack = split(zulipconf('nagios', 'hosts_fullstack', undef), ',')
+  $hosts_smokescreen = split(zulipconf('nagios', 'hosts_smokescreen', undef), ',')
+  $hosts_other = split(zulipconf('nagios', 'hosts_other', undef), ',')
+
+  $hosts = zulipconf_nagios_hosts()
+  $qualified_hosts = $hosts.map |$h| { if '.' in $h { $h } else { "${h}.${default_host_domain}" }}
+  Kandra::User_Dotfiles['nagios'] {
+    keys        => 'nagios',
+    known_hosts => $qualified_hosts,
+  }
+
+  file { '/etc/nagios4/':
+    recurse => true,
+    purge   => false,
+    require => Package[nagios4],
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    source  => 'puppet:///modules/kandra/nagios4/',
+    notify  => Service['nagios4'],
+  }
+
+  file { '/etc/apache2/sites-available/nagios.conf':
+    purge   => false,
+    require => Package[apache2],
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0640',
+    content => template('kandra/nagios_apache_site.conf.template.erb'),
+  }
+  apache2site { 'nagios':
+    ensure  => present,
+    require => [
+      File['/etc/apache2/sites-available/nagios.conf'],
+      Apache2mod['headers'], Apache2mod['ssl'],
+    ],
+    notify  => Service['apache2'],
+  }
+  kandra::teleport::application{ 'nagios':
+    description => 'Monitoring: nagios',
+    port        => '3000',
+  }
+
+  file { '/etc/nagios4/conf.d/contacts.cfg':
+    require => Package[nagios4],
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    content => template('kandra/nagios4/contacts.cfg.template.erb'),
+    notify  => Service['nagios4'],
+  }
+  file { '/etc/nagios4/conf.d/hosts.cfg':
+    require => Package[nagios4],
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    content => template('kandra/nagios4/hosts.cfg.template.erb'),
+    notify  => Service['nagios4'],
+  }
+  file { '/etc/nagios4/conf.d/localhost.cfg':
+    require => Package[nagios4],
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    content => template('kandra/nagios4/localhost.cfg.template.erb'),
+    notify  => Service['nagios4'],
+  }
+
+  file { '/etc/nagios4/cgi.cfg':
+    require => Package[nagios4],
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    content => template('kandra/nagios4/cgi.cfg.template.erb'),
+    notify  => Service['nagios4'],
+  }
+
+  service { 'nagios4':
+    ensure  => running,
+    require => Kandra::User_Dotfiles['nagios'],
+  }
+
+  file { [
+    '/etc/nagios4/conf.d/extinfo_nagios2.cfg',
+    '/etc/nagios4/conf.d/services_nagios2.cfg',
+    '/etc/nagios4/conf.d/contacts_nagios2.cfg',
+    '/etc/nagios4/conf.d/hostgroups_nagios2.cfg',
+    '/etc/nagios4/conf.d/localhost_nagios2.cfg',
+    '/etc/nagios4/conf.d/zulip_nagios.cfg',
+  ]:
+    ensure => absent,
+  }
+
+  file { "${zulip::common::supervisor_conf_dir}/autossh_tunnels.conf":
+    ensure  => file,
+    require => [
+      Package['supervisor', 'autossh'],
+      Kandra::User_Dotfiles['nagios'],
+    ],
+    mode    => '0644',
+    owner   => 'root',
+    group   => 'root',
+    content => template('kandra/supervisor/conf.d/autossh_tunnels.conf.erb'),
+    notify  => Service['supervisor'],
+  }
+  file { '/etc/nagios4/conf.d/zulip_autossh.cfg':
+    ensure  => file,
+    mode    => '0644',
+    owner   => 'root',
+    group   => 'root',
+    content => template('kandra/nagios_autossh.template.erb'),
+    notify  => Service['nagios4'],
+  }
+
+  file { '/var/lib/nagios/msmtprc':
+    ensure  => file,
+    mode    => '0600',
+    owner   => 'nagios',
+    group   => 'nagios',
+    content => template('kandra/msmtprc_nagios.template.erb'),
+    require => File['/var/lib/nagios'],
+  }
+
+  file { '/var/lib/nagios/.ssh/config':
+    ensure => file,
+    mode   => '0644',
+    owner  => 'nagios',
+    group  => 'nagios',
+    source => 'puppet:///modules/kandra/nagios_ssh_config',
+  }
+
+  # Disable apparmor for msmtp so it can read the above config file
+  file { '/etc/apparmor.d/disable/usr.bin.msmtp':
+    ensure => link,
+    target => '/etc/apparmor.d/usr.bin.msmtp',
+    notify => Exec['reload apparmor'],
+  }
+  exec {'reload apparmor':
+    command     => '/sbin/apparmor_parser -R /etc/apparmor.d/usr.bin.msmtp',
+    refreshonly => true,
+  }
+
+  exec { 'fix_nagios_permissions':
+    command => 'dpkg-statoverride --update --add nagios www-data 2710 /var/lib/nagios4/rw',
+    unless  => 'bash -c "ls -ld /var/lib/nagios4/rw | grep ^drwx--s--- -q"',
+    notify  => Service['nagios4'],
+  }
+  exec { 'fix_nagios_permissions2':
+    command => 'dpkg-statoverride --update --add nagios nagios 751 /var/lib/nagios4',
+    unless  => 'bash -c "ls -ld /var/lib/nagios4 | grep ^drwxr-x--x -q"',
+    notify  => Service['nagios4'],
+  }
+
+  # TODO: Install our API
+}

puppet/kandra/manifests/profile/postgresql.pp

@@ -0,0 +1,74 @@
+class kandra::profile::postgresql inherits kandra::profile::base {
+  include kandra::teleport::db
+  include kandra::prometheus::postgresql
+
+  # We key off of `listen_addresses` being set to know if we should be starting PostgreSQL.
+  $listen_addresses = zulipconf('postgresql', 'listen_addresses', undef)
+  $is_stage1 = ($listen_addresses == undef)
+  class { 'zulip::profile::postgresql':
+    start => ! $is_stage1,
+  }
+
+  package { ['xfsprogs', 'nvme-cli']: ensure => installed }
+
+  kandra::firewall_allow{ 'postgresql': }
+
+  zulip::sysctl { 'postgresql-swappiness':
+    key   => 'vm.swappiness',
+    value => '0',
+  }
+  zulip::sysctl { 'postgresql-overcommit':
+    key   => 'vm.overcommit_memory',
+    value => '2',
+  }
+
+  file { '/root/setup_disks.sh':
+    ensure => file,
+    owner  => 'root',
+    group  => 'root',
+    mode   => '0744',
+    source => 'puppet:///modules/kandra/postgresql/setup_disks.sh',
+  }
+  exec { 'setup_disks':
+    command => '/root/setup_disks.sh',
+    # We need to not have started installing the non-AWS kernel, so
+    # the xfs module gets installed for the running kernel, and we can
+    # mount it.
+    before  => Package['linux-image-virtual'],
+    require => Package["postgresql-${zulip::postgresql_common::version}", 'xfsprogs', 'nvme-cli'],
+    unless  => 'test /var/lib/postgresql/ -ef /srv/data/postgresql/',
+  }
+
+  # This is the second stage, after secrets are configured
+  if (! $is_stage1) {
+    $replication_primary = zulipconf('postgresql', 'replication_primary', undef)
+    $replication_user = zulipconf('postgresql', 'replication_user', undef)
+    if $replication_primary != undef and $replication_user != undef {
+      file { '/root/setup_data.sh':
+        ensure => file,
+        owner  => 'root',
+        group  => 'root',
+        mode   => '0744',
+        source => 'puppet:///modules/kandra/postgresql/setup_data.sh',
+      }
+      exec { 'setup_data':
+        command => '/root/setup_data.sh',
+        require => [File['/usr/local/bin/env-wal-g'], Exec['setup_disks']],
+        unless  => "test -d /srv/data/postgresql/${zulip::postgresql_common::version}/main",
+        timeout => 0,
+        before  => File["${zulip::postgresql_base::postgresql_datadir}/standby.signal"],
+        notify  => Service['postgresql'],
+      }
+    }
+  }
+
+  file { "${zulip::postgresql_base::postgresql_confdir}/pg_hba.conf":
+    ensure  => file,
+    require => Package["postgresql-${zulip::postgresql_common::version}"],
+    notify  => Service['postgresql'],
+    owner   => 'postgres',
+    group   => 'postgres',
+    mode    => '0640',
+    source  => 'puppet:///modules/kandra/postgresql/pg_hba.conf',
+  }
+}

puppet/kandra/manifests/profile/prod_app_frontend.pp

@@ -0,0 +1,37 @@
+class kandra::profile::prod_app_frontend inherits kandra::profile::base {
+  include kandra::app_frontend
+  include zulip::hooks::zulip_notify
+
+  Kandra::User_Dotfiles['root'] {
+    keys => 'internal-limited-write-deploy-key',
+  }
+  Kandra::User_Dotfiles['zulip'] {
+    keys => 'internal-limited-write-deploy-key',
+  }
+
+  zulip::sysctl { 'conntrack':
+    comment => 'Increase conntrack kernel table size',
+    key     => 'net.nf_conntrack_max',
+    value   => zulipconf('application_server', 'conntrack_max', 262144),
+  }
+
+  file { '/etc/nginx/sites-available/zulip':
+    ensure  => file,
+    require => [Package['nginx-full'], Exec['generate-default-snakeoil']],
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    source  => 'puppet:///modules/kandra/nginx/sites-available/zulip',
+    notify  => Service['nginx'],
+  }
+
+  file { '/etc/nginx/sites-enabled/zulip':
+    ensure  => link,
+    require => Package['nginx-full'],
+    target  => '/etc/nginx/sites-available/zulip',
+    notify  => Service['nginx'],
+  }
+
+  # Prod has our Apple Push Notifications Service private key at
+  # /etc/ssl/django-private/apns-dist.pem
+}

puppet/kandra/manifests/profile/prometheus_server.pp

@@ -0,0 +1,87 @@
+# @summary Gathers Prometheus statistics from all nodes.
+#
+# Only one instance is necessary.
+#
+class kandra::profile::prometheus_server inherits kandra::profile::base {
+
+  include kandra::prometheus::base
+
+  # Service discovery via Teleport
+  include kandra::teleport::tbot
+  include kandra::teleport::sd
+
+  # This blackbox monitoring of the backup system runs locally
+  include kandra::prometheus::wal_g
+
+  # Ditto the Akamai logs
+  include kandra::prometheus::akamai
+
+  # Ditto Weblate
+  include kandra::prometheus::weblate
+
+  # The SES log ETL (writing to S3) runs on vector
+  include kandra::ses_logs
+
+  # Export prometheus stats to status.zulip.com
+  include kandra::statuspage
+
+  $version = $zulip::common::versions['prometheus']['version']
+  $dir = "/srv/zulip-prometheus-${version}"
+  $bin = "${dir}/prometheus"
+  $data_dir = '/var/lib/prometheus'
+
+  zulip::external_dep { 'prometheus':
+    version        => $version,
+    url            => "https://github.com/prometheus/prometheus/releases/download/v${version}/prometheus-${version}.linux-${zulip::common::goarch}.tar.gz",
+    tarball_prefix => "prometheus-${version}.linux-${zulip::common::goarch}",
+    bin            => [$bin, "${dir}/promtool"],
+    cleanup_after  => [Service[supervisor]],
+  }
+  file { '/usr/local/bin/promtool':
+    ensure  => link,
+    target  => "${dir}/promtool",
+    require => File["${dir}/promtool"],
+    before  => Exec['Cleanup prometheus'],
+  }
+
+  file { $data_dir:
+    ensure  => directory,
+    owner   => 'prometheus',
+    group   => 'prometheus',
+    require => [ User[prometheus], Group[prometheus] ],
+  }
+  file { '/etc/prometheus':
+    ensure => directory,
+    owner  => 'root',
+    group  => 'root',
+    mode   => '0755',
+  }
+
+  $czo = zulipconf('prometheus', 'czo', '')
+  $other_hosts = split(zulipconf('prometheus', 'other_hosts', ''), ',')
+  $backup_buckets = split(zulipconf('prometheus', 'walg_buckets', ''), ',')
+  file { '/etc/prometheus/prometheus.yaml':
+    ensure  => file,
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    content => template('kandra/prometheus/prometheus.yaml.template.erb'),
+    notify  => Service[supervisor],
+  }
+
+  file { "${zulip::common::supervisor_conf_dir}/prometheus.conf":
+    ensure  => file,
+    require => [
+      Package[supervisor],
+      File[$bin],
+      File[$data_dir],
+      File['/etc/prometheus/prometheus.yaml'],
+      Service['tbot'],
+    ],
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    content => template('kandra/supervisor/conf.d/prometheus.conf.template.erb'),
+    notify  => Service[supervisor],
+  }
+}

puppet/kandra/manifests/profile/redis.pp

@@ -0,0 +1,35 @@
+class kandra::profile::redis inherits kandra::profile::base {
+  include zulip::profile::redis
+  include kandra::prometheus::redis
+
+  zulip::sysctl { 'redis-somaxconn':
+    key   => 'net.core.somaxconn',
+    value => '65535',
+  }
+
+  # Need redis_password in its own file for Nagios
+  file { '/var/lib/nagios/redis_password':
+    ensure  => file,
+    mode    => '0600',
+    owner   => 'nagios',
+    group   => 'nagios',
+    content => "${zulip::profile::redis::redis_password}\n",
+  }
+
+  group { 'redistunnel':
+    ensure => present,
+    gid    => '1080',
+  }
+  user { 'redistunnel':
+    ensure     => present,
+    uid        => '1080',
+    gid        => '1080',
+    groups     => ['zulip'],
+    shell      => '/bin/true',
+    home       => '/home/redistunnel',
+    managehome => true,
+  }
+  kandra::user_dotfiles { 'redistunnel':
+    authorized_keys => true,
+  }
+}

puppet/kandra/manifests/profile/smokescreen.pp

@@ -0,0 +1,9 @@
+class kandra::profile::smokescreen inherits kandra::profile::base {
+
+
+  include zulip::profile::smokescreen
+  kandra::firewall_allow { 'smokescreen': port => '4750' }
+  kandra::teleport::prometheus_app { 'smokescreen': port => '4760' }
+
+  include kandra::camo
+}

puppet/kandra/manifests/profile/staging_app_frontend.pp

@@ -0,0 +1,28 @@
+class kandra::profile::staging_app_frontend inherits kandra::profile::base {
+
+  include kandra::app_frontend
+
+  file { '/etc/nginx/sites-available/zulip-staging':
+    ensure  => file,
+    require => [Package['nginx-full'], Exec['generate-default-snakeoil']],
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    source  => 'puppet:///modules/kandra/nginx/sites-available/zulip-staging',
+    notify  => Service['nginx'],
+  }
+  file { '/etc/nginx/sites-enabled/zulip-staging':
+    ensure  => link,
+    require => Package['nginx-full'],
+    target  => '/etc/nginx/sites-available/zulip-staging',
+    notify  => Service['nginx'],
+  }
+
+  # Eventually, this will go in a staging_app_frontend_once.pp
+  zulip::cron { 'check_send_receive_time':
+    hour      => '*',
+    minute    => '*',
+    command   => '/usr/lib/nagios/plugins/zulip_app_frontend/check_send_receive_time',
+    use_proxy => false,
+  }
+}

puppet/kandra/manifests/profile/teleport.pp

@@ -0,0 +1,23 @@
+class kandra::profile::teleport inherits kandra::profile::base {
+
+
+  file { '/etc/teleport_server.yaml':
+    owner  => 'root',
+    group  => 'root',
+    mode   => '0644',
+    source => 'puppet:///modules/kandra/teleport_server.yaml',
+    before => Kandra::Teleport::Part['server'],
+    notify => Exec['reload teleport_server'],
+  }
+  file { '/var/lib/teleport-server':
+    ensure => directory,
+    owner  => 'root',
+    group  => 'root',
+    mode   => '0700',
+    before => Kandra::Teleport::Part['server'],
+  }
+  kandra::teleport::part { 'server': }
+
+  # https://goteleport.com/docs/reference/deployment/networking/#ports-with-tls-routing
+  kandra::firewall_allow { 'teleport_server_ui': port => 443 }
+}

puppet/kandra/manifests/profile/zulipbot_zulip_org.pp

@@ -0,0 +1,8 @@
+class kandra::profile::zulipbot_zulip_org inherits kandra::profile::base {
+
+  kandra::firewall_allow { 'http': }
+  kandra::firewall_allow { 'https': }
+
+  # TODO: This does not do any configuration of zulipbot itself, or of
+  # caddy.
+}

puppet/kandra/manifests/prometheus/akamai.pp

@@ -0,0 +1,23 @@
+# @summary Prometheus monitoring of Akamai access logs
+#
+class kandra::prometheus::akamai {
+  include kandra::prometheus::base
+  include kandra::vector
+  include zulip::supervisor
+
+  $pipelines = {
+    'static' => zulipsecret('secrets', 'akamai_static_sqs_url', ''),
+    'realm'  => zulipsecret('secrets', 'akamai_realm_sqs_url', ''),
+  }
+
+  concat::fragment { 'vector_akamai':
+    target  => $kandra::vector::conf,
+    order   => '50',
+    content => template('kandra/vector_akamai.toml.template.erb'),
+  }
+  concat::fragment { 'vector_akamai_export':
+    target  => $kandra::vector::conf,
+    content => ',"akamai_logs2metrics*"',
+    order   => '90',
+  }
+}

puppet/kandra/manifests/prometheus/base.pp

@@ -0,0 +1,16 @@
+# @summary Configures a node for monitoring with Prometheus
+#
+class kandra::prometheus::base {
+  group { 'prometheus':
+    ensure => present,
+    gid    => '1060',
+  }
+  user { 'prometheus':
+    ensure     => present,
+    uid        => '1060',
+    gid        => '1060',
+    shell      => '/bin/bash',
+    home       => '/nonexistent',
+    managehome => false,
+  }
+}

puppet/kandra/manifests/prometheus/grok.pp

@@ -0,0 +1,45 @@
+# @summary Parses nginx access_log files
+#
+class kandra::prometheus::grok {
+  include kandra::prometheus::base
+  include zulip::supervisor
+
+  $version = $zulip::common::versions['grok_exporter']['version']
+  $dir = "/srv/zulip-grok_exporter-${version}"
+  $bin = "${dir}/grok_exporter"
+
+  zulip::external_dep { 'grok_exporter':
+    version        => $version,
+    url            => "https://github.com/fstab/grok_exporter/releases/download/v${version}/grok_exporter-${version}.linux-${zulip::common::goarch}.zip",
+    tarball_prefix => "grok_exporter-${version}.linux-${zulip::common::goarch}",
+    bin            => [$bin],
+    cleanup_after  => [Service[supervisor]],
+  }
+
+  $realm_names_regex = zulipconf('grok_exporter', 'realm_names_regex', '__impossible__')
+  $include_dir = "${dir}/patterns"
+  file { '/etc/grok_exporter.yaml':
+    ensure  => file,
+    owner   => zulip,
+    group   => zulip,
+    mode    => '0644',
+    content => template('kandra/prometheus/grok_exporter.yaml.template.erb'),
+    notify  => Service[supervisor],
+  }
+
+  kandra::teleport::prometheus_app { 'grok_exporter': port => '9144' }
+  file { "${zulip::common::supervisor_conf_dir}/prometheus_grok_exporter.conf":
+    ensure  => file,
+    require => [
+      User[zulip],
+      Package[supervisor],
+      File[$bin],
+      File['/etc/grok_exporter.yaml'],
+    ],
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    content => template('kandra/supervisor/conf.d/prometheus_grok_exporter.conf.template.erb'),
+    notify  => Service[supervisor],
+  }
+}

puppet/kandra/manifests/prometheus/memcached.pp

@@ -0,0 +1,39 @@
+# @summary Export memcached stats, with SASL auth
+#
+# We cannot use the stock
+# https://github.com/prometheus/memcached_exporter because it does not
+# support SASL auth, which we require.  Re-implement it in Python,
+# using bmemcached.
+class kandra::prometheus::memcached {
+  include kandra::prometheus::base
+  include zulip::supervisor
+
+  # We embed the hash of the contents into the name of the process, so
+  # that `supervisorctl reread` knows that it has updated.
+  $full_exporter_hash = sha256(file('kandra/memcached_exporter'))
+  $exporter_hash = $full_exporter_hash[0,8]
+
+  $bin = '/usr/local/bin/memcached_exporter'
+  file { $bin:
+    ensure => file,
+    owner  => 'root',
+    group  => 'root',
+    mode   => '0755',
+    source => 'puppet:///modules/kandra/memcached_exporter',
+  }
+
+  kandra::teleport::prometheus_app { 'memcached_exporter': port => '11212' }
+  file { "${zulip::common::supervisor_conf_dir}/memcached_exporter.conf":
+    ensure  => file,
+    require => [
+      User[zulip],
+      Package[supervisor],
+      File[$bin],
+    ],
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    content => template('kandra/supervisor/conf.d/memcached_exporter.conf.template.erb'),
+    notify  => Service[supervisor],
+  }
+}

puppet/kandra/manifests/prometheus/node.pp

@@ -0,0 +1,33 @@
+# @summary Configures a node for monitoring with Prometheus
+#
+class kandra::prometheus::node {
+  include kandra::prometheus::base
+  include zulip::supervisor
+
+  $version = $zulip::common::versions['node_exporter']['version']
+  $dir = "/srv/zulip-node_exporter-${version}"
+  $bin = "${dir}/node_exporter"
+
+  zulip::external_dep { 'node_exporter':
+    version        => $version,
+    url            => "https://github.com/prometheus/node_exporter/releases/download/v${version}/node_exporter-${version}.linux-${zulip::common::goarch}.tar.gz",
+    tarball_prefix => "node_exporter-${version}.linux-${zulip::common::goarch}",
+    bin            => [$bin],
+    cleanup_after  => [Service[supervisor]],
+  }
+
+  kandra::teleport::prometheus_app { 'node_exporter': port => '9100' }
+  file { "${zulip::common::supervisor_conf_dir}/prometheus_node_exporter.conf":
+    ensure  => file,
+    require => [
+      User[zulip],
+      Package[supervisor],
+      File[$bin],
+    ],
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    content => template('kandra/supervisor/conf.d/prometheus_node_exporter.conf.template.erb'),
+    notify  => Service[supervisor],
+  }
+}

puppet/kandra/manifests/prometheus/postgresql.pp

@@ -0,0 +1,83 @@
+# @summary Prometheus monitoring of postgresql servers
+#
+class kandra::prometheus::postgresql {
+  include kandra::prometheus::base
+  include zulip::supervisor
+  include zulip::golang
+
+  $version = $zulip::common::versions['postgres_exporter-src']['version']
+  $dir = "/srv/zulip-postgres_exporter-src-${version}"
+  $bin = "/usr/local/bin/postgres_exporter-${version}-go-${zulip::golang::version}"
+
+  # Binary builds: https://github.com/prometheus-community/postgres_exporter/releases/download/v${version}/postgres_exporter-${version}.linux-${zulip::common::goarch}.tar.gz
+
+  zulip::external_dep { 'postgres_exporter-src':
+    version        => $version,
+    url            => "https://github.com/alexmv/postgres_exporter/archive/${version}.tar.gz",
+    tarball_prefix => "postgres_exporter-${version}",
+  }
+
+  exec { 'compile postgres_exporter':
+    command     => "make build && cp ./postgres_exporter ${bin}",
+    cwd         => $dir,
+    # GOCACHE is required; nothing is written to GOPATH, but it is required to be set
+    environment => ['GOCACHE=/tmp/gocache', 'GOPATH=/root/go'],
+    path        => [
+      "${zulip::golang::dir}/bin",
+      '/usr/local/bin',
+      '/usr/bin',
+      '/bin',
+    ],
+    creates     => $bin,
+    require     => [
+      Zulip::External_Dep['golang'],
+      Zulip::External_Dep['postgres_exporter-src'],
+    ],
+    notify      => Exec['Cleanup postgres_exporter'],
+  }
+  # This resource is created by the 'compile postgres_exporter' step.
+  file { $bin:
+    ensure  => file,
+    require => Exec['compile postgres_exporter'],
+  }
+  exec { 'Cleanup postgres_exporter':
+    refreshonly => true,
+    provider    => shell,
+    onlyif      => "ls /usr/local/bin/postgres_exporter-* | grep -xv '${bin}'",
+    command     => "ls /usr/local/bin/postgres_exporter-* | grep -xv '${bin}' | xargs rm -r",
+    require     => [File[$bin], Service[supervisor]],
+  }
+
+  if false {
+    # This is left commented out, since it only makes sense to run
+    # against a server where the database exists and is writable --
+    # the former of which happens outside th scope of puppet right
+    # now, and the latter of which can only be determined after the
+    # database is in place.  Given that it has been run once, we do
+    # not expect to ever need it to run again; it is left here for
+    # completeness.
+    include zulip::postgresql_client
+    exec { 'create prometheus postgres user':
+      require => Class['zulip::postgresql_client'],
+      command => '/usr/bin/createuser -g pg_monitor prometheus',
+      unless  => 'test -f /usr/bin/psql && /usr/bin/psql -tAc "select usename from pg_user" | /bin/grep -xq prometheus)',
+      user    => 'postgres',
+      before  => File["${zulip::common::supervisor_conf_dir}/prometheus_postgres_exporter.conf"],
+    }
+  }
+
+  kandra::teleport::prometheus_app { 'postgres_exporter': port => '9187' }
+  file { "${zulip::common::supervisor_conf_dir}/prometheus_postgres_exporter.conf":
+    ensure  => file,
+    require => [
+      User[prometheus],
+      Package[supervisor],
+      File[$bin],
+    ],
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    content => template('kandra/supervisor/conf.d/prometheus_postgres_exporter.conf.template.erb'),
+    notify  => Service[supervisor],
+  }
+}

puppet/kandra/manifests/prometheus/process.pp

@@ -0,0 +1,43 @@
+# @summary Prometheus monitoring of Zulip server processes
+#
+class kandra::prometheus::process {
+  include kandra::prometheus::base
+  include zulip::supervisor
+
+  $version = $zulip::common::versions['process_exporter']['version']
+  $dir = "/srv/zulip-process_exporter-${version}"
+  $bin = "${dir}/process-exporter"
+  $conf = '/etc/zulip/process_exporter.yaml'
+
+  zulip::external_dep { 'process_exporter':
+    version        => $version,
+    url            => "https://github.com/ncabatoff/process-exporter/releases/download/v${version}/process-exporter-${version}.linux-${zulip::common::goarch}.tar.gz",
+    tarball_prefix => "process-exporter-${version}.linux-${zulip::common::goarch}",
+    bin            => [$bin],
+    cleanup_after  => [Service[supervisor]],
+  }
+
+  kandra::teleport::prometheus_app { 'process_exporter': port => '9256' }
+  file { $conf:
+    ensure  => file,
+    require => User[zulip],
+    owner   => 'zulip',
+    group   => 'zulip',
+    mode    => '0644',
+    source  => 'puppet:///modules/kandra/process_exporter.yaml',
+  }
+  file { "${zulip::common::supervisor_conf_dir}/prometheus_process_exporter.conf":
+    ensure  => file,
+    require => [
+      User[zulip],
+      Package[supervisor],
+      File[$bin],
+      File[$conf],
+    ],
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    content => template('kandra/supervisor/conf.d/prometheus_process_exporter.conf.template.erb'),
+    notify  => Service[supervisor],
+  }
+}

puppet/kandra/manifests/prometheus/pushgateway.pp

@@ -0,0 +1,32 @@
+# @summary Pushgateway for cron jobs
+#
+class kandra::prometheus::pushgateway {
+  include kandra::prometheus::base
+  include zulip::supervisor
+
+  $version = $zulip::common::versions['pushgateway']['version']
+  $dir = "/srv/zulip-pushgateway-${version}"
+  $bin = "${dir}/pushgateway"
+
+  zulip::external_dep { 'pushgateway':
+    version        => $version,
+    url            => "https://github.com/prometheus/pushgateway/releases/download/v${version}/pushgateway-${version}.linux-${zulip::common::goarch}.tar.gz",
+    tarball_prefix => "pushgateway-${version}.linux-${zulip::common::goarch}",
+    bin            => [$bin],
+    cleanup_after  => [Service[supervisor]],
+  }
+
+  file { "${zulip::common::supervisor_conf_dir}/prometheus_pushgateway.conf":
+    ensure  => file,
+    require => [
+      User[zulip],
+      Package[supervisor],
+      File[$bin],
+    ],
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    content => template('kandra/supervisor/conf.d/prometheus_pushgateway.conf.template.erb'),
+    notify  => Service[supervisor],
+  }
+}

puppet/kandra/manifests/prometheus/rabbitmq.pp

@@ -0,0 +1,14 @@
+# @summary Prometheus monitoring of rabbitmq server.  This is done via
+# the built-in prometheus plugin which serves on port 15692:
+# https://www.rabbitmq.com/prometheus.html
+#
+class kandra::prometheus::rabbitmq {
+  include kandra::prometheus::base
+
+  exec { 'enable rabbitmq-prometheus':
+    command => 'rabbitmq-plugins enable rabbitmq_prometheus',
+    unless  => 'grep -q rabbitmq_prometheus /etc/rabbitmq/enabled_plugins',
+    require => Service['rabbitmq-server'],
+  }
+  kandra::teleport::prometheus_app { 'rabbitmq': port => '15692' }
+}

puppet/kandra/manifests/prometheus/redis.pp

@@ -0,0 +1,33 @@
+# @summary Prometheus monitoring of redis servers
+#
+class kandra::prometheus::redis {
+  include kandra::prometheus::base
+  include zulip::supervisor
+
+  $version = $zulip::common::versions['redis_exporter']['version']
+  $dir = "/srv/zulip-redis_exporter-${version}"
+  $bin = "${dir}/redis_exporter"
+
+  zulip::external_dep { 'redis_exporter':
+    version        => $version,
+    url            => "https://github.com/oliver006/redis_exporter/releases/download/v${version}/redis_exporter-v${version}.linux-${zulip::common::goarch}.tar.gz",
+    tarball_prefix => "redis_exporter-v${version}.linux-${zulip::common::goarch}",
+    bin            => [$bin],
+    cleanup_after  => [Service[supervisor]],
+  }
+
+  kandra::teleport::prometheus_app { 'redis_exporter': port => '9121' }
+  file { "${zulip::common::supervisor_conf_dir}/prometheus_redis_exporter.conf":
+    ensure  => file,
+    require => [
+      User[zulip],
+      Package[supervisor],
+      File[$bin],
+    ],
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    content => template('kandra/supervisor/conf.d/prometheus_redis_exporter.conf.template.erb'),
+    notify  => Service[supervisor],
+  }
+}

puppet/kandra/manifests/prometheus/uwsgi.pp

@@ -0,0 +1,33 @@
+# @summary Prometheus monitoring of uwsgi servers
+#
+class kandra::prometheus::uwsgi {
+  include kandra::prometheus::base
+  include zulip::supervisor
+
+  $version = $zulip::common::versions['uwsgi_exporter']['version']
+  $dir = "/srv/zulip-uwsgi_exporter-${version}"
+  $bin = "${dir}/uwsgi_exporter"
+
+  zulip::external_dep { 'uwsgi_exporter':
+    version        => $version,
+    url            => "https://github.com/timonwong/uwsgi_exporter/releases/download/v${version}/uwsgi_exporter-${version}.linux-${zulip::common::goarch}.tar.gz",
+    tarball_prefix => "uwsgi_exporter-${version}.linux-${zulip::common::goarch}",
+    bin            => [$bin],
+    cleanup_after  => [Service[supervisor]],
+  }
+
+  kandra::teleport::prometheus_app { 'uwsgi_exporter': port => '9238' }
+  file { "${zulip::common::supervisor_conf_dir}/prometheus_uwsgi_exporter.conf":
+    ensure  => file,
+    require => [
+      User[zulip],
+      Package[supervisor],
+      File[$bin],
+    ],
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    content => template('kandra/supervisor/conf.d/prometheus_uwsgi_exporter.conf.template.erb'),
+    notify  => Service[supervisor],
+  }
+}

puppet/kandra/manifests/prometheus/wal_g.pp

@@ -0,0 +1,34 @@
+# @summary Prometheus monitoring of wal-g backups
+#
+class kandra::prometheus::wal_g {
+  include kandra::prometheus::base
+  include zulip::supervisor
+  include zulip::wal_g
+
+  file { '/usr/local/bin/wal-g-exporter':
+    ensure  => file,
+    require => User[zulip],
+    owner   => 'zulip',
+    group   => 'zulip',
+    mode    => '0755',
+    source  => 'puppet:///modules/zulip/postgresql/wal-g-exporter',
+  }
+
+  # We embed the hash of the contents into the name of the process, so
+  # that `supervisorctl reread` knows that it has updated.
+  $full_exporter_hash = sha256(file('zulip/postgresql/wal-g-exporter'))
+  $exporter_hash = $full_exporter_hash[0,8]
+  file { "${zulip::common::supervisor_conf_dir}/prometheus_wal_g_exporter.conf":
+    ensure  => file,
+    require => [
+      User[zulip],
+      Package[supervisor],
+      File['/usr/local/bin/wal-g-exporter'],
+    ],
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    content => template('kandra/supervisor/conf.d/prometheus_wal_g_exporter.conf.template.erb'),
+    notify  => Service[supervisor],
+  }
+}

puppet/kandra/manifests/prometheus/weblate.pp

@@ -0,0 +1,40 @@
+# @summary Export Weblate translation stats
+class kandra::prometheus::weblate {
+  include kandra::prometheus::base
+  include zulip::supervisor
+
+  # We embed the hash of the contents into the name of the process, so
+  # that `supervisorctl reread` knows that it has updated.
+  $full_exporter_hash = sha256(file('kandra/weblate_exporter'))
+  $exporter_hash = $full_exporter_hash[0,8]
+
+  $bin = '/usr/local/bin/weblate_exporter'
+  file { $bin:
+    ensure => file,
+    owner  => 'root',
+    group  => 'root',
+    mode   => '0755',
+    source => 'puppet:///modules/kandra/weblate_exporter',
+  }
+
+  file { "${zulip::common::supervisor_conf_dir}/weblate_exporter.conf":
+    ensure  => file,
+    require => [
+      User[zulip],
+      Package[supervisor],
+      File[$bin],
+    ],
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    content => template('kandra/supervisor/conf.d/weblate_exporter.conf.template.erb'),
+    notify  => Service[supervisor],
+  }
+
+  include kandra::prometheus::pushgateway
+  zulip::cron { 'weblate-to-pushgateway':
+    minute    => '*/15',
+    command   => 'curl http://localhost:9189/metrics | curl --data-binary @- http://localhost:9091/metrics/job/weblate',
+    use_proxy => false,
+  }
+}

puppet/kandra/manifests/ses_logs.pp

@@ -0,0 +1,12 @@
+class kandra::ses_logs {
+  include kandra::vector
+
+  $ses_logs_sqs_url = zulipsecret('secrets', 'ses_logs_sqs_url', '')
+  $ses_logs_s3_bucket = zulipsecret('secrets', 'ses_logs_s3_bucket', '')
+
+  concat::fragment { 'vector_ses_logs':
+    target  => $kandra::vector::conf,
+    order   => '50',
+    content => template('kandra/vector_ses.toml.template.erb'),
+  }
+}

puppet/kandra/manifests/ssh_authorized_keys.pp

@@ -0,0 +1,17 @@
+define kandra::ssh_authorized_keys(
+  $keys = true,
+) {
+  $user = $name
+  if $keys == true {
+    $keypath = "prod/ssh/authorized_keys/${user}"
+  } elsif $keys.is_a(Array) {
+    $keypath = join($keys.map |$k| {"prod/ssh/authorized_keys/${k}"}, ' ')
+  } else {
+    $keypath = "prod/ssh/authorized_keys/${keys}"
+  }
+  exec { "ssh_authorized_keys ${user}":
+    require => File['/usr/local/bin/install-ssh-authorized-keys'],
+    command => "/usr/local/bin/install-ssh-authorized-keys ${user} ${keypath}",
+    unless  => "[ -f /usr/local/bin/install-ssh-authorized-keys ] && /usr/local/bin/install-ssh-authorized-keys --check ${user} ${keypath}",
+  }
+}

puppet/kandra/manifests/ssh_keys.pp

@@ -0,0 +1,17 @@
+define kandra::ssh_keys(
+  $keys = true,
+) {
+  $user = $name
+  if $keys == true {
+    $keypath = "prod/ssh/keys/${user}"
+  } elsif $keys.is_a(Array) {
+    $keypath = join($keys.map |$k| {"prod/ssh/keys/${k}"}, ' ')
+  } else {
+    $keypath = "prod/ssh/keys/${keys}"
+  }
+  exec { "ssh_keys ${user}":
+    require => File['/usr/local/bin/install-ssh-keys'],
+    command => "/usr/local/bin/install-ssh-keys ${user} ${keypath}",
+    unless  => "[ -f /usr/local/bin/install-ssh-keys ] && /usr/local/bin/install-ssh-keys --check ${user} ${keypath}",
+  }
+}

puppet/kandra/manifests/statuspage.pp

@@ -0,0 +1,28 @@
+# @summary Plumb Prometheus stats into status.zulip.com
+#
+# Requires a /etc/zulip/statuspage.conf which maps statuspage.io
+# metric_ids to Prometheus queries.
+class kandra::statuspage {
+  $bin = '/usr/local/bin/statuspage-pusher'
+
+  file { $bin:
+    ensure => file,
+    owner  => 'root',
+    group  => 'root',
+    mode   => '0755',
+    source => 'puppet:///modules/kandra/statuspage-pusher',
+  }
+
+  file { "${zulip::common::supervisor_conf_dir}/statuspage-pusher.conf":
+    ensure  => file,
+    require => [
+      Package[supervisor],
+      File[$bin],
+    ],
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    content => template('kandra/supervisor/conf.d/statuspage-pusher.conf.template.erb'),
+    notify  => Service[supervisor],
+  }
+}

puppet/kandra/manifests/teleport/application.pp

@@ -0,0 +1,30 @@
+# @summary Adds an http "application" to the Teleport configuration for the host.
+#
+# See https://goteleport.com/docs/application-access/
+define kandra::teleport::application (
+  $port,
+  $description = '',
+  $order = '50',
+  $labels = {},
+) {
+  include kandra::teleport::application_top
+  $app_data = [
+    {
+      name        => $name,
+      description => $description,
+      uri         => "http://localhost:${port}",
+      labels      => {
+        name => $name,
+      } + $labels,
+    },
+  ]
+
+  # This is appended to puppet/kandra/files/teleport_node.yaml, so
+  # must be indented, and the leading document dashes stripped.
+  $app_yaml = to_yaml($app_data).regsubst(/\A---\s*\n/, '').regsubst(/^/, '    ', 'G')
+  concat::fragment { "teleport_app_${name}":
+    target  => '/etc/teleport_node.yaml',
+    order   => $order,
+    content => $app_yaml,
+  }
+}

puppet/kandra/manifests/teleport/application_top.pp

@@ -0,0 +1,10 @@
+# @summary Enables application support on the node; include once.
+#
+# See https://goteleport.com/docs/application-access/
+class kandra::teleport::application_top {
+  concat::fragment { 'teleport_app':
+    target => '/etc/teleport_node.yaml',
+    order  => '10',
+    source => 'puppet:///modules/kandra/teleport_app.yaml',
+  }
+}

puppet/kandra/manifests/teleport/base.pp

@@ -0,0 +1,18 @@
+class kandra::teleport::base {
+  include zulip::supervisor
+
+  $setup_apt_repo_file = "${facts['zulip_scripts_path']}/lib/setup-apt-repo"
+  exec{ 'setup-apt-repo-teleport':
+    command => "${setup_apt_repo_file} --list teleport",
+    unless  => "${setup_apt_repo_file} --list teleport --verify",
+  }
+  package { 'teleport':
+    ensure  => installed,
+    require => Exec['setup-apt-repo-teleport'],
+  }
+  service { 'teleport':
+    ensure  => stopped,
+    enable  => mask,
+    require => Package['teleport'],
+  }
+}

puppet/kandra/manifests/teleport/db.pp

@@ -0,0 +1,23 @@
+# @summary Provide Teleport SSH access to a node.
+#
+# https://goteleport.com/docs/admin-guide/#adding-nodes-to-the-cluster
+# details additional manual steps to allow a node to join the cluster.
+class kandra::teleport::db {
+  include kandra::teleport::base
+
+  $fqdn = $facts['networking']['fqdn']
+  $hostname = $facts['networking']['hostname']
+  $is_ec2 = zulipconf('machine', 'hosting_provider', 'ec2') == 'ec2'
+  $join_token = zulipsecret('secrets', 'teleport_join_token', '')
+  file { '/etc/teleport_db.yaml':
+    ensure  => file,
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    content => template('kandra/teleport_db.yaml.template.erb'),
+    before  => Kandra::Teleport::Part['db'],
+    notify  => Exec['reload teleport_db'],
+  }
+
+  kandra::teleport::part { 'db': }
+}

puppet/kandra/manifests/teleport/node.pp

@@ -0,0 +1,39 @@
+# @summary Provide Teleport SSH access to a node.
+#
+# EC2 nodes will automatically join the cluster; non-EC2 hosts will
+# need to set a teleport_join_token secret.  See
+# https://goteleport.com/docs/agents/join-services-to-your-cluster/join-token/#generate-a-token
+class kandra::teleport::node {
+  include kandra::teleport::base
+
+  $is_ec2 = zulipconf('machine', 'hosting_provider', 'ec2') == 'ec2'
+  $join_token = zulipsecret('secrets', 'teleport_join_token', '')
+  concat { '/etc/teleport_node.yaml':
+    ensure => present,
+    owner  => 'root',
+    group  => 'root',
+    mode   => '0644',
+    before => Kandra::Teleport::Part['node'],
+    notify => Exec['reload teleport_node'],
+  }
+  concat::fragment { 'teleport_node_base':
+    target  => '/etc/teleport_node.yaml',
+    content => template('kandra/teleport_node.yaml.template.erb'),
+    order   => '01',
+  }
+
+  kandra::teleport::part { 'node': }
+
+  $host_ca_path = '/etc/ssl/certs/teleport-ca-host.cert'
+  $host_ca_extract = @("EOT")
+    sqlite3 /var/lib/teleport/proc/sqlite.db "select value from kv where key = '/ids/node/current'" \
+      | jq -r .spec.tls_ca_certs[] \
+      | base64 -d \
+      > ${host_ca_path}
+    | EOT
+  exec { 'teleport host CA':
+    require => [Service['teleport_node'], Package['sqlite3']],
+    creates => $host_ca_path,
+    command => $host_ca_extract,
+  }
+}

puppet/kandra/manifests/teleport/part.pp

@@ -0,0 +1,32 @@
+# @summary Adds a systemd service named teleport_$name
+#
+define kandra::teleport::part() {
+  $part = $name
+
+  include zulip::systemd_daemon_reload
+  file { "/etc/systemd/system/teleport_${part}.service":
+    require => [
+      Package[teleport],
+    ],
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    content => template('kandra/teleport.service.template.erb'),
+    notify  => [Exec['reload systemd'], Service["teleport_${part}"]],
+  }
+
+  service {"teleport_${part}":
+    ensure  => running,
+    enable  => true,
+    require => [Service['supervisor'], Service['teleport'], Exec['reload systemd']],
+  }
+
+  # Notify this from YAML config changes (rather than the service)
+  # to get a SIGHUP-based graceful restart -- Teleport forks a new
+  # daemon and drains the old -- instead of a hard stop+start.
+  exec { "reload teleport_${part}":
+    command     => "/bin/systemctl reload teleport_${part}",
+    refreshonly => true,
+    require     => Service["teleport_${part}"],
+  }
+}

puppet/kandra/manifests/teleport/prometheus_app.pp

@@ -0,0 +1,24 @@
+define kandra::teleport::prometheus_app (
+  $port,
+  $description = '',
+  $order = '50',
+) {
+  $app_name = regsubst("${facts['networking']['hostname']}-${name}", '_', '-', 'G')
+  # Handle non-AWS hosts which don't have ec2 metadata
+  $fallback_role = $facts['networking']['hostname'] ? {
+    /^([^.-]+)/ => $1,
+    default     => '',
+  }
+  $role = pick_default($facts.dig('ec2_metadata', 'tags', 'instance', 'role'), $fallback_role)
+  kandra::teleport::application { $app_name:
+    port        => $port,
+    order       => $order,
+    description => $description,
+    labels      => {
+      type     => 'prometheus',
+      instance => $facts['networking']['hostname'],
+      exporter => $name,
+      role     => $role,
+    }
+  }
+}

puppet/kandra/manifests/teleport/sd.pp

@@ -0,0 +1,31 @@
+# @summary Returns Prometheus scrape targets based on Teleport apps
+#
+# Only one instance is necessary.
+#
+class kandra::teleport::sd {
+  include zulip::supervisor
+  include kandra::teleport::tbot
+
+  $version = $zulip::common::versions['teleport-sd']['version']
+  $bin = "/srv/zulip-teleport-sd-${version}"
+
+  zulip::external_dep { 'teleport-sd':
+    version       => $version,
+    url           => "https://github.com/alexmv/teleport-sd/releases/download/v${version}/teleport-sd-linux-${zulip::common::goarch}",
+    cleanup_after => [Service[supervisor]],
+  }
+
+  file { "${zulip::common::supervisor_conf_dir}/teleport-sd.conf":
+    ensure  => file,
+    require => [
+      Package[teleport],
+      Package[supervisor],
+      File[$bin],
+    ],
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    content => template('kandra/supervisor/conf.d/teleport-sd.conf.erb'),
+    notify  => Service[supervisor],
+  }
+}

puppet/kandra/manifests/teleport/tbot.pp

@@ -0,0 +1,48 @@
+# @summary Provide a local application proxy
+# https://goteleport.com/docs/reference/machine-workload-identity/configuration/#application-proxy
+class kandra::teleport::tbot {
+  include kandra::teleport::base
+
+  file { '/etc/tbot.yaml':
+    ensure => file,
+    owner  => 'root',
+    group  => 'root',
+    mode   => '0644',
+    source => 'puppet:///modules/kandra/tbot.yaml',
+    before => Service['tbot'],
+    notify => Exec['reload tbot'],
+  }
+
+  file { '/etc/systemd/system/tbot.service':
+    require => [
+      Package[teleport],
+    ],
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    source  => 'puppet:///modules/kandra/tbot.service',
+    notify  => [Exec['reload systemd'], Service['tbot']],
+  }
+
+  file { ['/var/lib/teleport/bot', '/var/lib/teleport/bot-identity']:
+    ensure  => directory,
+    owner   => 'teleport',
+    group   => 'teleport',
+    mode    => '0644',
+    require => Service['teleport'],
+    before  => Service['tbot'],
+  }
+
+  service {'tbot':
+    ensure  => running,
+    enable  => true,
+    require => [Service['teleport'], Exec['reload systemd']],
+  }
+
+  # See the equivalent in kandra::teleport::part.
+  exec { 'reload tbot':
+    command     => '/bin/systemctl reload tbot',
+    refreshonly => true,
+    require     => Service['tbot'],
+  }
+}

puppet/kandra/manifests/user_dotfiles.pp

@@ -0,0 +1,85 @@
+define kandra::user_dotfiles (
+  $home = '',
+  $keys = false,
+  $authorized_keys = false,
+  $known_hosts = false,
+) {
+  $user = $name
+
+  if $home == '' {
+    $homedir = "/home/${user}"
+  } else {
+    $homedir = $home
+  }
+
+  file { "${homedir}/.ssh":
+    ensure  => directory,
+    require => User[$user],
+    owner   => $user,
+    group   => $user,
+    mode    => '0700',
+  }
+
+  file { "${homedir}/.emacs":
+    ensure  => file,
+    require => User[$user],
+    owner   => $user,
+    group   => $user,
+    mode    => '0644',
+    source  => 'puppet:///modules/kandra/dotfiles/emacs.el',
+  }
+
+  # Suppress MOTD printing, to fix load problems with Nagios caused by
+  # Ubuntu's default MOTD tools for things like "checking for the next
+  # release" being super slow.
+  file { "${homedir}/.hushlogin":
+    ensure  => file,
+    require => User[$user],
+    owner   => $user,
+    group   => $user,
+    mode    => '0644',
+    content => '',
+  }
+
+  if $keys != false {
+    kandra::ssh_keys{ $user:
+      keys    => $keys,
+      require => File["${homedir}/.ssh"],
+    }
+  }
+  if $authorized_keys != false {
+    kandra::ssh_authorized_keys{ $user:
+      keys    => $authorized_keys,
+      require => File["${homedir}/.ssh"],
+    }
+  }
+  if $known_hosts != false {
+    file { "${homedir}/.ssh/known_hosts":
+      # We mark this as "present" to ensure that it exists, but not to
+      # directly control its contents.
+      ensure  => present,
+      owner   => $user,
+      group   => $user,
+      mode    => '0644',
+      require => File["${homedir}/.ssh"],
+    }
+    $known_hosts.each |Optional[String] $hostname| {
+      if $hostname == undef {
+        # pass
+      } elsif $hostname == 'github.com' {
+        $github_keys = file('kandra/github.keys')
+        exec { "${user} ssh known_hosts ${hostname}":
+          command => "echo '${github_keys}' >> ${homedir}/.ssh/known_hosts",
+          unless  => "grep ${hostname} ${homedir}/.ssh/known_hosts",
+          require => File["${homedir}/.ssh/known_hosts"],
+        }
+      } else {
+        exec { "${user} ssh known_hosts ${hostname}":
+          command => "ssh-keyscan ${hostname} >> ${homedir}/.ssh/known_hosts",
+          unless  => "grep ${hostname} ${homedir}/.ssh/known_hosts",
+          require => File["${homedir}/.ssh/known_hosts"],
+        }
+      }
+    }
+  }
+}

puppet/kandra/manifests/vector.pp

@@ -0,0 +1,73 @@
+# @summary Installs Vector to transform Prometheus data
+#
+class kandra::vector {
+  $version = $zulip::common::versions['vector']['version']
+  $dir = "/srv/zulip-vector-${version}"
+  $bin = "${dir}/bin/vector"
+  $conf = '/etc/vector.toml'
+
+  $arch = $facts['os']['architecture'] ? {
+    'amd64'   => 'x86_64',
+    'aarch64' => 'aarch64',
+  }
+
+  zulip::external_dep { 'vector':
+    version        => $version,
+    url            => "https://packages.timber.io/vector/${version}/vector-${version}-${arch}-unknown-linux-gnu.tar.gz",
+    tarball_prefix => "vector-${arch}-unknown-linux-gnu",
+    bin            => [$bin],
+    cleanup_after  => [Service[supervisor]],
+  }
+  file { "${zulip::common::supervisor_conf_dir}/vector.conf":
+    ensure  => file,
+    require => [
+      User[zulip],
+      Package[supervisor],
+      Concat[$conf],
+      File[$bin],
+    ],
+    before  => Exec['Cleanup vector'],
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    content => template('kandra/supervisor/conf.d/vector.conf.template.erb'),
+    notify  => Service[supervisor],
+  }
+
+  exec { 'reload vector':
+    command     => 'supervisorctl signal HUP vector',
+    require     => Service[supervisor],
+    refreshonly => true,
+  }
+  concat { $conf:
+    ensure => present,
+    owner  => 'root',
+    group  => 'root',
+    mode   => '0644',
+    notify => Exec['reload vector'],
+  }
+  # All of the pipelines need to be included in the Prometheus
+  # exporter; they insert their strings at order 90, with a leading
+  # comma, in the middle of the "inputs" block
+  $vector_export = @(VECTOR)
+    [sources.vector_metrics]
+      type = "internal_metrics"
+    [sinks.prometheus_exporter]
+      type = "prometheus_exporter"
+      address = "127.0.0.1:9081"
+      flush_period_secs = 120
+      suppress_timestamp = true
+      buckets = [0.001, 0.002, 0.005, 0.01, 0.02, 0.05, 0.1, 0.2, 0.5, 1, 2, 5]
+      inputs = ["vector_metrics"
+    |-VECTOR
+  concat::fragment { 'vector_export_prefix':
+    target  => $conf,
+    content => $vector_export,
+    order   => '89',
+  }
+  concat::fragment { 'vector_export_suffix':
+    target  => $conf,
+    content => "]\n",
+    order   => '99',
+  }
+}