Linux sysadmin

# always edit via visudo: it syntax-checks before saving
sudo visudo -f /etc/sudoers.d/deploy
sudo -l              # what may I run?
sudo -k              # drop cached credentials
sudoedit /etc/hosts  # edit as root with your own $EDITOR

# who    where=(as whom)   [tags:] commands
deploy   ALL=(root)        NOPASSWD: /usr/bin/systemctl restart app
%ops     ALL=(ALL:ALL)     ALL

chmod 640 app.env
chmod u+x,go-w deploy.sh
chmod -R u=rwX,go=rX site/   # X: exec only on dirs
chown -R app:app /srv/app
chown :www-data upload/      # group only
stat -c '%a %U:%G %n' file   # 640 app:app file
# audit setuid binaries
sudo find / -xdev -perm -4000 -type f

setfacl -m u:alice:rwX /srv/shared     # grant a user
setfacl -R -m g:web:rX /srv/site       # recursive, group
setfacl -d -m g:web:rwX /srv/uploads   # default ACL
getfacl /srv/shared
setfacl -x u:alice /srv/shared         # remove one entry
setfacl -b /srv/shared                 # remove all ACLs

kill 1234              # SIGTERM
kill -HUP 1234         # reload
kill -9 1234           # only after TERM failed
pkill -f 'bun run'     # match full command line
killall nginx          # by name
kill -l                # list signals

nice -n 10 ./reindex.sh     # -20 (greedy) .. 19 (polite)
renice -n 5 -p 1234
ionice -c3 tar czf b.tgz d  # idle I/O class
long-task &                 # background; Ctrl-Z then bg
jobs -l; fg %1; disown %1
nohup ./task.sh > task.log 2>&1 &
# better: a transient unit with logs + limits
sudo systemd-run --unit=reindex -p MemoryMax=2G \
  /srv/app/reindex.sh
journalctl -u reindex -f

[Unit]
Description=Queue worker
After=network-online.target postgresql.service
Wants=network-online.target
 
[Service]
Type=exec
User=worker
WorkingDirectory=/srv/worker
EnvironmentFile=/etc/worker/env
ExecStart=/usr/local/bin/bun run worker.ts
Restart=always
RestartSec=5
 
[Install]
WantedBy=multi-user.target

[Service]
ExecStart=
ExecStart=/usr/local/bin/bun run --smol worker.ts
Environment=LOG_LEVEL=debug
MemoryMax=512M

const server = Bun.serve({
  port: Number(process.env.PORT ?? 3000),
  fetch: () => new Response("ok"),
});
 
process.on("SIGTERM", async () => {
  await server.stop(); // let in-flight requests finish
  process.exit(0);
});

┌──────── minute        0-59
│ ┌────── hour          0-23
│ │ ┌──── day of month  1-31
│ │ │ ┌── month         1-12 or jan-dec
│ │ │ │ ┌ day of week   0-7 or sun-sat (0 and 7 = Sunday)
│ │ │ │ │
* * * * *  command

Types: deb
URIs: https://pkg.example.com/apt
Suites: stable
Components: main
Signed-By: /etc/apt/keyrings/example.gpg

curl -fsSL https://pkg.example.com/key.asc \
  | sudo gpg --dearmor -o /etc/apt/keyrings/example.gpg
sudo apt modernize-sources   # Debian 13: old .list → deb822
# Fedora (dnf5)
sudo dnf config-manager addrepo \
  --from-repofile=https://pkg.example.com/x.repo
# RHEL 10 (dnf4)
sudo dnf config-manager --add-repo \
  https://pkg.example.com/x.repo

# what                                      where  type  options                  dump pass
UUID=3f1c2a9e-0d4b-4c1e-9b7a-2e5f6d8c9a10  /data  ext4  defaults,noatime,nofail  0    2
/swapfile                                   none   swap  sw                       0    0

network:
  version: 2
  ethernets:
    eth0:
      dhcp4: false
      addresses: [10.0.0.10/24]
      routes:
        - to: default
          via: 10.0.0.1
      nameservers:
        addresses: [1.1.1.1, 9.9.9.9]

nmcli device status
nmcli con show
nmcli con mod eth0 ipv4.method manual \
  ipv4.addresses 10.0.0.10/24 ipv4.gateway 10.0.0.1 \
  ipv4.dns "1.1.1.1 9.9.9.9"
nmcli con up eth0

sudo ufw default deny incoming
sudo ufw default allow outgoing
sudo ufw allow OpenSSH           # or: ufw limit 22/tcp
sudo ufw allow 80,443/tcp
sudo ufw allow from 10.0.0.0/8 to any port 5432 proto tcp
sudo ufw enable
sudo ufw status numbered
sudo ufw delete 4

sudo firewall-cmd --get-active-zones
sudo firewall-cmd --list-all
sudo firewall-cmd --permanent --add-service=https
sudo firewall-cmd --permanent --add-port=3000/tcp
sudo firewall-cmd --permanent --zone=trusted \
  --add-source=10.0.0.0/8
sudo firewall-cmd --reload   # --permanent needs a reload

#!/usr/sbin/nft -f
flush ruleset
 
table inet filter {
  chain input {
    type filter hook input priority 0; policy drop;
    ct state established,related accept
    iif lo accept
    meta l4proto { icmp, ipv6-icmp } accept
    tcp dport { 22, 80, 443 } accept
  }
  chain forward {
    type filter hook forward priority 0; policy drop;
  }
}

[Journal]
SystemMaxUse=1G
MaxRetentionSec=1month

# top 10 client IPs in an access log
awk '{print $1}' access.log | sort | uniq -c \
  | sort -rn | head
# count HTTP status codes
awk '{print $9}' access.log | sort | uniq -c
# replace across files safely (NUL-separated)
rg -l -0 'old\.host' \
  | xargs -0 sed -i 's/old\.host/new.host/g'
# JSON logs: errors from the last hour
journalctl -u app --since -1h -o cat \
  | jq -c 'select(.level == "error")'

# who got OOM-killed?
journalctl -k -g 'out of memory|oom-kill' -b
# memory per unit (cgroups)
systemd-cgtop -m
# biggest RSS processes
ps -eo pid,rss,cmd --sort=-rss | head

APT::Periodic::Update-Package-Lists "1";
APT::Periodic::Unattended-Upgrade "1";

[DEFAULT]
bantime  = 1h
findtime = 10m
maxretry = 5
backend  = systemd
 
[sshd]
enabled = true

[Unit]
Description=My Bun app
After=network-online.target
Wants=network-online.target
 
[Service]
Type=exec
User=app
WorkingDirectory=/srv/app
EnvironmentFile=/etc/app/env
Environment=NODE_ENV=production PORT=3000
ExecStart=/usr/local/bin/bun run src/index.ts
Restart=on-failure
RestartSec=2
TimeoutStopSec=20
NoNewPrivileges=yes
ProtectSystem=strict
ProtectHome=yes
PrivateTmp=yes
StateDirectory=app
 
[Install]
WantedBy=multi-user.target

# /etc/systemd/system/backup.service
[Unit]
Description=Nightly backup
 
[Service]
Type=oneshot
User=backup
ExecStart=/usr/local/bin/backup.sh
 
# /etc/systemd/system/backup.timer
[Unit]
Description=Run backup nightly
 
[Timer]
OnCalendar=*-*-* 03:00:00
RandomizedDelaySec=15m
Persistent=true
 
[Install]
WantedBy=timers.target

sudo ss -ltnp 'sport = :3000'
sudo lsof -nP -iTCP:3000 -sTCP:LISTEN
systemctl status 12345        # which unit owns the PID
ps -o pid,user,etime,cmd -p 12345
sudo fuser -k -TERM 3000/tcp  # stop whatever holds it

df -hT -x tmpfs -x devtmpfs
sudo du -xh -d1 / 2>/dev/null | sort -h | tail -15
sudo ncdu -x /
sudo find / -xdev -type f -size +500M \
  -exec ls -lh {} + 2>/dev/null | sort -k5 -h
sudo journalctl --vacuum-size=500M
sudo apt clean              # or: sudo dnf clean all
docker system df            # then: docker system prune
# space not freed after rm? a process still holds it
sudo lsof -nP +L1

/var/log/app/*.log {
    daily
    rotate 14
    maxsize 100M
    compress
    delaycompress
    missingok
    notifempty
    create 0640 app app
    sharedscripts
    postrotate
        systemctl kill -s HUP app.service
    endscript
}

sudo fallocate -l 2G /swapfile
sudo chmod 600 /swapfile
sudo mkswap /swapfile
sudo swapon /swapfile
echo '/swapfile none swap sw 0 0' | sudo tee -a /etc/fstab
swapon --show && free -h
# prefer RAM; swap only under pressure
echo 'vm.swappiness=10' \
  | sudo tee /etc/sysctl.d/99-swappiness.conf
sudo sysctl --system

#!/usr/bin/env bash
# usage (as root): ./bootstrap.sh zach ~/zach.pub
set -euo pipefail
U=${1:?user}; KEY=$(cat "${2:?pubkey file}")
 
apt-get update && apt-get -y full-upgrade
apt-get -y install ufw fail2ban unattended-upgrades \
  curl git htop ncdu jq sysstat
 
id "$U" &>/dev/null || useradd -m -s /bin/bash -G sudo "$U"
passwd "$U"   # sudo will ask for it
install -d -m 700 -o "$U" -g "$U" "/home/$U/.ssh"
install -m 600 -o "$U" -g "$U" /dev/null \
  "/home/$U/.ssh/authorized_keys"
echo "$KEY" >> "/home/$U/.ssh/authorized_keys"
 
cat > /etc/ssh/sshd_config.d/00-hardening.conf <<'EOF'
PermitRootLogin no
PasswordAuthentication no
KbdInteractiveAuthentication no
EOF
sshd -t && systemctl try-reload-or-restart ssh
ufw allow OpenSSH && ufw --force enable
dpkg-reconfigure -f noninteractive unattended-upgrades