urocibg / monitoring.sh

#!/bin/bash # Цветове за по-добър изглед RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[1;33m' BLUE='\033[0;34m' NC='\033[0m' # No Color echo -e "${GREEN}================================${NC}" echo -e "${GREEN}Proxmox LXC Monitoring Stack${NC}" echo -e "${GREEN}================================${NC}\n" # Проверка за root права if [[ $EUID -ne 0 ]]; then echo -e "${RED}Този скрипт трябва да се изпълни като root!${NC}" exit 1 fi # Въвеждане на Telegram данни echo -e "${YELLOW}Моля въведете Telegram Bot Token:${NC}" read -r TELEGRAM_TOKEN echo -e "${YELLOW}Моля въведете Telegram Chat ID (БЕЗ кавички):${NC}" read -r TELEGRAM_CHAT_ID # Проверка дали са въведени if [ -z "$TELEGRAM_TOKEN" ] || [ -z "$TELEGRAM_CHAT_ID" ]; then echo -e "${RED}Telegram данните са задължителни!${NC}" exit 1 fi echo -e "\n${GREEN}Започва инсталацията в LXC контейнера...${NC}\n" # Актуализация на системата echo -e "${YELLOW}[1/9] Актуализация на системата...${NC}" apt-get update -qq apt-get install -y wget curl tar net-tools > /dev/null 2>&1 # Създаване на потребители echo -e "${YELLOW}[2/9] Създаване на системни потребители...${NC}" useradd --no-create-home --shell /bin/false prometheus 2>/dev/null || true useradd --no-create-home --shell /bin/false node_exporter 2>/dev/null || true useradd --no-create-home --shell /bin/false alertmanager 2>/dev/null || true # Инсталация на Prometheus echo -e "${YELLOW}[3/9] Инсталация на Prometheus...${NC}" PROM_VERSION="2.47.0" wget -q https://github.com/prometheus/prometheus/releases/download/v${PROM_VERSION}/prometheus-${PROM_VERSION}.linux-amd64.tar.gz tar xzf prometheus-${PROM_VERSION}.linux-amd64.tar.gz cp prometheus-${PROM_VERSION}.linux-amd64/prometheus /usr/local/bin/ cp prometheus-${PROM_VERSION}.linux-amd64/promtool /usr/local/bin/ chown prometheus:prometheus /usr/local/bin/prometheus /usr/local/bin/promtool rm -rf prometheus-${PROM_VERSION}.linux-amd64* mkdir -p /etc/prometheus /var/lib/prometheus chown prometheus:prometheus /etc/prometheus /var/lib/prometheus # Конфигурация на Prometheus за LXC контейнери cat > /etc/prometheus/prometheus.yml <<EOF global: scrape_interval: 15s evaluation_interval: 15s external_labels: monitor: 'proxmox-lxc-monitor' alerting: alertmanagers: - static_configs: - targets: - localhost:9093 rule_files: - "/etc/prometheus/alerts.yml" scrape_configs: - job_name: 'prometheus' static_configs: - targets: ['localhost:9090'] labels: alias: 'Prometheus Server' type: 'monitoring' - job_name: 'monitoring-container' static_configs: - targets: ['localhost:9100'] labels: alias: 'Monitoring LXC' type: 'lxc-container' - job_name: 'lxc-containers' static_configs: - targets: [] # Добави тук IP адресите на другите контейнери: # - targets: ['10.0.0.101:9100'] # labels: # alias: 'Web Server' # type: 'lxc-container' # - targets: ['10.0.0.102:9100'] # labels: # alias: 'Database Server' # type: 'lxc-container' - job_name: 'blackbox' metrics_path: /probe params: module: [http_2xx] static_configs: - targets: - http://localhost:9090 - https://www.google.com relabel_configs: - source_labels: [__address__] target_label: __param_target - source_labels: [__param_target] target_label: instance - target_label: __address__ replacement: localhost:9115 EOF # Създаване на alert правила cat > /etc/prometheus/alerts.yml <<EOF groups: - name: test_alerts interval: 10s rules: - alert: TestAlert expr: up{job="prometheus"} == 1 for: 5s labels: severity: info annotations: summary: "🎉 Proxmox мониторингът е активен!" description: "Prometheus е конфигуриран успешно и следи LXC контейнерите." - name: lxc_container_alerts interval: 30s rules: - alert: LXCContainerDown expr: up{type="lxc-container"} == 0 for: 1m labels: severity: critical annotations: summary: "⚠️ LXC контейнер {{ \$labels.alias }} е DOWN" description: "Контейнер {{ \$labels.instance }} не отговаря повече от 1 минута." - alert: HighCPUUsage expr: 100 - (avg by(instance, alias) (irate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 80 for: 2m labels: severity: warning annotations: summary: "🔥 Високо CPU натоварване" description: "{{ \$labels.alias }} ({{ \$labels.instance }}) използва {{ \$value | humanize }}% CPU" - alert: HighMemoryUsage expr: (node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) / node_memory_MemTotal_bytes * 100 > 80 for: 2m labels: severity: warning annotations: summary: "💾 Високо RAM използване" description: "{{ \$labels.alias }} ({{ \$labels.instance }}) използва {{ \$value | humanize }}% RAM" - alert: DiskSpaceLow expr: (node_filesystem_avail_bytes{fstype!="tmpfs",mountpoint="/"} / node_filesystem_size_bytes{fstype!="tmpfs",mountpoint="/"}) * 100 < 20 for: 2m labels: severity: warning annotations: summary: "💿 Малко дисково пространство" description: "{{ \$labels.alias }} ({{ \$labels.instance }}) има само {{ \$value | humanize }}% свободно място" - alert: HighLoadAverage expr: node_load5 / count(node_cpu_seconds_total{mode="idle"}) without (cpu, mode) > 2 for: 5m labels: severity: warning annotations: summary: "📊 Високо системно натоварване" description: "{{ \$labels.alias }} има Load Average: {{ \$value | humanize }}" - alert: NetworkErrors expr: rate(node_network_receive_errs_total[5m]) > 10 or rate(node_network_transmit_errs_total[5m]) > 10 for: 2m labels: severity: warning annotations: summary: "🌐 Мрежови грешки" description: "{{ \$labels.alias }} има {{ \$value | humanize }} грешки/сек на интерфейс {{ \$labels.device }}" EOF chown -R prometheus:prometheus /etc/prometheus # Systemd service за Prometheus cat > /etc/systemd/system/prometheus.service <<EOF [Unit] Description=Prometheus Monitoring for Proxmox LXC Wants=network-online.target After=network-online.target [Service] User=prometheus Group=prometheus Type=simple ExecStart=/usr/local/bin/prometheus \\ --config.file=/etc/prometheus/prometheus.yml \\ --storage.tsdb.path=/var/lib/prometheus/ \\ --web.console.templates=/etc/prometheus/consoles \\ --web.console.libraries=/etc/prometheus/console_libraries \\ --storage.tsdb.retention.time=30d [Install] WantedBy=multi-user.target EOF # Инсталация на Node Exporter за този контейнер echo -e "${YELLOW}[4/9] Инсталация на Node Exporter...${NC}" NODE_VERSION="1.6.1" wget -q https://github.com/prometheus/node_exporter/releases/download/v${NODE_VERSION}/node_exporter-${NODE_VERSION}.linux-amd64.tar.gz tar xzf node_exporter-${NODE_VERSION}.linux-amd64.tar.gz cp node_exporter-${NODE_VERSION}.linux-amd64/node_exporter /usr/local/bin/ chown node_exporter:node_exporter /usr/local/bin/node_exporter rm -rf node_exporter-${NODE_VERSION}.linux-amd64* cat > /etc/systemd/system/node_exporter.service <<EOF [Unit] Description=Node Exporter for LXC Container Wants=network-online.target After=network-online.target [Service] User=node_exporter Group=node_exporter Type=simple ExecStart=/usr/local/bin/node_exporter [Install] WantedBy=multi-user.target EOF # Инсталация на Blackbox Exporter echo -e "${YELLOW}[5/9] Инсталация на Blackbox Exporter...${NC}" BLACKBOX_VERSION="0.24.0" wget -q https://github.com/prometheus/blackbox_exporter/releases/download/v${BLACKBOX_VERSION}/blackbox_exporter-${BLACKBOX_VERSION}.linux-amd64.tar.gz tar xzf blackbox_exporter-${BLACKBOX_VERSION}.linux-amd64.tar.gz cp blackbox_exporter-${BLACKBOX_VERSION}.linux-amd64/blackbox_exporter /usr/local/bin/ chown prometheus:prometheus /usr/local/bin/blackbox_exporter rm -rf blackbox_exporter-${BLACKBOX_VERSION}.linux-amd64* mkdir -p /etc/blackbox_exporter cat > /etc/blackbox_exporter/config.yml <<EOF modules: http_2xx: prober: http timeout: 5s http: valid_http_versions: ["HTTP/1.1", "HTTP/2.0"] valid_status_codes: [] method: GET follow_redirects: true tcp_connect: prober: tcp timeout: 5s icmp: prober: icmp timeout: 5s EOF cat > /etc/systemd/system/blackbox_exporter.service <<EOF [Unit] Description=Blackbox Exporter Wants=network-online.target After=network-online.target [Service] User=prometheus Group=prometheus Type=simple ExecStart=/usr/local/bin/blackbox_exporter --config.file=/etc/blackbox_exporter/config.yml [Install] WantedBy=multi-user.target EOF # Инсталация на Alertmanager echo -e "${YELLOW}[6/9] Инсталация на Alertmanager...${NC}" ALERT_VERSION="0.26.0" wget -q https://github.com/prometheus/alertmanager/releases/download/v${ALERT_VERSION}/alertmanager-${ALERT_VERSION}.linux-amd64.tar.gz tar xzf alertmanager-${ALERT_VERSION}.linux-amd64.tar.gz cp alertmanager-${ALERT_VERSION}.linux-amd64/alertmanager /usr/local/bin/ cp alertmanager-${ALERT_VERSION}.linux-amd64/amtool /usr/local/bin/ chown alertmanager:alertmanager /usr/local/bin/alertmanager /usr/local/bin/amtool rm -rf alertmanager-${ALERT_VERSION}.linux-amd64* mkdir -p /etc/alertmanager /var/lib/alertmanager chown -R alertmanager:alertmanager /etc/alertmanager /var/lib/alertmanager # Конфигурация на Alertmanager с Telegram cat > /etc/alertmanager/alertmanager.yml <<EOF global: resolve_timeout: 5m route: group_by: ['alertname', 'instance', 'alias'] group_wait: 10s group_interval: 10s repeat_interval: 12h receiver: 'telegram' receivers: - name: 'telegram' telegram_configs: - bot_token: '${TELEGRAM_TOKEN}' chat_id: ${TELEGRAM_CHAT_ID} parse_mode: 'HTML' message: | <b>{{ .Status | toUpper }}</b> - Proxmox LXC Monitor {{ range .Alerts }} <b>🏷 Alert:</b> {{ .Labels.alertname }} <b>📍 Container:</b> {{ .Labels.alias }} ({{ .Labels.instance }}) <b>🔴 Severity:</b> {{ .Labels.severity }} <b>📝 Summary:</b> {{ .Annotations.summary }} <b>ℹ️ Details:</b> {{ .Annotations.description }} <b>⏰ Started:</b> {{ .StartsAt.Format "02.01.2006 15:04:05" }} {{ end }} inhibit_rules: - source_match: severity: 'critical' target_match: severity: 'warning' equal: ['alertname', 'instance'] EOF chown -R alertmanager:alertmanager /etc/alertmanager cat > /etc/systemd/system/alertmanager.service <<EOF [Unit] Description=Alertmanager for Proxmox LXC Wants=network-online.target After=network-online.target [Service] User=alertmanager Group=alertmanager Type=simple ExecStart=/usr/local/bin/alertmanager \\ --config.file=/etc/alertmanager/alertmanager.yml \\ --storage.path=/var/lib/alertmanager/ [Install] WantedBy=multi-user.target EOF # Инсталация на Grafana echo -e "${YELLOW}[7/9] Инсталация на Grafana...${NC}" apt-get install -y apt-transport-https software-properties-common > /dev/null 2>&1 wget -q -O /usr/share/keyrings/grafana.key https://apt.grafana.com/gpg.key echo "deb [signed-by=/usr/share/keyrings/grafana.key] https://apt.grafana.com stable main" | tee /etc/apt/sources.list.d/grafana.list > /dev/null apt-get update -qq apt-get install -y grafana > /dev/null 2>&1 # Конфигурация на Grafana datasource mkdir -p /etc/grafana/provisioning/datasources cat > /etc/grafana/provisioning/datasources/prometheus.yml <<EOF apiVersion: 1 datasources: - name: Prometheus type: prometheus access: proxy url: http://localhost:9090 isDefault: true editable: true jsonData: timeInterval: "15s" EOF # Конфигурация на Grafana dashboards mkdir -p /etc/grafana/provisioning/dashboards cat > /etc/grafana/provisioning/dashboards/default.yml <<EOF apiVersion: 1 providers: - name: 'Proxmox LXC' orgId: 1 folder: 'LXC Containers' type: file disableDeletion: false updateIntervalSeconds: 10 allowUiUpdates: true options: path: /var/lib/grafana/dashboards EOF mkdir -p /var/lib/grafana/dashboards # Изтегляне на dashboards echo -e "${YELLOW}[8/9] Изтегляне на Grafana dashboards...${NC}" wget -q -O /var/lib/grafana/dashboards/node-exporter.json https://grafana.com/api/dashboards/1860/revisions/latest/download wget -q -O /var/lib/grafana/dashboards/prometheus-stats.json https://grafana.com/api/dashboards/2/revisions/latest/download chown -R grafana:grafana /var/lib/grafana/dashboards # Стартиране на всички сервизи echo -e "${YELLOW}[9/9] Стартиране на сервизите...${NC}" systemctl daemon-reload systemctl enable prometheus node_exporter blackbox_exporter alertmanager grafana-server > /dev/null 2>&1 systemctl start node_exporter systemctl start blackbox_exporter systemctl start prometheus systemctl start alertmanager systemctl start grafana-server # Изчакване за стартиране sleep 5 # Проверка на статуса echo -e "\n${BLUE}Проверка на статуса на сервизите...${NC}\n" SERVICES=("prometheus" "node_exporter" "blackbox_exporter" "alertmanager" "grafana-server") ALL_OK=true for service in "${SERVICES[@]}"; do if systemctl is-active --quiet $service; then echo -e "${GREEN}✓${NC} $service: Running" else echo -e "${RED}✗${NC} $service: Failed" ALL_OK=false fi done # Тригване на тестов alert echo -e "\n${YELLOW}Изпращане на тестов alert...${NC}" sleep 3 curl -X POST http://localhost:9090/-/reload 2>/dev/null # Финален изход echo -e "\n${GREEN}================================${NC}" echo -e "${GREEN}Инсталацията завърши успешно!${NC}" echo -e "${GREEN}================================${NC}\n" SERVER_IP=$(hostname -I | awk '{print $1}') echo -e "${BLUE}╔════════════════════════════════════════╗${NC}" echo -e "${BLUE}║ Proxmox LXC Monitoring Stack ║${NC}" echo -e "${BLUE}╚════════════════════════════════════════╝${NC}\n" echo -e "${YELLOW}🌐 Достъп до услугите:${NC}" echo -e " Prometheus: http://${SERVER_IP}:9090" echo -e " Alertmanager: http://${SERVER_IP}:9093" echo -e " Grafana: http://${SERVER_IP}:3000" echo -e " └─ User: ${GREEN}admin${NC}" echo -e " └─ Pass: ${GREEN}admin${NC}" echo -e " Node Exporter: http://${SERVER_IP}:9100/metrics" echo -e " Blackbox Export: http://${SERVER_IP}:9115" echo -e "\n${YELLOW}📁 Конфигурационни файлове:${NC}" echo -e " Prometheus: /etc/prometheus/prometheus.yml" echo -e " Alerts: /etc/prometheus/alerts.yml" echo -e " Alertmanager: /etc/alertmanager/alertmanager.yml" echo -e "\n${YELLOW}📋 Следващи стъпки:${NC}" echo -e " 1. Инсталирай Node Exporter в другите LXC контейнери" echo -e " 2. Добави IP адресите им в: ${GREEN}/etc/prometheus/prometheus.yml${NC}" echo -e " 3. Рестартирай Prometheus: ${GREEN}systemctl restart prometheus${NC}" echo -e "\n${GREEN}🎉 Тестов alert ще бъде изпратен в Telegram след 5-10 секунди!${NC}" echo -e "${YELLOW}📱 Telegram Chat ID: ${TELEGRAM_CHAT_ID}${NC}\n" # Създаване на helper скриптове cat > /usr/local/bin/add-lxc-container <<'ADDSCRIPT' #!/bin/bash if [ $# -ne 2 ]; then echo "Usage: add-lxc-container <IP> <Alias>" echo "Example: add-lxc-container 10.0.0.101 'Web Server'" exit 1 fi IP=$1 ALIAS=$2 # Добавяне в Prometheus config преди последния празен targets sed -i "/- targets: \[\]/i\\ - targets: ['${IP}:9100']\n labels:\n alias: '${ALIAS}'\n type: 'lxc-container'" /etc/prometheus/prometheus.yml systemctl restart prometheus echo "✓ Добавен контейнер: $ALIAS ($IP)" echo "Провери в Prometheus: http://localhost:9090/targets" ADDSCRIPT chmod +x /usr/local/bin/add-lxc-container echo -e "${GREEN}✓ Създаден helper скрипт: ${YELLOW}add-lxc-container${NC}" echo -e " Използвай: ${GREEN}add-lxc-container 10.0.0.101 'Web Server'${NC}\n" if [ "$ALL_OK" = true ]; then exit 0 else echo -e "${RED}Някои сервизи не стартираха правилно. Проверете логовете.${NC}" exit 1 fi