#!/bin/bash
set -eo pipefail
#set -eou pipefail

R='\e[1;31m'
AR='\e[0;31m'

G='\e[1;92m'
AG='\e[0;92m'

C='\e[1;36m'  # 亮青色
AC='\e[0;36m' # 暗青色

Y='\e[1;33m'  # 亮黄色
AY='\e[0;33m' # 暗黄色

H='\e[0;90m'  # 暗灰色 (推荐)
LH='\e[1;31m' # 亮黑色
AH='\e[1;90m' # 亮灰色

B='\e[0;37m' # 白色
B='\e[0;30m' # 黑色
NC='\e[0m' # No Color

# ====================== 公共工具函数 ======================
# 错误输出函数
error() {
    echo -e "${AR}[E] $*${NC}" >&2
    exit 1
}

# 信息输出函数
info() {
    echo -e "${AG}[I]${NC} ${AC}$*${NC}"
}

# 警告输出函数
warn() {
    echo -e "${AY}[W] $*${NC}"
}

# 清理最后一行
cline() {
    echo -ne "\r\033[K"
}

COLUMNS=$(( $(tput cols 2>/dev/null || echo 80) - 8 ))
SSEP=$(printf "%${COLUMNS}s" "" | tr " " "-")
DSEP=$(printf "%${COLUMNS}s" "" | tr " " "=")

status_dnf_packages="/tmp/status_dnf_packages"

# ====================== 全局关联数组定义（核心优化） ======================
# 全局选项：verbose/debug/help/nodes 等，支持 GLOBAL_OPTS[v] 判断
declare -A GLOBAL_OPTS=(
    [v]=""          # -v
    [verbose]=""    # --verbose
    [d]=""          # -d
    [debug]=""      # --debug
    [h]=""          # -h
    [help]=""       # --help
    [nodes]=""      # -n/--nodes 节点值
)

# 子命令栈（数组：存储 exec/list node/serv autofs）
CMD_STACK=()
# 公共参数集合（-开头）
PUBLIC_ARGS=()
# 子命令专属参数（--开头）
PRIVATE_ARGS=()
# 默认节点列表(会自动从 nodes 文件覆盖)
DEFAULT_NODES=("localhost")
# 全局生效节点数组（所有子命令共享）
GLOBAL_NODES=()
declare -A GLOBAL_PORT
declare -A global_env
declare -A global_treeinfo

load_os_envs() {
    interfaces=$(ls /sys/class/net | grep -v lo)
    for iface in $interfaces; do
        readarray -t config <<< "$(nmcli -g IP4.ADDRESS,IP4.GATEWAY,IP4.DNS device show $iface 2>/dev/null)"

        # IP地址
        ip_addr=$(echo "${config[0]}" | cut -d: -f2 | cut -d'/' -f1)
        global_env["${iface}_ip"]="${ip_addr:-}"

        # 网关
        gateway=$(echo "${config[1]}" | cut -d: -f2)
        global_env["${iface}_gateway"]="${gateway:-}"

        # DNS
        dns=$(echo "${config[2]}" | cut -d: -f2- | tr ' ' ',' | sed 's/,$//')
        global_env["${iface}_dns"]="${dns:-223.5.5.5,223.6.6.6}"

        # CIDR掩码
        cidr=$(nmcli -g IP4.ADDRESS device show $iface 2>/dev/null | head -1 | cut -d'/' -f2)
        global_env["${iface}_mask"]="${cidr:-24}"
    done
}

load_conf_envs() {

    while read -r key val; do
        # 跳过空行、注释行
        [[ -z "$key" || "$key" == \#* ]] && continue

        global_env["$key"]="$val"
    done < "$config_file"
}

parse_treeinfo() {
    local treeinfo_file="${1:-.treeinfo}"
    local current_section=""

    while IFS= read -r line || [[ -n "$line" ]]; do
        # 跳过空行和注释行
        [[ -z "$line" || "$line" =~ ^[[:space:]]*\; ]] && continue

        # 匹配 section，例如 [general]
        if [[ "$line" =~ ^\[([^]]+)\]$ ]]; then
            current_section="${BASH_REMATCH[1]}"
            continue
        fi

        # 匹配 key = value
        if [[ "$line" =~ ^[[:space:]]*([^=]+)[[:space:]]*=[[:space:]]*(.*)$ ]]; then
            local key="${BASH_REMATCH[1]}"
            local value="${BASH_REMATCH[2]}"

            key="${key#"${key%%[![:space:]]*}"}"
            key="${key%"${key##*[![:space:]]}"}"

            # 去除值两端和中间的所有空格
            value="${value#"${value%%[![:space:]]*}"}"
            value="${value%"${value##*[![:space:]]}"}"
            value=$(echo "$value" | tr -d ' ' | tr '[:upper:]' '[:lower:]')

            # 存储到关联数组，使用 "section.key" 作为键名
            global_treeinfo["${current_section}.${key}"]="$value"
        fi
    done < "$treeinfo_file"
}

getFileArgs(){
    # 调用模式: getFileArgs myfile "true"  未指定文件时只返回错误码
    # 调用模式: result=$(getFileArgs myfile "true") 返回文件
    # 调用模式: getFileArgs myfile 未指定文件时直接退出脚本,默认值: false

    local -n filename=$1
    local isForce=${2:-"false"} # 默认值为 "false"

    # 检查 file 参数是否传入
    if [[ -z "${GLOBAL_OPTS[file]-}" ]]; then
        local err_msg="未指定文件，请使用 -f 或 --file 参数"

        if [[ "$isForce" == "false" ]]; then
            error "$err_msg"
        else
            warn "$err_msg"
            return 1 # 返回非0表示失败
        fi
    fi

    # 检查文件是否存在
    if [[ -f "${GLOBAL_OPTS[file]}" ]] || [[ -d "${GLOBAL_OPTS[file]}" ]]; then
        filename="${GLOBAL_OPTS[file]}"
        return 0
    else
        local err_msg="错误：文件或目录不存在 -> ${GLOBAL_OPTS[file]}"

        if [[ "$isForce" == "false" ]]; then
            error "$err_msg"
        else
            warn "$err_msg"
            return 1
        fi
    fi
}

getArgs() {
    # getArgs interface iface e "msg"
    local key="$1"
    local result="$2"
    local level="${3:-w}" # 默认级别为 warn
    local msg="${4:-}"
    local value=""

    # 优先级1：检查 GLOBAL_OPTS（命令行参数）
    if [[ -n "${GLOBAL_OPTS[$key]}" ]]; then
        value="${GLOBAL_OPTS[$key]}"
    # 优先级2：检查 global_env（配置文件）
    elif [[ -n "${global_env[$key]}" ]]; then
        value="${global_env[$key]}"
    fi

    # 检查 GLOBAL_OPTS 数组中是否存在该键
    if [[ -n "$value" ]]; then
        printf -v "$result" "%s" "$value"
        return 0
    else
        # 根据不同的级别处理错误
        case "$level" in
            e)
                [[ -n "$msg" ]] && error "$msg" || error "必须指定 --$key 参数"
                ;;
            w)
                [[ -n "$msg" ]] && warn "$msg" || warn "未指定 --$key 参数，将使用默认值或跳过"
                ;;
            i)
                [[ -n "$msg" ]] && info "$msg" || info "未指定 --$key 参数"
                ;;
            *)
                [[ -n "$msg" ]] && info "$msg" || info "未指定 --$key 参数"
                ;;
        esac
        return 1
    fi
}

# ============================================================
# 辅助函数：打印所有环境变量（调试用）
# ============================================================
print_global_env() {
    for key in "${!global_env[@]}"; do
        printf "%-20s = %s\n" "$key" "${global_env[$key]}"
    done
    echo "======================================"
}

# ============================================================
# 辅助函数：快速获取系统信息摘要
# ============================================================
show_system_summary() {
    cat <<EOF
========================================
        系统信息摘要
========================================
操作系统:   ${global_env[release]} (${global_env[osname]} ${global_env[os_version]})
内核版本:   ${global_env[kernel_release]}
主机名:     ${global_env[hostname]}
架构:       ${global_env[architecture]}
CPU:        ${global_env[cpu_model]}
CPU核心数:  ${global_env[cpu_cores]}
内存总量:   ${global_env[mem_total]} GB
默认网卡:   ${global_env[default_interface]}
系统负载:   ${global_env[load_1min]} (1min) / ${global_env[load_5min]} (5min)
磁盘使用:   ${global_env[disk_root_usage]}% (根分区)
虚拟化:     ${global_env[virt_type]}
当前用户:   ${global_env[username]} (UID: ${global_env[uid]})
========================================
EOF
}

# ============================================================
# 辅助函数：通用函数
# ============================================================

# 通用命令执行函数，带进度显示和日志记录
# 用法: run_with_progress "命令描述" "要执行的命令" [日志文件]
# 示例: run_with_progress "编译SLURM" "rpmbuild -ta slurm.tar.gz" "slurm_build.log"
run_with_progress() {
    local description="$1"
    local command="$2"
    local log_file="${3:-/tmp/command_$(date +%Y%m%d_%H%M%S).log}"
    local count=0
    local start_time=$(date +%s)
    local exit_code=0
    
    info "$DSEP"
    info "开始: $description"
    info "日志: $log_file"
    info "$DSEP"
    
    # 执行命令，同时保存日志和显示进度
    eval "$command" 2>&1 | tee "$log_file" | while IFS= read -r line; do
        count=$((count + 1))
        local current_time=$(date +%s)
        local elapsed=$((current_time - start_time))
        local elapsed_min=$((elapsed / 60))
        local elapsed_sec=$((elapsed % 60))
        
        # 提取关键信息（去掉路径等冗余信息）
        local short_line=$(echo "$line" | sed 's|/home/[^/]*/|~/|g' | cut -c1-60)
        
        printf "\r${AC}[B] [%s]${NR} ${H}[%d]${NR} %02d:%02d - %.60s" \
            "$description" $count $elapsed_min $elapsed_sec "$short_line"
    done
    
    exit_code=${PIPESTATUS[0]}
    
    # 清除最后一行进度信息
    echo -ne "\r\033[K"

    if [ $exit_code -eq 0 ]; then
        info "[$description] - 完成! (共 $count 步, 耗时 $(($(date +%s) - start_time))秒)"
        info "日志已保存: $log_file"
    else
        warn "❌ $description - 失败! (退出码: $exit_code, 共 $count 步)"
        warn "请查看日志: $log_file"
        return $exit_code
    fi
}

# 过滤安装包
filter_pkgs(){
    local -n input_array=$1   # 输入的包数组
    local -n output_array=$2  # 输出的包数组

    output_array=()             # 清空输出数组
    for pkg in ${input_array[@]}; do
        #grep -q $pkg $status_dnf_packages && continue
        if ! rpm -q "$pkg" &>/dev/null; then
            #echo "$pkg" >> $status_dnf_packages
            output_array+=("$pkg")
        else
            info "Skip installation, already installed - $pkg"
        fi
    done
}

cidr_to_netmask() {
    local cidr=$1
    local masks=(
        "0.0.0.0"           # 0
        "128.0.0.0"         # 1
        "192.0.0.0"         # 2
        "224.0.0.0"         # 3
        "240.0.0.0"         # 4
        "248.0.0.0"         # 5
        "252.0.0.0"         # 6
        "254.0.0.0"         # 7
        "255.0.0.0"         # 8
        "255.128.0.0"       # 9
        "255.192.0.0"       # 10
        "255.224.0.0"       # 11
        "255.240.0.0"       # 12
        "255.248.0.0"       # 13
        "255.252.0.0"       # 14
        "255.254.0.0"       # 15
        "255.255.0.0"       # 16
        "255.255.128.0"     # 17
        "255.255.192.0"     # 18
        "255.255.224.0"     # 19
        "255.255.240.0"     # 20
        "255.255.248.0"     # 21
        "255.255.252.0"     # 22
        "255.255.254.0"     # 23
        "255.255.255.0"     # 24
        "255.255.255.128"   # 25
        "255.255.255.192"   # 26
        "255.255.255.224"   # 27
        "255.255.255.240"   # 28
        "255.255.255.248"   # 29
        "255.255.255.252"   # 30
        "255.255.255.254"   # 31
        "255.255.255.255"   # 32
    )
    echo "${masks[$cidr]}"
}

get_subnet_address() {
    local ip="$1"
    local mask="$2"

    IFS=. read -r i1 i2 i3 i4 <<< "$ip"
    IFS=. read -r m1 m2 m3 m4 <<< "$mask"

    n1=$((i1 & m1))
    n2=$((i2 & m2))
    n3=$((i3 & m3))
    n4=$((i4 & m4))

    echo "$n1.$n2.$n3.$n4"

}
# ====================== pxe server functions ====================
build_httpd_srv() {
    log_debug "当前节点：${GLOBAL_NODES[*]}"
    log_debug "公共参数：${PUBLIC_ARGS[*]}"
    log_debug "私有参数：${PRIVATE_ARGS[*]}"


    nopkgs=()
    pkgs=("httpd")
    filter_pkgs pkgs nopkgs
 
    local cmd=("dnf -y install ${nopkgs[*]}")
    [[ -z ${nopkgs[@]} ]] || run_with_progress "DNF 安装HTTPD服务..." "$cmd" "$log_file"

    http_root_conf="/etc/httpd"
    http_html_dirs="/var/www/html"
    http_serv_conf="/usr/lib/systemd/system/httpd.service"

    custom_conf="$http_root_conf/conf.d/sunhpc.conf"
    {
        echo 'Alias /linux /var/www/html/linux'
        echo ''
        echo 'UseCanonicalName Off'
        echo "ServerName `hostname -f`"
        echo ''
        echo '<Directory "/var/www/html/linux">'
        echo "    Options Indexes FollowSymLinks"
        echo "    AllowOverride None"
        echo "    Require all granted"
        echo ""
        echo "    EnableSendfile on"
        echo "</Directory>"
    } > ${custom_conf}

    #systemctl daemon-reload && systemctl restart httpd
    #systemctl is-enabled httpd &>/dev/null || systemctl enable httpd &>/dev/null


    if systemctl is-active httpd &>/dev/null ; then 
        info "HTTP 服务部署成功"
        info "HTTP 服务端口: 80,443"
    else
        warn "HTTP 服务部署失败,请查看服务信息 systemctl status httpd.service"
    fi
}

build_dnsmasq_srv() {
    log_debug "当前节点：${GLOBAL_NODES[*]}"
    log_debug "公共参数：${PUBLIC_ARGS[*]}"
    log_debug "私有参数：${PRIVATE_ARGS[*]}"

    nopkgs=()
    pkgs=("syslinux dnsmasq dnsmasq-utils ipxe-bootimgs")
    filter_pkgs pkgs nopkgs

    local cmd=("dnf -y install ${nopkgs[*]}")
    [[ -z ${nopkgs[@]} ]] || run_with_progress "DNF 安装 dnsmasq 服务..." "$cmd" "$log_file"

    getArgs interface iface   e "必须指定一个网络接口,例如: -i/--interface eth1"
    getArgs server ipaddr     e "Check the configuration file $config_file: server <ipaddr>"
    getArgs tftpboot tftpboot e "Check the configuration file $config_file: tftpboot <fullpath>"
    getArgs startip start_ip  e "Check the configuration file $config_file: startip <ipaddr>"
    getArgs endip   ender_ip  e "Check the configuration file $config_file: endip <ipaddr>"
    getArgs dns dnsip         e "Check the configuration file $config_file: dns <dnsaddr>"

    getArgs bios biosboot     e "Check the configuration file $config_file: biosboot <fullpath>"
    getArgs uefi uefiboot     e "Check the configuration file $config_file: uefiboot <fullpath>"
    getArgs ipxeboot ipxeboot e "Check the configuration file $config_file: ipxeboot <url>"


    biosboot="${biosboot#$tftpboot/}"
    uefiboot="${uefiboot#$tftpboot/}"
    dnsmasq_conf="/etc/dnsmasq.d/sunhpc.conf"

    {
        echo "interface=$iface"
        echo "bind-interfaces"
        echo ""
        echo "enable-tftp"
        echo "tftp-root=${tftpboot}"
        echo ""
        echo "# Gateway:3, DNS:6"
        echo "dhcp-range=$start_ip,$ender_ip,12h"
        echo "dhcp-option=3,${ipaddr}"
        echo "dhcp-option=6,$dnsip"
        echo ""
        echo "dhcp-match=set:bios,option:client-arch,0"
        echo "dhcp-match=set:uefi,option:client-arch,7"
        echo "dhcp-match=set:uefi,option:client-arch,9"
        echo ""
        echo "dhcp-boot=tag:bios,$biosboot"
        echo "dhcp-boot=tag:uefi,$uefiboot"
        echo "dhcp-boot=tag:ipxe,$ipxeboot"
    } > ${dnsmasq_conf}

}
# ====================== server functions ========================
build_slurm_srv() {
    log_debug "当前节点：${GLOBAL_NODES[*]}"
    log_debug "公共参数：${PUBLIC_ARGS[*]}"
    log_debug "私有参数：${PRIVATE_ARGS[*]}"

    FILE_CONF="slurm_build.conf"
    REPOS_DIRS="/etc/yum.repos.d/*.repo"
    SEARCH_STR="mirrors.ustc.edu.cn"
    log_file="slurm_build.log"
    EPEL_C="/etc/yum.repos.d/epel.repo"
    REPO_C="/etc/yum.repos.d/slurm.repo"
    DST_DIR="/srv/repos/slurm/packages"

    STATUS="$DST_DIR/.installed"
    init_db_flags="/var/lib/mysql/.initialized"
    slurmctld_flags="/etc/slurm/.slurmctld"
    slurmdbd_flags="/etc/slurm/.slurmdbd"
    slurmd_flags="/etc/slurm/.slurmd"
    

    # 检查 file 参数是否传入
    if [[ -z "${GLOBAL_OPTS[file]-}" ]]; then
        echo "错误：未指定文件，请使用 -f 或 --file 参数"
        exit 1
    fi

    # 检查文件是否存在
    if [[ ! -f "${GLOBAL_OPTS[file]}" ]]; then
        echo "错误：文件不存在 → ${GLOBAL_OPTS[file]}"
        exit 1
    fi

    info "${DSEP}"
    info "🔨 开始 Slurm 构建安装（断点续跑模式）"
    info "${DSEP}"

    dnf config-manager --enable crb
    [ -f "$EPEL_C" ] && info "EPEL Repo is enabled" || dnf -y install epel-release

    local cmd="dnf -y install @development yum-utils dnf-utils rpm-build"
    [[ -f $STATUS ]] || run_with_progress "DNF 安装系统开发依赖包..." "$cmd" "$log_file"

    local pkgs="munge munge-devel mariadb mariadb-devel \
        gtk2 gtk2-devel gtk3 gtk3-devel http-parser http-parser-devel \
        json-c json-c-devel libyaml libyaml-devel libjwt libjwt-devel \
        wget python3 readline-devel pam-devel perl-ExtUtils-MakeMaker \
        perl-devel perl-JSON-PP createrepo_c hdf5 hdf5-devel man2html \
        man2html-core pam pam-devel freeipmi freeipmi-devel numactl  \
        numactl-devel pmix pmix-devel hwloc hwloc-devel lua lua-devel \
        ucx ucx-devel jq lz4-devel mariadb-server mariadb-devel" 

    nopkgs=()
    filter_pkgs pkgs nopkgs

    cmd=("dnf -y install ${nopkgs[*]}")
    [[ -z ${nopkgs[@]} ]] || run_with_progress "DNF 安装系统基础依赖包..." "$cmd" "$log_file"

    export CPPFLAGS="-I$PWD/deps_pkgs/cuda-13.0/targets/x86_64-linux/include ${CPPFLAGS}"
    export LDFLAGS="-L$PWD/deps_pkgs/cuda-13.0/targets/x86_64-linux/lib/stubs ${LDFLAGS}"

    [[ -f $STATUS ]] && info "Slurm 已经存在 ..." || info "开始制作 Slurm rpm 安装包 ..."

    TARFILE=${GLOBAL_OPTS[file]}
    local cmd="rpmbuild -ta \
        --with slurmrestd --with hdf5 --with hwloc --with muma \
        --with pmix --with nvml --with lua --with ucx --with jwt --with freeipmi \
        $TARFILE" 

    [[ -f $STATUS ]] || run_with_progress "Slurm RPM 构建..." "$cmd" "$log_file"

    mkdir -pv $DST_DIR
    find $HOME/rpmbuild/RPMS/ -iname "*.rpm" -exec mv {} $DST_DIR \;

    {
        echo "[slurm]"
        echo "name=Slurm Local Repository"
        echo "baseurl=file:///$DST_DIR"
        echo "gpgcheck=0"
        echo "enabled=1"
    } > "$REPO_C"
    touch $DST_DIR/.installed

    #createrepo_c $DST_DIR 2>&1 | tee $log_file
    info "SLURM 部署完成, Repos: $DST_DIR"

    local pkgs="pcp-pmda-slurm slurm slurm-perlapi \
        slurm-contribs slurm-devel slurm-libpmi slurm-pam_slurm \
        slurm-sackd slurm-slurmctld slurm-slurmd \
        slurm-slurmdbd slurm-slurmrestd \
        slurm-example-configs"

    nopkgs=()
    filter_pkgs pkgs nopkgs

    cmd=("dnf -y install ${nopkgs[*]}")
    [[ -z ${nopkgs[@]} ]] || run_with_progress "Slurm 服务安装..." "$cmd" "$log_file"

    # 开始配置 MariaDB 数据库
    slurm_db_conf="/etc/my.cnf.d/slurm.cnf"
    {
        echo "[mysqld]"
        echo "innodb_buffer_pool_size = 2G"
        echo "innodb_log_file_size = 64M"
        echo "innodb_lock_wait_timeout = 900"
    } > "$slurm_db_conf"
    systemctl is-active mariadb  &>/dev/null || systemctl start mariadb  &>/dev/null
    systemctl is-enabled mariadb &>/dev/null || systemctl enable mariadb &>/dev/null

    DB_ROOT_PASSWORD="admin_b101"
    # 静默执行初始化数据库: mysql_secure_installation
    [[ -f $init_db_flags ]] && info "Database already initialized, skipping mysql_secure_installation" \
    || mysql_secure_installation <<EOF

n
y
${DB_ROOT_PASSWORD}
${DB_ROOT_PASSWORD}
y
y
y
y
EOF
    if mysql -u root -p${DB_ROOT_PASSWORD} -e "select 1" >/dev/null 2>&1; then
        info "MariaDB initialization and securing completed successfully!"
    else
        warn "Error: Authentication failed. Please check the logs."
    fi
    info "数据库初始化密码: $DB_ROOT_PASSWORD"
    info "重新设置密码: "
    info "    systemctl stop mariadb"
    info "    mysqld_safe --skip-grant-tables &"
    info "    mysql -u root"
    info "    flush privileges;"
    info "    alter user 'root'@'localhost' identified by 'new_pass'"
    info "    exit"
    info "killall mysqld"
    info "systemctl start mariadb"

    SLURM_DB_PW="admin_b101"
    SLURM_DB_NM="slurm_acct_db"

    info "开始创建 Slurm 基础数据库: ${SLURM_DB_NM}"
    info "Slurm Database User: slurm"
    info "Slurm Database Pass: $SLURM_DB_PW"

    [[ -f $init_db_flags ]] || mysql -u root -p${DB_ROOT_PASSWORD} <<-EOSQL
CREATE DATABASE IF NOT EXISTS ${SLURM_DB_NM};
DROP USER IF EXISTS 'slurm'@'localhost';
CREATE USER IF NOT EXISTS 'slurm'@'localhost' identified by '${SLURM_DB_PW}';
GRANT ALL PRIVILEGES ON ${SLURM_DB_NM}.* TO 'slurm'@'localhost' WITH GRANT OPTION;
FLUSH PRIVILEGES;
EOSQL
    touch ${init_db_flags}

    # 配置 slurmdbd 服务
    slurm_root="/etc/slurm"
    slurm_dbdc="$slurm_root/slurmdbd.conf"
    slurm_ctld="$slurm_root/slurm.conf"
    getent passwd slurm &>/dev/null || useradd -r -b /var/lib -s /sbin/nologin slurm
    getent passwd slurm &>/dev/null || useradd -r -b /var/lib -s /sbin/nologin slurmrestd

    touch ${slurm_dbdc}
    chown slurm:slurm ${slurm_dbdc}
    chmod 600 ${slurm_dbdc}

    mkdir -p "/var/log/slurm"
    mkdir -p "/var/run/slurmdbd"
    chown -R slurm:slurm "/var/log/slurm"
    chown -R slurm:slurm "/var/run/slurmdbd"

    [[ -f $slurmdbd_flags ]] && info "SlurmDBD Service is configuration finisheld..." \
    || cat > ${slurm_dbdc} <<-EOSLURMDBD
AuthType=auth/munge
LogFile=/var/log/slurm/slurmdbd.log
PidFile=/var/run/slurmdbd/slurmdbd.pid

DbdHost=`hostname -s`
DbdPort=6819
SlurmUser=slurm
DebugLevel=info

StorageType=accounting_storage/mysql
StorageHost=localhost
StoragePort=3306
StorageUser=slurm
StoragePass=${SLURM_DB_PW}
StorageLoc=${SLURM_DB_NM}

ArchiveEvents=yes
ArchiveJobs=yes
ArchiveResvs=yes
ArchiveSteps=no
ArchiveSuspend=no
ArchiveTXN=no
ArchiveUsage=no
PurgeEventAfter=1month
PurgeJobAfter=12month
PurgeResvAfter=1month
PurgeStepAfter=1month
PurgeSuspendAfter=1month
PurgeTXNAfter=12month
PurgeUsageAfter=24month
EOSLURMDBD
    touch $slurmdbd_flags

    if [[ ! -f $slurmdbd_flags ]]; then
        systemctl restart slurmdbd  &>/dev/null
        systemctl is-enabled slurmdbd &>/dev/null || systemctl enable slurmdbd &>/dev/null
    fi

    # 配置 slurmctl 服务
    nodeinfo=$(slurmd -C | head -1)
    [[ -f $slurmctld_flags ]] && info "SlurmCTLD Service is configuration finisheld..." \
    || cat > ${slurm_ctld} <<-EOSLURMCTLD
ClusterName=cluster
ControlMachine=`hostname -s`
AuthType=auth/munge
SlurmUser=slurm
SlurmdUser=root
SlurmctldPort=6817
SlurmdPort=6818

StateSaveLocation=/var/spool/slurm/ctld
SlurmdSpoolDir=/var/spool/slurm/d
SlurmctldPidFile=/var/run/slurmctld.pid
SlurmdPidFile=/var/run/slurmd.pid
ProctrackType=proctrack/linuxproc
SchedulerType=sched/backfill
SelectType=select/cons_tres
SelectTypeParameters=CR_Core

AccountingStorageType=accounting_storage/slurmdbd
AccountingStorageHost=localhost
AccountingStoragePort=6819
MailProg=/bin/true

${nodeinfo} State=UNKNOWN
PartitionName=control Nodes=cluster Default=YES MaxTime=INFINITE State=UP
EOSLURMCTLD
    touch $slurmctld_flags

    if [[ ! -f $slurmctld_flags ]]; then
        systemctl restart slurmctld  &>/dev/null
        systemctl is-enabled slurmctld &>/dev/null || systemctl enable slurmctld &>/dev/null
    fi

    echo "$SSEP"
    /usr/bin/sacctmgr show clusters format=cluster,controlhost,controlport,qos
    #/usr/bin/sacctmgr show clusters format=cluster,controlhost,controlport --noheader
    echo "$SSEP"
    /usr/bin/sinfo -N -l | tail -n +2
    echo "$SSEP"
}

# ====================== 核心函数：节点解析 ======================
load_nodes_from_file() {
    local nodes_file="./nodes"

    # 如果文件不存在，使用默认内置节点
    if [[ ! -f "${nodes_file}" ]]; then
        log_debug "节点文件 ${nodes_file} 不存在，使用内置默认节点"
        return
    fi

    # 读取文件，去空行、去注释、去重、去前后空格
    mapfile -t lines < "${nodes_file}"
    DEFAULT_NODES=()
    for line in "${lines[@]}"; do
        # 跳过空行/空白行
        [ -z "${line// /}" ] && continue

        # 把一行切成数组(按空格分割)
        cols=($line)

        node="${cols[0]}"       # 节点名称(必须)
        port="${cols[1]:-22}"   # 端口不存在 -> 默认 22
        addr="${cols[2]:-}"     # IP 不存在  -> 默认空字符串

        # 按空格分割成数组
        GLOBAL_PORT["$node"]=${port}

        # 去掉空格 + 跳过空行/注释行
        clean_line=$(echo "${node}" | xargs)
        [[ -z "${clean_line}" || "${clean_line}" =~ ^# ]] && continue
        DEFAULT_NODES+=("${node}")
    done

    log_info "成功从 nodes 文件加载节点：${#DEFAULT_NODES[@]} 个"
    log_debug "nodes 文件节点列表：${DEFAULT_NODES[*]}"
}

parse_nodes() {
    local node_val="$1"
    GLOBAL_NODES=()
    
    if [[ "$node_val" == "all" ]]; then
        GLOBAL_NODES=("${DEFAULT_NODES[@]}")
    else
        IFS=',' read -ra tmp <<< "$node_val"
        GLOBAL_NODES=("${tmp[@]}")
    fi
    
    GLOBAL_OPTS[nodes]="$node_val"
}

# 检查节点状态（online/offline/auth/pass）
check_node_status() {
    local node="$1"
    port=${GLOBAL_PORT[$node]}

    # 检查网络连通性
    if ! ping -c 1 -W 1 "$node" &>/dev/null; then
        echo "offline"
        return
    fi

    # 检查是否是本机
    localhost=$(hostname -s)
    if [[ "$node" == "localhost" || "$node" == $localhost ]]; then
        echo "localhost"
        return
    fi

    # 检查SSH免密登录
    if ssh -o ConnectTimeout=1 -o BatchMode=yes -o StrictHostKeyChecking=accept-new  -p $port "$node" true &>/dev/null; then
        echo "online,auth"
    else
        echo "online,pass"
    fi
}

show_nodes() {
    echo "$SSEP"
    printf "${G}%-12s %-15s %-8s %s${NC}\n" "Nodename" "Status" "Port" "Days"
    echo "$SSEP"
    for node in "${GLOBAL_NODES[@]}"; do
        status=$(check_node_status "$node")
        port=${GLOBAL_PORT[$node]}

        if [[ "$status" == "offline" ]]; then
            uptime_days="---"
        else
            uptime_days=$(ssh -o ConnectTimeout=1 -o BatchMode=yes -p$port "$node" "uptime \
                | awk -F'up|days' '{print \$2}' | tr -d ',' | xargs" 2>/dev/null || echo "---")
        fi

        printf "%-12s %-15s %-8s %s\n" "$node" "$status" "$port" "$uptime_days"
    done
    echo "$SSEP"
}

run_cmd() {
    local node="$1"
    local cmds="$2"
    local output=""
    local exit_code=0

    # 获取本机主机名（自动识别管理节点）
    local localhost=$(hostname -s)
    if [[ "$node" == "$localhost" || "$node" == "localhost" ]]; then
        # ====================== 本机：直接执行 ======================
        output=$(eval "$cmds" 2>&1)
        exit_code=$?
    else
        # ====================== 远程：SSH 执行 ======================
        output=$(ssh -o ConnectTimeout=1 -o BatchMode=yes "$node" "$cmds" 2>&1)
        exit_code=$?
    fi


    if [[ $exit_code -eq 0 ]]; then
        info "执行成功: ${node}"
    else
        warn "执行失败: ${node} (${R}Code${NC}: ${R}$exit_code${NC})"
    fi
    echo "$DSEP"

    echo "$output"
}



# ====================== 核心函数: 同步文件路径处理 ======================
sync_file_path() {
    local src_file="$1"

    # ======================== 核心规则 ========================
    # 1. 如果源是 绝对路径（/开头），直接返回原路径（系统目录）
    if [[ "$src_file" == /* ]]; then
        echo "$src_file"
        return
    fi

    # 2. 如果源是 相对路径（linux/alls/* 或 linux/节点/*）
    #    自动去掉 linux/xxx/ 前缀，剩余部分自动变成 绝对路径（前面加 /）
    #    支持：etc usr opt var home root 任意目录，无限扩展
    # ==================================================
    local rel_path
    rel_path=$(echo "$src_file" | sed -E 's#^linux/(alls|[-_a-zA-Z0-9]+)/##')

    # 拼接为绝对路径
    echo "/$rel_path"
}

sync_file_all() {
    load_nodes "$NODE_ARG"

    # ===================== 美化打印节点列表（固定缩进 + 自动换行） =====================
    local indent="           "  # 11个空格
    local line=""
    local count=0
    local max_per_line=5  # 每行显示几个节点

    info "开始同步所有节点文件，节点列表："
    for node in "${NODES[@]}"; do
        line+="$node "
        count=$((count + 1))

        if [[ $count -ge $max_per_line ]]; then
            echo -e "${indent}${line}"
            line=""
            count=0
        fi
    done

    # 打印剩余不足一行的节点
    if [[ -n "$line" ]]; then
        echo -e "${indent}${line}"
    fi
    # ================================================================================
    for node_dir in linux/*; do
        [[ -d "$node_dir" ]] || continue

        for sub_dir in "$node_dir"/*; do
            [[ -d "$sub_dir" ]] || continue

            info "处理源目录：$sub_dir"

            for node in "${NODES[@]}"; do
                if [[ "$node_dir" == "linux/alls" || "$node_dir" == "linux/$node" ]]; then
                    target=$(sync_file_path "$sub_dir")
                    info "同步 ${Y}$sub_dir${NC} -> ${R}$node${NC}:${Y}$target${NC}"
                    rsync -avzAP --info=progress2 --exclude='.*' "$sub_dir"/ "$node":"$target"/
                fi
            done
        done
    done

    info "同步完成！"
}

sync_file_single() {
    local src_file="$1"
    [[ -z "$src_file" ]] && error "请使用 -f 指定文件路径"
    [[ -e "$src_file" ]] || error "文件不存在：$src_file"

    load_nodes "$NODE_ARG"

    local indent="           "  # 11个空格
    local line=""
    local count=0
    local max_per_line=5  # 每行显示几个节点

    info "开始同步所有节点文件，节点列表："
    for node in "${NODES[@]}"; do
        line+="$node "
        count=$((count + 1))

        if [[ $count -ge $max_per_line ]]; then
            echo -e "${indent}${line}"
            line=""
            count=0
        fi
    done

    # 打印剩余不足一行的节点
    if [[ -n "$line" ]]; then
        echo -e "${indent}${line}"
    fi

    target=$(sync_file_path "$src_file")
    for node in "${NODES[@]}"; do
        info "同步 ${Y}$src_file${NC} -> ${R}$node${NC}:${Y}$target${NC}"
        rsync -avzAP --info=progress2 "$src_file" "$node":"$target"
    done
    info "同步完成！"
}

# rsync 同步函数，支持 ISO 文件和普通目录
# 用法: rsync_sync_file <source> <destination> [rsync_options]
# 参数:
#   source        - 源路径（文件或目录，支持 ISO 文件）
#   destination   - 目标路径
#   rsync_options - 可选，额外的 rsync 参数
# 返回:
#   0 - 成功
#   1 - 失败
rsync_sync_file() {
    local source="$1"
    local destination="$2"
    local rsync_opts="${3:--ah --info=progress2}"  # 默认 rsync 选项
    local temp_mount=""
    local is_iso=false
    local exit_code=0
    
    # 检查参数
    if [[ -z "$source" ]] || [[ -z "$destination" ]]; then
        warn "必须指定源路径和目标路径" >&2
        warn "用法: rsync_sync_file <source> <destination> [rsync_options]" >&2
        return 1
    fi
    
    # 检查源路径是否存在
    if [[ ! -e "$source" ]]; then
        warn "源路径不存在: $source" >&2
        return 1
    fi
    
    # 检查是否为 ISO 文件
    if [[ -f "$source" && ( "$source" =~ \.(iso|ISO)$ ) ]]; then
        is_iso=true
        info "检测到 ISO 文件: $source"
        
        # 创建临时挂载点
        temp_mount=$(mktemp -d "/tmp/iso_mount_XXXXXX")
        info "创建临时挂载点: $temp_mount"
        
        # 挂载 ISO 文件
        if mount -o loop "$source" "$temp_mount" 2>/dev/null; then
            info "成功挂载 ISO 文件到: $temp_mount"
            source="$temp_mount"
        else
            warn "无法挂载 ISO 文件，可能需要 root 权限" >&2
            rmdir "$temp_mount" 2>/dev/null
            return 1
        fi
    fi
    
    # 确保目标目录存在
    # 提取目标目录(如果是文件,取目录部分)
    local dest_dir
    if [[ "$destination" == */ ]]; then
        dest_dir="$destination" # 目标 / 结尾, 视为目录
    else
        dest_dir="$(dirname "$destination")" # 目标是文件, 取目录部分
    fi

    if [[ ! -d "$dest_dir" ]]; then
        info "创建目标目录: $dest_dir"
        mkdir -p "$dest_dir" || {
            warn "无法创建目标目录: $dest_dir" >&2
            exit_code=1
            cleanup
        }
    fi
    
    # 确保源路径以斜杠结尾（rsync 目录同步规范）
    local rsync_source="$source"
    if [[ -d "$source" ]]; then
        # 如果是目录，确保以 / 结尾，这样同步目录内容而不是目录本身
        rsync_source="${source}/"
    fi
    
    # 执行 rsync 同步
    info "开始同步数据..."
    info "      源路径: $source"
    info "    目标路径: $destination"
    info "  rsync 选项: $rsync_opts"
    
    rsync $rsync_opts "$rsync_source" "$destination"
    exit_code=$?
    
    if [[ $exit_code -eq 0 ]]; then
        info "同步成功完成"
    else
        warn "rsync 同步失败，退出码: $exit_code" >&2
    fi
    
    # 清理：卸载临时挂载点
    cleanup() {
        if [[ "$is_iso" == "true" ]] && [[ -n "$temp_mount" ]]; then
            info "清理临时挂载点..."
            umount "$temp_mount" 2>/dev/null || {
                warn "无法卸载 $temp_mount，可能需要手动清理" >&2
            }
            rmdir "$temp_mount" 2>/dev/null
            info "已删除临时挂载点: $temp_mount"
        fi
        
        return $exit_code
    }
}
# ====================== 帮助文档 ======================
show_help() {
    cat <<EOF
SunHPC command tools

Usage: sunhpc <COMMAND> [OPTIONS]

Public Args:
  -v, --verbose       Verbose output
  -d, --debug         Debug Mode
  -h, --help          Display the help infomation
  -n, --nodes <list>  Specified node (all or node1,node2,node3)

Support commands:
  sunhpc exec                 	Batch execute command
  sunhpc init                 	Execute the init op
  sunhpc sync   file          	Run the synchronize file command

  sunhpc show   nodes          	Display nodes list
  sunhpc show   vimrc         	Dispaly Vimrc config
  sunhpc show   system        	Display the system infomation

  sunhpc build   httpd         	Build httpd   server
  sunhpc build   slurm		    Build slurm   server 
  sunhpc build   autofs		    Build autofs  server
  sunhpc build   dnsmasq	    Build dnsmasq server
  sunhpc create syncfile      	Display the help infomation

Report bugs to: <support@sunhpc.com>
Htool URL: https://gitea.sunhpc.com
Gitea URL: https://gitea.sunhpc.com/htool.git
EOF
}

show_help_sync() {
    cat <<EOF
SunHPC command tools

Usage: sunhpc <COMMAND> [OPTIONS]

Public Args:
  -v, --verbose       Verbose output
  -d, --debug         Debug Mode
  -h, --help          Display the help infomation
  -n, --nodes <list>  Specified node (all or node1,node2,node3)

Command Example:
  sunhpc sync file -f/--file <filepath>  Sync a single file
  sunhpc sync file -a/--all              Sync all files

Report bugs to: <support@sunhpc.com>
Htool URL: https://gitea.sunhpc.com
Gitea URL: https://gitea.sunhpc.com/htool.git
EOF
}

show_help_slurm() {
    cat <<EOF
SunHPC command tools

Usage: sunhpc <COMMAND> [OPTIONS]

Public Args:
  -v, --verbose       Verbose output
  -d, --debug         Debug Mode
  -h, --help          Display the help infomation
  -n, --nodes <list>  Specified node (all or node1,node2,node3)

Command Example:
  sunhpc serv slurm -b/--build			        Build Slurm rpm packages
  sunhpc serv slurm -i/--install -m/master     	Install Slurm to frontend
  sunhpc serv slurm -i/--install -c/compute     Install Slurm to computes

Report bugs to: <support@sunhpc.com>
Htool URL: https://gitea.sunhpc.com
Gitea URL: https://gitea.sunhpc.com/htool.git
EOF
}

show_vimrc() {
    cat <<-EOF
" 开启语法高亮
syntax enable

" 设置缩进为4个空格
set tabstop=4
set softtabstop=4
set shiftwidth=4

" 用空格替代Tab键（禁止输入真实Tab）
set expandtab

" 自动缩进（写代码更方便）
set autoindent
set smartindent
EOF
}
# ====================== Verbose/Debug 输出工具 ======================
log_info() {
    if [[ -n "${GLOBAL_OPTS[v]}" || -n "${GLOBAL_OPTS[verbose]}" ]]; then
        echo -e "\033[32m[I]\033[0m $1"
    fi
}

log_debug() {
    if [[ -n "${GLOBAL_OPTS[d]}" || -n "${GLOBAL_OPTS[debug]}" ]]; then
        echo -e "\033[33m[D]\033[0m $1"
    fi
}

# ====================== 子命令执行 ======================
init_run() {

    info "初次使用此工具,需要在当前目录创建一个配置文件."
    info "  1、手动创建配置文件,e.g., touch $config_file"
    info "  2、自动创建配置文件,e.g., "
    info "         sunhpc init -i <interface> -m /mnt/cdrom"
    echo ""

    getArgs interface iface e "必须指定一个网络接口,例如: -i/--interface eth1"
    getArgs mntroot mntroot e "必须指定 mnt 路径,例如: -m/--mntroot /mnt/cdrom"

    treeinfo=$(find -L $mntroot -name ".treeinfo")
    parse_treeinfo $treeinfo

    for key in "${!global_treeinfo[@]}"; do 
        log_debug "key: '$key', Value: '${global_treeinfo[$key]}'";
    done

    [[ -z "$treeinfo" ]] && warn "提供的 $mntroot 目录为空,或是不被支持的系统."

    nopkgs=()
    pkgs=("ipxe-bootimgs")
    filter_pkgs pkgs nopkgs
 
    local cmd=("dnf -y install ${nopkgs[*]}")
    [[ -z ${nopkgs[@]} ]] || run_with_progress "DNF Install ipxe tools..." "$cmd" "$log_file"

    srv_root="/srv"
    repos_root="$srv_root/repos"

    kernel_path="${global_treeinfo[images-x86_64.kernel]}"
    initrd_path="${global_treeinfo[images-x86_64.initrd]}"

    sys_os_rels="${global_treeinfo[general.family]}"  # RockyLinux
    sys_os_vers="${global_treeinfo[general.version]}" # 9.7
    url_os_path="$sys_os_rels/$sys_os_vers" # http://ip/ 后面系统路径 rockylinux/9.7

    ipaddr=${global_env[${iface}_ip]}
    netcidr=${global_env[${iface}_mask]}
    netmask=$(cidr_to_netmask 24)
    subnet=$(get_subnet_address $ipaddr $netmask)

    IFS=. read -r i1 i2 i3 i4 <<< "$ipaddr"
    start_ip="$i1.$i2.$i3.10"
    ender_ip="$i1.$i2.$i3.90"

    mnt_dst="$repos_root/$url_os_path"
    rsync_sync_file $mntroot $mnt_dst

    lrepo_path="$repos_root/$sys_os_rels"

    wwwroot="/var/www/html/linux"
    [[ -e $wwwroot ]] || mkdir -p $wwwroot

    ks_src="$PWD/pkgs/"
    ks_dst="$srv_root/htool/pkgs/"
    [[ -e $ks_src ]] && rsync_sync_file $ks_src $ks_dst

    tftpboot="/srv/pxelinux/tftpboot"

    efi_src="$mnt_dst/EFI/BOOT/BOOTX64.EFI"
    [[ -e $efi_src ]] && rsync_sync_file $efi_src $tftpboot/bootx64.efi \
        || warn "UEFI Boot file no found: $efi_src"

    ipxe_src="/usr/share/ipxe/undionly.kpxe"
    [[ -e $ipxe_src ]] && rsync_sync_file $ipxe_src $tftpboot/boot/ \
        || warn "iPXE Boot file no found: $ipxe_tools"

    ks_args="net.ifnames=0 biosdevname=0"
    ks_rhel="http://$ipaddr/linux/ks/rhel.ks"
    ks_dfmt="http://$ipaddr/linux/ks/dfmt.sh"
    ks_ipxe="http://$ipaddr/linux/ks/ipxe.sh"
    ks_init="http://$ipaddr/linux/ks/init.sh"
    ks_repo="http://$ipaddr/linux/$url_os_path"
    vmlinuz="http://$ipaddr/linux/$url_os_path/$kernel_path"
    initrds="http://$ipaddr/linux/$url_os_path/$initrd_path"

    {
        echo "# ===================================================="
        echo "# Generated by htool commands"
        echo "# ===================================================="
        echo ""
        echo "interface $iface"
        echo "server    $ipaddr"
        echo "pkgsroot  $ks_dst"
        echo "wwwroot   $wwwroot"
        echo "tftpboot  $tftpboot"
        echo "startip   $start_ip"
        echo "endip     $ender_ip"
        echo "dns       223.5.5.5"
        echo "osname    $sys_os_rels"
        echo "osvers    $sys_os_vers"
        echo "ospath    $url_os_path"
        echo "lrepo     $lrepo_path"
        echo "bios      $tftpboot/boot/undionly.kpxe"
        echo "uefi      $tftpboot/boot/bootx64.efi"
        echo "dfmt      $ks_dfmt"
        echo "ksaddr    $ks_rhel"
        echo "ipxeboot  $ks_ipxe"
        echo "repos     $ks_repo"
        echo "vmlinuz   $vmlinuz"
        echo "initrd    $initrds"
    } > $config_file

    pkgs_ipxe="$srv_root/htool/pkgs/ks/ipxe.sh"
    sed -i "s|@osname@|$sys_os_rels|"  $pkgs_ipxe
    sed -i "s|@osrels@|"${sys_os_rels^}"|"  $pkgs_ipxe
    sed -i "s|@osvers@|$sys_os_vers|"  $pkgs_ipxe
    sed -i "s|@vmlinuz@|$vmlinuz|"     $pkgs_ipxe
    sed -i "s|@initrds@|$initrds|"     $pkgs_ipxe
    sed -i "s|@kargs@|$ks_args|"       $pkgs_ipxe
    sed -i "s|@repos@|$ks_repo|"       $pkgs_ipxe
    sed -i "s|@ksarg@|$ks_rhel|"       $pkgs_ipxe

    pkgs_rhel="$srv_root/htool/pkgs/ks/rhel.ks"
    sed -i "s|@url@|$ks_repo|"  $pkgs_rhel
    sed -i "s|@dfmt@|$ks_dfmt|" $pkgs_rhel
    sed -i "s|@init@|$ks_init|" $pkgs_rhel


    pkgs_init="${ks_dst}ks/init.sh"
    sed -i "s|@osipaddr@|$ipaddr|"    $pkgs_init
    sed -i "s|@ospath@|$url_os_path|" $pkgs_init
}

exec_run() {
    log_debug "当前节点：${GLOBAL_NODES[*]}"
    log_debug "公共参数：${PUBLIC_ARGS[*]}"
    log_debug "私有参数：${PRIVATE_ARGS[*]}"

    local cmds=$1
    [[ -z "$cmds" ]] && error "请指定要执行的命令，例如：sunhpc exec \"df -h\""

    echo "$DSEP"
    info "执行命令: $cmds"

    localhost=$(hostname -s)
    for node in "${GLOBAL_NODES[@]}"; do
        if [[ "$node" != "$localhost" || "$node" != "localhost" ]]; then
            status=$(check_node_status "$node")
            if [[ "$status" == "offline" ]]; then
                warn "$node 节点离线，跳过执行"
                continue
            fi
            if [[ "$status" == "online,pass" ]]; then
                warn "$node 需要密码登录，跳过执行"
                continue
            fi
        fi

        # ========== 核心：自动本机/远程执行 ==========
        run_cmd "$node" "$cmds"

    done
    echo "$SSEP"
    info "命令执行完成！"
    echo "$SSEP"
}

sync_run() {
    log_debug "当前节点：${GLOBAL_NODES[*]}"
    log_debug "公共参数：${PUBLIC_ARGS[*]}"
    log_debug "私有参数：${PRIVATE_ARGS[*]}"

    #case "${PRIVATE_ARGS[@]}" in
    case "${GLOBAL_OPTS[@]}" in
         all)  echo 'all----->' ;;#sync_file_all ;;
        file)  echo 'file---->' ;;#sync_file_single ;;
           *)  show_help_sync ;;
    esac

}

create_sync_file() {
    dirspath=(
        "/etc/passwd"
        "/etc/shadow"
        "/etc/group"
        "/etc/slurm"
        "/etc/systemd/system"
        "/usr/lib/systemd/system"
    )

    local base="${1:-.}"
    local name="${2:-linux}"
    local all_dir="${base}/${name}/all"

    mkdir -p "${all_dir}" || return 1
    for d in "${dirspath}"; do
        if [[ -f $d ]]; then
            src_dir=$(dirname "$d")
            dst_dir="${all_dir}/$src_dir"
            mkdir -p "${dst_dir}"
            /usr/bin/cp $d ${dst_dir}
        else
            mkdir -p "${all_dir}/${d}"
        fi
    done

    for n in "${GLOBAL_NODES[@]}"; do
        mkdir -p "${base}/${name}/${n}"
    done
    
}

# ====================== 命令路由 ======================
dispatch() {
    # 优先显示帮助
    if [[ -n "${GLOBAL_OPTS[h]}" || -n "${GLOBAL_OPTS[help]}" || ${#CMD_STACK[@]} -eq 0 ]]; then
        [[ $helper -ne 1 ]] && show_help
        exit 0
    fi

    log_debug "$DSEP"
    log_debug "解析子命令：${CMD_STACK[*]}"
    log_debug "专属参数  ：${PRIVATE_ARGS[*]}"

    case "${CMD_STACK[0]}" in
        exec) 
            exec_run "${CMD_STACK[1]}"
            ;;
        init) 
            init_run "${CMD_STACK[1]}"
            ;;
        sync)
            case "${CMD_STACK[1]}" in 
                file)       sync_run  ;;
                *)          show_help ;;
            esac
            ;;
        show)
            case "${CMD_STACK[1]}" in
                 node)      show_nodes ;;
                 vimrc)     show_vimrc ;;
                system)     show_system_summary ;;
                *)          show_help;;
            esac
            ;;
        build)
            case "${CMD_STACK[1]}" in
                httpd)      build_httpd_srv ;;
                slurm)      build_slurm_srv ;;
                dnsmasq)    build_dnsmasq_srv ;;
                *)          show_help;;
            esac
            ;;
        create)
            case "${CMD_STACK[1]}" in 
                syncfile)   create_sync_file  ;;
                *)          show_help ;;
            esac
            ;;
        *) [[ $helper -ne 1 ]] && show_help ;;
    esac
}

# ====================== 核心函数：参数解析 ======================
parse_args() {
    while [[ $# -gt 0 ]]; do
        local arg="$1"
        case "$arg" in
            # 公共短参数：-v -d -h -n
            -v) GLOBAL_OPTS[v]=1; GLOBAL_OPTS[verbose]=1; PUBLIC_ARGS+=("$arg"); shift ;;
            -d) GLOBAL_OPTS[d]=1; GLOBAL_OPTS[debug]=1; PUBLIC_ARGS+=("$arg"); shift ;;
            -h) GLOBAL_OPTS[h]=1; GLOBAL_OPTS[help]=1; PUBLIC_ARGS+=("$arg"); shift ;;
            -m) GLOBAL_OPTS[mntroot]=$2; PUBLIC_ARGS+=("$arg" "$2"); shift 2 ;;
            -w) GLOBAL_OPTS[wwwroot]=$2; PUBLIC_ARGS+=("$arg" "$2"); shift 2 ;;
            -f) GLOBAL_OPTS[file]=$2; PUBLIC_ARGS+=("$arg" "$2"); shift 2 ;;
            -a) GLOBAL_OPTS[all]=$2; PUBLIC_ARGS+=("$arg" "$2"); shift 2 ;;
            -i) GLOBAL_OPTS[interface]=$2; PUBLIC_ARGS+=("$arg" "$2"); shift 2 ;;
            -n) parse_nodes "$2"; PUBLIC_ARGS+=("$arg" "$2"); shift 2 ;;

            # 公共长参数：--verbose --debug --help --nodes
            --verbose) GLOBAL_OPTS[verbose]=1; GLOBAL_OPTS[v]=1; PUBLIC_ARGS+=("$arg"); shift ;;
            --debug) GLOBAL_OPTS[debug]=1; GLOBAL_OPTS[d]=1; PUBLIC_ARGS+=("$arg"); shift ;;
            --help) GLOBAL_OPTS[help]=1; GLOBAL_OPTS[h]=1; PUBLIC_ARGS+=("$arg"); shift ;;
            --mntroot) GLOBAL_OPTS[mntroot]=$2; PUBLIC_ARGS+=("$arg" "$2"); shift 2 ;;
            --wwwroot) GLOBAL_OPTS[wwwroot]=$2; PUBLIC_ARGS+=("$arg" "$2"); shift 2 ;;
            --file) GLOBAL_OPTS[file]=$2; PUBLIC_ARGS+=("$arg" "$2"); shift 2 ;;
            --all) GLOBAL_OPTS[all]=$2; PUBLIC_ARGS+=("$arg" "$2"); shift 2 ;;
            --interface) GLOBAL_OPTS[interface]=$2; PUBLIC_ARGS+=("$arg" "$2"); shift 2 ;;
            --nodes) parse_nodes "$2"; PUBLIC_ARGS+=("$arg" "$2"); shift 2 ;;

            # -- 专属短参数（子命令私有）
            -*) PRIVATE_ARGS+=("$arg"); shift ;;
            # -- 专属长参数（子命令私有）
            --*) PRIVATE_ARGS+=("$arg"); shift ;;

            # 子命令（无 - / --）
            *) CMD_STACK+=("$arg"); shift ;;
        esac
    done
}

# ====================== 主入口 ======================
main() {
    # 配置文件
    config_file="htool.conf"

    # load system env
    load_os_envs

    # 启动时先加载 nodes 文件
    load_nodes_from_file

    # 默认使用文件中的节点
    GLOBAL_NODES=("${DEFAULT_NODES[@]}")

    # 获取基础信息存储变量
    if [[ ! -e $config_file ]]; then
        info "$DSEP"
        warn "配置文件不存在: $config_file "
        warn "首次使用此工具需要使用下面命令进行初始化操作:"
        warn "    sunhpc init -i eth1 -m /mnt/cdrom"
        info "$DSEP"
        echo ""
        helper=1
    fi

    [[ -e $config_file ]] && load_conf_envs

    parse_args "$@"
    dispatch
}

main "$@"