testing differnt configs + scripts

2026-09-27 20:39:37 -05:00
1 parent 36cc7825b3
commit f806d8ea95
@@ -1 +0,0 @@
-scripts
@@ -31,13 +31,13 @@ GLOBAL_PORT = 8989
 MY_IP = socket.gethostbyname(socket.gethostname())

 HEARTBEAT_INTERVAL = 0.25
-GOSSIP_INTERVAL = 0.35
+GOSSIP_INTERVAL = 0.25
 JOIN_RETRY_INTERVAL = 1.0
 GOSSIP_FANOUT = 2

-GOSSIP_FAILURE_TIMEOUT = 2.8
-GOSSIP_S_SUSPECT_TIMEOUT = 1.2
-GOSSIP_S_TIMEOUT_SUSPECT = 1.6
+GOSSIP_FAILURE_TIMEOUT = 3
+GOSSIP_S_SUSPECT_TIMEOUT = 1.5
+GOSSIP_S_TIMEOUT_SUSPECT = 1.5
 GOSSIP_FAILURE_TIMEOUT_CLEANUP = 10.0


@@ -0,0 +1,405 @@
+#!/usr/bin/env bash
+
+SESSION="cs425-debug"
+CMDS_WINDOW="cmds"
+
+NUM_SERVERS=10
+
+PORT=8989
+
+WORKDIR="fa26_cs425_.group-mp_.team-01/MP2"
+SCRIPT="peer.py"
+
+SSH_OPTS="-o BatchMode=yes \
+-o ConnectTimeout=2 \
+-o ServerAliveInterval=1 \
+-o ServerAliveCountMax=1"
+
+
+# HELPERS
+
+
+expand_targets() {
+
+    if [ "$1" = "all" ]; then
+        seq 1 "$NUM_SERVERS"
+    else
+        echo "$@"
+    fi
+}
+
+
+get_pane() {
+
+    local vm="$1"
+
+    # VM1  -> pane 0
+    # VM2  -> pane 1
+
+    echo "$SESSION:$CMDS_WINDOW.$((vm - 1))"
+}
+
+
+is_running() {
+
+    local vm="$1"
+
+    ssh $SSH_OPTS "server$vm" \
+        "pgrep -f '[p]ython3.*$SCRIPT' >/dev/null" \
+        2>/dev/null
+}
+
+
+
+# START
+
+
+start_vm() {
+
+    local vm="$1"
+    local pane
+
+    pane=$(get_pane "$vm")
+
+    if ! tmux has-session \
+        -t "$SESSION" \
+        2>/dev/null
+    then
+        echo "VM$vm: tmux session not running"
+        return 1
+    fi
+
+    echo "starting VM$vm..."
+
+    # Kill old process on remote VM
+
+    ssh $SSH_OPTS "server$vm" \
+        "fuser -k -9 $PORT/udp 2>/dev/null || true; \
+         fuser -k -9 $PORT/tcp 2>/dev/null || true; \
+         pkill -9 -f '[p]ython3.*$SCRIPT' 2>/dev/null || true" \
+        >/dev/null 2>&1
+
+    sleep 0.3
+
+    # Reset pane
+
+    tmux send-keys \
+        -t "$pane" \
+        C-c
+
+    sleep 0.05
+
+    tmux send-keys \
+        -t "$pane" \
+        "clear" \
+        C-m
+
+    # Start peer.py
+
+    tmux send-keys \
+        -t "$pane" \
+        "ssh -t $SSH_OPTS server$vm 'cd \$HOME/$WORKDIR && MACHINE_INDEX=$vm python3 $SCRIPT'" \
+        C-m
+}
+
+
+
+# KILL
+
+
+kill_vm() {
+
+    local vm="$1"
+
+    echo "killing VM$vm..."
+
+    ssh $SSH_OPTS "server$vm" \
+        "fuser -k -9 $PORT/udp 2>/dev/null || true; \
+         fuser -k -9 $PORT/tcp 2>/dev/null || true; \
+         pkill -9 -f '[p]ython3.*$SCRIPT' 2>/dev/null || true" \
+        >/dev/null 2>&1
+}
+
+
+
+# COMMAND
+
+
+cmd_to_vm() {
+
+    local vm="$1"
+    shift
+
+    local command="$*"
+    local pane
+    local pane_command
+
+    pane=$(get_pane "$vm")
+
+    if ! is_running "$vm"; then
+        echo "VM$vm skipped (STOPPED)"
+        return
+    fi
+
+    pane_command=$(
+        tmux display-message \
+            -p \
+            -t "$pane" \
+            '#{pane_current_command}' \
+            2>/dev/null
+    )
+
+    if [ "$pane_command" != "ssh" ]; then
+        echo "VM$vm skipped (pane not attached)"
+        return
+    fi
+
+    echo "VM$vm <- $command"
+
+    tmux send-keys \
+        -t "$pane" \
+        "$command" \
+        C-m
+}
+
+
+
+# LIST
+
+
+list_vm() {
+
+    local vm="$1"
+
+    echo
+    echo "========== VM$vm =========="
+
+    ssh $SSH_OPTS "server$vm" \
+        "cat \$HOME/$WORKDIR/membership_list.txt" \
+        2>/dev/null
+}
+
+
+
+# LOG
+
+
+log_vm() {
+
+    local vm="$1"
+
+    echo
+    echo "========== VM$vm =========="
+
+    ssh $SSH_OPTS "server$vm" \
+        "tail -n 30 \$HOME/$WORKDIR/machine.$vm.log" \
+        2>/dev/null
+}
+
+
+
+# STATUS
+
+
+status_vm() {
+
+    local vm="$1"
+    local pid
+
+    pid=$(
+        ssh $SSH_OPTS "server$vm" \
+            "pgrep -f '[p]ython3.*$SCRIPT' | tr '\n' ' '" \
+            2>/dev/null
+    )
+
+    if [ -n "$pid" ]; then
+        echo "VM$vm RUNNING, PID(s): $pid"
+    else
+        echo "VM$vm STOPPED"
+    fi
+}
+
+
+
+# MAIN
+
+
+action="$1"
+shift || true
+
+case "$action" in
+
+
+# START
+
+
+start)
+
+    if [ "$#" -eq 0 ]; then
+        echo "Usage: ./cmd.sh start <all|N...>"
+        exit 1
+    fi
+
+    targets=$(expand_targets "$@")
+
+    for vm in $targets; do
+        start_vm "$vm" &
+    done
+
+    wait
+
+    sleep 0.5
+
+    for vm in $targets; do
+        status_vm "$vm" &
+    done
+
+    wait
+    ;;
+
+
+
+# KILL
+
+
+kill)
+
+    if [ "$#" -eq 0 ]; then
+        echo "Usage: ./cmd.sh kill <all|N...>"
+        exit 1
+    fi
+
+    targets=$(expand_targets "$@")
+
+    for vm in $targets; do
+        kill_vm "$vm" &
+    done
+
+    wait
+    ;;
+
+
+
+# CMD
+
+
+cmd)
+
+    target="$1"
+    shift || true
+
+    if [ -z "$target" ] || [ "$#" -eq 0 ]; then
+        echo "Usage:"
+        echo "./cmd.sh cmd <all|N> \"command\""
+        exit 1
+    fi
+
+    if [ "$target" = "all" ]; then
+
+        for vm in $(seq 1 "$NUM_SERVERS"); do
+            cmd_to_vm "$vm" "$@" &
+        done
+
+        wait
+
+    else
+
+        cmd_to_vm "$target" "$@"
+
+    fi
+    ;;
+
+
+
+# LIST
+
+
+list)
+
+    if [ "$#" -eq 0 ]; then
+        targets=$(seq 1 "$NUM_SERVERS")
+    else
+        targets=$(expand_targets "$@")
+    fi
+
+    for vm in $targets; do
+        list_vm "$vm" &
+    done
+
+    wait
+    ;;
+
+
+
+# LOG
+
+
+log)
+
+    if [ "$#" -eq 0 ]; then
+        targets=$(seq 1 "$NUM_SERVERS")
+    else
+        targets=$(expand_targets "$@")
+    fi
+
+    for vm in $targets; do
+        log_vm "$vm" &
+    done
+
+    wait
+    ;;
+
+
+
+# STATUS
+
+
+status)
+
+    if [ "$#" -eq 0 ]; then
+        targets=$(seq 1 "$NUM_SERVERS")
+    else
+        targets=$(expand_targets "$@")
+    fi
+
+    for vm in $targets; do
+        status_vm "$vm" &
+    done
+
+    wait
+    ;;
+
+
+
+# HELP
+
+
+*)
+
+    echo
+    echo "Usage:"
+    echo
+    echo "  ./cmd.sh start  <all|N...>"
+    echo "  ./cmd.sh kill   <all|N...>"
+    echo "  ./cmd.sh cmd    <all|N> \"command\""
+    echo "  ./cmd.sh list   [all|N...]"
+    echo "  ./cmd.sh log    [all|N...]"
+    echo "  ./cmd.sh status [all|N...]"
+    echo
+    echo "Examples:"
+    echo
+    echo "  ./cmd.sh start 1"
+    echo "  ./cmd.sh start 2 3 4 5"
+    echo "  ./cmd.sh start 6 7 8 9 10"
+    echo
+    echo "  ./cmd.sh cmd all \"switch suspect\""
+    echo "  ./cmd.sh cmd all \"switch nosuspect\""
+    echo "  ./cmd.sh cmd all \"set_drop_rate 10\""
+    echo "  ./cmd.sh cmd all \"display_protocol\""
+    echo
+    echo "  ./cmd.sh kill 8 9"
+    echo "  ./cmd.sh log 8 9"
+    echo
+    exit 1
+    ;;
+esac
@@ -0,0 +1,254 @@
+#!/usr/bin/env python3
+
+import sys
+import ast
+import time
+
+
+IP_TO_VM = {
+    "172.22.156.3": "1",
+    "172.22.158.2": "2",
+    "172.22.94.2":  "3",
+    "172.22.156.4": "4",
+    "172.22.158.3": "5",
+    "172.22.94.3":  "6",
+    "172.22.156.5": "7",
+    "172.22.158.4": "8",
+    "172.22.94.4":  "9",
+    "172.22.156.6": "10",
+}
+
+
+
+# COLORS
+
+
+GREEN  = "\033[92m"
+YELLOW = "\033[93m"
+RED    = "\033[91m"
+CYAN   = "\033[96m"
+WHITE  = "\033[97m"
+GRAY   = "\033[90m"
+
+BOLD  = "\033[1m"
+RESET = "\033[0m"
+
+CLEAR_LINE = "\033[K"
+
+
+STATE_STYLE = {
+    "ALIVE":   (GREEN, "A"),
+    "SUSPECT": (YELLOW, "S"),
+    "FAILED":  (RED, "F"),
+}
+
+
+def render(lines):
+
+   
+    # STOPPED / SSH ERROR
+   
+
+    for line in lines:
+        if line.startswith("**STOPPED**"):
+
+            reason = line.replace(
+                "**STOPPED**",
+                ""
+            ).strip()
+
+            return (
+                f"{CLEAR_LINE}"
+                f"{BOLD}{RED}>>> STOPPED <<<{RESET}"
+                f"{CLEAR_LINE}\n"
+                f"{GRAY}{reason}{RESET}"
+                f"{CLEAR_LINE}\n"
+            )
+
+   
+    # PARSE
+   
+
+    members = []
+
+    mode = "?"
+    drop_rate = "?"
+
+    now = time.time()
+
+    for raw_line in lines:
+
+        line = raw_line.strip()
+
+        if not line:
+            continue
+
+        # ----------------------------------------------------
+        # MODE / DROP RATE LINE
+        # ----------------------------------------------------
+
+        if line.startswith("Mode:"):
+
+            try:
+                pieces = line.split(",")
+
+                mode = pieces[0].split(":", 1)[1].strip()
+                drop_rate = pieces[1].split(":", 1)[1].strip()
+
+            except Exception:
+                pass
+
+            continue
+
+        # ----------------------------------------------------
+        # MEMBERSHIP ENTRY
+        # ----------------------------------------------------
+
+        try:
+
+            id_text, data_text = line.split(": ", 1)
+
+            member_id = ast.literal_eval(id_text)
+            member_data = ast.literal_eval(data_text)
+
+            ip, port, timestamp = member_id
+
+            local_last_seen = member_data.get(
+                "local_last_seen",
+                now
+            )
+
+            members.append({
+                "vm": IP_TO_VM.get(ip, "?"),
+
+                "hb": member_data.get(
+                    "heart_beat",
+                    0
+                ),
+
+                "state": member_data.get(
+                    "state",
+                    "?"
+                ).upper(),
+
+                "age": max(
+                    0.0,
+                    now - local_last_seen
+                ),
+
+                "inc": member_data.get(
+                    "incarnation_counter",
+                    0
+                ),
+            })
+
+        except Exception:
+            continue
+
+   
+    # HEADER
+   
+
+    out = []
+
+    mode_upper = str(mode).upper()
+
+    if mode_upper == "SUSPECT":
+        mode_color = YELLOW
+    elif mode_upper == "NOSUSPECT":
+        mode_color = GREEN
+    else:
+        mode_color = GRAY
+
+    out.append(
+        f"{BOLD}{CYAN}MODE:{RESET} "
+        f"{mode_color}{BOLD}{mode_upper}{RESET} "
+        f"{GRAY}|{RESET} "
+        f"{BOLD}{CYAN}DROP:{RESET} "
+        f"{WHITE}{drop_rate}%{RESET}"
+        f"{CLEAR_LINE}"
+    )
+
+   
+    # NO DATA
+   
+
+    if not members:
+
+        out.append(
+            f"{BOLD}{YELLOW}"
+            f"waiting for membership..."
+            f"{RESET}"
+            f"{CLEAR_LINE}"
+        )
+
+        return "\n".join(out) + "\n"
+
+   
+    # SORT
+   
+
+    def vm_number(member):
+
+        try:
+            return int(member["vm"])
+
+        except Exception:
+            return 999
+
+    members.sort(
+        key=vm_number
+    )
+
+   
+    # TABLE HEADER
+   
+
+    out.append(
+        f"{BOLD}{CYAN}"
+        f"{'VM':<3} "
+        f"{'S':<2} "
+        f"{'HB':<6} "
+        f"{'AGE':<5} "
+        f"{'INC'}"
+        f"{RESET}"
+        f"{CLEAR_LINE}"
+    )
+
+   
+    # MEMBERS
+   
+
+    for member in members:
+
+        color, code = STATE_STYLE.get(
+            member["state"],
+            (GRAY, "?")
+        )
+
+        out.append(
+            f"{WHITE}"
+            f"{member['vm']:<3}"
+            f"{RESET} "
+
+            f"{color}{BOLD}"
+            f"{code:<2}"
+            f"{RESET} "
+
+            f"{member['hb']:<6} "
+
+            f"{member['age']:>4.1f} "
+
+            f"{member['inc']}"
+
+            f"{CLEAR_LINE}"
+        )
+
+    return "\n".join(out) + "\n"
+
+
+print(
+    render(list(sys.stdin)),
+    end="",
+    flush=True
+)
@@ -0,0 +1,147 @@
+#!/usr/bin/env bash
+
+SESSION="cs425-debug"
+NUM_SERVERS=10
+
+WORKDIR="fa26_cs425_.group-mp_.team-01/MP2"
+FORMATTER="$(cd "$(dirname "$0")" && pwd)/pretty_membership.py"
+
+SSH_OPTS="-o ConnectTimeout=2 \
+-o ServerAliveInterval=1 \
+-o ServerAliveCountMax=1 \
+-o BatchMode=yes"
+
+
+# RESET TMUX
+
+
+tmux kill-session -t "$SESSION" 2>/dev/null || true
+
+tmux new-session -d -s "$SESSION" -n "display"
+
+tmux set-option -t "$SESSION" pane-border-status top
+tmux set-option -t "$SESSION" pane-border-format " #{pane_title} "
+
+
+# DISPLAY COMMAND
+
+
+make_display_cmd() {
+    local vm="$1"
+
+    cat <<EOF
+clear; while true; do
+    printf '\033[H'
+
+    {
+        ssh $SSH_OPTS server$vm \
+        'FILE="\$HOME/$WORKDIR/membership_list.txt";
+         if [ ! -f "\$FILE" ]; then
+             echo "**STOPPED** no membership file";
+         else
+             NOW=\$(date +%s);
+             MTIME=\$(stat -c %Y "\$FILE" 2>/dev/null || echo 0);
+             AGE=\$((NOW - MTIME));
+
+             if [ "\$AGE" -gt 3 ]; then
+                 echo "**STOPPED** \${AGE}s since last update";
+             else
+                 cat "\$FILE";
+             fi;
+         fi' 2>&1 \
+        || echo "**STOPPED** SSH unavailable"
+    } | python3 "$FORMATTER"
+
+    printf '\033[J'
+    sleep 0.75
+done
+EOF
+}
+
+
+# DISPLAY WINDOW
+
+
+tmux select-pane \
+    -t "$SESSION:display.0" \
+    -T "VM1"
+
+tmux send-keys \
+    -t "$SESSION:display.0" \
+    "$(make_display_cmd 1)" \
+    C-m
+
+for i in $(seq 2 "$NUM_SERVERS"); do
+
+    PANE=$(
+        tmux split-window \
+            -t "$SESSION:display" \
+            -P \
+            -F '#{pane_id}'
+    )
+
+    tmux select-layout \
+        -t "$SESSION:display" \
+        tiled
+
+    tmux select-pane \
+        -t "$PANE" \
+        -T "VM$i"
+
+    tmux send-keys \
+        -t "$PANE" \
+        "$(make_display_cmd "$i")" \
+        C-m
+done
+
+tmux select-layout \
+    -t "$SESSION:display" \
+    tiled
+
+
+# COMMAND WINDOW
+
+
+tmux new-window \
+    -t "$SESSION" \
+    -n "cmds"
+
+tmux set-window-option \
+    -t "$SESSION:cmds" \
+    pane-base-index 0
+
+tmux select-pane \
+    -t "$SESSION:cmds.0" \
+    -T "VM1"
+
+for i in $(seq 2 "$NUM_SERVERS"); do
+
+    PANE=$(
+        tmux split-window \
+            -t "$SESSION:cmds" \
+            -P \
+            -F '#{pane_id}'
+    )
+
+    tmux select-layout \
+        -t "$SESSION:cmds" \
+        tiled
+
+    tmux select-pane \
+        -t "$PANE" \
+        -T "VM$i"
+done
+
+tmux select-layout \
+    -t "$SESSION:cmds" \
+    tiled
+
+
+# OPEN DISPLAY
+
+
+tmux select-window \
+    -t "$SESSION:display"
+
+tmux attach-session \
+    -t "$SESSION"