Merge pull request #3 from illinois-cs-coursework/aryan

MP DONE

MP1/client.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
 import socket
+import sys
 
 PORT = 5055
 ALL_IPS = [
@@ -15,19 +16,45 @@ ALL_IPS = [
     "172.22.156.6"
 ]
 
-def grep_all_ips(ips, cmd):
+def send_message(sock, message):
+    e_data = message.encode('utf-8')
+    #header for the message length
+    header = str(len(e_data)).zfill(10).encode('utf-8')
+    sock.sendall(header)
+    sock.sendall(e_data)
+
+def receive_exact(sock, num_bytes):
+    received_data = b""
+    while len(received_data) < num_bytes:
+        chunk = sock.recv(num_bytes - len(received_data))
+        if not chunk:
+            raise ConnectionError("connection closed before the full message arrived")
+        received_data += chunk
+    return received_data
+
+def receive_message(sock):
+    header = receive_exact(sock, 10)
+    if not header:
+        return None
+    message_length = int(header.decode('utf-8'))
+    received_data = receive_exact(sock, message_length)
+    return received_data.decode('utf-8')
+
+def grep_all_ips(ips, cmd, if_unit_test):
     match_count = []
 
     for i, ip in enumerate(ips):
         try:
             client_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
             client_socket.connect((ip, PORT))
-            # print(f"Connected to server at {ip}:{PORT}")
 
+            if if_unit_test:
+                full_cmd = cmd + f" ./machine.{i + 1}.test.log"
+            else:
                 full_cmd = cmd + f" ./machine.{i + 1}.log"
 
-            client_socket.sendall(full_cmd.encode())
+            send_message(client_socket, full_cmd)
-            data = client_socket.recv(65536).decode()
+            data = receive_message(client_socket)
 
             count = len(data.splitlines())
             match_count.append(count)
@@ -38,18 +65,28 @@ def grep_all_ips(ips, cmd):
 
         except Exception as e:
             print(f"Failed to connect to {ip}:{PORT} - {e}")
+            match_count.append(-1)
         finally:
             client_socket.close()
     
     return match_count
 
-while True:
+#main
-    cmd = input(">> ").strip()
+#if unit test flag exists, then use the test log files instead of the actual log files
-    match_count = grep_all_ips(ALL_IPS, cmd)
+cmd = "grep " + " ".join([arg for arg in sys.argv[1:] if arg != "-t"])
+match_count = grep_all_ips(ALL_IPS, cmd, if_unit_test=[x for x in sys.argv].count("-t") > 0)
 
 print("=========================== SUMMARY ===========================")
 for i, count in enumerate(match_count):
-        if count > 0:
+    if count == -1:
+        if [x for x in sys.argv].count("-t") > 0:
+            print(f"machine.{i + 1}.test.log: Connection failed")
+        else:   
+            print(f"machine.{i + 1}.log: Connection failed")
+    else:
+        if [x for x in sys.argv].count("-t") > 0:
+            print(f"machine.{i + 1}.test.log: {count} matches")
+        else:
             print(f"machine.{i + 1}.log: {count} matches")
-    print(f"Total matches: {sum(match_count)}")
+print(f"Total matches: {max(0, sum(match_count))}")
 print("===============================================================")

MP1/generate_log.py

@@ -1,8 +1,9 @@
 import random
+import sys
 
-machine_num = input("Enter the machine number: ")
+machine_num = int(sys.argv[1])
-machine_log_file = f"machine.{machine_num}.log"
+machine_log_file = f"machine.{machine_num}.test.log"
-num_lines = int(input("Enter the number of log lines to generate: "))
+num_lines = int(sys.argv[2])
 
 log_level = ["TRACE", "DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL", "AUDIT"]
 
@@ -20,7 +21,27 @@ log_messages = [
     "System update installed.",
 ]
 
+open(machine_log_file, "w").close()
+
 for i in range(num_lines):
-    log_entry = f"[{machine_num}-{random.randint(1000, 9999)}] [{log_level[random.randint(0, len(log_level) - 1)]}] {log_messages[random.randint(0, len(log_messages) - 1)]}\n"
+    #on all
-    with open(machine_log_file, "a") as file:
+    if i == 0:
-        file.write(log_entry)
+        log_entry = f"[{machine_num}-{i}] [INFO] TEST_ALL\n"
+    #unique     
+    elif i == 1 and machine_num in [2, 4, 6, 8, 10]:
+        log_entry = f"[{machine_num}-{i}] [DEBUG] TEST_UNIQUE\n"
+    #on some
+    elif i == 2 and machine_num in [1, 3, 5, 7, 9]:
+        log_entry = f"[{machine_num}-{i}] [ERROR] TEST_SOME\n"
+    
+    elif 3 <= i < 13:
+        log_entry = f"[{machine_num}-{i}] [INFO] TEST_MEDIUM\n"
+
+    else:
+        log_entry = f"[{machine_num}-{i}] [{random.choice(log_level)}] {random.choice(log_messages)}\n"
+    
+    with open(machine_log_file, "a") as f:
+        f.write(log_entry)
+    
+
+            

MP1/scripts/deploy_mp.sh

@@ -0,0 +1,9 @@
+#/usr/bin/env bash
+
+/Users/aryan/Documents/Github/fa26_cs425_.group-mp_.team-01/MP1/scripts/kill_mp.sh all
+
+for i in {1..10}; do
+    vpn ssh -n -f -o BatchMode=yes "server$i" "cd fa26_cs425_.group-mp_.team-01/MP1 && git pull origin aryan && ~/fa26_cs425_.group-mp_.team-01/MP1/scripts/install_cmd.sh"
+done
+
+/Users/aryan/Documents/Github/fa26_cs425_.group-mp_.team-01/MP1/scripts/run_mp.sh all

MP1/scripts/install_cmd.sh

@@ -1,6 +1,6 @@
-chmod +x ../client.py
+rm -rf "$HOME/bin/grepp"
-mkdir -p ~/bin
+mkdir -p "$HOME/bin"
-cp ../client.py ~/bin/grepp
+install -m 755 ~/fa26_cs425_.group-mp_.team-01/MP1/client.py "$HOME/bin/grepp"
-chmod +x ~/bin/grepp
+echo 'export PATH="$HOME/bin:$PATH"' >> "$HOME/.bashrc"
-export PATH="$HOME/bin:$PATH" > ~/.bashrc
+hash -r
-source ~/.bashrc
+source "$HOME/.bashrc"

MP1/scripts/kill_mp.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+if [[ "$1" == "all" ]]; then
+    for i in {1..10}; do
+        echo "Killing server$i..."
+        vpn ssh -o BatchMode=yes "server$i" "fuser -k -9 5055/tcp"
+        echo ""
+    done
+else
+    for i in "$@"; do
+        echo "Killing server$i..."
+        vpn ssh -o BatchMode=yes "server$i" "fuser -k -9 5055/tcp"
+        echo ""
+    done
+fi

MP1/scripts/run_mp.sh

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+
+CMD='cd fa26_cs425_.group-mp_.team-01/MP1 && nohup python3 server.py </dev/null >/dev/null 2>&1 &'
+
+if [[ "$#" -eq 0 ]]; then
+    echo "Need at least one server number as an argument."
+    exit 1
+fi
+
+if [[ "$1" == "all" ]]; then
+    TARGETS="1 2 3 4 5 6 7 8 9 10"
+else
+    TARGETS="$@"
+fi
+
+echo "Starting servers"
+
+for i in $TARGETS; do
+    vpn ssh -n -f -o BatchMode=yes "server$i" "$CMD"
+done
+
+sleep 2
+
+for i in $TARGETS; do
+    pid=$(vpn ssh -n -o BatchMode=yes "server$i" "pgrep -o -u \$USER -f '[p]ython3 server.py'")
+    
+    if [[ -n "$pid" ]]; then
+        pid_clean=$(echo "$pid" | tr '\n' ' ')
+        echo "Server$i is running with PID(s): $pid_clean"
+    else
+        echo "Server$i is not running."
+    fi
+done

MP1/server.py

@@ -2,6 +2,7 @@
 import socket
 import shlex
 import subprocess
+from pathlib import Path
 
 HOST = "0.0.0.0"
 PORT = 5055
@@ -11,28 +12,56 @@ server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
 server_socket.bind((HOST, PORT))
 server_socket.listen()
 
+SCRIPT_DIR = Path(__file__).resolve().parent
+
 print(f"Server listening on {HOST}:{PORT}")
 
+def send_message(sock, message):
+    e_data = message.encode('utf-8')
+    #header for the message length
+    header = str(len(e_data)).zfill(10).encode('utf-8')
+    sock.sendall(header)
+    sock.sendall(e_data)
+
+def receive_exact(sock, num_bytes):
+    received_data = b""
+    while len(received_data) < num_bytes:
+        chunk = sock.recv(num_bytes - len(received_data))
+        if not chunk:
+            raise ConnectionError("connection closed before the full message arrived")
+        received_data += chunk
+    return received_data
+
+def receive_message(sock):
+    header = receive_exact(sock, 10)
+    if not header:
+        return None
+    message_length = int(header.decode('utf-8'))
+    received_data = receive_exact(sock, message_length)
+    return received_data.decode('utf-8')
+
 while True:
     conn, addr = server_socket.accept()
     print(f"Connected by {addr}")
 
-    command = conn.recv(65536)
+    command = receive_message(conn)
-    # print("Received Grep Command:", command.decode())
+    if command is None:
+        conn.close()
+        continue
 
-    command_args = shlex.split(command.decode())
+    command_args = shlex.split(command)
 
     if not command_args or command_args[0] != "grep":
         response = "Invalid command. Please use the 'grep' command."
-        conn.sendall(response.encode())
+        send_message(conn, response)
         conn.close()
         continue
 
     command_args.insert(1, "-n")
 
-    result = subprocess.run(command_args, capture_output=True, text=True)
+    result = subprocess.run(command_args, cwd=SCRIPT_DIR, capture_output=True, text=True)
    
-    conn.sendall(result.stdout.encode())
+    send_message(conn, result.stdout + result.stderr)
 
     conn.close()
 

MP1/test.py

@@ -0,0 +1,315 @@
+import subprocess
+import random
+import sys
+
+PORT = 5055
+ALL_IPS = [
+    "172.22.156.3",
+    "172.22.158.2",
+    "172.22.94.2",
+    "172.22.156.4",
+    "172.22.158.3",
+    "172.22.94.3",
+    "172.22.156.5",
+    "172.22.158.4",
+    "172.22.94.4",
+    "172.22.156.6"
+]
+
+tests = ["TEST_ALL", "TEST_UNIQUE", "TEST_SOME", "TEST_MEDIUM"]
+results = ["""Filename: machine.1.log
+Matches: 1
+1:[1-0] [INFO] TEST_ALL
+
+Filename: machine.2.log
+Matches: 1
+1:[2-0] [INFO] TEST_ALL
+
+Filename: machine.3.log
+Matches: 1
+1:[3-0] [INFO] TEST_ALL
+
+Filename: machine.4.log
+Matches: 1
+1:[4-0] [INFO] TEST_ALL
+
+Filename: machine.5.log
+Matches: 1
+1:[5-0] [INFO] TEST_ALL
+
+Filename: machine.6.log
+Matches: 1
+1:[6-0] [INFO] TEST_ALL
+
+Filename: machine.7.log
+Matches: 1
+1:[7-0] [INFO] TEST_ALL
+
+Filename: machine.8.log
+Matches: 1
+1:[8-0] [INFO] TEST_ALL
+
+Filename: machine.9.log
+Matches: 1
+1:[9-0] [INFO] TEST_ALL
+
+Filename: machine.10.log
+Matches: 1
+1:[10-0] [INFO] TEST_ALL
+
+=========================== SUMMARY ===========================
+machine.1.test.log: 1 matches
+machine.2.test.log: 1 matches
+machine.3.test.log: 1 matches
+machine.4.test.log: 1 matches
+machine.5.test.log: 1 matches
+machine.6.test.log: 1 matches
+machine.7.test.log: 1 matches
+machine.8.test.log: 1 matches
+machine.9.test.log: 1 matches
+machine.10.test.log: 1 matches
+Total matches: 10
+===============================================================""", 
+"""
+Filename: machine.2.log
+Matches: 1
+2:[2-1] [DEBUG] TEST_UNIQUE
+
+Filename: machine.4.log
+Matches: 1
+2:[4-1] [DEBUG] TEST_UNIQUE
+
+Filename: machine.6.log
+Matches: 1
+2:[6-1] [DEBUG] TEST_UNIQUE
+
+Filename: machine.8.log
+Matches: 1
+2:[8-1] [DEBUG] TEST_UNIQUE
+
+Filename: machine.10.log
+Matches: 1
+2:[10-1] [DEBUG] TEST_UNIQUE
+
+=========================== SUMMARY ===========================
+machine.1.test.log: 0 matches
+machine.2.test.log: 1 matches
+machine.3.test.log: 0 matches
+machine.4.test.log: 1 matches
+machine.5.test.log: 0 matches
+machine.6.test.log: 1 matches
+machine.7.test.log: 0 matches
+machine.8.test.log: 1 matches
+machine.9.test.log: 0 matches
+machine.10.test.log: 1 matches
+Total matches: 5
+===============================================================
+""",
+"""
+Filename: machine.1.log
+Matches: 1
+3:[1-2] [ERROR] TEST_SOME
+
+Filename: machine.3.log
+Matches: 1
+3:[3-2] [ERROR] TEST_SOME
+
+Filename: machine.5.log
+Matches: 1
+3:[5-2] [ERROR] TEST_SOME
+
+Filename: machine.7.log
+Matches: 1
+3:[7-2] [ERROR] TEST_SOME
+
+Filename: machine.9.log
+Matches: 1
+3:[9-2] [ERROR] TEST_SOME
+
+=========================== SUMMARY ===========================
+machine.1.test.log: 1 matches
+machine.2.test.log: 0 matches
+machine.3.test.log: 1 matches
+machine.4.test.log: 0 matches
+machine.5.test.log: 1 matches
+machine.6.test.log: 0 matches
+machine.7.test.log: 1 matches
+machine.8.test.log: 0 matches
+machine.9.test.log: 1 matches
+machine.10.test.log: 0 matches
+Total matches: 5
+===============================================================
+""", 
+"""
+Filename: machine.1.log
+Matches: 10
+4:[1-3] [INFO] TEST_MEDIUM
+5:[1-4] [INFO] TEST_MEDIUM
+6:[1-5] [INFO] TEST_MEDIUM
+7:[1-6] [INFO] TEST_MEDIUM
+8:[1-7] [INFO] TEST_MEDIUM
+9:[1-8] [INFO] TEST_MEDIUM
+10:[1-9] [INFO] TEST_MEDIUM
+11:[1-10] [INFO] TEST_MEDIUM
+12:[1-11] [INFO] TEST_MEDIUM
+13:[1-12] [INFO] TEST_MEDIUM
+
+Filename: machine.2.log
+Matches: 10
+4:[2-3] [INFO] TEST_MEDIUM
+5:[2-4] [INFO] TEST_MEDIUM
+6:[2-5] [INFO] TEST_MEDIUM
+7:[2-6] [INFO] TEST_MEDIUM
+8:[2-7] [INFO] TEST_MEDIUM
+9:[2-8] [INFO] TEST_MEDIUM
+10:[2-9] [INFO] TEST_MEDIUM
+11:[2-10] [INFO] TEST_MEDIUM
+12:[2-11] [INFO] TEST_MEDIUM
+13:[2-12] [INFO] TEST_MEDIUM
+
+Filename: machine.3.log
+Matches: 10
+4:[3-3] [INFO] TEST_MEDIUM
+5:[3-4] [INFO] TEST_MEDIUM
+6:[3-5] [INFO] TEST_MEDIUM
+7:[3-6] [INFO] TEST_MEDIUM
+8:[3-7] [INFO] TEST_MEDIUM
+9:[3-8] [INFO] TEST_MEDIUM
+10:[3-9] [INFO] TEST_MEDIUM
+11:[3-10] [INFO] TEST_MEDIUM
+12:[3-11] [INFO] TEST_MEDIUM
+13:[3-12] [INFO] TEST_MEDIUM
+
+Filename: machine.4.log
+Matches: 10
+4:[4-3] [INFO] TEST_MEDIUM
+5:[4-4] [INFO] TEST_MEDIUM
+6:[4-5] [INFO] TEST_MEDIUM
+7:[4-6] [INFO] TEST_MEDIUM
+8:[4-7] [INFO] TEST_MEDIUM
+9:[4-8] [INFO] TEST_MEDIUM
+10:[4-9] [INFO] TEST_MEDIUM
+11:[4-10] [INFO] TEST_MEDIUM
+12:[4-11] [INFO] TEST_MEDIUM
+13:[4-12] [INFO] TEST_MEDIUM
+
+Filename: machine.5.log
+Matches: 10
+4:[5-3] [INFO] TEST_MEDIUM
+5:[5-4] [INFO] TEST_MEDIUM
+6:[5-5] [INFO] TEST_MEDIUM
+7:[5-6] [INFO] TEST_MEDIUM
+8:[5-7] [INFO] TEST_MEDIUM
+9:[5-8] [INFO] TEST_MEDIUM
+10:[5-9] [INFO] TEST_MEDIUM
+11:[5-10] [INFO] TEST_MEDIUM
+12:[5-11] [INFO] TEST_MEDIUM
+13:[5-12] [INFO] TEST_MEDIUM
+
+Filename: machine.6.log
+Matches: 10
+4:[6-3] [INFO] TEST_MEDIUM
+5:[6-4] [INFO] TEST_MEDIUM
+6:[6-5] [INFO] TEST_MEDIUM
+7:[6-6] [INFO] TEST_MEDIUM
+8:[6-7] [INFO] TEST_MEDIUM
+9:[6-8] [INFO] TEST_MEDIUM
+10:[6-9] [INFO] TEST_MEDIUM
+11:[6-10] [INFO] TEST_MEDIUM
+12:[6-11] [INFO] TEST_MEDIUM
+13:[6-12] [INFO] TEST_MEDIUM
+
+Filename: machine.7.log
+Matches: 10
+4:[7-3] [INFO] TEST_MEDIUM
+5:[7-4] [INFO] TEST_MEDIUM
+6:[7-5] [INFO] TEST_MEDIUM
+7:[7-6] [INFO] TEST_MEDIUM
+8:[7-7] [INFO] TEST_MEDIUM
+9:[7-8] [INFO] TEST_MEDIUM
+10:[7-9] [INFO] TEST_MEDIUM
+11:[7-10] [INFO] TEST_MEDIUM
+12:[7-11] [INFO] TEST_MEDIUM
+13:[7-12] [INFO] TEST_MEDIUM
+
+Filename: machine.8.log
+Matches: 10
+4:[8-3] [INFO] TEST_MEDIUM
+5:[8-4] [INFO] TEST_MEDIUM
+6:[8-5] [INFO] TEST_MEDIUM
+7:[8-6] [INFO] TEST_MEDIUM
+8:[8-7] [INFO] TEST_MEDIUM
+9:[8-8] [INFO] TEST_MEDIUM
+10:[8-9] [INFO] TEST_MEDIUM
+11:[8-10] [INFO] TEST_MEDIUM
+12:[8-11] [INFO] TEST_MEDIUM
+13:[8-12] [INFO] TEST_MEDIUM
+
+Filename: machine.9.log
+Matches: 10
+4:[9-3] [INFO] TEST_MEDIUM
+5:[9-4] [INFO] TEST_MEDIUM
+6:[9-5] [INFO] TEST_MEDIUM
+7:[9-6] [INFO] TEST_MEDIUM
+8:[9-7] [INFO] TEST_MEDIUM
+9:[9-8] [INFO] TEST_MEDIUM
+10:[9-9] [INFO] TEST_MEDIUM
+11:[9-10] [INFO] TEST_MEDIUM
+12:[9-11] [INFO] TEST_MEDIUM
+13:[9-12] [INFO] TEST_MEDIUM
+
+Filename: machine.10.log
+Matches: 10
+4:[10-3] [INFO] TEST_MEDIUM
+5:[10-4] [INFO] TEST_MEDIUM
+6:[10-5] [INFO] TEST_MEDIUM
+7:[10-6] [INFO] TEST_MEDIUM
+8:[10-7] [INFO] TEST_MEDIUM
+9:[10-8] [INFO] TEST_MEDIUM
+10:[10-9] [INFO] TEST_MEDIUM
+11:[10-10] [INFO] TEST_MEDIUM
+12:[10-11] [INFO] TEST_MEDIUM
+13:[10-12] [INFO] TEST_MEDIUM
+
+=========================== SUMMARY ===========================
+machine.1.test.log: 10 matches
+machine.2.test.log: 10 matches
+machine.3.test.log: 10 matches
+machine.4.test.log: 10 matches
+machine.5.test.log: 10 matches
+machine.6.test.log: 10 matches
+machine.7.test.log: 10 matches
+machine.8.test.log: 10 matches
+machine.9.test.log: 10 matches
+machine.10.test.log: 10 matches
+Total matches: 100
+===============================================================
+"""]
+
+
+#generate log files on all the machines before running this test
+print("Generating log files on all machines...")
+for i, ip in enumerate(ALL_IPS):
+    subprocess.run(["vpn", "ssh", f"asing271@{ip}", f"cd ~/fa26_cs425_.group-mp_.team-01/MP1 &&  python3 generate_log.py {i + 1} 100"])
+
+#start the server on all the machines before running this test
+print("Starting server on all machines...")
+for i, ip in enumerate(ALL_IPS):
+    print(f"Starting server on machine {i + 1} ({ip})...")
+    subprocess.run([
+    "vpn", "ssh", f"asing271@{ip}", "cd ~/fa26_cs425_.group-mp_.team-01/MP1 && setsid -f python3 server.py > server.log 2>&1 < /dev/null"])
+
+#ssh into one machine at randome
+machine_num = random.randint(1, len(ALL_IPS))
+print(f"Running grep command on a {machine_num} random machine...")
+for i in tests:
+    result = subprocess.run(["vpn", "ssh", f"asing271@{ALL_IPS[machine_num - 1]}", f"cd ~/fa26_cs425_.group-mp_.team-01/MP1 &&  grepp {i} -t"], capture_output=True, text=True)
+    if result.stdout.strip() == results[tests.index(i)].strip():
+        print("PASS {i} test")
+    else:
+        print("FAIL {i} test")
+
+#delete log files on all the machines after running this test
+print("Deleting log files on all machines...")
+for i, ip in enumerate(ALL_IPS):
+    subprocess.run(["vpn", "ssh", f"asing271@{ip}", "cd ~/fa26_cs425_.group-mp_.team-01/MP1 &&  rm -f ./machine.*.test.log"])