Update TODO list

- Also print a summary at the end.

README.txt

@@ -7,9 +7,29 @@
 Run commands across a set of hosts interactively!
 =================================================
 
-Basic configuration file is in multiball.cfg.example in the distribution. Customize to your liking. Host list is loaded from,
-by default, your ssh configuration (assuming you have a specific ssh configuration for your all hosts like Autonomic has). You
-can also use arbitrary host lists (documented in configuration file).
+Basic configuration file is in `multiball.cfg.example` in the distribution. Customize to your liking. 
 
+Host list is loaded from your ssh configuration by default (assuming you have a specific ssh configuration for your all hosts like Autonomic has). You can also use arbitrary host lists (documented in configuration file).
 
 Planned features, see __main__.py's comments.
+
+Installation
+-------------------------------------------------
+
+Install dependencies:
+
+* Debian: `sudo apt install python3 python3-venv git make`
+* Fedora: `sudo dnf install python3 python3-pip git make`
+
+Clone the repo:
+
+```
+git clone https://git.autonomic.zone/autonomic-cooperative/multiball.git
+```
+
+Install, including dependencies:
+
+```
+cd multiball
+make
+```

TODO.txt

@@ -0,0 +1,26 @@
+TODO
+- add local variable support that can be interpolated into commands
+- keep track of previous command outputs for last command, and any previous command with target list and command line
+  - Allow any given command to be tagged with a label, and the outputs would be stored in the output dictionary under that name (for scripting);
+    we assign a default name to it, and are able to refer to it based on either the label, or how many commands back it was;
+  - scripts entire output is stored as one label, but also
+  - add commands to show the list of captured outputs, along with the commands and targets
+  - add commands to search captured outputs
+  - add commands to search captured outputs and put them into a variable that can be used for /target etc.
+- calling commands and aliases from command line (e.g. multiball -t <targets> -c <command>)
+- tagging and filtering commands (so `noupgrade` gets a tag, and we can filter them)
+- saving the last host set in a .file
+- allow server groups / server labels
+- add 'watch' to run a command repeatedly until it succeeds (interruptable)
+- add the number of servers for each group of services too
+- notice when you don't sudo in front of command and it errors and ask you if you meant to sudo
+- autocomplete for environment variable names (requires heirarchical completer)
+- implement target aliases (-filteralias) which gives a label to an argument to -hosts
+- ad-hoc host groupings with assigned names, and a host grouping stack
+- allow scripts that use -safe to prompt for safety / restore safety after running
+- implement various commented commands in the command list
+- implement interactive alias system
+- Catch more exceptions in fabtools, and also add retries
+- Make the runner aware of multiple commands so that it can combine outputs and make 'overall success' or 'overall failure'
+- make C-c break the connections not the program
+- make a /summary command that *only* outputs the summary of a remote command, not the stdout/stderr

makefile

@@ -0,0 +1,21 @@
+default: install
+
+.venv:
+	python3 -m venv .venv
+
+pip: .venv
+	@.venv/bin/pip install -e .
+
+$(HOME)/.local/bin/:
+	@echo "'$(HOME)/.local/bin' was not found; creating it"
+	mkdir -p $(HOME)/.local/bin
+	@echo 'Be sure to add this directory to your $$PATH, using e.g.'
+	@echo "echo $(HOME)/.local/bin >> $(HOME)/.bashrc"
+
+$(HOME)/.local/bin/multiball: $(HOME)/.local/bin/ pip
+	ln -s $(PWD)/.venv/bin/multiball $(HOME)/.local/bin/multiball
+
+install: $(HOME)/.local/bin/multiball	
+
+.PHONY: pip install
+

multiball/__main__.py

@@ -3,27 +3,6 @@
 #
 
 #
-# TODO
-# - keep track of previous command outputs for last command, and any previous command with target list and command line
-# - calling commands and alieses from command line
-# - tagging and filtering commands (so #noupgrade gets a tag, and we can filter them)
-# - saving the last host set in a .file
-# - allow server groups
-# - allow grepping of output from a command, and then populating the target list from matchintg on it
-# - add 'watch' to run a command repeatedly until it succeeds (interruptable)
-# - add the number of servers for each group of services too
-# - notice when you don't sudo in front of command and it errors and ask you if you meant to sudo
-# - autocomplete for environment variable names (requires heirarchical completer)
-# - implement target aliases (-filteralias) which gives a label to an argument to -hosts
-# - ad-hoc host groupings with assigned names, and a host grouping stack
-# - Assign each output group to a number that can be easily selected with a -target etc. But also allow matching
-#   against those groups. Output groupings survive until the next command.
-# - allow scripts that use -safe to prompt for safety / restore safety after running
-# - Add variables that can be set and passed to commands
-# - implement various commented commands in the command list
-# - implement interactive alias system
-# - Catch more exceptions in fabtools, and also add retries
-
 
 import argparse
 import datetime

multiball/fabtools.py

@@ -1,22 +1,39 @@
+import socket
 import time
-
+from enum import Enum
 from pathlib import Path
-from threading import Thread, Lock
-
-from tqdm import tqdm
-
-from fabric2 import ThreadingGroup, SerialGroup, Config, Connection
+from threading import Lock, Thread
 
 import paramiko
+from fabric2 import Config, Connection, SerialGroup, ThreadingGroup
+from tqdm import tqdm
 
 
+class RunResult(Enum):
+    Success = 0
+    CommandFailure = 1
+    Error = 2
+    Cancelled = 3
+    Timeout = 4 # FIXME support host timeouts
+
 def thread_run(connection, command, result_lock, result_queue):
+    runresult = RunResult.Success
     try:
         res = connection.run(command, warn=True, hide=True)
-    except (paramiko.ssh_exception.NoValidConnectionsError, paramiko.ssh_exception.SSHException) as inst:
+        if (res.exited != 0):
+            runresult = RunResult.CommandFailure
+    except (paramiko.ssh_exception.NoValidConnectionsError, paramiko.ssh_exception.SSHException, socket.gaierror) as inst:
         res = f"Could not connect to host: {inst}"
+        runresult = RunResult.Error
+    except KeyboardInterrupt as inst:
+        res = f"Canceled host {inst} from C-c"
+        runresult = RunResult.Cancelled
+    except Exception as inst:
+        res = f"Unknown error for host: {inst}"
+        runresult = RunResult.Error
     with result_lock:
-        result_queue.append((connection, res))
+        result_queue.append((connection, res, runresult))
+
 
 # A set of hosts we can target with a series of commands and also can collect output of each command
 class HostSet:
@@ -58,24 +75,37 @@ class HostSet:
                 if ((time.time() - tupdate) > 10.0):
                     tupdate = time.time()
                     print("Still waiting on ", [thread.host for thread in threads])
+                    # FIXME have the thread killed if it's still waiting after $TIMEOUT
         prog.close()
 
         # Gather up results by output
         gathered = {}
-        for connection, result in resq:
+        success = []
+        error = []
+        commandfail = []
+
+        for connection, result, runresult in resq:
+            # print(connection, result, runresult)
             rstr = str(result)
             if not rstr in gathered:
                 gathered[rstr] = []
             gathered[rstr].append(connection)
+            if runresult in (RunResult.Error, RunResult.Cancelled):
+                error.append(connection)
+            elif runresult == RunResult.CommandFailure:
+                commandfail.append(connection)
+            else:
+                success.append(connection)
 
         # display results
         for result, connections in gathered.items():
             print('-----> [{}]'.format(' '.join(connection.original_host for connection in connections)))
             print(result)
-        # ## gather_output
-        # for connection in self.connections:
-        #     # import pdb
-        #     # pdb.set_trace()
-        #     res = connection.run(command)
-        #     print('---- {} ----'.format(connection.original_host))
-        #     print(res)
+
+        print('#####> RUN SUMMARY <#####')
+        if (success):
+            print('        Succeeded: ', ', '.join(connection.original_host for connection in success))
+        if (commandfail):
+            print('     Command fail: ', ', '.join(connection.original_host for connection in commandfail))
+        if (error):
+            print('Failed with Error: ', ','.join(connection.original_host for connection in error))