Add the start of output logging!

bumped to version 0.5.0.

- Fix some typos.
- Make fabtools HostSet save the command output to itself.
- Make the multiball runner save the most recent output.
- (command: /save) Add command to save most recent output to file.
- Add a debug command for development purposes.

README.txt

@@ -7,9 +7,29 @@
 Run commands across a set of hosts interactively!
 =================================================
 
-Basic configuration file is in multiball.cfg.example in the distribution. Customize to your liking. Host list is loaded from,
+Basic configuration file is in `multiball.cfg.example` in the distribution. Customize to your liking. 
-by default, your ssh configuration (assuming you have a specific ssh configuration for your all hosts like Autonomic has). You
-can also use arbitrary host lists (documented in configuration file).
 
+Host list is loaded from your ssh configuration by default (assuming you have a specific ssh configuration for your all hosts like Autonomic has). You can also use arbitrary host lists (documented in configuration file).
 
 Planned features, see __main__.py's comments.
+
+Installation
+-------------------------------------------------
+
+Install dependencies:
+
+* Debian: `sudo apt install python3 python3-venv git make`
+* Fedora: `sudo dnf install python3 python3-pip git make`
+
+Clone the repo:
+
+```
+git clone https://git.autonomic.zone/autonomic-cooperative/multiball.git
+```
+
+Install, including dependencies:
+
+```
+cd multiball
+make
+```

TODO.txt

@@ -0,0 +1,26 @@
+TODO
+- add local variable support that can be interpolated into commands
+- keep track of previous command outputs for last command, and any previous command with target list and command line
+  - Allow any given command to be tagged with a label, and the outputs would be stored in the output dictionary under that name (for scripting);
+    we assign a default name to it, and are able to refer to it based on either the label, or how many commands back it was;
+  - scripts entire output is stored as one label, but also
+  - add commands to show the list of captured outputs, along with the commands and targets
+  - add commands to search captured outputs
+  - add commands to search captured outputs and put them into a variable that can be used for /target etc.
+- calling commands and aliases from command line (e.g. multiball -t <targets> -c <command>)
+- tagging and filtering commands (so `noupgrade` gets a tag, and we can filter them)
+- saving the last host set in a .file
+- allow server groups / server labels
+- add 'watch' to run a command repeatedly until it succeeds (interruptable)
+- add the number of servers for each group of services too
+- notice when you don't sudo in front of command and it errors and ask you if you meant to sudo
+- autocomplete for environment variable names (requires heirarchical completer)
+- implement target aliases (-filteralias) which gives a label to an argument to -hosts
+- ad-hoc host groupings with assigned names, and a host grouping stack
+- allow scripts that use -safe to prompt for safety / restore safety after running
+- implement various commented commands in the command list
+- implement interactive alias system
+- Catch more exceptions in fabtools, and also add retries
+- Make the runner aware of multiple commands so that it can combine outputs and make 'overall success' or 'overall failure'
+- make C-c break the connections not the program
+- make a /summary command that *only* outputs the summary of a remote command, not the stdout/stderr

makefile

@@ -0,0 +1,21 @@
+default: install
+
+.venv:
+	python3 -m venv .venv
+
+pip: .venv
+	@.venv/bin/pip install -e .
+
+$(HOME)/.local/bin/:
+	@echo "'$(HOME)/.local/bin' was not found; creating it"
+	mkdir -p $(HOME)/.local/bin
+	@echo 'Be sure to add this directory to your $$PATH, using e.g.'
+	@echo "echo $(HOME)/.local/bin >> $(HOME)/.bashrc"
+
+$(HOME)/.local/bin/multiball: $(HOME)/.local/bin/ pip
+	ln -s $(PWD)/.venv/bin/multiball $(HOME)/.local/bin/multiball
+
+install: $(HOME)/.local/bin/multiball	
+
+.PHONY: pip install
+

multiball/__init__.py

@@ -1 +1 @@
-__version__ = "0.0.0"
+__version__ = "0.5.0"

multiball/__main__.py

@@ -3,27 +3,6 @@
 #
 
 #
-# TODO
-# - keep track of previous command outputs for last command, and any previous command with target list and command line
-# - calling commands and alieses from command line
-# - tagging and filtering commands (so #noupgrade gets a tag, and we can filter them)
-# - saving the last host set in a .file
-# - allow server groups
-# - allow grepping of output from a command, and then populating the target list from matchintg on it
-# - add 'watch' to run a command repeatedly until it succeeds (interruptable)
-# - add the number of servers for each group of services too
-# - notice when you don't sudo in front of command and it errors and ask you if you meant to sudo
-# - autocomplete for environment variable names (requires heirarchical completer)
-# - implement target aliases (-filteralias) which gives a label to an argument to -hosts
-# - ad-hoc host groupings with assigned names, and a host grouping stack
-# - Assign each output group to a number that can be easily selected with a -target etc. But also allow matching
-#   against those groups. Output groupings survive until the next command.
-# - allow scripts that use -safe to prompt for safety / restore safety after running
-# - Add variables that can be set and passed to commands
-# - implement various commented commands in the command list
-# - implement interactive alias system
-# - Catch more exceptions in fabtools, and also add retries
-
 
 import argparse
 import datetime
@@ -46,6 +25,7 @@ from prompt_toolkit.completion import WordCompleter
 from prompt_toolkit.history import InMemoryHistory, FileHistory
 from prompt_toolkit.shortcuts import PromptSession
 
+from . import __version__
 from .fabtools import HostSet
 from .tools import load_sshconfig_server_list, load_bare_server_list, load_config
 
@@ -85,17 +65,19 @@ COMMANDS = {
     'clearenv': (('/clearenv',), "Clear environment (entire or single variable)"),
     'safety': (('/safety', 'safety'), "Turn on safety or turn off with `off`."),
     'safe': (('/safe', ), "Run this command as if it were safe even if safety is on."),
-    'confirm': (('/confirm',), "[scriptpfing] Prompt for confirmation"),
+    'confirm': (('/confirm',), "[scripting] Prompt for confirmation"),
     'echo': (('/echo', ), "[scripting] print string"),
     'arguments': (('/arguments',), "[scripting] Abort if no arguments are specified."),
     # fixme add interpolation
-    'unsafe': (('/unsafe', ), "[scriptiong] Abort command if safety is on."),
+    'unsafe': (('/unsafe', ), "[scripting] Abort command if safety is on."),
+    'save': (('/save', ),"Save last run log to file."),
+    'debug': (('/////debug', ), "[devel] enter PDB"),
     # 'history': (('/history', '/h'), "Show the last 10 commands in the history."),
     # 'filteralias': (('/filteralias',), "Create a filter alias which allows quick filtering."),
     # 'undo': (('/undo',), "Undo last host or environment command."),
     # 'alias': (('/alias', ), "Create an alias interactively.")
     ## Fixme unambiguate the above commands which list things into this command so things are more consistent
-    # 'list': (('/list',), "List various things (environmet, allhosts, hosts, targets)")
+    # 'list': (('/list',), "List various things (environment, allhosts, hosts, targets)")
     # '!': (('!', '/local'), "Run a local command and capture its output.")
 }
 
@@ -111,6 +93,7 @@ class Multiball:
             'safety': self.command_safety,
             'safe': self.command_safe,
             'clearenv': self.command_clearenv,
+            'save': self.command_save,
         }
         configdict = load_config(Path("multiball.cfg"))
         self.allhosts: set = set()
@@ -118,6 +101,7 @@ class Multiball:
         self.environment = {}
         self.safety = True
         self.ssh_config = None
+        self.last_run = {}
 
         if config := configdict.get('config', {}):
             if 'history' in config:
@@ -257,6 +241,27 @@ class Multiball:
         print("cleared environment")
         self.environment = dict()
 
+    def command_save(self, command, args):
+        # FIXME: We need to make aliases store multiple entries, and save should take the whole alias not just
+        # the last command.
+        if (not self.last_run):
+            print("Nothing to log.")
+            return
+        tstamp = datetime.datetime.now().strftime("%Y-%m-%d-%H%M%Z")
+        outfilename = f"{tstamp}.multiball.log"
+        if (args):
+            outfilename = args[0]
+            if (not outfilename.endswith(".log")):
+                outfilename = outfilename + ".log"
+
+        print(f"Writing {outfilename} for command `{self.last_run['cmd']}`.")
+        with (open(outfilename, 'w')) as out:
+            out.writelines([f"command: {self.last_run['cmd']}\n",
+                            "-------------------------------------------\n"
+                            ""])
+            out.write(self.last_run['results'].getvalue())
+            out.write("\n")
+
     def _print_targetlist(self, tlist=None, keyword='Targeting'):
         if not tlist:
             tlist = self.targethosts
@@ -288,8 +293,11 @@ class Multiball:
 
         if abst == 'exit':
             raise MBExit
-        if abst == 'help':
+        elif abst == 'help':
             self._print_help()
+        elif abst == 'debug':
+            print("==== DEBUGGING ====")
+            __import__("pdb").set_trace()
         elif abst in self.handlers:
             self.handlers[abst](command[0], cargs)
         elif abst == 'clear':
@@ -363,6 +371,10 @@ class Multiball:
         remotecmd.append(command)
         h = HostSet(self.targethosts, self.ssh_config)
         h.run(';'.join(remotecmd))
+        self.last_run = {
+            'cmd': remotecmd,
+            'results': h.results
+        }
 
     def do_cmd(self, command):
         try:
@@ -397,7 +409,7 @@ class Multiball:
 
 
 def multiball(*args, **kwargs):
-    print("Welcome to Multiball. type /help for help.")
+    print(f"Welcome to Multiball {__version__}. type /help for help.")
     mb = Multiball()
     # loop on prompt
     while (True):

multiball/fabtools.py

@@ -1,26 +1,50 @@
+import io
+import socket
 import time
-
+from enum import Enum
 from pathlib import Path
-from threading import Thread, Lock
+from threading import Lock, Thread
-
-from tqdm import tqdm
-
-from fabric2 import ThreadingGroup, SerialGroup, Config, Connection
 
 import paramiko
+from fabric2 import Config, Connection, SerialGroup, ThreadingGroup
+from tqdm import tqdm
 
 
+class RunResult(Enum):
+    Success = 0
+    CommandFailure = 1
+    Error = 2
+    Cancelled = 3
+    Timeout = 4 # FIXME support host timeouts
+
 def thread_run(connection, command, result_lock, result_queue):
+    runresult = RunResult.Success
     try:
         res = connection.run(command, warn=True, hide=True)
-    except (paramiko.ssh_exception.NoValidConnectionsError, paramiko.ssh_exception.SSHException) as inst:
+        if (res.exited != 0):
+            runresult = RunResult.CommandFailure
+    except (paramiko.ssh_exception.NoValidConnectionsError, paramiko.ssh_exception.SSHException, socket.gaierror) as inst:
         res = f"Could not connect to host: {inst}"
+        runresult = RunResult.Error
+    except KeyboardInterrupt as inst:
+        res = f"Canceled host {inst} from C-c"
+        runresult = RunResult.Cancelled
+    except Exception as inst:
+        res = f"Unknown error for host: {inst}"
+        runresult = RunResult.Error
     with result_lock:
-        result_queue.append((connection, res))
+        result_queue.append((connection, res, runresult))
+
 
 # A set of hosts we can target with a series of commands and also can collect output of each command
 class HostSet:
+    def _lprint(self, *args):
+        s = " ".join([str(a) for a in args]) + '\n'
+        self.results.write(s)
+        print(*args)
+
     def __init__(self, hostlist: list, ssh_config_path: Path=None):
+        self.results = io.StringIO()
         if ssh_config_path is None:
             ssh_config_path = Path("~/.ssh/config")
         ssh_config_path = ssh_config_path.expanduser()
@@ -58,24 +82,37 @@ class HostSet:
                 if ((time.time() - tupdate) > 10.0):
                     tupdate = time.time()
                     print("Still waiting on ", [thread.host for thread in threads])
+                    # FIXME have the thread killed if it's still waiting after $TIMEOUT
         prog.close()
 
         # Gather up results by output
         gathered = {}
-        for connection, result in resq:
+        success = []
+        error = []
+        commandfail = []
+
+        for connection, result, runresult in resq:
+            # print(connection, result, runresult)
             rstr = str(result)
             if not rstr in gathered:
                 gathered[rstr] = []
             gathered[rstr].append(connection)
+            if runresult in (RunResult.Error, RunResult.Cancelled):
+                error.append(connection)
+            elif runresult == RunResult.CommandFailure:
+                commandfail.append(connection)
+            else:
+                success.append(connection)
 
         # display results
         for result, connections in gathered.items():
-            print('-----> [{}]'.format(' '.join(connection.original_host for connection in connections)))
+            self._lprint('-----> [{}]'.format(' '.join(connection.original_host for connection in connections)))
-            print(result)
+            self._lprint(result)
-        # ## gather_output
+
-        # for connection in self.connections:
+        self._lprint('#####> RUN SUMMARY <#####')
-        #     # import pdb
+        if (success):
-        #     # pdb.set_trace()
+            self._lprint('        Succeeded: ', ', '.join(connection.original_host for connection in success))
-        #     res = connection.run(command)
+        if (commandfail):
-        #     print('---- {} ----'.format(connection.original_host))
+            self._lprint('     Command fail: ', ', '.join(connection.original_host for connection in commandfail))
-        #     print(res)
+        if (error):
+            self._lprint('Failed with Error: ', ','.join(connection.original_host for connection in error))