0.7.1 style fixes

Fixes potential bugs with printing things. Just really cleans up the style printing.

README.txt

@@ -7,9 +7,29 @@
 Run commands across a set of hosts interactively!
 =================================================
 
-Basic configuration file is in multiball.cfg.example in the distribution. Customize to your liking. Host list is loaded from,
-by default, your ssh configuration (assuming you have a specific ssh configuration for your all hosts like Autonomic has). You
-can also use arbitrary host lists (documented in configuration file).
+Basic configuration file is in `multiball.cfg.example` in the distribution. Customize to your liking. 
 
+Host list is loaded from your ssh configuration by default (assuming you have a specific ssh configuration for your all hosts like Autonomic has). You can also use arbitrary host lists (documented in configuration file).
 
 Planned features, see __main__.py's comments.
+
+Installation
+-------------------------------------------------
+
+Install dependencies:
+
+* Debian: `sudo apt install python3 python3-venv git make`
+* Fedora: `sudo dnf install python3 python3-pip git make`
+
+Clone the repo:
+
+```
+git clone https://git.autonomic.zone/autonomic-cooperative/multiball.git
+```
+
+Install, including dependencies:
+
+```
+cd multiball
+make
+```

TODO.txt

@@ -0,0 +1,37 @@
+TODO
+- We should throw out fabric; paramiko does everything we need - we can open a shell and send signals to handle control-C
+  and other sorts of situation handling. We'll basically have to rewrite how fabtools works but it should be relatively
+  trivial since we already basically handle everything at the paramiko level. We may have to parse sshconfig ourselves
+  but it shouldn't be too big a deal.
+- fix handling of control-c (sorta works sometimes)
+- redo the command loop to make some fuckin sense why do we use eceptions to do modal stuff
+- make styles optional
+- expose styles in config file or personal config
+- add local variable support that can be interpolated into commands
+- keep track of previous command outputs for last command, and any previous command with target list and command line
+  - Allow any given command to be tagged with a label, and the outputs would be stored in the output dictionary under that name (for scripting);
+    we assign a default name to it, and are able to refer to it based on either the label, or how many commands back it was;
+  - scripts entire output is stored as one label, but also
+  - add commands to show the list of captured outputs, along with the commands and targets
+  - add commands to search captured outputs
+  - add commands to search captured outputs and put them into a variable that can be used for /target etc.
+- tagging and filtering commands (so `noupgrade` gets a tag, and we can filter them)
+- saving the last host set in a .file
+- allow server groups / server labels
+- add 'watch' to run a command repeatedly until it succeeds (interruptable)
+- add the number of servers for each group of services too
+- notice when you don't sudo in front of command and it errors and ask you if you meant to sudo
+- autocomplete for environment variable names (requires heirarchical completer)
+- implement target aliases (-filteralias) which gives a label to an argument to -hosts
+- ad-hoc host groupings with assigned names, and a host grouping stack
+- allow scripts that use -safe to prompt for safety / restore safety after running
+- implement various commented commands in the command list
+- implement interactive alias system
+- Catch more exceptions in fabtools, and also add retries
+- Make the runner aware of multiple commands so that it can combine outputs and make 'overall success' or 'overall failure'
+- make C-c break the connections not the program (this requires throwing out fabric)
+- make a /summary command that *only* outputs the summary of a remote command, not the stdout/stderr
+- document more things about multiball
+- allow a user-specific configuration that isn't in the current directory
+- make a set of commands that modulate the /all list, so /all goes back to the real default not the total default
+- the ability to deploy files

makefile

@@ -0,0 +1,21 @@
+default: install
+
+.venv:
+	python3 -m venv .venv
+
+pip: .venv
+	@.venv/bin/pip install -e .
+
+$(HOME)/.local/bin/:
+	@echo "'$(HOME)/.local/bin' was not found; creating it"
+	mkdir -p $(HOME)/.local/bin
+	@echo 'Be sure to add this directory to your $$PATH, using e.g.'
+	@echo "echo $(HOME)/.local/bin >> $(HOME)/.bashrc"
+
+$(HOME)/.local/bin/multiball: $(HOME)/.local/bin/ pip
+	ln -s $(PWD)/.venv/bin/multiball $(HOME)/.local/bin/multiball
+
+install: $(HOME)/.local/bin/multiball	
+
+.PHONY: pip install
+

multiball/__init__.py

@@ -1 +1 @@
-__version__ = "0.0.0"
+__version__ = "0.7.1"

multiball/__main__.py

@@ -3,27 +3,6 @@
 #
 
 #
-# TODO
-# - keep track of previous command outputs for last command, and any previous command with target list and command line
-# - calling commands and alieses from command line
-# - tagging and filtering commands (so #noupgrade gets a tag, and we can filter them)
-# - saving the last host set in a .file
-# - allow server groups
-# - allow grepping of output from a command, and then populating the target list from matchintg on it
-# - add 'watch' to run a command repeatedly until it succeeds (interruptable)
-# - add the number of servers for each group of services too
-# - notice when you don't sudo in front of command and it errors and ask you if you meant to sudo
-# - autocomplete for environment variable names (requires heirarchical completer)
-# - implement target aliases (-filteralias) which gives a label to an argument to -hosts
-# - ad-hoc host groupings with assigned names, and a host grouping stack
-# - Assign each output group to a number that can be easily selected with a -target etc. But also allow matching
-#   against those groups. Output groupings survive until the next command.
-# - allow scripts that use -safe to prompt for safety / restore safety after running
-# - Add variables that can be set and passed to commands
-# - implement various commented commands in the command list
-# - implement interactive alias system
-# - Catch more exceptions in fabtools, and also add retries
-
 
 import argparse
 import datetime
@@ -32,23 +11,29 @@ import logging
 import os
 import pprint
 import re
-import shutil
 import shlex
+import shutil
 import sys
 import traceback
 from itertools import islice, zip_longest
 from pathlib import Path
 from typing import Iterable, Optional, Sequence, Union
 
-import yaml
 import prompt_toolkit
+import yaml
 from prompt_toolkit.completion import WordCompleter
-from prompt_toolkit.history import InMemoryHistory, FileHistory
+from prompt_toolkit.history import FileHistory, InMemoryHistory
 from prompt_toolkit.shortcuts import PromptSession
+from prompt_toolkit.styles import Style
 
+from . import __version__
 from .fabtools import HostSet
-from .tools import load_sshconfig_server_list, load_bare_server_list, load_config
+from .tools import (load_bare_server_list, load_config,
+                    load_sshconfig_server_list)
 
+_print = print
+from .style import print_style as print
+from .style import STYLES
 
 class MultiballCommandException(BaseException):
     pass
@@ -85,17 +70,19 @@ COMMANDS = {
     'clearenv': (('/clearenv',), "Clear environment (entire or single variable)"),
     'safety': (('/safety', 'safety'), "Turn on safety or turn off with `off`."),
     'safe': (('/safe', ), "Run this command as if it were safe even if safety is on."),
-    'confirm': (('/confirm',), "[scriptpfing] Prompt for confirmation"),
+    'confirm': (('/confirm',), "[scripting] Prompt for confirmation"),
     'echo': (('/echo', ), "[scripting] print string"),
     'arguments': (('/arguments',), "[scripting] Abort if no arguments are specified."),
     # fixme add interpolation
-    'unsafe': (('/unsafe', ), "[scriptiong] Abort command if safety is on."),
+    'unsafe': (('/unsafe', ), "[scripting] Abort command if safety is on."),
+    'save': (('/save', ),"Save last run log to file."),
+    'debug': (('/////debug', ), "[devel] enter PDB"),
     # 'history': (('/history', '/h'), "Show the last 10 commands in the history."),
     # 'filteralias': (('/filteralias',), "Create a filter alias which allows quick filtering."),
     # 'undo': (('/undo',), "Undo last host or environment command."),
     # 'alias': (('/alias', ), "Create an alias interactively.")
     ## Fixme unambiguate the above commands which list things into this command so things are more consistent
-    # 'list': (('/list',), "List various things (environmet, allhosts, hosts, targets)")
+    # 'list': (('/list',), "List various things (environment, allhosts, hosts, targets)")
     # '!': (('!', '/local'), "Run a local command and capture its output.")
 }
 
@@ -111,6 +98,7 @@ class Multiball:
             'safety': self.command_safety,
             'safe': self.command_safe,
             'clearenv': self.command_clearenv,
+            'save': self.command_save,
         }
         configdict = load_config(Path("multiball.cfg"))
         self.allhosts: set = set()
@@ -118,6 +106,7 @@ class Multiball:
         self.environment = {}
         self.safety = True
         self.ssh_config = None
+        self.last_run = {}
 
         if config := configdict.get('config', {}):
             if 'history' in config:
@@ -129,19 +118,19 @@ class Multiball:
                         self.ssh_config = Path(sc).expanduser()
                     newhosts = load_sshconfig_server_list(
                         Path(sc).expanduser())
-                    print(f"Loaded {len(newhosts)} from {sc}")
+                    print(f"Loaded {len(newhosts)} host(s) from {sc}")
                     self.allhosts.update(newhosts)
 
             if bareconfigs := config.get('hostlists'):
                 for sc in bareconfigs.split(','):
                     newhosts = load_bare_server_list(Path(sc).expanduser())
-                    print(f"Loaded {len(newhosts)} from {sc}")
+                    print(f"Loaded {len(newhosts)} host(s) from {sc}")
                     self.allhosts.update(newhosts)
 
         else:
             self.ssh_config = Path("~/.ssh/ssh_config").expanduser()
             print(
-                f"Warning: no [config] section in {configFile}. Using fallbacks.")
+                f"Warning: no [config] section in {configFile}. Using fallbacks.", style="error")
             self.allhosts = load_sshconfig_server_list(self.ssh_config)
 
         # setup state and environment
@@ -177,17 +166,17 @@ class Multiball:
         """
         if args.lower().startswith("off"):
             self.safety = False
-            print("** Safety OFF. Now unsafe.")
+            print("** Safety OFF. Now unsafe!", style="unsafe")
         else:
             self.safety = True
-            print("** Safety ON.")
+            print("** Safety ON.", style="safe")
 
     def command_echo(self, command, args):
         """
         Echo string
         fixme put string interpolation here so that state variables can be printed
         """
-        print(args)
+        print(f"{args}")
 
     def command_environment(self, command, args):
         """
@@ -216,8 +205,10 @@ class Multiball:
         """
         Prompt user with a yes/no question. Default responses result in a CommandAbort exception.
         """
-        result = prompt_toolkit.prompt(
-            "<" + args + " [Yys/Nn] > ")
+        result = prompt_toolkit.prompt([("class:promptyn",
+                                         "<" + args + " [Yys/Nn]>"),
+                                        ("class:message",
+                                         " ")], style=STYLES)
         if result and (result[0] in 'Yys'):
             return True
         else:
@@ -233,9 +224,9 @@ class Multiball:
         """
         if not self._is_scripting or not self._script_frame_arguments:
             if not self._is_scripting:
-                print("does nothing outside of script")
+                print("does nothing outside of script", style="alert")
             else:
-                print("arguments required")
+                print("arguments required", style="error")
             raise MBCommandAbort
 
         return
@@ -246,9 +237,9 @@ class Multiball:
         """
         if not self._is_scripting or self.safety:
             if not self._is_scripting:
-                print("does nothing outside of script")
+                print("does nothing outside of script", style="alert")
             else:
-                print("cannot run in safety mode")
+                print("cannot run in safety mode", style="error")
             raise MBCommandAbort
 
         return
@@ -257,6 +248,27 @@ class Multiball:
         print("cleared environment")
         self.environment = dict()
 
+    def command_save(self, command, args):
+        # FIXME: We need to make aliases store multiple entries, and save should take the whole alias not just
+        # the last command.
+        if (not self.last_run):
+            print("Nothing to log.", style="alert")
+            return
+        tstamp = datetime.datetime.now().strftime("%Y-%m-%d-%H%M%Z")
+        outfilename = f"{tstamp}.multiball.log"
+        if (args):
+            outfilename = args[0]
+            if (not outfilename.endswith(".log")):
+                outfilename = outfilename + ".log"
+
+        print(f"Writing {outfilename} for command `{self.last_run['cmd']}`.", style="alert")
+        with (open(outfilename, 'w')) as out:
+            out.writelines([f"command: {self.last_run['cmd']}\n",
+                            "-------------------------------------------\n"
+                            ""])
+            out.write(self.last_run['results'].getvalue())
+            out.write("\n")
+
     def _print_targetlist(self, tlist=None, keyword='Targeting'):
         if not tlist:
             tlist = self.targethosts
@@ -265,7 +277,8 @@ class Multiball:
 
     def _print_help(self):
         for item in COMMANDS.values():
-            print(', '.join(item[0]) + ' - ' + item[1])
+            m = ', '.join(item[0]) + ' - ' + item[1]
+            print(f"{m}", style="bright")
         print("Other entries run command on targets.")
 
     def _run_command(self, command: str):
@@ -288,8 +301,11 @@ class Multiball:
 
         if abst == 'exit':
             raise MBExit
-        if abst == 'help':
+        elif abst == 'help':
             self._print_help()
+        elif abst == 'debug':
+            print("==== DEBUGGING ====")
+            __import__("pdb").set_trace()
         elif abst in self.handlers:
             self.handlers[abst](command[0], cargs)
         elif abst == 'clear':
@@ -363,6 +379,10 @@ class Multiball:
         remotecmd.append(command)
         h = HostSet(self.targethosts, self.ssh_config)
         h.run(';'.join(remotecmd))
+        self.last_run = {
+            'cmd': remotecmd,
+            'results': h.results
+        }
 
     def do_cmd(self, command):
         try:
@@ -370,10 +390,10 @@ class Multiball:
         except MBCommandRun:
             # remote command!
             if self.safety:
-                print(f"would run `{command}` on targethosts")
-                print("Safety is ON. `/safety off` to turn off.")
+                print(f"would run `{command}` on targethosts", style="bright")
+                print("Safety is ON. `/safety off` to turn off.", style="alert")
             else:
-                print(f"running `{command}` on targethosts")
+                print(f"running `{command}` on targethosts", style="bright")
                 self._run_remote_command(command)
 
     def cmd_prompt(self):
@@ -385,22 +405,73 @@ class Multiball:
         completer = WordCompleter(list(self.allhosts) + allcommands)
         if self.safety:
             prompt = "!"
+            pclass = "class:safe"
         else:
             prompt = '$'
-        command = self.session.prompt(
-            f'{len(self.targethosts)} -> {prompt} ', completer=completer)
+            pclass = "class:unsafe"
+        command = self.session.prompt([("class:count", f'{len(self.targethosts)}'),
+                                      ("class:prompt", '->'),
+                                      (pclass, prompt),
+                                      ("class:default", " ")], completer=completer, style=STYLES)
         try:
             self.do_cmd(command)
         except MBCommandAbort:
             # Nothing of importance happens here.
             ...
 
+def parse_args(argv):
+    if (len(argv) > 1):
+        parser = argparse.ArgumentParser(
+            prog='multiball',
+            description='Run commands on a number of hosts.',
+            epilog='With no arguments enter interactive mode.',
+        )
+        parser.add_argument('--confirm', action='store_true', help='Skip confirmation step, use with caution.')
+        parser.add_argument('-t', '--target', type=str, nargs='?', action='append', help='Select one or more targets.')
+        parser.add_argument('-a', '--all', action='store_true', help='Select default target list.')
+        parser.add_argument('-c', '--command', type=str, nargs='+', action='append', help='Command to run.')
+        res = parser.parse_args(argv)
+        return res
+    else:
+        return None
 
-def multiball(*args, **kwargs):
-    print("Welcome to Multiball. type /help for help.")
+
+def multiball(argv):
+    args = parse_args(argv)
+
+    ## Command mode
+    if (args):
+        if (not args.all and (len(args.target) <= 0)):
+            print('Need either --all or at least one --target')
+            return 1
+        if (len(args.command) <= 0):
+            print('Need at least one --command')
+            return 1
+
+        print(f"Multiball {__version__}.", style="alert")
+        mb = Multiball()
+
+        if (args.all):
+            mb.targethosts = mb.allhosts
+        else:
+            mb.targethosts = args.target
+
+        for command in args.command:
+            mb._print_targetlist()
+            command = ' '.join(command)
+            if (not args.confirm):
+                try:
+                    mb.command_confirm('', args=f"Run `{command}`?")
+                except MBCommandAbort:
+                    continue
+            mb._run_remote_command(command)
+        return 0
+
+    ## Interactive mode
+    print(f"Welcome to Multiball {__version__}. type /help for help.", style="alert")
     mb = Multiball()
-    # loop on prompt
     while (True):
+        # loop on prompt
         try:
             mb.cmd_prompt()
         except MBExit:
@@ -409,12 +480,12 @@ def multiball(*args, **kwargs):
 
 def main():
     try:
-        sys.exit(multiball(sys.argv))
+        sys.exit(multiball(sys.argv[1:]))
     except EOFError:
         sys.exit(0)
     except Exception as inst:
         # fixme log exception
-        print(f"Exception! {inst}")
+        print(f"Exception! {inst}", style="error")
         print(traceback.format_exc())
         sys.exit(-1)
 

multiball/fabtools.py

@@ -1,26 +1,59 @@
+import io
+import socket
 import time
-
+from enum import Enum
 from pathlib import Path
-from threading import Thread, Lock
-
-from tqdm import tqdm
-
-from fabric2 import ThreadingGroup, SerialGroup, Config, Connection
+from threading import Lock, Thread
 
 import paramiko
+from fabric2 import Config, Connection, SerialGroup, ThreadingGroup
+from tqdm import tqdm
 
+from .style import print_style
+
+class RunResult(Enum):
+    Success = 0
+    CommandFailure = 1
+    Error = 2
+    Cancelled = 3
+    Timeout = 4 # FIXME support host timeouts
 
 def thread_run(connection, command, result_lock, result_queue):
+    runresult = RunResult.Success
     try:
+        res = connection.open()
         res = connection.run(command, warn=True, hide=True)
-    except (paramiko.ssh_exception.NoValidConnectionsError, paramiko.ssh_exception.SSHException) as inst:
+        if (res.exited != 0):
+            runresult = RunResult.CommandFailure
+    except (paramiko.ssh_exception.NoValidConnectionsError, paramiko.ssh_exception.SSHException, socket.gaierror) as inst:
         res = f"Could not connect to host: {inst}"
+        runresult = RunResult.Error
+    except KeyboardInterrupt as inst:
+        res = f"Canceled host {inst} from C-c"
+        runresult = RunResult.Cancelled
+    except Exception as inst:
+        res = f"Unknown error for host: {inst}"
+        runresult = RunResult.Error
+    finally:
+        connection.close()
+
     with result_lock:
-        result_queue.append((connection, res))
+        result_queue.append((connection, res, runresult))
+
+
 
 # A set of hosts we can target with a series of commands and also can collect output of each command
 class HostSet:
+    def _lprint(self, *args, **kwargs):
+        s = " ".join([str(a) for a in args]) + '\n'
+        self.results.write(s)
+        if ("style" in kwargs):
+           print_style(s.rstrip(), style=kwargs["style"])
+        else:
+            print(*args)
+
     def __init__(self, hostlist: list, ssh_config_path: Path=None):
+        self.results = io.StringIO()
         if ssh_config_path is None:
             ssh_config_path = Path("~/.ssh/config")
         ssh_config_path = ssh_config_path.expanduser()
@@ -38,44 +71,66 @@ class HostSet:
         for connection in self.connections:
             t = Thread(target=thread_run, args=[connection, command, reslock, resq])
             t.host = connection.original_host
+            t.connection = connection
             t.start()
             threads.append(t)
 
         # display status about threads, and join them when they finish
         while (True):
-            nt = []
-            for i in threads:
-                if not i.is_alive():
-                    i.join()
-                    prog.update()
-                    #print('.', end='', flush=True)
+            try:
+                nt = []
+                for i in threads:
+                    if not i.is_alive():
+                        i.join()
+                        prog.update()
+                        #print('.', end='', flush=True)
+                    else:
+                        nt.append(i)
+                threads = nt
+                if len(threads) == 0:
+                    break
                 else:
-                    nt.append(i)
-            threads = nt
-            if len(threads) == 0:
-                break
-            else:
-                if ((time.time() - tupdate) > 10.0):
-                    tupdate = time.time()
-                    print("Still waiting on ", [thread.host for thread in threads])
+                    if ((time.time() - tupdate) > 10.0):
+                        tupdate = time.time()
+                        m = "Still waiting on " + str([thread.host for thread in threads])
+                        print_style(m, style="alert")
+                        # FIXME have the thread killed if it's still waiting after $TIMEOUT
+            except KeyboardInterrupt as inst:
+                # cancel remaining servers
+                print_style("(break)", style="error")
+                for thread in threads:
+                    thread.connection.close()
+
         prog.close()
 
         # Gather up results by output
         gathered = {}
-        for connection, result in resq:
+        success = []
+        error = []
+        commandfail = []
+
+        for connection, result, runresult in resq:
+            # print(connection, result, runresult)
             rstr = str(result)
             if not rstr in gathered:
                 gathered[rstr] = []
             gathered[rstr].append(connection)
+            if runresult in (RunResult.Error, RunResult.Cancelled):
+                error.append(connection)
+            elif runresult == RunResult.CommandFailure:
+                commandfail.append(connection)
+            else:
+                success.append(connection)
 
         # display results
         for result, connections in gathered.items():
-            print('-----> [{}]'.format(' '.join(connection.original_host for connection in connections)))
-            print(result)
-        # ## gather_output
-        # for connection in self.connections:
-        #     # import pdb
-        #     # pdb.set_trace()
-        #     res = connection.run(command)
-        #     print('---- {} ----'.format(connection.original_host))
-        #     print(res)
+            self._lprint('-----> [{}]'.format(' '.join(connection.original_host for connection in connections)))
+            self._lprint(result)
+
+        self._lprint('#####> RUN SUMMARY <#####', style="alert")
+        if (success):
+            self._lprint('        Succeeded: ', ', '.join(connection.original_host for connection in success), style="good")
+        if (commandfail):
+            self._lprint('     Command fail: ', ', '.join(connection.original_host for connection in commandfail), style="warning")
+        if (error):
+            self._lprint('Failed with Error: ', ','.join(connection.original_host for connection in error), style="error")

multiball/style.py

@@ -0,0 +1,27 @@
+from prompt_toolkit import print_formatted_text, HTML
+from prompt_toolkit.styles import Style
+from prompt_toolkit.formatted_text import FormattedText
+
+STYLES = Style.from_dict({
+    "safe":     "#012 bg:#00ff88",
+    "unsafe":   "#012 bg:#ff2200",
+    "message":  "#666699 bg:#000",
+    "count":    "#6666ff bg:#000",
+    "prompt":   "#777700 bg:#000",
+    "alert":    "bold #f0f bg:#000",
+    "error":    "bold #f00 bg:#000",
+    "warning":  "bold #f50 bg:#000",
+    "good":     "bold #090 bg:#000",
+    "promptyn": "bold #000 bg:#ff0",
+    "default":  "#880 bg:#000",
+    "bright":   "#ff0 bg:#000",
+    })
+
+def print_style(*args, **kwargs):
+    style="message"
+    if ("style" in kwargs):
+        style=kwargs["style"]
+
+    message = " ".join([str(a) for a in args])
+
+    print_formatted_text(FormattedText([(f"class:{style}", message)]), style=STYLES)