Add create_members and create_bans

README.md

@@ -1,5 +1,7 @@
 # adversarial-mailman-2to3 
 
-> be you
+migrating mailman2 to mailman3?
-> migrating mailman2 to mailman3
+
-> shared hosting doesn't give you access to mailman2 config
+shared hosting doesn't give you access to mailman2 config?
+
+enter adversarial-mailman-2to3!

export_mailman2_settings.py

@@ -1,47 +1,285 @@
 import argparse
 import json
 import sys
+from functools import partial
+from copy import deepcopy
+from math import floor
+
 from bs4 import BeautifulSoup
 
-from mailman.utilities.importer import NAME_MAPPINGS
+from mailman.interfaces.action import Action
-
+from mailman.interfaces.archiver import ArchivePolicy
-parser = argparse.ArgumentParser(
+from mailman.interfaces.digests import DigestFrequency
-    prog=sys.argv[0], description="Munge Mailman config data"
+from mailman.utilities.importer import (
+    NAME_MAPPINGS,
+    EXCLUDES as EXCLUDES_ORIGINAL,
+    TYPES as TYPES_ORIGINAL,
+    member_moderation_action_mapping,
+    dmarc_action_mapping, 
+    member_roster_visibility_mapping
+)
+from mailman.interfaces.nntp import NewsgroupModeration
+from mailman.interfaces.mailinglist import (
+    DMARCMitigateAction,
+    Personalization,
+    ReplyToMunging,
+    SubscriptionPolicy,
 )
 
-parser.add_argument("file", type=argparse.FileType('r', encoding='utf-8'))
+KEYFILTER = ('submit')
 
-args = parser.parse_args()
+EXCLUDES = set(
+    tuple(EXCLUDES_ORIGINAL) + tuple({
+    # from `convert_to_uri` in mailman's `importer.py`
+    'goodbye_msg': 'list:user:notice:goodbye',
+    'msg_header': 'list:member:regular:header',
+    'msg_footer': 'list:member:regular:footer',
+    'digest_header': 'list:member:digest:header',
+    'digest_footer': 'list:member:digest:footer',
+}.keys()) + (
+    # some settings to manually skip, which don't seem to be handled by the
+    # conversion script
+    'archive_volume_frequency',
+    'autoresponse_postings_text_upload',
+    'autoresponse_admin_text_upload',
+    'autoresponse_request_text_upload',
+    'bounce_notify_owner_on_bounce_increment',
+    'nondigestable',
+    'digestable',
+    'digest_is_default',
+    'mime_is_default_digest',
+    'digest_size_threshhold',
+    '_new_volume',
+    '_send_digest_now',
+    'umbrella_list',
+    'umbrella_member_suffix',
+    'send_reminders',
+    'admin_member_chunksize',
+    'host_name',
+    'new_member_options',
+    'include_sender_header',
+    'drop_cc',
+    'available_languages',
+    'encode_ascii_prefixes',
+    'regular_exclude_lists',
+    'regular_include_lists',
+    'scrub_nondigest',
+    'regular_exclude_ignore',
+    'subscribe_auto_approval',
+    'unsubscribe_policy',
+    'obscure_addresses',
+    'member_moderation_notice',
+    'equivalent_domains',
+    'nonmember_rejection_notice',
+    'default_member_moderation',
+    'member_verbosity_threshold',
+    'member_verbosity_interval',
+    'member_moderation_action',
+    # TODO investigate
+    'dmarc_quarantine_moderation_action',
+    'dmarc_none_moderation_action',
+    'forward_auto_discards',
+    'hdrfilter_rebox_01',
+    'hdrfilter_action_01',
+    'bounce_matching_headers',
+    'topic_rebox_01',
+    'topic_desc_01',
+    'topic_box_01',
+    # None of the lists has topic filtering enabled
+    'topics_enabled',
+    'topics_bodylines_limit',
+    # ... or DMARC addresses
+    'dmarc_moderation_addresses',
+))
 
-data_clean = {}
+TYPES = deepcopy(TYPES_ORIGINAL)
 
-soup = BeautifulSoup(args.file.read(), 'html.parser')
+TYPES['process_bounces'] = lambda x: bool(x)
+TYPES['bounce_notify_owner_on_disable'] = lambda x: bool(x)
+TYPES['bounce_notify_owner_on_removal'] = lambda x: bool(x)
+TYPES['filter_content'] = lambda x: bool(x)
+TYPES['collapse_alternatives'] = lambda x: bool(x)
+TYPES['convert_html_to_plaintext'] = lambda x: bool(x)
+TYPES['digest_send_periodic'] = lambda x: bool(x)
+TYPES['anonymous_list'] = lambda x: bool(x)
+TYPES['first_strip_reply_to'] = lambda x: bool(x)
+TYPES['send_welcome_message'] = lambda x: bool(x)
+TYPES['send_goodbye_message'] = lambda x: bool(x)
+TYPES['admin_immed_notify'] = lambda x: bool(x)
+TYPES['admin_notify_mchanges'] = lambda x: bool(x)
+TYPES['respond_to_post_requests'] = lambda x: bool(x)
+TYPES['emergency'] = lambda x: bool(x)
+TYPES['administrivia'] = lambda x: bool(x)
+TYPES['include_rfc2369_headers'] = lambda x: bool(x)
+TYPES['allow_list_posts'] = lambda x: bool(x)
+TYPES['advertised'] = lambda x: bool(x)
+TYPES['require_explicit_destination'] = lambda x: bool(x)
+TYPES['bounce_score_threshold'] = lambda x: floor(float(x))
 
-for field in soup.find_all('textarea'):
+TYPES_EXTRA = {
-    name = NAME_MAPPINGS.get(field['name'], field['name'])
+    'autorespond_requests': lambda x, y: x(y).name,
+    'autorespond_owner': lambda x, y: x(y).name,
+    'autorespond_postings': lambda x, y: x(y).name,
+    'autoresponse_grace_period': lambda x, y: '{}d'.format(x(y).days),
+    'bounce_info_stale_after': lambda x, y: '{}s'.format(x(y).seconds),
+    'bounce_you_are_disabled_warnings_interval': lambda x, y: '{}s'.format(x(y).seconds),
+    'forward_unrecognized_bounces_to': lambda x, y: x(y).name,
+    'filter_action': lambda x, y: x(y).name,
+    'digest_volume_frequency': lambda x, y: x(y).name,
+    'reply_goes_to_list': lambda x, y: x(y).name,
+    'newsgroup_moderation': lambda x, y: x(y).name,
+    'subscription_policy': lambda x, y: x(y).name,
+    'default_nonmember_action': lambda x, y: x(y).name,
+}
 
-    if 'msg' in name:
+for key, func in TYPES_EXTRA.items():
-        continue
+    TYPES[key] = partial(func, TYPES_ORIGINAL[key])
-
-    data_clean[name] = [l for l in field.get_text().split('\n') if l != ""]
 
 
-for field in soup.find_all('input'):
+def msg(*args, **kwargs):
-    if field['type'] == 'hidden':
+    print(*args, file=sys.stderr, **kwargs)
-        continue
 
-    if field['type'] == 'RADIO':
+
-        if 'checked' not in field.attrs:
+def get_form_data(htmlfile):
+    data_clean = {}
+
+    soup = BeautifulSoup(htmlfile.read(), 'html.parser')
+
+    for field in soup.find_all('textarea'):
+        if field['name'] in EXCLUDES:
             continue
 
-    name = NAME_MAPPINGS.get(field['name'], field['name'])
+        name = NAME_MAPPINGS.get(field['name'], field['name'])
 
-    if 'msg' in name:
+        if 'msg' in name:
-        continue
+            continue
 
-    try:
+        data_clean[name] = [line for line in field.get_text().split('\n') if
-        data_clean[name] = field['value']
+                            line != ""]
-    except KeyError:
-        data_clean[name] = ""
 
-print(json.dumps(data_clean))
+    for field in soup.find_all('input'):
+        if field['name'] in EXCLUDES:
+            continue
+
+        if field['type'].lower() in ('hidden', 'submit'):
+            continue
+
+        if field['type'].lower() == 'radio':
+            if 'checked' not in field.attrs:
+                continue
+
+        name = NAME_MAPPINGS.get(field['name'], field['name'])
+
+        if 'msg' in name:
+            continue
+
+        try:
+            value = field['value']
+            try:
+                value = int(value)
+            except ValueError:
+                ...
+        except KeyError:
+            value = ''
+
+        converter = TYPES.get(name)
+        try:
+            if converter is not None:
+                value = converter(value)
+        except (TypeError, KeyError, ValueError):
+            from pdb import set_trace; set_trace()
+
+            print('Type conversion error for key "{}": {}'.format(
+                name, value), file=sys.stderr)
+
+        data_clean[name] = value
+
+    return data_clean
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        prog=sys.argv[0], description="Munge Mailman config data"
+    )
+    parser.add_argument("config", help="A list of files (named specifically) to import as configurations",
+                        nargs='+',
+                        action="append")
+    args = parser.parse_args()
+
+    globalconfig = {}
+    for page in args.config[0]:
+        msg(page)
+        with open(page) as inf:
+            result = get_form_data(inf)
+        for key, value in result.items():
+            if key in globalconfig:
+                msg(f"warning - duplicate key {key}")
+            else:
+                globalconfig[key] = value
+
+    # Handle the moderation policy.
+    #
+    # The mlist.default_member_action and mlist.default_nonmember_action enum
+    # values are different in Mailman 2.1, because they have been merged into a
+    # single enum in Mailman 3.
+    #
+    # Unmoderated lists used to have default_member_moderation set to a false
+    # value; this translates to the Defer default action.  Moderated lists with
+    # the default_member_moderation set to a true value used to store the
+    # action in the member_moderation_action flag, the values were: 0==Hold,
+    # 1=Reject, 2==Discard
+    if bool(globalconfig.get('default_member_moderation', 0)):
+        globalconfig['default_member_action'] = member_moderation_action_mapping(
+            globalconfig.get('member_moderation_action')).name
+        del globalconfig['default_member_moderation']
+    else:
+        globalconfig['default_member_action'] = Action.defer.name
+    # Handle DMARC mitigations.
+    # This would be straightforward except for from_is_list.  The issue
+    # is in MM 2.1 the from_is_list action applies if dmarc_moderation_action
+    # doesn't apply and they can be different.
+    # We will map as follows:
+    # from_is_list > dmarc_moderation_action
+    #    dmarc_mitigate_action = from_is_list action
+    #    dmarc_mitigate_unconditionally = True
+    # from_is_list <= dmarc_moderation_action
+    #    dmarc_mitigate_action = dmarc_moderation_action
+    #    dmarc_mitigate_unconditionally = False
+    # The text attributes are handled above.
+    if (globalconfig.get('from_is_list', 0) >
+            globalconfig.get('dmarc_moderation_action', 0)):
+        globalconfig['dmarc_mitigate_action'] = dmarc_action_mapping(
+            globalconfig.get('from_is_list', 0)).name
+        globalconfig['dmarc_mitigate_unconditionally'] = True
+    else:
+        globalconfig['dmarc_mitigate_action'] = dmarc_action_mapping(
+            globalconfig.get('dmarc_moderation_action', 0)).name
+        globalconfig['dmarc_mitigate_unconditionally'] = False
+    if 'from_is_list' in globalconfig.keys():
+        del globalconfig['from_is_list']
+    if 'dmarc_moderation_action' in globalconfig.keys():
+        del globalconfig['dmarc_moderation_action']
+    # Handle the archiving policy.  In MM2.1 there were two boolean options
+    # but only three of the four possible states were valid.  Now there's just
+    # an enum.
+    if globalconfig.get('archive'):
+        # For maximum safety, if for some strange reason there's no
+        # archive_private key, treat the list as having private archives.
+        if globalconfig.get('archive_private', True):
+            globalconfig['archive_policy'] = ArchivePolicy.private.name
+        else:
+            globalconfig['archive_policy'] = ArchivePolicy.public.name
+        del globalconfig['archive']
+    else:
+        globalconfig['archive_policy'] = ArchivePolicy.never.name
+    if 'archive_private' in globalconfig.keys():
+        del globalconfig['archive_private']
+
+    # Handle roster visibility.
+    mapping = member_roster_visibility_mapping(
+        globalconfig.get('private_roster', None))
+    if mapping is not None:
+        globalconfig['member_roster_visibility'] = mapping.name
+        del globalconfig['private_roster']
+
+    print(json.dumps(globalconfig))

import_mailman3_list_config.py

@@ -9,7 +9,9 @@ from pathlib import Path
 from requests.auth import HTTPBasicAuth
 
 REST_USER="restadmin"
-REST_PATH="http://127.0.0.1:8001/3.0/lists"
+REST_PATH="http://127.0.0.1:8001/3.0"
+REST_PASS = Path("/run/secrets/mailman_rest_password").read_text().strip()
+REST_AUTH = HTTPBasicAuth(REST_USER, REST_PASS)
 
 BANNER="""
             _) |                 __ /                _|_)
@@ -19,6 +21,55 @@ _|_|_|\__,_|_|_|_|_|_|\__,_|_| _|___/\__|\___/_| _|_|  _|\__, |
 """
 
 
+# def create_user(email):
+#     result = requests.get(f"{REST_PATH}/users/{email}")
+#     print(result)
+#     print(result.text)
+
+#     if not result.ok:
+#         print(f"user {email} not found, creating")
+#         result = requests.post(f"{REST_PATH}/users", auth=REST_AUTH, json={"email": email})
+#         if (result.ok):
+#             print("success")
+#             print(result)
+#             print(result.text)
+#         else:
+#             print("failed")
+#             print(result)
+#             print(result.reason)
+#             return False
+#     return True
+
+def create_member(email, mlist, role):
+    result = requests.post(f"{REST_PATH}/members", auth=REST_AUTH, json={"list_id": mlist.replace('@', '.'), "subscriber": email, "role": role})
+    if (result.ok):
+        print(f"{email} <- {role} success")
+        print(result)
+        print(result.text)
+        return True
+    else:
+        print(f"{email} <- {role} failed")
+        print(result)
+        print(result.reason)
+        print(result.text)
+        return False
+
+
+def create_ban(email, mlist):
+    result = requests.post(f"{REST_PATH}/lists/{mlist}/bans", auth=REST_AUTH, json={"email": email})
+    if (result.ok):
+        print(f"{email} <- BAN success")
+        print(result)
+        print(result.text)
+        return True
+    else:
+        print(f"{email} <- BAN failed")
+        print(result)
+        print(result.reason)
+        print(result.text)
+        return False
+
+
 def parse_args(args):
     parser = argparse.ArgumentParser("load a json config into local mailman configuration")
 
@@ -37,16 +88,13 @@ def main(args):
     with pargs.patch.open() as inf:
         p = json.load(inf)
 
-    rest_pass = Path("/run/secrets/mailman_rest_password").read_text().strip()
-    rest_auth = HTTPBasicAuth(REST_USER, rest_pass)
-
     # determine if list exists, if not create with defaults
-    result = requests.get(REST_PATH, auth=rest_auth)
+    result = requests.get(REST_PATH + '/lists', auth=REST_AUTH)
     if result.ok:
         if (not (pargs.list in [x['fqdn_listname'] for x in result.json()['entries']])):
             # create list
             print("list not found: creating")
-            result = requests.post(f"{REST_PATH}", auth=rest_auth, json={"fqdn_listname": pargs.list})
+            result = requests.post(f"{REST_PATH}/lists", auth=REST_AUTH, json={"fqdn_listname": pargs.list})
             if (result.ok):
                 print("success")
                 print(result)
@@ -55,6 +103,7 @@ def main(args):
                 print("failed")
                 print(result)
                 print(result.reason)
+                print(result.text)
                 return 1
             # list exists
     else:
@@ -64,17 +113,51 @@ def main(args):
         return 1
 
 
-    # patch config
+    # fixme pull out owner, moderator keys, we'll poke them in separately
-    result = requests.patch(f"{REST_PATH}/{pargs.list}/config", auth=rest_auth, json=p)
+    owners = []
-    if result.ok:
+    moderators = []
-        print("success")
+    ban_list = []
-        print(result)
+    if "owner" in p:
-        print(result.text)
+        owners = p["owner"]
-    else:
+        del p["owner"]
-        print("failed")
+    if "moderator" in p:
-        print(result)
+        moderators = p["moderator"]
-        print(result.reason)
+        del p["moderator"]
+    if "ban_list" in p:
+        ban_list = p["ban_list"]
+        del p["ban_list"]
 
+    # patch config
+    cnt = 0
+    for k, v in p.items():
+        if (type(v) == bool):
+            v = str(v)
+        result = requests.patch(f"{REST_PATH}/lists/{pargs.list}/config", auth=REST_AUTH, json={k: v})
+        if result.ok:
+            cnt += 1
+            print(f"*** [OK] {k} success")
+            print(result)
+            print(result.text)
+            print("")
+        else:
+            print(f"*** [!!] {k} failed")
+            print(result)
+            print(result.reason)
+            print(result.text)
+            print("")
+
+    print(f"- {cnt} / {len(p) - 1} succeeded.")
+
+    print("assigning roles")
+    for user in owners:
+        create_member(user, pargs.list, "owner")
+
+    for user in moderators:
+        create_member(user, pargs.list, "moderator")
+
+    print("banning")
+    for user in ban_list:
+        create_ban(user, pargs.list)
 
 if __name__ == "__main__":
     sys.exit(main(sys.argv))