adversarial-mailman2to3/export_mailman2_settings.py

import argparse
import json
import sys
from functools import partial
from copy import deepcopy
from math import floor

from bs4 import BeautifulSoup

from mailman.interfaces.action import Action
from mailman.interfaces.archiver import ArchivePolicy
from mailman.interfaces.digests import DigestFrequency
from mailman.utilities.importer import (
    NAME_MAPPINGS,
    EXCLUDES as EXCLUDES_ORIGINAL,
    TYPES as TYPES_ORIGINAL,
    member_moderation_action_mapping,
    dmarc_action_mapping, 
    member_roster_visibility_mapping
)
from mailman.interfaces.nntp import NewsgroupModeration
from mailman.interfaces.mailinglist import (
    DMARCMitigateAction,
    Personalization,
    ReplyToMunging,
    SubscriptionPolicy,
)

KEYFILTER = ('submit')

EXCLUDES = set(
    tuple(EXCLUDES_ORIGINAL) + tuple({
    # from `convert_to_uri` in mailman's `importer.py`
    'goodbye_msg': 'list:user:notice:goodbye',
    'msg_header': 'list:member:regular:header',
    'msg_footer': 'list:member:regular:footer',
    'digest_header': 'list:member:digest:header',
    'digest_footer': 'list:member:digest:footer',
}.keys()) + (
    # some settings to manually skip, which don't seem to be handled by the
    # conversion script
    'archive_volume_frequency',
    'autoresponse_postings_text_upload',
    'autoresponse_admin_text_upload',
    'autoresponse_request_text_upload',
    'bounce_notify_owner_on_bounce_increment',
    'nondigestable',
    'digestable',
    'digest_is_default',
    'mime_is_default_digest',
    'digest_size_threshhold',
    '_new_volume',
    '_send_digest_now',
    'umbrella_list',
    'umbrella_member_suffix',
    'send_reminders',
    'admin_member_chunksize',
    'host_name',
    'new_member_options',
    'include_sender_header',
    'drop_cc',
    'available_languages',
    'encode_ascii_prefixes',
    'regular_exclude_lists',
    'regular_include_lists',
    'scrub_nondigest',
    'regular_exclude_ignore',
    'subscribe_auto_approval',
    'unsubscribe_policy',
    'obscure_addresses',
    'member_moderation_notice',
    'equivalent_domains',
    'nonmember_rejection_notice',
    'default_member_moderation',
    'member_verbosity_threshold',
    'member_verbosity_interval',
    'member_moderation_action',
    # TODO investigate
    'dmarc_quarantine_moderation_action',
    'dmarc_none_moderation_action',
    'forward_auto_discards',
    'hdrfilter_rebox_01',
    'hdrfilter_action_01',
    'bounce_matching_headers',
    'topic_rebox_01',
    'topic_desc_01',
    'topic_box_01',
    # None of the lists has topic filtering enabled
    'topics_enabled',
    'topics_bodylines_limit',
    # ... or DMARC addresses
    'dmarc_moderation_addresses',
))

TYPES = deepcopy(TYPES_ORIGINAL)

TYPES['process_bounces'] = lambda x: bool(x)
TYPES['bounce_notify_owner_on_disable'] = lambda x: bool(x)
TYPES['bounce_notify_owner_on_removal'] = lambda x: bool(x)
TYPES['filter_content'] = lambda x: bool(x)
TYPES['collapse_alternatives'] = lambda x: bool(x)
TYPES['convert_html_to_plaintext'] = lambda x: bool(x)
TYPES['digest_send_periodic'] = lambda x: bool(x)
TYPES['anonymous_list'] = lambda x: bool(x)
TYPES['first_strip_reply_to'] = lambda x: bool(x)
TYPES['send_welcome_message'] = lambda x: bool(x)
TYPES['send_goodbye_message'] = lambda x: bool(x)
TYPES['admin_immed_notify'] = lambda x: bool(x)
TYPES['admin_notify_mchanges'] = lambda x: bool(x)
TYPES['respond_to_post_requests'] = lambda x: bool(x)
TYPES['emergency'] = lambda x: bool(x)
TYPES['administrivia'] = lambda x: bool(x)
TYPES['include_rfc2369_headers'] = lambda x: bool(x)
TYPES['allow_list_posts'] = lambda x: bool(x)
TYPES['advertised'] = lambda x: bool(x)
TYPES['require_explicit_destination'] = lambda x: bool(x)
TYPES['bounce_score_threshold'] = lambda x: floor(float(x))

TYPES_EXTRA = {
    'autorespond_requests': lambda x, y: x(y).name,
    'autorespond_owner': lambda x, y: x(y).name,
    'autorespond_postings': lambda x, y: x(y).name,
    'autoresponse_grace_period': lambda x, y: '{}d'.format(x(y).days),
    'bounce_info_stale_after': lambda x, y: '{}s'.format(x(y).seconds),
    'bounce_you_are_disabled_warnings_interval': lambda x, y: '{}s'.format(x(y).seconds),
    'forward_unrecognized_bounces_to': lambda x, y: x(y).name,
    'filter_action': lambda x, y: x(y).name,
    'digest_volume_frequency': lambda x, y: x(y).name,
    'reply_goes_to_list': lambda x, y: x(y).name,
    'newsgroup_moderation': lambda x, y: x(y).name,
    'subscription_policy': lambda x, y: x(y).name,
    'default_nonmember_action': lambda x, y: x(y).name,
}

for key, func in TYPES_EXTRA.items():
    TYPES[key] = partial(func, TYPES_ORIGINAL[key])


def msg(*args, **kwargs):
    print(*args, file=sys.stderr, **kwargs)


def get_form_data(htmlfile):
    data_clean = {}

    soup = BeautifulSoup(htmlfile.read(), 'html.parser')

    for field in soup.find_all('textarea'):
        if field['name'] in EXCLUDES:
            continue

        name = NAME_MAPPINGS.get(field['name'], field['name'])

        if 'msg' in name:
            continue

        data_clean[name] = [line for line in field.get_text().split('\n') if
                            line != ""]

    for field in soup.find_all('input'):
        if field['name'] in EXCLUDES:
            continue

        if field['type'].lower() in ('hidden', 'submit'):
            continue

        if field['type'].lower() == 'radio':
            if 'checked' not in field.attrs:
                continue

        name = NAME_MAPPINGS.get(field['name'], field['name'])

        if 'msg' in name:
            continue

        try:
            value = field['value']
            try:
                value = int(value)
            except ValueError:
                ...
        except KeyError:
            value = ''

        converter = TYPES.get(name)
        try:
            if converter is not None:
                value = converter(value)
        except (TypeError, KeyError, ValueError):
            from pdb import set_trace; set_trace()

            print('Type conversion error for key "{}": {}'.format(
                name, value), file=sys.stderr)

        data_clean[name] = value

    return data_clean


if __name__ == "__main__":
    parser = argparse.ArgumentParser(
        prog=sys.argv[0], description="Munge Mailman config data"
    )
    parser.add_argument("config", help="A list of files (named specifically) to import as configurations",
                        nargs='+',
                        action="append")
    args = parser.parse_args()

    globalconfig = {}
    for page in args.config[0]:
        msg(page)
        with open(page) as inf:
            result = get_form_data(inf)
        for key, value in result.items():
            if key in globalconfig:
                msg(f"warning - duplicate key {key}")
            else:
                globalconfig[key] = value

    # Handle the moderation policy.
    #
    # The mlist.default_member_action and mlist.default_nonmember_action enum
    # values are different in Mailman 2.1, because they have been merged into a
    # single enum in Mailman 3.
    #
    # Unmoderated lists used to have default_member_moderation set to a false
    # value; this translates to the Defer default action.  Moderated lists with
    # the default_member_moderation set to a true value used to store the
    # action in the member_moderation_action flag, the values were: 0==Hold,
    # 1=Reject, 2==Discard
    if bool(globalconfig.get('default_member_moderation', 0)):
        globalconfig['default_member_action'] = member_moderation_action_mapping(
            globalconfig.get('member_moderation_action')).name
        del globalconfig['default_member_moderation']
    else:
        globalconfig['default_member_action'] = Action.defer.name
    # Handle DMARC mitigations.
    # This would be straightforward except for from_is_list.  The issue
    # is in MM 2.1 the from_is_list action applies if dmarc_moderation_action
    # doesn't apply and they can be different.
    # We will map as follows:
    # from_is_list > dmarc_moderation_action
    #    dmarc_mitigate_action = from_is_list action
    #    dmarc_mitigate_unconditionally = True
    # from_is_list <= dmarc_moderation_action
    #    dmarc_mitigate_action = dmarc_moderation_action
    #    dmarc_mitigate_unconditionally = False
    # The text attributes are handled above.
    if (globalconfig.get('from_is_list', 0) >
            globalconfig.get('dmarc_moderation_action', 0)):
        globalconfig['dmarc_mitigate_action'] = dmarc_action_mapping(
            globalconfig.get('from_is_list', 0)).name
        globalconfig['dmarc_mitigate_unconditionally'] = True
    else:
        globalconfig['dmarc_mitigate_action'] = dmarc_action_mapping(
            globalconfig.get('dmarc_moderation_action', 0)).name
        globalconfig['dmarc_mitigate_unconditionally'] = False
    if 'from_is_list' in globalconfig.keys():
        del globalconfig['from_is_list']
    if 'dmarc_moderation_action' in globalconfig.keys():
        del globalconfig['dmarc_moderation_action']
    # Handle the archiving policy.  In MM2.1 there were two boolean options
    # but only three of the four possible states were valid.  Now there's just
    # an enum.
    if globalconfig.get('archive'):
        # For maximum safety, if for some strange reason there's no
        # archive_private key, treat the list as having private archives.
        if globalconfig.get('archive_private', True):
            globalconfig['archive_policy'] = ArchivePolicy.private.name
        else:
            globalconfig['archive_policy'] = ArchivePolicy.public.name
        del globalconfig['archive']
    else:
        globalconfig['archive_policy'] = ArchivePolicy.never.name
    if 'archive_private' in globalconfig.keys():
        del globalconfig['archive_private']

    # Handle roster visibility.
    mapping = member_roster_visibility_mapping(
        globalconfig.get('private_roster', None))
    if mapping is not None:
        globalconfig['member_roster_visibility'] = mapping.name
        del globalconfig['private_roster']

    print(json.dumps(globalconfig))