286 lines
9.9 KiB
Python
286 lines
9.9 KiB
Python
import argparse
|
|
import json
|
|
import sys
|
|
from functools import partial
|
|
from copy import deepcopy
|
|
from math import floor
|
|
|
|
from bs4 import BeautifulSoup
|
|
|
|
from mailman.interfaces.action import Action
|
|
from mailman.interfaces.archiver import ArchivePolicy
|
|
from mailman.interfaces.digests import DigestFrequency
|
|
from mailman.utilities.importer import (
|
|
NAME_MAPPINGS,
|
|
EXCLUDES as EXCLUDES_ORIGINAL,
|
|
TYPES as TYPES_ORIGINAL,
|
|
member_moderation_action_mapping,
|
|
dmarc_action_mapping,
|
|
member_roster_visibility_mapping
|
|
)
|
|
from mailman.interfaces.nntp import NewsgroupModeration
|
|
from mailman.interfaces.mailinglist import (
|
|
DMARCMitigateAction,
|
|
Personalization,
|
|
ReplyToMunging,
|
|
SubscriptionPolicy,
|
|
)
|
|
|
|
KEYFILTER = ('submit')
|
|
|
|
EXCLUDES = set(
|
|
tuple(EXCLUDES_ORIGINAL) + tuple({
|
|
# from `convert_to_uri` in mailman's `importer.py`
|
|
'goodbye_msg': 'list:user:notice:goodbye',
|
|
'msg_header': 'list:member:regular:header',
|
|
'msg_footer': 'list:member:regular:footer',
|
|
'digest_header': 'list:member:digest:header',
|
|
'digest_footer': 'list:member:digest:footer',
|
|
}.keys()) + (
|
|
# some settings to manually skip, which don't seem to be handled by the
|
|
# conversion script
|
|
'archive_volume_frequency',
|
|
'autoresponse_postings_text_upload',
|
|
'autoresponse_admin_text_upload',
|
|
'autoresponse_request_text_upload',
|
|
'bounce_notify_owner_on_bounce_increment',
|
|
'nondigestable',
|
|
'digestable',
|
|
'digest_is_default',
|
|
'mime_is_default_digest',
|
|
'digest_size_threshhold',
|
|
'_new_volume',
|
|
'_send_digest_now',
|
|
'umbrella_list',
|
|
'umbrella_member_suffix',
|
|
'send_reminders',
|
|
'admin_member_chunksize',
|
|
'host_name',
|
|
'new_member_options',
|
|
'include_sender_header',
|
|
'drop_cc',
|
|
'available_languages',
|
|
'encode_ascii_prefixes',
|
|
'regular_exclude_lists',
|
|
'regular_include_lists',
|
|
'scrub_nondigest',
|
|
'regular_exclude_ignore',
|
|
'subscribe_auto_approval',
|
|
'unsubscribe_policy',
|
|
'obscure_addresses',
|
|
'member_moderation_notice',
|
|
'equivalent_domains',
|
|
'nonmember_rejection_notice',
|
|
'default_member_moderation',
|
|
'member_verbosity_threshold',
|
|
'member_verbosity_interval',
|
|
'member_moderation_action',
|
|
# TODO investigate
|
|
'dmarc_quarantine_moderation_action',
|
|
'dmarc_none_moderation_action',
|
|
'forward_auto_discards',
|
|
'hdrfilter_rebox_01',
|
|
'hdrfilter_action_01',
|
|
'bounce_matching_headers',
|
|
'topic_rebox_01',
|
|
'topic_desc_01',
|
|
'topic_box_01',
|
|
# None of the lists has topic filtering enabled
|
|
'topics_enabled',
|
|
'topics_bodylines_limit',
|
|
# ... or DMARC addresses
|
|
'dmarc_moderation_addresses',
|
|
))
|
|
|
|
TYPES = deepcopy(TYPES_ORIGINAL)
|
|
|
|
TYPES['process_bounces'] = lambda x: bool(x)
|
|
TYPES['bounce_notify_owner_on_disable'] = lambda x: bool(x)
|
|
TYPES['bounce_notify_owner_on_removal'] = lambda x: bool(x)
|
|
TYPES['filter_content'] = lambda x: bool(x)
|
|
TYPES['collapse_alternatives'] = lambda x: bool(x)
|
|
TYPES['convert_html_to_plaintext'] = lambda x: bool(x)
|
|
TYPES['digest_send_periodic'] = lambda x: bool(x)
|
|
TYPES['anonymous_list'] = lambda x: bool(x)
|
|
TYPES['first_strip_reply_to'] = lambda x: bool(x)
|
|
TYPES['send_welcome_message'] = lambda x: bool(x)
|
|
TYPES['send_goodbye_message'] = lambda x: bool(x)
|
|
TYPES['admin_immed_notify'] = lambda x: bool(x)
|
|
TYPES['admin_notify_mchanges'] = lambda x: bool(x)
|
|
TYPES['respond_to_post_requests'] = lambda x: bool(x)
|
|
TYPES['emergency'] = lambda x: bool(x)
|
|
TYPES['administrivia'] = lambda x: bool(x)
|
|
TYPES['include_rfc2369_headers'] = lambda x: bool(x)
|
|
TYPES['allow_list_posts'] = lambda x: bool(x)
|
|
TYPES['advertised'] = lambda x: bool(x)
|
|
TYPES['require_explicit_destination'] = lambda x: bool(x)
|
|
TYPES['bounce_score_threshold'] = lambda x: floor(float(x))
|
|
|
|
TYPES_EXTRA = {
|
|
'autorespond_requests': lambda x, y: x(y).name,
|
|
'autorespond_owner': lambda x, y: x(y).name,
|
|
'autorespond_postings': lambda x, y: x(y).name,
|
|
'autoresponse_grace_period': lambda x, y: '{}d'.format(x(y).days),
|
|
'bounce_info_stale_after': lambda x, y: '{}s'.format(x(y).seconds),
|
|
'bounce_you_are_disabled_warnings_interval': lambda x, y: '{}s'.format(x(y).seconds),
|
|
'forward_unrecognized_bounces_to': lambda x, y: x(y).name,
|
|
'filter_action': lambda x, y: x(y).name,
|
|
'digest_volume_frequency': lambda x, y: x(y).name,
|
|
'reply_goes_to_list': lambda x, y: x(y).name,
|
|
'newsgroup_moderation': lambda x, y: x(y).name,
|
|
'subscription_policy': lambda x, y: x(y).name,
|
|
'default_nonmember_action': lambda x, y: x(y).name,
|
|
}
|
|
|
|
for key, func in TYPES_EXTRA.items():
|
|
TYPES[key] = partial(func, TYPES_ORIGINAL[key])
|
|
|
|
|
|
def msg(*args, **kwargs):
|
|
print(*args, file=sys.stderr, **kwargs)
|
|
|
|
|
|
def get_form_data(htmlfile):
|
|
data_clean = {}
|
|
|
|
soup = BeautifulSoup(htmlfile.read(), 'html.parser')
|
|
|
|
for field in soup.find_all('textarea'):
|
|
if field['name'] in EXCLUDES:
|
|
continue
|
|
|
|
name = NAME_MAPPINGS.get(field['name'], field['name'])
|
|
|
|
if 'msg' in name:
|
|
continue
|
|
|
|
data_clean[name] = [line for line in field.get_text().split('\n') if
|
|
line != ""]
|
|
|
|
for field in soup.find_all('input'):
|
|
if field['name'] in EXCLUDES:
|
|
continue
|
|
|
|
if field['type'].lower() in ('hidden', 'submit'):
|
|
continue
|
|
|
|
if field['type'].lower() == 'radio':
|
|
if 'checked' not in field.attrs:
|
|
continue
|
|
|
|
name = NAME_MAPPINGS.get(field['name'], field['name'])
|
|
|
|
if 'msg' in name:
|
|
continue
|
|
|
|
try:
|
|
value = field['value']
|
|
try:
|
|
value = int(value)
|
|
except ValueError:
|
|
...
|
|
except KeyError:
|
|
value = ''
|
|
|
|
converter = TYPES.get(name)
|
|
try:
|
|
if converter is not None:
|
|
value = converter(value)
|
|
except (TypeError, KeyError, ValueError):
|
|
from pdb import set_trace; set_trace()
|
|
|
|
print('Type conversion error for key "{}": {}'.format(
|
|
name, value), file=sys.stderr)
|
|
|
|
data_clean[name] = value
|
|
|
|
return data_clean
|
|
|
|
|
|
if __name__ == "__main__":
|
|
parser = argparse.ArgumentParser(
|
|
prog=sys.argv[0], description="Munge Mailman config data"
|
|
)
|
|
parser.add_argument("config", help="A list of files (named specifically) to import as configurations",
|
|
nargs='+',
|
|
action="append")
|
|
args = parser.parse_args()
|
|
|
|
globalconfig = {}
|
|
for page in args.config[0]:
|
|
msg(page)
|
|
with open(page) as inf:
|
|
result = get_form_data(inf)
|
|
for key, value in result.items():
|
|
if key in globalconfig:
|
|
msg(f"warning - duplicate key {key}")
|
|
else:
|
|
globalconfig[key] = value
|
|
|
|
# Handle the moderation policy.
|
|
#
|
|
# The mlist.default_member_action and mlist.default_nonmember_action enum
|
|
# values are different in Mailman 2.1, because they have been merged into a
|
|
# single enum in Mailman 3.
|
|
#
|
|
# Unmoderated lists used to have default_member_moderation set to a false
|
|
# value; this translates to the Defer default action. Moderated lists with
|
|
# the default_member_moderation set to a true value used to store the
|
|
# action in the member_moderation_action flag, the values were: 0==Hold,
|
|
# 1=Reject, 2==Discard
|
|
if bool(globalconfig.get('default_member_moderation', 0)):
|
|
globalconfig['default_member_action'] = member_moderation_action_mapping(
|
|
globalconfig.get('member_moderation_action')).name
|
|
del globalconfig['default_member_moderation']
|
|
else:
|
|
globalconfig['default_member_action'] = Action.defer.name
|
|
# Handle DMARC mitigations.
|
|
# This would be straightforward except for from_is_list. The issue
|
|
# is in MM 2.1 the from_is_list action applies if dmarc_moderation_action
|
|
# doesn't apply and they can be different.
|
|
# We will map as follows:
|
|
# from_is_list > dmarc_moderation_action
|
|
# dmarc_mitigate_action = from_is_list action
|
|
# dmarc_mitigate_unconditionally = True
|
|
# from_is_list <= dmarc_moderation_action
|
|
# dmarc_mitigate_action = dmarc_moderation_action
|
|
# dmarc_mitigate_unconditionally = False
|
|
# The text attributes are handled above.
|
|
if (globalconfig.get('from_is_list', 0) >
|
|
globalconfig.get('dmarc_moderation_action', 0)):
|
|
globalconfig['dmarc_mitigate_action'] = dmarc_action_mapping(
|
|
globalconfig.get('from_is_list', 0)).name
|
|
globalconfig['dmarc_mitigate_unconditionally'] = True
|
|
else:
|
|
globalconfig['dmarc_mitigate_action'] = dmarc_action_mapping(
|
|
globalconfig.get('dmarc_moderation_action', 0)).name
|
|
globalconfig['dmarc_mitigate_unconditionally'] = False
|
|
if 'from_is_list' in globalconfig.keys():
|
|
del globalconfig['from_is_list']
|
|
if 'dmarc_moderation_action' in globalconfig.keys():
|
|
del globalconfig['dmarc_moderation_action']
|
|
# Handle the archiving policy. In MM2.1 there were two boolean options
|
|
# but only three of the four possible states were valid. Now there's just
|
|
# an enum.
|
|
if globalconfig.get('archive'):
|
|
# For maximum safety, if for some strange reason there's no
|
|
# archive_private key, treat the list as having private archives.
|
|
if globalconfig.get('archive_private', True):
|
|
globalconfig['archive_policy'] = ArchivePolicy.private.name
|
|
else:
|
|
globalconfig['archive_policy'] = ArchivePolicy.public.name
|
|
del globalconfig['archive']
|
|
else:
|
|
globalconfig['archive_policy'] = ArchivePolicy.never.name
|
|
if 'archive_private' in globalconfig.keys():
|
|
del globalconfig['archive_private']
|
|
|
|
# Handle roster visibility.
|
|
mapping = member_roster_visibility_mapping(
|
|
globalconfig.get('private_roster', None))
|
|
if mapping is not None:
|
|
globalconfig['member_roster_visibility'] = mapping.name
|
|
del globalconfig['private_roster']
|
|
|
|
print(json.dumps(globalconfig))
|