Merge branch 'trunk' of ssh://git.autonomic.zone:2222/autonomic-cooperative/civicrm-confdump into trunk

confdump.py

@@ -20,7 +20,8 @@ import traceback
 
 import MySQLdb as mysql
 
-from typing import Any, Dict
+from collections import defaultdict
+from typing import Any, Dict, List
 
 from civicrmapi4.civicrmapi4 import APIv4
 
@@ -62,8 +63,7 @@ LOAD_TRIVIAL = ["FinancialType",
                 "CustomGroup",
                 "OptionGroup",
                 "OptionValue",
-                "Domain",
+                "Domain",]
-                "Contact"]
 
 # This is a payment processor we can assign contribution pages to in order for them to work.
 # FIXME this seems to produce a non-working setup.
@@ -88,12 +88,28 @@ STANDIN_PAYMENT_PROCESSOR = {"id": "7",
                              "payment_instrument_id": "9"}
 
 
+CUSTOM_FIELD_TYPE_MAP = {
+    "String": "VARCHAR(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL",
+    "Integer": "INT(11) DEFAULT NULL",
+    "Int": "INT(11) DEFAULT NULL",
+    "Memo": "VARCHAR(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL",
+    "Date": "datetime DEFAULT NULL",
+    "Boolean": "tinyint(4) DEFAULT NULL",
+    "Money": "decimal(20, 2) DEFAULT NULL",
+    "ContactReference": "INT(10) DEFAULT NULL"
+}
+
 def object_to_table(instr: str) -> str:
+    """
+
+    """
     words = re.findall(r'[A-Z](?:[a-z]+|[A-Z]*(?=[A-Z]|$))', instr)
     return 'civicrm_' + '_'.join([x.lower() for x in words])
 
 
 def python_value_to_sql(val: Any) -> str:
+    """
+    """
     if type(val) == bool:
         if val:
             return "TRUE"
@@ -110,11 +126,29 @@ def python_value_to_sql(val: Any) -> str:
 
 
 def dict_to_insert(table: str, objdict: Dict) -> str:
+    """
+
+    """
     columns = tuple(x for x in objdict.keys())
     values = tuple(python_value_to_sql(objdict[x]) for x in columns)
     return "REPLACE INTO {} ({}) VALUES ({});".format(table, ",".join(columns), ",".join(values))
 
 
+def create_custom_value_table(custom_group_record: Dict, custom_value_rows: List[Dict]) -> str:
+    """
+    Return a sequence of SQL commands to create the custom value table based on a civicrm_custom_group record and a
+    series of value rows.
+
+    """
+    fields = ",".join(["{} {}".format(x["column_name"], CUSTOM_FIELD_TYPE_MAP[x["data_type"]]) for x in custom_value_rows])
+
+    return """DROP TABLE IF EXISTS {cgr_name}; CREATE TABLE {cgr_name} (
+    id INT(10) unsigned NOT NULL AUTO_INCREMENT PRIMARY KEY,
+    entity_id int(10) unsigned NOT NULL,
+    {fields},
+    UNIQUE(entity_id)
+    );""".format(cgr_name=custom_group_record['table_name'], fields=fields)
+
 
 def parse_arguments() -> argparse.Namespace:
     parser = argparse.ArgumentParser(prog="confdump.py",
@@ -238,6 +272,18 @@ def main() -> int:
             for row in indata:
                 query = dict_to_insert(table_name, row)
                 cursor.execute(query)
+
+        # create custom field data tables
+        custom_fields = defaultdict(list)
+        with open((args.input / "CustomField.json")) as inf:
+            for field in json.load(inf):
+                custom_fields[field["custom_group_id"]].append(field)
+        with open((args.input / "CustomGroup.json")) as inf:
+            custom_groups = json.load(inf)
+
+        for group in custom_groups:
+            cursor.execute(create_custom_value_table(group, custom_fields[group["id"]]))
+
         cursor.execute("SET FOREIGN_KEY_CHECKS=1;")
         cursor.close()