Fix potential import error

When it was "=" it didn't get ones with multiple subtypes.

.gitignore

@@ -3,6 +3,7 @@
 # security leaks?;
 .env
 mydata
+confdump.venv
 
 # Byte-compiled / optimized / DLL files
 __pycache__/

README.md

@@ -5,9 +5,9 @@
 - Install requirements `pip install -r requirements.txt`
 - `cp example.env .env` 
 - Edit `.env` and add creds for the production, test or development server you'll retrieve the data from. 
-- Execute dump to retrieve base data: `env $(cat example.env) python ./confdump.py dump -o mydata INSTANCEPATH` where INSTANCEPATH is something like https://crm.dev.caat.org.uk/.
+- Execute dump to retrieve base data: `env $(cat .env) python ./confdump.py dump -o mydata INSTANCEPATH` where INSTANCEPATH is something like https://crm.dev.caat.org.uk/.
 - **IMPORTANT!!** - If you sourced the data from live, you must delete the `.env` file or delete the creds from inside it. If you leave them in it will negate the whole purpose of having this conf dump utility, which is to provide a way of creating a local site without any sensitive data.
-- If your civicrm is in a docker container, load data into running local instance with: `confdump.py mysql -i mydata/ -p 63306`
+- If your civicrm is in a docker container, load data into running local instance with: `python confdump.py mysql -i mydata/ -p 63306`
 - Otherwise use `confdump.py mysql -i mydata/ --host=<host> --db=<db> --user=<user> --password=<password>`
 - Clear the cache in CiviCRM: 
   * DOCKER: `make shell`, and then inside the shell `cd /app; ./vendor/bin/drush cc all`

confdump.py

@@ -20,8 +20,10 @@ import traceback
 
 import MySQLdb as mysql
 
-from typing import Any, Dict
+from collections import defaultdict
+from typing import Any, Dict, List
 
+import civicrmapi4
 from civicrmapi4.civicrmapi4 import APIv4
 
 import phpserialize
@@ -38,7 +40,9 @@ DUMP_TRIVIAL = ["FinancialType",
                 "CustomGroup",
                 "OptionGroup",
                 "OptionValue",
-                "Domain"]
+                "Domain",
+                "SavedSearch",
+                "Tag"]
 
 
 # "ContributionPage", needs payment processors & payment_processor column formatted correctly.
@@ -63,7 +67,14 @@ LOAD_TRIVIAL = ["FinancialType",
                 "OptionGroup",
                 "OptionValue",
                 "Domain",
-                "Contact"]
+                "Contact",
+                "SavedSearch",
+                "Tag"]
+
+WEIRD_LIST = [
+    ("civicrm_contact", "contact_sub_type"),
+    ("civicrm_custom_group", "extends_entity_column_value")
+]
 
 # This is a payment processor we can assign contribution pages to in order for them to work.
 # FIXME this seems to produce a non-working setup.
@@ -88,12 +99,37 @@ STANDIN_PAYMENT_PROCESSOR = {"id": "7",
                              "payment_instrument_id": "9"}
 
 
+CUSTOM_FIELD_TYPE_MAP = {
+    "String": "VARCHAR(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL",
+    "Integer": "INT(11) DEFAULT NULL",
+    "Int": "INT(11) DEFAULT NULL",
+    "Memo": "VARCHAR(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL",
+    "Date": "datetime DEFAULT NULL",
+    "Boolean": "tinyint(4) DEFAULT NULL",
+    "Money": "decimal(20, 2) DEFAULT NULL",
+    "ContactReference": "INT(10) DEFAULT NULL"
+}
+
 def object_to_table(instr: str) -> str:
+    """
+
+    """
     words = re.findall(r'[A-Z](?:[a-z]+|[A-Z]*(?=[A-Z]|$))', instr)
     return 'civicrm_' + '_'.join([x.lower() for x in words])
 
 
+def array_to_weird_array(val: List) -> str:
+    if (val is None):
+        return "NULL"
+
+    return '"\x01' + ('\x01'.join([str(x) for x in val])) + '\x01"'
+
+def value_to_php_serialized(val: Any) -> str:
+    return "'{}'".format(mysql.escape_string(phpserialize.dumps(val).decode()).decode())
+
 def python_value_to_sql(val: Any) -> str:
+    """
+    """
     if type(val) == bool:
         if val:
             return "TRUE"
@@ -103,18 +139,45 @@ def python_value_to_sql(val: Any) -> str:
     if (isinstance(val, (int, float, complex))):
         return str(val)
     if (type(val) == list):
-        return "'{}'".format(",".join([str(v) for v in val]))
+        # weird list serialization
+        return "'" + ','.join([str(x) for x in val]) + "'"
     if (type(val) == dict):
-        return "'{}'".format(mysql.escape_string(phpserialize.dumps(val).decode()).decode())
+        return value_to_php_serialized(val)
     return "'{}'".format(mysql.escape_string(val).decode())
 
 
 def dict_to_insert(table: str, objdict: Dict) -> str:
+    """
+
+    """
     columns = tuple(x for x in objdict.keys())
-    values = tuple(python_value_to_sql(objdict[x]) for x in columns)
+    values = list()
+    for col in columns:
+        # any weird array we have to process here if there are others
+        if (table, col) in WEIRD_LIST:
+            values.append(array_to_weird_array(objdict[col]))
+        elif table == "civicrm_saved_search" and col == "form_values":
+            values.append(value_to_php_serialized(objdict[col]))
+        else:
+            values.append(python_value_to_sql(objdict[col]))
     return "REPLACE INTO {} ({}) VALUES ({});".format(table, ",".join(columns), ",".join(values))
 
 
+def create_custom_value_table(custom_group_record: Dict, custom_value_rows: List[Dict]) -> str:
+    """
+    Return a sequence of SQL commands to create the custom value table based on a civicrm_custom_group record and a
+    series of value rows.
+
+    """
+    fields = ",".join(["{} {}".format(x["column_name"], CUSTOM_FIELD_TYPE_MAP[x["data_type"]]) for x in custom_value_rows])
+
+    return """DROP TABLE IF EXISTS {cgr_name}; CREATE TABLE {cgr_name} (
+    id INT(10) unsigned NOT NULL AUTO_INCREMENT PRIMARY KEY,
+    entity_id int(10) unsigned NOT NULL,
+    {fields},
+    UNIQUE(entity_id)
+    );""".format(cgr_name=custom_group_record['table_name'], fields=fields)
+
 
 def parse_arguments() -> argparse.Namespace:
     parser = argparse.ArgumentParser(prog="confdump.py",
@@ -160,6 +223,16 @@ def parse_arguments() -> argparse.Namespace:
     return parser.parse_args()
 
 
+def wrap_api_get(api, obj, args=None):
+    if args is None:
+        args = []
+    try:
+        return api.get(obj, args)
+    except civicrmapi4.civicrmapi4.CallFailed:
+        logging.error("Could not fetch {}".format(obj))
+    return []
+
+
 def main() -> int:
     args = parse_arguments()
 
@@ -199,15 +272,14 @@ def main() -> int:
 
         for table in DUMP_TRIVIAL:
             output = args.output / (table + ".json")
-            data = api.get(table)
+            data = wrap_api_get(api, table)
             if data:
                 print("dumping", table)
                 with output.open("w") as of:
                     of.write(json.dumps(data))
-
-        # dump org contacts
+                    # dump org contacts
         output = args.output / ("Contact.json")
-        data = api.get("Contact", where=[["contact_sub_type", "=", "Political_Party"]])
+        data = wrap_api_get(api, "Contact", where=[["contact_sub_type", "CONTAINS", "Political_Party"]])
         if data:
             print("dumping parties")
             with output.open("w") as of:
@@ -218,16 +290,15 @@ def main() -> int:
             print("input directory does not exist")
             return 1
 
-        connection = mysql.connect(host=args.host, port=args.port, user=args.user, password=args.password, db=args.db)
+        connection = mysql.connect(host=args.host, port=args.port, user=args.user, password=args.password, db=args.db, charset='utf8', use_unicode=True)
         connection.autocommit(True)
         cursor = connection.cursor()
         cursor.execute("SET FOREIGN_KEY_CHECKS=0;")
 
         query = dict_to_insert("civicrm_payment_processor", STANDIN_PAYMENT_PROCESSOR)
-        print(query)
-        print(cursor.execute(query))
         for table in LOAD_TRIVIAL:
             # exceptions that require extra processing
+            print(table)
             with open((args.input / (table + ".json"))) as inf:
                 indata = json.load(inf)
             table_name = object_to_table(table)
@@ -235,9 +306,23 @@ def main() -> int:
             if table == "ContributionPage":
                 for row in indata:
                     row['payment_processor'] = STANDIN_PAYMENT_PROCESSOR_ID
+            tot = 0;
             for row in indata:
                 query = dict_to_insert(table_name, row)
-                cursor.execute(query)
+                tot += cursor.execute(query)
+            print(tot)
+
+        # create custom field data tables
+        custom_fields = defaultdict(list)
+        with open((args.input / "CustomField.json")) as inf:
+            for field in json.load(inf):
+                custom_fields[field["custom_group_id"]].append(field)
+        with open((args.input / "CustomGroup.json")) as inf:
+            custom_groups = json.load(inf)
+
+        for group in custom_groups:
+            cursor.execute(create_custom_value_table(group, custom_fields[group["id"]]))
+
         cursor.execute("SET FOREIGN_KEY_CHECKS=1;")
         cursor.close()
 

requirements.txt

@@ -1,3 +1,3 @@
--e git+https://git.autonomic.zone/cas/civicrmapi4#egg=civicrmapi4
+-e git+https://git.autonomic.zone/autonomic-cooperative/civicrmapi4#egg=civicrmapi4
 phpserialize
 mysqlclient~=1.4.6