Compare commits
12 Commits
3344394fbc
...
trunk
Author | SHA1 | Date | |
---|---|---|---|
7dbc4ae55c | |||
401572996d | |||
f874ed1206 | |||
d7cfc5419e | |||
62099efa85 | |||
851a784b36 | |||
107f10f704 | |||
a341654334 | |||
a91879cc08 | |||
e888c08d30 | |||
12d094c392 | |||
145a0f698d |
@ -5,9 +5,9 @@
|
||||
- Install requirements `pip install -r requirements.txt`
|
||||
- `cp example.env .env`
|
||||
- Edit `.env` and add creds for the production, test or development server you'll retrieve the data from.
|
||||
- Execute dump to retrieve base data: `env $(cat example.env) python ./confdump.py dump -o mydata INSTANCEPATH` where INSTANCEPATH is something like https://crm.dev.caat.org.uk/.
|
||||
- Execute dump to retrieve base data: `env $(cat .env) python ./confdump.py dump -o mydata INSTANCEPATH` where INSTANCEPATH is something like https://crm.dev.caat.org.uk/.
|
||||
- **IMPORTANT!!** - If you sourced the data from live, you must delete the `.env` file or delete the creds from inside it. If you leave them in it will negate the whole purpose of having this conf dump utility, which is to provide a way of creating a local site without any sensitive data.
|
||||
- If your civicrm is in a docker container, load data into running local instance with: `confdump.py mysql -i mydata/ -p 63306`
|
||||
- If your civicrm is in a docker container, load data into running local instance with: `python confdump.py mysql -i mydata/ -p 63306`
|
||||
- Otherwise use `confdump.py mysql -i mydata/ --host=<host> --db=<db> --user=<user> --password=<password>`
|
||||
- Clear the cache in CiviCRM:
|
||||
* DOCKER: `make shell`, and then inside the shell `cd /app; ./vendor/bin/drush cc all`
|
||||
|
63
confdump.py
63
confdump.py
@ -23,6 +23,7 @@ import MySQLdb as mysql
|
||||
from collections import defaultdict
|
||||
from typing import Any, Dict, List
|
||||
|
||||
import civicrmapi4
|
||||
from civicrmapi4.civicrmapi4 import APIv4
|
||||
|
||||
import phpserialize
|
||||
@ -39,7 +40,9 @@ DUMP_TRIVIAL = ["FinancialType",
|
||||
"CustomGroup",
|
||||
"OptionGroup",
|
||||
"OptionValue",
|
||||
"Domain"]
|
||||
"Domain",
|
||||
"SavedSearch",
|
||||
"Tag"]
|
||||
|
||||
|
||||
# "ContributionPage", needs payment processors & payment_processor column formatted correctly.
|
||||
@ -63,7 +66,15 @@ LOAD_TRIVIAL = ["FinancialType",
|
||||
"CustomGroup",
|
||||
"OptionGroup",
|
||||
"OptionValue",
|
||||
"Domain",]
|
||||
"Domain",
|
||||
"Contact",
|
||||
"SavedSearch",
|
||||
"Tag"]
|
||||
|
||||
WEIRD_LIST = [
|
||||
("civicrm_contact", "contact_sub_type"),
|
||||
("civicrm_custom_group", "extends_entity_column_value")
|
||||
]
|
||||
|
||||
# This is a payment processor we can assign contribution pages to in order for them to work.
|
||||
# FIXME this seems to produce a non-working setup.
|
||||
@ -107,6 +118,15 @@ def object_to_table(instr: str) -> str:
|
||||
return 'civicrm_' + '_'.join([x.lower() for x in words])
|
||||
|
||||
|
||||
def array_to_weird_array(val: List) -> str:
|
||||
if (val is None):
|
||||
return "NULL"
|
||||
|
||||
return '"\x01' + ('\x01'.join([str(x) for x in val])) + '\x01"'
|
||||
|
||||
def value_to_php_serialized(val: Any) -> str:
|
||||
return "'{}'".format(mysql.escape_string(phpserialize.dumps(val).decode()).decode())
|
||||
|
||||
def python_value_to_sql(val: Any) -> str:
|
||||
"""
|
||||
"""
|
||||
@ -119,9 +139,10 @@ def python_value_to_sql(val: Any) -> str:
|
||||
if (isinstance(val, (int, float, complex))):
|
||||
return str(val)
|
||||
if (type(val) == list):
|
||||
return "'{}'".format(",".join([str(v) for v in val]))
|
||||
# weird list serialization
|
||||
return "'" + ','.join([str(x) for x in val]) + "'"
|
||||
if (type(val) == dict):
|
||||
return "'{}'".format(mysql.escape_string(phpserialize.dumps(val).decode()).decode())
|
||||
return value_to_php_serialized(val)
|
||||
return "'{}'".format(mysql.escape_string(val).decode())
|
||||
|
||||
|
||||
@ -130,7 +151,15 @@ def dict_to_insert(table: str, objdict: Dict) -> str:
|
||||
|
||||
"""
|
||||
columns = tuple(x for x in objdict.keys())
|
||||
values = tuple(python_value_to_sql(objdict[x]) for x in columns)
|
||||
values = list()
|
||||
for col in columns:
|
||||
# any weird array we have to process here if there are others
|
||||
if (table, col) in WEIRD_LIST:
|
||||
values.append(array_to_weird_array(objdict[col]))
|
||||
elif table == "civicrm_saved_search" and col == "form_values":
|
||||
values.append(value_to_php_serialized(objdict[col]))
|
||||
else:
|
||||
values.append(python_value_to_sql(objdict[col]))
|
||||
return "REPLACE INTO {} ({}) VALUES ({});".format(table, ",".join(columns), ",".join(values))
|
||||
|
||||
|
||||
@ -194,6 +223,16 @@ def parse_arguments() -> argparse.Namespace:
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def wrap_api_get(api, obj, args=None):
|
||||
if args is None:
|
||||
args = []
|
||||
try:
|
||||
return api.get(obj, args)
|
||||
except civicrmapi4.civicrmapi4.CallFailed:
|
||||
logging.error("Could not fetch {}".format(obj))
|
||||
return []
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parse_arguments()
|
||||
|
||||
@ -233,15 +272,14 @@ def main() -> int:
|
||||
|
||||
for table in DUMP_TRIVIAL:
|
||||
output = args.output / (table + ".json")
|
||||
data = api.get(table)
|
||||
data = wrap_api_get(api, table)
|
||||
if data:
|
||||
print("dumping", table)
|
||||
with output.open("w") as of:
|
||||
of.write(json.dumps(data))
|
||||
|
||||
# dump org contacts
|
||||
# dump org contacts
|
||||
output = args.output / ("Contact.json")
|
||||
data = api.get("Contact", where=[["contact_sub_type", "=", "Political_Party"]])
|
||||
data = wrap_api_get(api, "Contact", where=[["contact_sub_type", "CONTAINS", "Political_Party"]])
|
||||
if data:
|
||||
print("dumping parties")
|
||||
with output.open("w") as of:
|
||||
@ -258,10 +296,9 @@ def main() -> int:
|
||||
cursor.execute("SET FOREIGN_KEY_CHECKS=0;")
|
||||
|
||||
query = dict_to_insert("civicrm_payment_processor", STANDIN_PAYMENT_PROCESSOR)
|
||||
print(query)
|
||||
print(cursor.execute(query))
|
||||
for table in LOAD_TRIVIAL:
|
||||
# exceptions that require extra processing
|
||||
print(table)
|
||||
with open((args.input / (table + ".json"))) as inf:
|
||||
indata = json.load(inf)
|
||||
table_name = object_to_table(table)
|
||||
@ -269,9 +306,11 @@ def main() -> int:
|
||||
if table == "ContributionPage":
|
||||
for row in indata:
|
||||
row['payment_processor'] = STANDIN_PAYMENT_PROCESSOR_ID
|
||||
tot = 0;
|
||||
for row in indata:
|
||||
query = dict_to_insert(table_name, row)
|
||||
cursor.execute(query)
|
||||
tot += cursor.execute(query)
|
||||
print(tot)
|
||||
|
||||
# create custom field data tables
|
||||
custom_fields = defaultdict(list)
|
||||
|
@ -1,3 +1,3 @@
|
||||
-e git+https://git.autonomic.zone/cas/civicrmapi4#egg=civicrmapi4
|
||||
-e git+https://git.autonomic.zone/autonomic-cooperative/civicrmapi4#egg=civicrmapi4
|
||||
phpserialize
|
||||
mysqlclient~=1.4.6
|
||||
|
Reference in New Issue
Block a user