The key fix here is that the fields for merging to household are no longer added to the temporary table.
They are fully loaded in php anyway so there is no performance gain using a temp table.
Instead we iterate them in php as we process each row
// CRM-7675
const EXPORT_ROW_COUNT = 100000;
- /**
- * Key representing the head of household in the relationship array.
- *
- * e.g. ['8_b_a' => 'Household Member Is', '8_a_b = 'Household Member Of'.....]
- *
- * @var
- */
- protected static $relationshipTypes = [];
-
/**
* Get default return property for export based on mode
*
$processor = new CRM_Export_BAO_ExportProcessor($exportMode, $fields, $queryOperator, $mergeSameHousehold, $isPostalOnly);
$returnProperties = array();
- self::$relationshipTypes = $processor->getRelationshipTypes();
if ($fields) {
foreach ($fields as $key => $value) {
$fieldName = CRM_Utils_Array::value(1, $value);
- if (!$fieldName) {
+ if (!$fieldName || $processor->isHouseholdMergeRelationshipTypeKey($fieldName)) {
continue;
}
$returnProperties['id'] = 1;
}
- foreach ($returnProperties as $key => $value) {
- if (!$processor->isRelationshipTypeKey($key)) {
- foreach ($processor->getHouseholdRelationshipTypes() as $householdRelationshipType) {
- if (!in_array($key, ['location_type', 'im_provider'])) {
- $returnProperties[$householdRelationshipType][$key] = $value;
- }
- }
- // @todo - don't use returnProperties above.
- $processor->setHouseholdMergeReturnProperties($returnProperties[$householdRelationshipType]);
- }
- }
+ $processor->setHouseholdMergeReturnProperties(array_diff_key($returnProperties, array_fill_keys(['location_type', 'im_provider'], 1)));
}
self::buildRelatedContactArray($selectAll, $ids, $processor, $componentTable);
$count++;
$rowsThisIteration++;
$row = $processor->buildRow($query, $iterationDAO, $outputColumns, $metadata, $paymentDetails, $addPaymentHeader, $paymentTableId);
+ if ($row === FALSE) {
+ continue;
+ }
// add component info
// write the row to a file
self::mergeSameAddress($exportTempTable, $sqlColumns, $exportParams);
}
- // merge the records if they have corresponding households
- if ($mergeSameHousehold) {
- foreach ($processor->getHouseholdRelationshipTypes() as $householdRelationshipType) {
- self::mergeSameHousehold($exportTempTable, $sqlColumns, $householdRelationshipType);
- }
- }
-
// call export hook
CRM_Utils_Hook::export($exportTempTable, $headerRows, $sqlColumns, $exportMode, $componentTable, $ids);
return $merge;
}
- /**
- * Merge household record into the individual record
- * if exists
- *
- * @param string $exportTempTable
- * Temporary temp table that stores the records.
- * @param array $sqlColumns
- * Array of names of the table columns of the temp table.
- * @param string $prefix
- * Name of the relationship type that is prefixed to the table columns.
- */
- public static function mergeSameHousehold($exportTempTable, &$sqlColumns, $prefix) {
- $prefixColumn = $prefix . '_';
- $replaced = array();
-
- // name map of the non standard fields in header rows & sql columns
- $mappingFields = array(
- 'civicrm_primary_id' => 'id',
- 'provider_id' => 'im_service_provider',
- );
-
- //figure out which columns are to be replaced by which ones
- foreach ($sqlColumns as $columnNames => $dontCare) {
- if ($rep = CRM_Utils_Array::value($columnNames, $mappingFields)) {
- $replaced[$columnNames] = CRM_Utils_String::munge($prefixColumn . $rep, '_', 64);
- }
- else {
- $householdColName = CRM_Utils_String::munge($prefixColumn . $columnNames, '_', 64);
-
- if (!empty($sqlColumns[$householdColName])) {
- $replaced[$columnNames] = $householdColName;
- }
- }
- }
- $query = "UPDATE $exportTempTable SET ";
-
- $clause = array();
- foreach ($replaced as $from => $to) {
- $clause[] = "$from = $to ";
- unset($sqlColumns[$to]);
- }
- $query .= implode(",\n", $clause);
- $query .= " WHERE {$replaced['civicrm_primary_id']} != ''";
-
- CRM_Core_DAO::executeQuery($query);
-
- //drop the table columns that store redundant household info
- $dropQuery = "ALTER TABLE $exportTempTable ";
- foreach ($replaced as $householdColumns) {
- $dropClause[] = " DROP $householdColumns ";
- }
- $dropQuery .= implode(",\n", $dropClause);
-
- CRM_Core_DAO::executeQuery($dropQuery);
-
- // also drop the temp table if exists
- $sql = "DROP TABLE IF EXISTS {$exportTempTable}_temp";
- CRM_Core_DAO::executeQuery($sql);
-
- // clean up duplicate records
- $query = "
-CREATE TABLE {$exportTempTable}_temp SELECT *
-FROM {$exportTempTable}
-GROUP BY civicrm_primary_id ";
-
- CRM_Core_DAO::disableFullGroupByMode();
- CRM_Core_DAO::executeQuery($query);
- CRM_Core_DAO::reenableFullGroupByMode();
-
- $query = "DROP TABLE $exportTempTable";
- CRM_Core_DAO::executeQuery($query);
-
- $query = "ALTER TABLE {$exportTempTable}_temp RENAME TO {$exportTempTable}";
- CRM_Core_DAO::executeQuery($query);
- }
-
/**
* @param $exportTempTable
* @param $headerRows
*/
protected $relationshipReturnProperties = [];
+ /**
+ * IDs of households that have already been exported.
+ *
+ * @var array
+ */
+ protected $exportedHouseholds = [];
+
/**
* Get return properties by relationship.
* @return array
return isset($this->relatedContactValues[$relationshipType][$contactID][$field]) ? $this->relatedContactValues[$relationshipType][$contactID][$field] : '';
}
+ /**
+ * Get the id of the related household.
+ *
+ * @param int $contactID
+ * @param string $relationshipType
+ *
+ * @return int
+ */
+ public function getRelatedHouseholdID($contactID, $relationshipType) {
+ return $this->relatedContactValues[$relationshipType][$contactID]['id'];
+ }
+
+ /**
+ * Has the household already been exported.
+ *
+ * @param int $housholdContactID
+ *
+ * @return bool
+ */
+ public function isHouseholdExported($housholdContactID) {
+ return isset($this->exportedHouseholds[$housholdContactID]);
+
+ }
+
/**
* @return bool
*/
* @param $addPaymentHeader
* @param $paymentTableId
*
- * @return array
+ * @return array|bool
*/
public function buildRow($query, $iterationDAO, $outputColumns, $metadata, $paymentDetails, $addPaymentHeader, $paymentTableId) {
$phoneTypes = CRM_Core_PseudoConstant::get('CRM_Core_DAO_Phone', 'phone_type_id');
$imProviders = CRM_Core_PseudoConstant::get('CRM_Core_DAO_IM', 'provider_id');
$row = [];
+ $householdMergeRelationshipType = $this->getHouseholdMergeTypeForRow($iterationDAO->contact_id);
+ if ($householdMergeRelationshipType) {
+ $householdID = $this->getRelatedHouseholdID($iterationDAO->contact_id, $householdMergeRelationshipType);
+ if ($this->isHouseholdExported($householdID)) {
+ return FALSE;
+ }
+ foreach (array_keys($outputColumns) as $column) {
+ $row[$column] = $this->getRelationshipValue($householdMergeRelationshipType, $iterationDAO->contact_id, $column);
+ }
+ $this->markHouseholdExported($householdID);
+ return $row;
+ }
+
$query->convertToPseudoNames($iterationDAO);
//first loop through output columns so that we return what is required, and in same order.
foreach ($outputColumns as $field => $value) {
-
// add im_provider to $dao object
if ($field == 'im_provider' && property_exists($iterationDAO, 'provider_id')) {
$iterationDAO->im_provider = $iterationDAO->provider_id;
}
if ($this->isRelationshipTypeKey($field)) {
- foreach (array_keys($value) as $property) {
- if ($property === 'location') {
- // @todo just undo all this nasty location wrangling!
- foreach ($value['location'] as $locationKey => $locationFields) {
- foreach (array_keys($locationFields) as $locationField) {
- $fieldKey = str_replace(' ', '_', $locationKey . '-' . $locationField);
- $row[$field . '_' . $fieldKey] = $this->getRelationshipValue($field, $iterationDAO->contact_id, $fieldKey);
- }
- }
- }
- else {
- $row[$field . '_' . $property] = $this->getRelationshipValue($field, $iterationDAO->contact_id, $property);
- }
- }
+ $this->buildRelationshipFieldsForRow($row, $iterationDAO->contact_id, $value, $field);
}
else {
$row[$field] = $this->getTransformedFieldValue($field, $iterationDAO, $fieldValue, $metadata, $paymentDetails);
return $row;
}
+ /**
+ * If this row has a household whose details we should use get the relationship type key.
+ *
+ * @param $contactID
+ *
+ * @return bool
+ */
+ public function getHouseholdMergeTypeForRow($contactID) {
+ if (!$this->isMergeSameHousehold()) {
+ return FALSE;
+ }
+ foreach ($this->getHouseholdRelationshipTypes() as $relationshipType) {
+ if (isset($this->relatedContactValues[$relationshipType][$contactID])) {
+ return $relationshipType;
+ }
+ }
+ }
+
+ /**
+ * Mark the given household as already exported.
+ *
+ * @param $householdID
+ */
+ public function markHouseholdExported($householdID) {
+ $this->exportedHouseholds[$householdID] = $householdID;
+ }
+
/**
* @param $field
* @param $iterationDAO
return $params;
}
+ /**
+ * @param $row
+ * @param $contactID
+ * @param $value
+ * @param $field
+ */
+ protected function buildRelationshipFieldsForRow(&$row, $contactID, $value, $field) {
+ foreach (array_keys($value) as $property) {
+ if ($property === 'location') {
+ // @todo just undo all this nasty location wrangling!
+ foreach ($value['location'] as $locationKey => $locationFields) {
+ foreach (array_keys($locationFields) as $locationField) {
+ $fieldKey = str_replace(' ', '_', $locationKey . '-' . $locationField);
+ $row[$field . '_' . $fieldKey] = $this->getRelationshipValue($field, $contactID, $fieldKey);
+ }
+ }
+ }
+ else {
+ $row[$field . '_' . $property] = $this->getRelationshipValue($field, $contactID, $property);
+ }
+ }
+ }
+
}
* Test exporting relationships.
*/
public function testExportRelationshipsMergeToHouseholdAllFields() {
- $this->markTestIncomplete('Does not yet work under CI due to mysql limitation (number of columns in table). Works on some boxes');
list($householdID) = $this->setUpHousehold();
list($tableName) = CRM_Export_BAO_Export::exportComponents(
FALSE,
);
$dao = CRM_Core_DAO::executeQuery("SELECT * FROM {$tableName}");
while ($dao->fetch()) {
+ $this->assertEquals('Unit Test household', $dao->display_name);
$this->assertEquals('Portland', $dao->city);
$this->assertEquals('ME', $dao->state_province);
$this->assertEquals($householdID, $dao->civicrm_primary_id);
$this->assertEquals($householdID, $dao->civicrm_primary_id);
- $this->assertEquals('Unit Test Household', $dao->addressee);
- $this->assertEquals('Unit Test Household', $dao->display_name);
+ $this->assertEquals('Unit Test household', $dao->addressee);
+ $this->assertEquals(1, $dao->N);
}
}
projects / civicrm-core.git / commitdiff