3 +--------------------------------------------------------------------+
5 +--------------------------------------------------------------------+
6 | Copyright CiviCRM LLC (c) 2004-2019 |
7 +--------------------------------------------------------------------+
8 | This file is a part of CiviCRM. |
10 | CiviCRM is free software; you can copy, modify, and distribute it |
11 | under the terms of the GNU Affero General Public License |
12 | Version 3, 19 November 2007 and the CiviCRM Licensing Exception. |
14 | CiviCRM is distributed in the hope that it will be useful, but |
15 | WITHOUT ANY WARRANTY; without even the implied warranty of |
16 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. |
17 | See the GNU Affero General Public License for more details. |
19 | You should have received a copy of the GNU Affero General Public |
20 | License and the CiviCRM Licensing Exception along |
21 | with this program; if not, contact CiviCRM LLC |
22 | at info[AT]civicrm[DOT]org. If you have questions about the |
23 | GNU Affero General Public License or the licensing of CiviCRM, |
24 | see the CiviCRM license FAQ at http://civicrm.org/licensing |
25 +--------------------------------------------------------------------+
31 * @copyright CiviCRM LLC (c) 2004-2019
35 * This class contains the functions for Component export
38 class CRM_Export_BAO_Export
{
39 // increase this number a lot to avoid making too many queries
40 // LIMIT is not much faster than a no LIMIT query
42 const EXPORT_ROW_COUNT
= 100000;
45 * Get Export component
47 * @param int $exportMode
51 * CiviCRM Export Component
53 public static function exportComponent($exportMode) {
54 switch ($exportMode) {
55 case CRM_Export_Form_Select
::CONTRIBUTE_EXPORT
:
56 $component = 'civicrm_contribution';
59 case CRM_Export_Form_Select
::EVENT_EXPORT
:
60 $component = 'civicrm_participant';
63 case CRM_Export_Form_Select
::MEMBER_EXPORT
:
64 $component = 'civicrm_membership';
67 case CRM_Export_Form_Select
::PLEDGE_EXPORT
:
68 $component = 'civicrm_pledge';
71 case CRM_Export_Form_Select
::GRANT_EXPORT
:
72 $component = 'civicrm_grant';
79 * Get the list the export fields.
81 * @param int $selectAll
82 * User preference while export.
85 * @param array $params
86 * Associated array of fields.
87 * @param string $order
89 * @param array $fields
90 * Associated array of fields.
91 * @param array $moreReturnProperties
92 * Additional return fields.
93 * @param int $exportMode
95 * @param string $componentClause
97 * @param string $componentTable
99 * @param bool $mergeSameAddress
100 * Merge records if they have same address.
101 * @param bool $mergeSameHousehold
102 * Merge records if they belong to the same household.
104 * @param array $exportParams
105 * @param string $queryOperator
108 * An array can be requested from within a unit test.
110 * @throws \CRM_Core_Exception
112 public static function exportComponents(
118 $moreReturnProperties = NULL,
119 $exportMode = CRM_Export_Form_Select
::CONTACT_EXPORT
,
120 $componentClause = NULL,
121 $componentTable = NULL,
122 $mergeSameAddress = FALSE,
123 $mergeSameHousehold = FALSE,
125 $queryOperator = 'AND'
129 isset($exportParams['postal_mailing_export']['postal_mailing_export']) &&
130 $exportParams['postal_mailing_export']['postal_mailing_export'] == 1
133 if (!$selectAll && $componentTable && !empty($exportParams['additional_group'])) {
134 // If an Additional Group is selected, then all contacts in that group are
135 // added to the export set (filtering out duplicates).
136 // Really - the calling function could do this ... just saying
137 // @todo take a whip to the calling function.
138 CRM_Core_DAO
::executeQuery("
139 INSERT INTO {$componentTable} SELECT distinct gc.contact_id FROM civicrm_group_contact gc WHERE gc.group_id = {$exportParams['additional_group']} ON DUPLICATE KEY UPDATE {$componentTable}.contact_id = gc.contact_id"
142 // rectify params to what proximity search expects if there is a value for prox_distance
144 // @todo - move this back to the calling functions
145 if (!empty($params)) {
146 CRM_Contact_BAO_ProximityQuery
::fixInputParams($params);
148 // @todo everything from this line up should go back to the calling functions.
149 $processor = new CRM_Export_BAO_ExportProcessor($exportMode, $fields, $queryOperator, $mergeSameHousehold, $isPostalOnly, $mergeSameAddress);
150 if ($moreReturnProperties) {
151 $processor->setAdditionalRequestedReturnProperties($moreReturnProperties);
153 $processor->setComponentTable($componentTable);
154 $processor->setComponentClause($componentClause);
156 list($query, $queryString) = $processor->runQuery($params, $order);
158 // This perhaps only needs calling when $mergeSameHousehold == 1
159 self
::buildRelatedContactArray($selectAll, $ids, $processor, $componentTable);
161 $addPaymentHeader = FALSE;
163 list($outputColumns, $metadata) = $processor->getExportStructureArrays();
165 if ($processor->isMergeSameAddress()) {
166 //make sure the addressee fields are selected
167 //while using merge same address feature
168 // some columns are required for assistance incase they are not already present
169 $exportParams['merge_same_address']['temp_columns'] = $processor->getAdditionalFieldsForSameAddressMerge();
170 // This is silly - we should do this at the point when the array is used...
171 if (isset($exportParams['merge_same_address']['temp_columns']['id'])) {
172 unset($exportParams['merge_same_address']['temp_columns']['id']);
173 $exportParams['merge_same_address']['temp_columns']['civicrm_primary_id'] = 1;
175 // @todo - this is a temp fix - ideally later we don't set stuff only to unset it.
176 // test exists covering this...
177 foreach (array_keys($exportParams['merge_same_address']['temp_columns']) as $field) {
178 $processor->setColumnAsCalculationOnly($field);
182 $paymentDetails = [];
183 if ($processor->isExportPaymentFields()) {
184 // get payment related in for event and members
185 $paymentDetails = CRM_Contribute_BAO_Contribution
::getContributionDetails($exportMode, $ids);
186 //get all payment headers.
187 // If we haven't selected specific payment fields, load in all the
189 if (!$processor->isExportSpecifiedPaymentFields()) {
190 if (!empty($paymentDetails)) {
191 $addPaymentHeader = TRUE;
192 foreach (array_keys($processor->getPaymentHeaders()) as $paymentField) {
193 $processor->addOutputSpecification($paymentField);
199 $componentDetails = [];
201 $rowCount = self
::EXPORT_ROW_COUNT
;
203 // we write to temp table often to avoid using too much memory
208 $headerRows = $processor->getHeaderRows();
209 $sqlColumns = $processor->getSQLColumns();
210 $processor->setTemporaryTable(self
::createTempTable($sqlColumns));
211 $limitReached = FALSE;
213 while (!$limitReached) {
214 $limitQuery = "{$queryString} LIMIT {$offset}, {$rowCount}";
215 CRM_Core_DAO
::disableFullGroupByMode();
216 $iterationDAO = CRM_Core_DAO
::executeQuery($limitQuery);
217 CRM_Core_DAO
::reenableFullGroupByMode();
218 // If this is less than our limit by the end of the iteration we do not need to run the query again to
219 // check if some remain.
220 $rowsThisIteration = 0;
222 while ($iterationDAO->fetch()) {
224 $rowsThisIteration++
;
225 $row = $processor->buildRow($query, $iterationDAO, $outputColumns, $metadata, $paymentDetails, $addPaymentHeader);
226 if ($row === FALSE) {
230 // add component info
231 // write the row to a file
232 $componentDetails[] = $row;
234 // output every $tempRowCount rows
235 if ($count %
$tempRowCount == 0) {
236 self
::writeDetailsToTable($processor, $componentDetails, $sqlColumns);
237 $componentDetails = [];
240 if ($rowsThisIteration < self
::EXPORT_ROW_COUNT
) {
241 $limitReached = TRUE;
243 $offset +
= $rowCount;
246 if ($processor->getTemporaryTable()) {
247 self
::writeDetailsToTable($processor, $componentDetails, $sqlColumns);
249 // do merge same address and merge same household processing
250 if ($mergeSameAddress) {
251 $processor->mergeSameAddress($sqlColumns, $exportParams);
255 $table = $processor->getTemporaryTable();
256 CRM_Utils_Hook
::export($table, $headerRows, $sqlColumns, $exportMode, $componentTable, $ids);
257 if ($table !== $processor->getTemporaryTable()) {
258 CRM_Core_Error
::deprecatedFunctionWarning('altering the export table in the hook is deprecated (in some flows the table itself will be)');
259 $processor->setTemporaryTable($table);
262 // In order to be able to write a unit test against this function we need to suppress
263 // the csv writing. In future hopefully the csv writing & the main processing will be in separate functions.
264 if (empty($exportParams['suppress_csv_for_testing'])) {
265 self
::writeCSVFromTable($headerRows, $sqlColumns, $processor);
268 // return tableName sqlColumns headerRows in test context
269 return [$processor->getTemporaryTable(), $sqlColumns, $headerRows, $processor];
272 // delete the export temp table and component table
273 $sql = "DROP TABLE IF EXISTS " . $processor->getTemporaryTable();
274 CRM_Core_DAO
::executeQuery($sql);
275 CRM_Core_DAO
::reenableFullGroupByMode();
276 CRM_Utils_System
::civiExit(0, ['processor' => $processor]);
279 CRM_Core_DAO
::reenableFullGroupByMode();
280 throw new CRM_Core_Exception(ts('No records to export'));
285 * Handle import error file creation.
287 public static function invoke() {
288 $type = CRM_Utils_Request
::retrieve('type', 'Positive');
289 $parserName = CRM_Utils_Request
::retrieve('parser', 'String');
290 if (empty($parserName) ||
empty($type)) {
294 // clean and ensure parserName is a valid string
295 $parserName = CRM_Utils_String
::munge($parserName);
296 $parserClass = explode('_', $parserName);
298 // make sure parserClass is in the CRM namespace and
299 // at least 3 levels deep
300 if ($parserClass[0] == 'CRM' &&
301 count($parserClass) >= 3
303 require_once str_replace('_', DIRECTORY_SEPARATOR
, $parserName) . ".php";
304 // ensure the functions exists
305 if (method_exists($parserName, 'errorFileName') &&
306 method_exists($parserName, 'saveFileName')
308 $errorFileName = $parserName::errorFileName($type);
309 $saveFileName = $parserName::saveFileName($type);
310 if (!empty($errorFileName) && !empty($saveFileName)) {
311 CRM_Utils_System
::setHttpHeader('Cache-Control', 'must-revalidate, post-check=0, pre-check=0');
312 CRM_Utils_System
::setHttpHeader('Content-Description', 'File Transfer');
313 CRM_Utils_System
::setHttpHeader('Content-Type', 'text/csv');
314 CRM_Utils_System
::setHttpHeader('Content-Length', filesize($errorFileName));
315 CRM_Utils_System
::setHttpHeader('Content-Disposition', 'attachment; filename=' . $saveFileName);
317 readfile($errorFileName);
321 CRM_Utils_System
::civiExit();
325 * @param $customSearchClass
329 public static function exportCustom($customSearchClass, $formValues, $order) {
330 $ext = CRM_Extension_System
::singleton()->getMapper();
331 if (!$ext->isExtensionClass($customSearchClass)) {
332 require_once str_replace('_', DIRECTORY_SEPARATOR
, $customSearchClass) . '.php';
335 require_once $ext->classToPath($customSearchClass);
337 $search = new $customSearchClass($formValues);
339 $includeContactIDs = FALSE;
340 if ($formValues['radio_ts'] == 'ts_sel') {
341 $includeContactIDs = TRUE;
344 $sql = $search->all(0, 0, $order, $includeContactIDs);
346 $columns = $search->columns();
348 $header = array_keys($columns);
349 $fields = array_values($columns);
352 $dao = CRM_Core_DAO
::executeQuery($sql);
354 if (method_exists($search, 'alterRow')) {
357 while ($dao->fetch()) {
360 foreach ($fields as $field) {
361 $unqualified_field = CRM_Utils_Array
::First(array_slice(explode('.', $field), -1));
362 $row[$field] = $dao->$unqualified_field;
365 $search->alterRow($row);
370 CRM_Core_Report_Excel
::writeCSVFile(ts('CiviCRM Contact Search'), $header, $rows);
371 CRM_Utils_System
::civiExit();
375 * @param \CRM_Export_BAO_ExportProcessor $processor
379 public static function writeDetailsToTable($processor, $details, $sqlColumns) {
380 $tableName = $processor->getTemporaryTable();
381 if (empty($details)) {
390 $id = CRM_Core_DAO
::singleValueQuery($sql);
397 foreach ($details as $row) {
399 $valueString = [$id];
400 foreach ($row as $value) {
402 $valueString[] = "''";
405 $valueString[] = "'" . CRM_Core_DAO
::escapeString($value) . "'";
408 $sqlClause[] = '(' . implode(',', $valueString) . ')';
410 $sqlColumns = array_merge(['id' => 1], $sqlColumns);
411 $sqlColumnString = '(' . implode(',', array_keys($sqlColumns)) . ')';
413 $sqlValueString = implode(",\n", $sqlClause);
416 INSERT INTO $tableName $sqlColumnString
417 VALUES $sqlValueString
419 CRM_Core_DAO
::executeQuery($sql);
427 public static function createTempTable($sqlColumns) {
428 //creating a temporary table for the search result that need be exported
429 $exportTempTable = CRM_Utils_SQL_TempTable
::build()->setDurable()->setCategory('export');
431 // also create the sql table
432 $exportTempTable->drop();
434 $sql = " id int unsigned NOT NULL AUTO_INCREMENT, ";
435 if (!empty($sqlColumns)) {
436 $sql .= implode(",\n", array_values($sqlColumns)) . ',';
439 $sql .= "\n PRIMARY KEY ( id )";
441 // add indexes for street_address and household_name if present
445 'civicrm_primary_id',
448 foreach ($addIndices as $index) {
449 if (isset($sqlColumns[$index])) {
451 INDEX index_{$index}( $index )
456 $exportTempTable->createWithColumns($sql);
457 return $exportTempTable->getName();
463 * @param \CRM_Export_BAO_ExportProcessor $processor
465 public static function writeCSVFromTable($headerRows, $sqlColumns, $processor) {
466 $exportTempTable = $processor->getTemporaryTable();
467 $exportMode = $processor->getExportMode();
470 $limit = self
::EXPORT_ROW_COUNT
;
472 $query = "SELECT * FROM $exportTempTable";
475 $limitQuery = $query . "
476 LIMIT $offset, $limit
478 $dao = CRM_Core_DAO
::executeQuery($limitQuery);
484 $componentDetails = [];
485 while ($dao->fetch()) {
488 foreach ($sqlColumns as $column => $dontCare) {
489 $row[$column] = $dao->$column;
491 $componentDetails[] = $row;
493 CRM_Core_Report_Excel
::writeCSVFile($processor->getExportFileName(),
500 $writeHeader = FALSE;
506 * Build componentPayment fields.
508 * This is no longer used by export but BAO_Mapping still calls it & we
509 * should find a generic way to handle this or move this to that class.
513 public static function componentPaymentFields() {
514 static $componentPaymentFields;
515 if (!isset($componentPaymentFields)) {
516 $componentPaymentFields = [
517 'componentPaymentField_total_amount' => ts('Total Amount'),
518 'componentPaymentField_contribution_status' => ts('Contribution Status'),
519 'componentPaymentField_received_date' => ts('Date Received'),
520 'componentPaymentField_payment_instrument' => ts('Payment Method'),
521 'componentPaymentField_transaction_id' => ts('Transaction ID'),
524 return $componentPaymentFields;
528 * Get the values of linked household contact.
530 * @param CRM_Core_DAO $relDAO
531 * @param array $value
532 * @param string $field
535 private static function fetchRelationshipDetails($relDAO, $value, $field, &$row) {
536 $phoneTypes = CRM_Core_PseudoConstant
::get('CRM_Core_DAO_Phone', 'phone_type_id');
537 $imProviders = CRM_Core_PseudoConstant
::get('CRM_Core_DAO_IM', 'provider_id');
538 $i18n = CRM_Core_I18n
::singleton();
539 $field = $field . '_';
541 foreach ($value as $relationField => $relationValue) {
542 if (is_object($relDAO) && property_exists($relDAO, $relationField)) {
543 $fieldValue = $relDAO->$relationField;
544 if ($relationField == 'phone_type_id') {
545 $fieldValue = $phoneTypes[$relationValue];
547 elseif ($relationField == 'provider_id') {
548 $fieldValue = CRM_Utils_Array
::value($relationValue, $imProviders);
551 elseif (is_object($relDAO) && in_array($relationField, [
556 //special case for greeting replacement
557 $fldValue = "{$relationField}_display";
558 $fieldValue = $relDAO->$fldValue;
561 elseif (is_object($relDAO) && $relationField == 'state_province') {
562 $fieldValue = CRM_Core_PseudoConstant
::stateProvince($relDAO->state_province_id
);
564 elseif (is_object($relDAO) && $relationField == 'country') {
565 $fieldValue = CRM_Core_PseudoConstant
::country($relDAO->country_id
);
570 $relPrefix = $field . $relationField;
572 if (is_object($relDAO) && $relationField == 'id') {
573 $row[$relPrefix] = $relDAO->contact_id
;
575 elseif (is_array($relationValue) && $relationField == 'location') {
576 foreach ($relationValue as $ltype => $val) {
577 // If the location name has a space in it the we need to handle that. This
578 // is kinda hacky but specifically covered in the ExportTest so later efforts to
579 // improve it should be secure in the knowled it will be caught.
580 $ltype = str_replace(' ', '_', $ltype);
581 foreach (array_keys($val) as $fld) {
582 $type = explode('-', $fld);
583 $fldValue = "{$ltype}-" . $type[0];
584 if (!empty($type[1])) {
585 $fldValue .= "-" . $type[1];
587 // CRM-3157: localise country, region (both have ‘country’ context)
588 // and state_province (‘province’ context)
590 case (!is_object($relDAO)):
591 $row[$field . '_' . $fldValue] = '';
594 case in_array('country', $type):
595 case in_array('world_region', $type):
596 $row[$field . '_' . $fldValue] = $i18n->crm_translate($relDAO->$fldValue,
597 ['context' => 'country']
601 case in_array('state_province', $type):
602 $row[$field . '_' . $fldValue] = $i18n->crm_translate($relDAO->$fldValue,
603 ['context' => 'province']
608 $row[$field . '_' . $fldValue] = $relDAO->$fldValue;
614 elseif (isset($fieldValue) && $fieldValue != '') {
615 //check for custom data
616 if ($cfID = CRM_Core_BAO_CustomField
::getKeyID($relationField)) {
617 $row[$relPrefix] = CRM_Core_BAO_CustomField
::displayValue($fieldValue, $cfID);
620 //normal relationship fields
621 // CRM-3157: localise country, region (both have ‘country’ context) and state_province (‘province’ context)
622 switch ($relationField) {
625 $row[$relPrefix] = $i18n->crm_translate($fieldValue, ['context' => 'country']);
628 case 'state_province':
629 $row[$relPrefix] = $i18n->crm_translate($fieldValue, ['context' => 'province']);
633 $row[$relPrefix] = $fieldValue;
639 // if relation field is empty or null
640 $row[$relPrefix] = '';
646 * Get the ids that we want to get related contact details for.
649 * @param int $exportMode
653 protected static function getIDsForRelatedContact($ids, $exportMode) {
654 if ($exportMode == CRM_Export_Form_Select
::CONTACT_EXPORT
) {
657 if ($exportMode == CRM_Export_Form_Select
::ACTIVITY_EXPORT
) {
659 $sourceID = CRM_Core_PseudoConstant
::getKey('CRM_Activity_BAO_ActivityContact', 'record_type_id', 'Activity Source');
660 $dao = CRM_Core_DAO
::executeQuery("
661 SELECT contact_id FROM civicrm_activity_contact
662 WHERE activity_id IN ( " . implode(',', $ids) . ") AND
663 record_type_id = {$sourceID}
666 while ($dao->fetch()) {
667 $relIDs[] = $dao->contact_id
;
671 $component = self
::exportComponent($exportMode);
673 if ($exportMode == CRM_Export_Form_Select
::CASE_EXPORT
) {
674 return CRM_Case_BAO_Case
::retrieveContactIdsByCaseId($ids);
677 return CRM_Core_DAO
::getContactIDsFromComponent($ids, $component);
684 * @param \CRM_Export_BAO_ExportProcessor $processor
685 * @param $componentTable
687 protected static function buildRelatedContactArray($selectAll, $ids, $processor, $componentTable) {
688 $allRelContactArray = $relationQuery = [];
689 $queryMode = $processor->getQueryMode();
690 $exportMode = $processor->getExportMode();
692 foreach ($processor->getRelationshipReturnProperties() as $relationshipKey => $relationReturnProperties) {
693 $allRelContactArray[$relationshipKey] = [];
694 // build Query for each relationship
695 $relationQuery = new CRM_Contact_BAO_Query(NULL, $relationReturnProperties,
696 NULL, FALSE, FALSE, $queryMode
698 list($relationSelect, $relationFrom, $relationWhere, $relationHaving) = $relationQuery->query();
700 list($id, $direction) = explode('_', $relationshipKey, 2);
701 // identify the relationship direction
702 $contactA = 'contact_id_a';
703 $contactB = 'contact_id_b';
704 if ($direction == 'b_a') {
705 $contactA = 'contact_id_b';
706 $contactB = 'contact_id_a';
708 $relIDs = self
::getIDsForRelatedContact($ids, $exportMode);
710 $relationshipJoin = $relationshipClause = '';
711 if (!$selectAll && $componentTable) {
712 $relationshipJoin = " INNER JOIN {$componentTable} ctTable ON ctTable.contact_id = {$contactA}";
714 elseif (!empty($relIDs)) {
715 $relID = implode(',', $relIDs);
716 $relationshipClause = " AND crel.{$contactA} IN ( {$relID} )";
719 $relationFrom = " {$relationFrom}
720 INNER JOIN civicrm_relationship crel ON crel.{$contactB} = contact_a.id AND crel.relationship_type_id = {$id}
721 {$relationshipJoin} ";
723 //check for active relationship status only
724 $today = date('Ymd');
725 $relationActive = " AND (crel.is_active = 1 AND ( crel.end_date is NULL OR crel.end_date >= {$today} ) )";
726 $relationWhere = " WHERE contact_a.is_deleted = 0 {$relationshipClause} {$relationActive}";
727 CRM_Core_DAO
::disableFullGroupByMode();
728 $relationSelect = "{$relationSelect}, {$contactA} as refContact ";
729 $relationQueryString = "$relationSelect $relationFrom $relationWhere $relationHaving GROUP BY crel.{$contactA}";
731 $allRelContactDAO = CRM_Core_DAO
::executeQuery($relationQueryString);
732 CRM_Core_DAO
::reenableFullGroupByMode();
734 while ($allRelContactDAO->fetch()) {
735 $relationQuery->convertToPseudoNames($allRelContactDAO);
737 // @todo pass processor to fetchRelationshipDetails and set fields directly within it.
738 self
::fetchRelationshipDetails($allRelContactDAO, $relationReturnProperties, $relationshipKey, $row);
739 foreach (array_keys($relationReturnProperties) as $property) {
740 if ($property === 'location') {
741 // @todo - simplify location in self::fetchRelationshipDetails - remove handling here. Or just call
742 // $processor->setRelationshipValue from fetchRelationshipDetails
743 foreach ($relationReturnProperties['location'] as $locationName => $locationValues) {
744 foreach (array_keys($locationValues) as $locationValue) {
745 $key = str_replace(' ', '_', $locationName) . '-' . $locationValue;
746 $processor->setRelationshipValue($relationshipKey, $allRelContactDAO->refContact
, $key, $row[$relationshipKey . '__' . $key]);
751 $processor->setRelationshipValue($relationshipKey, $allRelContactDAO->refContact
, $property, $row[$relationshipKey . '_' . $property]);