CRM/Export/BAO/Export.php

   1 <?php
   2 /*
   3  +--------------------------------------------------------------------+
   4  | CiviCRM version 5                                                  |
   5  +--------------------------------------------------------------------+
   6  | Copyright CiviCRM LLC (c) 2004-2019                                |
   7  +--------------------------------------------------------------------+
   8  | This file is a part of CiviCRM.                                    |
   9  |                                                                    |
  10  | CiviCRM is free software; you can copy, modify, and distribute it  |
  11  | under the terms of the GNU Affero General Public License           |
  12  | Version 3, 19 November 2007 and the CiviCRM Licensing Exception.   |
  13  |                                                                    |
  14  | CiviCRM is distributed in the hope that it will be useful, but     |
  15  | WITHOUT ANY WARRANTY; without even the implied warranty of         |
  16  | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.               |
  17  | See the GNU Affero General Public License for more details.        |
  18  |                                                                    |
  19  | You should have received a copy of the GNU Affero General Public   |
  20  | License and the CiviCRM Licensing Exception along                  |
  21  | with this program; if not, contact CiviCRM LLC                     |
  22  | at info[AT]civicrm[DOT]org. If you have questions about the        |
  23  | GNU Affero General Public License or the licensing of CiviCRM,     |
  24  | see the CiviCRM license FAQ at http://civicrm.org/licensing        |
  25  +--------------------------------------------------------------------+
  26  */
  27
  28 /**
  29  *
  30  * @package CRM
  31  * @copyright CiviCRM LLC (c) 2004-2019
  32  */
  33
  34 /**
  35  * This class contains the functions for Component export
  36  *
  37  */
  38 class CRM_Export_BAO_Export {
  39   // increase this number a lot to avoid making too many queries
  40   // LIMIT is not much faster than a no LIMIT query
  41   // CRM-7675
  42   const EXPORT_ROW_COUNT = 100000;
  43
  44   /**
  45    * Get the list the export fields.
  46    *
  47    * @param int $selectAll
  48    *   User preference while export.
  49    * @param array $ids
  50    *   Contact ids.
  51    * @param array $params
  52    *   Associated array of fields.
  53    * @param string $order
  54    *   Order by clause.
  55    * @param array $fields
  56    *   Associated array of fields.
  57    * @param array $moreReturnProperties
  58    *   Additional return fields.
  59    * @param int $exportMode
  60    *   Export mode.
  61    * @param string $componentClause
  62    *   Component clause.
  63    * @param string $componentTable
  64    *   Component table.
  65    * @param bool $mergeSameAddress
  66    *   Merge records if they have same address.
  67    * @param bool $mergeSameHousehold
  68    *   Merge records if they belong to the same household.
  69    *
  70    * @param array $exportParams
  71    * @param string $queryOperator
  72    *
  73    * @throws \CRM_Core_Exception
  74    */
  75   public static function exportComponents(
  76     $selectAll,
  77     $ids,
  78     $params,
  79     $order = NULL,
  80     $fields = NULL,
  81     $moreReturnProperties = NULL,
  82     $exportMode = CRM_Export_Form_Select::CONTACT_EXPORT,
  83     $componentClause = NULL,
  84     $componentTable = NULL,
  85     $mergeSameAddress = FALSE,
  86     $mergeSameHousehold = FALSE,
  87     $exportParams = [],
  88     $queryOperator = 'AND'
  89   ) {
  90
  91     $isPostalOnly = (
  92       isset($exportParams['postal_mailing_export']['postal_mailing_export']) &&
  93       $exportParams['postal_mailing_export']['postal_mailing_export'] == 1
  94     );
  95
  96     if (!$selectAll && $componentTable && !empty($exportParams['additional_group'])) {
  97       // If an Additional Group is selected, then all contacts in that group are
  98       // added to the export set (filtering out duplicates).
  99       // Really - the calling function could do this ... just saying
 100       // @todo take a whip to the calling function.
 101       CRM_Core_DAO::executeQuery("
 102 INSERT INTO {$componentTable} SELECT distinct gc.contact_id FROM civicrm_group_contact gc WHERE gc.group_id = {$exportParams['additional_group']} ON DUPLICATE KEY UPDATE {$componentTable}.contact_id = gc.contact_id"
 103       );
 104     }
 105     // rectify params to what proximity search expects if there is a value for prox_distance
 106     // CRM-7021
 107     // @todo - move this back to the calling functions
 108     if (!empty($params)) {
 109       CRM_Contact_BAO_ProximityQuery::fixInputParams($params);
 110     }
 111     // @todo everything from this line up should go back to the calling functions.
 112     $processor = new CRM_Export_BAO_ExportProcessor($exportMode, $fields, $queryOperator, $mergeSameHousehold, $isPostalOnly, $mergeSameAddress, $exportParams);
 113     if ($moreReturnProperties) {
 114       $processor->setAdditionalRequestedReturnProperties($moreReturnProperties);
 115     }
 116     $processor->setComponentTable($componentTable);
 117     $processor->setComponentClause($componentClause);
 118
 119     list($query, $queryString) = $processor->runQuery($params, $order);
 120
 121     // This perhaps only needs calling when $mergeSameHousehold == 1
 122     self::buildRelatedContactArray($selectAll, $ids, $processor, $componentTable);
 123
 124     $addPaymentHeader = FALSE;
 125
 126     list($outputColumns, $metadata) = $processor->getExportStructureArrays();
 127
 128     if ($processor->isMergeSameAddress()) {
 129       foreach (array_keys($processor->getAdditionalFieldsForSameAddressMerge()) as $field) {
 130         $processor->setColumnAsCalculationOnly($field);
 131       }
 132     }
 133
 134     $paymentDetails = [];
 135     if ($processor->isExportPaymentFields()) {
 136       // get payment related in for event and members
 137       $paymentDetails = CRM_Contribute_BAO_Contribution::getContributionDetails($exportMode, $ids);
 138       //get all payment headers.
 139       // If we haven't selected specific payment fields, load in all the
 140       // payment headers.
 141       if (!$processor->isExportSpecifiedPaymentFields()) {
 142         if (!empty($paymentDetails)) {
 143           $addPaymentHeader = TRUE;
 144           foreach (array_keys($processor->getPaymentHeaders()) as $paymentField) {
 145             $processor->addOutputSpecification($paymentField);
 146           }
 147         }
 148       }
 149     }
 150
 151     $componentDetails = [];
 152
 153     $rowCount = self::EXPORT_ROW_COUNT;
 154     $offset = 0;
 155     // we write to temp table often to avoid using too much memory
 156     $tempRowCount = 100;
 157
 158     $count = -1;
 159
 160     $sqlColumns = $processor->getSQLColumns();
 161     $processor->createTempTable();
 162     $limitReached = FALSE;
 163
 164     while (!$limitReached) {
 165       $limitQuery = "{$queryString} LIMIT {$offset}, {$rowCount}";
 166       CRM_Core_DAO::disableFullGroupByMode();
 167       $iterationDAO = CRM_Core_DAO::executeQuery($limitQuery);
 168       CRM_Core_DAO::reenableFullGroupByMode();
 169       // If this is less than our limit by the end of the iteration we do not need to run the query again to
 170       // check if some remain.
 171       $rowsThisIteration = 0;
 172
 173       while ($iterationDAO->fetch()) {
 174         $count++;
 175         $rowsThisIteration++;
 176         $row = $processor->buildRow($query, $iterationDAO, $outputColumns, $metadata, $paymentDetails, $addPaymentHeader);
 177         if ($row === FALSE) {
 178           continue;
 179         }
 180
 181         // add component info
 182         // write the row to a file
 183         $componentDetails[] = $row;
 184
 185         // output every $tempRowCount rows
 186         if ($count % $tempRowCount == 0) {
 187           self::writeDetailsToTable($processor, $componentDetails, $sqlColumns);
 188           $componentDetails = [];
 189         }
 190       }
 191       if ($rowsThisIteration < self::EXPORT_ROW_COUNT) {
 192         $limitReached = TRUE;
 193       }
 194       $offset += $rowCount;
 195     }
 196
 197     if ($processor->getTemporaryTable()) {
 198       self::writeDetailsToTable($processor, $componentDetails);
 199
 200       // do merge same address and merge same household processing
 201       if ($mergeSameAddress) {
 202         $processor->mergeSameAddress();
 203       }
 204
 205       $processor->writeCSVFromTable();
 206
 207       // delete the export temp table and component table
 208       $sql = "DROP TABLE IF EXISTS " . $processor->getTemporaryTable();
 209       CRM_Core_DAO::executeQuery($sql);
 210       CRM_Core_DAO::reenableFullGroupByMode();
 211       CRM_Utils_System::civiExit(0, ['processor' => $processor]);
 212     }
 213     else {
 214       CRM_Core_DAO::reenableFullGroupByMode();
 215       throw new CRM_Core_Exception(ts('No records to export'));
 216     }
 217   }
 218
 219   /**
 220    * Handle import error file creation.
 221    */
 222   public static function invoke() {
 223     $type = CRM_Utils_Request::retrieve('type', 'Positive');
 224     $parserName = CRM_Utils_Request::retrieve('parser', 'String');
 225     if (empty($parserName) || empty($type)) {
 226       return;
 227     }
 228
 229     // clean and ensure parserName is a valid string
 230     $parserName = CRM_Utils_String::munge($parserName);
 231     $parserClass = explode('_', $parserName);
 232
 233     // make sure parserClass is in the CRM namespace and
 234     // at least 3 levels deep
 235     if ($parserClass[0] == 'CRM' &&
 236       count($parserClass) >= 3
 237     ) {
 238       require_once str_replace('_', DIRECTORY_SEPARATOR, $parserName) . ".php";
 239       // ensure the functions exists
 240       if (method_exists($parserName, 'errorFileName') &&
 241         method_exists($parserName, 'saveFileName')
 242       ) {
 243         $errorFileName = $parserName::errorFileName($type);
 244         $saveFileName = $parserName::saveFileName($type);
 245         if (!empty($errorFileName) && !empty($saveFileName)) {
 246           CRM_Utils_System::setHttpHeader('Cache-Control', 'must-revalidate, post-check=0, pre-check=0');
 247           CRM_Utils_System::setHttpHeader('Content-Description', 'File Transfer');
 248           CRM_Utils_System::setHttpHeader('Content-Type', 'text/csv');
 249           CRM_Utils_System::setHttpHeader('Content-Length', filesize($errorFileName));
 250           CRM_Utils_System::setHttpHeader('Content-Disposition', 'attachment; filename=' . $saveFileName);
 251
 252           readfile($errorFileName);
 253         }
 254       }
 255     }
 256     CRM_Utils_System::civiExit();
 257   }
 258
 259   /**
 260    * @param $customSearchClass
 261    * @param $formValues
 262    * @param $order
 263    */
 264   public static function exportCustom($customSearchClass, $formValues, $order) {
 265     $ext = CRM_Extension_System::singleton()->getMapper();
 266     if (!$ext->isExtensionClass($customSearchClass)) {
 267       require_once str_replace('_', DIRECTORY_SEPARATOR, $customSearchClass) . '.php';
 268     }
 269     else {
 270       require_once $ext->classToPath($customSearchClass);
 271     }
 272     $search = new $customSearchClass($formValues);
 273
 274     $includeContactIDs = FALSE;
 275     if ($formValues['radio_ts'] == 'ts_sel') {
 276       $includeContactIDs = TRUE;
 277     }
 278
 279     $sql = $search->all(0, 0, $order, $includeContactIDs);
 280
 281     $columns = $search->columns();
 282
 283     $header = array_keys($columns);
 284     $fields = array_values($columns);
 285
 286     $rows = [];
 287     $dao = CRM_Core_DAO::executeQuery($sql);
 288     $alterRow = FALSE;
 289     if (method_exists($search, 'alterRow')) {
 290       $alterRow = TRUE;
 291     }
 292     while ($dao->fetch()) {
 293       $row = [];
 294
 295       foreach ($fields as $field) {
 296         $unqualified_field = CRM_Utils_Array::First(array_slice(explode('.', $field), -1));
 297         $row[$field] = $dao->$unqualified_field;
 298       }
 299       if ($alterRow) {
 300         $search->alterRow($row);
 301       }
 302       $rows[] = $row;
 303     }
 304
 305     CRM_Core_Report_Excel::writeCSVFile(ts('CiviCRM Contact Search'), $header, $rows);
 306     CRM_Utils_System::civiExit();
 307   }
 308
 309   /**
 310    * @param \CRM_Export_BAO_ExportProcessor $processor
 311    * @param $details
 312    */
 313   public static function writeDetailsToTable($processor, $details) {
 314     $tableName = $processor->getTemporaryTable();
 315     if (empty($details)) {
 316       return;
 317     }
 318
 319     $sql = "
 320 SELECT max(id)
 321 FROM   $tableName
 322 ";
 323
 324     $id = CRM_Core_DAO::singleValueQuery($sql);
 325     if (!$id) {
 326       $id = 0;
 327     }
 328
 329     $sqlClause = [];
 330
 331     foreach ($details as $row) {
 332       $id++;
 333       $valueString = [$id];
 334       foreach ($row as $value) {
 335         if (empty($value)) {
 336           $valueString[] = "''";
 337         }
 338         else {
 339           $valueString[] = "'" . CRM_Core_DAO::escapeString($value) . "'";
 340         }
 341       }
 342       $sqlClause[] = '(' . implode(',', $valueString) . ')';
 343     }
 344     $sqlColumns = array_merge(['id' => 1], $processor->getSQLColumns());
 345     $sqlColumnString = '(' . implode(',', array_keys($sqlColumns)) . ')';
 346
 347     $sqlValueString = implode(",\n", $sqlClause);
 348
 349     $sql = "
 350 INSERT INTO $tableName $sqlColumnString
 351 VALUES $sqlValueString
 352 ";
 353     CRM_Core_DAO::executeQuery($sql);
 354   }
 355
 356   /**
 357    * Get the ids that we want to get related contact details for.
 358    *
 359    * @param array $ids
 360    * @param int $exportMode
 361    *
 362    * @return array
 363    */
 364   protected static function getIDsForRelatedContact($ids, $exportMode) {
 365     if ($exportMode == CRM_Export_Form_Select::CONTACT_EXPORT) {
 366       return $ids;
 367     }
 368     if ($exportMode == CRM_Export_Form_Select::ACTIVITY_EXPORT) {
 369       $relIDs = [];
 370       $sourceID = CRM_Core_PseudoConstant::getKey('CRM_Activity_BAO_ActivityContact', 'record_type_id', 'Activity Source');
 371       $dao = CRM_Core_DAO::executeQuery("
 372             SELECT contact_id FROM civicrm_activity_contact
 373             WHERE activity_id IN ( " . implode(',', $ids) . ") AND
 374             record_type_id = {$sourceID}
 375           ");
 376
 377       while ($dao->fetch()) {
 378         $relIDs[] = $dao->contact_id;
 379       }
 380       return $relIDs;
 381     }
 382     $componentMapping = [
 383       CRM_Export_Form_Select::CONTRIBUTE_EXPORT => 'civicrm_contribution',
 384       CRM_Export_Form_Select::EVENT_EXPORT => 'civicrm_participant',
 385       CRM_Export_Form_Select::MEMBER_EXPORT => 'civicrm_membership',
 386       CRM_Export_Form_Select::PLEDGE_EXPORT => 'civicrm_pledge',
 387       CRM_Export_Form_Select::GRANT_EXPORT => 'civicrm_grant',
 388     ];
 389
 390     if ($exportMode == CRM_Export_Form_Select::CASE_EXPORT) {
 391       return CRM_Case_BAO_Case::retrieveContactIdsByCaseId($ids);
 392     }
 393     else {
 394       return CRM_Core_DAO::getContactIDsFromComponent($ids, $componentMapping[$exportMode]);
 395     }
 396   }
 397
 398   /**
 399    * @param $selectAll
 400    * @param $ids
 401    * @param \CRM_Export_BAO_ExportProcessor $processor
 402    * @param $componentTable
 403    */
 404   protected static function buildRelatedContactArray($selectAll, $ids, $processor, $componentTable) {
 405     $allRelContactArray = $relationQuery = [];
 406     $queryMode = $processor->getQueryMode();
 407     $exportMode = $processor->getExportMode();
 408
 409     foreach ($processor->getRelationshipReturnProperties() as $relationshipKey => $relationReturnProperties) {
 410       $allRelContactArray[$relationshipKey] = [];
 411       // build Query for each relationship
 412       $relationQuery = new CRM_Contact_BAO_Query(NULL, $relationReturnProperties,
 413         NULL, FALSE, FALSE, $queryMode
 414       );
 415       list($relationSelect, $relationFrom, $relationWhere, $relationHaving) = $relationQuery->query();
 416
 417       list($id, $direction) = explode('_', $relationshipKey, 2);
 418       // identify the relationship direction
 419       $contactA = 'contact_id_a';
 420       $contactB = 'contact_id_b';
 421       if ($direction == 'b_a') {
 422         $contactA = 'contact_id_b';
 423         $contactB = 'contact_id_a';
 424       }
 425       $relIDs = self::getIDsForRelatedContact($ids, $exportMode);
 426
 427       $relationshipJoin = $relationshipClause = '';
 428       if (!$selectAll && $componentTable) {
 429         $relationshipJoin = " INNER JOIN {$componentTable} ctTable ON ctTable.contact_id = {$contactA}";
 430       }
 431       elseif (!empty($relIDs)) {
 432         $relID = implode(',', $relIDs);
 433         $relationshipClause = " AND crel.{$contactA} IN ( {$relID} )";
 434       }
 435
 436       $relationFrom = " {$relationFrom}
 437               INNER JOIN civicrm_relationship crel ON crel.{$contactB} = contact_a.id AND crel.relationship_type_id = {$id}
 438               {$relationshipJoin} ";
 439
 440       //check for active relationship status only
 441       $today = date('Ymd');
 442       $relationActive = " AND (crel.is_active = 1 AND ( crel.end_date is NULL OR crel.end_date >= {$today} ) )";
 443       $relationWhere = " WHERE contact_a.is_deleted = 0 {$relationshipClause} {$relationActive}";
 444       CRM_Core_DAO::disableFullGroupByMode();
 445       $relationSelect = "{$relationSelect}, {$contactA} as refContact ";
 446       $relationQueryString = "$relationSelect $relationFrom $relationWhere $relationHaving GROUP BY crel.{$contactA}";
 447
 448       $allRelContactDAO = CRM_Core_DAO::executeQuery($relationQueryString);
 449       CRM_Core_DAO::reenableFullGroupByMode();
 450
 451       while ($allRelContactDAO->fetch()) {
 452         $relationQuery->convertToPseudoNames($allRelContactDAO);
 453         $row = [];
 454         // @todo pass processor to fetchRelationshipDetails and set fields directly within it.
 455         $processor->fetchRelationshipDetails($allRelContactDAO, $relationReturnProperties, $relationshipKey, $row);
 456         foreach (array_keys($relationReturnProperties) as $property) {
 457           if ($property === 'location') {
 458             // @todo - simplify location in fetchRelationshipDetails - remove handling here. Or just call
 459             // $processor->setRelationshipValue from fetchRelationshipDetails
 460             foreach ($relationReturnProperties['location'] as $locationName => $locationValues) {
 461               foreach (array_keys($locationValues) as $locationValue) {
 462                 $key = str_replace(' ', '_', $locationName) . '-' . $locationValue;
 463                 $processor->setRelationshipValue($relationshipKey, $allRelContactDAO->refContact, $key, $row[$relationshipKey . '__' . $key]);
 464               }
 465             }
 466           }
 467           else {
 468             $processor->setRelationshipValue($relationshipKey, $allRelContactDAO->refContact, $property, $row[$relationshipKey . '_' . $property]);
 469           }
 470         }
 471       }
 472     }
 473   }
 474
 475 }