CRM/Member/Import/Parser/Membership.php

   1 <?php
   2 /*
   3  +--------------------------------------------------------------------+
   4  | Copyright CiviCRM LLC. All rights reserved.                        |
   5  |                                                                    |
   6  | This work is published under the GNU AGPLv3 license with some      |
   7  | permitted exceptions and without any warranty. For full license    |
   8  | and copyright information, see https://civicrm.org/licensing       |
   9  +--------------------------------------------------------------------+
  10  */
  11
  12 /**
  13  *
  14  * @package CRM
  15  * @copyright CiviCRM LLC https://civicrm.org/licensing
  16  */
  17
  18 /**
  19  * class to parse membership csv files
  20  */
  21 class CRM_Member_Import_Parser_Membership extends CRM_Import_Parser {
  22
  23   protected $_mapperKeys;
  24
  25   /**
  26    * Array of metadata for all available fields.
  27    *
  28    * @var array
  29    */
  30   protected $fieldMetadata = [];
  31
  32   /**
  33    * Array of successfully imported membership id's
  34    *
  35    * @var array
  36    */
  37   protected $_newMemberships;
  38
  39   protected $_fileName;
  40
  41   /**
  42    * Imported file size
  43    * @var int
  44    */
  45   protected $_fileSize;
  46
  47   /**
  48    * Separator being used
  49    * @var string
  50    */
  51   protected $_separator;
  52
  53   /**
  54    * Total number of lines in file
  55    * @var int
  56    */
  57   protected $_lineCount;
  58
  59   /**
  60    * Class constructor.
  61    *
  62    * @param $mapperKeys
  63    */
  64   public function __construct($mapperKeys = []) {
  65     parent::__construct();
  66     $this->_mapperKeys = $mapperKeys;
  67   }
  68
  69   /**
  70    * @param string $fileName
  71    * @param string $separator
  72    * @param $mapper
  73    * @param bool $skipColumnHeader
  74    * @param int $mode
  75    * @param int $contactType
  76    * @param int $onDuplicate
  77    * @param int $statusID
  78    *
  79    * @return mixed
  80    * @throws Exception
  81    */
  82   public function run(
  83     $fileName,
  84     $separator,
  85     $mapper,
  86     $skipColumnHeader = FALSE,
  87     $mode = self::MODE_PREVIEW,
  88     $contactType = self::CONTACT_INDIVIDUAL,
  89     $onDuplicate = self::DUPLICATE_SKIP,
  90     $statusID = NULL
  91   ) {
  92     $this->_contactType = $this->getContactType();
  93     $this->init();
  94
  95     $this->_lineCount = 0;
  96     $this->_invalidRowCount = $this->_validCount = 0;
  97     $this->_totalCount = 0;
  98
  99     $this->_errors = [];
 100     $this->_warnings = [];
 101     if ($statusID) {
 102       $this->progressImport($statusID);
 103       $startTimestamp = $currTimestamp = $prevTimestamp = CRM_Utils_Time::time();
 104     }
 105     $dataSource = $this->getDataSourceObject();
 106     $totalRowCount = $dataSource->getRowCount(['new']);
 107     $dataSource->setStatuses(['new']);
 108     while ($row = $dataSource->getRow()) {
 109       $values = array_values($row);
 110       if ($mode == self::MODE_IMPORT) {
 111         $this->import($values);
 112         if ($statusID && (($this->_lineCount % 50) == 0)) {
 113           $prevTimestamp = $this->progressImport($statusID, FALSE, $startTimestamp, $prevTimestamp, $totalRowCount);
 114         }
 115       }
 116     }
 117   }
 118
 119   /**
 120    * Given a list of the importable field keys that the user has selected
 121    * set the active fields array to this list
 122    *
 123    * @param array $fieldKeys mapped array of values
 124    *
 125    * @return void
 126    */
 127   public function setActiveFields($fieldKeys) {
 128     $this->_activeFieldCount = count($fieldKeys);
 129     foreach ($fieldKeys as $key) {
 130       if (empty($this->_fields[$key])) {
 131         $this->_activeFields[] = new CRM_Member_Import_Field('', ts('- do not import -'));
 132       }
 133       else {
 134         $this->_activeFields[] = clone($this->_fields[$key]);
 135       }
 136     }
 137   }
 138
 139   /**
 140    * Format the field values for input to the api.
 141    *
 142    * @return array
 143    *   (reference ) associative array of name/value pairs
 144    */
 145   public function getParams() {
 146     $this->getSubmittedValue('mapper');
 147     $params = [];
 148     for ($i = 0; $i < $this->_activeFieldCount; $i++) {
 149       if (isset($this->_activeFields[$i]->_value)
 150         && !isset($params[$this->_activeFields[$i]->_name])
 151         && !isset($this->_activeFields[$i]->_related)
 152       ) {
 153
 154         $params[$this->_activeFields[$i]->_name] = $this->_activeFields[$i]->_value;
 155       }
 156     }
 157     return $params;
 158   }
 159
 160   /**
 161    * @param string $name
 162    * @param $title
 163    * @param int $type
 164    * @param string $headerPattern
 165    * @param string $dataPattern
 166    */
 167   public function addField($name, $title, $type = CRM_Utils_Type::T_INT, $headerPattern = '//', $dataPattern = '//') {
 168     if (empty($name)) {
 169       $this->_fields['doNotImport'] = new CRM_Member_Import_Field($name, $title, $type, $headerPattern, $dataPattern);
 170     }
 171     else {
 172
 173       //$tempField = CRM_Contact_BAO_Contact::importableFields('Individual', null );
 174       $tempField = CRM_Contact_BAO_Contact::importableFields('All', NULL);
 175       if (!array_key_exists($name, $tempField)) {
 176         $this->_fields[$name] = new CRM_Member_Import_Field($name, $title, $type, $headerPattern, $dataPattern);
 177       }
 178       else {
 179         $this->_fields[$name] = new CRM_Contact_Import_Field($name, $title, $type, $headerPattern, $dataPattern,
 180           CRM_Utils_Array::value('hasLocationType', $tempField[$name])
 181         );
 182       }
 183     }
 184   }
 185
 186   /**
 187    * Store parser values.
 188    *
 189    * @param CRM_Core_Session $store
 190    *
 191    * @param int $mode
 192    *
 193    * @return void
 194    */
 195   public function set($store, $mode = self::MODE_SUMMARY) {
 196     $store->set('lineCount', $this->_lineCount);
 197     $store->set('validRowCount', $this->_validCount);
 198     $store->set('invalidRowCount', $this->_invalidRowCount);
 199
 200     if ($this->_invalidRowCount) {
 201       $store->set('errorsFileName', $this->_errorFileName);
 202     }
 203
 204     if ($mode == self::MODE_IMPORT) {
 205       $store->set('duplicateRowCount', $this->_duplicateCount);
 206       if ($this->_duplicateCount) {
 207         $store->set('duplicatesFileName', $this->_duplicateFileName);
 208       }
 209     }
 210   }
 211
 212   /**
 213    * Export data to a CSV file.
 214    *
 215    * @param string $fileName
 216    * @param array $header
 217    * @param array $data
 218    *
 219    * @return void
 220    */
 221   public static function exportCSV($fileName, $header, $data) {
 222     $output = [];
 223     $fd = fopen($fileName, 'w');
 224
 225     foreach ($header as $key => $value) {
 226       $header[$key] = "\"$value\"";
 227     }
 228     $config = CRM_Core_Config::singleton();
 229     $output[] = implode($config->fieldSeparator, $header);
 230
 231     foreach ($data as $datum) {
 232       foreach ($datum as $key => $value) {
 233         if (is_array($value)) {
 234           foreach ($value[0] as $k1 => $v1) {
 235             if ($k1 == 'location_type_id') {
 236               continue;
 237             }
 238             $datum[$k1] = $v1;
 239           }
 240         }
 241         else {
 242           $datum[$key] = "\"$value\"";
 243         }
 244       }
 245       $output[] = implode($config->fieldSeparator, $datum);
 246     }
 247     fwrite($fd, implode("\n", $output));
 248     fclose($fd);
 249   }
 250
 251   /**
 252    * The initializer code, called before the processing
 253    *
 254    * @return void
 255    */
 256   public function init() {
 257     // Force re-load of user job.
 258     unset($this->userJob);
 259     $this->setFieldMetadata();
 260
 261     $this->_newMemberships = [];
 262
 263     $this->setActiveFields($this->_mapperKeys);
 264   }
 265
 266   /**
 267    * Validate the values.
 268    *
 269    * @param array $values
 270    *   The array of values belonging to this line.
 271    */
 272   public function validateValues($values): void {
 273     $params = $this->getMappedRow($values);
 274     $errors = [];
 275     foreach ($params as $key => $value) {
 276       $errors = array_merge($this->getInvalidValues($value, $key), $errors);
 277     }
 278
 279     if (empty($params['membership_type_id'])) {
 280       $errors[] = ts('Missing required fields');
 281       return;
 282     }
 283
 284     //To check whether start date or join date is provided
 285     if (empty($params['start_date']) && empty($params['join_date'])) {
 286       $errors[] = 'Membership Start Date is required to create a memberships.';
 287     }
 288     if ($errors) {
 289       throw new CRM_Core_Exception('Invalid value for field(s) : ' . implode(',', $errors));
 290     }
 291   }
 292
 293   /**
 294    * Handle the values in import mode.
 295    *
 296    * @param array $values
 297    *   The array of values belonging to this line.
 298    *
 299    * @return int|void|null
 300    *   the result of this processing - which is ignored
 301    */
 302   public function import($values) {
 303     $onDuplicate = $this->getSubmittedValue('onDuplicate');
 304     $rowNumber = (int) ($values[array_key_last($values)]);
 305     try {
 306       $params = $this->getMappedRow($values);
 307
 308       //assign join date equal to start date if join date is not provided
 309       if (empty($params['join_date']) && !empty($params['start_date'])) {
 310         $params['join_date'] = $params['start_date'];
 311       }
 312
 313       $formatted = $params;
 314       // don't add to recent items, CRM-4399
 315       $formatted['skipRecentView'] = TRUE;
 316
 317       $formatValues = [];
 318       foreach ($params as $key => $field) {
 319         // ignore empty values or empty arrays etc
 320         if (CRM_Utils_System::isNull($field)) {
 321           continue;
 322         }
 323
 324         $formatValues[$key] = $field;
 325       }
 326
 327       //format params to meet api v2 requirements.
 328       //@todo find a way to test removing this formatting
 329       $formatError = $this->membership_format_params($formatValues, $formatted, TRUE);
 330
 331       if ($onDuplicate != CRM_Import_Parser::DUPLICATE_UPDATE) {
 332         $formatted['custom'] = CRM_Core_BAO_CustomField::postProcess($formatted,
 333           NULL,
 334           'Membership'
 335         );
 336       }
 337       else {
 338         //fix for CRM-2219 Update Membership
 339         // onDuplicate == CRM_Import_Parser::DUPLICATE_UPDATE
 340         if (!empty($formatted['is_override']) && empty($formatted['status_id'])) {
 341           throw new CRM_Core_Exception('Required parameter missing: Status', CRM_Import_Parser::ERROR);
 342         }
 343
 344         if (!empty($formatValues['membership_id'])) {
 345           $dao = new CRM_Member_BAO_Membership();
 346           $dao->id = $formatValues['membership_id'];
 347           $dates = ['join_date', 'start_date', 'end_date'];
 348           foreach ($dates as $v) {
 349             if (empty($formatted[$v])) {
 350               $formatted[$v] = CRM_Core_DAO::getFieldValue('CRM_Member_DAO_Membership', $formatValues['membership_id'], $v);
 351             }
 352           }
 353
 354           $formatted['custom'] = CRM_Core_BAO_CustomField::postProcess($formatted,
 355             $formatValues['membership_id'],
 356             'Membership'
 357           );
 358           if ($dao->find(TRUE)) {
 359             if (empty($params['line_item']) && !empty($formatted['membership_type_id'])) {
 360               CRM_Price_BAO_LineItem::getLineItemArray($formatted, NULL, 'membership', $formatted['membership_type_id']);
 361             }
 362
 363             $newMembership = civicrm_api3('Membership', 'create', $formatted);
 364             $this->_newMemberships[] = $newMembership['id'];
 365             $this->setImportStatus($rowNumber, 'IMPORTED', 'Required parameter missing: Status');
 366             return CRM_Import_Parser::VALID;
 367           }
 368           throw new CRM_Core_Exception('Matching Membership record not found for Membership ID ' . $formatValues['membership_id'] . '. Row was skipped.', CRM_Import_Parser::ERROR);
 369         }
 370       }
 371
 372       //Format dates
 373       $startDate = $formatted['start_date'];
 374       $endDate = $formatted['end_date'] ?? NULL;
 375       $joinDate = $formatted['join_date'];
 376
 377       if (empty($formatValues['id']) && empty($formatValues['contact_id'])) {
 378         $error = $this->checkContactDuplicate($formatValues);
 379
 380         if (CRM_Core_Error::isAPIError($error, CRM_Core_ERROR::DUPLICATE_CONTACT)) {
 381           $matchedIDs = explode(',', $error['error_message']['params'][0]);
 382           if (count($matchedIDs) > 1) {
 383             throw new CRM_Core_Exception('Multiple matching contact records detected for this row. The membership was not imported', CRM_Import_Parser::ERROR);
 384           }
 385           else {
 386             $cid = $matchedIDs[0];
 387             $formatted['contact_id'] = $cid;
 388
 389             //fix for CRM-1924
 390             $calcDates = CRM_Member_BAO_MembershipType::getDatesForMembershipType($formatted['membership_type_id'],
 391               $joinDate,
 392               $startDate,
 393               $endDate
 394             );
 395             self::formattedDates($calcDates, $formatted);
 396
 397             //fix for CRM-3570, exclude the statuses those having is_admin = 1
 398             //now user can import is_admin if is override is true.
 399             $excludeIsAdmin = FALSE;
 400             if (empty($formatted['is_override'])) {
 401               $formatted['exclude_is_admin'] = $excludeIsAdmin = TRUE;
 402             }
 403             $calcStatus = CRM_Member_BAO_MembershipStatus::getMembershipStatusByDate($startDate,
 404               $endDate,
 405               $joinDate,
 406               'now',
 407               $excludeIsAdmin,
 408               $formatted['membership_type_id'],
 409               $formatted
 410             );
 411
 412             if (empty($formatted['status_id'])) {
 413               $formatted['status_id'] = $calcStatus['id'];
 414             }
 415             elseif (empty($formatted['is_override'])) {
 416               if (empty($calcStatus)) {
 417                 throw new CRM_Core_Exception('Status in import row (' . $formatValues['status_id'] . ') does not match calculated status based on your configured Membership Status Rules. Record was not imported.', CRM_Import_Parser::ERROR);
 418               }
 419               if ($formatted['status_id'] != $calcStatus['id']) {
 420                 //Status Hold" is either NOT mapped or is FALSE
 421                 throw new CRM_Core_Exception('Status in import row (' . $formatValues['status_id'] . ') does not match calculated status based on your configured Membership Status Rules (' . $calcStatus['name'] . '). Record was not imported.', CRM_Import_Parser::ERROR);
 422               }
 423             }
 424
 425             $newMembership = civicrm_api3('membership', 'create', $formatted);
 426
 427             $this->_newMemberships[] = $newMembership['id'];
 428             $this->setImportStatus($rowNumber, 'IMPORTED', '');
 429             return CRM_Import_Parser::VALID;
 430           }
 431         }
 432         else {
 433           // Using new Dedupe rule.
 434           $ruleParams = [
 435             'contact_type' => $this->_contactType,
 436             'used' => 'Unsupervised',
 437           ];
 438           $fieldsArray = CRM_Dedupe_BAO_DedupeRule::dedupeRuleFields($ruleParams);
 439           $disp = '';
 440
 441           foreach ($fieldsArray as $value) {
 442             if (array_key_exists(trim($value), $params)) {
 443               $paramValue = $params[trim($value)];
 444               if (is_array($paramValue)) {
 445                 $disp .= $params[trim($value)][0][trim($value)] . " ";
 446               }
 447               else {
 448                 $disp .= $params[trim($value)] . " ";
 449               }
 450             }
 451           }
 452
 453           if (!empty($params['external_identifier'])) {
 454             if ($disp) {
 455               $disp .= "AND {$params['external_identifier']}";
 456             }
 457             else {
 458               $disp = $params['external_identifier'];
 459             }
 460           }
 461           throw new CRM_Core_Exception('No matching Contact found for (' . $disp . ')', CRM_Import_Parser::ERROR);
 462         }
 463       }
 464       else {
 465         if (!empty($formatValues['external_identifier'])) {
 466           $checkCid = new CRM_Contact_DAO_Contact();
 467           $checkCid->external_identifier = $formatValues['external_identifier'];
 468           $checkCid->find(TRUE);
 469           if ($checkCid->id != $formatted['contact_id']) {
 470             throw new CRM_Core_Exception('Mismatch of External ID:' . $formatValues['external_identifier'] . ' and Contact Id:' . $formatted['contact_id'], CRM_Import_Parser::ERROR);
 471           }
 472         }
 473
 474         //to calculate dates
 475         $calcDates = CRM_Member_BAO_MembershipType::getDatesForMembershipType($formatted['membership_type_id'],
 476           $joinDate,
 477           $startDate,
 478           $endDate
 479         );
 480         self::formattedDates($calcDates, $formatted);
 481         //end of date calculation part
 482
 483         //fix for CRM-3570, exclude the statuses those having is_admin = 1
 484         //now user can import is_admin if is override is true.
 485         $excludeIsAdmin = FALSE;
 486         if (empty($formatted['is_override'])) {
 487           $formatted['exclude_is_admin'] = $excludeIsAdmin = TRUE;
 488         }
 489         $calcStatus = CRM_Member_BAO_MembershipStatus::getMembershipStatusByDate($startDate,
 490           $endDate,
 491           $joinDate,
 492           'now',
 493           $excludeIsAdmin,
 494           $formatted['membership_type_id'],
 495           $formatted
 496         );
 497         if (empty($formatted['status_id'])) {
 498           $formatted['status_id'] = $calcStatus['id'] ?? NULL;
 499         }
 500         elseif (empty($formatted['is_override'])) {
 501           if (empty($calcStatus)) {
 502             throw new CRM_Core_Exception('Status in import row (' . CRM_Utils_Array::value('status_id', $formatValues) . ') does not match calculated status based on your configured Membership Status Rules. Record was not imported.', CRM_Import_Parser::ERROR);
 503           }
 504           if ($formatted['status_id'] != $calcStatus['id']) {
 505             //Status Hold" is either NOT mapped or is FALSE
 506             throw new CRM_Core_Exception($rowNumber, 'ERROR', 'Status in import row (' . CRM_Utils_Array::value('status_id', $formatValues) . ') does not match calculated status based on your configured Membership Status Rules (' . $calcStatus['name'] . '). Record was not imported.', CRM_Import_Parser::ERROR);
 507           }
 508         }
 509
 510         $newMembership = civicrm_api3('membership', 'create', $formatted);
 511
 512         $this->_newMemberships[] = $newMembership['id'];
 513         $this->setImportStatus($rowNumber, 'IMPORTED', '');
 514         return CRM_Import_Parser::VALID;
 515       }
 516     }
 517     catch (CRM_Core_Exception $e) {
 518       $this->setImportStatus($rowNumber, 'ERROR', $e->getMessage());
 519       return CRM_Import_Parser::ERROR;
 520     }
 521     catch (CiviCRM_API3_Exception $e) {
 522       $this->setImportStatus($rowNumber, 'ERROR', $e->getMessage());
 523       return CRM_Import_Parser::ERROR;
 524     }
 525   }
 526
 527   /**
 528    * Get the array of successfully imported membership id's
 529    *
 530    * @return array
 531    */
 532   public function &getImportedMemberships() {
 533     return $this->_newMemberships;
 534   }
 535
 536   /**
 537    *  to calculate join, start and end dates
 538    *
 539    * @param array $calcDates
 540    *   Array of dates returned by getDatesForMembershipType().
 541    *
 542    * @param $formatted
 543    *
 544    */
 545   public function formattedDates($calcDates, &$formatted) {
 546     $dates = [
 547       'join_date',
 548       'start_date',
 549       'end_date',
 550     ];
 551
 552     foreach ($dates as $d) {
 553       if (isset($formatted[$d]) &&
 554         !CRM_Utils_System::isNull($formatted[$d])
 555       ) {
 556         $formatted[$d] = CRM_Utils_Date::isoToMysql($formatted[$d]);
 557       }
 558       elseif (isset($calcDates[$d])) {
 559         $formatted[$d] = CRM_Utils_Date::isoToMysql($calcDates[$d]);
 560       }
 561     }
 562   }
 563
 564   /**
 565    * @deprecated - this function formats params according to v2 standards but
 566    * need to be sure about the impact of not calling it so retaining on the import class
 567    * take the input parameter list as specified in the data model and
 568    * convert it into the same format that we use in QF and BAO object
 569    *
 570    * @param array $params
 571    *   Associative array of property name/value.
 572    *                             pairs to insert in new contact.
 573    * @param array $values
 574    *   The reformatted properties that we can use internally.
 575    *
 576    * @param array|bool $create Is the formatted Values array going to
 577    *                             be used for CRM_Member_BAO_Membership:create()
 578    *
 579    * @throws Exception
 580    * @return array|error
 581    */
 582   public function membership_format_params($params, &$values, $create = FALSE) {
 583     require_once 'api/v3/utils.php';
 584     $fields = CRM_Member_DAO_Membership::fields();
 585     _civicrm_api3_store_values($fields, $params, $values);
 586
 587     $customFields = CRM_Core_BAO_CustomField::getFields('Membership');
 588
 589     foreach ($params as $key => $value) {
 590
 591       switch ($key) {
 592         case 'contact_id':
 593           if (!CRM_Utils_Rule::integer($value)) {
 594             throw new Exception("contact_id not valid: $value");
 595           }
 596           $dao = new CRM_Core_DAO();
 597           $qParams = [];
 598           $svq = $dao->singleValueQuery("SELECT id FROM civicrm_contact WHERE id = $value",
 599             $qParams
 600           );
 601           if (!$svq) {
 602             throw new Exception("Invalid Contact ID: There is no contact record with contact_id = $value.");
 603           }
 604           $values[$key] = $value;
 605           break;
 606
 607         default:
 608           break;
 609       }
 610     }
 611
 612     return NULL;
 613   }
 614
 615   /**
 616    * Set field metadata.
 617    */
 618   protected function setFieldMetadata(): void {
 619     if (empty($this->importableFieldsMetadata)) {
 620       $metadata = CRM_Member_BAO_Membership::importableFields($this->getContactType(), FALSE);
 621
 622       foreach ($metadata as $name => $field) {
 623         // @todo - we don't really need to do all this.... fieldMetadata is just fine to use as is.
 624         $field['type'] = CRM_Utils_Array::value('type', $field, CRM_Utils_Type::T_INT);
 625         $field['dataPattern'] = CRM_Utils_Array::value('dataPattern', $field, '//');
 626         $field['headerPattern'] = CRM_Utils_Array::value('headerPattern', $field, '//');
 627         $this->addField($name, $field['title'], $field['type'], $field['headerPattern'], $field['dataPattern']);
 628       }
 629       // We are consolidating on `importableFieldsMetadata` - but both still used.
 630       $this->importableFieldsMetadata = $this->fieldMetadata = $metadata;
 631     }
 632   }
 633
 634   /**
 635    * Get the metadata field for which importable fields does not key the actual field name.
 636    *
 637    * @return string[]
 638    */
 639   protected function getOddlyMappedMetadataFields(): array {
 640     $uniqueNames = ['membership_id', 'membership_contact_id'];
 641     $fields = [];
 642     foreach ($uniqueNames as $name) {
 643       $fields[$this->importableFieldsMetadata[$name]['name']] = $name;
 644     }
 645     // Include the parent fields as they could be present if required for matching ...in theory.
 646     return array_merge($fields, parent::getOddlyMappedMetadataFields());
 647   }
 648
 649 }