CRM/Activity/Import/Parser/Activity.php

   1 <?php
   2 /*
   3  +--------------------------------------------------------------------+
   4  | Copyright CiviCRM LLC. All rights reserved.                        |
   5  |                                                                    |
   6  | This work is published under the GNU AGPLv3 license with some      |
   7  | permitted exceptions and without any warranty. For full license    |
   8  | and copyright information, see https://civicrm.org/licensing       |
   9  +--------------------------------------------------------------------+
  10  */
  11
  12 /**
  13  *
  14  * @package CRM
  15  * @copyright CiviCRM LLC https://civicrm.org/licensing
  16  */
  17
  18
  19 /**
  20  * Class to parse activity csv files.
  21  */
  22 class CRM_Activity_Import_Parser_Activity extends CRM_Import_Parser {
  23
  24   protected $_mapperKeys;
  25
  26   private $_contactIdIndex;
  27
  28   /**
  29    * Array of successfully imported activity id's
  30    *
  31    * @var array
  32    */
  33   protected $_newActivity;
  34
  35   protected $_fileName;
  36
  37   /**
  38    * Imported file size.
  39    * @var int
  40    */
  41   protected $_fileSize;
  42
  43   /**
  44    * Separator being used.
  45    * @var string
  46    */
  47   protected $_separator;
  48
  49   /**
  50    * Total number of lines in file.
  51    * @var int
  52    */
  53   protected $_lineCount;
  54
  55   /**
  56    * Whether the file has a column header or not.
  57    *
  58    * @var bool
  59    */
  60   protected $_haveColumnHeader;
  61
  62   /**
  63    * Class constructor.
  64    *
  65    * @param array $mapperKeys
  66    */
  67   public function __construct($mapperKeys = []) {
  68     parent::__construct();
  69     $this->_mapperKeys = $mapperKeys;
  70   }
  71
  72   /**
  73    * The initializer code, called before the processing.
  74    */
  75   public function init() {
  76     $activityContact = CRM_Activity_BAO_ActivityContact::import();
  77     $activityTarget['target_contact_id'] = $activityContact['contact_id'];
  78     $fields = array_merge(CRM_Activity_BAO_Activity::importableFields(),
  79       $activityTarget
  80     );
  81
  82     $fields = array_merge($fields, [
  83       'source_contact_id' => [
  84         'title' => ts('Source Contact'),
  85         'headerPattern' => '/Source.Contact?/i',
  86       ],
  87       'activity_label' => [
  88         'title' => ts('Activity Type Label'),
  89         'headerPattern' => '/(activity.)?type label?/i',
  90       ],
  91     ]);
  92
  93     foreach ($fields as $name => $field) {
  94       $field['type'] = CRM_Utils_Array::value('type', $field, CRM_Utils_Type::T_INT);
  95       $field['dataPattern'] = CRM_Utils_Array::value('dataPattern', $field, '//');
  96       $field['headerPattern'] = CRM_Utils_Array::value('headerPattern', $field, '//');
  97       if (!empty($field['custom_group_id'])) {
  98         $field['title'] = $field["groupTitle"] . ' :: ' . $field["title"];
  99       }
 100       $this->addField($name, $field['title'], $field['type'], $field['headerPattern'], $field['dataPattern']);
 101     }
 102
 103     $this->_newActivity = [];
 104
 105     $this->setActiveFields($this->_mapperKeys);
 106
 107     // FIXME: we should do this in one place together with Form/MapField.php
 108     $this->_contactIdIndex = -1;
 109
 110     $index = 0;
 111     foreach ($this->_mapperKeys as $key) {
 112       switch ($key) {
 113         case 'target_contact_id':
 114         case 'external_identifier':
 115           $this->_contactIdIndex = $index;
 116           break;
 117       }
 118       $index++;
 119     }
 120   }
 121
 122   /**
 123    * Handle the values in summary mode.
 124    *
 125    * @param array $values
 126    *   The array of values belonging to this line.
 127    *
 128    * @return int
 129    *   CRM_Import_Parser::VALID for success or
 130    *   CRM_Import_Parser::ERROR for error.
 131    */
 132   public function summary(&$values) {
 133     try {
 134       $this->validateValues($values);
 135     }
 136     catch (CRM_Core_Exception $e) {
 137       return $this->addError($values, [$e->getMessage()]);
 138     }
 139
 140     return CRM_Import_Parser::VALID;
 141   }
 142
 143   /**
 144    * Handle the values in import mode.
 145    *
 146    * @param int $onDuplicate
 147    *   The code for what action to take on duplicates.
 148    * @param array $values
 149    *   The array of values belonging to this line.
 150    *
 151    * @return int
 152    *   CRM_Import_Parser::VALID for success or
 153    *   CRM_Import_Parser::ERROR for error.
 154    *
 155    * @throws \CRM_Core_Exception
 156    */
 157   public function import($onDuplicate, &$values) {
 158     // First make sure this is a valid line
 159     try {
 160       $this->validateValues($values);
 161     }
 162     catch (CRM_Core_Exception $e) {
 163       return $this->addError($values, [$e->getMessage()]);
 164     }
 165     $params = $this->getApiReadyParams($values);
 166     // For date-Formats.
 167     $session = CRM_Core_Session::singleton();
 168     $dateType = $session->get('dateTypes');
 169
 170     $customFields = CRM_Core_BAO_CustomField::getFields('Activity');
 171
 172     foreach ($params as $key => $val) {
 173       if ($customFieldID = CRM_Core_BAO_CustomField::getKeyID($key)) {
 174         if (!empty($customFields[$customFieldID]) && $customFields[$customFieldID]['data_type'] == 'Date') {
 175           CRM_Contact_Import_Parser_Contact::formatCustomDate($params, $params, $dateType, $key);
 176         }
 177         elseif (!empty($customFields[$customFieldID]) && $customFields[$customFieldID]['data_type'] == 'Boolean') {
 178           $params[$key] = CRM_Utils_String::strtoboolstr($val);
 179         }
 180       }
 181       elseif ($key === 'activity_date_time') {
 182         $params[$key] = CRM_Utils_Date::formatDate($val, $dateType);
 183       }
 184       elseif ($key === 'activity_subject') {
 185         $params['subject'] = $val;
 186       }
 187     }
 188
 189     if ($this->_contactIdIndex < 0) {
 190
 191       // Retrieve contact id using contact dedupe rule.
 192       // Since we are supporting only individual's activity import.
 193       $params['contact_type'] = 'Individual';
 194       $params['version'] = 3;
 195       $error = _civicrm_api3_deprecated_duplicate_formatted_contact($params);
 196
 197       if (CRM_Core_Error::isAPIError($error, CRM_Core_ERROR::DUPLICATE_CONTACT)) {
 198         $matchedIDs = explode(',', $error['error_message']['params'][0]);
 199         if (count($matchedIDs) > 1) {
 200           array_unshift($values, 'Multiple matching contact records detected for this row. The activity was not imported');
 201           return CRM_Import_Parser::ERROR;
 202         }
 203         $cid = $matchedIDs[0];
 204         $params['target_contact_id'] = $cid;
 205         $params['version'] = 3;
 206         $newActivity = civicrm_api('activity', 'create', $params);
 207         if (!empty($newActivity['is_error'])) {
 208           array_unshift($values, $newActivity['error_message']);
 209           return CRM_Import_Parser::ERROR;
 210         }
 211
 212         $this->_newActivity[] = $newActivity['id'];
 213         return CRM_Import_Parser::VALID;
 214
 215       }
 216       // Using new Dedupe rule.
 217       $ruleParams = [
 218         'contact_type' => 'Individual',
 219         'used' => 'Unsupervised',
 220       ];
 221       $fieldsArray = CRM_Dedupe_BAO_DedupeRule::dedupeRuleFields($ruleParams);
 222
 223       $disp = NULL;
 224       foreach ($fieldsArray as $value) {
 225         if (array_key_exists(trim($value), $params)) {
 226           $paramValue = $params[trim($value)];
 227           if (is_array($paramValue)) {
 228             $disp .= $params[trim($value)][0][trim($value)] . " ";
 229           }
 230           else {
 231             $disp .= $params[trim($value)] . " ";
 232           }
 233         }
 234       }
 235
 236       if (!empty($params['external_identifier'])) {
 237         if ($disp) {
 238           $disp .= "AND {$params['external_identifier']}";
 239         }
 240         else {
 241           $disp = $params['external_identifier'];
 242         }
 243       }
 244
 245       array_unshift($values, 'No matching Contact found for (' . $disp . ')');
 246       return CRM_Import_Parser::ERROR;
 247     }
 248     if (!empty($params['external_identifier'])) {
 249       $targetContactId = CRM_Core_DAO::getFieldValue('CRM_Contact_DAO_Contact',
 250         $params['external_identifier'], 'id', 'external_identifier'
 251       );
 252
 253       if (!empty($params['target_contact_id']) &&
 254         $params['target_contact_id'] != $targetContactId
 255       ) {
 256         array_unshift($values, 'Mismatch of External ID:' . $params['external_identifier'] . ' and Contact Id:' . $params['target_contact_id']);
 257         return CRM_Import_Parser::ERROR;
 258       }
 259       if ($targetContactId) {
 260         $params['target_contact_id'] = $targetContactId;
 261       }
 262       else {
 263         array_unshift($values, 'No Matching Contact for External ID:' . $params['external_identifier']);
 264         return CRM_Import_Parser::ERROR;
 265       }
 266     }
 267
 268     $params['version'] = 3;
 269     $newActivity = civicrm_api('activity', 'create', $params);
 270     if (!empty($newActivity['is_error'])) {
 271       array_unshift($values, $newActivity['error_message']);
 272       return CRM_Import_Parser::ERROR;
 273     }
 274
 275     $this->_newActivity[] = $newActivity['id'];
 276     return CRM_Import_Parser::VALID;
 277   }
 278
 279   /**
 280    *
 281    * Get the value for the given field from the row of values.
 282    *
 283    * @param array $row
 284    * @param string $fieldName
 285    *
 286    * @return null|string
 287    */
 288   protected function getFieldValue(array $row, string $fieldName) {
 289     if (!is_numeric($this->getFieldIndex($fieldName))) {
 290       return NULL;
 291     }
 292     return $row[$this->getFieldIndex($fieldName)] ?? NULL;
 293   }
 294
 295   /**
 296    * Get the index for the given field.
 297    *
 298    * @param string $fieldName
 299    *
 300    * @return false|int
 301    */
 302   protected function getFieldIndex(string $fieldName) {
 303     return array_search($fieldName, $this->_mapperKeys, TRUE);
 304
 305   }
 306
 307   /**
 308    * Add an error to the values.
 309    *
 310    * @param array $values
 311    * @param array $error
 312    *
 313    * @return int
 314    */
 315   protected function addError(array &$values, array $error): int {
 316     array_unshift($values, implode(';', $error));
 317     return CRM_Import_Parser::ERROR;
 318   }
 319
 320   /**
 321    * Validate that the activity type id does not conflict with the label.
 322    *
 323    * @param array $values
 324    *
 325    * @return void
 326    * @throws \CRM_Core_Exception
 327    */
 328   protected function validateActivityTypeIDAndLabel(array $values): void {
 329     $activityLabel = $this->getFieldValue($values, 'activity_label');
 330     $activityTypeID = $this->getFieldValue($values, 'activity_type_id');
 331     if ($activityLabel && $activityTypeID
 332       && $activityLabel !== CRM_Core_PseudoConstant::getLabel('CRM_Activity_BAO_Activity', 'activity_type_id', $activityTypeID)) {
 333       throw new CRM_Core_Exception(ts('Activity type label and Activity type ID are in conflict'));
 334     }
 335   }
 336
 337   /**
 338    * Is the supplied date field valid based on selected date format.
 339    *
 340    * @param string $value
 341    *
 342    * @return bool
 343    */
 344   protected function isValidDate(string $value): bool {
 345     return (bool) CRM_Utils_Date::formatDate($value, CRM_Core_Session::singleton()->get('dateTypes'));
 346   }
 347
 348   /**
 349    * Is the supplied field a valid contact id.
 350    *
 351    * @param string|int $value
 352    *
 353    * @return bool
 354    */
 355   protected function isValidContactID($value): bool {
 356     if (!CRM_Utils_Rule::integer($value)) {
 357       return FALSE;
 358     }
 359     if (!CRM_Core_DAO::singleValueQuery("SELECT id FROM civicrm_contact WHERE id = " . (int) $value)) {
 360       return FALSE;
 361     }
 362     return TRUE;
 363   }
 364
 365   /**
 366    * Validate custom fields.
 367    *
 368    * @param array $values
 369    *
 370    * @throws \CRM_Core_Exception
 371    */
 372   protected function validateCustomFields($values):void {
 373     $this->setActiveFieldValues($values);
 374     $params = $this->getActiveFieldParams();
 375     $errorMessage = NULL;
 376     // Checking error in custom data.
 377     $params['contact_type'] = 'Activity';
 378     CRM_Contact_Import_Parser_Contact::isErrorInCustomData($params, $errorMessage);
 379     if ($errorMessage) {
 380       throw new CRM_Core_Exception('Invalid value for field(s) : ' . $errorMessage);
 381     }
 382   }
 383
 384   /**
 385    * @param array $values
 386    *
 387    * @throws \CRM_Core_Exception
 388    */
 389   protected function validateValues(array $values): void {
 390     // Check required fields if this is not an update.
 391     if (!$this->getFieldValue($values, 'activity_id')) {
 392       if (!$this->getFieldValue($values, 'activity_label')
 393         && !$this->getFieldValue($values, 'activity_type_id')) {
 394         throw new CRM_Core_Exception(ts('Missing required fields: Activity type label or Activity type ID'));
 395       }
 396       if (!$this->getFieldValue($values, 'activity_date_time')) {
 397         throw new CRM_Core_Exception(ts('Missing required fields'));
 398       }
 399     }
 400
 401     $this->validateActivityTypeIDAndLabel($values);
 402     if ($this->getFieldValue($values, 'activity_date_time')
 403       && !$this->isValidDate($this->getFieldValue($values, 'activity_date_time'))) {
 404       throw new CRM_Core_Exception(ts('Invalid Activity Date'));
 405     }
 406
 407     if ($this->getFieldValue($values, 'activity_engagement_level')
 408       && !CRM_Utils_Rule::positiveInteger($this->getFieldValue($values, 'activity_engagement_level'))) {
 409       throw new CRM_Core_Exception(ts('Activity Engagement Index'));
 410     }
 411
 412     $targetContactID = $this->getFieldValue($values, 'target_contact_id');
 413     if ($targetContactID && !$this->isValidContactID($targetContactID)) {
 414       throw new CRM_Core_Exception("Invalid Contact ID: There is no contact record with contact_id = " . CRM_Utils_Type::escape($targetContactID, 'String'));
 415     }
 416     $this->validateCustomFields($values);
 417   }
 418
 419   /**
 420    * Get array of parameters formatted for the api from the submitted values.
 421    *
 422    * @param array $values
 423    *
 424    * @return array
 425    */
 426   protected function getApiReadyParams(array $values): array {
 427     $this->setActiveFieldValues($values);
 428     $params = $this->getActiveFieldParams();
 429     if ($this->getFieldValue($values, 'activity_label')) {
 430       $params['activity_type_id'] = array_search(
 431          $this->getFieldValue($values, 'activity_label'),
 432          CRM_Activity_BAO_Activity::buildOptions('activity_type_id', 'create'),
 433         TRUE
 434       );
 435     }
 436     return $params;
 437   }
 438
 439   /**
 440    * @param array $fileName
 441    * @param string $separator
 442    * @param $mapper
 443    * @param bool $skipColumnHeader
 444    * @param int $mode
 445    * @param int $onDuplicate
 446    * @param int $statusID
 447    * @param int $totalRowCount
 448    *
 449    * @return mixed
 450    * @throws Exception
 451    */
 452   public function run(
 453     array $fileName,
 454           $separator,
 455           $mapper,
 456           $skipColumnHeader = FALSE,
 457           $mode = self::MODE_PREVIEW,
 458           $onDuplicate = self::DUPLICATE_SKIP,
 459           $statusID = NULL,
 460           $totalRowCount = NULL
 461   ) {
 462
 463     $fileName = $fileName['name'];
 464
 465     $this->init();
 466
 467     $this->_haveColumnHeader = $skipColumnHeader;
 468
 469     $this->_separator = $separator;
 470
 471     $fd = fopen($fileName, "r");
 472     if (!$fd) {
 473       return FALSE;
 474     }
 475
 476     $this->_lineCount = 0;
 477     $this->_invalidRowCount = $this->_validCount = 0;
 478     $this->_totalCount = 0;
 479
 480     $this->_errors = [];
 481     $this->_warnings = [];
 482
 483     $this->_fileSize = number_format(filesize($fileName) / 1024.0, 2);
 484
 485     if ($mode == self::MODE_MAPFIELD) {
 486       $this->_rows = [];
 487     }
 488     else {
 489       $this->_activeFieldCount = count($this->_activeFields);
 490     }
 491     if ($statusID) {
 492       $this->progressImport($statusID);
 493       $startTimestamp = $currTimestamp = $prevTimestamp = time();
 494     }
 495
 496     while (!feof($fd)) {
 497       $this->_lineCount++;
 498
 499       $values = fgetcsv($fd, 8192, $separator);
 500       if (!$values) {
 501         continue;
 502       }
 503
 504       self::encloseScrub($values);
 505
 506       // skip column header if we're not in mapfield mode
 507       if ($mode != self::MODE_MAPFIELD && $skipColumnHeader) {
 508         $skipColumnHeader = FALSE;
 509         continue;
 510       }
 511
 512       // Trim whitespace around the values.
 513
 514       $empty = TRUE;
 515       foreach ($values as $k => $v) {
 516         $values[$k] = trim($v, " \t\r\n");
 517       }
 518
 519       if (CRM_Utils_System::isNull($values)) {
 520         continue;
 521       }
 522
 523       $this->_totalCount++;
 524
 525       if ($mode == self::MODE_MAPFIELD) {
 526         $returnCode = CRM_Import_Parser::VALID;
 527       }
 528       // Note that MODE_SUMMARY seems to be never used.
 529       elseif ($mode == self::MODE_PREVIEW || $mode == self::MODE_SUMMARY) {
 530         $returnCode = $this->summary($values);
 531       }
 532       elseif ($mode == self::MODE_IMPORT) {
 533         $returnCode = $this->import($onDuplicate, $values);
 534         if ($statusID && (($this->_lineCount % 50) == 0)) {
 535           $prevTimestamp = $this->progressImport($statusID, FALSE, $startTimestamp, $prevTimestamp, $totalRowCount);
 536         }
 537       }
 538       else {
 539         $returnCode = self::ERROR;
 540       }
 541
 542       // note that a line could be valid but still produce a warning
 543       if ($returnCode & self::VALID) {
 544         $this->_validCount++;
 545         if ($mode == self::MODE_MAPFIELD) {
 546           $this->_rows[] = $values;
 547           $this->_activeFieldCount = max($this->_activeFieldCount, count($values));
 548         }
 549       }
 550
 551       if ($returnCode & self::ERROR) {
 552         $this->_invalidRowCount++;
 553         $recordNumber = $this->_lineCount;
 554         if ($this->_haveColumnHeader) {
 555           $recordNumber--;
 556         }
 557         array_unshift($values, $recordNumber);
 558         $this->_errors[] = $values;
 559       }
 560
 561       // if we are done processing the maxNumber of lines, break
 562       if ($this->_maxLinesToProcess > 0 && $this->_validCount >= $this->_maxLinesToProcess) {
 563         break;
 564       }
 565     }
 566
 567     fclose($fd);
 568
 569     if ($mode == self::MODE_PREVIEW || $mode == self::MODE_IMPORT) {
 570       $customHeaders = $mapper;
 571
 572       $customfields = CRM_Core_BAO_CustomField::getFields('Activity');
 573       foreach ($customHeaders as $key => $value) {
 574         if ($id = CRM_Core_BAO_CustomField::getKeyID($value)) {
 575           $customHeaders[$key] = $customfields[$id][0];
 576         }
 577       }
 578       if ($this->_invalidRowCount) {
 579         // removed view url for invlaid contacts
 580         $headers = array_merge(
 581           [ts('Line Number'), ts('Reason')],
 582           $customHeaders
 583         );
 584         $this->_errorFileName = self::errorFileName(self::ERROR);
 585         self::exportCSV($this->_errorFileName, $headers, $this->_errors);
 586       }
 587     }
 588   }
 589
 590   /**
 591    * Given a list of the importable field keys that the user has selected set the active fields array to this list.
 592    *
 593    * @param array $fieldKeys
 594    */
 595   public function setActiveFields($fieldKeys) {
 596     $this->_activeFieldCount = count($fieldKeys);
 597     foreach ($fieldKeys as $key) {
 598       if (empty($this->_fields[$key])) {
 599         $this->_activeFields[] = new CRM_Activity_Import_Field('', ts('- do not import -'));
 600       }
 601       else {
 602         $this->_activeFields[] = clone($this->_fields[$key]);
 603       }
 604     }
 605   }
 606
 607   /**
 608    * @param string $name
 609    * @param $title
 610    * @param int $type
 611    * @param string $headerPattern
 612    * @param string $dataPattern
 613    */
 614   public function addField($name, $title, $type = CRM_Utils_Type::T_INT, $headerPattern = '//', $dataPattern = '//') {
 615     if (empty($name)) {
 616       $this->_fields['doNotImport'] = new CRM_Activity_Import_Field($name, $title, $type, $headerPattern, $dataPattern);
 617     }
 618     else {
 619
 620       $tempField = CRM_Contact_BAO_Contact::importableFields('Individual', NULL);
 621       if (!array_key_exists($name, $tempField)) {
 622         $this->_fields[$name] = new CRM_Activity_Import_Field($name, $title, $type, $headerPattern, $dataPattern);
 623       }
 624       else {
 625         $this->_fields[$name] = new CRM_Contact_Import_Field($name, $title, $type, $headerPattern, $dataPattern, CRM_Utils_Array::value('hasLocationType', $tempField[$name]));
 626       }
 627     }
 628   }
 629
 630   /**
 631    * Store parser values.
 632    *
 633    * @param CRM_Core_Session $store
 634    */
 635   public function set($store) {
 636     $store->set('fileSize', $this->_fileSize);
 637     $store->set('lineCount', $this->_lineCount);
 638     $store->set('separator', $this->_separator);
 639     $store->set('fields', $this->getSelectValues());
 640
 641     $store->set('headerPatterns', $this->getHeaderPatterns());
 642     $store->set('dataPatterns', $this->getDataPatterns());
 643     $store->set('columnCount', $this->_activeFieldCount);
 644
 645     $store->set('totalRowCount', $this->_totalCount);
 646     $store->set('validRowCount', $this->_validCount);
 647     $store->set('invalidRowCount', $this->_invalidRowCount);
 648
 649     if ($this->_invalidRowCount) {
 650       $store->set('errorsFileName', $this->_errorFileName);
 651     }
 652
 653     if (isset($this->_rows) && !empty($this->_rows)) {
 654       $store->set('dataValues', $this->_rows);
 655     }
 656   }
 657
 658   /**
 659    * Export data to a CSV file.
 660    *
 661    * @param string $fileName
 662    * @param array $header
 663    * @param array $data
 664    */
 665   public static function exportCSV($fileName, $header, $data) {
 666     $output = [];
 667     $fd = fopen($fileName, 'w');
 668
 669     foreach ($header as $key => $value) {
 670       $header[$key] = "\"$value\"";
 671     }
 672     $config = CRM_Core_Config::singleton();
 673     $output[] = implode($config->fieldSeparator, $header);
 674
 675     foreach ($data as $datum) {
 676       foreach ($datum as $key => $value) {
 677         $datum[$key] = "\"$value\"";
 678       }
 679       $output[] = implode($config->fieldSeparator, $datum);
 680     }
 681     fwrite($fd, implode("\n", $output));
 682     fclose($fd);
 683   }
 684
 685 }