CRM/Activity/Import/Parser/Activity.php

   1 <?php
   2 /*
   3  +--------------------------------------------------------------------+
   4  | Copyright CiviCRM LLC. All rights reserved.                        |
   5  |                                                                    |
   6  | This work is published under the GNU AGPLv3 license with some      |
   7  | permitted exceptions and without any warranty. For full license    |
   8  | and copyright information, see https://civicrm.org/licensing       |
   9  +--------------------------------------------------------------------+
  10  */
  11
  12 /**
  13  *
  14  * @package CRM
  15  * @copyright CiviCRM LLC https://civicrm.org/licensing
  16  */
  17
  18
  19 /**
  20  * Class to parse activity csv files.
  21  */
  22 class CRM_Activity_Import_Parser_Activity extends CRM_Import_Parser {
  23
  24   protected $_mapperKeys;
  25
  26   /**
  27    * Array of successfully imported activity id's
  28    *
  29    * @var array
  30    */
  31   protected $_newActivity;
  32
  33   protected $_fileName;
  34
  35   /**
  36    * Imported file size.
  37    * @var int
  38    */
  39   protected $_fileSize;
  40
  41   /**
  42    * Separator being used.
  43    * @var string
  44    */
  45   protected $_separator;
  46
  47   /**
  48    * Total number of lines in file.
  49    * @var int
  50    */
  51   protected $_lineCount;
  52
  53   /**
  54    * Whether the file has a column header or not.
  55    *
  56    * @var bool
  57    */
  58   protected $_haveColumnHeader;
  59
  60   /**
  61    * Class constructor.
  62    *
  63    * @param array $mapperKeys
  64    */
  65   public function __construct($mapperKeys = []) {
  66     parent::__construct();
  67     $this->_mapperKeys = $mapperKeys;
  68   }
  69
  70   /**
  71    * The initializer code, called before the processing.
  72    */
  73   public function init() {
  74     $activityContact = CRM_Activity_BAO_ActivityContact::import();
  75     $activityTarget['target_contact_id'] = $activityContact['contact_id'];
  76     $fields = array_merge(CRM_Activity_BAO_Activity::importableFields(),
  77       $activityTarget
  78     );
  79
  80     $fields = array_merge($fields, [
  81       'source_contact_id' => [
  82         'title' => ts('Source Contact'),
  83         'headerPattern' => '/Source.Contact?/i',
  84       ],
  85       'activity_label' => [
  86         'title' => ts('Activity Type Label'),
  87         'headerPattern' => '/(activity.)?type label?/i',
  88       ],
  89     ]);
  90
  91     foreach ($fields as $name => $field) {
  92       $field['type'] = CRM_Utils_Array::value('type', $field, CRM_Utils_Type::T_INT);
  93       $field['dataPattern'] = CRM_Utils_Array::value('dataPattern', $field, '//');
  94       $field['headerPattern'] = CRM_Utils_Array::value('headerPattern', $field, '//');
  95       if (!empty($field['custom_group_id'])) {
  96         $field['title'] = $field["groupTitle"] . ' :: ' . $field["title"];
  97       }
  98       $this->addField($name, $field['title'], $field['type'], $field['headerPattern'], $field['dataPattern']);
  99     }
 100
 101     $this->_newActivity = [];
 102
 103     $this->setActiveFields($this->_mapperKeys);
 104   }
 105
 106   /**
 107    * Handle the values in summary mode.
 108    *
 109    * @param array $values
 110    *   The array of values belonging to this line.
 111    *
 112    * @return int
 113    *   CRM_Import_Parser::VALID for success or
 114    *   CRM_Import_Parser::ERROR for error.
 115    */
 116   public function summary(&$values) {
 117     try {
 118       $this->validateValues($values);
 119     }
 120     catch (CRM_Core_Exception $e) {
 121       return $this->addError($values, [$e->getMessage()]);
 122     }
 123
 124     return CRM_Import_Parser::VALID;
 125   }
 126
 127   /**
 128    * Handle the values in import mode.
 129    *
 130    * @param int $onDuplicate
 131    *   The code for what action to take on duplicates.
 132    * @param array $values
 133    *   The array of values belonging to this line.
 134    *
 135    * @return int
 136    *   CRM_Import_Parser::VALID for success or
 137    *   CRM_Import_Parser::ERROR for error.
 138    *
 139    * @throws \CRM_Core_Exception
 140    */
 141   public function import($onDuplicate, &$values) {
 142     // First make sure this is a valid line
 143     try {
 144       $this->validateValues($values);
 145     }
 146     catch (CRM_Core_Exception $e) {
 147       return $this->addError($values, [$e->getMessage()]);
 148     }
 149     $params = $this->getApiReadyParams($values);
 150     // For date-Formats.
 151     $session = CRM_Core_Session::singleton();
 152     $dateType = $session->get('dateTypes');
 153
 154     $customFields = CRM_Core_BAO_CustomField::getFields('Activity');
 155
 156     foreach ($params as $key => $val) {
 157       if ($customFieldID = CRM_Core_BAO_CustomField::getKeyID($key)) {
 158         if (!empty($customFields[$customFieldID]) && $customFields[$customFieldID]['data_type'] == 'Date') {
 159           CRM_Contact_Import_Parser_Contact::formatCustomDate($params, $params, $dateType, $key);
 160         }
 161         elseif (!empty($customFields[$customFieldID]) && $customFields[$customFieldID]['data_type'] == 'Boolean') {
 162           $params[$key] = CRM_Utils_String::strtoboolstr($val);
 163         }
 164       }
 165       elseif ($key === 'activity_date_time') {
 166         $params[$key] = CRM_Utils_Date::formatDate($val, $dateType);
 167       }
 168       elseif ($key === 'activity_subject') {
 169         $params['subject'] = $val;
 170       }
 171     }
 172
 173     if (empty($params['external_identifier']) && empty($params['target_contact_id'])) {
 174
 175       // Retrieve contact id using contact dedupe rule.
 176       // Since we are supporting only individual's activity import.
 177       $params['contact_type'] = 'Individual';
 178       $params['version'] = 3;
 179       $matchedIDs = CRM_Contact_BAO_Contact::getDuplicateContacts($params, 'Individual');
 180
 181       if (!empty($matchedIDs)) {
 182         if (count($matchedIDs) > 1) {
 183           array_unshift($values, 'Multiple matching contact records detected for this row. The activity was not imported');
 184           return CRM_Import_Parser::ERROR;
 185         }
 186         $cid = $matchedIDs[0];
 187         $params['target_contact_id'] = $cid;
 188         $params['version'] = 3;
 189         $newActivity = civicrm_api('activity', 'create', $params);
 190         if (!empty($newActivity['is_error'])) {
 191           array_unshift($values, $newActivity['error_message']);
 192           return CRM_Import_Parser::ERROR;
 193         }
 194
 195         $this->_newActivity[] = $newActivity['id'];
 196         return CRM_Import_Parser::VALID;
 197
 198       }
 199       // Using new Dedupe rule.
 200       $ruleParams = [
 201         'contact_type' => 'Individual',
 202         'used' => 'Unsupervised',
 203       ];
 204       $fieldsArray = CRM_Dedupe_BAO_DedupeRule::dedupeRuleFields($ruleParams);
 205
 206       $disp = NULL;
 207       foreach ($fieldsArray as $value) {
 208         if (array_key_exists(trim($value), $params)) {
 209           $paramValue = $params[trim($value)];
 210           if (is_array($paramValue)) {
 211             $disp .= $params[trim($value)][0][trim($value)] . " ";
 212           }
 213           else {
 214             $disp .= $params[trim($value)] . " ";
 215           }
 216         }
 217       }
 218
 219       if (!empty($params['external_identifier'])) {
 220         if ($disp) {
 221           $disp .= "AND {$params['external_identifier']}";
 222         }
 223         else {
 224           $disp = $params['external_identifier'];
 225         }
 226       }
 227
 228       array_unshift($values, 'No matching Contact found for (' . $disp . ')');
 229       return CRM_Import_Parser::ERROR;
 230     }
 231     if (!empty($params['external_identifier'])) {
 232       $targetContactId = CRM_Core_DAO::getFieldValue('CRM_Contact_DAO_Contact',
 233         $params['external_identifier'], 'id', 'external_identifier'
 234       );
 235
 236       if (!empty($params['target_contact_id']) &&
 237         $params['target_contact_id'] != $targetContactId
 238       ) {
 239         array_unshift($values, 'Mismatch of External ID:' . $params['external_identifier'] . ' and Contact Id:' . $params['target_contact_id']);
 240         return CRM_Import_Parser::ERROR;
 241       }
 242       if ($targetContactId) {
 243         $params['target_contact_id'] = $targetContactId;
 244       }
 245       else {
 246         array_unshift($values, 'No Matching Contact for External ID:' . $params['external_identifier']);
 247         return CRM_Import_Parser::ERROR;
 248       }
 249     }
 250
 251     $params['version'] = 3;
 252     $newActivity = civicrm_api('activity', 'create', $params);
 253     if (!empty($newActivity['is_error'])) {
 254       array_unshift($values, $newActivity['error_message']);
 255       return CRM_Import_Parser::ERROR;
 256     }
 257
 258     $this->_newActivity[] = $newActivity['id'];
 259     return CRM_Import_Parser::VALID;
 260   }
 261
 262   /**
 263    *
 264    * Get the value for the given field from the row of values.
 265    *
 266    * @param array $row
 267    * @param string $fieldName
 268    *
 269    * @return null|string
 270    */
 271   protected function getFieldValue(array $row, string $fieldName) {
 272     if (!is_numeric($this->getFieldIndex($fieldName))) {
 273       return NULL;
 274     }
 275     return $row[$this->getFieldIndex($fieldName)] ?? NULL;
 276   }
 277
 278   /**
 279    * Get the index for the given field.
 280    *
 281    * @param string $fieldName
 282    *
 283    * @return false|int
 284    */
 285   protected function getFieldIndex(string $fieldName) {
 286     return array_search($fieldName, $this->_mapperKeys, TRUE);
 287
 288   }
 289
 290   /**
 291    * Add an error to the values.
 292    *
 293    * @param array $values
 294    * @param array $error
 295    *
 296    * @return int
 297    */
 298   protected function addError(array &$values, array $error): int {
 299     array_unshift($values, implode(';', $error));
 300     return CRM_Import_Parser::ERROR;
 301   }
 302
 303   /**
 304    * Validate that the activity type id does not conflict with the label.
 305    *
 306    * @param array $values
 307    *
 308    * @return void
 309    * @throws \CRM_Core_Exception
 310    */
 311   protected function validateActivityTypeIDAndLabel(array $values): void {
 312     $activityLabel = $this->getFieldValue($values, 'activity_label');
 313     $activityTypeID = $this->getFieldValue($values, 'activity_type_id');
 314     if ($activityLabel && $activityTypeID
 315       && $activityLabel !== CRM_Core_PseudoConstant::getLabel('CRM_Activity_BAO_Activity', 'activity_type_id', $activityTypeID)) {
 316       throw new CRM_Core_Exception(ts('Activity type label and Activity type ID are in conflict'));
 317     }
 318   }
 319
 320   /**
 321    * Is the supplied date field valid based on selected date format.
 322    *
 323    * @param string $value
 324    *
 325    * @return bool
 326    */
 327   protected function isValidDate(string $value): bool {
 328     return (bool) CRM_Utils_Date::formatDate($value, CRM_Core_Session::singleton()->get('dateTypes'));
 329   }
 330
 331   /**
 332    * Is the supplied field a valid contact id.
 333    *
 334    * @param string|int $value
 335    *
 336    * @return bool
 337    */
 338   protected function isValidContactID($value): bool {
 339     if (!CRM_Utils_Rule::integer($value)) {
 340       return FALSE;
 341     }
 342     if (!CRM_Core_DAO::singleValueQuery("SELECT id FROM civicrm_contact WHERE id = " . (int) $value)) {
 343       return FALSE;
 344     }
 345     return TRUE;
 346   }
 347
 348   /**
 349    * Validate custom fields.
 350    *
 351    * @param array $values
 352    *
 353    * @throws \CRM_Core_Exception
 354    */
 355   protected function validateCustomFields($values):void {
 356     $this->setActiveFieldValues($values);
 357     $params = $this->getActiveFieldParams();
 358     $errorMessage = NULL;
 359     // Checking error in custom data.
 360     $params['contact_type'] = 'Activity';
 361     CRM_Contact_Import_Parser_Contact::isErrorInCustomData($params, $errorMessage);
 362     if ($errorMessage) {
 363       throw new CRM_Core_Exception('Invalid value for field(s) : ' . $errorMessage);
 364     }
 365   }
 366
 367   /**
 368    * @param array $values
 369    *
 370    * @throws \CRM_Core_Exception
 371    */
 372   protected function validateValues(array $values): void {
 373     // Check required fields if this is not an update.
 374     if (!$this->getFieldValue($values, 'activity_id')) {
 375       if (!$this->getFieldValue($values, 'activity_label')
 376         && !$this->getFieldValue($values, 'activity_type_id')) {
 377         throw new CRM_Core_Exception(ts('Missing required fields: Activity type label or Activity type ID'));
 378       }
 379       if (!$this->getFieldValue($values, 'activity_date_time')) {
 380         throw new CRM_Core_Exception(ts('Missing required fields'));
 381       }
 382     }
 383
 384     $this->validateActivityTypeIDAndLabel($values);
 385     if ($this->getFieldValue($values, 'activity_date_time')
 386       && !$this->isValidDate($this->getFieldValue($values, 'activity_date_time'))) {
 387       throw new CRM_Core_Exception(ts('Invalid Activity Date'));
 388     }
 389
 390     if ($this->getFieldValue($values, 'activity_engagement_level')
 391       && !CRM_Utils_Rule::positiveInteger($this->getFieldValue($values, 'activity_engagement_level'))) {
 392       throw new CRM_Core_Exception(ts('Activity Engagement Index'));
 393     }
 394
 395     $targetContactID = $this->getFieldValue($values, 'target_contact_id');
 396     if ($targetContactID && !$this->isValidContactID($targetContactID)) {
 397       throw new CRM_Core_Exception("Invalid Contact ID: There is no contact record with contact_id = " . CRM_Utils_Type::escape($targetContactID, 'String'));
 398     }
 399     $this->validateCustomFields($values);
 400   }
 401
 402   /**
 403    * Get array of parameters formatted for the api from the submitted values.
 404    *
 405    * @param array $values
 406    *
 407    * @return array
 408    */
 409   protected function getApiReadyParams(array $values): array {
 410     $this->setActiveFieldValues($values);
 411     $params = $this->getActiveFieldParams();
 412     if ($this->getFieldValue($values, 'activity_label')) {
 413       $params['activity_type_id'] = array_search(
 414          $this->getFieldValue($values, 'activity_label'),
 415          CRM_Activity_BAO_Activity::buildOptions('activity_type_id', 'create'),
 416         TRUE
 417       );
 418     }
 419     return $params;
 420   }
 421
 422   /**
 423    * @param array $fileName
 424    * @param string $separator
 425    * @param $mapper
 426    * @param bool $skipColumnHeader
 427    * @param int $mode
 428    * @param int $onDuplicate
 429    * @param int $statusID
 430    * @param int $totalRowCount
 431    *
 432    * @return mixed
 433    * @throws Exception
 434    */
 435   public function run(
 436     array $fileName,
 437           $separator,
 438           $mapper,
 439           $skipColumnHeader = FALSE,
 440           $mode = self::MODE_PREVIEW,
 441           $onDuplicate = self::DUPLICATE_SKIP,
 442           $statusID = NULL,
 443           $totalRowCount = NULL
 444   ) {
 445
 446     $fileName = $fileName['name'];
 447
 448     $this->init();
 449
 450     $this->_haveColumnHeader = $skipColumnHeader;
 451
 452     $this->_separator = $separator;
 453
 454     $fd = fopen($fileName, "r");
 455     if (!$fd) {
 456       return FALSE;
 457     }
 458
 459     $this->_lineCount = 0;
 460     $this->_invalidRowCount = $this->_validCount = 0;
 461     $this->_totalCount = 0;
 462
 463     $this->_errors = [];
 464     $this->_warnings = [];
 465
 466     $this->_fileSize = number_format(filesize($fileName) / 1024.0, 2);
 467
 468     if ($mode == self::MODE_MAPFIELD) {
 469       $this->_rows = [];
 470     }
 471     else {
 472       $this->_activeFieldCount = count($this->_activeFields);
 473     }
 474     if ($statusID) {
 475       $this->progressImport($statusID);
 476       $startTimestamp = $currTimestamp = $prevTimestamp = time();
 477     }
 478
 479     while (!feof($fd)) {
 480       $this->_lineCount++;
 481
 482       $values = fgetcsv($fd, 8192, $separator);
 483       if (!$values) {
 484         continue;
 485       }
 486
 487       self::encloseScrub($values);
 488
 489       // skip column header if we're not in mapfield mode
 490       if ($mode != self::MODE_MAPFIELD && $skipColumnHeader) {
 491         $skipColumnHeader = FALSE;
 492         continue;
 493       }
 494
 495       // Trim whitespace around the values.
 496
 497       $empty = TRUE;
 498       foreach ($values as $k => $v) {
 499         $values[$k] = trim($v, " \t\r\n");
 500       }
 501
 502       if (CRM_Utils_System::isNull($values)) {
 503         continue;
 504       }
 505
 506       $this->_totalCount++;
 507
 508       if ($mode == self::MODE_MAPFIELD) {
 509         $returnCode = CRM_Import_Parser::VALID;
 510       }
 511       // Note that MODE_SUMMARY seems to be never used.
 512       elseif ($mode == self::MODE_PREVIEW || $mode == self::MODE_SUMMARY) {
 513         $returnCode = $this->summary($values);
 514       }
 515       elseif ($mode == self::MODE_IMPORT) {
 516         $returnCode = $this->import($onDuplicate, $values);
 517         if ($statusID && (($this->_lineCount % 50) == 0)) {
 518           $prevTimestamp = $this->progressImport($statusID, FALSE, $startTimestamp, $prevTimestamp, $totalRowCount);
 519         }
 520       }
 521       else {
 522         $returnCode = self::ERROR;
 523       }
 524
 525       // note that a line could be valid but still produce a warning
 526       if ($returnCode & self::VALID) {
 527         $this->_validCount++;
 528         if ($mode == self::MODE_MAPFIELD) {
 529           $this->_rows[] = $values;
 530           $this->_activeFieldCount = max($this->_activeFieldCount, count($values));
 531         }
 532       }
 533
 534       if ($returnCode & self::ERROR) {
 535         $this->_invalidRowCount++;
 536         $recordNumber = $this->_lineCount;
 537         if ($this->_haveColumnHeader) {
 538           $recordNumber--;
 539         }
 540         array_unshift($values, $recordNumber);
 541         $this->_errors[] = $values;
 542       }
 543
 544       // if we are done processing the maxNumber of lines, break
 545       if ($this->_maxLinesToProcess > 0 && $this->_validCount >= $this->_maxLinesToProcess) {
 546         break;
 547       }
 548     }
 549
 550     fclose($fd);
 551
 552     if ($mode == self::MODE_PREVIEW || $mode == self::MODE_IMPORT) {
 553       $customHeaders = $mapper;
 554
 555       $customfields = CRM_Core_BAO_CustomField::getFields('Activity');
 556       foreach ($customHeaders as $key => $value) {
 557         if ($id = CRM_Core_BAO_CustomField::getKeyID($value)) {
 558           $customHeaders[$key] = $customfields[$id][0];
 559         }
 560       }
 561       if ($this->_invalidRowCount) {
 562         // removed view url for invlaid contacts
 563         $headers = array_merge(
 564           [ts('Line Number'), ts('Reason')],
 565           $customHeaders
 566         );
 567         $this->_errorFileName = self::errorFileName(self::ERROR);
 568         self::exportCSV($this->_errorFileName, $headers, $this->_errors);
 569       }
 570     }
 571   }
 572
 573   /**
 574    * Given a list of the importable field keys that the user has selected set the active fields array to this list.
 575    *
 576    * @param array $fieldKeys
 577    */
 578   public function setActiveFields($fieldKeys) {
 579     $this->_activeFieldCount = count($fieldKeys);
 580     foreach ($fieldKeys as $key) {
 581       if (empty($this->_fields[$key])) {
 582         $this->_activeFields[] = new CRM_Activity_Import_Field('', ts('- do not import -'));
 583       }
 584       else {
 585         $this->_activeFields[] = clone($this->_fields[$key]);
 586       }
 587     }
 588   }
 589
 590   /**
 591    * @param string $name
 592    * @param $title
 593    * @param int $type
 594    * @param string $headerPattern
 595    * @param string $dataPattern
 596    */
 597   public function addField($name, $title, $type = CRM_Utils_Type::T_INT, $headerPattern = '//', $dataPattern = '//') {
 598     if (empty($name)) {
 599       $this->_fields['doNotImport'] = new CRM_Activity_Import_Field($name, $title, $type, $headerPattern, $dataPattern);
 600     }
 601     else {
 602
 603       $tempField = CRM_Contact_BAO_Contact::importableFields('Individual', NULL);
 604       if (!array_key_exists($name, $tempField)) {
 605         $this->_fields[$name] = new CRM_Activity_Import_Field($name, $title, $type, $headerPattern, $dataPattern);
 606       }
 607       else {
 608         $this->_fields[$name] = new CRM_Contact_Import_Field($name, $title, $type, $headerPattern, $dataPattern, CRM_Utils_Array::value('hasLocationType', $tempField[$name]));
 609       }
 610     }
 611   }
 612
 613   /**
 614    * Store parser values.
 615    *
 616    * @param CRM_Core_Session $store
 617    */
 618   public function set($store) {
 619     $store->set('fileSize', $this->_fileSize);
 620     $store->set('lineCount', $this->_lineCount);
 621     $store->set('separator', $this->_separator);
 622     $store->set('fields', $this->getSelectValues());
 623
 624     $store->set('headerPatterns', $this->getHeaderPatterns());
 625     $store->set('dataPatterns', $this->getDataPatterns());
 626     $store->set('columnCount', $this->_activeFieldCount);
 627
 628     $store->set('totalRowCount', $this->_totalCount);
 629     $store->set('validRowCount', $this->_validCount);
 630     $store->set('invalidRowCount', $this->_invalidRowCount);
 631
 632     if ($this->_invalidRowCount) {
 633       $store->set('errorsFileName', $this->_errorFileName);
 634     }
 635
 636     if (isset($this->_rows) && !empty($this->_rows)) {
 637       $store->set('dataValues', $this->_rows);
 638     }
 639   }
 640
 641   /**
 642    * Export data to a CSV file.
 643    *
 644    * @param string $fileName
 645    * @param array $header
 646    * @param array $data
 647    */
 648   public static function exportCSV($fileName, $header, $data) {
 649     $output = [];
 650     $fd = fopen($fileName, 'w');
 651
 652     foreach ($header as $key => $value) {
 653       $header[$key] = "\"$value\"";
 654     }
 655     $config = CRM_Core_Config::singleton();
 656     $output[] = implode($config->fieldSeparator, $header);
 657
 658     foreach ($data as $datum) {
 659       foreach ($datum as $key => $value) {
 660         $datum[$key] = "\"$value\"";
 661       }
 662       $output[] = implode($config->fieldSeparator, $datum);
 663     }
 664     fwrite($fd, implode("\n", $output));
 665     fclose($fd);
 666   }
 667
 668 }