Merge pull request #23343 from eileenmcnaughton/import_parser_job
[civicrm-core.git] / CRM / Activity / Import / Parser / Activity.php
1 <?php
2 /*
3 +--------------------------------------------------------------------+
4 | Copyright CiviCRM LLC. All rights reserved. |
5 | |
6 | This work is published under the GNU AGPLv3 license with some |
7 | permitted exceptions and without any warranty. For full license |
8 | and copyright information, see https://civicrm.org/licensing |
9 +--------------------------------------------------------------------+
10 */
11
12 /**
13 *
14 * @package CRM
15 * @copyright CiviCRM LLC https://civicrm.org/licensing
16 */
17
18
19 /**
20 * Class to parse activity csv files.
21 */
22 class CRM_Activity_Import_Parser_Activity extends CRM_Import_Parser {
23
24 protected $_mapperKeys;
25
26 private $_contactIdIndex;
27
28 /**
29 * Array of successfully imported activity id's
30 *
31 * @var array
32 */
33 protected $_newActivity;
34
35 protected $_fileName;
36
37 /**
38 * Imported file size.
39 * @var int
40 */
41 protected $_fileSize;
42
43 /**
44 * Separator being used.
45 * @var string
46 */
47 protected $_separator;
48
49 /**
50 * Total number of lines in file.
51 * @var int
52 */
53 protected $_lineCount;
54
55 /**
56 * Whether the file has a column header or not.
57 *
58 * @var bool
59 */
60 protected $_haveColumnHeader;
61
62 /**
63 * Class constructor.
64 *
65 * @param array $mapperKeys
66 */
67 public function __construct($mapperKeys = []) {
68 parent::__construct();
69 $this->_mapperKeys = $mapperKeys;
70 }
71
72 /**
73 * The initializer code, called before the processing.
74 */
75 public function init() {
76 $activityContact = CRM_Activity_BAO_ActivityContact::import();
77 $activityTarget['target_contact_id'] = $activityContact['contact_id'];
78 $fields = array_merge(CRM_Activity_BAO_Activity::importableFields(),
79 $activityTarget
80 );
81
82 $fields = array_merge($fields, [
83 'source_contact_id' => [
84 'title' => ts('Source Contact'),
85 'headerPattern' => '/Source.Contact?/i',
86 ],
87 'activity_label' => [
88 'title' => ts('Activity Type Label'),
89 'headerPattern' => '/(activity.)?type label?/i',
90 ],
91 ]);
92
93 foreach ($fields as $name => $field) {
94 $field['type'] = CRM_Utils_Array::value('type', $field, CRM_Utils_Type::T_INT);
95 $field['dataPattern'] = CRM_Utils_Array::value('dataPattern', $field, '//');
96 $field['headerPattern'] = CRM_Utils_Array::value('headerPattern', $field, '//');
97 if (!empty($field['custom_group_id'])) {
98 $field['title'] = $field["groupTitle"] . ' :: ' . $field["title"];
99 }
100 $this->addField($name, $field['title'], $field['type'], $field['headerPattern'], $field['dataPattern']);
101 }
102
103 $this->_newActivity = [];
104
105 $this->setActiveFields($this->_mapperKeys);
106
107 // FIXME: we should do this in one place together with Form/MapField.php
108 $this->_contactIdIndex = -1;
109
110 $index = 0;
111 foreach ($this->_mapperKeys as $key) {
112 switch ($key) {
113 case 'target_contact_id':
114 case 'external_identifier':
115 $this->_contactIdIndex = $index;
116 break;
117 }
118 $index++;
119 }
120 }
121
122 /**
123 * Handle the values in preview mode.
124 *
125 * @param array $values
126 * The array of values belonging to this line.
127 *
128 * @return bool
129 * the result of this processing
130 */
131 public function preview(&$values) {
132 return $this->summary($values);
133 }
134
135 /**
136 * Handle the values in summary mode.
137 *
138 * @param array $values
139 * The array of values belonging to this line.
140 *
141 * @return bool
142 * the result of this processing
143 */
144 public function summary(&$values) {
145 try {
146 $this->validateValues($values);
147 }
148 catch (CRM_Core_Exception $e) {
149 return $this->addError($values, [$e->getMessage()]);
150 }
151
152 return CRM_Import_Parser::VALID;
153 }
154
155 /**
156 * Handle the values in import mode.
157 *
158 * @param int $onDuplicate
159 * The code for what action to take on duplicates.
160 * @param array $values
161 * The array of values belonging to this line.
162 *
163 * @return bool
164 * the result of this processing
165 * @throws \CRM_Core_Exception
166 */
167 public function import($onDuplicate, &$values) {
168 // First make sure this is a valid line
169 try {
170 $this->validateValues($values);
171 }
172 catch (CRM_Core_Exception $e) {
173 return $this->addError($values, [$e->getMessage()]);
174 }
175 $params = $this->getApiReadyParams($values);
176 // For date-Formats.
177 $session = CRM_Core_Session::singleton();
178 $dateType = $session->get('dateTypes');
179
180 $customFields = CRM_Core_BAO_CustomField::getFields('Activity');
181
182 foreach ($params as $key => $val) {
183 if ($customFieldID = CRM_Core_BAO_CustomField::getKeyID($key)) {
184 if (!empty($customFields[$customFieldID]) && $customFields[$customFieldID]['data_type'] == 'Date') {
185 CRM_Contact_Import_Parser_Contact::formatCustomDate($params, $params, $dateType, $key);
186 }
187 elseif (!empty($customFields[$customFieldID]) && $customFields[$customFieldID]['data_type'] == 'Boolean') {
188 $params[$key] = CRM_Utils_String::strtoboolstr($val);
189 }
190 }
191 elseif ($key === 'activity_date_time') {
192 $params[$key] = CRM_Utils_Date::formatDate($val, $dateType);
193 }
194 elseif ($key === 'activity_subject') {
195 $params['subject'] = $val;
196 }
197 }
198
199 if ($this->_contactIdIndex < 0) {
200
201 // Retrieve contact id using contact dedupe rule.
202 // Since we are supporting only individual's activity import.
203 $params['contact_type'] = 'Individual';
204 $params['version'] = 3;
205 $error = _civicrm_api3_deprecated_duplicate_formatted_contact($params);
206
207 if (CRM_Core_Error::isAPIError($error, CRM_Core_ERROR::DUPLICATE_CONTACT)) {
208 $matchedIDs = explode(',', $error['error_message']['params'][0]);
209 if (count($matchedIDs) > 1) {
210 array_unshift($values, 'Multiple matching contact records detected for this row. The activity was not imported');
211 return CRM_Import_Parser::ERROR;
212 }
213 $cid = $matchedIDs[0];
214 $params['target_contact_id'] = $cid;
215 $params['version'] = 3;
216 $newActivity = civicrm_api('activity', 'create', $params);
217 if (!empty($newActivity['is_error'])) {
218 array_unshift($values, $newActivity['error_message']);
219 return CRM_Import_Parser::ERROR;
220 }
221
222 $this->_newActivity[] = $newActivity['id'];
223 return CRM_Import_Parser::VALID;
224
225 }
226 // Using new Dedupe rule.
227 $ruleParams = [
228 'contact_type' => 'Individual',
229 'used' => 'Unsupervised',
230 ];
231 $fieldsArray = CRM_Dedupe_BAO_DedupeRule::dedupeRuleFields($ruleParams);
232
233 $disp = NULL;
234 foreach ($fieldsArray as $value) {
235 if (array_key_exists(trim($value), $params)) {
236 $paramValue = $params[trim($value)];
237 if (is_array($paramValue)) {
238 $disp .= $params[trim($value)][0][trim($value)] . " ";
239 }
240 else {
241 $disp .= $params[trim($value)] . " ";
242 }
243 }
244 }
245
246 if (!empty($params['external_identifier'])) {
247 if ($disp) {
248 $disp .= "AND {$params['external_identifier']}";
249 }
250 else {
251 $disp = $params['external_identifier'];
252 }
253 }
254
255 array_unshift($values, 'No matching Contact found for (' . $disp . ')');
256 return CRM_Import_Parser::ERROR;
257 }
258 if (!empty($params['external_identifier'])) {
259 $targetContactId = CRM_Core_DAO::getFieldValue('CRM_Contact_DAO_Contact',
260 $params['external_identifier'], 'id', 'external_identifier'
261 );
262
263 if (!empty($params['target_contact_id']) &&
264 $params['target_contact_id'] != $targetContactId
265 ) {
266 array_unshift($values, 'Mismatch of External ID:' . $params['external_identifier'] . ' and Contact Id:' . $params['target_contact_id']);
267 return CRM_Import_Parser::ERROR;
268 }
269 if ($targetContactId) {
270 $params['target_contact_id'] = $targetContactId;
271 }
272 else {
273 array_unshift($values, 'No Matching Contact for External ID:' . $params['external_identifier']);
274 return CRM_Import_Parser::ERROR;
275 }
276 }
277
278 $params['version'] = 3;
279 $newActivity = civicrm_api('activity', 'create', $params);
280 if (!empty($newActivity['is_error'])) {
281 array_unshift($values, $newActivity['error_message']);
282 return CRM_Import_Parser::ERROR;
283 }
284
285 $this->_newActivity[] = $newActivity['id'];
286 return CRM_Import_Parser::VALID;
287 }
288
289 /**
290 *
291 * Get the value for the given field from the row of values.
292 *
293 * @param array $row
294 * @param string $fieldName
295 *
296 * @return null|string
297 */
298 protected function getFieldValue(array $row, string $fieldName) {
299 if (!is_numeric($this->getFieldIndex($fieldName))) {
300 return NULL;
301 }
302 return $row[$this->getFieldIndex($fieldName)] ?? NULL;
303 }
304
305 /**
306 * Get the index for the given field.
307 *
308 * @param string $fieldName
309 *
310 * @return false|int
311 */
312 protected function getFieldIndex(string $fieldName) {
313 return array_search($fieldName, $this->_mapperKeys, TRUE);
314
315 }
316
317 /**
318 * Add an error to the values.
319 *
320 * @param array $values
321 * @param array $error
322 *
323 * @return int
324 */
325 protected function addError(array &$values, array $error): int {
326 array_unshift($values, implode(';', $error));
327 return CRM_Import_Parser::ERROR;
328 }
329
330 /**
331 * Validate that the activity type id does not conflict with the label.
332 *
333 * @param array $values
334 *
335 * @return void
336 * @throws \CRM_Core_Exception
337 */
338 protected function validateActivityTypeIDAndLabel(array $values): void {
339 $activityLabel = $this->getFieldValue($values, 'activity_label');
340 $activityTypeID = $this->getFieldValue($values, 'activity_type_id');
341 if ($activityLabel && $activityTypeID
342 && $activityLabel !== CRM_Core_PseudoConstant::getLabel('CRM_Activity_BAO_Activity', 'activity_type_id', $activityTypeID)) {
343 throw new CRM_Core_Exception(ts('Activity type label and Activity type ID are in conflict'));
344 }
345 }
346
347 /**
348 * Is the supplied date field valid based on selected date format.
349 *
350 * @param string $value
351 *
352 * @return bool
353 */
354 protected function isValidDate(string $value): bool {
355 return (bool) CRM_Utils_Date::formatDate($value, CRM_Core_Session::singleton()->get('dateTypes'));
356 }
357
358 /**
359 * Is the supplied field a valid contact id.
360 *
361 * @param string|int $value
362 *
363 * @return bool
364 */
365 protected function isValidContactID($value): bool {
366 if (!CRM_Utils_Rule::integer($value)) {
367 return FALSE;
368 }
369 if (!CRM_Core_DAO::singleValueQuery("SELECT id FROM civicrm_contact WHERE id = " . (int) $value)) {
370 return FALSE;
371 }
372 return TRUE;
373 }
374
375 /**
376 * Validate custom fields.
377 *
378 * @param array $values
379 *
380 * @throws \CRM_Core_Exception
381 */
382 protected function validateCustomFields($values):void {
383 $this->setActiveFieldValues($values);
384 $params = $this->getActiveFieldParams();
385 $errorMessage = NULL;
386 // Checking error in custom data.
387 $params['contact_type'] = 'Activity';
388 CRM_Contact_Import_Parser_Contact::isErrorInCustomData($params, $errorMessage);
389 if ($errorMessage) {
390 throw new CRM_Core_Exception('Invalid value for field(s) : ' . $errorMessage);
391 }
392 }
393
394 /**
395 * @param array $values
396 *
397 * @throws \CRM_Core_Exception
398 */
399 protected function validateValues(array $values): void {
400 // Check required fields if this is not an update.
401 if (!$this->getFieldValue($values, 'activity_id')) {
402 if (!$this->getFieldValue($values, 'activity_label')
403 && !$this->getFieldValue($values, 'activity_type_id')) {
404 throw new CRM_Core_Exception(ts('Missing required fields: Activity type label or Activity type ID'));
405 }
406 if (!$this->getFieldValue($values, 'activity_date_time')) {
407 throw new CRM_Core_Exception(ts('Missing required fields'));
408 }
409 }
410
411 $this->validateActivityTypeIDAndLabel($values);
412 if ($this->getFieldValue($values, 'activity_date_time')
413 && !$this->isValidDate($this->getFieldValue($values, 'activity_date_time'))) {
414 throw new CRM_Core_Exception(ts('Invalid Activity Date'));
415 }
416
417 if ($this->getFieldValue($values, 'activity_engagement_level')
418 && !CRM_Utils_Rule::positiveInteger($this->getFieldValue($values, 'activity_engagement_level'))) {
419 throw new CRM_Core_Exception(ts('Activity Engagement Index'));
420 }
421
422 $targetContactID = $this->getFieldValue($values, 'target_contact_id');
423 if ($targetContactID && !$this->isValidContactID($targetContactID)) {
424 throw new CRM_Core_Exception("Invalid Contact ID: There is no contact record with contact_id = " . CRM_Utils_Type::escape($targetContactID, 'String'));
425 }
426 $this->validateCustomFields($values);
427 }
428
429 /**
430 * Get array of parameters formatted for the api from the submitted values.
431 *
432 * @param array $values
433 *
434 * @return array
435 */
436 protected function getApiReadyParams(array $values): array {
437 $this->setActiveFieldValues($values);
438 $params = $this->getActiveFieldParams();
439 if ($this->getFieldValue($values, 'activity_label')) {
440 $params['activity_type_id'] = array_search(
441 $this->getFieldValue($values, 'activity_label'),
442 CRM_Activity_BAO_Activity::buildOptions('activity_type_id', 'create'),
443 TRUE
444 );
445 }
446 return $params;
447 }
448
449 /**
450 * @param array $fileName
451 * @param string $separator
452 * @param $mapper
453 * @param bool $skipColumnHeader
454 * @param int $mode
455 * @param int $onDuplicate
456 * @param int $statusID
457 * @param int $totalRowCount
458 *
459 * @return mixed
460 * @throws Exception
461 */
462 public function run(
463 array $fileName,
464 $separator,
465 $mapper,
466 $skipColumnHeader = FALSE,
467 $mode = self::MODE_PREVIEW,
468 $onDuplicate = self::DUPLICATE_SKIP,
469 $statusID = NULL,
470 $totalRowCount = NULL
471 ) {
472
473 $fileName = $fileName['name'];
474
475 $this->init();
476
477 $this->_haveColumnHeader = $skipColumnHeader;
478
479 $this->_separator = $separator;
480
481 $fd = fopen($fileName, "r");
482 if (!$fd) {
483 return FALSE;
484 }
485
486 $this->_lineCount = 0;
487 $this->_invalidRowCount = $this->_validCount = 0;
488 $this->_totalCount = 0;
489
490 $this->_errors = [];
491 $this->_warnings = [];
492
493 $this->_fileSize = number_format(filesize($fileName) / 1024.0, 2);
494
495 if ($mode == self::MODE_MAPFIELD) {
496 $this->_rows = [];
497 }
498 else {
499 $this->_activeFieldCount = count($this->_activeFields);
500 }
501 if ($statusID) {
502 $this->progressImport($statusID);
503 $startTimestamp = $currTimestamp = $prevTimestamp = time();
504 }
505
506 while (!feof($fd)) {
507 $this->_lineCount++;
508
509 $values = fgetcsv($fd, 8192, $separator);
510 if (!$values) {
511 continue;
512 }
513
514 self::encloseScrub($values);
515
516 // skip column header if we're not in mapfield mode
517 if ($mode != self::MODE_MAPFIELD && $skipColumnHeader) {
518 $skipColumnHeader = FALSE;
519 continue;
520 }
521
522 // Trim whitespace around the values.
523
524 $empty = TRUE;
525 foreach ($values as $k => $v) {
526 $values[$k] = trim($v, " \t\r\n");
527 }
528
529 if (CRM_Utils_System::isNull($values)) {
530 continue;
531 }
532
533 $this->_totalCount++;
534
535 if ($mode == self::MODE_MAPFIELD) {
536 $returnCode = CRM_Import_Parser::VALID;
537 }
538 elseif ($mode == self::MODE_PREVIEW) {
539 $returnCode = $this->preview($values);
540 }
541 elseif ($mode == self::MODE_SUMMARY) {
542 $returnCode = $this->summary($values);
543 }
544 elseif ($mode == self::MODE_IMPORT) {
545 $returnCode = $this->import($onDuplicate, $values);
546 if ($statusID && (($this->_lineCount % 50) == 0)) {
547 $prevTimestamp = $this->progressImport($statusID, FALSE, $startTimestamp, $prevTimestamp, $totalRowCount);
548 }
549 }
550 else {
551 $returnCode = self::ERROR;
552 }
553
554 // note that a line could be valid but still produce a warning
555 if ($returnCode & self::VALID) {
556 $this->_validCount++;
557 if ($mode == self::MODE_MAPFIELD) {
558 $this->_rows[] = $values;
559 $this->_activeFieldCount = max($this->_activeFieldCount, count($values));
560 }
561 }
562
563 if ($returnCode & self::ERROR) {
564 $this->_invalidRowCount++;
565 $recordNumber = $this->_lineCount;
566 if ($this->_haveColumnHeader) {
567 $recordNumber--;
568 }
569 array_unshift($values, $recordNumber);
570 $this->_errors[] = $values;
571 }
572
573 if ($returnCode & self::DUPLICATE) {
574 $this->_duplicateCount++;
575 $recordNumber = $this->_lineCount;
576 if ($this->_haveColumnHeader) {
577 $recordNumber--;
578 }
579 array_unshift($values, $recordNumber);
580 $this->_duplicates[] = $values;
581 if ($onDuplicate != self::DUPLICATE_SKIP) {
582 $this->_validCount++;
583 }
584 }
585
586 // if we are done processing the maxNumber of lines, break
587 if ($this->_maxLinesToProcess > 0 && $this->_validCount >= $this->_maxLinesToProcess) {
588 break;
589 }
590 }
591
592 fclose($fd);
593
594 if ($mode == self::MODE_PREVIEW || $mode == self::MODE_IMPORT) {
595 $customHeaders = $mapper;
596
597 $customfields = CRM_Core_BAO_CustomField::getFields('Activity');
598 foreach ($customHeaders as $key => $value) {
599 if ($id = CRM_Core_BAO_CustomField::getKeyID($value)) {
600 $customHeaders[$key] = $customfields[$id][0];
601 }
602 }
603 if ($this->_invalidRowCount) {
604 // removed view url for invlaid contacts
605 $headers = array_merge(
606 [ts('Line Number'), ts('Reason')],
607 $customHeaders
608 );
609 $this->_errorFileName = self::errorFileName(self::ERROR);
610 self::exportCSV($this->_errorFileName, $headers, $this->_errors);
611 }
612
613 if ($this->_duplicateCount) {
614 $headers = array_merge(
615 [ts('Line Number'), ts('View Activity History URL')],
616 $customHeaders
617 );
618
619 $this->_duplicateFileName = self::errorFileName(self::DUPLICATE);
620 self::exportCSV($this->_duplicateFileName, $headers, $this->_duplicates);
621 }
622 }
623 }
624
625 /**
626 * Given a list of the importable field keys that the user has selected set the active fields array to this list.
627 *
628 * @param array $fieldKeys
629 */
630 public function setActiveFields($fieldKeys) {
631 $this->_activeFieldCount = count($fieldKeys);
632 foreach ($fieldKeys as $key) {
633 if (empty($this->_fields[$key])) {
634 $this->_activeFields[] = new CRM_Activity_Import_Field('', ts('- do not import -'));
635 }
636 else {
637 $this->_activeFields[] = clone($this->_fields[$key]);
638 }
639 }
640 }
641
642 /**
643 * @param string $name
644 * @param $title
645 * @param int $type
646 * @param string $headerPattern
647 * @param string $dataPattern
648 */
649 public function addField($name, $title, $type = CRM_Utils_Type::T_INT, $headerPattern = '//', $dataPattern = '//') {
650 if (empty($name)) {
651 $this->_fields['doNotImport'] = new CRM_Activity_Import_Field($name, $title, $type, $headerPattern, $dataPattern);
652 }
653 else {
654
655 $tempField = CRM_Contact_BAO_Contact::importableFields('Individual', NULL);
656 if (!array_key_exists($name, $tempField)) {
657 $this->_fields[$name] = new CRM_Activity_Import_Field($name, $title, $type, $headerPattern, $dataPattern);
658 }
659 else {
660 $this->_fields[$name] = new CRM_Contact_Import_Field($name, $title, $type, $headerPattern, $dataPattern, CRM_Utils_Array::value('hasLocationType', $tempField[$name]));
661 }
662 }
663 }
664
665 /**
666 * Store parser values.
667 *
668 * @param CRM_Core_Session $store
669 *
670 * @param int $mode
671 */
672 public function set($store, $mode = self::MODE_SUMMARY) {
673 $store->set('fileSize', $this->_fileSize);
674 $store->set('lineCount', $this->_lineCount);
675 $store->set('separator', $this->_separator);
676 $store->set('fields', $this->getSelectValues());
677
678 $store->set('headerPatterns', $this->getHeaderPatterns());
679 $store->set('dataPatterns', $this->getDataPatterns());
680 $store->set('columnCount', $this->_activeFieldCount);
681
682 $store->set('totalRowCount', $this->_totalCount);
683 $store->set('validRowCount', $this->_validCount);
684 $store->set('invalidRowCount', $this->_invalidRowCount);
685
686 if ($this->_invalidRowCount) {
687 $store->set('errorsFileName', $this->_errorFileName);
688 }
689
690 if (isset($this->_rows) && !empty($this->_rows)) {
691 $store->set('dataValues', $this->_rows);
692 }
693
694 if ($mode == self::MODE_IMPORT) {
695 $store->set('duplicateRowCount', $this->_duplicateCount);
696 if ($this->_duplicateCount) {
697 $store->set('duplicatesFileName', $this->_duplicateFileName);
698 }
699 }
700 }
701
702 /**
703 * Export data to a CSV file.
704 *
705 * @param string $fileName
706 * @param array $header
707 * @param array $data
708 */
709 public static function exportCSV($fileName, $header, $data) {
710 $output = [];
711 $fd = fopen($fileName, 'w');
712
713 foreach ($header as $key => $value) {
714 $header[$key] = "\"$value\"";
715 }
716 $config = CRM_Core_Config::singleton();
717 $output[] = implode($config->fieldSeparator, $header);
718
719 foreach ($data as $datum) {
720 foreach ($datum as $key => $value) {
721 $datum[$key] = "\"$value\"";
722 }
723 $output[] = implode($config->fieldSeparator, $datum);
724 }
725 fwrite($fd, implode("\n", $output));
726 fclose($fd);
727 }
728
729 }