3 +--------------------------------------------------------------------+
4 | Copyright CiviCRM LLC. All rights reserved. |
6 | This work is published under the GNU AGPLv3 license with some |
7 | permitted exceptions and without any warranty. For full license |
8 | and copyright information, see https://civicrm.org/licensing |
9 +--------------------------------------------------------------------+
15 * @copyright CiviCRM LLC https://civicrm.org/licensing
20 * Class to parse activity csv files.
22 class CRM_Activity_Import_Parser_Activity
extends CRM_Import_Parser
{
24 protected $_mapperKeys;
26 private $_contactIdIndex;
29 * Array of successfully imported activity id's
33 protected $_newActivity;
44 * Separator being used.
47 protected $_separator;
50 * Total number of lines in file.
53 protected $_lineCount;
56 * Whether the file has a column header or not.
60 protected $_haveColumnHeader;
65 * @param array $mapperKeys
67 public function __construct($mapperKeys = []) {
68 parent
::__construct();
69 $this->_mapperKeys
= $mapperKeys;
73 * The initializer code, called before the processing.
75 public function init() {
76 $activityContact = CRM_Activity_BAO_ActivityContact
::import();
77 $activityTarget['target_contact_id'] = $activityContact['contact_id'];
78 $fields = array_merge(CRM_Activity_BAO_Activity
::importableFields(),
82 $fields = array_merge($fields, [
83 'source_contact_id' => [
84 'title' => ts('Source Contact'),
85 'headerPattern' => '/Source.Contact?/i',
88 'title' => ts('Activity Type Label'),
89 'headerPattern' => '/(activity.)?type label?/i',
93 foreach ($fields as $name => $field) {
94 $field['type'] = CRM_Utils_Array
::value('type', $field, CRM_Utils_Type
::T_INT
);
95 $field['dataPattern'] = CRM_Utils_Array
::value('dataPattern', $field, '//');
96 $field['headerPattern'] = CRM_Utils_Array
::value('headerPattern', $field, '//');
97 if (!empty($field['custom_group_id'])) {
98 $field['title'] = $field["groupTitle"] . ' :: ' . $field["title"];
100 $this->addField($name, $field['title'], $field['type'], $field['headerPattern'], $field['dataPattern']);
103 $this->_newActivity
= [];
105 $this->setActiveFields($this->_mapperKeys
);
107 // FIXME: we should do this in one place together with Form/MapField.php
108 $this->_contactIdIndex
= -1;
111 foreach ($this->_mapperKeys
as $key) {
113 case 'target_contact_id':
114 case 'external_identifier':
115 $this->_contactIdIndex
= $index;
123 * Handle the values in preview mode.
125 * @param array $values
126 * The array of values belonging to this line.
129 * the result of this processing
131 public function preview(&$values) {
132 return $this->summary($values);
136 * Handle the values in summary mode.
138 * @param array $values
139 * The array of values belonging to this line.
142 * the result of this processing
144 public function summary(&$values) {
146 $this->validateValues($values);
148 catch (CRM_Core_Exception
$e) {
149 return $this->addError($values, [$e->getMessage()]);
152 return CRM_Import_Parser
::VALID
;
156 * Handle the values in import mode.
158 * @param int $onDuplicate
159 * The code for what action to take on duplicates.
160 * @param array $values
161 * The array of values belonging to this line.
164 * the result of this processing
165 * @throws \CRM_Core_Exception
167 public function import($onDuplicate, &$values) {
168 // First make sure this is a valid line
170 $this->validateValues($values);
172 catch (CRM_Core_Exception
$e) {
173 return $this->addError($values, [$e->getMessage()]);
175 $params = $this->getApiReadyParams($values);
177 $session = CRM_Core_Session
::singleton();
178 $dateType = $session->get('dateTypes');
180 $customFields = CRM_Core_BAO_CustomField
::getFields('Activity');
182 foreach ($params as $key => $val) {
183 if ($customFieldID = CRM_Core_BAO_CustomField
::getKeyID($key)) {
184 if (!empty($customFields[$customFieldID]) && $customFields[$customFieldID]['data_type'] == 'Date') {
185 CRM_Contact_Import_Parser_Contact
::formatCustomDate($params, $params, $dateType, $key);
187 elseif (!empty($customFields[$customFieldID]) && $customFields[$customFieldID]['data_type'] == 'Boolean') {
188 $params[$key] = CRM_Utils_String
::strtoboolstr($val);
191 elseif ($key === 'activity_date_time') {
192 $params[$key] = CRM_Utils_Date
::formatDate($val, $dateType);
194 elseif ($key === 'activity_subject') {
195 $params['subject'] = $val;
199 if ($this->_contactIdIndex
< 0) {
201 // Retrieve contact id using contact dedupe rule.
202 // Since we are supporting only individual's activity import.
203 $params['contact_type'] = 'Individual';
204 $params['version'] = 3;
205 $error = _civicrm_api3_deprecated_duplicate_formatted_contact($params);
207 if (CRM_Core_Error
::isAPIError($error, CRM_Core_ERROR
::DUPLICATE_CONTACT
)) {
208 $matchedIDs = explode(',', $error['error_message']['params'][0]);
209 if (count($matchedIDs) > 1) {
210 array_unshift($values, 'Multiple matching contact records detected for this row. The activity was not imported');
211 return CRM_Import_Parser
::ERROR
;
213 $cid = $matchedIDs[0];
214 $params['target_contact_id'] = $cid;
215 $params['version'] = 3;
216 $newActivity = civicrm_api('activity', 'create', $params);
217 if (!empty($newActivity['is_error'])) {
218 array_unshift($values, $newActivity['error_message']);
219 return CRM_Import_Parser
::ERROR
;
222 $this->_newActivity
[] = $newActivity['id'];
223 return CRM_Import_Parser
::VALID
;
226 // Using new Dedupe rule.
228 'contact_type' => 'Individual',
229 'used' => 'Unsupervised',
231 $fieldsArray = CRM_Dedupe_BAO_DedupeRule
::dedupeRuleFields($ruleParams);
234 foreach ($fieldsArray as $value) {
235 if (array_key_exists(trim($value), $params)) {
236 $paramValue = $params[trim($value)];
237 if (is_array($paramValue)) {
238 $disp .= $params[trim($value)][0][trim($value)] . " ";
241 $disp .= $params[trim($value)] . " ";
246 if (!empty($params['external_identifier'])) {
248 $disp .= "AND {$params['external_identifier']}";
251 $disp = $params['external_identifier'];
255 array_unshift($values, 'No matching Contact found for (' . $disp . ')');
256 return CRM_Import_Parser
::ERROR
;
258 if (!empty($params['external_identifier'])) {
259 $targetContactId = CRM_Core_DAO
::getFieldValue('CRM_Contact_DAO_Contact',
260 $params['external_identifier'], 'id', 'external_identifier'
263 if (!empty($params['target_contact_id']) &&
264 $params['target_contact_id'] != $targetContactId
266 array_unshift($values, 'Mismatch of External ID:' . $params['external_identifier'] . ' and Contact Id:' . $params['target_contact_id']);
267 return CRM_Import_Parser
::ERROR
;
269 if ($targetContactId) {
270 $params['target_contact_id'] = $targetContactId;
273 array_unshift($values, 'No Matching Contact for External ID:' . $params['external_identifier']);
274 return CRM_Import_Parser
::ERROR
;
278 $params['version'] = 3;
279 $newActivity = civicrm_api('activity', 'create', $params);
280 if (!empty($newActivity['is_error'])) {
281 array_unshift($values, $newActivity['error_message']);
282 return CRM_Import_Parser
::ERROR
;
285 $this->_newActivity
[] = $newActivity['id'];
286 return CRM_Import_Parser
::VALID
;
291 * Get the value for the given field from the row of values.
294 * @param string $fieldName
296 * @return null|string
298 protected function getFieldValue(array $row, string $fieldName) {
299 if (!is_numeric($this->getFieldIndex($fieldName))) {
302 return $row[$this->getFieldIndex($fieldName)] ??
NULL;
306 * Get the index for the given field.
308 * @param string $fieldName
312 protected function getFieldIndex(string $fieldName) {
313 return array_search($fieldName, $this->_mapperKeys
, TRUE);
318 * Add an error to the values.
320 * @param array $values
321 * @param array $error
325 protected function addError(array &$values, array $error): int {
326 array_unshift($values, implode(';', $error));
327 return CRM_Import_Parser
::ERROR
;
331 * Validate that the activity type id does not conflict with the label.
333 * @param array $values
336 * @throws \CRM_Core_Exception
338 protected function validateActivityTypeIDAndLabel(array $values): void
{
339 $activityLabel = $this->getFieldValue($values, 'activity_label');
340 $activityTypeID = $this->getFieldValue($values, 'activity_type_id');
341 if ($activityLabel && $activityTypeID
342 && $activityLabel !== CRM_Core_PseudoConstant
::getLabel('CRM_Activity_BAO_Activity', 'activity_type_id', $activityTypeID)) {
343 throw new CRM_Core_Exception(ts('Activity type label and Activity type ID are in conflict'));
348 * Is the supplied date field valid based on selected date format.
350 * @param string $value
354 protected function isValidDate(string $value): bool {
355 return (bool) CRM_Utils_Date
::formatDate($value, CRM_Core_Session
::singleton()->get('dateTypes'));
359 * Is the supplied field a valid contact id.
361 * @param string|int $value
365 protected function isValidContactID($value): bool {
366 if (!CRM_Utils_Rule
::integer($value)) {
369 if (!CRM_Core_DAO
::singleValueQuery("SELECT id FROM civicrm_contact WHERE id = " . (int) $value)) {
376 * Validate custom fields.
378 * @param array $values
380 * @throws \CRM_Core_Exception
382 protected function validateCustomFields($values):void
{
383 $this->setActiveFieldValues($values);
384 $params = $this->getActiveFieldParams();
385 $errorMessage = NULL;
386 // Checking error in custom data.
387 $params['contact_type'] = 'Activity';
388 CRM_Contact_Import_Parser_Contact
::isErrorInCustomData($params, $errorMessage);
390 throw new CRM_Core_Exception('Invalid value for field(s) : ' . $errorMessage);
395 * @param array $values
397 * @throws \CRM_Core_Exception
399 protected function validateValues(array $values): void
{
400 // Check required fields if this is not an update.
401 if (!$this->getFieldValue($values, 'activity_id')) {
402 if (!$this->getFieldValue($values, 'activity_label')
403 && !$this->getFieldValue($values, 'activity_type_id')) {
404 throw new CRM_Core_Exception(ts('Missing required fields: Activity type label or Activity type ID'));
406 if (!$this->getFieldValue($values, 'activity_date_time')) {
407 throw new CRM_Core_Exception(ts('Missing required fields'));
411 $this->validateActivityTypeIDAndLabel($values);
412 if ($this->getFieldValue($values, 'activity_date_time')
413 && !$this->isValidDate($this->getFieldValue($values, 'activity_date_time'))) {
414 throw new CRM_Core_Exception(ts('Invalid Activity Date'));
417 if ($this->getFieldValue($values, 'activity_engagement_level')
418 && !CRM_Utils_Rule
::positiveInteger($this->getFieldValue($values, 'activity_engagement_level'))) {
419 throw new CRM_Core_Exception(ts('Activity Engagement Index'));
422 $targetContactID = $this->getFieldValue($values, 'target_contact_id');
423 if ($targetContactID && !$this->isValidContactID($targetContactID)) {
424 throw new CRM_Core_Exception("Invalid Contact ID: There is no contact record with contact_id = " . CRM_Utils_Type
::escape($targetContactID, 'String'));
426 $this->validateCustomFields($values);
430 * Get array of parameters formatted for the api from the submitted values.
432 * @param array $values
436 protected function getApiReadyParams(array $values): array {
437 $this->setActiveFieldValues($values);
438 $params = $this->getActiveFieldParams();
439 if ($this->getFieldValue($values, 'activity_label')) {
440 $params['activity_type_id'] = array_search(
441 $this->getFieldValue($values, 'activity_label'),
442 CRM_Activity_BAO_Activity
::buildOptions('activity_type_id', 'create'),
450 * @param array $fileName
451 * @param string $separator
453 * @param bool $skipColumnHeader
455 * @param int $onDuplicate
456 * @param int $statusID
457 * @param int $totalRowCount
466 $skipColumnHeader = FALSE,
467 $mode = self
::MODE_PREVIEW
,
468 $onDuplicate = self
::DUPLICATE_SKIP
,
470 $totalRowCount = NULL
473 $fileName = $fileName['name'];
477 $this->_haveColumnHeader
= $skipColumnHeader;
479 $this->_separator
= $separator;
481 $fd = fopen($fileName, "r");
486 $this->_lineCount
= 0;
487 $this->_invalidRowCount
= $this->_validCount
= 0;
488 $this->_totalCount
= $this->_conflictCount
= 0;
491 $this->_warnings
= [];
492 $this->_conflicts
= [];
494 $this->_fileSize
= number_format(filesize($fileName) / 1024.0, 2);
496 if ($mode == self
::MODE_MAPFIELD
) {
500 $this->_activeFieldCount
= count($this->_activeFields
);
503 $this->progressImport($statusID);
504 $startTimestamp = $currTimestamp = $prevTimestamp = time();
510 $values = fgetcsv($fd, 8192, $separator);
515 self
::encloseScrub($values);
517 // skip column header if we're not in mapfield mode
518 if ($mode != self
::MODE_MAPFIELD
&& $skipColumnHeader) {
519 $skipColumnHeader = FALSE;
523 // Trim whitespace around the values.
526 foreach ($values as $k => $v) {
527 $values[$k] = trim($v, " \t\r\n");
530 if (CRM_Utils_System
::isNull($values)) {
534 $this->_totalCount++
;
536 if ($mode == self
::MODE_MAPFIELD
) {
537 $returnCode = CRM_Import_Parser
::VALID
;
539 elseif ($mode == self
::MODE_PREVIEW
) {
540 $returnCode = $this->preview($values);
542 elseif ($mode == self
::MODE_SUMMARY
) {
543 $returnCode = $this->summary($values);
545 elseif ($mode == self
::MODE_IMPORT
) {
546 $returnCode = $this->import($onDuplicate, $values);
547 if ($statusID && (($this->_lineCount %
50) == 0)) {
548 $prevTimestamp = $this->progressImport($statusID, FALSE, $startTimestamp, $prevTimestamp, $totalRowCount);
552 $returnCode = self
::ERROR
;
555 // note that a line could be valid but still produce a warning
556 if ($returnCode & self
::VALID
) {
557 $this->_validCount++
;
558 if ($mode == self
::MODE_MAPFIELD
) {
559 $this->_rows
[] = $values;
560 $this->_activeFieldCount
= max($this->_activeFieldCount
, count($values));
564 if ($returnCode & self
::ERROR
) {
565 $this->_invalidRowCount++
;
566 $recordNumber = $this->_lineCount
;
567 if ($this->_haveColumnHeader
) {
570 array_unshift($values, $recordNumber);
571 $this->_errors
[] = $values;
574 if ($returnCode & self
::CONFLICT
) {
575 $this->_conflictCount++
;
576 $recordNumber = $this->_lineCount
;
577 if ($this->_haveColumnHeader
) {
580 array_unshift($values, $recordNumber);
581 $this->_conflicts
[] = $values;
584 if ($returnCode & self
::DUPLICATE
) {
585 $this->_duplicateCount++
;
586 $recordNumber = $this->_lineCount
;
587 if ($this->_haveColumnHeader
) {
590 array_unshift($values, $recordNumber);
591 $this->_duplicates
[] = $values;
592 if ($onDuplicate != self
::DUPLICATE_SKIP
) {
593 $this->_validCount++
;
597 // we give the derived class a way of aborting the process
598 // note that the return code could be multiple code or'ed together
599 if ($returnCode & self
::STOP
) {
603 // if we are done processing the maxNumber of lines, break
604 if ($this->_maxLinesToProcess
> 0 && $this->_validCount
>= $this->_maxLinesToProcess
) {
611 if ($mode == self
::MODE_PREVIEW ||
$mode == self
::MODE_IMPORT
) {
612 $customHeaders = $mapper;
614 $customfields = CRM_Core_BAO_CustomField
::getFields('Activity');
615 foreach ($customHeaders as $key => $value) {
616 if ($id = CRM_Core_BAO_CustomField
::getKeyID($value)) {
617 $customHeaders[$key] = $customfields[$id][0];
620 if ($this->_invalidRowCount
) {
621 // removed view url for invlaid contacts
622 $headers = array_merge(
623 [ts('Line Number'), ts('Reason')],
626 $this->_errorFileName
= self
::errorFileName(self
::ERROR
);
627 self
::exportCSV($this->_errorFileName
, $headers, $this->_errors
);
629 if ($this->_conflictCount
) {
630 $headers = array_merge(
631 [ts('Line Number'), ts('Reason')],
634 $this->_conflictFileName
= self
::errorFileName(self
::CONFLICT
);
635 self
::exportCSV($this->_conflictFileName
, $headers, $this->_conflicts
);
637 if ($this->_duplicateCount
) {
638 $headers = array_merge(
639 [ts('Line Number'), ts('View Activity History URL')],
643 $this->_duplicateFileName
= self
::errorFileName(self
::DUPLICATE
);
644 self
::exportCSV($this->_duplicateFileName
, $headers, $this->_duplicates
);
650 * Given a list of the importable field keys that the user has selected set the active fields array to this list.
652 * @param array $fieldKeys
654 public function setActiveFields($fieldKeys) {
655 $this->_activeFieldCount
= count($fieldKeys);
656 foreach ($fieldKeys as $key) {
657 if (empty($this->_fields
[$key])) {
658 $this->_activeFields
[] = new CRM_Activity_Import_Field('', ts('- do not import -'));
661 $this->_activeFields
[] = clone($this->_fields
[$key]);
667 * @param string $name
670 * @param string $headerPattern
671 * @param string $dataPattern
673 public function addField($name, $title, $type = CRM_Utils_Type
::T_INT
, $headerPattern = '//', $dataPattern = '//') {
675 $this->_fields
['doNotImport'] = new CRM_Activity_Import_Field($name, $title, $type, $headerPattern, $dataPattern);
679 $tempField = CRM_Contact_BAO_Contact
::importableFields('Individual', NULL);
680 if (!array_key_exists($name, $tempField)) {
681 $this->_fields
[$name] = new CRM_Activity_Import_Field($name, $title, $type, $headerPattern, $dataPattern);
684 $this->_fields
[$name] = new CRM_Contact_Import_Field($name, $title, $type, $headerPattern, $dataPattern, CRM_Utils_Array
::value('hasLocationType', $tempField[$name]));
690 * Store parser values.
692 * @param CRM_Core_Session $store
696 public function set($store, $mode = self
::MODE_SUMMARY
) {
697 $store->set('fileSize', $this->_fileSize
);
698 $store->set('lineCount', $this->_lineCount
);
699 $store->set('separator', $this->_separator
);
700 $store->set('fields', $this->getSelectValues());
702 $store->set('headerPatterns', $this->getHeaderPatterns());
703 $store->set('dataPatterns', $this->getDataPatterns());
704 $store->set('columnCount', $this->_activeFieldCount
);
706 $store->set('totalRowCount', $this->_totalCount
);
707 $store->set('validRowCount', $this->_validCount
);
708 $store->set('invalidRowCount', $this->_invalidRowCount
);
709 $store->set('conflictRowCount', $this->_conflictCount
);
711 if ($this->_invalidRowCount
) {
712 $store->set('errorsFileName', $this->_errorFileName
);
714 if ($this->_conflictCount
) {
715 $store->set('conflictsFileName', $this->_conflictFileName
);
717 if (isset($this->_rows
) && !empty($this->_rows
)) {
718 $store->set('dataValues', $this->_rows
);
721 if ($mode == self
::MODE_IMPORT
) {
722 $store->set('duplicateRowCount', $this->_duplicateCount
);
723 if ($this->_duplicateCount
) {
724 $store->set('duplicatesFileName', $this->_duplicateFileName
);
730 * Export data to a CSV file.
732 * @param string $fileName
733 * @param array $header
736 public static function exportCSV($fileName, $header, $data) {
738 $fd = fopen($fileName, 'w');
740 foreach ($header as $key => $value) {
741 $header[$key] = "\"$value\"";
743 $config = CRM_Core_Config
::singleton();
744 $output[] = implode($config->fieldSeparator
, $header);
746 foreach ($data as $datum) {
747 foreach ($datum as $key => $value) {
748 $datum[$key] = "\"$value\"";
750 $output[] = implode($config->fieldSeparator
, $datum);
752 fwrite($fd, implode("\n", $output));