3 +--------------------------------------------------------------------+
4 | Copyright CiviCRM LLC. All rights reserved. |
6 | This work is published under the GNU AGPLv3 license with some |
7 | permitted exceptions and without any warranty. For full license |
8 | and copyright information, see https://civicrm.org/licensing |
9 +--------------------------------------------------------------------+
15 * @copyright CiviCRM LLC https://civicrm.org/licensing
20 * Class to parse activity csv files.
22 class CRM_Activity_Import_Parser_Activity
extends CRM_Import_Parser
{
24 protected $_mapperKeys;
27 * Array of successfully imported activity id's
31 protected $_newActivity;
42 * Separator being used.
45 protected $_separator;
48 * Total number of lines in file.
51 protected $_lineCount;
54 * Whether the file has a column header or not.
58 protected $_haveColumnHeader;
63 * @param array $mapperKeys
65 public function __construct($mapperKeys = []) {
66 parent
::__construct();
67 $this->_mapperKeys
= $mapperKeys;
71 * The initializer code, called before the processing.
73 public function init() {
74 $activityContact = CRM_Activity_BAO_ActivityContact
::import();
75 $activityTarget['target_contact_id'] = $activityContact['contact_id'];
76 $fields = array_merge(CRM_Activity_BAO_Activity
::importableFields(),
80 $fields = array_merge($fields, [
81 'source_contact_id' => [
82 'title' => ts('Source Contact'),
83 'headerPattern' => '/Source.Contact?/i',
86 'title' => ts('Activity Type Label'),
87 'headerPattern' => '/(activity.)?type label?/i',
91 foreach ($fields as $name => $field) {
92 $field['type'] = CRM_Utils_Array
::value('type', $field, CRM_Utils_Type
::T_INT
);
93 $field['dataPattern'] = CRM_Utils_Array
::value('dataPattern', $field, '//');
94 $field['headerPattern'] = CRM_Utils_Array
::value('headerPattern', $field, '//');
95 if (!empty($field['custom_group_id'])) {
96 $field['title'] = $field["groupTitle"] . ' :: ' . $field["title"];
98 $this->addField($name, $field['title'], $field['type'], $field['headerPattern'], $field['dataPattern']);
101 $this->_newActivity
= [];
103 $this->setActiveFields($this->_mapperKeys
);
107 * Handle the values in summary mode.
109 * @param array $values
110 * The array of values belonging to this line.
113 * CRM_Import_Parser::VALID for success or
114 * CRM_Import_Parser::ERROR for error.
116 public function summary(&$values) {
118 $this->validateValues($values);
120 catch (CRM_Core_Exception
$e) {
121 return $this->addError($values, [$e->getMessage()]);
124 return CRM_Import_Parser
::VALID
;
128 * Handle the values in import mode.
130 * @param int $onDuplicate
131 * The code for what action to take on duplicates.
132 * @param array $values
133 * The array of values belonging to this line.
136 * CRM_Import_Parser::VALID for success or
137 * CRM_Import_Parser::ERROR for error.
139 * @throws \CRM_Core_Exception
141 public function import($onDuplicate, &$values) {
142 // First make sure this is a valid line
144 $this->validateValues($values);
146 catch (CRM_Core_Exception
$e) {
147 return $this->addError($values, [$e->getMessage()]);
149 $params = $this->getApiReadyParams($values);
151 $session = CRM_Core_Session
::singleton();
152 $dateType = $session->get('dateTypes');
154 $customFields = CRM_Core_BAO_CustomField
::getFields('Activity');
156 foreach ($params as $key => $val) {
157 if ($customFieldID = CRM_Core_BAO_CustomField
::getKeyID($key)) {
158 if (!empty($customFields[$customFieldID]) && $customFields[$customFieldID]['data_type'] == 'Date') {
159 CRM_Contact_Import_Parser_Contact
::formatCustomDate($params, $params, $dateType, $key);
161 elseif (!empty($customFields[$customFieldID]) && $customFields[$customFieldID]['data_type'] == 'Boolean') {
162 $params[$key] = CRM_Utils_String
::strtoboolstr($val);
165 elseif ($key === 'activity_date_time') {
166 $params[$key] = CRM_Utils_Date
::formatDate($val, $dateType);
168 elseif ($key === 'activity_subject') {
169 $params['subject'] = $val;
173 if (empty($params['external_identifier']) && empty($params['target_contact_id'])) {
175 // Retrieve contact id using contact dedupe rule.
176 // Since we are supporting only individual's activity import.
177 $params['contact_type'] = 'Individual';
178 $params['version'] = 3;
179 $matchedIDs = CRM_Contact_BAO_Contact
::getDuplicateContacts($params, 'Individual');
181 if (!empty($matchedIDs)) {
182 if (count($matchedIDs) > 1) {
183 array_unshift($values, 'Multiple matching contact records detected for this row. The activity was not imported');
184 return CRM_Import_Parser
::ERROR
;
186 $cid = $matchedIDs[0];
187 $params['target_contact_id'] = $cid;
188 $params['version'] = 3;
189 $newActivity = civicrm_api('activity', 'create', $params);
190 if (!empty($newActivity['is_error'])) {
191 array_unshift($values, $newActivity['error_message']);
192 return CRM_Import_Parser
::ERROR
;
195 $this->_newActivity
[] = $newActivity['id'];
196 return CRM_Import_Parser
::VALID
;
199 // Using new Dedupe rule.
201 'contact_type' => 'Individual',
202 'used' => 'Unsupervised',
204 $fieldsArray = CRM_Dedupe_BAO_DedupeRule
::dedupeRuleFields($ruleParams);
207 foreach ($fieldsArray as $value) {
208 if (array_key_exists(trim($value), $params)) {
209 $paramValue = $params[trim($value)];
210 if (is_array($paramValue)) {
211 $disp .= $params[trim($value)][0][trim($value)] . " ";
214 $disp .= $params[trim($value)] . " ";
219 if (!empty($params['external_identifier'])) {
221 $disp .= "AND {$params['external_identifier']}";
224 $disp = $params['external_identifier'];
228 array_unshift($values, 'No matching Contact found for (' . $disp . ')');
229 return CRM_Import_Parser
::ERROR
;
231 if (!empty($params['external_identifier'])) {
232 $targetContactId = CRM_Core_DAO
::getFieldValue('CRM_Contact_DAO_Contact',
233 $params['external_identifier'], 'id', 'external_identifier'
236 if (!empty($params['target_contact_id']) &&
237 $params['target_contact_id'] != $targetContactId
239 array_unshift($values, 'Mismatch of External ID:' . $params['external_identifier'] . ' and Contact Id:' . $params['target_contact_id']);
240 return CRM_Import_Parser
::ERROR
;
242 if ($targetContactId) {
243 $params['target_contact_id'] = $targetContactId;
246 array_unshift($values, 'No Matching Contact for External ID:' . $params['external_identifier']);
247 return CRM_Import_Parser
::ERROR
;
251 $params['version'] = 3;
252 $newActivity = civicrm_api('activity', 'create', $params);
253 if (!empty($newActivity['is_error'])) {
254 array_unshift($values, $newActivity['error_message']);
255 return CRM_Import_Parser
::ERROR
;
258 $this->_newActivity
[] = $newActivity['id'];
259 return CRM_Import_Parser
::VALID
;
264 * Get the value for the given field from the row of values.
267 * @param string $fieldName
269 * @return null|string
271 protected function getFieldValue(array $row, string $fieldName) {
272 if (!is_numeric($this->getFieldIndex($fieldName))) {
275 return $row[$this->getFieldIndex($fieldName)] ??
NULL;
279 * Get the index for the given field.
281 * @param string $fieldName
285 protected function getFieldIndex(string $fieldName) {
286 return array_search($fieldName, $this->_mapperKeys
, TRUE);
291 * Add an error to the values.
293 * @param array $values
294 * @param array $error
298 protected function addError(array &$values, array $error): int {
299 array_unshift($values, implode(';', $error));
300 return CRM_Import_Parser
::ERROR
;
304 * Validate that the activity type id does not conflict with the label.
306 * @param array $values
309 * @throws \CRM_Core_Exception
311 protected function validateActivityTypeIDAndLabel(array $values): void
{
312 $activityLabel = $this->getFieldValue($values, 'activity_label');
313 $activityTypeID = $this->getFieldValue($values, 'activity_type_id');
314 if ($activityLabel && $activityTypeID
315 && $activityLabel !== CRM_Core_PseudoConstant
::getLabel('CRM_Activity_BAO_Activity', 'activity_type_id', $activityTypeID)) {
316 throw new CRM_Core_Exception(ts('Activity type label and Activity type ID are in conflict'));
321 * Is the supplied date field valid based on selected date format.
323 * @param string $value
327 protected function isValidDate(string $value): bool {
328 return (bool) CRM_Utils_Date
::formatDate($value, CRM_Core_Session
::singleton()->get('dateTypes'));
332 * Is the supplied field a valid contact id.
334 * @param string|int $value
338 protected function isValidContactID($value): bool {
339 if (!CRM_Utils_Rule
::integer($value)) {
342 if (!CRM_Core_DAO
::singleValueQuery("SELECT id FROM civicrm_contact WHERE id = " . (int) $value)) {
349 * Validate custom fields.
351 * @param array $values
353 * @throws \CRM_Core_Exception
355 protected function validateCustomFields($values):void
{
356 $this->setActiveFieldValues($values);
357 $params = $this->getActiveFieldParams();
358 $errorMessage = NULL;
359 // Checking error in custom data.
360 $params['contact_type'] = 'Activity';
361 CRM_Contact_Import_Parser_Contact
::isErrorInCustomData($params, $errorMessage);
363 throw new CRM_Core_Exception('Invalid value for field(s) : ' . $errorMessage);
368 * @param array $values
370 * @throws \CRM_Core_Exception
372 protected function validateValues(array $values): void
{
373 // Check required fields if this is not an update.
374 if (!$this->getFieldValue($values, 'activity_id')) {
375 if (!$this->getFieldValue($values, 'activity_label')
376 && !$this->getFieldValue($values, 'activity_type_id')) {
377 throw new CRM_Core_Exception(ts('Missing required fields: Activity type label or Activity type ID'));
379 if (!$this->getFieldValue($values, 'activity_date_time')) {
380 throw new CRM_Core_Exception(ts('Missing required fields'));
384 $this->validateActivityTypeIDAndLabel($values);
385 if ($this->getFieldValue($values, 'activity_date_time')
386 && !$this->isValidDate($this->getFieldValue($values, 'activity_date_time'))) {
387 throw new CRM_Core_Exception(ts('Invalid Activity Date'));
390 if ($this->getFieldValue($values, 'activity_engagement_level')
391 && !CRM_Utils_Rule
::positiveInteger($this->getFieldValue($values, 'activity_engagement_level'))) {
392 throw new CRM_Core_Exception(ts('Activity Engagement Index'));
395 $targetContactID = $this->getFieldValue($values, 'target_contact_id');
396 if ($targetContactID && !$this->isValidContactID($targetContactID)) {
397 throw new CRM_Core_Exception("Invalid Contact ID: There is no contact record with contact_id = " . CRM_Utils_Type
::escape($targetContactID, 'String'));
399 $this->validateCustomFields($values);
403 * Get array of parameters formatted for the api from the submitted values.
405 * @param array $values
409 protected function getApiReadyParams(array $values): array {
410 $this->setActiveFieldValues($values);
411 $params = $this->getActiveFieldParams();
412 if ($this->getFieldValue($values, 'activity_label')) {
413 $params['activity_type_id'] = array_search(
414 $this->getFieldValue($values, 'activity_label'),
415 CRM_Activity_BAO_Activity
::buildOptions('activity_type_id', 'create'),
423 * @param array $fileName
424 * @param string $separator
426 * @param bool $skipColumnHeader
428 * @param int $onDuplicate
429 * @param int $statusID
430 * @param int $totalRowCount
439 $skipColumnHeader = FALSE,
440 $mode = self
::MODE_PREVIEW
,
441 $onDuplicate = self
::DUPLICATE_SKIP
,
443 $totalRowCount = NULL
446 $fileName = $fileName['name'];
450 $this->_haveColumnHeader
= $skipColumnHeader;
452 $this->_separator
= $separator;
454 $fd = fopen($fileName, "r");
459 $this->_lineCount
= 0;
460 $this->_invalidRowCount
= $this->_validCount
= 0;
461 $this->_totalCount
= 0;
464 $this->_warnings
= [];
466 $this->_fileSize
= number_format(filesize($fileName) / 1024.0, 2);
468 if ($mode == self
::MODE_MAPFIELD
) {
472 $this->_activeFieldCount
= count($this->_activeFields
);
475 $this->progressImport($statusID);
476 $startTimestamp = $currTimestamp = $prevTimestamp = time();
482 $values = fgetcsv($fd, 8192, $separator);
487 self
::encloseScrub($values);
489 // skip column header if we're not in mapfield mode
490 if ($mode != self
::MODE_MAPFIELD
&& $skipColumnHeader) {
491 $skipColumnHeader = FALSE;
495 // Trim whitespace around the values.
498 foreach ($values as $k => $v) {
499 $values[$k] = trim($v, " \t\r\n");
502 if (CRM_Utils_System
::isNull($values)) {
506 $this->_totalCount++
;
508 if ($mode == self
::MODE_MAPFIELD
) {
509 $returnCode = CRM_Import_Parser
::VALID
;
511 // Note that MODE_SUMMARY seems to be never used.
512 elseif ($mode == self
::MODE_PREVIEW ||
$mode == self
::MODE_SUMMARY
) {
513 $returnCode = $this->summary($values);
515 elseif ($mode == self
::MODE_IMPORT
) {
516 $returnCode = $this->import($onDuplicate, $values);
517 if ($statusID && (($this->_lineCount %
50) == 0)) {
518 $prevTimestamp = $this->progressImport($statusID, FALSE, $startTimestamp, $prevTimestamp, $totalRowCount);
522 $returnCode = self
::ERROR
;
525 // note that a line could be valid but still produce a warning
526 if ($returnCode & self
::VALID
) {
527 $this->_validCount++
;
528 if ($mode == self
::MODE_MAPFIELD
) {
529 $this->_rows
[] = $values;
530 $this->_activeFieldCount
= max($this->_activeFieldCount
, count($values));
534 if ($returnCode & self
::ERROR
) {
535 $this->_invalidRowCount++
;
536 $recordNumber = $this->_lineCount
;
537 if ($this->_haveColumnHeader
) {
540 array_unshift($values, $recordNumber);
541 $this->_errors
[] = $values;
544 // if we are done processing the maxNumber of lines, break
545 if ($this->_maxLinesToProcess
> 0 && $this->_validCount
>= $this->_maxLinesToProcess
) {
552 if ($mode == self
::MODE_PREVIEW ||
$mode == self
::MODE_IMPORT
) {
553 $customHeaders = $mapper;
555 $customfields = CRM_Core_BAO_CustomField
::getFields('Activity');
556 foreach ($customHeaders as $key => $value) {
557 if ($id = CRM_Core_BAO_CustomField
::getKeyID($value)) {
558 $customHeaders[$key] = $customfields[$id][0];
561 if ($this->_invalidRowCount
) {
562 // removed view url for invlaid contacts
563 $headers = array_merge(
564 [ts('Line Number'), ts('Reason')],
567 $this->_errorFileName
= self
::errorFileName(self
::ERROR
);
568 self
::exportCSV($this->_errorFileName
, $headers, $this->_errors
);
574 * Given a list of the importable field keys that the user has selected set the active fields array to this list.
576 * @param array $fieldKeys
578 public function setActiveFields($fieldKeys) {
579 $this->_activeFieldCount
= count($fieldKeys);
580 foreach ($fieldKeys as $key) {
581 if (empty($this->_fields
[$key])) {
582 $this->_activeFields
[] = new CRM_Activity_Import_Field('', ts('- do not import -'));
585 $this->_activeFields
[] = clone($this->_fields
[$key]);
591 * @param string $name
594 * @param string $headerPattern
595 * @param string $dataPattern
597 public function addField($name, $title, $type = CRM_Utils_Type
::T_INT
, $headerPattern = '//', $dataPattern = '//') {
599 $this->_fields
['doNotImport'] = new CRM_Activity_Import_Field($name, $title, $type, $headerPattern, $dataPattern);
603 $tempField = CRM_Contact_BAO_Contact
::importableFields('Individual', NULL);
604 if (!array_key_exists($name, $tempField)) {
605 $this->_fields
[$name] = new CRM_Activity_Import_Field($name, $title, $type, $headerPattern, $dataPattern);
608 $this->_fields
[$name] = new CRM_Contact_Import_Field($name, $title, $type, $headerPattern, $dataPattern, CRM_Utils_Array
::value('hasLocationType', $tempField[$name]));
614 * Store parser values.
616 * @param CRM_Core_Session $store
618 public function set($store) {
619 $store->set('fileSize', $this->_fileSize
);
620 $store->set('lineCount', $this->_lineCount
);
621 $store->set('separator', $this->_separator
);
622 $store->set('fields', $this->getSelectValues());
624 $store->set('headerPatterns', $this->getHeaderPatterns());
625 $store->set('dataPatterns', $this->getDataPatterns());
626 $store->set('columnCount', $this->_activeFieldCount
);
628 $store->set('totalRowCount', $this->_totalCount
);
629 $store->set('validRowCount', $this->_validCount
);
630 $store->set('invalidRowCount', $this->_invalidRowCount
);
632 if ($this->_invalidRowCount
) {
633 $store->set('errorsFileName', $this->_errorFileName
);
636 if (isset($this->_rows
) && !empty($this->_rows
)) {
637 $store->set('dataValues', $this->_rows
);
642 * Export data to a CSV file.
644 * @param string $fileName
645 * @param array $header
648 public static function exportCSV($fileName, $header, $data) {
650 $fd = fopen($fileName, 'w');
652 foreach ($header as $key => $value) {
653 $header[$key] = "\"$value\"";
655 $config = CRM_Core_Config
::singleton();
656 $output[] = implode($config->fieldSeparator
, $header);
658 foreach ($data as $datum) {
659 foreach ($datum as $key => $value) {
660 $datum[$key] = "\"$value\"";
662 $output[] = implode($config->fieldSeparator
, $datum);
664 fwrite($fd, implode("\n", $output));