3 +--------------------------------------------------------------------+
4 | Copyright CiviCRM LLC. All rights reserved. |
6 | This work is published under the GNU AGPLv3 license with some |
7 | permitted exceptions and without any warranty. For full license |
8 | and copyright information, see https://civicrm.org/licensing |
9 +--------------------------------------------------------------------+
15 * @copyright CiviCRM LLC https://civicrm.org/licensing
20 * Class to parse activity csv files.
22 class CRM_Activity_Import_Parser_Activity
extends CRM_Import_Parser
{
24 protected $_mapperKeys;
26 private $_contactIdIndex;
29 * Array of successfully imported activity id's
33 protected $_newActivity;
44 * Separator being used.
47 protected $_separator;
50 * Total number of lines in file.
53 protected $_lineCount;
56 * Whether the file has a column header or not.
60 protected $_haveColumnHeader;
65 * @param array $mapperKeys
67 public function __construct($mapperKeys = []) {
68 parent
::__construct();
69 $this->_mapperKeys
= $mapperKeys;
73 * The initializer code, called before the processing.
75 public function init() {
76 $activityContact = CRM_Activity_BAO_ActivityContact
::import();
77 $activityTarget['target_contact_id'] = $activityContact['contact_id'];
78 $fields = array_merge(CRM_Activity_BAO_Activity
::importableFields(),
82 $fields = array_merge($fields, [
83 'source_contact_id' => [
84 'title' => ts('Source Contact'),
85 'headerPattern' => '/Source.Contact?/i',
88 'title' => ts('Activity Type Label'),
89 'headerPattern' => '/(activity.)?type label?/i',
93 foreach ($fields as $name => $field) {
94 $field['type'] = CRM_Utils_Array
::value('type', $field, CRM_Utils_Type
::T_INT
);
95 $field['dataPattern'] = CRM_Utils_Array
::value('dataPattern', $field, '//');
96 $field['headerPattern'] = CRM_Utils_Array
::value('headerPattern', $field, '//');
97 if (!empty($field['custom_group_id'])) {
98 $field['title'] = $field["groupTitle"] . ' :: ' . $field["title"];
100 $this->addField($name, $field['title'], $field['type'], $field['headerPattern'], $field['dataPattern']);
103 $this->_newActivity
= [];
105 $this->setActiveFields($this->_mapperKeys
);
107 // FIXME: we should do this in one place together with Form/MapField.php
108 $this->_contactIdIndex
= -1;
111 foreach ($this->_mapperKeys
as $key) {
113 case 'target_contact_id':
114 case 'external_identifier':
115 $this->_contactIdIndex
= $index;
123 * Handle the values in summary mode.
125 * @param array $values
126 * The array of values belonging to this line.
129 * CRM_Import_Parser::VALID for success or
130 * CRM_Import_Parser::ERROR for error.
132 public function summary(&$values) {
134 $this->validateValues($values);
136 catch (CRM_Core_Exception
$e) {
137 return $this->addError($values, [$e->getMessage()]);
140 return CRM_Import_Parser
::VALID
;
144 * Handle the values in import mode.
146 * @param int $onDuplicate
147 * The code for what action to take on duplicates.
148 * @param array $values
149 * The array of values belonging to this line.
152 * CRM_Import_Parser::VALID for success or
153 * CRM_Import_Parser::ERROR for error.
155 * @throws \CRM_Core_Exception
157 public function import($onDuplicate, &$values) {
158 // First make sure this is a valid line
160 $this->validateValues($values);
162 catch (CRM_Core_Exception
$e) {
163 return $this->addError($values, [$e->getMessage()]);
165 $params = $this->getApiReadyParams($values);
167 $session = CRM_Core_Session
::singleton();
168 $dateType = $session->get('dateTypes');
170 $customFields = CRM_Core_BAO_CustomField
::getFields('Activity');
172 foreach ($params as $key => $val) {
173 if ($customFieldID = CRM_Core_BAO_CustomField
::getKeyID($key)) {
174 if (!empty($customFields[$customFieldID]) && $customFields[$customFieldID]['data_type'] == 'Date') {
175 CRM_Contact_Import_Parser_Contact
::formatCustomDate($params, $params, $dateType, $key);
177 elseif (!empty($customFields[$customFieldID]) && $customFields[$customFieldID]['data_type'] == 'Boolean') {
178 $params[$key] = CRM_Utils_String
::strtoboolstr($val);
181 elseif ($key === 'activity_date_time') {
182 $params[$key] = CRM_Utils_Date
::formatDate($val, $dateType);
184 elseif ($key === 'activity_subject') {
185 $params['subject'] = $val;
189 if ($this->_contactIdIndex
< 0) {
191 // Retrieve contact id using contact dedupe rule.
192 // Since we are supporting only individual's activity import.
193 $params['contact_type'] = 'Individual';
194 $params['version'] = 3;
195 $error = _civicrm_api3_deprecated_duplicate_formatted_contact($params);
197 if (CRM_Core_Error
::isAPIError($error, CRM_Core_ERROR
::DUPLICATE_CONTACT
)) {
198 $matchedIDs = explode(',', $error['error_message']['params'][0]);
199 if (count($matchedIDs) > 1) {
200 array_unshift($values, 'Multiple matching contact records detected for this row. The activity was not imported');
201 return CRM_Import_Parser
::ERROR
;
203 $cid = $matchedIDs[0];
204 $params['target_contact_id'] = $cid;
205 $params['version'] = 3;
206 $newActivity = civicrm_api('activity', 'create', $params);
207 if (!empty($newActivity['is_error'])) {
208 array_unshift($values, $newActivity['error_message']);
209 return CRM_Import_Parser
::ERROR
;
212 $this->_newActivity
[] = $newActivity['id'];
213 return CRM_Import_Parser
::VALID
;
216 // Using new Dedupe rule.
218 'contact_type' => 'Individual',
219 'used' => 'Unsupervised',
221 $fieldsArray = CRM_Dedupe_BAO_DedupeRule
::dedupeRuleFields($ruleParams);
224 foreach ($fieldsArray as $value) {
225 if (array_key_exists(trim($value), $params)) {
226 $paramValue = $params[trim($value)];
227 if (is_array($paramValue)) {
228 $disp .= $params[trim($value)][0][trim($value)] . " ";
231 $disp .= $params[trim($value)] . " ";
236 if (!empty($params['external_identifier'])) {
238 $disp .= "AND {$params['external_identifier']}";
241 $disp = $params['external_identifier'];
245 array_unshift($values, 'No matching Contact found for (' . $disp . ')');
246 return CRM_Import_Parser
::ERROR
;
248 if (!empty($params['external_identifier'])) {
249 $targetContactId = CRM_Core_DAO
::getFieldValue('CRM_Contact_DAO_Contact',
250 $params['external_identifier'], 'id', 'external_identifier'
253 if (!empty($params['target_contact_id']) &&
254 $params['target_contact_id'] != $targetContactId
256 array_unshift($values, 'Mismatch of External ID:' . $params['external_identifier'] . ' and Contact Id:' . $params['target_contact_id']);
257 return CRM_Import_Parser
::ERROR
;
259 if ($targetContactId) {
260 $params['target_contact_id'] = $targetContactId;
263 array_unshift($values, 'No Matching Contact for External ID:' . $params['external_identifier']);
264 return CRM_Import_Parser
::ERROR
;
268 $params['version'] = 3;
269 $newActivity = civicrm_api('activity', 'create', $params);
270 if (!empty($newActivity['is_error'])) {
271 array_unshift($values, $newActivity['error_message']);
272 return CRM_Import_Parser
::ERROR
;
275 $this->_newActivity
[] = $newActivity['id'];
276 return CRM_Import_Parser
::VALID
;
281 * Get the value for the given field from the row of values.
284 * @param string $fieldName
286 * @return null|string
288 protected function getFieldValue(array $row, string $fieldName) {
289 if (!is_numeric($this->getFieldIndex($fieldName))) {
292 return $row[$this->getFieldIndex($fieldName)] ??
NULL;
296 * Get the index for the given field.
298 * @param string $fieldName
302 protected function getFieldIndex(string $fieldName) {
303 return array_search($fieldName, $this->_mapperKeys
, TRUE);
308 * Add an error to the values.
310 * @param array $values
311 * @param array $error
315 protected function addError(array &$values, array $error): int {
316 array_unshift($values, implode(';', $error));
317 return CRM_Import_Parser
::ERROR
;
321 * Validate that the activity type id does not conflict with the label.
323 * @param array $values
326 * @throws \CRM_Core_Exception
328 protected function validateActivityTypeIDAndLabel(array $values): void
{
329 $activityLabel = $this->getFieldValue($values, 'activity_label');
330 $activityTypeID = $this->getFieldValue($values, 'activity_type_id');
331 if ($activityLabel && $activityTypeID
332 && $activityLabel !== CRM_Core_PseudoConstant
::getLabel('CRM_Activity_BAO_Activity', 'activity_type_id', $activityTypeID)) {
333 throw new CRM_Core_Exception(ts('Activity type label and Activity type ID are in conflict'));
338 * Is the supplied date field valid based on selected date format.
340 * @param string $value
344 protected function isValidDate(string $value): bool {
345 return (bool) CRM_Utils_Date
::formatDate($value, CRM_Core_Session
::singleton()->get('dateTypes'));
349 * Is the supplied field a valid contact id.
351 * @param string|int $value
355 protected function isValidContactID($value): bool {
356 if (!CRM_Utils_Rule
::integer($value)) {
359 if (!CRM_Core_DAO
::singleValueQuery("SELECT id FROM civicrm_contact WHERE id = " . (int) $value)) {
366 * Validate custom fields.
368 * @param array $values
370 * @throws \CRM_Core_Exception
372 protected function validateCustomFields($values):void
{
373 $this->setActiveFieldValues($values);
374 $params = $this->getActiveFieldParams();
375 $errorMessage = NULL;
376 // Checking error in custom data.
377 $params['contact_type'] = 'Activity';
378 CRM_Contact_Import_Parser_Contact
::isErrorInCustomData($params, $errorMessage);
380 throw new CRM_Core_Exception('Invalid value for field(s) : ' . $errorMessage);
385 * @param array $values
387 * @throws \CRM_Core_Exception
389 protected function validateValues(array $values): void
{
390 // Check required fields if this is not an update.
391 if (!$this->getFieldValue($values, 'activity_id')) {
392 if (!$this->getFieldValue($values, 'activity_label')
393 && !$this->getFieldValue($values, 'activity_type_id')) {
394 throw new CRM_Core_Exception(ts('Missing required fields: Activity type label or Activity type ID'));
396 if (!$this->getFieldValue($values, 'activity_date_time')) {
397 throw new CRM_Core_Exception(ts('Missing required fields'));
401 $this->validateActivityTypeIDAndLabel($values);
402 if ($this->getFieldValue($values, 'activity_date_time')
403 && !$this->isValidDate($this->getFieldValue($values, 'activity_date_time'))) {
404 throw new CRM_Core_Exception(ts('Invalid Activity Date'));
407 if ($this->getFieldValue($values, 'activity_engagement_level')
408 && !CRM_Utils_Rule
::positiveInteger($this->getFieldValue($values, 'activity_engagement_level'))) {
409 throw new CRM_Core_Exception(ts('Activity Engagement Index'));
412 $targetContactID = $this->getFieldValue($values, 'target_contact_id');
413 if ($targetContactID && !$this->isValidContactID($targetContactID)) {
414 throw new CRM_Core_Exception("Invalid Contact ID: There is no contact record with contact_id = " . CRM_Utils_Type
::escape($targetContactID, 'String'));
416 $this->validateCustomFields($values);
420 * Get array of parameters formatted for the api from the submitted values.
422 * @param array $values
426 protected function getApiReadyParams(array $values): array {
427 $this->setActiveFieldValues($values);
428 $params = $this->getActiveFieldParams();
429 if ($this->getFieldValue($values, 'activity_label')) {
430 $params['activity_type_id'] = array_search(
431 $this->getFieldValue($values, 'activity_label'),
432 CRM_Activity_BAO_Activity
::buildOptions('activity_type_id', 'create'),
440 * @param array $fileName
441 * @param string $separator
443 * @param bool $skipColumnHeader
445 * @param int $onDuplicate
446 * @param int $statusID
447 * @param int $totalRowCount
456 $skipColumnHeader = FALSE,
457 $mode = self
::MODE_PREVIEW
,
458 $onDuplicate = self
::DUPLICATE_SKIP
,
460 $totalRowCount = NULL
463 $fileName = $fileName['name'];
467 $this->_haveColumnHeader
= $skipColumnHeader;
469 $this->_separator
= $separator;
471 $fd = fopen($fileName, "r");
476 $this->_lineCount
= 0;
477 $this->_invalidRowCount
= $this->_validCount
= 0;
478 $this->_totalCount
= 0;
481 $this->_warnings
= [];
483 $this->_fileSize
= number_format(filesize($fileName) / 1024.0, 2);
485 if ($mode == self
::MODE_MAPFIELD
) {
489 $this->_activeFieldCount
= count($this->_activeFields
);
492 $this->progressImport($statusID);
493 $startTimestamp = $currTimestamp = $prevTimestamp = time();
499 $values = fgetcsv($fd, 8192, $separator);
504 self
::encloseScrub($values);
506 // skip column header if we're not in mapfield mode
507 if ($mode != self
::MODE_MAPFIELD
&& $skipColumnHeader) {
508 $skipColumnHeader = FALSE;
512 // Trim whitespace around the values.
515 foreach ($values as $k => $v) {
516 $values[$k] = trim($v, " \t\r\n");
519 if (CRM_Utils_System
::isNull($values)) {
523 $this->_totalCount++
;
525 if ($mode == self
::MODE_MAPFIELD
) {
526 $returnCode = CRM_Import_Parser
::VALID
;
528 // Note that MODE_SUMMARY seems to be never used.
529 elseif ($mode == self
::MODE_PREVIEW ||
$mode == self
::MODE_SUMMARY
) {
530 $returnCode = $this->summary($values);
532 elseif ($mode == self
::MODE_IMPORT
) {
533 $returnCode = $this->import($onDuplicate, $values);
534 if ($statusID && (($this->_lineCount %
50) == 0)) {
535 $prevTimestamp = $this->progressImport($statusID, FALSE, $startTimestamp, $prevTimestamp, $totalRowCount);
539 $returnCode = self
::ERROR
;
542 // note that a line could be valid but still produce a warning
543 if ($returnCode & self
::VALID
) {
544 $this->_validCount++
;
545 if ($mode == self
::MODE_MAPFIELD
) {
546 $this->_rows
[] = $values;
547 $this->_activeFieldCount
= max($this->_activeFieldCount
, count($values));
551 if ($returnCode & self
::ERROR
) {
552 $this->_invalidRowCount++
;
553 $recordNumber = $this->_lineCount
;
554 if ($this->_haveColumnHeader
) {
557 array_unshift($values, $recordNumber);
558 $this->_errors
[] = $values;
561 // if we are done processing the maxNumber of lines, break
562 if ($this->_maxLinesToProcess
> 0 && $this->_validCount
>= $this->_maxLinesToProcess
) {
569 if ($mode == self
::MODE_PREVIEW ||
$mode == self
::MODE_IMPORT
) {
570 $customHeaders = $mapper;
572 $customfields = CRM_Core_BAO_CustomField
::getFields('Activity');
573 foreach ($customHeaders as $key => $value) {
574 if ($id = CRM_Core_BAO_CustomField
::getKeyID($value)) {
575 $customHeaders[$key] = $customfields[$id][0];
578 if ($this->_invalidRowCount
) {
579 // removed view url for invlaid contacts
580 $headers = array_merge(
581 [ts('Line Number'), ts('Reason')],
584 $this->_errorFileName
= self
::errorFileName(self
::ERROR
);
585 self
::exportCSV($this->_errorFileName
, $headers, $this->_errors
);
591 * Given a list of the importable field keys that the user has selected set the active fields array to this list.
593 * @param array $fieldKeys
595 public function setActiveFields($fieldKeys) {
596 $this->_activeFieldCount
= count($fieldKeys);
597 foreach ($fieldKeys as $key) {
598 if (empty($this->_fields
[$key])) {
599 $this->_activeFields
[] = new CRM_Activity_Import_Field('', ts('- do not import -'));
602 $this->_activeFields
[] = clone($this->_fields
[$key]);
608 * @param string $name
611 * @param string $headerPattern
612 * @param string $dataPattern
614 public function addField($name, $title, $type = CRM_Utils_Type
::T_INT
, $headerPattern = '//', $dataPattern = '//') {
616 $this->_fields
['doNotImport'] = new CRM_Activity_Import_Field($name, $title, $type, $headerPattern, $dataPattern);
620 $tempField = CRM_Contact_BAO_Contact
::importableFields('Individual', NULL);
621 if (!array_key_exists($name, $tempField)) {
622 $this->_fields
[$name] = new CRM_Activity_Import_Field($name, $title, $type, $headerPattern, $dataPattern);
625 $this->_fields
[$name] = new CRM_Contact_Import_Field($name, $title, $type, $headerPattern, $dataPattern, CRM_Utils_Array
::value('hasLocationType', $tempField[$name]));
631 * Store parser values.
633 * @param CRM_Core_Session $store
635 public function set($store) {
636 $store->set('fileSize', $this->_fileSize
);
637 $store->set('lineCount', $this->_lineCount
);
638 $store->set('separator', $this->_separator
);
639 $store->set('fields', $this->getSelectValues());
641 $store->set('headerPatterns', $this->getHeaderPatterns());
642 $store->set('dataPatterns', $this->getDataPatterns());
643 $store->set('columnCount', $this->_activeFieldCount
);
645 $store->set('totalRowCount', $this->_totalCount
);
646 $store->set('validRowCount', $this->_validCount
);
647 $store->set('invalidRowCount', $this->_invalidRowCount
);
649 if ($this->_invalidRowCount
) {
650 $store->set('errorsFileName', $this->_errorFileName
);
653 if (isset($this->_rows
) && !empty($this->_rows
)) {
654 $store->set('dataValues', $this->_rows
);
659 * Export data to a CSV file.
661 * @param string $fileName
662 * @param array $header
665 public static function exportCSV($fileName, $header, $data) {
667 $fd = fopen($fileName, 'w');
669 foreach ($header as $key => $value) {
670 $header[$key] = "\"$value\"";
672 $config = CRM_Core_Config
::singleton();
673 $output[] = implode($config->fieldSeparator
, $header);
675 foreach ($data as $datum) {
676 foreach ($datum as $key => $value) {
677 $datum[$key] = "\"$value\"";
679 $output[] = implode($config->fieldSeparator
, $datum);
681 fwrite($fd, implode("\n", $output));