Merge pull request #15435 from MegaphoneJon/reporting-21
[civicrm-core.git] / CRM / Contact / Import / Parser.php
1 <?php
2 /*
3 +--------------------------------------------------------------------+
4 | Copyright CiviCRM LLC. All rights reserved. |
5 | |
6 | This work is published under the GNU AGPLv3 license with some |
7 | permitted exceptions and without any warranty. For full license |
8 | and copyright information, see https://civicrm.org/licensing |
9 +--------------------------------------------------------------------+
10 */
11
12 /**
13 *
14 * @package CRM
15 * @copyright CiviCRM LLC https://civicrm.org/licensing
16 */
17 abstract class CRM_Contact_Import_Parser extends CRM_Import_Parser {
18
19 protected $_tableName;
20
21 /**
22 * Total number of lines in file
23 *
24 * @var int
25 */
26 protected $_rowCount;
27
28 /**
29 * Running total number of un-matched Contacts.
30 *
31 * @var int
32 */
33 protected $_unMatchCount;
34
35 /**
36 * Array of unmatched lines.
37 *
38 * @var array
39 */
40 protected $_unMatch;
41
42 /**
43 * Total number of contacts with unparsed addresses
44 * @var int
45 */
46 protected $_unparsedAddressCount;
47
48 /**
49 * Filename of mismatch data
50 *
51 * @var string
52 */
53 protected $_misMatchFilemName;
54
55 protected $_primaryKeyName;
56 protected $_statusFieldName;
57
58 protected $fieldMetadata = [];
59 /**
60 * On duplicate
61 *
62 * @var int
63 */
64 public $_onDuplicate;
65
66 /**
67 * Dedupe rule group id to use if set
68 *
69 * @var int
70 */
71 public $_dedupeRuleGroupID = NULL;
72
73 /**
74 * Run import.
75 *
76 * @param string $tableName
77 * @param array $mapper
78 * @param int $mode
79 * @param int $contactType
80 * @param string $primaryKeyName
81 * @param string $statusFieldName
82 * @param int $onDuplicate
83 * @param int $statusID
84 * @param int $totalRowCount
85 * @param bool $doGeocodeAddress
86 * @param int $timeout
87 * @param string $contactSubType
88 * @param int $dedupeRuleGroupID
89 *
90 * @return mixed
91 */
92 public function run(
93 $tableName,
94 $mapper = [],
95 $mode = self::MODE_PREVIEW,
96 $contactType = self::CONTACT_INDIVIDUAL,
97 $primaryKeyName = '_id',
98 $statusFieldName = '_status',
99 $onDuplicate = self::DUPLICATE_SKIP,
100 $statusID = NULL,
101 $totalRowCount = NULL,
102 $doGeocodeAddress = FALSE,
103 $timeout = CRM_Contact_Import_Parser::DEFAULT_TIMEOUT,
104 $contactSubType = NULL,
105 $dedupeRuleGroupID = NULL
106 ) {
107
108 // TODO: Make the timeout actually work
109 $this->_onDuplicate = $onDuplicate;
110 $this->_dedupeRuleGroupID = $dedupeRuleGroupID;
111
112 switch ($contactType) {
113 case CRM_Import_Parser::CONTACT_INDIVIDUAL:
114 $this->_contactType = 'Individual';
115 break;
116
117 case CRM_Import_Parser::CONTACT_HOUSEHOLD:
118 $this->_contactType = 'Household';
119 break;
120
121 case CRM_Import_Parser::CONTACT_ORGANIZATION:
122 $this->_contactType = 'Organization';
123 }
124
125 $this->_contactSubType = $contactSubType;
126
127 $this->init();
128
129 $this->_rowCount = $this->_warningCount = 0;
130 $this->_invalidRowCount = $this->_validCount = 0;
131 $this->_totalCount = $this->_conflictCount = 0;
132
133 $this->_errors = [];
134 $this->_warnings = [];
135 $this->_conflicts = [];
136 $this->_unparsedAddresses = [];
137
138 $this->_tableName = $tableName;
139 $this->_primaryKeyName = $primaryKeyName;
140 $this->_statusFieldName = $statusFieldName;
141
142 if ($mode == self::MODE_MAPFIELD) {
143 $this->_rows = [];
144 }
145 else {
146 $this->_activeFieldCount = count($this->_activeFields);
147 }
148
149 if ($mode == self::MODE_IMPORT) {
150 //get the key of email field
151 foreach ($mapper as $key => $value) {
152 if (strtolower($value) == 'email') {
153 $emailKey = $key;
154 break;
155 }
156 }
157 }
158
159 if ($statusID) {
160 $this->progressImport($statusID);
161 $startTimestamp = $currTimestamp = $prevTimestamp = time();
162 }
163 // get the contents of the temp. import table
164 $query = "SELECT * FROM $tableName";
165 if ($mode == self::MODE_IMPORT) {
166 $query .= " WHERE $statusFieldName = 'NEW'";
167 }
168
169 $result = CRM_Core_DAO::executeQuery($query);
170
171 while ($result->fetch()) {
172 $values = array_values($result->toArray());
173 $this->_rowCount++;
174
175 /* trim whitespace around the values */
176 foreach ($values as $k => $v) {
177 $values[$k] = trim($v, " \t\r\n");
178 }
179 if (CRM_Utils_System::isNull($values)) {
180 continue;
181 }
182
183 $this->_totalCount++;
184
185 if ($mode == self::MODE_MAPFIELD) {
186 $returnCode = $this->mapField($values);
187 }
188 elseif ($mode == self::MODE_PREVIEW) {
189 $returnCode = $this->preview($values);
190 }
191 elseif ($mode == self::MODE_SUMMARY) {
192 $returnCode = $this->summary($values);
193 }
194 elseif ($mode == self::MODE_IMPORT) {
195 //print "Running parser in import mode<br/>\n";
196 $returnCode = $this->import($onDuplicate, $values, $doGeocodeAddress);
197 if ($statusID && (($this->_rowCount % 50) == 0)) {
198 $prevTimestamp = $this->progressImport($statusID, FALSE, $startTimestamp, $prevTimestamp, $totalRowCount);
199 }
200 }
201 else {
202 $returnCode = self::ERROR;
203 }
204
205 // note that a line could be valid but still produce a warning
206 if ($returnCode & self::VALID) {
207 $this->_validCount++;
208 if ($mode == self::MODE_MAPFIELD) {
209 $this->_rows[] = $values;
210 $this->_activeFieldCount = max($this->_activeFieldCount, count($values));
211 }
212 }
213
214 if ($returnCode & self::WARNING) {
215 $this->_warningCount++;
216 if ($this->_warningCount < $this->_maxWarningCount) {
217 $this->_warningCount[] = $line;
218 }
219 }
220
221 if ($returnCode & self::ERROR) {
222 $this->_invalidRowCount++;
223 array_unshift($values, $this->_rowCount);
224 $this->_errors[] = $values;
225 }
226
227 if ($returnCode & self::CONFLICT) {
228 $this->_conflictCount++;
229 array_unshift($values, $this->_rowCount);
230 $this->_conflicts[] = $values;
231 }
232
233 if ($returnCode & self::NO_MATCH) {
234 $this->_unMatchCount++;
235 array_unshift($values, $this->_rowCount);
236 $this->_unMatch[] = $values;
237 }
238
239 if ($returnCode & self::DUPLICATE) {
240 if ($returnCode & self::MULTIPLE_DUPE) {
241 /* TODO: multi-dupes should be counted apart from singles
242 * on non-skip action */
243 }
244 $this->_duplicateCount++;
245 array_unshift($values, $this->_rowCount);
246 $this->_duplicates[] = $values;
247 if ($onDuplicate != self::DUPLICATE_SKIP) {
248 $this->_validCount++;
249 }
250 }
251
252 if ($returnCode & self::UNPARSED_ADDRESS_WARNING) {
253 $this->_unparsedAddressCount++;
254 array_unshift($values, $this->_rowCount);
255 $this->_unparsedAddresses[] = $values;
256 }
257 // we give the derived class a way of aborting the process
258 // note that the return code could be multiple code or'ed together
259 if ($returnCode & self::STOP) {
260 break;
261 }
262
263 // if we are done processing the maxNumber of lines, break
264 if ($this->_maxLinesToProcess > 0 && $this->_validCount >= $this->_maxLinesToProcess) {
265 break;
266 }
267
268 // see if we've hit our timeout yet
269 /* if ( $the_thing_with_the_stuff ) {
270 do_something( );
271 } */
272 }
273
274 if ($mode == self::MODE_PREVIEW || $mode == self::MODE_IMPORT) {
275 $customHeaders = $mapper;
276
277 $customfields = CRM_Core_BAO_CustomField::getFields($this->_contactType);
278 foreach ($customHeaders as $key => $value) {
279 if ($id = CRM_Core_BAO_CustomField::getKeyID($value)) {
280 $customHeaders[$key] = $customfields[$id][0];
281 }
282 }
283
284 if ($this->_invalidRowCount) {
285 // removed view url for invlaid contacts
286 $headers = array_merge([
287 ts('Line Number'),
288 ts('Reason'),
289 ], $customHeaders);
290 $this->_errorFileName = self::errorFileName(self::ERROR);
291 self::exportCSV($this->_errorFileName, $headers, $this->_errors);
292 }
293 if ($this->_conflictCount) {
294 $headers = array_merge([
295 ts('Line Number'),
296 ts('Reason'),
297 ], $customHeaders);
298 $this->_conflictFileName = self::errorFileName(self::CONFLICT);
299 self::exportCSV($this->_conflictFileName, $headers, $this->_conflicts);
300 }
301 if ($this->_duplicateCount) {
302 $headers = array_merge([
303 ts('Line Number'),
304 ts('View Contact URL'),
305 ], $customHeaders);
306
307 $this->_duplicateFileName = self::errorFileName(self::DUPLICATE);
308 self::exportCSV($this->_duplicateFileName, $headers, $this->_duplicates);
309 }
310 if ($this->_unMatchCount) {
311 $headers = array_merge([
312 ts('Line Number'),
313 ts('Reason'),
314 ], $customHeaders);
315
316 $this->_misMatchFilemName = self::errorFileName(self::NO_MATCH);
317 self::exportCSV($this->_misMatchFilemName, $headers, $this->_unMatch);
318 }
319 if ($this->_unparsedAddressCount) {
320 $headers = array_merge([
321 ts('Line Number'),
322 ts('Contact Edit URL'),
323 ], $customHeaders);
324 $this->_errorFileName = self::errorFileName(self::UNPARSED_ADDRESS_WARNING);
325 self::exportCSV($this->_errorFileName, $headers, $this->_unparsedAddresses);
326 }
327 }
328 //echo "$this->_totalCount,$this->_invalidRowCount,$this->_conflictCount,$this->_duplicateCount";
329 return $this->fini();
330 }
331
332 /**
333 * Given a list of the importable field keys that the user has selected.
334 * set the active fields array to this list
335 *
336 * @param array $fieldKeys
337 * Mapped array of values.
338 */
339 public function setActiveFields($fieldKeys) {
340 $this->_activeFieldCount = count($fieldKeys);
341 foreach ($fieldKeys as $key) {
342 if (empty($this->_fields[$key])) {
343 $this->_activeFields[] = new CRM_Contact_Import_Field('', ts('- do not import -'));
344 }
345 else {
346 $this->_activeFields[] = clone($this->_fields[$key]);
347 }
348 }
349 }
350
351 /**
352 * @param $elements
353 */
354 public function setActiveFieldLocationTypes($elements) {
355 for ($i = 0; $i < count($elements); $i++) {
356 $this->_activeFields[$i]->_hasLocationType = $elements[$i];
357 }
358 }
359
360 /**
361 * @param $elements
362 */
363
364 /**
365 * @param $elements
366 */
367 public function setActiveFieldPhoneTypes($elements) {
368 for ($i = 0; $i < count($elements); $i++) {
369 $this->_activeFields[$i]->_phoneType = $elements[$i];
370 }
371 }
372
373 /**
374 * @param $elements
375 */
376 public function setActiveFieldWebsiteTypes($elements) {
377 for ($i = 0; $i < count($elements); $i++) {
378 $this->_activeFields[$i]->_websiteType = $elements[$i];
379 }
380 }
381
382 /**
383 * Set IM Service Provider type fields.
384 *
385 * @param array $elements
386 * IM service provider type ids.
387 */
388 public function setActiveFieldImProviders($elements) {
389 for ($i = 0; $i < count($elements); $i++) {
390 $this->_activeFields[$i]->_imProvider = $elements[$i];
391 }
392 }
393
394 /**
395 * @param $elements
396 */
397 public function setActiveFieldRelated($elements) {
398 for ($i = 0; $i < count($elements); $i++) {
399 $this->_activeFields[$i]->_related = $elements[$i];
400 }
401 }
402
403 /**
404 * @param $elements
405 */
406 public function setActiveFieldRelatedContactType($elements) {
407 for ($i = 0; $i < count($elements); $i++) {
408 $this->_activeFields[$i]->_relatedContactType = $elements[$i];
409 }
410 }
411
412 /**
413 * @param $elements
414 */
415 public function setActiveFieldRelatedContactDetails($elements) {
416 for ($i = 0; $i < count($elements); $i++) {
417 $this->_activeFields[$i]->_relatedContactDetails = $elements[$i];
418 }
419 }
420
421 /**
422 * @param $elements
423 */
424 public function setActiveFieldRelatedContactLocType($elements) {
425 for ($i = 0; $i < count($elements); $i++) {
426 $this->_activeFields[$i]->_relatedContactLocType = $elements[$i];
427 }
428 }
429
430 /**
431 * Set active field for related contact's phone type.
432 *
433 * @param array $elements
434 */
435 public function setActiveFieldRelatedContactPhoneType($elements) {
436 for ($i = 0; $i < count($elements); $i++) {
437 $this->_activeFields[$i]->_relatedContactPhoneType = $elements[$i];
438 }
439 }
440
441 /**
442 * @param $elements
443 */
444 public function setActiveFieldRelatedContactWebsiteType($elements) {
445 for ($i = 0; $i < count($elements); $i++) {
446 $this->_activeFields[$i]->_relatedContactWebsiteType = $elements[$i];
447 }
448 }
449
450 /**
451 * Set IM Service Provider type fields for related contacts.
452 *
453 * @param array $elements
454 * IM service provider type ids of related contact.
455 */
456 public function setActiveFieldRelatedContactImProvider($elements) {
457 for ($i = 0; $i < count($elements); $i++) {
458 $this->_activeFields[$i]->_relatedContactImProvider = $elements[$i];
459 }
460 }
461
462 /**
463 * Format the field values for input to the api.
464 *
465 * @return array
466 * (reference ) associative array of name/value pairs
467 */
468 public function &getActiveFieldParams() {
469 $params = [];
470
471 for ($i = 0; $i < $this->_activeFieldCount; $i++) {
472 if ($this->_activeFields[$i]->_name == 'do_not_import') {
473 continue;
474 }
475
476 if (isset($this->_activeFields[$i]->_value)) {
477 if (isset($this->_activeFields[$i]->_hasLocationType)) {
478 if (!isset($params[$this->_activeFields[$i]->_name])) {
479 $params[$this->_activeFields[$i]->_name] = [];
480 }
481
482 $value = [
483 $this->_activeFields[$i]->_name => $this->_activeFields[$i]->_value,
484 'location_type_id' => $this->_activeFields[$i]->_hasLocationType,
485 ];
486
487 if (isset($this->_activeFields[$i]->_phoneType)) {
488 $value['phone_type_id'] = $this->_activeFields[$i]->_phoneType;
489 }
490
491 // get IM service Provider type id
492 if (isset($this->_activeFields[$i]->_imProvider)) {
493 $value['provider_id'] = $this->_activeFields[$i]->_imProvider;
494 }
495
496 $params[$this->_activeFields[$i]->_name][] = $value;
497 }
498 elseif (isset($this->_activeFields[$i]->_websiteType)) {
499 $value = [
500 $this->_activeFields[$i]->_name => $this->_activeFields[$i]->_value,
501 'website_type_id' => $this->_activeFields[$i]->_websiteType,
502 ];
503
504 $params[$this->_activeFields[$i]->_name][] = $value;
505 }
506
507 if (!isset($params[$this->_activeFields[$i]->_name])) {
508 if (!isset($this->_activeFields[$i]->_related)) {
509 $params[$this->_activeFields[$i]->_name] = $this->_activeFields[$i]->_value;
510 }
511 }
512
513 //minor fix for CRM-4062
514 if (isset($this->_activeFields[$i]->_related)) {
515 if (!isset($params[$this->_activeFields[$i]->_related])) {
516 $params[$this->_activeFields[$i]->_related] = [];
517 }
518
519 if (!isset($params[$this->_activeFields[$i]->_related]['contact_type']) && !empty($this->_activeFields[$i]->_relatedContactType)) {
520 $params[$this->_activeFields[$i]->_related]['contact_type'] = $this->_activeFields[$i]->_relatedContactType;
521 }
522
523 if (isset($this->_activeFields[$i]->_relatedContactLocType) && !empty($this->_activeFields[$i]->_value)) {
524 if (!empty($params[$this->_activeFields[$i]->_related][$this->_activeFields[$i]->_relatedContactDetails]) &&
525 !is_array($params[$this->_activeFields[$i]->_related][$this->_activeFields[$i]->_relatedContactDetails])
526 ) {
527 $params[$this->_activeFields[$i]->_related][$this->_activeFields[$i]->_relatedContactDetails] = [];
528 }
529 $value = [
530 $this->_activeFields[$i]->_relatedContactDetails => $this->_activeFields[$i]->_value,
531 'location_type_id' => $this->_activeFields[$i]->_relatedContactLocType,
532 ];
533
534 if (isset($this->_activeFields[$i]->_relatedContactPhoneType)) {
535 $value['phone_type_id'] = $this->_activeFields[$i]->_relatedContactPhoneType;
536 }
537
538 // get IM service Provider type id for related contact
539 if (isset($this->_activeFields[$i]->_relatedContactImProvider)) {
540 $value['provider_id'] = $this->_activeFields[$i]->_relatedContactImProvider;
541 }
542
543 $params[$this->_activeFields[$i]->_related][$this->_activeFields[$i]->_relatedContactDetails][] = $value;
544 }
545 elseif (isset($this->_activeFields[$i]->_relatedContactWebsiteType)) {
546 $params[$this->_activeFields[$i]->_related][$this->_activeFields[$i]->_relatedContactDetails][] = [
547 'url' => $this->_activeFields[$i]->_value,
548 'website_type_id' => $this->_activeFields[$i]->_relatedContactWebsiteType,
549 ];
550 }
551 elseif (empty($this->_activeFields[$i]->_value) && isset($this->_activeFields[$i]->_relatedContactLocType)) {
552 if (empty($params[$this->_activeFields[$i]->_related][$this->_activeFields[$i]->_relatedContactDetails])) {
553 $params[$this->_activeFields[$i]->_related][$this->_activeFields[$i]->_relatedContactDetails] = [];
554 }
555 }
556 else {
557 $params[$this->_activeFields[$i]->_related][$this->_activeFields[$i]->_relatedContactDetails] = $this->_activeFields[$i]->_value;
558 }
559 }
560 }
561 }
562
563 return $params;
564 }
565
566 /**
567 * @return array
568 */
569 public function getColumnPatterns() {
570 CRM_Core_Error::deprecatedFunctionWarning('no longer used- use CRM_Contact_Import_MetadataTrait');
571 $values = [];
572 foreach ($this->_fields as $name => $field) {
573 $values[$name] = $field->_columnPattern;
574 }
575 return $values;
576 }
577
578 /**
579 * @param string $name
580 * @param $title
581 * @param int $type
582 * @param string $headerPattern
583 * @param string $dataPattern
584 * @param bool $hasLocationType
585 */
586 public function addField(
587 $name, $title, $type = CRM_Utils_Type::T_INT,
588 $headerPattern = '//', $dataPattern = '//',
589 $hasLocationType = FALSE
590 ) {
591 $this->_fields[$name] = new CRM_Contact_Import_Field($name, $title, $type, $headerPattern, $dataPattern, $hasLocationType);
592 if (empty($name)) {
593 $this->_fields['doNotImport'] = new CRM_Contact_Import_Field($name, $title, $type, $headerPattern, $dataPattern, $hasLocationType);
594 }
595 }
596
597 /**
598 * Store parser values.
599 *
600 * @param CRM_Core_Session $store
601 *
602 * @param int $mode
603 */
604 public function set($store, $mode = self::MODE_SUMMARY) {
605 $store->set('rowCount', $this->_rowCount);
606 $store->set('fields', $this->getSelectValues());
607 $store->set('fieldTypes', $this->getSelectTypes());
608
609 $store->set('columnCount', $this->_activeFieldCount);
610
611 $store->set('totalRowCount', $this->_totalCount);
612 $store->set('validRowCount', $this->_validCount);
613 $store->set('invalidRowCount', $this->_invalidRowCount);
614 $store->set('conflictRowCount', $this->_conflictCount);
615 $store->set('unMatchCount', $this->_unMatchCount);
616
617 switch ($this->_contactType) {
618 case 'Individual':
619 $store->set('contactType', CRM_Import_Parser::CONTACT_INDIVIDUAL);
620 break;
621
622 case 'Household':
623 $store->set('contactType', CRM_Import_Parser::CONTACT_HOUSEHOLD);
624 break;
625
626 case 'Organization':
627 $store->set('contactType', CRM_Import_Parser::CONTACT_ORGANIZATION);
628 }
629
630 if ($this->_invalidRowCount) {
631 $store->set('errorsFileName', $this->_errorFileName);
632 }
633 if ($this->_conflictCount) {
634 $store->set('conflictsFileName', $this->_conflictFileName);
635 }
636 if (isset($this->_rows) && !empty($this->_rows)) {
637 $store->set('dataValues', $this->_rows);
638 }
639
640 if ($this->_unMatchCount) {
641 $store->set('mismatchFileName', $this->_misMatchFilemName);
642 }
643
644 if ($mode == self::MODE_IMPORT) {
645 $store->set('duplicateRowCount', $this->_duplicateCount);
646 $store->set('unparsedAddressCount', $this->_unparsedAddressCount);
647 if ($this->_duplicateCount) {
648 $store->set('duplicatesFileName', $this->_duplicateFileName);
649 }
650 if ($this->_unparsedAddressCount) {
651 $store->set('errorsFileName', $this->_errorFileName);
652 }
653 }
654 //echo "$this->_totalCount,$this->_invalidRowCount,$this->_conflictCount,$this->_duplicateCount";
655 }
656
657 /**
658 * Export data to a CSV file.
659 *
660 * @param string $fileName
661 * @param array $header
662 * @param array $data
663 */
664 public static function exportCSV($fileName, $header, $data) {
665
666 if (file_exists($fileName) && !is_writable($fileName)) {
667 CRM_Core_Error::movedSiteError($fileName);
668 }
669 //hack to remove '_status', '_statusMsg' and '_id' from error file
670 $errorValues = [];
671 $dbRecordStatus = ['IMPORTED', 'ERROR', 'DUPLICATE', 'INVALID', 'NEW'];
672 foreach ($data as $rowCount => $rowValues) {
673 $count = 0;
674 foreach ($rowValues as $key => $val) {
675 if (in_array($val, $dbRecordStatus) && $count == (count($rowValues) - 3)) {
676 break;
677 }
678 $errorValues[$rowCount][$key] = $val;
679 $count++;
680 }
681 }
682 $data = $errorValues;
683
684 $output = [];
685 $fd = fopen($fileName, 'w');
686
687 foreach ($header as $key => $value) {
688 $header[$key] = "\"$value\"";
689 }
690 $config = CRM_Core_Config::singleton();
691 $output[] = implode($config->fieldSeparator, $header);
692
693 foreach ($data as $datum) {
694 foreach ($datum as $key => $value) {
695 $datum[$key] = "\"$value\"";
696 }
697 $output[] = implode($config->fieldSeparator, $datum);
698 }
699 fwrite($fd, implode("\n", $output));
700 fclose($fd);
701 }
702
703 /**
704 * Update the record with PK $id in the import database table.
705 *
706 * @param int $id
707 * @param array $params
708 */
709 public function updateImportRecord($id, &$params) {
710 $statusFieldName = $this->_statusFieldName;
711 $primaryKeyName = $this->_primaryKeyName;
712
713 if ($statusFieldName && $primaryKeyName) {
714 $dao = new CRM_Core_DAO();
715 $db = $dao->getDatabaseConnection();
716
717 $query = "UPDATE $this->_tableName
718 SET $statusFieldName = ?,
719 ${statusFieldName}Msg = ?
720 WHERE $primaryKeyName = ?";
721 $args = [
722 $params[$statusFieldName],
723 CRM_Utils_Array::value("${statusFieldName}Msg", $params),
724 $id,
725 ];
726
727 //print "Running query: $query<br/>With arguments: ".$params[$statusFieldName].", ".$params["${statusFieldName}Msg"].", $id<br/>";
728
729 $db->query($query, $args);
730 }
731 }
732
733 /**
734 * Format common params data to proper format to store.
735 *
736 * @param array $params
737 * Contain record values.
738 * @param array $formatted
739 * Array of formatted data.
740 * @param array $contactFields
741 * Contact DAO fields.
742 */
743 public function formatCommonData($params, &$formatted, &$contactFields) {
744 $csType = [
745 CRM_Utils_Array::value('contact_type', $formatted),
746 ];
747
748 //CRM-5125
749 //add custom fields for contact sub type
750 if (!empty($this->_contactSubType)) {
751 $csType = $this->_contactSubType;
752 }
753
754 if ($relCsType = CRM_Utils_Array::value('contact_sub_type', $formatted)) {
755 $csType = $relCsType;
756 }
757
758 $customFields = CRM_Core_BAO_CustomField::getFields($formatted['contact_type'], FALSE, FALSE, $csType);
759
760 $addressCustomFields = CRM_Core_BAO_CustomField::getFields('Address');
761 $customFields = $customFields + $addressCustomFields;
762
763 //if a Custom Email Greeting, Custom Postal Greeting or Custom Addressee is mapped, and no "Greeting / Addressee Type ID" is provided, then automatically set the type = Customized, CRM-4575
764 $elements = [
765 'email_greeting_custom' => 'email_greeting',
766 'postal_greeting_custom' => 'postal_greeting',
767 'addressee_custom' => 'addressee',
768 ];
769 foreach ($elements as $k => $v) {
770 if (array_key_exists($k, $params) && !(array_key_exists($v, $params))) {
771 $label = key(CRM_Core_OptionGroup::values($v, TRUE, NULL, NULL, 'AND v.name = "Customized"'));
772 $params[$v] = $label;
773 }
774 }
775
776 //format date first
777 $session = CRM_Core_Session::singleton();
778 $dateType = $session->get("dateTypes");
779 foreach ($params as $key => $val) {
780 $customFieldID = CRM_Core_BAO_CustomField::getKeyID($key);
781 if ($customFieldID &&
782 !array_key_exists($customFieldID, $addressCustomFields)
783 ) {
784 //we should not update Date to null, CRM-4062
785 if ($val && ($customFields[$customFieldID]['data_type'] == 'Date')) {
786 //CRM-21267
787 CRM_Contact_Import_Parser_Contact::formatCustomDate($params, $formatted, $dateType, $key);
788 }
789 elseif ($customFields[$customFieldID]['data_type'] == 'Boolean') {
790 if (empty($val) && !is_numeric($val) && $this->_onDuplicate == CRM_Import_Parser::DUPLICATE_FILL) {
791 //retain earlier value when Import mode is `Fill`
792 unset($params[$key]);
793 }
794 else {
795 $params[$key] = CRM_Utils_String::strtoboolstr($val);
796 }
797 }
798 }
799
800 if ($key == 'birth_date' && $val) {
801 CRM_Utils_Date::convertToDefaultDate($params, $dateType, $key);
802 }
803 elseif ($key == 'deceased_date' && $val) {
804 CRM_Utils_Date::convertToDefaultDate($params, $dateType, $key);
805 $params['is_deceased'] = 1;
806 }
807 elseif ($key == 'is_deceased' && $val) {
808 $params[$key] = CRM_Utils_String::strtoboolstr($val);
809 }
810 }
811
812 //now format custom data.
813 foreach ($params as $key => $field) {
814 if (is_array($field)) {
815 $isAddressCustomField = FALSE;
816 foreach ($field as $value) {
817 $break = FALSE;
818 if (is_array($value)) {
819 foreach ($value as $name => $testForEmpty) {
820 if ($addressCustomFieldID = CRM_Core_BAO_CustomField::getKeyID($name)) {
821 $isAddressCustomField = TRUE;
822 break;
823 }
824 // check if $value does not contain IM provider or phoneType
825 if (($name !== 'phone_type_id' || $name !== 'provider_id') && ($testForEmpty === '' || $testForEmpty == NULL)) {
826 $break = TRUE;
827 break;
828 }
829 }
830 }
831 else {
832 $break = TRUE;
833 }
834
835 if (!$break) {
836 if (!empty($value['location_type_id'])) {
837 $this->formatLocationBlock($value, $formatted);
838 }
839 else {
840 // @todo - this is still reachable - e.g. import with related contact info like firstname,lastname,spouse-first-name,spouse-last-name,spouse-home-phone
841 CRM_Core_Error::deprecatedFunctionWarning('this is not expected to be reachable now');
842 $this->formatContactParameters($value, $formatted);
843 }
844 }
845 }
846 if (!$isAddressCustomField) {
847 continue;
848 }
849 }
850
851 $formatValues = [
852 $key => $field,
853 ];
854
855 if (($key !== 'preferred_communication_method') && (array_key_exists($key, $contactFields))) {
856 // due to merging of individual table and
857 // contact table, we need to avoid
858 // preferred_communication_method forcefully
859 $formatValues['contact_type'] = $formatted['contact_type'];
860 }
861
862 if ($key == 'id' && isset($field)) {
863 $formatted[$key] = $field;
864 }
865 $this->formatContactParameters($formatValues, $formatted);
866
867 //Handling Custom Data
868 // note: Address custom fields will be handled separately inside formatContactParameters
869 if (($customFieldID = CRM_Core_BAO_CustomField::getKeyID($key)) &&
870 array_key_exists($customFieldID, $customFields) &&
871 !array_key_exists($customFieldID, $addressCustomFields)
872 ) {
873
874 $extends = $customFields[$customFieldID]['extends'] ?? NULL;
875 $htmlType = $customFields[$customFieldID]['html_type'] ?? NULL;
876 $dataType = $customFields[$customFieldID]['data_type'] ?? NULL;
877 $serialized = CRM_Core_BAO_CustomField::isSerialized($customFields[$customFieldID]);
878
879 if (!$serialized && in_array($htmlType, ['Select', 'Radio', 'Autocomplete-Select']) && in_array($dataType, ['String', 'Int'])) {
880 $customOption = CRM_Core_BAO_CustomOption::getCustomOption($customFieldID, TRUE);
881 foreach ($customOption as $customValue) {
882 $val = $customValue['value'] ?? NULL;
883 $label = strtolower($customValue['label'] ?? '');
884 $value = strtolower(trim($formatted[$key]));
885 if (($value == $label) || ($value == strtolower($val))) {
886 $params[$key] = $formatted[$key] = $val;
887 }
888 }
889 }
890 elseif ($serialized && !empty($formatted[$key]) && !empty($params[$key])) {
891 $mulValues = explode(',', $formatted[$key]);
892 $customOption = CRM_Core_BAO_CustomOption::getCustomOption($customFieldID, TRUE);
893 $formatted[$key] = [];
894 $params[$key] = [];
895 foreach ($mulValues as $v1) {
896 foreach ($customOption as $v2) {
897 if ((strtolower($v2['label']) == strtolower(trim($v1))) ||
898 (strtolower($v2['value']) == strtolower(trim($v1)))
899 ) {
900 if ($htmlType == 'CheckBox') {
901 $params[$key][$v2['value']] = $formatted[$key][$v2['value']] = 1;
902 }
903 else {
904 $params[$key][] = $formatted[$key][] = $v2['value'];
905 }
906 }
907 }
908 }
909 }
910 }
911 }
912
913 if (!empty($key) && ($customFieldID = CRM_Core_BAO_CustomField::getKeyID($key)) && array_key_exists($customFieldID, $customFields) &&
914 !array_key_exists($customFieldID, $addressCustomFields)
915 ) {
916 // @todo calling api functions directly is not supported
917 _civicrm_api3_custom_format_params($params, $formatted, $extends);
918 }
919
920 // to check if not update mode and unset the fields with empty value.
921 if (!$this->_updateWithId && array_key_exists('custom', $formatted)) {
922 foreach ($formatted['custom'] as $customKey => $customvalue) {
923 if (empty($formatted['custom'][$customKey][-1]['is_required'])) {
924 $formatted['custom'][$customKey][-1]['is_required'] = $customFields[$customKey]['is_required'];
925 }
926 $emptyValue = $customvalue[-1]['value'] ?? NULL;
927 if (!isset($emptyValue)) {
928 unset($formatted['custom'][$customKey]);
929 }
930 }
931 }
932
933 // parse street address, CRM-5450
934 if ($this->_parseStreetAddress) {
935 if (array_key_exists('address', $formatted) && is_array($formatted['address'])) {
936 foreach ($formatted['address'] as $instance => & $address) {
937 $streetAddress = $address['street_address'] ?? NULL;
938 if (empty($streetAddress)) {
939 continue;
940 }
941 // parse address field.
942 $parsedFields = CRM_Core_BAO_Address::parseStreetAddress($streetAddress);
943
944 //street address consider to be parsed properly,
945 //If we get street_name and street_number.
946 if (empty($parsedFields['street_name']) || empty($parsedFields['street_number'])) {
947 $parsedFields = array_fill_keys(array_keys($parsedFields), '');
948 }
949
950 // merge parse address w/ main address block.
951 $address = array_merge($address, $parsedFields);
952 }
953 }
954 }
955 }
956
957 /**
958 * Format contact parameters.
959 *
960 * @todo this function needs re-writing & re-merging into the main function.
961 *
962 * Here be dragons.
963 *
964 * @param array $values
965 * @param array $params
966 *
967 * @return bool
968 */
969 protected function formatContactParameters(&$values, &$params) {
970 // Crawl through the possible classes:
971 // Contact
972 // Individual
973 // Household
974 // Organization
975 // Location
976 // Address
977 // Email
978 // Phone
979 // IM
980 // Note
981 // Custom
982
983 // first add core contact values since for other Civi modules they are not added
984 $contactFields = CRM_Contact_DAO_Contact::fields();
985 _civicrm_api3_store_values($contactFields, $values, $params);
986
987 if (isset($values['contact_type'])) {
988 // we're an individual/household/org property
989
990 $fields[$values['contact_type']] = CRM_Contact_DAO_Contact::fields();
991
992 _civicrm_api3_store_values($fields[$values['contact_type']], $values, $params);
993 return TRUE;
994 }
995
996 // Cache the various object fields
997 // @todo - remove this after confirming this is just a compilation of other-wise-cached fields.
998 static $fields = [];
999
1000 if (isset($values['individual_prefix'])) {
1001 if (!empty($params['prefix_id'])) {
1002 $prefixes = CRM_Core_PseudoConstant::get('CRM_Contact_DAO_Contact', 'prefix_id');
1003 $params['prefix'] = $prefixes[$params['prefix_id']];
1004 }
1005 else {
1006 $params['prefix'] = $values['individual_prefix'];
1007 }
1008 return TRUE;
1009 }
1010
1011 if (isset($values['individual_suffix'])) {
1012 if (!empty($params['suffix_id'])) {
1013 $suffixes = CRM_Core_PseudoConstant::get('CRM_Contact_DAO_Contact', 'suffix_id');
1014 $params['suffix'] = $suffixes[$params['suffix_id']];
1015 }
1016 else {
1017 $params['suffix'] = $values['individual_suffix'];
1018 }
1019 return TRUE;
1020 }
1021
1022 // CRM-4575
1023 if (isset($values['email_greeting'])) {
1024 if (!empty($params['email_greeting_id'])) {
1025 $emailGreetingFilter = [
1026 'contact_type' => $params['contact_type'] ?? NULL,
1027 'greeting_type' => 'email_greeting',
1028 ];
1029 $emailGreetings = CRM_Core_PseudoConstant::greeting($emailGreetingFilter);
1030 $params['email_greeting'] = $emailGreetings[$params['email_greeting_id']];
1031 }
1032 else {
1033 $params['email_greeting'] = $values['email_greeting'];
1034 }
1035
1036 return TRUE;
1037 }
1038
1039 if (isset($values['postal_greeting'])) {
1040 if (!empty($params['postal_greeting_id'])) {
1041 $postalGreetingFilter = [
1042 'contact_type' => $params['contact_type'] ?? NULL,
1043 'greeting_type' => 'postal_greeting',
1044 ];
1045 $postalGreetings = CRM_Core_PseudoConstant::greeting($postalGreetingFilter);
1046 $params['postal_greeting'] = $postalGreetings[$params['postal_greeting_id']];
1047 }
1048 else {
1049 $params['postal_greeting'] = $values['postal_greeting'];
1050 }
1051 return TRUE;
1052 }
1053
1054 if (isset($values['addressee'])) {
1055 if (!empty($params['addressee_id'])) {
1056 $addresseeFilter = [
1057 'contact_type' => $params['contact_type'] ?? NULL,
1058 'greeting_type' => 'addressee',
1059 ];
1060 $addressee = CRM_Core_PseudoConstant::addressee($addresseeFilter);
1061 $params['addressee'] = $addressee[$params['addressee_id']];
1062 }
1063 else {
1064 $params['addressee'] = $values['addressee'];
1065 }
1066 return TRUE;
1067 }
1068
1069 if (isset($values['gender'])) {
1070 if (!empty($params['gender_id'])) {
1071 $genders = CRM_Core_PseudoConstant::get('CRM_Contact_DAO_Contact', 'gender_id');
1072 $params['gender'] = $genders[$params['gender_id']];
1073 }
1074 else {
1075 $params['gender'] = $values['gender'];
1076 }
1077 return TRUE;
1078 }
1079
1080 if (!empty($values['preferred_communication_method'])) {
1081 $comm = [];
1082 $pcm = array_change_key_case(array_flip(CRM_Core_PseudoConstant::get('CRM_Contact_DAO_Contact', 'preferred_communication_method')), CASE_LOWER);
1083
1084 $preffComm = explode(',', $values['preferred_communication_method']);
1085 foreach ($preffComm as $v) {
1086 $v = strtolower(trim($v));
1087 if (array_key_exists($v, $pcm)) {
1088 $comm[$pcm[$v]] = 1;
1089 }
1090 }
1091
1092 $params['preferred_communication_method'] = $comm;
1093 return TRUE;
1094 }
1095
1096 // format the website params.
1097 if (!empty($values['url'])) {
1098 static $websiteFields;
1099 if (!is_array($websiteFields)) {
1100 $websiteFields = CRM_Core_DAO_Website::fields();
1101 }
1102 if (!array_key_exists('website', $params) ||
1103 !is_array($params['website'])
1104 ) {
1105 $params['website'] = [];
1106 }
1107
1108 $websiteCount = count($params['website']);
1109 _civicrm_api3_store_values($websiteFields, $values,
1110 $params['website'][++$websiteCount]
1111 );
1112
1113 return TRUE;
1114 }
1115
1116 // get the formatted location blocks into params - w/ 3.0 format, CRM-4605
1117 if (!empty($values['location_type_id'])) {
1118 CRM_Core_Error::deprecatedFunctionWarning('this is not expected to be reachable now');
1119 return $this->formatLocationBlock($values, $params);
1120 }
1121
1122 if (isset($values['note'])) {
1123 // add a note field
1124 if (!isset($params['note'])) {
1125 $params['note'] = [];
1126 }
1127 $noteBlock = count($params['note']) + 1;
1128
1129 $params['note'][$noteBlock] = [];
1130 if (!isset($fields['Note'])) {
1131 $fields['Note'] = CRM_Core_DAO_Note::fields();
1132 }
1133
1134 // get the current logged in civicrm user
1135 $session = CRM_Core_Session::singleton();
1136 $userID = $session->get('userID');
1137
1138 if ($userID) {
1139 $values['contact_id'] = $userID;
1140 }
1141
1142 _civicrm_api3_store_values($fields['Note'], $values, $params['note'][$noteBlock]);
1143
1144 return TRUE;
1145 }
1146
1147 // Check for custom field values
1148 $customFields = CRM_Core_BAO_CustomField::getFields(CRM_Utils_Array::value('contact_type', $values),
1149 FALSE, FALSE, NULL, NULL, FALSE, FALSE, FALSE
1150 );
1151
1152 foreach ($values as $key => $value) {
1153 if ($customFieldID = CRM_Core_BAO_CustomField::getKeyID($key)) {
1154 // check if it's a valid custom field id
1155
1156 if (!array_key_exists($customFieldID, $customFields)) {
1157 return civicrm_api3_create_error('Invalid custom field ID');
1158 }
1159 else {
1160 $params[$key] = $value;
1161 }
1162 }
1163 }
1164 return TRUE;
1165 }
1166
1167 /**
1168 * Format location block ready for importing.
1169 *
1170 * There is some test coverage for this in CRM_Contact_Import_Parser_ContactTest
1171 * e.g. testImportPrimaryAddress.
1172 *
1173 * @param array $values
1174 * @param array $params
1175 *
1176 * @return bool
1177 */
1178 protected function formatLocationBlock(&$values, &$params) {
1179 $blockTypes = [
1180 'phone' => 'Phone',
1181 'email' => 'Email',
1182 'im' => 'IM',
1183 'openid' => 'OpenID',
1184 'phone_ext' => 'Phone',
1185 ];
1186 foreach ($blockTypes as $blockFieldName => $block) {
1187 if (!array_key_exists($blockFieldName, $values)) {
1188 continue;
1189 }
1190 $blockIndex = $values['location_type_id'] . (!empty($values['phone_type_id']) ? '_' . $values['phone_type_id'] : '');
1191
1192 // block present in value array.
1193 if (!array_key_exists($blockFieldName, $params) || !is_array($params[$blockFieldName])) {
1194 $params[$blockFieldName] = [];
1195 }
1196
1197 $fields[$block] = $this->getMetadataForEntity($block);
1198
1199 // copy value to dao field name.
1200 if ($blockFieldName == 'im') {
1201 $values['name'] = $values[$blockFieldName];
1202 }
1203
1204 _civicrm_api3_store_values($fields[$block], $values,
1205 $params[$blockFieldName][$blockIndex]
1206 );
1207
1208 $this->fillPrimary($params[$blockFieldName][$blockIndex], $values, $block, CRM_Utils_Array::value('id', $params));
1209
1210 if (empty($params['id']) && (count($params[$blockFieldName]) == 1)) {
1211 $params[$blockFieldName][$blockIndex]['is_primary'] = TRUE;
1212 }
1213
1214 // we only process single block at a time.
1215 return TRUE;
1216 }
1217
1218 // handle address fields.
1219 if (!array_key_exists('address', $params) || !is_array($params['address'])) {
1220 $params['address'] = [];
1221 }
1222
1223 // Note: we doing multiple value formatting here for address custom fields, plus putting into right format.
1224 // The actual formatting (like date, country ..etc) for address custom fields is taken care of while saving
1225 // the address in CRM_Core_BAO_Address::create method
1226 if (!empty($values['location_type_id'])) {
1227 static $customFields = [];
1228 if (empty($customFields)) {
1229 $customFields = CRM_Core_BAO_CustomField::getFields('Address');
1230 }
1231 // make a copy of values, as we going to make changes
1232 $newValues = $values;
1233 foreach ($values as $key => $val) {
1234 $customFieldID = CRM_Core_BAO_CustomField::getKeyID($key);
1235 if ($customFieldID && array_key_exists($customFieldID, $customFields)) {
1236
1237 $htmlType = $customFields[$customFieldID]['html_type'] ?? NULL;
1238 if (CRM_Core_BAO_CustomField::isSerialized($customFields[$customFieldID]) && $val) {
1239 $mulValues = explode(',', $val);
1240 $customOption = CRM_Core_BAO_CustomOption::getCustomOption($customFieldID, TRUE);
1241 $newValues[$key] = [];
1242 foreach ($mulValues as $v1) {
1243 foreach ($customOption as $v2) {
1244 if ((strtolower($v2['label']) == strtolower(trim($v1))) ||
1245 (strtolower($v2['value']) == strtolower(trim($v1)))
1246 ) {
1247 if ($htmlType == 'CheckBox') {
1248 $newValues[$key][$v2['value']] = 1;
1249 }
1250 else {
1251 $newValues[$key][] = $v2['value'];
1252 }
1253 }
1254 }
1255 }
1256 }
1257 }
1258 }
1259 // consider new values
1260 $values = $newValues;
1261 }
1262
1263 $fields['Address'] = $this->getMetadataForEntity('Address');
1264 // @todo this is kinda replicated below....
1265 _civicrm_api3_store_values($fields['Address'], $values, $params['address'][$values['location_type_id']]);
1266
1267 $addressFields = [
1268 'county',
1269 'country',
1270 'state_province',
1271 'supplemental_address_1',
1272 'supplemental_address_2',
1273 'supplemental_address_3',
1274 'StateProvince.name',
1275 ];
1276 foreach (array_keys($customFields) as $customFieldID) {
1277 $addressFields[] = 'custom_' . $customFieldID;
1278 }
1279
1280 foreach ($addressFields as $field) {
1281 if (array_key_exists($field, $values)) {
1282 if (!array_key_exists('address', $params)) {
1283 $params['address'] = [];
1284 }
1285 $params['address'][$values['location_type_id']][$field] = $values[$field];
1286 }
1287 }
1288
1289 $this->fillPrimary($params['address'][$values['location_type_id']], $values, 'address', CRM_Utils_Array::value('id', $params));
1290 return TRUE;
1291 }
1292
1293 /**
1294 * Get the field metadata for the relevant entity.
1295 *
1296 * @param string $entity
1297 *
1298 * @return array
1299 */
1300 protected function getMetadataForEntity($entity) {
1301 if (!isset($this->fieldMetadata[$entity])) {
1302 $className = "CRM_Core_DAO_$entity";
1303 $this->fieldMetadata[$entity] = $className::fields();
1304 }
1305 return $this->fieldMetadata[$entity];
1306 }
1307
1308 /**
1309 * Fill in the primary location.
1310 *
1311 * If the contact has a primary address we update it. Otherwise
1312 * we add an address of the default location type.
1313 *
1314 * @param array $params
1315 * Address block parameters
1316 * @param array $values
1317 * Input values
1318 * @param string $entity
1319 * - address, email, phone
1320 * @param int|null $contactID
1321 *
1322 * @throws \CiviCRM_API3_Exception
1323 */
1324 protected function fillPrimary(&$params, $values, $entity, $contactID) {
1325 if ($values['location_type_id'] === 'Primary') {
1326 if ($contactID) {
1327 $primary = civicrm_api3($entity, 'get', [
1328 'return' => 'location_type_id',
1329 'contact_id' => $contactID,
1330 'is_primary' => 1,
1331 'sequential' => 1,
1332 ]);
1333 }
1334 $defaultLocationType = CRM_Core_BAO_LocationType::getDefault();
1335 $params['location_type_id'] = (int) (isset($primary) && $primary['count']) ? $primary['values'][0]['location_type_id'] : $defaultLocationType->id;
1336 $params['is_primary'] = 1;
1337 }
1338 }
1339
1340 }