Commit | Line | Data |
---|---|---|
ec3811b1 CW |
1 | <?php |
2 | /* | |
3 | +--------------------------------------------------------------------+ | |
bc77d7c0 | 4 | | Copyright CiviCRM LLC. All rights reserved. | |
ec3811b1 | 5 | | | |
bc77d7c0 TO |
6 | | This work is published under the GNU AGPLv3 license with some | |
7 | | permitted exceptions and without any warranty. For full license | | |
8 | | and copyright information, see https://civicrm.org/licensing | | |
ec3811b1 | 9 | +--------------------------------------------------------------------+ |
d25dd0ee | 10 | */ |
ec3811b1 | 11 | |
80e9f1a2 | 12 | use Civi\Api4\CustomField; |
7b057b66 EM |
13 | use Civi\Api4\UserJob; |
14 | ||
ec3811b1 CW |
15 | /** |
16 | * | |
17 | * @package CRM | |
ca5cec67 | 18 | * @copyright CiviCRM LLC https://civicrm.org/licensing |
ec3811b1 | 19 | */ |
ec3811b1 CW |
20 | abstract class CRM_Import_Parser { |
21 | /** | |
22 | * Settings | |
23 | */ | |
ca2057ea | 24 | const MAX_WARNINGS = 25, DEFAULT_TIMEOUT = 30; |
ec3811b1 CW |
25 | |
26 | /** | |
27 | * Return codes | |
28 | */ | |
7da04cde | 29 | const VALID = 1, WARNING = 2, ERROR = 4, CONFLICT = 8, STOP = 16, DUPLICATE = 32, MULTIPLE_DUPE = 64, NO_MATCH = 128, UNPARSED_ADDRESS_WARNING = 256; |
ec3811b1 CW |
30 | |
31 | /** | |
32 | * Parser modes | |
33 | */ | |
7da04cde | 34 | const MODE_MAPFIELD = 1, MODE_PREVIEW = 2, MODE_SUMMARY = 4, MODE_IMPORT = 8; |
ec3811b1 CW |
35 | |
36 | /** | |
37 | * Codes for duplicate record handling | |
38 | */ | |
a87fbfbd | 39 | const DUPLICATE_SKIP = 1, DUPLICATE_UPDATE = 4, DUPLICATE_FILL = 8, DUPLICATE_NOCHECK = 16; |
ec3811b1 CW |
40 | |
41 | /** | |
42 | * Contact types | |
43 | */ | |
7da04cde | 44 | const CONTACT_INDIVIDUAL = 1, CONTACT_HOUSEHOLD = 2, CONTACT_ORGANIZATION = 4; |
69a4c20a | 45 | |
7b057b66 EM |
46 | /** |
47 | * User job id. | |
48 | * | |
49 | * This is the primary key of the civicrm_user_job table which is used to | |
50 | * track the import. | |
51 | * | |
52 | * @var int | |
53 | */ | |
54 | protected $userJobID; | |
55 | ||
578e4db3 EM |
56 | /** |
57 | * The user job in use. | |
58 | * | |
59 | * @var array | |
60 | */ | |
61 | protected $userJob; | |
62 | ||
018c9e26 EM |
63 | /** |
64 | * Potentially ambiguous options. | |
65 | * | |
66 | * For example 'UT' is a state in more than one country. | |
67 | * | |
68 | * @var array | |
69 | */ | |
70 | protected $ambiguousOptions = []; | |
71 | ||
72 | /** | |
73 | * States to country mapping. | |
74 | * | |
75 | * @var array | |
76 | */ | |
77 | protected $statesByCountry = []; | |
78 | ||
7b057b66 EM |
79 | /** |
80 | * @return int|null | |
81 | */ | |
82 | public function getUserJobID(): ?int { | |
83 | return $this->userJobID; | |
84 | } | |
85 | ||
578e4db3 EM |
86 | /** |
87 | * Ids of contacts created this iteration. | |
88 | * | |
89 | * @var array | |
90 | */ | |
91 | protected $createdContacts = []; | |
92 | ||
7b057b66 EM |
93 | /** |
94 | * Set user job ID. | |
95 | * | |
96 | * @param int $userJobID | |
79d21b5b EM |
97 | * |
98 | * @return self | |
7b057b66 | 99 | */ |
79d21b5b | 100 | public function setUserJobID(int $userJobID): self { |
7b057b66 | 101 | $this->userJobID = $userJobID; |
79d21b5b | 102 | return $this; |
7b057b66 EM |
103 | } |
104 | ||
e0ce85b6 EM |
105 | /** |
106 | * Countries that the site is restricted to | |
107 | * | |
108 | * @var array|false | |
109 | */ | |
110 | private $availableCountries; | |
111 | ||
7b057b66 EM |
112 | /** |
113 | * Get User Job. | |
114 | * | |
115 | * API call to retrieve the userJob row. | |
116 | * | |
117 | * @return array | |
118 | * | |
119 | * @throws \API_Exception | |
120 | */ | |
121 | protected function getUserJob(): array { | |
578e4db3 EM |
122 | if (empty($this->userJob)) { |
123 | $this->userJob = UserJob::get() | |
124 | ->addWhere('id', '=', $this->getUserJobID()) | |
125 | ->execute() | |
126 | ->first(); | |
127 | } | |
128 | return $this->userJob; | |
7b057b66 EM |
129 | } |
130 | ||
5e21b588 EM |
131 | /** |
132 | * Get the relevant datasource object. | |
133 | * | |
134 | * @return \CRM_Import_DataSource|null | |
135 | * | |
136 | * @throws \API_Exception | |
137 | */ | |
138 | protected function getDataSourceObject(): ?CRM_Import_DataSource { | |
139 | $className = $this->getSubmittedValue('dataSource'); | |
140 | if ($className) { | |
141 | /* @var CRM_Import_DataSource $dataSource */ | |
142 | return new $className($this->getUserJobID()); | |
143 | } | |
144 | return NULL; | |
145 | } | |
146 | ||
52bd01f5 EM |
147 | /** |
148 | * Get the submitted value, as stored on the user job. | |
149 | * | |
150 | * @param string $fieldName | |
151 | * | |
152 | * @return mixed | |
153 | * | |
77c96d86 EM |
154 | * @noinspection PhpDocMissingThrowsInspection |
155 | * @noinspection PhpUnhandledExceptionInspection | |
52bd01f5 EM |
156 | */ |
157 | protected function getSubmittedValue(string $fieldName) { | |
158 | return $this->getUserJob()['metadata']['submitted_values'][$fieldName]; | |
159 | } | |
160 | ||
5e21b588 EM |
161 | /** |
162 | * Has the import completed. | |
163 | * | |
164 | * @return bool | |
165 | * | |
166 | * @throws \API_Exception | |
167 | * @throws \CRM_Core_Exception | |
168 | */ | |
169 | public function isComplete() :bool { | |
170 | return $this->getDataSourceObject()->isCompleted(); | |
171 | } | |
172 | ||
52bd01f5 EM |
173 | /** |
174 | * Get configured contact type. | |
175 | * | |
176 | * @throws \API_Exception | |
177 | */ | |
178 | protected function getContactType() { | |
179 | if (!$this->_contactType) { | |
180 | $contactTypeMapping = [ | |
181 | CRM_Import_Parser::CONTACT_INDIVIDUAL => 'Individual', | |
182 | CRM_Import_Parser::CONTACT_HOUSEHOLD => 'Household', | |
183 | CRM_Import_Parser::CONTACT_ORGANIZATION => 'Organization', | |
184 | ]; | |
185 | $this->_contactType = $contactTypeMapping[$this->getSubmittedValue('contactType')]; | |
186 | } | |
187 | return $this->_contactType; | |
188 | } | |
189 | ||
80cb71bb EM |
190 | /** |
191 | * Get configured contact type. | |
192 | * | |
193 | * @return string|null | |
194 | * | |
195 | * @throws \API_Exception | |
196 | */ | |
197 | public function getContactSubType() { | |
198 | if (!$this->_contactSubType) { | |
199 | $this->_contactSubType = $this->getSubmittedValue('contactSubType'); | |
200 | } | |
201 | return $this->_contactSubType; | |
202 | } | |
203 | ||
69a4c20a | 204 | /** |
100fef9d | 205 | * Total number of non empty lines |
971e129b | 206 | * @var int |
69a4c20a CW |
207 | */ |
208 | protected $_totalCount; | |
209 | ||
210 | /** | |
100fef9d | 211 | * Running total number of valid lines |
971e129b | 212 | * @var int |
69a4c20a CW |
213 | */ |
214 | protected $_validCount; | |
215 | ||
216 | /** | |
100fef9d | 217 | * Running total number of invalid rows |
971e129b | 218 | * @var int |
69a4c20a CW |
219 | */ |
220 | protected $_invalidRowCount; | |
221 | ||
222 | /** | |
100fef9d | 223 | * Maximum number of non-empty/comment lines to process |
69a4c20a CW |
224 | * |
225 | * @var int | |
226 | */ | |
227 | protected $_maxLinesToProcess; | |
228 | ||
69a4c20a | 229 | /** |
100fef9d | 230 | * Array of error lines, bounded by MAX_ERROR |
971e129b | 231 | * @var array |
69a4c20a CW |
232 | */ |
233 | protected $_errors; | |
234 | ||
69a4c20a | 235 | /** |
100fef9d | 236 | * Total number of duplicate (from database) lines |
971e129b | 237 | * @var int |
69a4c20a CW |
238 | */ |
239 | protected $_duplicateCount; | |
240 | ||
241 | /** | |
100fef9d | 242 | * Array of duplicate lines |
971e129b | 243 | * @var array |
69a4c20a CW |
244 | */ |
245 | protected $_duplicates; | |
246 | ||
69a4c20a | 247 | /** |
100fef9d | 248 | * Maximum number of warnings to store |
971e129b | 249 | * @var int |
69a4c20a CW |
250 | */ |
251 | protected $_maxWarningCount = self::MAX_WARNINGS; | |
252 | ||
253 | /** | |
100fef9d | 254 | * Array of warning lines, bounded by MAX_WARNING |
971e129b | 255 | * @var array |
69a4c20a CW |
256 | */ |
257 | protected $_warnings; | |
258 | ||
259 | /** | |
100fef9d | 260 | * Array of all the fields that could potentially be part |
69a4c20a CW |
261 | * of this import process |
262 | * @var array | |
263 | */ | |
264 | protected $_fields; | |
265 | ||
64cafaa3 | 266 | /** |
267 | * Metadata for all available fields, keyed by unique name. | |
268 | * | |
269 | * This is intended to supercede $_fields which uses a special sauce format which | |
270 | * importableFieldsMetadata uses the standard getfields type format. | |
271 | * | |
272 | * @var array | |
273 | */ | |
274 | protected $importableFieldsMetadata = []; | |
275 | ||
276 | /** | |
277 | * Get metadata for all importable fields in std getfields style format. | |
278 | * | |
279 | * @return array | |
280 | */ | |
281 | public function getImportableFieldsMetadata(): array { | |
282 | return $this->importableFieldsMetadata; | |
283 | } | |
284 | ||
285 | /** | |
286 | * Set metadata for all importable fields in std getfields style format. | |
f25114b4 | 287 | * |
64cafaa3 | 288 | * @param array $importableFieldsMetadata |
289 | */ | |
f25114b4 | 290 | public function setImportableFieldsMetadata(array $importableFieldsMetadata): void { |
64cafaa3 | 291 | $this->importableFieldsMetadata = $importableFieldsMetadata; |
292 | } | |
293 | ||
73edfc10 EM |
294 | /** |
295 | * Gets the fields available for importing in a key-name, title format. | |
296 | * | |
297 | * @return array | |
298 | * eg. ['first_name' => 'First Name'.....] | |
299 | * | |
300 | * @throws \API_Exception | |
301 | * | |
302 | * @todo - we are constructing the metadata before we | |
303 | * have set the contact type so we re-do it here. | |
304 | * | |
305 | * Once we have cleaned up the way the mapper is handled | |
306 | * we can ditch all the existing _construct parameters in favour | |
307 | * of just the userJobID - there are current open PRs towards this end. | |
308 | */ | |
309 | public function getAvailableFields(): array { | |
310 | $this->setFieldMetadata(); | |
311 | $return = []; | |
312 | foreach ($this->getImportableFieldsMetadata() as $name => $field) { | |
313 | if ($name === 'id' && $this->isSkipDuplicates()) { | |
314 | // Duplicates are being skipped so id matching is not availble. | |
315 | continue; | |
316 | } | |
cbc11a37 | 317 | $return[$name] = $field['html']['label'] ?? $field['title']; |
73edfc10 EM |
318 | } |
319 | return $return; | |
320 | } | |
321 | ||
322 | /** | |
323 | * Did the user specify duplicates should be skipped and not imported. | |
324 | * | |
325 | * @return bool | |
326 | * | |
327 | * @throws \API_Exception | |
328 | */ | |
329 | protected function isSkipDuplicates(): bool { | |
330 | return ((int) $this->getSubmittedValue('onDuplicate')) === CRM_Import_Parser::DUPLICATE_SKIP; | |
331 | } | |
332 | ||
77c96d86 EM |
333 | /** |
334 | * Did the user specify duplicates should be filled with missing data. | |
335 | * | |
336 | * @return bool | |
337 | */ | |
338 | protected function isFillDuplicates(): bool { | |
339 | return ((int) $this->getSubmittedValue('onDuplicate')) === CRM_Import_Parser::DUPLICATE_FILL; | |
340 | } | |
341 | ||
69a4c20a | 342 | /** |
100fef9d | 343 | * Array of the fields that are actually part of the import process |
69a4c20a CW |
344 | * the position in the array also dictates their position in the import |
345 | * file | |
346 | * @var array | |
347 | */ | |
288db2d2 | 348 | protected $_activeFields = []; |
69a4c20a CW |
349 | |
350 | /** | |
100fef9d | 351 | * Cache the count of active fields |
69a4c20a CW |
352 | * |
353 | * @var int | |
354 | */ | |
355 | protected $_activeFieldCount; | |
356 | ||
357 | /** | |
100fef9d | 358 | * Cache of preview rows |
69a4c20a CW |
359 | * |
360 | * @var array | |
361 | */ | |
362 | protected $_rows; | |
363 | ||
364 | /** | |
100fef9d | 365 | * Filename of error data |
69a4c20a CW |
366 | * |
367 | * @var string | |
368 | */ | |
369 | protected $_errorFileName; | |
370 | ||
69a4c20a | 371 | /** |
100fef9d | 372 | * Filename of duplicate data |
69a4c20a CW |
373 | * |
374 | * @var string | |
375 | */ | |
376 | protected $_duplicateFileName; | |
377 | ||
378 | /** | |
100fef9d | 379 | * Contact type |
69a4c20a | 380 | * |
52bd01f5 | 381 | * @var string |
69a4c20a CW |
382 | */ |
383 | public $_contactType; | |
80cb71bb | 384 | |
0d46885c EM |
385 | /** |
386 | * @param string $contactType | |
387 | * | |
388 | * @return CRM_Import_Parser | |
389 | */ | |
390 | public function setContactType(string $contactType): CRM_Import_Parser { | |
391 | $this->_contactType = $contactType; | |
392 | return $this; | |
393 | } | |
394 | ||
e87ff4ce | 395 | /** |
396 | * Contact sub-type | |
397 | * | |
80cb71bb | 398 | * @var int|null |
e87ff4ce | 399 | */ |
400 | public $_contactSubType; | |
69a4c20a | 401 | |
80cb71bb EM |
402 | /** |
403 | * @param int|null $contactSubType | |
404 | * | |
405 | * @return self | |
406 | */ | |
407 | public function setContactSubType(?int $contactSubType): self { | |
408 | $this->_contactSubType = $contactSubType; | |
409 | return $this; | |
410 | } | |
411 | ||
69a4c20a | 412 | /** |
e87ff4ce | 413 | * Class constructor. |
69a4c20a | 414 | */ |
00be9182 | 415 | public function __construct() { |
69a4c20a | 416 | $this->_maxLinesToProcess = 0; |
69a4c20a CW |
417 | } |
418 | ||
69a4c20a | 419 | /** |
fe482240 | 420 | * Set and validate field values. |
69a4c20a | 421 | * |
5a4f6742 | 422 | * @param array $elements |
16b10e64 | 423 | * array. |
69a4c20a | 424 | */ |
1006edc9 | 425 | public function setActiveFieldValues($elements): void { |
69a4c20a CW |
426 | $maxCount = count($elements) < $this->_activeFieldCount ? count($elements) : $this->_activeFieldCount; |
427 | for ($i = 0; $i < $maxCount; $i++) { | |
428 | $this->_activeFields[$i]->setValue($elements[$i]); | |
429 | } | |
430 | ||
431 | // reset all the values that we did not have an equivalent import element | |
432 | for (; $i < $this->_activeFieldCount; $i++) { | |
433 | $this->_activeFields[$i]->resetValue(); | |
434 | } | |
69a4c20a CW |
435 | } |
436 | ||
437 | /** | |
fe482240 | 438 | * Format the field values for input to the api. |
69a4c20a | 439 | * |
a6c01b45 CW |
440 | * @return array |
441 | * (reference) associative array of name/value pairs | |
69a4c20a | 442 | */ |
00be9182 | 443 | public function &getActiveFieldParams() { |
be2fb01f | 444 | $params = []; |
69a4c20a CW |
445 | for ($i = 0; $i < $this->_activeFieldCount; $i++) { |
446 | if (isset($this->_activeFields[$i]->_value) | |
447 | && !isset($params[$this->_activeFields[$i]->_name]) | |
448 | && !isset($this->_activeFields[$i]->_related) | |
449 | ) { | |
450 | ||
451 | $params[$this->_activeFields[$i]->_name] = $this->_activeFields[$i]->_value; | |
452 | } | |
453 | } | |
454 | return $params; | |
455 | } | |
456 | ||
8cebffb2 | 457 | /** |
badf5061 JP |
458 | * Add progress bar to the import process. Calculates time remaining, status etc. |
459 | * | |
8cebffb2 | 460 | * @param $statusID |
badf5061 | 461 | * status id of the import process saved in $config->uploadDir. |
8cebffb2 JP |
462 | * @param bool $startImport |
463 | * True when progress bar is to be initiated. | |
464 | * @param $startTimestamp | |
f25114b4 | 465 | * Initial timestamp when the import was started. |
8cebffb2 JP |
466 | * @param $prevTimestamp |
467 | * Previous timestamp when this function was last called. | |
468 | * @param $totalRowCount | |
469 | * Total number of rows in the import file. | |
470 | * | |
471 | * @return NULL|$currTimestamp | |
472 | */ | |
473 | public function progressImport($statusID, $startImport = TRUE, $startTimestamp = NULL, $prevTimestamp = NULL, $totalRowCount = NULL) { | |
f25114b4 | 474 | $statusFile = CRM_Core_Config::singleton()->uploadDir . "status_{$statusID}.txt"; |
8cebffb2 JP |
475 | |
476 | if ($startImport) { | |
477 | $status = "<div class='description'> " . ts('No processing status reported yet.') . "</div>"; | |
478 | //do not force the browser to display the save dialog, CRM-7640 | |
be2fb01f | 479 | $contents = json_encode([0, $status]); |
8cebffb2 JP |
480 | file_put_contents($statusFile, $contents); |
481 | } | |
482 | else { | |
2e1f50d6 | 483 | $rowCount = $this->_rowCount ?? $this->_lineCount; |
8cebffb2 | 484 | $currTimestamp = time(); |
8cebffb2 JP |
485 | $time = ($currTimestamp - $prevTimestamp); |
486 | $recordsLeft = $totalRowCount - $rowCount; | |
487 | if ($recordsLeft < 0) { | |
488 | $recordsLeft = 0; | |
489 | } | |
490 | $estimatedTime = ($recordsLeft / 50) * $time; | |
491 | $estMinutes = floor($estimatedTime / 60); | |
492 | $timeFormatted = ''; | |
493 | if ($estMinutes > 1) { | |
494 | $timeFormatted = $estMinutes . ' ' . ts('minutes') . ' '; | |
495 | $estimatedTime = $estimatedTime - ($estMinutes * 60); | |
496 | } | |
497 | $timeFormatted .= round($estimatedTime) . ' ' . ts('seconds'); | |
498 | $processedPercent = (int ) (($rowCount * 100) / $totalRowCount); | |
499 | $statusMsg = ts('%1 of %2 records - %3 remaining', | |
be2fb01f | 500 | [1 => $rowCount, 2 => $totalRowCount, 3 => $timeFormatted] |
8cebffb2 JP |
501 | ); |
502 | $status = "<div class=\"description\"> <strong>{$statusMsg}</strong></div>"; | |
be2fb01f | 503 | $contents = json_encode([$processedPercent, $status]); |
8cebffb2 JP |
504 | |
505 | file_put_contents($statusFile, $contents); | |
506 | return $currTimestamp; | |
507 | } | |
508 | } | |
509 | ||
e0ef6999 EM |
510 | /** |
511 | * @return array | |
512 | */ | |
f25114b4 | 513 | public function getSelectValues(): array { |
be2fb01f | 514 | $values = []; |
69a4c20a CW |
515 | foreach ($this->_fields as $name => $field) { |
516 | $values[$name] = $field->_title; | |
517 | } | |
518 | return $values; | |
519 | } | |
520 | ||
e0ef6999 EM |
521 | /** |
522 | * @return array | |
523 | */ | |
00be9182 | 524 | public function getSelectTypes() { |
be2fb01f | 525 | $values = []; |
79d21b5b EM |
526 | // This is only called from the MapField form in isolation now, |
527 | // so we need to set the metadata. | |
528 | $this->init(); | |
69a4c20a CW |
529 | foreach ($this->_fields as $name => $field) { |
530 | if (isset($field->_hasLocationType)) { | |
531 | $values[$name] = $field->_hasLocationType; | |
532 | } | |
533 | } | |
534 | return $values; | |
535 | } | |
536 | ||
e0ef6999 EM |
537 | /** |
538 | * @return array | |
539 | */ | |
4d9f4d69 | 540 | public function getHeaderPatterns(): array { |
be2fb01f | 541 | $values = []; |
69a4c20a CW |
542 | foreach ($this->_fields as $name => $field) { |
543 | if (isset($field->_headerPattern)) { | |
544 | $values[$name] = $field->_headerPattern; | |
545 | } | |
546 | } | |
547 | return $values; | |
548 | } | |
549 | ||
e0ef6999 EM |
550 | /** |
551 | * @return array | |
552 | */ | |
4d9f4d69 | 553 | public function getDataPatterns():array { |
be2fb01f | 554 | $values = []; |
69a4c20a CW |
555 | foreach ($this->_fields as $name => $field) { |
556 | $values[$name] = $field->_dataPattern; | |
557 | } | |
558 | return $values; | |
559 | } | |
560 | ||
561 | /** | |
2b4bc760 | 562 | * Remove single-quote enclosures from a value array (row). |
69a4c20a CW |
563 | * |
564 | * @param array $values | |
565 | * @param string $enclosure | |
566 | * | |
567 | * @return void | |
69a4c20a | 568 | */ |
00be9182 | 569 | public static function encloseScrub(&$values, $enclosure = "'") { |
69a4c20a CW |
570 | if (empty($values)) { |
571 | return; | |
572 | } | |
573 | ||
574 | foreach ($values as $k => $v) { | |
575 | $values[$k] = preg_replace("/^$enclosure(.*)$enclosure$/", '$1', $v); | |
576 | } | |
577 | } | |
578 | ||
579 | /** | |
fe482240 | 580 | * Setter function. |
69a4c20a CW |
581 | * |
582 | * @param int $max | |
583 | * | |
584 | * @return void | |
69a4c20a | 585 | */ |
00be9182 | 586 | public function setMaxLinesToProcess($max) { |
69a4c20a CW |
587 | $this->_maxLinesToProcess = $max; |
588 | } | |
589 | ||
7e56b830 EM |
590 | /** |
591 | * Validate that we have the required fields to create the contact or find it to update. | |
592 | * | |
593 | * Note that the users duplicate selection affects this as follows | |
594 | * - if they did not select an update variant then the id field is not | |
595 | * permitted in the mapping - so we can assume the presence of id means | |
596 | * we should use it | |
597 | * - the external_identifier field is valid in place of the other fields | |
598 | * when they have chosen update or fill - in this case we are only looking | |
599 | * to update an existing contact. | |
600 | * | |
601 | * @param string $contactType | |
602 | * @param array $params | |
603 | * @param bool $isPermitExistingMatchFields | |
7d2012dc EM |
604 | * True if the it is enough to have fields which will enable us to find |
605 | * an existing contact (eg. external_identifier). | |
606 | * @param string $prefixString | |
607 | * String to include in the exception (e.g '(Child of)' if we are validating | |
608 | * a related contact. | |
7e56b830 EM |
609 | * |
610 | * @return void | |
611 | * @throws \CRM_Core_Exception | |
612 | */ | |
7d2012dc | 613 | protected function validateRequiredContactFields(string $contactType, array $params, bool $isPermitExistingMatchFields = TRUE, $prefixString = ''): void { |
7e56b830 EM |
614 | if (!empty($params['id'])) { |
615 | return; | |
616 | } | |
617 | $requiredFields = [ | |
618 | 'Individual' => [ | |
619 | 'first_name_last_name' => ['first_name' => ts('First Name'), 'last_name' => ts('Last Name')], | |
620 | 'email' => ts('Email Address'), | |
621 | ], | |
622 | 'Organization' => ['organization_name' => ts('Organization Name')], | |
623 | 'Household' => ['household_name' => ts('Household Name')], | |
624 | ][$contactType]; | |
625 | if ($isPermitExistingMatchFields) { | |
626 | $requiredFields['external_identifier'] = ts('External Identifier'); | |
627 | // Historically just an email has been accepted as it is 'usually good enough' | |
628 | // for a dedupe rule look up - but really this is a stand in for | |
629 | // whatever is needed to find an existing matching contact using the | |
630 | // specified dedupe rule (or the default Unsupervised if not specified). | |
631 | $requiredFields['email'] = ts('Email Address'); | |
632 | } | |
7d2012dc | 633 | $this->validateRequiredFields($requiredFields, $params, $prefixString); |
7e56b830 EM |
634 | } |
635 | ||
578e4db3 EM |
636 | protected function doPostImportActions() { |
637 | $userJob = $this->getUserJob(); | |
992a3d9e EM |
638 | $summaryInfo = $userJob['metadata']['summary_info'] ?? []; |
639 | $actions = $userJob['metadata']['post_actions'] ?? []; | |
578e4db3 EM |
640 | if (!empty($actions['group'])) { |
641 | $groupAdditions = $this->addImportedContactsToNewGroup($this->createdContacts, $actions['group']); | |
642 | foreach ($actions['group'] as $groupID) { | |
643 | $summaryInfo['groups'][$groupID]['added'] += $groupAdditions[$groupID]['added']; | |
644 | $summaryInfo['groups'][$groupID]['notAdded'] += $groupAdditions[$groupID]['notAdded']; | |
645 | } | |
646 | } | |
647 | if (!empty($actions['tag'])) { | |
648 | $tagAdditions = $this->tagImportedContactsWithNewTag($this->createdContacts, $actions['tag']); | |
649 | foreach ($actions['tag'] as $tagID) { | |
650 | $summaryInfo['tags'][$tagID]['added'] += $tagAdditions[$tagID]['added']; | |
651 | $summaryInfo['tags'][$tagID]['notAdded'] += $tagAdditions[$tagID]['notAdded']; | |
652 | } | |
653 | } | |
654 | ||
655 | $this->userJob['metadata']['summary_info'] = $summaryInfo; | |
656 | UserJob::update(FALSE)->addWhere('id', '=', $userJob['id'])->setValues(['metadata' => $this->userJob['metadata']])->execute(); | |
657 | } | |
658 | ||
3592a5e4 EM |
659 | public function queue() { |
660 | $dataSource = $this->getDataSourceObject(); | |
661 | $totalRowCount = $totalRows = $dataSource->getRowCount(['new']); | |
662 | $queue = Civi::queue('user_job_' . $this->getUserJobID(), ['type' => 'Sql', 'error' => 'abort']); | |
663 | $offset = 0; | |
664 | $batchSize = 5; | |
665 | while ($totalRows > 0) { | |
666 | if ($totalRows < $batchSize) { | |
667 | $batchSize = $totalRows; | |
668 | } | |
669 | $task = new CRM_Queue_Task( | |
670 | [get_class($this), 'runImport'], | |
671 | ['userJobID' => $this->getUserJobID(), 'limit' => $batchSize], | |
672 | ts('Processed %1 rows out of %2', [1 => $offset + $batchSize, 2 => $totalRowCount]) | |
673 | ); | |
674 | $queue->createItem($task); | |
675 | $totalRows -= $batchSize; | |
676 | $offset += $batchSize; | |
677 | } | |
678 | ||
679 | } | |
680 | ||
578e4db3 EM |
681 | /** |
682 | * Add imported contacts to groups. | |
683 | * | |
684 | * @param array $contactIDs | |
685 | * @param array $groups | |
686 | * | |
687 | * @return array | |
688 | */ | |
689 | private function addImportedContactsToNewGroup(array $contactIDs, array $groups): array { | |
690 | $groupAdditions = []; | |
691 | foreach ($groups as $groupID) { | |
692 | // @todo - this function has been in use historically but it does not seem | |
693 | // to add much efficiency of get + create api calls | |
694 | // and it doesn't give enough control over cache flushing for smaller batches. | |
695 | // Note that the import updates a lot of enities & checking & updating the group | |
696 | // shouldn't add much performance wise. However, cache flushing will | |
697 | $addCount = CRM_Contact_BAO_GroupContact::addContactsToGroup($contactIDs, $groupID); | |
698 | $groupAdditions[$groupID] = [ | |
699 | 'added' => (int) $addCount[1], | |
700 | 'notAdded' => (int) $addCount[2], | |
701 | ]; | |
702 | } | |
703 | return $groupAdditions; | |
704 | } | |
705 | ||
706 | /** | |
707 | * Tag imported contacts. | |
708 | * | |
709 | * @param array $contactIDs | |
710 | * @param array $tags | |
711 | * | |
712 | * @return array | |
713 | */ | |
714 | private function tagImportedContactsWithNewTag(array $contactIDs, array $tags) { | |
715 | $tagAdditions = []; | |
716 | foreach ($tags as $tagID) { | |
717 | // @todo - this function has been in use historically but it does not seem | |
718 | // to add much efficiency of get + create api calls | |
719 | // and it doesn't give enough control over cache flushing for smaller batches. | |
720 | // Note that the import updates a lot of enities & checking & updating the group | |
721 | // shouldn't add much performance wise. However, cache flushing will | |
722 | $outcome = CRM_Core_BAO_EntityTag::addEntitiesToTag($contactIDs, $tagID, 'civicrm_contact', FALSE); | |
723 | $tagAdditions[$tagID] = ['added' => $outcome[1], 'notAdded' => $outcome[2]]; | |
724 | } | |
725 | return $tagAdditions; | |
726 | } | |
727 | ||
69a4c20a | 728 | /** |
fe482240 | 729 | * Determines the file extension based on error code. |
69a4c20a | 730 | * |
f54e87d9 | 731 | * @var int $type error code constant |
69a4c20a | 732 | * @return string |
69a4c20a | 733 | */ |
00be9182 | 734 | public static function errorFileName($type) { |
69a4c20a CW |
735 | $fileName = NULL; |
736 | if (empty($type)) { | |
737 | return $fileName; | |
738 | } | |
739 | ||
740 | $config = CRM_Core_Config::singleton(); | |
741 | $fileName = $config->uploadDir . "sqlImport"; | |
742 | switch ($type) { | |
743 | case self::ERROR: | |
744 | $fileName .= '.errors'; | |
745 | break; | |
746 | ||
69a4c20a CW |
747 | case self::DUPLICATE: |
748 | $fileName .= '.duplicates'; | |
749 | break; | |
750 | ||
751 | case self::NO_MATCH: | |
752 | $fileName .= '.mismatch'; | |
753 | break; | |
754 | ||
755 | case self::UNPARSED_ADDRESS_WARNING: | |
756 | $fileName .= '.unparsedAddress'; | |
757 | break; | |
758 | } | |
759 | ||
760 | return $fileName; | |
761 | } | |
762 | ||
763 | /** | |
fe482240 | 764 | * Determines the file name based on error code. |
69a4c20a CW |
765 | * |
766 | * @var $type error code constant | |
767 | * @return string | |
69a4c20a | 768 | */ |
00be9182 | 769 | public static function saveFileName($type) { |
69a4c20a CW |
770 | $fileName = NULL; |
771 | if (empty($type)) { | |
772 | return $fileName; | |
773 | } | |
774 | switch ($type) { | |
775 | case self::ERROR: | |
776 | $fileName = 'Import_Errors.csv'; | |
777 | break; | |
778 | ||
69a4c20a CW |
779 | case self::DUPLICATE: |
780 | $fileName = 'Import_Duplicates.csv'; | |
781 | break; | |
782 | ||
783 | case self::NO_MATCH: | |
784 | $fileName = 'Import_Mismatch.csv'; | |
785 | break; | |
786 | ||
787 | case self::UNPARSED_ADDRESS_WARNING: | |
788 | $fileName = 'Import_Unparsed_Address.csv'; | |
789 | break; | |
790 | } | |
791 | ||
792 | return $fileName; | |
793 | } | |
794 | ||
56316747 | 795 | /** |
796 | * Check if contact is a duplicate . | |
797 | * | |
798 | * @param array $formatValues | |
799 | * | |
800 | * @return array | |
801 | */ | |
802 | protected function checkContactDuplicate(&$formatValues) { | |
803 | //retrieve contact id using contact dedupe rule | |
01c21f7e | 804 | $formatValues['contact_type'] = $formatValues['contact_type'] ?? $this->getContactType(); |
56316747 | 805 | $formatValues['version'] = 3; |
806 | require_once 'CRM/Utils/DeprecatedUtils.php'; | |
bd7c6219 | 807 | $params = $formatValues; |
808 | static $cIndieFields = NULL; | |
809 | static $defaultLocationId = NULL; | |
810 | ||
811 | $contactType = $params['contact_type']; | |
812 | if ($cIndieFields == NULL) { | |
813 | $cTempIndieFields = CRM_Contact_BAO_Contact::importableFields($contactType); | |
814 | $cIndieFields = $cTempIndieFields; | |
815 | ||
816 | $defaultLocation = CRM_Core_BAO_LocationType::getDefault(); | |
817 | ||
818 | // set the value to default location id else set to 1 | |
819 | if (!$defaultLocationId = (int) $defaultLocation->id) { | |
820 | $defaultLocationId = 1; | |
821 | } | |
822 | } | |
823 | ||
824 | $locationFields = CRM_Contact_BAO_Query::$_locationSpecificFields; | |
825 | ||
826 | $contactFormatted = []; | |
827 | foreach ($params as $key => $field) { | |
828 | if ($field == NULL || $field === '') { | |
829 | continue; | |
830 | } | |
831 | // CRM-17040, Considering only primary contact when importing contributions. So contribution inserts into primary contact | |
832 | // instead of soft credit contact. | |
01c21f7e | 833 | if (is_array($field) && $key !== "soft_credit") { |
bd7c6219 | 834 | foreach ($field as $value) { |
835 | $break = FALSE; | |
836 | if (is_array($value)) { | |
837 | foreach ($value as $name => $testForEmpty) { | |
838 | if ($name !== 'phone_type' && | |
839 | ($testForEmpty === '' || $testForEmpty == NULL) | |
840 | ) { | |
841 | $break = TRUE; | |
842 | break; | |
843 | } | |
844 | } | |
845 | } | |
846 | else { | |
847 | $break = TRUE; | |
848 | } | |
849 | if (!$break) { | |
f8909307 | 850 | $this->_civicrm_api3_deprecated_add_formatted_param($value, $contactFormatted); |
bd7c6219 | 851 | } |
852 | } | |
853 | continue; | |
854 | } | |
855 | ||
856 | $value = [$key => $field]; | |
857 | ||
858 | // check if location related field, then we need to add primary location type | |
859 | if (in_array($key, $locationFields)) { | |
860 | $value['location_type_id'] = $defaultLocationId; | |
861 | } | |
862 | elseif (array_key_exists($key, $cIndieFields)) { | |
863 | $value['contact_type'] = $contactType; | |
864 | } | |
865 | ||
f8909307 | 866 | $this->_civicrm_api3_deprecated_add_formatted_param($value, $contactFormatted); |
bd7c6219 | 867 | } |
868 | ||
869 | $contactFormatted['contact_type'] = $contactType; | |
870 | ||
871 | return _civicrm_api3_deprecated_duplicate_formatted_contact($contactFormatted); | |
56316747 | 872 | } |
873 | ||
f8909307 EM |
874 | /** |
875 | * This function adds the contact variable in $values to the | |
876 | * parameter list $params. For most cases, $values should have length 1. If | |
877 | * the variable being added is a child of Location, a location_type_id must | |
878 | * also be included. If it is a child of phone, a phone_type must be included. | |
879 | * | |
880 | * @param array $values | |
881 | * The variable(s) to be added. | |
882 | * @param array $params | |
883 | * The structured parameter list. | |
884 | * | |
885 | * @return bool|CRM_Utils_Error | |
886 | */ | |
887 | private function _civicrm_api3_deprecated_add_formatted_param(&$values, &$params) { | |
888 | // @todo - like most functions in import ... most of this is cruft.... | |
889 | // Crawl through the possible classes: | |
890 | // Contact | |
891 | // Individual | |
892 | // Household | |
893 | // Organization | |
894 | // Location | |
895 | // Address | |
896 | ||
897 | // Phone | |
898 | // IM | |
899 | // Note | |
900 | // Custom | |
901 | ||
902 | // Cache the various object fields | |
903 | static $fields = NULL; | |
904 | ||
905 | if ($fields == NULL) { | |
906 | $fields = []; | |
907 | } | |
908 | ||
909 | // first add core contact values since for other Civi modules they are not added | |
910 | require_once 'CRM/Contact/BAO/Contact.php'; | |
911 | $contactFields = CRM_Contact_DAO_Contact::fields(); | |
912 | _civicrm_api3_store_values($contactFields, $values, $params); | |
913 | ||
914 | if (isset($values['contact_type'])) { | |
915 | // we're an individual/household/org property | |
916 | ||
917 | $fields[$values['contact_type']] = CRM_Contact_DAO_Contact::fields(); | |
918 | ||
919 | _civicrm_api3_store_values($fields[$values['contact_type']], $values, $params); | |
920 | return TRUE; | |
921 | } | |
922 | ||
923 | if (isset($values['individual_prefix'])) { | |
924 | if (!empty($params['prefix_id'])) { | |
925 | $prefixes = CRM_Core_PseudoConstant::get('CRM_Contact_DAO_Contact', 'prefix_id'); | |
926 | $params['prefix'] = $prefixes[$params['prefix_id']]; | |
927 | } | |
928 | else { | |
929 | $params['prefix'] = $values['individual_prefix']; | |
930 | } | |
931 | return TRUE; | |
932 | } | |
933 | ||
934 | if (isset($values['individual_suffix'])) { | |
935 | if (!empty($params['suffix_id'])) { | |
936 | $suffixes = CRM_Core_PseudoConstant::get('CRM_Contact_DAO_Contact', 'suffix_id'); | |
937 | $params['suffix'] = $suffixes[$params['suffix_id']]; | |
938 | } | |
939 | else { | |
940 | $params['suffix'] = $values['individual_suffix']; | |
941 | } | |
942 | return TRUE; | |
943 | } | |
944 | ||
f8909307 EM |
945 | if (isset($values['gender'])) { |
946 | if (!empty($params['gender_id'])) { | |
947 | $genders = CRM_Core_PseudoConstant::get('CRM_Contact_DAO_Contact', 'gender_id'); | |
948 | $params['gender'] = $genders[$params['gender_id']]; | |
949 | } | |
950 | else { | |
951 | $params['gender'] = $values['gender']; | |
952 | } | |
953 | return TRUE; | |
954 | } | |
955 | ||
f8909307 EM |
956 | // format the website params. |
957 | if (!empty($values['url'])) { | |
958 | static $websiteFields; | |
959 | if (!is_array($websiteFields)) { | |
960 | require_once 'CRM/Core/DAO/Website.php'; | |
961 | $websiteFields = CRM_Core_DAO_Website::fields(); | |
962 | } | |
963 | if (!array_key_exists('website', $params) || | |
964 | !is_array($params['website']) | |
965 | ) { | |
966 | $params['website'] = []; | |
967 | } | |
968 | ||
969 | $websiteCount = count($params['website']); | |
970 | _civicrm_api3_store_values($websiteFields, $values, | |
971 | $params['website'][++$websiteCount] | |
972 | ); | |
973 | ||
974 | return TRUE; | |
975 | } | |
976 | ||
977 | // get the formatted location blocks into params - w/ 3.0 format, CRM-4605 | |
978 | if (!empty($values['location_type_id'])) { | |
979 | static $fields = NULL; | |
980 | if ($fields == NULL) { | |
981 | $fields = []; | |
982 | } | |
983 | ||
984 | foreach (['Phone', 'Email', 'IM', 'OpenID', 'Phone_Ext'] as $block) { | |
985 | $name = strtolower($block); | |
986 | if (!array_key_exists($name, $values)) { | |
987 | continue; | |
988 | } | |
989 | ||
990 | if ($name === 'phone_ext') { | |
991 | $block = 'Phone'; | |
992 | } | |
993 | ||
994 | // block present in value array. | |
995 | if (!array_key_exists($name, $params) || !is_array($params[$name])) { | |
996 | $params[$name] = []; | |
997 | } | |
998 | ||
999 | if (!array_key_exists($block, $fields)) { | |
1000 | $className = "CRM_Core_DAO_$block"; | |
1001 | $fields[$block] =& $className::fields(); | |
1002 | } | |
1003 | ||
1004 | $blockCnt = count($params[$name]); | |
1005 | ||
1006 | // copy value to dao field name. | |
1007 | if ($name == 'im') { | |
1008 | $values['name'] = $values[$name]; | |
1009 | } | |
1010 | ||
1011 | _civicrm_api3_store_values($fields[$block], $values, | |
1012 | $params[$name][++$blockCnt] | |
1013 | ); | |
1014 | ||
1015 | if (empty($params['id']) && ($blockCnt == 1)) { | |
1016 | $params[$name][$blockCnt]['is_primary'] = TRUE; | |
1017 | } | |
1018 | ||
1019 | // we only process single block at a time. | |
1020 | return TRUE; | |
1021 | } | |
1022 | ||
1023 | // handle address fields. | |
1024 | if (!array_key_exists('address', $params) || !is_array($params['address'])) { | |
1025 | $params['address'] = []; | |
1026 | } | |
1027 | ||
1028 | $addressCnt = 1; | |
1029 | foreach ($params['address'] as $cnt => $addressBlock) { | |
1030 | if (CRM_Utils_Array::value('location_type_id', $values) == | |
1031 | CRM_Utils_Array::value('location_type_id', $addressBlock) | |
1032 | ) { | |
1033 | $addressCnt = $cnt; | |
1034 | break; | |
1035 | } | |
1036 | $addressCnt++; | |
1037 | } | |
1038 | ||
1039 | if (!array_key_exists('Address', $fields)) { | |
1040 | $fields['Address'] = CRM_Core_DAO_Address::fields(); | |
1041 | } | |
1042 | ||
1043 | // Note: we doing multiple value formatting here for address custom fields, plus putting into right format. | |
1044 | // The actual formatting (like date, country ..etc) for address custom fields is taken care of while saving | |
1045 | // the address in CRM_Core_BAO_Address::create method | |
1046 | if (!empty($values['location_type_id'])) { | |
1047 | static $customFields = []; | |
1048 | if (empty($customFields)) { | |
1049 | $customFields = CRM_Core_BAO_CustomField::getFields('Address'); | |
1050 | } | |
1051 | // make a copy of values, as we going to make changes | |
1052 | $newValues = $values; | |
1053 | foreach ($values as $key => $val) { | |
1054 | $customFieldID = CRM_Core_BAO_CustomField::getKeyID($key); | |
1055 | if ($customFieldID && array_key_exists($customFieldID, $customFields)) { | |
1056 | // mark an entry in fields array since we want the value of custom field to be copied | |
1057 | $fields['Address'][$key] = NULL; | |
1058 | ||
1059 | $htmlType = $customFields[$customFieldID]['html_type'] ?? NULL; | |
1060 | if (CRM_Core_BAO_CustomField::isSerialized($customFields[$customFieldID]) && $val) { | |
1061 | $mulValues = explode(',', $val); | |
1062 | $customOption = CRM_Core_BAO_CustomOption::getCustomOption($customFieldID, TRUE); | |
1063 | $newValues[$key] = []; | |
1064 | foreach ($mulValues as $v1) { | |
1065 | foreach ($customOption as $v2) { | |
1066 | if ((strtolower($v2['label']) == strtolower(trim($v1))) || | |
1067 | (strtolower($v2['value']) == strtolower(trim($v1))) | |
1068 | ) { | |
1069 | if ($htmlType == 'CheckBox') { | |
1070 | $newValues[$key][$v2['value']] = 1; | |
1071 | } | |
1072 | else { | |
1073 | $newValues[$key][] = $v2['value']; | |
1074 | } | |
1075 | } | |
1076 | } | |
1077 | } | |
1078 | } | |
1079 | } | |
1080 | } | |
1081 | // consider new values | |
1082 | $values = $newValues; | |
1083 | } | |
1084 | ||
1085 | _civicrm_api3_store_values($fields['Address'], $values, $params['address'][$addressCnt]); | |
1086 | ||
1087 | $addressFields = [ | |
1088 | 'county', | |
1089 | 'country', | |
1090 | 'state_province', | |
1091 | 'supplemental_address_1', | |
1092 | 'supplemental_address_2', | |
1093 | 'supplemental_address_3', | |
1094 | 'StateProvince.name', | |
1095 | ]; | |
1096 | ||
1097 | foreach ($addressFields as $field) { | |
1098 | if (array_key_exists($field, $values)) { | |
1099 | if (!array_key_exists('address', $params)) { | |
1100 | $params['address'] = []; | |
1101 | } | |
1102 | $params['address'][$addressCnt][$field] = $values[$field]; | |
1103 | } | |
1104 | } | |
1105 | ||
1106 | if ($addressCnt == 1) { | |
1107 | ||
1108 | $params['address'][$addressCnt]['is_primary'] = TRUE; | |
1109 | } | |
1110 | return TRUE; | |
1111 | } | |
1112 | ||
1113 | if (isset($values['note'])) { | |
1114 | // add a note field | |
1115 | if (!isset($params['note'])) { | |
1116 | $params['note'] = []; | |
1117 | } | |
1118 | $noteBlock = count($params['note']) + 1; | |
1119 | ||
1120 | $params['note'][$noteBlock] = []; | |
1121 | if (!isset($fields['Note'])) { | |
1122 | $fields['Note'] = CRM_Core_DAO_Note::fields(); | |
1123 | } | |
1124 | ||
1125 | // get the current logged in civicrm user | |
1126 | $session = CRM_Core_Session::singleton(); | |
1127 | $userID = $session->get('userID'); | |
1128 | ||
1129 | if ($userID) { | |
1130 | $values['contact_id'] = $userID; | |
1131 | } | |
1132 | ||
1133 | _civicrm_api3_store_values($fields['Note'], $values, $params['note'][$noteBlock]); | |
1134 | ||
1135 | return TRUE; | |
1136 | } | |
1137 | ||
1138 | // Check for custom field values | |
1139 | ||
1140 | if (empty($fields['custom'])) { | |
1141 | $fields['custom'] = &CRM_Core_BAO_CustomField::getFields(CRM_Utils_Array::value('contact_type', $values), | |
1142 | FALSE, FALSE, NULL, NULL, FALSE, FALSE, FALSE | |
1143 | ); | |
1144 | } | |
1145 | ||
1146 | foreach ($values as $key => $value) { | |
1147 | if ($customFieldID = CRM_Core_BAO_CustomField::getKeyID($key)) { | |
1148 | // check if it's a valid custom field id | |
1149 | ||
1150 | if (!array_key_exists($customFieldID, $fields['custom'])) { | |
1151 | return civicrm_api3_create_error('Invalid custom field ID'); | |
1152 | } | |
1153 | else { | |
1154 | $params[$key] = $value; | |
1155 | } | |
1156 | } | |
1157 | } | |
1158 | } | |
1159 | ||
14b9e069 | 1160 | /** |
1161 | * Parse a field which could be represented by a label or name value rather than the DB value. | |
1162 | * | |
9ae10cd7 | 1163 | * We will try to match name first or (per https://lab.civicrm.org/dev/core/issues/1285 if we have an id. |
1164 | * | |
1165 | * but if not available then see if we have a label that can be converted to a name. | |
14b9e069 | 1166 | * |
1167 | * @param string|int|null $submittedValue | |
1168 | * @param array $fieldSpec | |
1169 | * Metadata for the field | |
1170 | * | |
1171 | * @return mixed | |
1172 | */ | |
1173 | protected function parsePseudoConstantField($submittedValue, $fieldSpec) { | |
0b742997 SL |
1174 | // dev/core#1289 Somehow we have wound up here but the BAO has not been specified in the fieldspec so we need to check this but future us problem, for now lets just return the submittedValue |
1175 | if (!isset($fieldSpec['bao'])) { | |
1176 | return $submittedValue; | |
1177 | } | |
14b9e069 | 1178 | /* @var \CRM_Core_DAO $bao */ |
1179 | $bao = $fieldSpec['bao']; | |
1180 | // For historical reasons use validate as context - ie disabled name matches ARE permitted. | |
1181 | $nameOptions = $bao::buildOptions($fieldSpec['name'], 'validate'); | |
9ae10cd7 | 1182 | if (isset($nameOptions[$submittedValue])) { |
1183 | return $submittedValue; | |
1184 | } | |
1185 | if (in_array($submittedValue, $nameOptions)) { | |
1186 | return array_search($submittedValue, $nameOptions, TRUE); | |
1187 | } | |
1188 | ||
1189 | $labelOptions = array_flip($bao::buildOptions($fieldSpec['name'], 'match')); | |
1190 | if (isset($labelOptions[$submittedValue])) { | |
1191 | return array_search($labelOptions[$submittedValue], $nameOptions, TRUE); | |
14b9e069 | 1192 | } |
1193 | return ''; | |
1194 | } | |
1195 | ||
be40742b CW |
1196 | /** |
1197 | * This is code extracted from 4 places where this exact snippet was being duplicated. | |
1198 | * | |
1199 | * FIXME: Extracting this was a first step, but there's also | |
1200 | * 1. Inconsistency in the way other select options are handled. | |
1201 | * Contribution adds handling for Select/Radio/Autocomplete | |
1202 | * Participant/Activity only handles Select/Radio and misses Autocomplete | |
1203 | * Membership is missing all of it | |
1204 | * 2. Inconsistency with the way this works vs. how it's implemented in Contact import. | |
1205 | * | |
1206 | * @param $customFieldID | |
1207 | * @param $value | |
1208 | * @param $fieldType | |
1209 | * @return array | |
1210 | */ | |
1211 | public static function unserializeCustomValue($customFieldID, $value, $fieldType) { | |
1212 | $mulValues = explode(',', $value); | |
1213 | $customOption = CRM_Core_BAO_CustomOption::getCustomOption($customFieldID, TRUE); | |
1214 | $values = []; | |
1215 | foreach ($mulValues as $v1) { | |
1216 | foreach ($customOption as $customValueID => $customLabel) { | |
1217 | $customValue = $customLabel['value']; | |
1218 | if ((strtolower(trim($customLabel['label'])) == strtolower(trim($v1))) || | |
1219 | (strtolower(trim($customValue)) == strtolower(trim($v1))) | |
1220 | ) { | |
f6fc1b15 | 1221 | $values[] = $customValue; |
be40742b CW |
1222 | } |
1223 | } | |
1224 | } | |
1225 | return $values; | |
1226 | } | |
1227 | ||
7e56b830 EM |
1228 | /** |
1229 | * Validate that the field requirements are met in the params. | |
1230 | * | |
1231 | * @param array $requiredFields | |
1232 | * @param array $params | |
1233 | * An array of required fields (fieldName => label) | |
1234 | * - note this follows the and / or array nesting we see in permission checks | |
1235 | * eg. | |
1236 | * [ | |
1237 | * 'email' => ts('Email'), | |
1238 | * ['first_name' => ts('First Name'), 'last_name' => ts('Last Name')] | |
1239 | * ] | |
1240 | * Means 'email' OR 'first_name AND 'last_name'. | |
7d2012dc | 1241 | * @param string $prefixString |
7e56b830 | 1242 | * |
7d2012dc | 1243 | * @throws \CRM_Core_Exception Exception thrown if field requirements are not met. |
7e56b830 | 1244 | */ |
7d2012dc | 1245 | protected function validateRequiredFields(array $requiredFields, array $params, $prefixString): void { |
7e56b830 EM |
1246 | $missingFields = []; |
1247 | foreach ($requiredFields as $key => $required) { | |
1248 | if (!is_array($required)) { | |
1249 | $importParameter = $params[$key] ?? []; | |
1250 | if (!is_array($importParameter)) { | |
1251 | if (!empty($importParameter)) { | |
1252 | return; | |
1253 | } | |
1254 | } | |
1255 | else { | |
1256 | foreach ($importParameter as $locationValues) { | |
1257 | if (!empty($locationValues[$key])) { | |
1258 | return; | |
1259 | } | |
1260 | } | |
1261 | } | |
1262 | ||
1263 | $missingFields[$key] = $required; | |
1264 | } | |
1265 | else { | |
1266 | foreach ($required as $field => $label) { | |
1267 | if (empty($params[$field])) { | |
1268 | $missing[$field] = $label; | |
1269 | } | |
1270 | } | |
1271 | if (empty($missing)) { | |
1272 | return; | |
1273 | } | |
1274 | $missingFields[$key] = implode(' ' . ts('and') . ' ', $missing); | |
1275 | } | |
1276 | } | |
24948d41 | 1277 | throw new CRM_Core_Exception($prefixString . ts('Missing required fields:') . ' ' . implode(' ' . ts('OR') . ' ', $missingFields)); |
7e56b830 EM |
1278 | } |
1279 | ||
19f33b09 EM |
1280 | /** |
1281 | * Get the field value, transformed by metadata. | |
1282 | * | |
1283 | * @param string $fieldName | |
1284 | * @param string|int $importedValue | |
1285 | * Value as it came in from the datasource. | |
1286 | * | |
1287 | * @return string|array|bool|int | |
1288 | * @throws \API_Exception | |
1289 | */ | |
1290 | protected function getTransformedFieldValue(string $fieldName, $importedValue) { | |
018c9e26 | 1291 | if (empty($importedValue)) { |
19f33b09 EM |
1292 | return $importedValue; |
1293 | } | |
b7d52f5e | 1294 | $fieldMetadata = $this->getFieldMetadata($fieldName); |
02374bae EM |
1295 | if (!empty($fieldMetadata['serialize']) && count(explode(',', $importedValue)) > 1) { |
1296 | $values = []; | |
1297 | foreach (explode(',', $importedValue) as $value) { | |
1298 | $values[] = $this->getTransformedFieldValue($fieldName, $value); | |
1299 | } | |
1300 | return $values; | |
1301 | } | |
639e4f37 EM |
1302 | if ($fieldName === 'url') { |
1303 | return CRM_Utils_Rule::url($importedValue) ? $importedValue : 'invalid_import_value'; | |
1304 | } | |
1305 | ||
1306 | if ($fieldName === 'email') { | |
1307 | return CRM_Utils_Rule::email($importedValue) ? $importedValue : 'invalid_import_value'; | |
1308 | } | |
1309 | ||
424df54f EM |
1310 | if ($fieldMetadata['type'] === CRM_Utils_Type::T_FLOAT) { |
1311 | return CRM_Utils_Rule::numeric($importedValue) ? $importedValue : 'invalid_import_value'; | |
1312 | } | |
4b58c5c4 EM |
1313 | if ($fieldMetadata['type'] === CRM_Utils_Type::T_MONEY) { |
1314 | return CRM_Utils_Rule::money($importedValue, TRUE) ? CRM_Utils_Rule::cleanMoney($importedValue) : 'invalid_import_value'; | |
1315 | } | |
b7d52f5e EM |
1316 | if ($fieldMetadata['type'] === CRM_Utils_Type::T_BOOLEAN) { |
1317 | $value = CRM_Utils_String::strtoboolstr($importedValue); | |
1318 | if ($value !== FALSE) { | |
1319 | return (bool) $value; | |
1320 | } | |
1321 | return 'invalid_import_value'; | |
1322 | } | |
4b58c5c4 | 1323 | if ($fieldMetadata['type'] === CRM_Utils_Type::T_DATE || $fieldMetadata['type'] === (CRM_Utils_Type::T_DATE + CRM_Utils_Type::T_TIME) || $fieldMetadata['type'] === CRM_Utils_Type::T_TIMESTAMP) { |
b7d52f5e EM |
1324 | $value = CRM_Utils_Date::formatDate($importedValue, $this->getSubmittedValue('dateFormats')); |
1325 | return ($value) ?: 'invalid_import_value'; | |
1326 | } | |
24948d41 EM |
1327 | $options = $this->getFieldOptions($fieldName); |
1328 | if ($options !== FALSE) { | |
018c9e26 EM |
1329 | if ($this->isAmbiguous($fieldName, $importedValue)) { |
1330 | // We can't transform it at this stage. Perhaps later we can with | |
1331 | // other information such as country. | |
1332 | return $importedValue; | |
1333 | } | |
1334 | ||
24948d41 EM |
1335 | $comparisonValue = is_numeric($importedValue) ? $importedValue : mb_strtolower($importedValue); |
1336 | return $options[$comparisonValue] ?? 'invalid_import_value'; | |
1337 | } | |
1338 | return $importedValue; | |
19f33b09 EM |
1339 | } |
1340 | ||
1341 | /** | |
1342 | * @param string $fieldName | |
1343 | * | |
1344 | * @return false|array | |
1345 | * | |
1346 | * @throws \API_Exception | |
1347 | */ | |
1348 | protected function getFieldOptions(string $fieldName) { | |
1349 | return $this->getFieldMetadata($fieldName, TRUE)['options']; | |
1350 | } | |
1351 | ||
1352 | /** | |
1353 | * Get the metadata for the field. | |
1354 | * | |
1355 | * @param string $fieldName | |
1356 | * @param bool $loadOptions | |
e0b8f9a9 EM |
1357 | * @param bool $limitToContactType |
1358 | * Only show fields for the type to import (not appropriate when looking up | |
1359 | * related contact fields). | |
1360 | * | |
19f33b09 EM |
1361 | * |
1362 | * @return array | |
1363 | * @throws \API_Exception | |
e0b8f9a9 | 1364 | * @throws \Civi\API\Exception\NotImplementedException |
19f33b09 | 1365 | */ |
e0b8f9a9 | 1366 | protected function getFieldMetadata(string $fieldName, bool $loadOptions = FALSE, $limitToContactType = FALSE): array { |
24948d41 | 1367 | |
80e9f1a2 | 1368 | $fieldMap = $this->getOddlyMappedMetadataFields(); |
24948d41 EM |
1369 | $fieldMapName = empty($fieldMap[$fieldName]) ? $fieldName : $fieldMap[$fieldName]; |
1370 | ||
80e9f1a2 EM |
1371 | // This whole business of only loading metadata for one type when we actually need it for all is ... dubious. |
1372 | if (empty($this->getImportableFieldsMetadata()[$fieldMapName])) { | |
1373 | if ($loadOptions || !$limitToContactType) { | |
1374 | $this->importableFieldsMetadata[$fieldMapName] = CRM_Contact_BAO_Contact::importableFields('All')[$fieldMapName]; | |
1375 | } | |
1376 | } | |
24948d41 | 1377 | |
80e9f1a2 EM |
1378 | $fieldMetadata = $this->getImportableFieldsMetadata()[$fieldMapName]; |
1379 | if ($loadOptions && !isset($fieldMetadata['options'])) { | |
018c9e26 EM |
1380 | if (($fieldMetadata['data_type'] ?? '') === 'StateProvince') { |
1381 | // Probably already loaded and also supports abbreviations - eg. NSW. | |
1382 | // Supporting for core AND custom state fields is more consistent. | |
1383 | $this->importableFieldsMetadata[$fieldMapName]['options'] = $this->getFieldOptions('state_province_id'); | |
1384 | return $this->importableFieldsMetadata[$fieldMapName]; | |
1385 | } | |
1386 | if (($fieldMetadata['data_type'] ?? '') === 'Country') { | |
1387 | // Probably already loaded and also supports abbreviations - eg. NSW. | |
1388 | // Supporting for core AND custom state fields is more consistent. | |
1389 | $this->importableFieldsMetadata[$fieldMapName]['options'] = $this->getFieldOptions('country_id'); | |
1390 | return $this->importableFieldsMetadata[$fieldMapName]; | |
1391 | } | |
80e9f1a2 | 1392 | $optionFieldName = empty($fieldMap[$fieldName]) ? $fieldMetadata['name'] : $fieldName; |
018c9e26 | 1393 | |
80e9f1a2 | 1394 | if (!empty($fieldMetadata['custom_group_id'])) { |
018c9e26 EM |
1395 | $customField = CustomField::get(FALSE) |
1396 | ->addWhere('id', '=', $fieldMetadata['custom_field_id']) | |
1397 | ->addSelect('name', 'custom_group_id.name') | |
1398 | ->execute() | |
1399 | ->first(); | |
80e9f1a2 EM |
1400 | $optionFieldName = $customField['custom_group_id.name'] . '.' . $customField['name']; |
1401 | } | |
24948d41 | 1402 | $options = civicrm_api4($this->getFieldEntity($fieldName), 'getFields', [ |
80e9f1a2 EM |
1403 | 'loadOptions' => ['id', 'name', 'label', 'abbr'], |
1404 | 'where' => [['name', '=', $optionFieldName]], | |
24948d41 EM |
1405 | 'select' => ['options'], |
1406 | ])->first()['options']; | |
639e4f37 EM |
1407 | if (is_array($options)) { |
1408 | // We create an array of the possible variants - notably including | |
1409 | // name AND label as either might be used. We also lower case before checking | |
1410 | $values = []; | |
1411 | foreach ($options as $option) { | |
80e9f1a2 EM |
1412 | $idKey = is_numeric($option['id']) ? $option['id'] : mb_strtolower($option['id']); |
1413 | $values[$idKey] = $option['id']; | |
1414 | foreach (['name', 'label', 'abbr'] as $key) { | |
1415 | $optionValue = mb_strtolower($option[$key] ?? ''); | |
1416 | if ($optionValue !== '') { | |
018c9e26 EM |
1417 | if (isset($values[$optionValue]) && $values[$optionValue] !== $option['id']) { |
1418 | if (!isset($this->ambiguousOptions[$fieldName][$optionValue])) { | |
1419 | $this->ambiguousOptions[$fieldName][$optionValue] = [$values[$optionValue]]; | |
1420 | } | |
1421 | $this->ambiguousOptions[$fieldName][$optionValue][] = $option['id']; | |
1422 | } | |
1423 | else { | |
1424 | $values[$optionValue] = $option['id']; | |
1425 | } | |
80e9f1a2 EM |
1426 | } |
1427 | } | |
639e4f37 EM |
1428 | } |
1429 | $this->importableFieldsMetadata[$fieldMapName]['options'] = $values; | |
1430 | } | |
1431 | else { | |
1432 | $this->importableFieldsMetadata[$fieldMapName]['options'] = $options; | |
19f33b09 | 1433 | } |
24948d41 | 1434 | return $this->importableFieldsMetadata[$fieldMapName]; |
19f33b09 EM |
1435 | } |
1436 | return $fieldMetadata; | |
1437 | } | |
1438 | ||
b1994c0b EM |
1439 | /** |
1440 | * @param $customFieldID | |
1441 | * @param $value | |
1442 | * @param array $fieldMetaData | |
1443 | * @param $dateType | |
1444 | * | |
1445 | * @return ?string | |
1446 | */ | |
1447 | protected function validateCustomField($customFieldID, $value, array $fieldMetaData, $dateType): ?string { | |
b1994c0b EM |
1448 | /* validate the data against the CF type */ |
1449 | ||
1450 | if ($value) { | |
1451 | $dataType = $fieldMetaData['data_type']; | |
1452 | $htmlType = $fieldMetaData['html_type']; | |
1453 | $isSerialized = CRM_Core_BAO_CustomField::isSerialized($fieldMetaData); | |
1454 | if ($dataType === 'Date') { | |
1455 | $params = ['date_field' => $value]; | |
1456 | if (CRM_Utils_Date::convertToDefaultDate($params, $dateType, 'date_field')) { | |
1457 | return NULL; | |
1458 | } | |
1459 | return $fieldMetaData['label']; | |
1460 | } | |
0b57e93c | 1461 | elseif ($dataType === 'Boolean') { |
b1994c0b EM |
1462 | if (CRM_Utils_String::strtoboolstr($value) === FALSE) { |
1463 | return $fieldMetaData['label'] . '::' . $fieldMetaData['groupTitle']; | |
1464 | } | |
1465 | } | |
1466 | // need not check for label filed import | |
1467 | $selectHtmlTypes = [ | |
1468 | 'CheckBox', | |
1469 | 'Select', | |
1470 | 'Radio', | |
1471 | ]; | |
1472 | if ((!$isSerialized && !in_array($htmlType, $selectHtmlTypes)) || $dataType == 'Boolean' || $dataType == 'ContactReference') { | |
1473 | $valid = CRM_Core_BAO_CustomValue::typecheck($dataType, $value); | |
1474 | if (!$valid) { | |
1475 | return $fieldMetaData['label']; | |
1476 | } | |
1477 | } | |
1478 | ||
1479 | // check for values for custom fields for checkboxes and multiselect | |
1480 | if ($isSerialized && $dataType != 'ContactReference') { | |
ccf5ff23 | 1481 | $mulValues = array_filter(explode(',', str_replace('|', ',', trim($value))), 'strlen'); |
b1994c0b EM |
1482 | $customOption = CRM_Core_BAO_CustomOption::getCustomOption($customFieldID, TRUE); |
1483 | foreach ($mulValues as $v1) { | |
b1994c0b EM |
1484 | |
1485 | $flag = FALSE; | |
1486 | foreach ($customOption as $v2) { | |
1487 | if ((strtolower(trim($v2['label'])) == strtolower(trim($v1))) || (strtolower(trim($v2['value'])) == strtolower(trim($v1)))) { | |
1488 | $flag = TRUE; | |
1489 | } | |
1490 | } | |
1491 | ||
1492 | if (!$flag) { | |
1493 | return $fieldMetaData['label']; | |
1494 | } | |
1495 | } | |
1496 | } | |
1497 | elseif ($htmlType == 'Select' || ($htmlType == 'Radio' && $dataType != 'Boolean')) { | |
1498 | $customOption = CRM_Core_BAO_CustomOption::getCustomOption($customFieldID, TRUE); | |
1499 | $flag = FALSE; | |
1500 | foreach ($customOption as $v2) { | |
1501 | if ((strtolower(trim($v2['label'])) == strtolower(trim($value))) || (strtolower(trim($v2['value'])) == strtolower(trim($value)))) { | |
1502 | $flag = TRUE; | |
1503 | } | |
1504 | } | |
1505 | if (!$flag) { | |
1506 | return $fieldMetaData['label']; | |
1507 | } | |
1508 | } | |
1509 | } | |
1510 | ||
1511 | return NULL; | |
1512 | } | |
1513 | ||
e0ce85b6 EM |
1514 | /** |
1515 | * Get the entity for the given field. | |
1516 | * | |
1517 | * @param string $fieldName | |
1518 | * | |
1519 | * @return mixed|null | |
1520 | * @throws \API_Exception | |
1521 | */ | |
1522 | protected function getFieldEntity(string $fieldName) { | |
1523 | if ($fieldName === 'do_not_import') { | |
1524 | return NULL; | |
1525 | } | |
80e9f1a2 EM |
1526 | if (in_array($fieldName, ['email_greeting_id', 'postal_greeting_id', 'addressee_id'], TRUE)) { |
1527 | return 'Contact'; | |
1528 | } | |
e0ce85b6 EM |
1529 | $metadata = $this->getFieldMetadata($fieldName); |
1530 | if (!isset($metadata['entity'])) { | |
1531 | return in_array($metadata['extends'], ['Individual', 'Organization', 'Household'], TRUE) ? 'Contact' : $metadata['extends']; | |
1532 | } | |
1533 | ||
1534 | // Our metadata for these is fugly. Handling the fugliness during retrieval. | |
1535 | if (in_array($metadata['entity'], ['Country', 'StateProvince', 'County'], TRUE)) { | |
1536 | return 'Address'; | |
1537 | } | |
1538 | return $metadata['entity']; | |
1539 | } | |
1540 | ||
9eec6f2b EM |
1541 | /** |
1542 | * Validate the import file, updating the import table with results. | |
1543 | * | |
1544 | * @throws \API_Exception | |
1545 | * @throws \CRM_Core_Exception | |
1546 | */ | |
1547 | public function validate(): void { | |
1548 | $dataSource = $this->getDataSourceObject(); | |
1549 | while ($row = $dataSource->getRow()) { | |
1550 | try { | |
1551 | $rowNumber = $row['_id']; | |
1552 | $values = array_values($row); | |
1553 | $this->validateValues($values); | |
1554 | $this->setImportStatus($rowNumber, 'NEW', ''); | |
1555 | } | |
1556 | catch (CRM_Core_Exception $e) { | |
1557 | $this->setImportStatus($rowNumber, 'ERROR', $e->getMessage()); | |
1558 | } | |
1559 | } | |
1560 | } | |
1561 | ||
e0ce85b6 EM |
1562 | /** |
1563 | * Search the value for the string 'invalid_import_value'. | |
1564 | * | |
1565 | * If the string is found it indicates the fields was rejected | |
1566 | * during `getTransformedValue` as not having valid data. | |
1567 | * | |
1568 | * @param string|array|int $value | |
1569 | * @param string $key | |
1570 | * @param string $prefixString | |
1571 | * | |
1572 | * @return array | |
1573 | * @throws \API_Exception | |
1574 | */ | |
1c82489b | 1575 | protected function getInvalidValues($value, string $key = '', string $prefixString = ''): array { |
e0ce85b6 EM |
1576 | $errors = []; |
1577 | if ($value === 'invalid_import_value') { | |
992a3d9e EM |
1578 | $metadata = $this->getFieldMetadata($key); |
1579 | $errors[] = $prefixString . ($metadata['html']['label'] ?? $metadata['title']); | |
e0ce85b6 EM |
1580 | } |
1581 | elseif (is_array($value)) { | |
1582 | foreach ($value as $innerKey => $innerValue) { | |
1583 | $result = $this->getInvalidValues($innerValue, $innerKey, $prefixString); | |
1584 | if (!empty($result)) { | |
1585 | $errors = array_merge($result, $errors); | |
1586 | } | |
1587 | } | |
1588 | } | |
1589 | return array_filter($errors); | |
1590 | } | |
1591 | ||
1592 | /** | |
1593 | * Get the available countries. | |
1594 | * | |
1595 | * If the site is not configured with a restriction then all countries are valid | |
1596 | * but otherwise only a select array are. | |
1597 | * | |
1598 | * @return array|false | |
1599 | * FALSE indicates no restrictions. | |
1600 | */ | |
1601 | protected function getAvailableCountries() { | |
1602 | if ($this->availableCountries === NULL) { | |
1603 | $availableCountries = Civi::settings()->get('countryLimit'); | |
1604 | $this->availableCountries = !empty($availableCountries) ? array_fill_keys($availableCountries, TRUE) : FALSE; | |
1605 | } | |
1606 | return $this->availableCountries; | |
1607 | } | |
1608 | ||
80e9f1a2 EM |
1609 | /** |
1610 | * Get the metadata field for which importable fields does not key the actual field name. | |
1611 | * | |
1612 | * @return string[] | |
1613 | */ | |
1614 | protected function getOddlyMappedMetadataFields(): array { | |
1615 | return [ | |
1616 | 'country_id' => 'country', | |
1617 | 'state_province_id' => 'state_province', | |
1618 | 'county_id' => 'county', | |
1619 | 'email_greeting_id' => 'email_greeting', | |
1620 | 'postal_greeting_id' => 'postal_greeting', | |
1621 | 'addressee_id' => 'addressee', | |
1622 | ]; | |
1623 | } | |
1624 | ||
018c9e26 EM |
1625 | /** |
1626 | * Get the default country for the site. | |
1627 | * | |
1628 | * @return int | |
1629 | */ | |
1630 | protected function getSiteDefaultCountry(): int { | |
1631 | if (!isset($this->siteDefaultCountry)) { | |
1632 | $this->siteDefaultCountry = (int) Civi::settings()->get('defaultContactCountry'); | |
1633 | } | |
1634 | return $this->siteDefaultCountry; | |
1635 | } | |
1636 | ||
1637 | /** | |
1638 | * Is the option ambiguous. | |
1639 | * | |
1640 | * @param string $fieldName | |
1641 | * @param string $importedValue | |
1642 | */ | |
1643 | protected function isAmbiguous(string $fieldName, $importedValue): bool { | |
1644 | return !empty($this->ambiguousOptions[$fieldName][mb_strtolower($importedValue)]); | |
1645 | } | |
1646 | ||
4a07230c EM |
1647 | /** |
1648 | * Get the civicrm_mapping_field appropriate layout for the mapper input. | |
1649 | * | |
1650 | * For simple parsers (not contribution or contact) the input looks like | |
1651 | * ['first_name', 'custom_32'] | |
1652 | * and it is converted to | |
1653 | * | |
1654 | * ['name' => 'first_name', 'mapping_id' => 1, 'column_number' => 5], | |
1655 | * | |
1656 | * @param array $fieldMapping | |
1657 | * @param int $mappingID | |
1658 | * @param int $columnNumber | |
1659 | * | |
1660 | * @return array | |
1661 | */ | |
1662 | public function getMappingFieldFromMapperInput(array $fieldMapping, int $mappingID, int $columnNumber): array { | |
1663 | return [ | |
1664 | 'name' => $fieldMapping[0], | |
1665 | 'mapping_id' => $mappingID, | |
1666 | 'column_number' => $columnNumber, | |
1667 | ]; | |
1668 | } | |
1669 | ||
992a3d9e EM |
1670 | /** |
1671 | * @param array $mappedField | |
1672 | * Field detail as would be saved in field_mapping table | |
1673 | * or as returned from getMappingFieldFromMapperInput | |
1674 | * | |
1675 | * @return string | |
1676 | * @throws \API_Exception | |
1677 | */ | |
1678 | public function getMappedFieldLabel(array $mappedField): string { | |
1679 | $this->setFieldMetadata(); | |
1680 | return $this->getFieldMetadata($mappedField['name'])['title']; | |
1681 | } | |
1682 | ||
1683 | /** | |
1684 | * Get the row from the csv mapped to our parameters. | |
1685 | * | |
1686 | * @param array $values | |
1687 | * | |
1688 | * @return array | |
1689 | * @throws \API_Exception | |
1690 | */ | |
1691 | public function getMappedRow(array $values): array { | |
1692 | $params = []; | |
1693 | foreach ($this->getFieldMappings() as $i => $mappedField) { | |
1694 | if ($mappedField['name'] === 'do_not_import') { | |
1695 | continue; | |
1696 | } | |
1697 | if ($mappedField['name']) { | |
1698 | $params[$this->getFieldMetadata($mappedField['name'])['name']] = $this->getTransformedFieldValue($mappedField['name'], $values[$i]); | |
1699 | } | |
1700 | } | |
1701 | return $params; | |
1702 | } | |
1703 | ||
288db2d2 EM |
1704 | /** |
1705 | * Get the field mappings for the import. | |
1706 | * | |
1707 | * This is the same format as saved in civicrm_mapping_field except | |
1708 | * that location_type_id = 'Primary' rather than empty where relevant. | |
1709 | * Also 'im_provider_id' is mapped to the 'real' field name 'provider_id' | |
1710 | * | |
1711 | * @return array | |
1712 | * @throws \API_Exception | |
1713 | */ | |
1714 | protected function getFieldMappings(): array { | |
1715 | $mappedFields = []; | |
992a3d9e EM |
1716 | $mapper = $this->getSubmittedValue('mapper'); |
1717 | foreach ($mapper as $i => $mapperRow) { | |
288db2d2 EM |
1718 | $mappedField = $this->getMappingFieldFromMapperInput($mapperRow, 0, $i); |
1719 | // Just for clarity since 0 is a pseudo-value | |
1720 | unset($mappedField['mapping_id']); | |
1721 | $mappedFields[] = $mappedField; | |
1722 | } | |
1723 | return $mappedFields; | |
1724 | } | |
1725 | ||
2a9fc517 EM |
1726 | /** |
1727 | * Run import. | |
1728 | * | |
1729 | * @param \CRM_Queue_TaskContext $taskContext | |
1730 | * | |
1731 | * @param int $userJobID | |
1732 | * @param int $limit | |
1733 | * | |
1734 | * @return bool | |
1735 | * @throws \API_Exception | |
1736 | * @throws \CRM_Core_Exception | |
1737 | */ | |
1738 | public static function runImport($taskContext, $userJobID, $limit) { | |
1739 | $userJob = UserJob::get()->addWhere('id', '=', $userJobID)->addSelect('type_id')->execute()->first(); | |
1740 | $parserClass = NULL; | |
1741 | foreach (CRM_Core_BAO_UserJob::getTypes() as $userJobType) { | |
1742 | if ($userJob['type_id'] === $userJobType['id']) { | |
1743 | $parserClass = $userJobType['class']; | |
1744 | } | |
1745 | } | |
3592a5e4 | 1746 | /* @var \CRM_Import_Parser $parser */ |
2a9fc517 EM |
1747 | $parser = new $parserClass(); |
1748 | $parser->setUserJobID($userJobID); | |
1749 | // Not sure if we still need to init.... | |
1750 | $parser->init(); | |
1751 | $dataSource = $parser->getDataSourceObject(); | |
1752 | $dataSource->setStatuses(['new']); | |
1753 | $dataSource->setLimit($limit); | |
1754 | ||
1755 | while ($row = $dataSource->getRow()) { | |
1756 | $values = array_values($row); | |
b75fe839 | 1757 | $parser->import($values); |
2a9fc517 EM |
1758 | } |
1759 | $parser->doPostImportActions(); | |
1760 | return TRUE; | |
1761 | } | |
1762 | ||
e95f7d12 EM |
1763 | /** |
1764 | * Check if an error in custom data. | |
1765 | * | |
1766 | * @deprecated all of this is duplicated if getTransformedValue is used. | |
1767 | * | |
1768 | * @param array $params | |
1769 | * @param string $errorMessage | |
1770 | * A string containing all the error-fields. | |
1771 | * | |
1772 | * @param null $csType | |
1773 | */ | |
1774 | public function isErrorInCustomData($params, &$errorMessage, $csType = NULL) { | |
1775 | $dateType = CRM_Core_Session::singleton()->get("dateTypes"); | |
1776 | $errors = []; | |
1777 | ||
1778 | if (!empty($params['contact_sub_type'])) { | |
1779 | $csType = $params['contact_sub_type'] ?? NULL; | |
1780 | } | |
1781 | ||
1782 | if (empty($params['contact_type'])) { | |
1783 | $params['contact_type'] = 'Individual'; | |
1784 | } | |
1785 | ||
1786 | // get array of subtypes - CRM-18708 | |
1787 | if (in_array($csType, CRM_Contact_BAO_ContactType::basicTypes(TRUE), TRUE)) { | |
1788 | $csType = $this->getSubtypes($params['contact_type']); | |
1789 | } | |
1790 | ||
1791 | if (is_array($csType)) { | |
1792 | // fetch custom fields for every subtype and add it to $customFields array | |
1793 | // CRM-18708 | |
1794 | $customFields = []; | |
1795 | foreach ($csType as $cType) { | |
1796 | $customFields += CRM_Core_BAO_CustomField::getFields($params['contact_type'], FALSE, FALSE, $cType); | |
1797 | } | |
1798 | } | |
1799 | else { | |
1800 | $customFields = CRM_Core_BAO_CustomField::getFields($params['contact_type'], FALSE, FALSE, $csType); | |
1801 | } | |
1802 | ||
1803 | foreach ($params as $key => $value) { | |
1804 | if ($customFieldID = CRM_Core_BAO_CustomField::getKeyID($key)) { | |
1805 | //For address custom fields, we do get actual custom field value as an inner array of | |
1806 | //values so need to modify | |
1807 | if (!array_key_exists($customFieldID, $customFields)) { | |
1808 | return ts('field ID'); | |
1809 | } | |
1810 | /* check if it's a valid custom field id */ | |
1811 | $errors[] = $this->validateCustomField($customFieldID, $value, $customFields[$customFieldID], $dateType); | |
1812 | } | |
1813 | } | |
1814 | if ($errors) { | |
1815 | $errorMessage .= ($errorMessage ? '; ' : '') . implode('; ', array_filter($errors)); | |
1816 | } | |
1817 | } | |
1818 | ||
1819 | /** | |
1820 | * get subtypes given the contact type | |
1821 | * | |
1822 | * @param string $contactType | |
1823 | * @return array $subTypes | |
1824 | */ | |
1825 | protected function getSubtypes($contactType) { | |
1826 | $subTypes = []; | |
1827 | $types = CRM_Contact_BAO_ContactType::subTypeInfo($contactType); | |
1828 | ||
1829 | if (count($types) > 0) { | |
1830 | foreach ($types as $type) { | |
1831 | $subTypes[] = $type['name']; | |
1832 | } | |
1833 | } | |
1834 | return $subTypes; | |
1835 | } | |
1836 | ||
83078312 EM |
1837 | /** |
1838 | * Update the status of the import row to reflect the processing outcome. | |
1839 | * | |
1840 | * @param int $id | |
1841 | * @param string $status | |
1842 | * @param string $message | |
1843 | * @param int|null $entityID | |
1844 | * Optional created entity ID | |
1845 | * | |
2d306c45 EM |
1846 | * @noinspection PhpDocMissingThrowsInspection |
1847 | * @noinspection PhpUnhandledExceptionInspection | |
83078312 EM |
1848 | */ |
1849 | protected function setImportStatus(int $id, string $status, string $message, ?int $entityID = NULL): void { | |
1850 | $this->getDataSourceObject()->updateStatus($id, $status, $message, $entityID); | |
1851 | } | |
1852 | ||
4b58c5c4 EM |
1853 | /** |
1854 | * Convert any given date string to default date array. | |
1855 | * | |
1856 | * @param array $params | |
1857 | * Has given date-format. | |
1858 | * @param array $formatted | |
1859 | * Store formatted date in this array. | |
1860 | * @param int $dateType | |
1861 | * Type of date. | |
1862 | * @param string $dateParam | |
1863 | * Index of params. | |
1864 | */ | |
1865 | public static function formatCustomDate(&$params, &$formatted, $dateType, $dateParam) { | |
1866 | //fix for CRM-2687 | |
1867 | CRM_Utils_Date::convertToDefaultDate($params, $dateType, $dateParam); | |
1868 | $formatted[$dateParam] = CRM_Utils_Date::processDate($params[$dateParam]); | |
1869 | } | |
1870 | ||
ec3811b1 | 1871 | } |