3 +--------------------------------------------------------------------+
5 +--------------------------------------------------------------------+
6 | Copyright CiviCRM LLC (c) 2004-2019 |
7 +--------------------------------------------------------------------+
8 | This file is a part of CiviCRM. |
10 | CiviCRM is free software; you can copy, modify, and distribute it |
11 | under the terms of the GNU Affero General Public License |
12 | Version 3, 19 November 2007 and the CiviCRM Licensing Exception. |
14 | CiviCRM is distributed in the hope that it will be useful, but |
15 | WITHOUT ANY WARRANTY; without even the implied warranty of |
16 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. |
17 | See the GNU Affero General Public License for more details. |
19 | You should have received a copy of the GNU Affero General Public |
20 | License and the CiviCRM Licensing Exception along |
21 | with this program; if not, contact CiviCRM LLC |
22 | at info[AT]civicrm[DOT]org. If you have questions about the |
23 | GNU Affero General Public License or the licensing of CiviCRM, |
24 | see the CiviCRM license FAQ at http://civicrm.org/licensing |
25 +--------------------------------------------------------------------+
31 * @copyright CiviCRM LLC (c) 2004-2019
33 abstract class CRM_Import_Parser
{
37 const MAX_WARNINGS
= 25, DEFAULT_TIMEOUT
= 30;
42 const VALID
= 1, WARNING
= 2, ERROR
= 4, CONFLICT
= 8, STOP
= 16, DUPLICATE
= 32, MULTIPLE_DUPE
= 64, NO_MATCH
= 128, UNPARSED_ADDRESS_WARNING
= 256;
47 const MODE_MAPFIELD
= 1, MODE_PREVIEW
= 2, MODE_SUMMARY
= 4, MODE_IMPORT
= 8;
50 * Codes for duplicate record handling
52 const DUPLICATE_SKIP
= 1, DUPLICATE_REPLACE
= 2, DUPLICATE_UPDATE
= 4, DUPLICATE_FILL
= 8, DUPLICATE_NOCHECK
= 16;
57 const CONTACT_INDIVIDUAL
= 1, CONTACT_HOUSEHOLD
= 2, CONTACT_ORGANIZATION
= 4;
61 * Total number of non empty lines
64 protected $_totalCount;
67 * Running total number of valid lines
70 protected $_validCount;
73 * Running total number of invalid rows
76 protected $_invalidRowCount;
79 * Maximum number of non-empty/comment lines to process
83 protected $_maxLinesToProcess;
86 * Array of error lines, bounded by MAX_ERROR
92 * Total number of conflict lines
95 protected $_conflictCount;
98 * Array of conflict lines
101 protected $_conflicts;
104 * Total number of duplicate (from database) lines
107 protected $_duplicateCount;
110 * Array of duplicate lines
113 protected $_duplicates;
116 * Running total number of warnings
119 protected $_warningCount;
122 * Maximum number of warnings to store
125 protected $_maxWarningCount = self
::MAX_WARNINGS
;
128 * Array of warning lines, bounded by MAX_WARNING
131 protected $_warnings;
134 * Array of all the fields that could potentially be part
135 * of this import process
141 * Array of the fields that are actually part of the import process
142 * the position in the array also dictates their position in the import
146 protected $_activeFields;
149 * Cache the count of active fields
153 protected $_activeFieldCount;
156 * Cache of preview rows
163 * Filename of error data
167 protected $_errorFileName;
170 * Filename of conflict data
174 protected $_conflictFileName;
177 * Filename of duplicate data
181 protected $_duplicateFileName;
188 public $_contactType;
194 public $_contactSubType;
199 public function __construct() {
200 $this->_maxLinesToProcess
= 0;
204 * Abstract function definitions.
206 abstract protected function init();
211 abstract protected function fini();
216 * @param array $values
220 abstract protected function mapField(&$values);
225 * @param array $values
229 abstract protected function preview(&$values);
236 abstract protected function summary(&$values);
239 * @param $onDuplicate
244 abstract protected function import($onDuplicate, &$values);
247 * Set and validate field values.
249 * @param array $elements
251 * @param $erroneousField
256 public function setActiveFieldValues($elements, &$erroneousField) {
257 $maxCount = count($elements) < $this->_activeFieldCount ?
count($elements) : $this->_activeFieldCount
;
258 for ($i = 0; $i < $maxCount; $i++
) {
259 $this->_activeFields
[$i]->setValue($elements[$i]);
262 // reset all the values that we did not have an equivalent import element
263 for (; $i < $this->_activeFieldCount
; $i++
) {
264 $this->_activeFields
[$i]->resetValue();
267 // now validate the fields and return false if error
268 $valid = self
::VALID
;
269 for ($i = 0; $i < $this->_activeFieldCount
; $i++
) {
270 if (!$this->_activeFields
[$i]->validate()) {
271 // no need to do any more validation
272 $erroneousField = $i;
273 $valid = self
::ERROR
;
281 * Format the field values for input to the api.
284 * (reference) associative array of name/value pairs
286 public function &getActiveFieldParams() {
288 for ($i = 0; $i < $this->_activeFieldCount
; $i++
) {
289 if (isset($this->_activeFields
[$i]->_value
)
290 && !isset($params[$this->_activeFields
[$i]->_name
])
291 && !isset($this->_activeFields
[$i]->_related
)
294 $params[$this->_activeFields
[$i]->_name
] = $this->_activeFields
[$i]->_value
;
301 * Add progress bar to the import process. Calculates time remaining, status etc.
304 * status id of the import process saved in $config->uploadDir.
305 * @param bool $startImport
306 * True when progress bar is to be initiated.
307 * @param $startTimestamp
308 * Initial timstamp when the import was started.
309 * @param $prevTimestamp
310 * Previous timestamp when this function was last called.
311 * @param $totalRowCount
312 * Total number of rows in the import file.
314 * @return NULL|$currTimestamp
316 public function progressImport($statusID, $startImport = TRUE, $startTimestamp = NULL, $prevTimestamp = NULL, $totalRowCount = NULL) {
317 $config = CRM_Core_Config
::singleton();
318 $statusFile = "{$config->uploadDir}status_{$statusID}.txt";
321 $status = "<div class='description'> " . ts('No processing status reported yet.') . "</div>";
322 //do not force the browser to display the save dialog, CRM-7640
323 $contents = json_encode([0, $status]);
324 file_put_contents($statusFile, $contents);
327 $rowCount = isset($this->_rowCount
) ?
$this->_rowCount
: $this->_lineCount
;
328 $currTimestamp = time();
329 $totalTime = ($currTimestamp - $startTimestamp);
330 $time = ($currTimestamp - $prevTimestamp);
331 $recordsLeft = $totalRowCount - $rowCount;
332 if ($recordsLeft < 0) {
335 $estimatedTime = ($recordsLeft / 50) * $time;
336 $estMinutes = floor($estimatedTime / 60);
338 if ($estMinutes > 1) {
339 $timeFormatted = $estMinutes . ' ' . ts('minutes') . ' ';
340 $estimatedTime = $estimatedTime - ($estMinutes * 60);
342 $timeFormatted .= round($estimatedTime) . ' ' . ts('seconds');
343 $processedPercent = (int ) (($rowCount * 100) / $totalRowCount);
344 $statusMsg = ts('%1 of %2 records - %3 remaining',
345 [1 => $rowCount, 2 => $totalRowCount, 3 => $timeFormatted]
347 $status = "<div class=\"description\"> <strong>{$statusMsg}</strong></div>";
348 $contents = json_encode([$processedPercent, $status]);
350 file_put_contents($statusFile, $contents);
351 return $currTimestamp;
358 public function getSelectValues() {
360 foreach ($this->_fields
as $name => $field) {
361 $values[$name] = $field->_title
;
369 public function getSelectTypes() {
371 foreach ($this->_fields
as $name => $field) {
372 if (isset($field->_hasLocationType
)) {
373 $values[$name] = $field->_hasLocationType
;
382 public function getHeaderPatterns() {
384 foreach ($this->_fields
as $name => $field) {
385 if (isset($field->_headerPattern
)) {
386 $values[$name] = $field->_headerPattern
;
395 public function getDataPatterns() {
397 foreach ($this->_fields
as $name => $field) {
398 $values[$name] = $field->_dataPattern
;
404 * Remove single-quote enclosures from a value array (row).
406 * @param array $values
407 * @param string $enclosure
411 public static function encloseScrub(&$values, $enclosure = "'") {
412 if (empty($values)) {
416 foreach ($values as $k => $v) {
417 $values[$k] = preg_replace("/^$enclosure(.*)$enclosure$/", '$1', $v);
428 public function setMaxLinesToProcess($max) {
429 $this->_maxLinesToProcess
= $max;
433 * Determines the file extension based on error code.
435 * @var $type error code constant
438 public static function errorFileName($type) {
444 $config = CRM_Core_Config
::singleton();
445 $fileName = $config->uploadDir
. "sqlImport";
448 $fileName .= '.errors';
452 $fileName .= '.conflicts';
455 case self
::DUPLICATE
:
456 $fileName .= '.duplicates';
460 $fileName .= '.mismatch';
463 case self
::UNPARSED_ADDRESS_WARNING
:
464 $fileName .= '.unparsedAddress';
472 * Determines the file name based on error code.
474 * @var $type error code constant
477 public static function saveFileName($type) {
484 $fileName = 'Import_Errors.csv';
488 $fileName = 'Import_Conflicts.csv';
491 case self
::DUPLICATE
:
492 $fileName = 'Import_Duplicates.csv';
496 $fileName = 'Import_Mismatch.csv';
499 case self
::UNPARSED_ADDRESS_WARNING
:
500 $fileName = 'Import_Unparsed_Address.csv';
508 * Check if contact is a duplicate .
510 * @param array $formatValues
514 protected function checkContactDuplicate(&$formatValues) {
515 //retrieve contact id using contact dedupe rule
516 $formatValues['contact_type'] = $this->_contactType
;
517 $formatValues['version'] = 3;
518 require_once 'CRM/Utils/DeprecatedUtils.php';
519 $error = _civicrm_api3_deprecated_check_contact_dedupe($formatValues);
524 * Parse a field which could be represented by a label or name value rather than the DB value.
526 * We will try to match name first but if not available then see if we have a label that can be converted to a name.
528 * @param string|int|null $submittedValue
529 * @param array $fieldSpec
530 * Metadata for the field
534 protected function parsePseudoConstantField($submittedValue, $fieldSpec) {
535 /* @var \CRM_Core_DAO $bao */
536 $bao = $fieldSpec['bao'];
537 // For historical reasons use validate as context - ie disabled name matches ARE permitted.
538 $nameOptions = $bao::buildOptions($fieldSpec['name'], 'validate');
539 if (!isset($nameOptions[$submittedValue])) {
540 $labelOptions = array_flip($bao::buildOptions($fieldSpec['name'], 'match'));
541 if (isset($labelOptions[$submittedValue])) {
542 return array_search($labelOptions[$submittedValue], $nameOptions, TRUE);