3 +--------------------------------------------------------------------+
5 +--------------------------------------------------------------------+
6 | Copyright CiviCRM LLC (c) 2004-2018 |
7 +--------------------------------------------------------------------+
8 | This file is a part of CiviCRM. |
10 | CiviCRM is free software; you can copy, modify, and distribute it |
11 | under the terms of the GNU Affero General Public License |
12 | Version 3, 19 November 2007 and the CiviCRM Licensing Exception. |
14 | CiviCRM is distributed in the hope that it will be useful, but |
15 | WITHOUT ANY WARRANTY; without even the implied warranty of |
16 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. |
17 | See the GNU Affero General Public License for more details. |
19 | You should have received a copy of the GNU Affero General Public |
20 | License and the CiviCRM Licensing Exception along |
21 | with this program; if not, contact CiviCRM LLC |
22 | at info[AT]civicrm[DOT]org. If you have questions about the |
23 | GNU Affero General Public License or the licensing of CiviCRM, |
24 | see the CiviCRM license FAQ at http://civicrm.org/licensing |
25 +--------------------------------------------------------------------+
31 * @copyright CiviCRM LLC (c) 2004-2018
33 abstract class CRM_Import_Parser
{
37 const MAX_WARNINGS
= 25, DEFAULT_TIMEOUT
= 30;
42 const VALID
= 1, WARNING
= 2, ERROR
= 4, CONFLICT
= 8, STOP
= 16, DUPLICATE
= 32, MULTIPLE_DUPE
= 64, NO_MATCH
= 128, UNPARSED_ADDRESS_WARNING
= 256;
47 const MODE_MAPFIELD
= 1, MODE_PREVIEW
= 2, MODE_SUMMARY
= 4, MODE_IMPORT
= 8;
50 * Codes for duplicate record handling
52 const DUPLICATE_SKIP
= 1, DUPLICATE_REPLACE
= 2, DUPLICATE_UPDATE
= 4, DUPLICATE_FILL
= 8, DUPLICATE_NOCHECK
= 16;
57 const CONTACT_INDIVIDUAL
= 1, CONTACT_HOUSEHOLD
= 2, CONTACT_ORGANIZATION
= 4;
61 * Total number of non empty lines
63 protected $_totalCount;
66 * Running total number of valid lines
68 protected $_validCount;
71 * Running total number of invalid rows
73 protected $_invalidRowCount;
76 * Maximum number of non-empty/comment lines to process
80 protected $_maxLinesToProcess;
83 * Array of error lines, bounded by MAX_ERROR
88 * Total number of conflict lines
90 protected $_conflictCount;
93 * Array of conflict lines
95 protected $_conflicts;
98 * Total number of duplicate (from database) lines
100 protected $_duplicateCount;
103 * Array of duplicate lines
105 protected $_duplicates;
108 * Running total number of warnings
110 protected $_warningCount;
113 * Maximum number of warnings to store
115 protected $_maxWarningCount = self
::MAX_WARNINGS
;
118 * Array of warning lines, bounded by MAX_WARNING
120 protected $_warnings;
123 * Array of all the fields that could potentially be part
124 * of this import process
130 * Array of the fields that are actually part of the import process
131 * the position in the array also dictates their position in the import
135 protected $_activeFields;
138 * Cache the count of active fields
142 protected $_activeFieldCount;
145 * Cache of preview rows
152 * Filename of error data
156 protected $_errorFileName;
159 * Filename of conflict data
163 protected $_conflictFileName;
166 * Filename of duplicate data
170 protected $_duplicateFileName;
177 public $_contactType;
183 public $_contactSubType;
188 public function __construct() {
189 $this->_maxLinesToProcess
= 0;
193 * Abstract function definitions.
195 abstract protected function init();
200 abstract protected function fini();
205 * @param array $values
209 abstract protected function mapField(&$values);
214 * @param array $values
218 abstract protected function preview(&$values);
225 abstract protected function summary(&$values);
228 * @param $onDuplicate
233 abstract protected function import($onDuplicate, &$values);
236 * Set and validate field values.
238 * @param array $elements
240 * @param $erroneousField
245 public function setActiveFieldValues($elements, &$erroneousField) {
246 $maxCount = count($elements) < $this->_activeFieldCount ?
count($elements) : $this->_activeFieldCount
;
247 for ($i = 0; $i < $maxCount; $i++
) {
248 $this->_activeFields
[$i]->setValue($elements[$i]);
251 // reset all the values that we did not have an equivalent import element
252 for (; $i < $this->_activeFieldCount
; $i++
) {
253 $this->_activeFields
[$i]->resetValue();
256 // now validate the fields and return false if error
257 $valid = self
::VALID
;
258 for ($i = 0; $i < $this->_activeFieldCount
; $i++
) {
259 if (!$this->_activeFields
[$i]->validate()) {
260 // no need to do any more validation
261 $erroneousField = $i;
262 $valid = self
::ERROR
;
270 * Format the field values for input to the api.
273 * (reference) associative array of name/value pairs
275 public function &getActiveFieldParams() {
277 for ($i = 0; $i < $this->_activeFieldCount
; $i++
) {
278 if (isset($this->_activeFields
[$i]->_value
)
279 && !isset($params[$this->_activeFields
[$i]->_name
])
280 && !isset($this->_activeFields
[$i]->_related
)
283 $params[$this->_activeFields
[$i]->_name
] = $this->_activeFields
[$i]->_value
;
290 * Add progress bar to the import process. Calculates time remaining, status etc.
293 * status id of the import process saved in $config->uploadDir.
294 * @param bool $startImport
295 * True when progress bar is to be initiated.
296 * @param $startTimestamp
297 * Initial timstamp when the import was started.
298 * @param $prevTimestamp
299 * Previous timestamp when this function was last called.
300 * @param $totalRowCount
301 * Total number of rows in the import file.
303 * @return NULL|$currTimestamp
305 public function progressImport($statusID, $startImport = TRUE, $startTimestamp = NULL, $prevTimestamp = NULL, $totalRowCount = NULL) {
306 $config = CRM_Core_Config
::singleton();
307 $statusFile = "{$config->uploadDir}status_{$statusID}.txt";
310 $status = "<div class='description'> " . ts('No processing status reported yet.') . "</div>";
311 //do not force the browser to display the save dialog, CRM-7640
312 $contents = json_encode(array(0, $status));
313 file_put_contents($statusFile, $contents);
316 $rowCount = isset($this->_rowCount
) ?
$this->_rowCount
: $this->_lineCount
;
317 $currTimestamp = time();
318 $totalTime = ($currTimestamp - $startTimestamp);
319 $time = ($currTimestamp - $prevTimestamp);
320 $recordsLeft = $totalRowCount - $rowCount;
321 if ($recordsLeft < 0) {
324 $estimatedTime = ($recordsLeft / 50) * $time;
325 $estMinutes = floor($estimatedTime / 60);
327 if ($estMinutes > 1) {
328 $timeFormatted = $estMinutes . ' ' . ts('minutes') . ' ';
329 $estimatedTime = $estimatedTime - ($estMinutes * 60);
331 $timeFormatted .= round($estimatedTime) . ' ' . ts('seconds');
332 $processedPercent = (int ) (($rowCount * 100) / $totalRowCount);
333 $statusMsg = ts('%1 of %2 records - %3 remaining',
334 array(1 => $rowCount, 2 => $totalRowCount, 3 => $timeFormatted)
336 $status = "<div class=\"description\"> <strong>{$statusMsg}</strong></div>";
337 $contents = json_encode(array($processedPercent, $status));
339 file_put_contents($statusFile, $contents);
340 return $currTimestamp;
347 public function getSelectValues() {
349 foreach ($this->_fields
as $name => $field) {
350 $values[$name] = $field->_title
;
358 public function getSelectTypes() {
360 foreach ($this->_fields
as $name => $field) {
361 if (isset($field->_hasLocationType
)) {
362 $values[$name] = $field->_hasLocationType
;
371 public function getHeaderPatterns() {
373 foreach ($this->_fields
as $name => $field) {
374 if (isset($field->_headerPattern
)) {
375 $values[$name] = $field->_headerPattern
;
384 public function getDataPatterns() {
386 foreach ($this->_fields
as $name => $field) {
387 $values[$name] = $field->_dataPattern
;
393 * Remove single-quote enclosures from a value array (row).
395 * @param array $values
396 * @param string $enclosure
400 public static function encloseScrub(&$values, $enclosure = "'") {
401 if (empty($values)) {
405 foreach ($values as $k => $v) {
406 $values[$k] = preg_replace("/^$enclosure(.*)$enclosure$/", '$1', $v);
417 public function setMaxLinesToProcess($max) {
418 $this->_maxLinesToProcess
= $max;
422 * Determines the file extension based on error code.
424 * @var $type error code constant
427 public static function errorFileName($type) {
433 $config = CRM_Core_Config
::singleton();
434 $fileName = $config->uploadDir
. "sqlImport";
437 $fileName .= '.errors';
441 $fileName .= '.conflicts';
444 case self
::DUPLICATE
:
445 $fileName .= '.duplicates';
449 $fileName .= '.mismatch';
452 case self
::UNPARSED_ADDRESS_WARNING
:
453 $fileName .= '.unparsedAddress';
461 * Determines the file name based on error code.
463 * @var $type error code constant
466 public static function saveFileName($type) {
473 $fileName = 'Import_Errors.csv';
477 $fileName = 'Import_Conflicts.csv';
480 case self
::DUPLICATE
:
481 $fileName = 'Import_Duplicates.csv';
485 $fileName = 'Import_Mismatch.csv';
488 case self
::UNPARSED_ADDRESS_WARNING
:
489 $fileName = 'Import_Unparsed_Address.csv';
497 * Check if contact is a duplicate .
499 * @param array $formatValues
503 protected function checkContactDuplicate(&$formatValues) {
504 //retrieve contact id using contact dedupe rule
505 $formatValues['contact_type'] = $this->_contactType
;
506 $formatValues['version'] = 3;
507 require_once 'CRM/Utils/DeprecatedUtils.php';
508 $error = _civicrm_api3_deprecated_check_contact_dedupe($formatValues);