3 +--------------------------------------------------------------------+
4 | CiviCRM version 4.6 |
5 +--------------------------------------------------------------------+
6 | Copyright CiviCRM LLC (c) 2004-2015 |
7 +--------------------------------------------------------------------+
8 | This file is a part of CiviCRM. |
10 | CiviCRM is free software; you can copy, modify, and distribute it |
11 | under the terms of the GNU Affero General Public License |
12 | Version 3, 19 November 2007 and the CiviCRM Licensing Exception. |
14 | CiviCRM is distributed in the hope that it will be useful, but |
15 | WITHOUT ANY WARRANTY; without even the implied warranty of |
16 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. |
17 | See the GNU Affero General Public License for more details. |
19 | You should have received a copy of the GNU Affero General Public |
20 | License and the CiviCRM Licensing Exception along |
21 | with this program; if not, contact CiviCRM LLC |
22 | at info[AT]civicrm[DOT]org. If you have questions about the |
23 | GNU Affero General Public License or the licensing of CiviCRM, |
24 | see the CiviCRM license FAQ at http://civicrm.org/licensing |
25 +--------------------------------------------------------------------+
31 * @copyright CiviCRM LLC (c) 2004-2015
35 abstract class CRM_Contact_Import_Parser
extends CRM_Import_Parser
{
37 protected $_tableName;
44 * Total number of lines in file
49 * Running total number of un matched Conact
51 protected $_unMatchCount;
54 * Array of unmatched lines
59 * Total number of contacts with unparsed addresses
61 protected $_unparsedAddressCount;
64 * Filename of mismatch data
68 protected $_misMatchFilemName;
70 protected $_primaryKeyName;
71 protected $_statusFieldName;
81 * Dedupe rule group id to use if set
85 public $_dedupeRuleGroupID = NULL;
88 * @param string $tableName
91 * @param int $contactType
92 * @param string $primaryKeyName
93 * @param string $statusFieldName
94 * @param int $onDuplicate
95 * @param int $statusID
96 * @param null $totalRowCount
97 * @param bool $doGeocodeAddress
99 * @param null $contactSubType
100 * @param int $dedupeRuleGroupID
107 $mode = self
::MODE_PREVIEW
,
108 $contactType = self
::CONTACT_INDIVIDUAL
,
109 $primaryKeyName = '_id',
110 $statusFieldName = '_status',
111 $onDuplicate = self
::DUPLICATE_SKIP
,
113 $totalRowCount = NULL,
114 $doGeocodeAddress = FALSE,
115 $timeout = CRM_Contact_Import_Parser
::DEFAULT_TIMEOUT
,
116 $contactSubType = NULL,
117 $dedupeRuleGroupID = NULL
120 // TODO: Make the timeout actually work
121 $this->_onDuplicate
= $onDuplicate;
122 $this->_dedupeRuleGroupID
= $dedupeRuleGroupID;
124 switch ($contactType) {
125 case CRM_Import_Parser
::CONTACT_INDIVIDUAL
:
126 $this->_contactType
= 'Individual';
129 case CRM_Import_Parser
::CONTACT_HOUSEHOLD
:
130 $this->_contactType
= 'Household';
133 case CRM_Import_Parser
::CONTACT_ORGANIZATION
:
134 $this->_contactType
= 'Organization';
137 $this->_contactSubType
= $contactSubType;
141 $this->_rowCount
= $this->_warningCount
= 0;
142 $this->_invalidRowCount
= $this->_validCount
= 0;
143 $this->_totalCount
= $this->_conflictCount
= 0;
145 $this->_errors
= array();
146 $this->_warnings
= array();
147 $this->_conflicts
= array();
148 $this->_unparsedAddresses
= array();
152 $this->_tableName
= $tableName;
153 $this->_primaryKeyName
= $primaryKeyName;
154 $this->_statusFieldName
= $statusFieldName;
156 if ($mode == self
::MODE_MAPFIELD
) {
157 $this->_rows
= array();
160 $this->_activeFieldCount
= count($this->_activeFields
);
163 if ($mode == self
::MODE_IMPORT
) {
164 //get the key of email field
165 foreach ($mapper as $key => $value) {
166 if (strtolower($value) == 'email') {
176 $config = CRM_Core_Config
::singleton();
177 $statusFile = "{$config->uploadDir}status_{$statusID}.txt";
178 $status = "<div class='description'> " . ts('No processing status reported yet.') . "</div>";
180 //do not force the browser to display the save dialog, CRM-7640
181 $contents = json_encode(array(0, $status));
183 file_put_contents($statusFile, $contents);
185 $startTimestamp = $currTimestamp = $prevTimestamp = time();
188 // get the contents of the temp. import table
189 $query = "SELECT * FROM $tableName";
190 if ($mode == self
::MODE_IMPORT
) {
191 $query .= " WHERE $statusFieldName = 'NEW'";
193 $dao = new CRM_Core_DAO();
194 $db = $dao->getDatabaseConnection();
195 $result = $db->query($query);
197 while ($values = $result->fetchRow(DB_FETCHMODE_ORDERED
)) {
200 /* trim whitespace around the values */
203 foreach ($values as $k => $v) {
204 $values[$k] = trim($v, " \t\r\n");
206 if (CRM_Utils_System
::isNull($values)) {
210 $this->_totalCount++
;
212 if ($mode == self
::MODE_MAPFIELD
) {
213 $returnCode = $this->mapField($values);
215 elseif ($mode == self
::MODE_PREVIEW
) {
216 $returnCode = $this->preview($values);
218 elseif ($mode == self
::MODE_SUMMARY
) {
219 $returnCode = $this->summary($values);
221 elseif ($mode == self
::MODE_IMPORT
) {
222 //print "Running parser in import mode<br/>\n";
223 $returnCode = $this->import($onDuplicate, $values, $doGeocodeAddress);
224 if ($statusID && (($this->_rowCount %
$skip) == 0)) {
225 $currTimestamp = time();
226 $totalTime = ($currTimestamp - $startTimestamp);
227 $time = ($currTimestamp - $prevTimestamp);
228 $recordsLeft = $totalRowCount - $this->_rowCount
;
229 if ($recordsLeft < 0) {
232 $estimatedTime = ($recordsLeft / $skip) * $time;
233 $estMinutes = floor($estimatedTime / 60);
235 if ($estMinutes > 1) {
236 $timeFormatted = $estMinutes . ' ' . ts('minutes') . ' ';
237 $estimatedTime = $estimatedTime - ($estMinutes * 60);
239 $timeFormatted .= round($estimatedTime) . ' ' . ts('seconds');
240 $processedPercent = (int ) (($this->_rowCount
* 100) / $totalRowCount);
241 $statusMsg = ts('%1 of %2 records - %3 remaining',
242 array(1 => $this->_rowCount
, 2 => $totalRowCount, 3 => $timeFormatted)
245 <div class=\"description\">
246 <strong>{$statusMsg}</strong>
250 $contents = json_encode(array($processedPercent, $status));
252 file_put_contents($statusFile, $contents);
254 $prevTimestamp = $currTimestamp;
259 $returnCode = self
::ERROR
;
262 // note that a line could be valid but still produce a warning
263 if ($returnCode & self
::VALID
) {
264 $this->_validCount++
;
265 if ($mode == self
::MODE_MAPFIELD
) {
266 $this->_rows
[] = $values;
267 $this->_activeFieldCount
= max($this->_activeFieldCount
, count($values));
271 if ($returnCode & self
::WARNING
) {
272 $this->_warningCount++
;
273 if ($this->_warningCount
< $this->_maxWarningCount
) {
274 $this->_warningCount
[] = $line;
278 if ($returnCode & self
::ERROR
) {
279 $this->_invalidRowCount++
;
280 if ($this->_invalidRowCount
< $this->_maxErrorCount
) {
281 array_unshift($values, $this->_rowCount
);
282 $this->_errors
[] = $values;
286 if ($returnCode & self
::CONFLICT
) {
287 $this->_conflictCount++
;
288 array_unshift($values, $this->_rowCount
);
289 $this->_conflicts
[] = $values;
292 if ($returnCode & self
::NO_MATCH
) {
293 $this->_unMatchCount++
;
294 array_unshift($values, $this->_rowCount
);
295 $this->_unMatch
[] = $values;
298 if ($returnCode & self
::DUPLICATE
) {
299 if ($returnCode & self
::MULTIPLE_DUPE
) {
300 /* TODO: multi-dupes should be counted apart from singles
301 * on non-skip action */
303 $this->_duplicateCount++
;
304 array_unshift($values, $this->_rowCount
);
305 $this->_duplicates
[] = $values;
306 if ($onDuplicate != self
::DUPLICATE_SKIP
) {
307 $this->_validCount++
;
311 if ($returnCode & self
::UNPARSED_ADDRESS_WARNING
) {
312 $this->_unparsedAddressCount++
;
313 array_unshift($values, $this->_rowCount
);
314 $this->_unparsedAddresses
[] = $values;
316 // we give the derived class a way of aborting the process
317 // note that the return code could be multiple code or'ed together
318 if ($returnCode & self
::STOP
) {
322 // if we are done processing the maxNumber of lines, break
323 if ($this->_maxLinesToProcess
> 0 && $this->_validCount
>= $this->_maxLinesToProcess
) {
327 // clean up memory from dao's
328 CRM_Core_DAO
::freeResult();
330 // see if we've hit our timeout yet
331 /* if ( $the_thing_with_the_stuff ) {
336 if ($mode == self
::MODE_PREVIEW ||
$mode == self
::MODE_IMPORT
) {
337 $customHeaders = $mapper;
339 $customfields = CRM_Core_BAO_CustomField
::getFields($this->_contactType
);
340 foreach ($customHeaders as $key => $value) {
341 if ($id = CRM_Core_BAO_CustomField
::getKeyID($value)) {
342 $customHeaders[$key] = $customfields[$id][0];
346 if ($this->_invalidRowCount
) {
347 // removed view url for invlaid contacts
348 $headers = array_merge(array(
354 $this->_errorFileName
= self
::errorFileName(self
::ERROR
);
355 self
::exportCSV($this->_errorFileName
, $headers, $this->_errors
);
357 if ($this->_conflictCount
) {
358 $headers = array_merge(array(
364 $this->_conflictFileName
= self
::errorFileName(self
::CONFLICT
);
365 self
::exportCSV($this->_conflictFileName
, $headers, $this->_conflicts
);
367 if ($this->_duplicateCount
) {
368 $headers = array_merge(array(
370 ts('View Contact URL'),
375 $this->_duplicateFileName
= self
::errorFileName(self
::DUPLICATE
);
376 self
::exportCSV($this->_duplicateFileName
, $headers, $this->_duplicates
);
378 if ($this->_unMatchCount
) {
379 $headers = array_merge(array(
386 $this->_misMatchFilemName
= self
::errorFileName(self
::NO_MATCH
);
387 self
::exportCSV($this->_misMatchFilemName
, $headers, $this->_unMatch
);
389 if ($this->_unparsedAddressCount
) {
390 $headers = array_merge(array(
392 ts('Contact Edit URL'),
396 $this->_errorFileName
= self
::errorFileName(self
::UNPARSED_ADDRESS_WARNING
);
397 self
::exportCSV($this->_errorFileName
, $headers, $this->_unparsedAddresses
);
400 //echo "$this->_totalCount,$this->_invalidRowCount,$this->_conflictCount,$this->_duplicateCount";
401 return $this->fini();
405 * Given a list of the importable field keys that the user has selected.
406 * set the active fields array to this list
408 * @param array $fieldKeys
409 * Mapped array of values.
411 public function setActiveFields($fieldKeys) {
412 $this->_activeFieldCount
= count($fieldKeys);
413 foreach ($fieldKeys as $key) {
414 if (empty($this->_fields
[$key])) {
415 $this->_activeFields
[] = new CRM_Contact_Import_Field('', ts('- do not import -'));
418 $this->_activeFields
[] = clone($this->_fields
[$key]);
426 public function setActiveFieldLocationTypes($elements) {
427 for ($i = 0; $i < count($elements); $i++
) {
428 $this->_activeFields
[$i]->_hasLocationType
= $elements[$i];
438 public function setActiveFieldPhoneTypes($elements) {
439 for ($i = 0; $i < count($elements); $i++
) {
440 $this->_activeFields
[$i]->_phoneType
= $elements[$i];
447 public function setActiveFieldWebsiteTypes($elements) {
448 for ($i = 0; $i < count($elements); $i++
) {
449 $this->_activeFields
[$i]->_websiteType
= $elements[$i];
454 * Set IM Service Provider type fields.
456 * @param array $elements
457 * IM service provider type ids.
461 public function setActiveFieldImProviders($elements) {
462 for ($i = 0; $i < count($elements); $i++
) {
463 $this->_activeFields
[$i]->_imProvider
= $elements[$i];
470 public function setActiveFieldRelated($elements) {
471 for ($i = 0; $i < count($elements); $i++
) {
472 $this->_activeFields
[$i]->_related
= $elements[$i];
479 public function setActiveFieldRelatedContactType($elements) {
480 for ($i = 0; $i < count($elements); $i++
) {
481 $this->_activeFields
[$i]->_relatedContactType
= $elements[$i];
488 public function setActiveFieldRelatedContactDetails($elements) {
489 for ($i = 0; $i < count($elements); $i++
) {
490 $this->_activeFields
[$i]->_relatedContactDetails
= $elements[$i];
497 public function setActiveFieldRelatedContactLocType($elements) {
498 for ($i = 0; $i < count($elements); $i++
) {
499 $this->_activeFields
[$i]->_relatedContactLocType
= $elements[$i];
506 public function setActiveFieldRelatedContactPhoneType($elements) {
507 for ($i = 0; $i < count($elements); $i++
) {
508 $this->_activeFields
[$i]->_relatedContactPhoneType
= $elements[$i];
515 public function setActiveFieldRelatedContactWebsiteType($elements) {
516 for ($i = 0; $i < count($elements); $i++
) {
517 $this->_activeFields
[$i]->_relatedContactWebsiteType
= $elements[$i];
522 * Set IM Service Provider type fields for related contacts.
524 * @param array $elements
525 * IM service provider type ids of related contact.
529 public function setActiveFieldRelatedContactImProvider($elements) {
530 for ($i = 0; $i < count($elements); $i++
) {
531 $this->_activeFields
[$i]->_relatedContactImProvider
= $elements[$i];
536 * Format the field values for input to the api.
539 * (reference ) associative array of name/value pairs
541 public function &getActiveFieldParams() {
544 for ($i = 0; $i < $this->_activeFieldCount
; $i++
) {
545 if ($this->_activeFields
[$i]->_name
== 'do_not_import') {
549 if (isset($this->_activeFields
[$i]->_value
)) {
550 if (isset($this->_activeFields
[$i]->_hasLocationType
)) {
551 if (!isset($params[$this->_activeFields
[$i]->_name
])) {
552 $params[$this->_activeFields
[$i]->_name
] = array();
556 $this->_activeFields
[$i]->_name
=> $this->_activeFields
[$i]->_value
,
557 'location_type_id' => $this->_activeFields
[$i]->_hasLocationType
,
560 if (isset($this->_activeFields
[$i]->_phoneType
)) {
561 $value['phone_type_id'] = $this->_activeFields
[$i]->_phoneType
;
564 // get IM service Provider type id
565 if (isset($this->_activeFields
[$i]->_imProvider
)) {
566 $value['provider_id'] = $this->_activeFields
[$i]->_imProvider
;
569 $params[$this->_activeFields
[$i]->_name
][] = $value;
571 elseif (isset($this->_activeFields
[$i]->_websiteType
)) {
573 $this->_activeFields
[$i]->_name
=> $this->_activeFields
[$i]->_value
,
574 'website_type_id' => $this->_activeFields
[$i]->_websiteType
,
577 $params[$this->_activeFields
[$i]->_name
][] = $value;
580 if (!isset($params[$this->_activeFields
[$i]->_name
])) {
581 if (!isset($this->_activeFields
[$i]->_related
)) {
582 $params[$this->_activeFields
[$i]->_name
] = $this->_activeFields
[$i]->_value
;
586 //minor fix for CRM-4062
587 if (isset($this->_activeFields
[$i]->_related
)) {
588 if (!isset($params[$this->_activeFields
[$i]->_related
])) {
589 $params[$this->_activeFields
[$i]->_related
] = array();
592 if (!isset($params[$this->_activeFields
[$i]->_related
]['contact_type']) && !empty($this->_activeFields
[$i]->_relatedContactType
)) {
593 $params[$this->_activeFields
[$i]->_related
]['contact_type'] = $this->_activeFields
[$i]->_relatedContactType
;
596 if (isset($this->_activeFields
[$i]->_relatedContactLocType
) && !empty($this->_activeFields
[$i]->_value
)) {
597 if (!empty($params[$this->_activeFields
[$i]->_related
][$this->_activeFields
[$i]->_relatedContactDetails
]) &&
598 !is_array($params[$this->_activeFields
[$i]->_related
][$this->_activeFields
[$i]->_relatedContactDetails
])
600 $params[$this->_activeFields
[$i]->_related
][$this->_activeFields
[$i]->_relatedContactDetails
] = array();
603 $this->_activeFields
[$i]->_relatedContactDetails
=> $this->_activeFields
[$i]->_value
,
604 'location_type_id' => $this->_activeFields
[$i]->_relatedContactLocType
,
607 if (isset($this->_activeFields
[$i]->_relatedContactPhoneType
)) {
608 $value['phone_type_id'] = $this->_activeFields
[$i]->_relatedContactPhoneType
;
611 // get IM service Provider type id for related contact
612 if (isset($this->_activeFields
[$i]->_relatedContactImProvider
)) {
613 $value['provider_id'] = $this->_activeFields
[$i]->_relatedContactImProvider
;
616 $params[$this->_activeFields
[$i]->_related
][$this->_activeFields
[$i]->_relatedContactDetails
][] = $value;
618 elseif (isset($this->_activeFields
[$i]->_relatedContactWebsiteType
)) {
619 $params[$this->_activeFields
[$i]->_related
][$this->_activeFields
[$i]->_relatedContactDetails
][] = array(
620 'url' => $this->_activeFields
[$i]->_value
,
621 'website_type_id' => $this->_activeFields
[$i]->_relatedContactWebsiteType
,
625 $params[$this->_activeFields
[$i]->_related
][$this->_activeFields
[$i]->_relatedContactDetails
] = $this->_activeFields
[$i]->_value
;
637 public function getColumnPatterns() {
639 foreach ($this->_fields
as $name => $field) {
640 $values[$name] = $field->_columnPattern
;
646 * @param string $name
649 * @param string $headerPattern
650 * @param string $dataPattern
651 * @param bool $hasLocationType
653 public function addField(
654 $name, $title, $type = CRM_Utils_Type
::T_INT
,
655 $headerPattern = '//', $dataPattern = '//',
656 $hasLocationType = FALSE
658 $this->_fields
[$name] = new CRM_Contact_Import_Field($name, $title, $type, $headerPattern, $dataPattern, $hasLocationType);
660 $this->_fields
['doNotImport'] = new CRM_Contact_Import_Field($name, $title, $type, $headerPattern, $dataPattern, $hasLocationType);
665 * Store parser values.
667 * @param CRM_Core_Session $store
673 public function set($store, $mode = self
::MODE_SUMMARY
) {
674 $store->set('rowCount', $this->_rowCount
);
675 $store->set('fields', $this->getSelectValues());
676 $store->set('fieldTypes', $this->getSelectTypes());
678 $store->set('columnPatterns', $this->getColumnPatterns());
679 $store->set('dataPatterns', $this->getDataPatterns());
680 $store->set('columnCount', $this->_activeFieldCount
);
682 $store->set('totalRowCount', $this->_totalCount
);
683 $store->set('validRowCount', $this->_validCount
);
684 $store->set('invalidRowCount', $this->_invalidRowCount
);
685 $store->set('conflictRowCount', $this->_conflictCount
);
686 $store->set('unMatchCount', $this->_unMatchCount
);
688 switch ($this->_contactType
) {
690 $store->set('contactType', CRM_Import_Parser
::CONTACT_INDIVIDUAL
);
694 $store->set('contactType', CRM_Import_Parser
::CONTACT_HOUSEHOLD
);
698 $store->set('contactType', CRM_Import_Parser
::CONTACT_ORGANIZATION
);
701 if ($this->_invalidRowCount
) {
702 $store->set('errorsFileName', $this->_errorFileName
);
704 if ($this->_conflictCount
) {
705 $store->set('conflictsFileName', $this->_conflictFileName
);
707 if (isset($this->_rows
) && !empty($this->_rows
)) {
708 $store->set('dataValues', $this->_rows
);
711 if ($this->_unMatchCount
) {
712 $store->set('mismatchFileName', $this->_misMatchFilemName
);
715 if ($mode == self
::MODE_IMPORT
) {
716 $store->set('duplicateRowCount', $this->_duplicateCount
);
717 $store->set('unparsedAddressCount', $this->_unparsedAddressCount
);
718 if ($this->_duplicateCount
) {
719 $store->set('duplicatesFileName', $this->_duplicateFileName
);
721 if ($this->_unparsedAddressCount
) {
722 $store->set('errorsFileName', $this->_errorFileName
);
725 //echo "$this->_totalCount,$this->_invalidRowCount,$this->_conflictCount,$this->_duplicateCount";
729 * Export data to a CSV file.
731 * @param string $fileName
732 * @param array $header
737 public static function exportCSV($fileName, $header, $data) {
739 if (file_exists($fileName) && !is_writable($fileName)) {
740 CRM_Core_Error
::movedSiteError($fileName);
742 //hack to remove '_status', '_statusMsg' and '_id' from error file
743 $errorValues = array();
744 $dbRecordStatus = array('IMPORTED', 'ERROR', 'DUPLICATE', 'INVALID', 'NEW');
745 foreach ($data as $rowCount => $rowValues) {
747 foreach ($rowValues as $key => $val) {
748 if (in_array($val, $dbRecordStatus) && $count == (count($rowValues) - 3)) {
751 $errorValues[$rowCount][$key] = $val;
755 $data = $errorValues;
758 $fd = fopen($fileName, 'w');
760 foreach ($header as $key => $value) {
761 $header[$key] = "\"$value\"";
763 $config = CRM_Core_Config
::singleton();
764 $output[] = implode($config->fieldSeparator
, $header);
766 foreach ($data as $datum) {
767 foreach ($datum as $key => $value) {
768 $datum[$key] = "\"$value\"";
770 $output[] = implode($config->fieldSeparator
, $datum);
772 fwrite($fd, implode("\n", $output));
777 * Update the record with PK $id in the import database table
780 * @param array $params
784 public function updateImportRecord($id, &$params) {
785 $statusFieldName = $this->_statusFieldName
;
786 $primaryKeyName = $this->_primaryKeyName
;
788 if ($statusFieldName && $primaryKeyName) {
789 $dao = new CRM_Core_DAO();
790 $db = $dao->getDatabaseConnection();
792 $query = "UPDATE $this->_tableName
793 SET $statusFieldName = ?,
794 ${statusFieldName}Msg = ?
795 WHERE $primaryKeyName = ?";
797 $params[$statusFieldName],
798 CRM_Utils_Array
::value("${statusFieldName}Msg", $params),
802 //print "Running query: $query<br/>With arguments: ".$params[$statusFieldName].", ".$params["${statusFieldName}Msg"].", $id<br/>";
804 $db->query($query, $args);