3 +--------------------------------------------------------------------+
4 | CiviCRM version 4.5 |
5 +--------------------------------------------------------------------+
6 | Copyright CiviCRM LLC (c) 2004-2014 |
7 +--------------------------------------------------------------------+
8 | This file is a part of CiviCRM. |
10 | CiviCRM is free software; you can copy, modify, and distribute it |
11 | under the terms of the GNU Affero General Public License |
12 | Version 3, 19 November 2007 and the CiviCRM Licensing Exception. |
14 | CiviCRM is distributed in the hope that it will be useful, but |
15 | WITHOUT ANY WARRANTY; without even the implied warranty of |
16 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. |
17 | See the GNU Affero General Public License for more details. |
19 | You should have received a copy of the GNU Affero General Public |
20 | License and the CiviCRM Licensing Exception along |
21 | with this program; if not, contact CiviCRM LLC |
22 | at info[AT]civicrm[DOT]org. If you have questions about the |
23 | GNU Affero General Public License or the licensing of CiviCRM, |
24 | see the CiviCRM license FAQ at http://civicrm.org/licensing |
25 +--------------------------------------------------------------------+
31 * @copyright CiviCRM LLC (c) 2004-2014
37 abstract class CRM_Contact_Import_Parser
extends CRM_Import_Parser
{
39 protected $_tableName;
47 * total number of lines in file
52 * running total number of un matched Conact
54 protected $_unMatchCount;
57 * array of unmatched lines
62 * total number of contacts with unparsed addresses
64 protected $_unparsedAddressCount;
67 * filename of mismatch data
71 protected $_misMatchFilemName;
73 protected $_primaryKeyName;
74 protected $_statusFieldName;
84 * dedupe rule group id to use if set
88 public $_dedupeRuleGroupID = NULL;
94 * @param int $contactType
95 * @param string $primaryKeyName
96 * @param string $statusFieldName
97 * @param int $onDuplicate
98 * @param null $statusID
99 * @param null $totalRowCount
100 * @param bool $doGeocodeAddress
101 * @param int $timeout
102 * @param null $contactSubType
103 * @param null $dedupeRuleGroupID
107 function run($tableName,
109 $mode = self
::MODE_PREVIEW
,
110 $contactType = self
::CONTACT_INDIVIDUAL
,
111 $primaryKeyName = '_id',
112 $statusFieldName = '_status',
113 $onDuplicate = self
::DUPLICATE_SKIP
,
115 $totalRowCount = NULL,
116 $doGeocodeAddress = FALSE,
117 $timeout = CRM_Contact_Import_Parser
::DEFAULT_TIMEOUT
,
118 $contactSubType = NULL,
119 $dedupeRuleGroupID = NULL
122 // TODO: Make the timeout actually work
123 $this->_onDuplicate
= $onDuplicate;
124 $this->_dedupeRuleGroupID
= $dedupeRuleGroupID;
126 switch ($contactType) {
127 case CRM_Import_Parser
::CONTACT_INDIVIDUAL
:
128 $this->_contactType
= 'Individual';
131 case CRM_Import_Parser
::CONTACT_HOUSEHOLD
:
132 $this->_contactType
= 'Household';
135 case CRM_Import_Parser
::CONTACT_ORGANIZATION
:
136 $this->_contactType
= 'Organization';
139 $this->_contactSubType
= $contactSubType;
143 $this->_rowCount
= $this->_warningCount
= 0;
144 $this->_invalidRowCount
= $this->_validCount
= 0;
145 $this->_totalCount
= $this->_conflictCount
= 0;
147 $this->_errors
= array();
148 $this->_warnings
= array();
149 $this->_conflicts
= array();
150 $this->_unparsedAddresses
= array();
154 $this->_tableName
= $tableName;
155 $this->_primaryKeyName
= $primaryKeyName;
156 $this->_statusFieldName
= $statusFieldName;
158 if ($mode == self
::MODE_MAPFIELD
) {
159 $this->_rows
= array();
162 $this->_activeFieldCount
= count($this->_activeFields
);
165 if ($mode == self
::MODE_IMPORT
) {
166 //get the key of email field
167 foreach ($mapper as $key => $value) {
168 if (strtolower($value) == 'email') {
178 $config = CRM_Core_Config
::singleton();
179 $statusFile = "{$config->uploadDir}status_{$statusID}.txt";
180 $status = "<div class='description'> " . ts('No processing status reported yet.') . "</div>";
182 //do not force the browser to display the save dialog, CRM-7640
183 $contents = json_encode(array(0, $status));
185 file_put_contents($statusFile, $contents);
187 $startTimestamp = $currTimestamp = $prevTimestamp = time();
190 // get the contents of the temp. import table
191 $query = "SELECT * FROM $tableName";
192 if ($mode == self
::MODE_IMPORT
) {
193 $query .= " WHERE $statusFieldName = 'NEW'";
195 $dao = new CRM_Core_DAO();
196 $db = $dao->getDatabaseConnection();
197 $result = $db->query($query);
199 while ($values = $result->fetchRow(DB_FETCHMODE_ORDERED
)) {
202 /* trim whitespace around the values */
205 foreach ($values as $k => $v) {
206 $values[$k] = trim($v, " \t\r\n");
208 if (CRM_Utils_System
::isNull($values)) {
212 $this->_totalCount++
;
214 if ($mode == self
::MODE_MAPFIELD
) {
215 $returnCode = $this->mapField($values);
217 elseif ($mode == self
::MODE_PREVIEW
) {
218 $returnCode = $this->preview($values);
220 elseif ($mode == self
::MODE_SUMMARY
) {
221 $returnCode = $this->summary($values);
223 elseif ($mode == self
::MODE_IMPORT
) {
224 //print "Running parser in import mode<br/>\n";
225 $returnCode = $this->import($onDuplicate, $values, $doGeocodeAddress);
226 if ($statusID && (($this->_rowCount %
$skip) == 0)) {
227 $currTimestamp = time();
228 $totalTime = ($currTimestamp - $startTimestamp);
229 $time = ($currTimestamp - $prevTimestamp);
230 $recordsLeft = $totalRowCount - $this->_rowCount
;
231 if ($recordsLeft < 0) {
234 $estimatedTime = ($recordsLeft / $skip) * $time;
235 $estMinutes = floor($estimatedTime / 60);
237 if ($estMinutes > 1) {
238 $timeFormatted = $estMinutes . ' ' . ts('minutes') . ' ';
239 $estimatedTime = $estimatedTime - ($estMinutes * 60);
241 $timeFormatted .= round($estimatedTime) . ' ' . ts('seconds');
242 $processedPercent = (int )(($this->_rowCount
* 100) / $totalRowCount);
243 $statusMsg = ts('%1 of %2 records - %3 remaining',
244 array(1 => $this->_rowCount
, 2 => $totalRowCount, 3 => $timeFormatted)
247 <div class=\"description\">
248 <strong>{$statusMsg}</strong>
252 $contents = json_encode (array($processedPercent, $status));
254 file_put_contents($statusFile, $contents);
256 $prevTimestamp = $currTimestamp;
261 $returnCode = self
::ERROR
;
264 // note that a line could be valid but still produce a warning
265 if ($returnCode & self
::VALID
) {
266 $this->_validCount++
;
267 if ($mode == self
::MODE_MAPFIELD
) {
268 $this->_rows
[] = $values;
269 $this->_activeFieldCount
= max($this->_activeFieldCount
, count($values));
273 if ($returnCode & self
::WARNING
) {
274 $this->_warningCount++
;
275 if ($this->_warningCount
< $this->_maxWarningCount
) {
276 $this->_warningCount
[] = $line;
280 if ($returnCode & self
::ERROR
) {
281 $this->_invalidRowCount++
;
282 if ($this->_invalidRowCount
< $this->_maxErrorCount
) {
283 array_unshift($values, $this->_rowCount
);
284 $this->_errors
[] = $values;
288 if ($returnCode & self
::CONFLICT
) {
289 $this->_conflictCount++
;
290 array_unshift($values, $this->_rowCount
);
291 $this->_conflicts
[] = $values;
294 if ($returnCode & self
::NO_MATCH
) {
295 $this->_unMatchCount++
;
296 array_unshift($values, $this->_rowCount
);
297 $this->_unMatch
[] = $values;
300 if ($returnCode & self
::DUPLICATE
) {
301 if ($returnCode & self
::MULTIPLE_DUPE
) {
302 /* TODO: multi-dupes should be counted apart from singles
303 * on non-skip action */
305 $this->_duplicateCount++
;
306 array_unshift($values, $this->_rowCount
);
307 $this->_duplicates
[] = $values;
308 if ($onDuplicate != self
::DUPLICATE_SKIP
) {
309 $this->_validCount++
;
313 if ($returnCode & self
::UNPARSED_ADDRESS_WARNING
) {
314 $this->_unparsedAddressCount++
;
315 array_unshift($values, $this->_rowCount
);
316 $this->_unparsedAddresses
[] = $values;
318 // we give the derived class a way of aborting the process
319 // note that the return code could be multiple code or'ed together
320 if ($returnCode & self
::STOP
) {
324 // if we are done processing the maxNumber of lines, break
325 if ($this->_maxLinesToProcess
> 0 && $this->_validCount
>= $this->_maxLinesToProcess
) {
329 // clean up memory from dao's
330 CRM_Core_DAO
::freeResult();
332 // see if we've hit our timeout yet
333 /* if ( $the_thing_with_the_stuff ) {
339 if ($mode == self
::MODE_PREVIEW ||
$mode == self
::MODE_IMPORT
) {
340 $customHeaders = $mapper;
342 $customfields = CRM_Core_BAO_CustomField
::getFields($this->_contactType
);
343 foreach ($customHeaders as $key => $value) {
344 if ($id = CRM_Core_BAO_CustomField
::getKeyID($value)) {
345 $customHeaders[$key] = $customfields[$id][0];
349 if ($this->_invalidRowCount
) {
350 // removed view url for invlaid contacts
351 $headers = array_merge(array(ts('Line Number'),
356 $this->_errorFileName
= self
::errorFileName(self
::ERROR
);
357 self
::exportCSV($this->_errorFileName
, $headers, $this->_errors
);
359 if ($this->_conflictCount
) {
360 $headers = array_merge(array(ts('Line Number'),
365 $this->_conflictFileName
= self
::errorFileName(self
::CONFLICT
);
366 self
::exportCSV($this->_conflictFileName
, $headers, $this->_conflicts
);
368 if ($this->_duplicateCount
) {
369 $headers = array_merge(array(ts('Line Number'),
370 ts('View Contact URL'),
375 $this->_duplicateFileName
= self
::errorFileName(self
::DUPLICATE
);
376 self
::exportCSV($this->_duplicateFileName
, $headers, $this->_duplicates
);
378 if ($this->_unMatchCount
) {
379 $headers = array_merge(array(ts('Line Number'),
385 $this->_misMatchFilemName
= self
::errorFileName(self
::NO_MATCH
);
386 self
::exportCSV($this->_misMatchFilemName
, $headers, $this->_unMatch
);
388 if ($this->_unparsedAddressCount
) {
389 $headers = array_merge(array(ts('Line Number'),
390 ts('Contact Edit URL'),
394 $this->_errorFileName
= self
::errorFileName(self
::UNPARSED_ADDRESS_WARNING
);
395 self
::exportCSV($this->_errorFileName
, $headers, $this->_unparsedAddresses
);
398 //echo "$this->_totalCount,$this->_invalidRowCount,$this->_conflictCount,$this->_duplicateCount";
399 return $this->fini();
403 * Given a list of the importable field keys that the user has selected
404 * set the active fields array to this list
406 * @param array mapped array of values
411 function setActiveFields($fieldKeys) {
412 $this->_activeFieldCount
= count($fieldKeys);
413 foreach ($fieldKeys as $key) {
414 if (empty($this->_fields
[$key])) {
415 $this->_activeFields
[] = new CRM_Contact_Import_Field('', ts('- do not import -'));
418 $this->_activeFields
[] = clone($this->_fields
[$key]);
426 function setActiveFieldLocationTypes($elements) {
427 for ($i = 0; $i < count($elements); $i++
) {
428 $this->_activeFields
[$i]->_hasLocationType
= $elements[$i];
438 function setActiveFieldPhoneTypes($elements) {
439 for ($i = 0; $i < count($elements); $i++
) {
440 $this->_activeFields
[$i]->_phoneType
= $elements[$i];
447 function setActiveFieldWebsiteTypes($elements) {
448 for ($i = 0; $i < count($elements); $i++
) {
449 $this->_activeFields
[$i]->_websiteType
= $elements[$i];
454 * Function to set IM Service Provider type fields
456 * @param array $elements IM service provider type ids
461 function setActiveFieldImProviders($elements) {
462 for ($i = 0; $i < count($elements); $i++
) {
463 $this->_activeFields
[$i]->_imProvider
= $elements[$i];
470 function setActiveFieldRelated($elements) {
471 for ($i = 0; $i < count($elements); $i++
) {
472 $this->_activeFields
[$i]->_related
= $elements[$i];
479 function setActiveFieldRelatedContactType($elements) {
480 for ($i = 0; $i < count($elements); $i++
) {
481 $this->_activeFields
[$i]->_relatedContactType
= $elements[$i];
488 function setActiveFieldRelatedContactDetails($elements) {
489 for ($i = 0; $i < count($elements); $i++
) {
490 $this->_activeFields
[$i]->_relatedContactDetails
= $elements[$i];
497 function setActiveFieldRelatedContactLocType($elements) {
498 for ($i = 0; $i < count($elements); $i++
) {
499 $this->_activeFields
[$i]->_relatedContactLocType
= $elements[$i];
506 function setActiveFieldRelatedContactPhoneType($elements) {
507 for ($i = 0; $i < count($elements); $i++
) {
508 $this->_activeFields
[$i]->_relatedContactPhoneType
= $elements[$i];
515 function setActiveFieldRelatedContactWebsiteType($elements) {
516 for ($i = 0; $i < count($elements); $i++
) {
517 $this->_activeFields
[$i]->_relatedContactWebsiteType
= $elements[$i];
522 * Function to set IM Service Provider type fields for related contacts
524 * @param array $elements IM service provider type ids of related contact
529 function setActiveFieldRelatedContactImProvider($elements) {
530 for ($i = 0; $i < count($elements); $i++
) {
531 $this->_activeFields
[$i]->_relatedContactImProvider
= $elements[$i];
536 * function to format the field values for input to the api
538 * @return array (reference ) associative array of name/value pairs
541 function &getActiveFieldParams() {
544 //CRM_Core_Error::debug( 'Count', $this->_activeFieldCount );
545 for ($i = 0; $i < $this->_activeFieldCount
; $i++
) {
546 if ($this->_activeFields
[$i]->_name
== 'do_not_import') {
550 if (isset($this->_activeFields
[$i]->_value
)) {
551 if (isset($this->_activeFields
[$i]->_hasLocationType
)) {
552 if (!isset($params[$this->_activeFields
[$i]->_name
])) {
553 $params[$this->_activeFields
[$i]->_name
] = array();
557 $this->_activeFields
[$i]->_name
=>
558 $this->_activeFields
[$i]->_value
,
559 'location_type_id' =>
560 $this->_activeFields
[$i]->_hasLocationType
,
563 if (isset($this->_activeFields
[$i]->_phoneType
)) {
564 $value['phone_type_id'] = $this->_activeFields
[$i]->_phoneType
;
567 // get IM service Provider type id
568 if (isset($this->_activeFields
[$i]->_imProvider
)) {
569 $value['provider_id'] = $this->_activeFields
[$i]->_imProvider
;
572 $params[$this->_activeFields
[$i]->_name
][] = $value;
574 elseif (isset($this->_activeFields
[$i]->_websiteType
)) {
576 $this->_activeFields
[$i]->_name
=> $this->_activeFields
[$i]->_value
,
577 'website_type_id' => $this->_activeFields
[$i]->_websiteType
,
580 $params[$this->_activeFields
[$i]->_name
][] = $value;
583 if (!isset($params[$this->_activeFields
[$i]->_name
])) {
584 if (!isset($this->_activeFields
[$i]->_related
)) {
585 $params[$this->_activeFields
[$i]->_name
] = $this->_activeFields
[$i]->_value
;
589 //minor fix for CRM-4062
590 if (isset($this->_activeFields
[$i]->_related
)) {
591 if (!isset($params[$this->_activeFields
[$i]->_related
])) {
592 $params[$this->_activeFields
[$i]->_related
] = array();
595 if (!isset($params[$this->_activeFields
[$i]->_related
]['contact_type']) && !empty($this->_activeFields
[$i]->_relatedContactType
)) {
596 $params[$this->_activeFields
[$i]->_related
]['contact_type'] = $this->_activeFields
[$i]->_relatedContactType
;
599 if (isset($this->_activeFields
[$i]->_relatedContactLocType
) && !empty($this->_activeFields
[$i]->_value
)) {
600 if (!is_array($params[$this->_activeFields
[$i]->_related
][$this->_activeFields
[$i]->_relatedContactDetails
])) {
601 $params[$this->_activeFields
[$i]->_related
][$this->_activeFields
[$i]->_relatedContactDetails
] = array();
604 $this->_activeFields
[$i]->_relatedContactDetails
=> $this->_activeFields
[$i]->_value
,
605 'location_type_id' => $this->_activeFields
[$i]->_relatedContactLocType
,
608 if (isset($this->_activeFields
[$i]->_relatedContactPhoneType
)) {
609 $value['phone_type_id'] = $this->_activeFields
[$i]->_relatedContactPhoneType
;
612 // get IM service Provider type id for related contact
613 if (isset($this->_activeFields
[$i]->_relatedContactImProvider
)) {
614 $value['provider_id'] = $this->_activeFields
[$i]->_relatedContactImProvider
;
617 $params[$this->_activeFields
[$i]->_related
][$this->_activeFields
[$i]->_relatedContactDetails
][] = $value;
619 elseif (isset($this->_activeFields
[$i]->_relatedContactWebsiteType
)) {
620 $params[$this->_activeFields
[$i]->_related
][$this->_activeFields
[$i]->_relatedContactDetails
][] = array(
621 'url' => $this->_activeFields
[$i]->_value
,
622 'website_type_id' => $this->_activeFields
[$i]->_relatedContactWebsiteType
,
626 $params[$this->_activeFields
[$i]->_related
][$this->_activeFields
[$i]->_relatedContactDetails
] = $this->_activeFields
[$i]->_value
;
638 function getColumnPatterns() {
640 foreach ($this->_fields
as $name => $field) {
641 $values[$name] = $field->_columnPattern
;
650 * @param string $headerPattern
651 * @param string $dataPattern
652 * @param bool $hasLocationType
654 function addField($name, $title, $type = CRM_Utils_Type
::T_INT
,
655 $headerPattern = '//', $dataPattern = '//',
656 $hasLocationType = FALSE
658 $this->_fields
[$name] = new CRM_Contact_Import_Field($name, $title, $type, $headerPattern, $dataPattern, $hasLocationType);
660 $this->_fields
['doNotImport'] = new CRM_Contact_Import_Field($name, $title, $type, $headerPattern, $dataPattern, $hasLocationType);
665 * Store parser values
667 * @param CRM_Core_Session $store
674 function set($store, $mode = self
::MODE_SUMMARY
) {
675 $store->set('rowCount', $this->_rowCount
);
676 $store->set('fields', $this->getSelectValues());
677 $store->set('fieldTypes', $this->getSelectTypes());
679 $store->set('columnPatterns', $this->getColumnPatterns());
680 $store->set('dataPatterns', $this->getDataPatterns());
681 $store->set('columnCount', $this->_activeFieldCount
);
683 $store->set('totalRowCount', $this->_totalCount
);
684 $store->set('validRowCount', $this->_validCount
);
685 $store->set('invalidRowCount', $this->_invalidRowCount
);
686 $store->set('conflictRowCount', $this->_conflictCount
);
687 $store->set('unMatchCount', $this->_unMatchCount
);
689 switch ($this->_contactType
) {
691 $store->set('contactType', CRM_Import_Parser
::CONTACT_INDIVIDUAL
);
695 $store->set('contactType', CRM_Import_Parser
::CONTACT_HOUSEHOLD
);
699 $store->set('contactType', CRM_Import_Parser
::CONTACT_ORGANIZATION
);
702 if ($this->_invalidRowCount
) {
703 $store->set('errorsFileName', $this->_errorFileName
);
705 if ($this->_conflictCount
) {
706 $store->set('conflictsFileName', $this->_conflictFileName
);
708 if (isset($this->_rows
) && !empty($this->_rows
)) {
709 $store->set('dataValues', $this->_rows
);
712 if ($this->_unMatchCount
) {
713 $store->set('mismatchFileName', $this->_misMatchFilemName
);
716 if ($mode == self
::MODE_IMPORT
) {
717 $store->set('duplicateRowCount', $this->_duplicateCount
);
718 $store->set('unparsedAddressCount', $this->_unparsedAddressCount
);
719 if ($this->_duplicateCount
) {
720 $store->set('duplicatesFileName', $this->_duplicateFileName
);
722 if ($this->_unparsedAddressCount
) {
723 $store->set('errorsFileName', $this->_errorFileName
);
726 //echo "$this->_totalCount,$this->_invalidRowCount,$this->_conflictCount,$this->_duplicateCount";
730 * Export data to a CSV file
733 * @param array $header
736 * @internal param string $filename
740 static function exportCSV($fileName, $header, $data) {
742 if (file_exists($fileName) && !is_writable($fileName)) {
743 CRM_Core_Error
::movedSiteError($fileName);
745 //hack to remove '_status', '_statusMsg' and '_id' from error file
746 $errorValues = array();
747 $dbRecordStatus = array('IMPORTED', 'ERROR', 'DUPLICATE', 'INVALID', 'NEW');
748 foreach ($data as $rowCount => $rowValues) {
750 foreach ($rowValues as $key => $val) {
751 if (in_array($val, $dbRecordStatus) && $count == (count($rowValues) - 3)) {
754 $errorValues[$rowCount][$key] = $val;
758 $data = $errorValues;
761 $fd = fopen($fileName, 'w');
763 foreach ($header as $key => $value) {
764 $header[$key] = "\"$value\"";
766 $config = CRM_Core_Config
::singleton();
767 $output[] = implode($config->fieldSeparator
, $header);
769 foreach ($data as $datum) {
770 foreach ($datum as $key => $value) {
771 $datum[$key] = "\"$value\"";
773 $output[] = implode($config->fieldSeparator
, $datum);
775 fwrite($fd, implode("\n", $output));
780 * Update the record with PK $id in the import database table
783 * @param array $params
788 public function updateImportRecord($id, &$params) {
789 $statusFieldName = $this->_statusFieldName
;
790 $primaryKeyName = $this->_primaryKeyName
;
792 if ($statusFieldName && $primaryKeyName) {
793 $dao = new CRM_Core_DAO();
794 $db = $dao->getDatabaseConnection();
796 $query = "UPDATE $this->_tableName
797 SET $statusFieldName = ?,
798 ${statusFieldName}Msg = ?
799 WHERE $primaryKeyName = ?";
801 $params[$statusFieldName],
802 CRM_Utils_Array
::value("${statusFieldName}Msg", $params),
806 //print "Running query: $query<br/>With arguments: ".$params[$statusFieldName].", ".$params["${statusFieldName}Msg"].", $id<br/>";
808 $db->query($query, $args);