3 +--------------------------------------------------------------------+
4 | CiviCRM version 4.6 |
5 +--------------------------------------------------------------------+
6 | Copyright CiviCRM LLC (c) 2004-2014 |
7 +--------------------------------------------------------------------+
8 | This file is a part of CiviCRM. |
10 | CiviCRM is free software; you can copy, modify, and distribute it |
11 | under the terms of the GNU Affero General Public License |
12 | Version 3, 19 November 2007 and the CiviCRM Licensing Exception. |
14 | CiviCRM is distributed in the hope that it will be useful, but |
15 | WITHOUT ANY WARRANTY; without even the implied warranty of |
16 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. |
17 | See the GNU Affero General Public License for more details. |
19 | You should have received a copy of the GNU Affero General Public |
20 | License and the CiviCRM Licensing Exception along |
21 | with this program; if not, contact CiviCRM LLC |
22 | at info[AT]civicrm[DOT]org. If you have questions about the |
23 | GNU Affero General Public License or the licensing of CiviCRM, |
24 | see the CiviCRM license FAQ at http://civicrm.org/licensing |
25 +--------------------------------------------------------------------+
31 * @copyright CiviCRM LLC (c) 2004-2014
37 abstract class CRM_Contact_Import_Parser
extends CRM_Import_Parser
{
39 protected $_tableName;
46 * Total number of lines in file
51 * Running total number of un matched Conact
53 protected $_unMatchCount;
56 * Array of unmatched lines
61 * Total number of contacts with unparsed addresses
63 protected $_unparsedAddressCount;
66 * Filename of mismatch data
70 protected $_misMatchFilemName;
72 protected $_primaryKeyName;
73 protected $_statusFieldName;
83 * Dedupe rule group id to use if set
87 public $_dedupeRuleGroupID = NULL;
90 * @param string $tableName
93 * @param int $contactType
94 * @param string $primaryKeyName
95 * @param string $statusFieldName
96 * @param int $onDuplicate
97 * @param int $statusID
98 * @param null $totalRowCount
99 * @param bool $doGeocodeAddress
100 * @param int $timeout
101 * @param null $contactSubType
102 * @param int $dedupeRuleGroupID
106 function run($tableName,
108 $mode = self
::MODE_PREVIEW
,
109 $contactType = self
::CONTACT_INDIVIDUAL
,
110 $primaryKeyName = '_id',
111 $statusFieldName = '_status',
112 $onDuplicate = self
::DUPLICATE_SKIP
,
114 $totalRowCount = NULL,
115 $doGeocodeAddress = FALSE,
116 $timeout = CRM_Contact_Import_Parser
::DEFAULT_TIMEOUT
,
117 $contactSubType = NULL,
118 $dedupeRuleGroupID = NULL
121 // TODO: Make the timeout actually work
122 $this->_onDuplicate
= $onDuplicate;
123 $this->_dedupeRuleGroupID
= $dedupeRuleGroupID;
125 switch ($contactType) {
126 case CRM_Import_Parser
::CONTACT_INDIVIDUAL
:
127 $this->_contactType
= 'Individual';
130 case CRM_Import_Parser
::CONTACT_HOUSEHOLD
:
131 $this->_contactType
= 'Household';
134 case CRM_Import_Parser
::CONTACT_ORGANIZATION
:
135 $this->_contactType
= 'Organization';
138 $this->_contactSubType
= $contactSubType;
142 $this->_rowCount
= $this->_warningCount
= 0;
143 $this->_invalidRowCount
= $this->_validCount
= 0;
144 $this->_totalCount
= $this->_conflictCount
= 0;
146 $this->_errors
= array();
147 $this->_warnings
= array();
148 $this->_conflicts
= array();
149 $this->_unparsedAddresses
= array();
153 $this->_tableName
= $tableName;
154 $this->_primaryKeyName
= $primaryKeyName;
155 $this->_statusFieldName
= $statusFieldName;
157 if ($mode == self
::MODE_MAPFIELD
) {
158 $this->_rows
= array();
161 $this->_activeFieldCount
= count($this->_activeFields
);
164 if ($mode == self
::MODE_IMPORT
) {
165 //get the key of email field
166 foreach ($mapper as $key => $value) {
167 if (strtolower($value) == 'email') {
177 $config = CRM_Core_Config
::singleton();
178 $statusFile = "{$config->uploadDir}status_{$statusID}.txt";
179 $status = "<div class='description'> " . ts('No processing status reported yet.') . "</div>";
181 //do not force the browser to display the save dialog, CRM-7640
182 $contents = json_encode(array(0, $status));
184 file_put_contents($statusFile, $contents);
186 $startTimestamp = $currTimestamp = $prevTimestamp = time();
189 // get the contents of the temp. import table
190 $query = "SELECT * FROM $tableName";
191 if ($mode == self
::MODE_IMPORT
) {
192 $query .= " WHERE $statusFieldName = 'NEW'";
194 $dao = new CRM_Core_DAO();
195 $db = $dao->getDatabaseConnection();
196 $result = $db->query($query);
198 while ($values = $result->fetchRow(DB_FETCHMODE_ORDERED
)) {
201 /* trim whitespace around the values */
204 foreach ($values as $k => $v) {
205 $values[$k] = trim($v, " \t\r\n");
207 if (CRM_Utils_System
::isNull($values)) {
211 $this->_totalCount++
;
213 if ($mode == self
::MODE_MAPFIELD
) {
214 $returnCode = $this->mapField($values);
216 elseif ($mode == self
::MODE_PREVIEW
) {
217 $returnCode = $this->preview($values);
219 elseif ($mode == self
::MODE_SUMMARY
) {
220 $returnCode = $this->summary($values);
222 elseif ($mode == self
::MODE_IMPORT
) {
223 //print "Running parser in import mode<br/>\n";
224 $returnCode = $this->import($onDuplicate, $values, $doGeocodeAddress);
225 if ($statusID && (($this->_rowCount %
$skip) == 0)) {
226 $currTimestamp = time();
227 $totalTime = ($currTimestamp - $startTimestamp);
228 $time = ($currTimestamp - $prevTimestamp);
229 $recordsLeft = $totalRowCount - $this->_rowCount
;
230 if ($recordsLeft < 0) {
233 $estimatedTime = ($recordsLeft / $skip) * $time;
234 $estMinutes = floor($estimatedTime / 60);
236 if ($estMinutes > 1) {
237 $timeFormatted = $estMinutes . ' ' . ts('minutes') . ' ';
238 $estimatedTime = $estimatedTime - ($estMinutes * 60);
240 $timeFormatted .= round($estimatedTime) . ' ' . ts('seconds');
241 $processedPercent = (int ) (($this->_rowCount
* 100) / $totalRowCount);
242 $statusMsg = ts('%1 of %2 records - %3 remaining',
243 array(1 => $this->_rowCount
, 2 => $totalRowCount, 3 => $timeFormatted)
246 <div class=\"description\">
247 <strong>{$statusMsg}</strong>
251 $contents = json_encode(array($processedPercent, $status));
253 file_put_contents($statusFile, $contents);
255 $prevTimestamp = $currTimestamp;
260 $returnCode = self
::ERROR
;
263 // note that a line could be valid but still produce a warning
264 if ($returnCode & self
::VALID
) {
265 $this->_validCount++
;
266 if ($mode == self
::MODE_MAPFIELD
) {
267 $this->_rows
[] = $values;
268 $this->_activeFieldCount
= max($this->_activeFieldCount
, count($values));
272 if ($returnCode & self
::WARNING
) {
273 $this->_warningCount++
;
274 if ($this->_warningCount
< $this->_maxWarningCount
) {
275 $this->_warningCount
[] = $line;
279 if ($returnCode & self
::ERROR
) {
280 $this->_invalidRowCount++
;
281 if ($this->_invalidRowCount
< $this->_maxErrorCount
) {
282 array_unshift($values, $this->_rowCount
);
283 $this->_errors
[] = $values;
287 if ($returnCode & self
::CONFLICT
) {
288 $this->_conflictCount++
;
289 array_unshift($values, $this->_rowCount
);
290 $this->_conflicts
[] = $values;
293 if ($returnCode & self
::NO_MATCH
) {
294 $this->_unMatchCount++
;
295 array_unshift($values, $this->_rowCount
);
296 $this->_unMatch
[] = $values;
299 if ($returnCode & self
::DUPLICATE
) {
300 if ($returnCode & self
::MULTIPLE_DUPE
) {
301 /* TODO: multi-dupes should be counted apart from singles
302 * on non-skip action */
304 $this->_duplicateCount++
;
305 array_unshift($values, $this->_rowCount
);
306 $this->_duplicates
[] = $values;
307 if ($onDuplicate != self
::DUPLICATE_SKIP
) {
308 $this->_validCount++
;
312 if ($returnCode & self
::UNPARSED_ADDRESS_WARNING
) {
313 $this->_unparsedAddressCount++
;
314 array_unshift($values, $this->_rowCount
);
315 $this->_unparsedAddresses
[] = $values;
317 // we give the derived class a way of aborting the process
318 // note that the return code could be multiple code or'ed together
319 if ($returnCode & self
::STOP
) {
323 // if we are done processing the maxNumber of lines, break
324 if ($this->_maxLinesToProcess
> 0 && $this->_validCount
>= $this->_maxLinesToProcess
) {
328 // clean up memory from dao's
329 CRM_Core_DAO
::freeResult();
331 // see if we've hit our timeout yet
332 /* if ( $the_thing_with_the_stuff ) {
337 if ($mode == self
::MODE_PREVIEW ||
$mode == self
::MODE_IMPORT
) {
338 $customHeaders = $mapper;
340 $customfields = CRM_Core_BAO_CustomField
::getFields($this->_contactType
);
341 foreach ($customHeaders as $key => $value) {
342 if ($id = CRM_Core_BAO_CustomField
::getKeyID($value)) {
343 $customHeaders[$key] = $customfields[$id][0];
347 if ($this->_invalidRowCount
) {
348 // removed view url for invlaid contacts
349 $headers = array_merge(array(ts('Line Number'),
354 $this->_errorFileName
= self
::errorFileName(self
::ERROR
);
355 self
::exportCSV($this->_errorFileName
, $headers, $this->_errors
);
357 if ($this->_conflictCount
) {
358 $headers = array_merge(array(ts('Line Number'),
363 $this->_conflictFileName
= self
::errorFileName(self
::CONFLICT
);
364 self
::exportCSV($this->_conflictFileName
, $headers, $this->_conflicts
);
366 if ($this->_duplicateCount
) {
367 $headers = array_merge(array(ts('Line Number'),
368 ts('View Contact URL'),
373 $this->_duplicateFileName
= self
::errorFileName(self
::DUPLICATE
);
374 self
::exportCSV($this->_duplicateFileName
, $headers, $this->_duplicates
);
376 if ($this->_unMatchCount
) {
377 $headers = array_merge(array(ts('Line Number'),
383 $this->_misMatchFilemName
= self
::errorFileName(self
::NO_MATCH
);
384 self
::exportCSV($this->_misMatchFilemName
, $headers, $this->_unMatch
);
386 if ($this->_unparsedAddressCount
) {
387 $headers = array_merge(array(ts('Line Number'),
388 ts('Contact Edit URL'),
392 $this->_errorFileName
= self
::errorFileName(self
::UNPARSED_ADDRESS_WARNING
);
393 self
::exportCSV($this->_errorFileName
, $headers, $this->_unparsedAddresses
);
396 //echo "$this->_totalCount,$this->_invalidRowCount,$this->_conflictCount,$this->_duplicateCount";
397 return $this->fini();
401 * Given a list of the importable field keys that the user has selected
402 * set the active fields array to this list
404 * @param array mapped array of values
408 public function setActiveFields($fieldKeys) {
409 $this->_activeFieldCount
= count($fieldKeys);
410 foreach ($fieldKeys as $key) {
411 if (empty($this->_fields
[$key])) {
412 $this->_activeFields
[] = new CRM_Contact_Import_Field('', ts('- do not import -'));
415 $this->_activeFields
[] = clone($this->_fields
[$key]);
423 public function setActiveFieldLocationTypes($elements) {
424 for ($i = 0; $i < count($elements); $i++
) {
425 $this->_activeFields
[$i]->_hasLocationType
= $elements[$i];
435 public function setActiveFieldPhoneTypes($elements) {
436 for ($i = 0; $i < count($elements); $i++
) {
437 $this->_activeFields
[$i]->_phoneType
= $elements[$i];
444 public function setActiveFieldWebsiteTypes($elements) {
445 for ($i = 0; $i < count($elements); $i++
) {
446 $this->_activeFields
[$i]->_websiteType
= $elements[$i];
451 * Set IM Service Provider type fields
453 * @param array $elements
454 * IM service provider type ids.
458 public function setActiveFieldImProviders($elements) {
459 for ($i = 0; $i < count($elements); $i++
) {
460 $this->_activeFields
[$i]->_imProvider
= $elements[$i];
467 public function setActiveFieldRelated($elements) {
468 for ($i = 0; $i < count($elements); $i++
) {
469 $this->_activeFields
[$i]->_related
= $elements[$i];
476 public function setActiveFieldRelatedContactType($elements) {
477 for ($i = 0; $i < count($elements); $i++
) {
478 $this->_activeFields
[$i]->_relatedContactType
= $elements[$i];
485 public function setActiveFieldRelatedContactDetails($elements) {
486 for ($i = 0; $i < count($elements); $i++
) {
487 $this->_activeFields
[$i]->_relatedContactDetails
= $elements[$i];
494 public function setActiveFieldRelatedContactLocType($elements) {
495 for ($i = 0; $i < count($elements); $i++
) {
496 $this->_activeFields
[$i]->_relatedContactLocType
= $elements[$i];
503 public function setActiveFieldRelatedContactPhoneType($elements) {
504 for ($i = 0; $i < count($elements); $i++
) {
505 $this->_activeFields
[$i]->_relatedContactPhoneType
= $elements[$i];
512 public function setActiveFieldRelatedContactWebsiteType($elements) {
513 for ($i = 0; $i < count($elements); $i++
) {
514 $this->_activeFields
[$i]->_relatedContactWebsiteType
= $elements[$i];
519 * Set IM Service Provider type fields for related contacts
521 * @param array $elements
522 * IM service provider type ids of related contact.
526 public function setActiveFieldRelatedContactImProvider($elements) {
527 for ($i = 0; $i < count($elements); $i++
) {
528 $this->_activeFields
[$i]->_relatedContactImProvider
= $elements[$i];
533 * Format the field values for input to the api
535 * @return array (reference ) associative array of name/value pairs
537 public function &getActiveFieldParams() {
540 for ($i = 0; $i < $this->_activeFieldCount
; $i++
) {
541 if ($this->_activeFields
[$i]->_name
== 'do_not_import') {
545 if (isset($this->_activeFields
[$i]->_value
)) {
546 if (isset($this->_activeFields
[$i]->_hasLocationType
)) {
547 if (!isset($params[$this->_activeFields
[$i]->_name
])) {
548 $params[$this->_activeFields
[$i]->_name
] = array();
552 $this->_activeFields
[$i]->_name
=>
553 $this->_activeFields
[$i]->_value
,
554 'location_type_id' =>
555 $this->_activeFields
[$i]->_hasLocationType
,
558 if (isset($this->_activeFields
[$i]->_phoneType
)) {
559 $value['phone_type_id'] = $this->_activeFields
[$i]->_phoneType
;
562 // get IM service Provider type id
563 if (isset($this->_activeFields
[$i]->_imProvider
)) {
564 $value['provider_id'] = $this->_activeFields
[$i]->_imProvider
;
567 $params[$this->_activeFields
[$i]->_name
][] = $value;
569 elseif (isset($this->_activeFields
[$i]->_websiteType
)) {
571 $this->_activeFields
[$i]->_name
=> $this->_activeFields
[$i]->_value
,
572 'website_type_id' => $this->_activeFields
[$i]->_websiteType
,
575 $params[$this->_activeFields
[$i]->_name
][] = $value;
578 if (!isset($params[$this->_activeFields
[$i]->_name
])) {
579 if (!isset($this->_activeFields
[$i]->_related
)) {
580 $params[$this->_activeFields
[$i]->_name
] = $this->_activeFields
[$i]->_value
;
584 //minor fix for CRM-4062
585 if (isset($this->_activeFields
[$i]->_related
)) {
586 if (!isset($params[$this->_activeFields
[$i]->_related
])) {
587 $params[$this->_activeFields
[$i]->_related
] = array();
590 if (!isset($params[$this->_activeFields
[$i]->_related
]['contact_type']) && !empty($this->_activeFields
[$i]->_relatedContactType
)) {
591 $params[$this->_activeFields
[$i]->_related
]['contact_type'] = $this->_activeFields
[$i]->_relatedContactType
;
594 if (isset($this->_activeFields
[$i]->_relatedContactLocType
) && !empty($this->_activeFields
[$i]->_value
)) {
595 if (!empty($params[$this->_activeFields
[$i]->_related
][$this->_activeFields
[$i]->_relatedContactDetails
]) &&
596 !is_array($params[$this->_activeFields
[$i]->_related
][$this->_activeFields
[$i]->_relatedContactDetails
])) {
597 $params[$this->_activeFields
[$i]->_related
][$this->_activeFields
[$i]->_relatedContactDetails
] = array();
600 $this->_activeFields
[$i]->_relatedContactDetails
=> $this->_activeFields
[$i]->_value
,
601 'location_type_id' => $this->_activeFields
[$i]->_relatedContactLocType
,
604 if (isset($this->_activeFields
[$i]->_relatedContactPhoneType
)) {
605 $value['phone_type_id'] = $this->_activeFields
[$i]->_relatedContactPhoneType
;
608 // get IM service Provider type id for related contact
609 if (isset($this->_activeFields
[$i]->_relatedContactImProvider
)) {
610 $value['provider_id'] = $this->_activeFields
[$i]->_relatedContactImProvider
;
613 $params[$this->_activeFields
[$i]->_related
][$this->_activeFields
[$i]->_relatedContactDetails
][] = $value;
615 elseif (isset($this->_activeFields
[$i]->_relatedContactWebsiteType
)) {
616 $params[$this->_activeFields
[$i]->_related
][$this->_activeFields
[$i]->_relatedContactDetails
][] = array(
617 'url' => $this->_activeFields
[$i]->_value
,
618 'website_type_id' => $this->_activeFields
[$i]->_relatedContactWebsiteType
,
622 $params[$this->_activeFields
[$i]->_related
][$this->_activeFields
[$i]->_relatedContactDetails
] = $this->_activeFields
[$i]->_value
;
634 public function getColumnPatterns() {
636 foreach ($this->_fields
as $name => $field) {
637 $values[$name] = $field->_columnPattern
;
643 * @param string $name
646 * @param string $headerPattern
647 * @param string $dataPattern
648 * @param bool $hasLocationType
650 function addField($name, $title, $type = CRM_Utils_Type
::T_INT
,
651 $headerPattern = '//', $dataPattern = '//',
652 $hasLocationType = FALSE
654 $this->_fields
[$name] = new CRM_Contact_Import_Field($name, $title, $type, $headerPattern, $dataPattern, $hasLocationType);
656 $this->_fields
['doNotImport'] = new CRM_Contact_Import_Field($name, $title, $type, $headerPattern, $dataPattern, $hasLocationType);
661 * Store parser values
663 * @param CRM_Core_Session $store
669 public function set($store, $mode = self
::MODE_SUMMARY
) {
670 $store->set('rowCount', $this->_rowCount
);
671 $store->set('fields', $this->getSelectValues());
672 $store->set('fieldTypes', $this->getSelectTypes());
674 $store->set('columnPatterns', $this->getColumnPatterns());
675 $store->set('dataPatterns', $this->getDataPatterns());
676 $store->set('columnCount', $this->_activeFieldCount
);
678 $store->set('totalRowCount', $this->_totalCount
);
679 $store->set('validRowCount', $this->_validCount
);
680 $store->set('invalidRowCount', $this->_invalidRowCount
);
681 $store->set('conflictRowCount', $this->_conflictCount
);
682 $store->set('unMatchCount', $this->_unMatchCount
);
684 switch ($this->_contactType
) {
686 $store->set('contactType', CRM_Import_Parser
::CONTACT_INDIVIDUAL
);
690 $store->set('contactType', CRM_Import_Parser
::CONTACT_HOUSEHOLD
);
694 $store->set('contactType', CRM_Import_Parser
::CONTACT_ORGANIZATION
);
697 if ($this->_invalidRowCount
) {
698 $store->set('errorsFileName', $this->_errorFileName
);
700 if ($this->_conflictCount
) {
701 $store->set('conflictsFileName', $this->_conflictFileName
);
703 if (isset($this->_rows
) && !empty($this->_rows
)) {
704 $store->set('dataValues', $this->_rows
);
707 if ($this->_unMatchCount
) {
708 $store->set('mismatchFileName', $this->_misMatchFilemName
);
711 if ($mode == self
::MODE_IMPORT
) {
712 $store->set('duplicateRowCount', $this->_duplicateCount
);
713 $store->set('unparsedAddressCount', $this->_unparsedAddressCount
);
714 if ($this->_duplicateCount
) {
715 $store->set('duplicatesFileName', $this->_duplicateFileName
);
717 if ($this->_unparsedAddressCount
) {
718 $store->set('errorsFileName', $this->_errorFileName
);
721 //echo "$this->_totalCount,$this->_invalidRowCount,$this->_conflictCount,$this->_duplicateCount";
725 * Export data to a CSV file
727 * @param string $fileName
728 * @param array $header
733 public static function exportCSV($fileName, $header, $data) {
735 if (file_exists($fileName) && !is_writable($fileName)) {
736 CRM_Core_Error
::movedSiteError($fileName);
738 //hack to remove '_status', '_statusMsg' and '_id' from error file
739 $errorValues = array();
740 $dbRecordStatus = array('IMPORTED', 'ERROR', 'DUPLICATE', 'INVALID', 'NEW');
741 foreach ($data as $rowCount => $rowValues) {
743 foreach ($rowValues as $key => $val) {
744 if (in_array($val, $dbRecordStatus) && $count == (count($rowValues) - 3)) {
747 $errorValues[$rowCount][$key] = $val;
751 $data = $errorValues;
754 $fd = fopen($fileName, 'w');
756 foreach ($header as $key => $value) {
757 $header[$key] = "\"$value\"";
759 $config = CRM_Core_Config
::singleton();
760 $output[] = implode($config->fieldSeparator
, $header);
762 foreach ($data as $datum) {
763 foreach ($datum as $key => $value) {
764 $datum[$key] = "\"$value\"";
766 $output[] = implode($config->fieldSeparator
, $datum);
768 fwrite($fd, implode("\n", $output));
773 * Update the record with PK $id in the import database table
776 * @param array $params
780 public function updateImportRecord($id, &$params) {
781 $statusFieldName = $this->_statusFieldName
;
782 $primaryKeyName = $this->_primaryKeyName
;
784 if ($statusFieldName && $primaryKeyName) {
785 $dao = new CRM_Core_DAO();
786 $db = $dao->getDatabaseConnection();
788 $query = "UPDATE $this->_tableName
789 SET $statusFieldName = ?,
790 ${statusFieldName}Msg = ?
791 WHERE $primaryKeyName = ?";
793 $params[$statusFieldName],
794 CRM_Utils_Array
::value("${statusFieldName}Msg", $params),
798 //print "Running query: $query<br/>With arguments: ".$params[$statusFieldName].", ".$params["${statusFieldName}Msg"].", $id<br/>";
800 $db->query($query, $args);