3 +--------------------------------------------------------------------+
4 | CiviCRM version 4.4 |
5 +--------------------------------------------------------------------+
6 | Copyright CiviCRM LLC (c) 2004-2013 |
7 +--------------------------------------------------------------------+
8 | This file is a part of CiviCRM. |
10 | CiviCRM is free software; you can copy, modify, and distribute it |
11 | under the terms of the GNU Affero General Public License |
12 | Version 3, 19 November 2007 and the CiviCRM Licensing Exception. |
14 | CiviCRM is distributed in the hope that it will be useful, but |
15 | WITHOUT ANY WARRANTY; without even the implied warranty of |
16 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. |
17 | See the GNU Affero General Public License for more details. |
19 | You should have received a copy of the GNU Affero General Public |
20 | License and the CiviCRM Licensing Exception along |
21 | with this program; if not, contact CiviCRM LLC |
22 | at info[AT]civicrm[DOT]org. If you have questions about the |
23 | GNU Affero General Public License or the licensing of CiviCRM, |
24 | see the CiviCRM license FAQ at http://civicrm.org/licensing |
25 +--------------------------------------------------------------------+
31 * @copyright CiviCRM LLC (c) 2004-2013
37 abstract class CRM_Contact_Import_Parser
extends CRM_Import_Parser
{
39 protected $_tableName;
47 * total number of lines in file
52 * running total number of un matched Conact
54 protected $_unMatchCount;
57 * array of unmatched lines
62 * total number of contacts with unparsed addresses
64 protected $_unparsedAddressCount;
67 * filename of mismatch data
71 protected $_misMatchFilemName;
73 protected $_primaryKeyName;
74 protected $_statusFieldName;
84 * dedupe rule group id to use if set
88 public $_dedupeRuleGroupID = NULL;
90 function run($tableName,
92 $mode = self
::MODE_PREVIEW
,
93 $contactType = self
::CONTACT_INDIVIDUAL
,
94 $primaryKeyName = '_id',
95 $statusFieldName = '_status',
96 $onDuplicate = self
::DUPLICATE_SKIP
,
98 $totalRowCount = NULL,
99 $doGeocodeAddress = FALSE,
100 $timeout = CRM_Contact_Import_Parser
::DEFAULT_TIMEOUT
,
101 $contactSubType = NULL,
102 $dedupeRuleGroupID = NULL
105 // TODO: Make the timeout actually work
106 $this->_onDuplicate
= $onDuplicate;
107 $this->_dedupeRuleGroupID
= $dedupeRuleGroupID;
109 switch ($contactType) {
110 case CRM_Import_Parser
::CONTACT_INDIVIDUAL
:
111 $this->_contactType
= 'Individual';
114 case CRM_Import_Parser
::CONTACT_HOUSEHOLD
:
115 $this->_contactType
= 'Household';
118 case CRM_Import_Parser
::CONTACT_ORGANIZATION
:
119 $this->_contactType
= 'Organization';
122 $this->_contactSubType
= $contactSubType;
126 $this->_rowCount
= $this->_warningCount
= 0;
127 $this->_invalidRowCount
= $this->_validCount
= 0;
128 $this->_totalCount
= $this->_conflictCount
= 0;
130 $this->_errors
= array();
131 $this->_warnings
= array();
132 $this->_conflicts
= array();
133 $this->_unparsedAddresses
= array();
137 $this->_tableName
= $tableName;
138 $this->_primaryKeyName
= $primaryKeyName;
139 $this->_statusFieldName
= $statusFieldName;
141 if ($mode == self
::MODE_MAPFIELD
) {
142 $this->_rows
= array();
145 $this->_activeFieldCount
= count($this->_activeFields
);
148 if ($mode == self
::MODE_IMPORT
) {
149 //get the key of email field
150 foreach ($mapper as $key => $value) {
151 if (strtolower($value) == 'email') {
161 $config = CRM_Core_Config
::singleton();
162 $statusFile = "{$config->uploadDir}status_{$statusID}.txt";
163 $status = "<div class='description'> " . ts('No processing status reported yet.') . "</div>";
165 //do not force the browser to display the save dialog, CRM-7640
166 $contents = json_encode(array(0, $status));
168 file_put_contents($statusFile, $contents);
170 $startTimestamp = $currTimestamp = $prevTimestamp = time();
173 // get the contents of the temp. import table
174 $query = "SELECT * FROM $tableName";
175 if ($mode == self
::MODE_IMPORT
) {
176 $query .= " WHERE $statusFieldName = 'NEW'";
178 $dao = new CRM_Core_DAO();
179 $db = $dao->getDatabaseConnection();
180 $result = $db->query($query);
182 while ($values = $result->fetchRow(DB_FETCHMODE_ORDERED
)) {
185 /* trim whitespace around the values */
188 foreach ($values as $k => $v) {
189 $values[$k] = trim($v, " \t\r\n");
191 if (CRM_Utils_System
::isNull($values)) {
195 $this->_totalCount++
;
197 if ($mode == self
::MODE_MAPFIELD
) {
198 $returnCode = $this->mapField($values);
200 elseif ($mode == self
::MODE_PREVIEW
) {
201 $returnCode = $this->preview($values);
203 elseif ($mode == self
::MODE_SUMMARY
) {
204 $returnCode = $this->summary($values);
206 elseif ($mode == self
::MODE_IMPORT
) {
207 //print "Running parser in import mode<br/>\n";
208 $returnCode = $this->import($onDuplicate, $values, $doGeocodeAddress);
209 if ($statusID && (($this->_rowCount %
$skip) == 0)) {
210 $currTimestamp = time();
211 $totalTime = ($currTimestamp - $startTimestamp);
212 $time = ($currTimestamp - $prevTimestamp);
213 $recordsLeft = $totalRowCount - $this->_rowCount
;
214 if ($recordsLeft < 0) {
217 $estimatedTime = ($recordsLeft / $skip) * $time;
218 $estMinutes = floor($estimatedTime / 60);
220 if ($estMinutes > 1) {
221 $timeFormatted = $estMinutes . ' ' . ts('minutes') . ' ';
222 $estimatedTime = $estimatedTime - ($estMinutes * 60);
224 $timeFormatted .= round($estimatedTime) . ' ' . ts('seconds');
225 $processedPercent = (int )(($this->_rowCount
* 100) / $totalRowCount);
226 $statusMsg = ts('%1 of %2 records - %3 remaining',
227 array(1 => $this->_rowCount
, 2 => $totalRowCount, 3 => $timeFormatted)
230 <div class=\"description\">
231 <strong>{$statusMsg}</strong>
235 $contents = json_encode (array($processedPercent, $status));
237 file_put_contents($statusFile, $contents);
239 $prevTimestamp = $currTimestamp;
244 $returnCode = self
::ERROR
;
247 // note that a line could be valid but still produce a warning
248 if ($returnCode & self
::VALID
) {
249 $this->_validCount++
;
250 if ($mode == self
::MODE_MAPFIELD
) {
251 $this->_rows
[] = $values;
252 $this->_activeFieldCount
= max($this->_activeFieldCount
, count($values));
256 if ($returnCode & self
::WARNING
) {
257 $this->_warningCount++
;
258 if ($this->_warningCount
< $this->_maxWarningCount
) {
259 $this->_warningCount
[] = $line;
263 if ($returnCode & self
::ERROR
) {
264 $this->_invalidRowCount++
;
265 if ($this->_invalidRowCount
< $this->_maxErrorCount
) {
266 array_unshift($values, $this->_rowCount
);
267 $this->_errors
[] = $values;
271 if ($returnCode & self
::CONFLICT
) {
272 $this->_conflictCount++
;
273 array_unshift($values, $this->_rowCount
);
274 $this->_conflicts
[] = $values;
277 if ($returnCode & self
::NO_MATCH
) {
278 $this->_unMatchCount++
;
279 array_unshift($values, $this->_rowCount
);
280 $this->_unMatch
[] = $values;
283 if ($returnCode & self
::DUPLICATE
) {
284 if ($returnCode & self
::MULTIPLE_DUPE
) {
285 /* TODO: multi-dupes should be counted apart from singles
286 * on non-skip action */
288 $this->_duplicateCount++
;
289 array_unshift($values, $this->_rowCount
);
290 $this->_duplicates
[] = $values;
291 if ($onDuplicate != self
::DUPLICATE_SKIP
) {
292 $this->_validCount++
;
296 if ($returnCode & self
::UNPARSED_ADDRESS_WARNING
) {
297 $this->_unparsedAddressCount++
;
298 array_unshift($values, $this->_rowCount
);
299 $this->_unparsedAddresses
[] = $values;
301 // we give the derived class a way of aborting the process
302 // note that the return code could be multiple code or'ed together
303 if ($returnCode & self
::STOP
) {
307 // if we are done processing the maxNumber of lines, break
308 if ($this->_maxLinesToProcess
> 0 && $this->_validCount
>= $this->_maxLinesToProcess
) {
312 // clean up memory from dao's
313 CRM_Core_DAO
::freeResult();
315 // see if we've hit our timeout yet
316 /* if ( $the_thing_with_the_stuff ) {
322 if ($mode == self
::MODE_PREVIEW ||
$mode == self
::MODE_IMPORT
) {
323 $customHeaders = $mapper;
325 $customfields = CRM_Core_BAO_CustomField
::getFields($this->_contactType
);
326 foreach ($customHeaders as $key => $value) {
327 if ($id = CRM_Core_BAO_CustomField
::getKeyID($value)) {
328 $customHeaders[$key] = $customfields[$id][0];
332 if ($this->_invalidRowCount
) {
333 // removed view url for invlaid contacts
334 $headers = array_merge(array(ts('Line Number'),
339 $this->_errorFileName
= self
::errorFileName(self
::ERROR
);
340 self
::exportCSV($this->_errorFileName
, $headers, $this->_errors
);
342 if ($this->_conflictCount
) {
343 $headers = array_merge(array(ts('Line Number'),
348 $this->_conflictFileName
= self
::errorFileName(self
::CONFLICT
);
349 self
::exportCSV($this->_conflictFileName
, $headers, $this->_conflicts
);
351 if ($this->_duplicateCount
) {
352 $headers = array_merge(array(ts('Line Number'),
353 ts('View Contact URL'),
358 $this->_duplicateFileName
= self
::errorFileName(self
::DUPLICATE
);
359 self
::exportCSV($this->_duplicateFileName
, $headers, $this->_duplicates
);
361 if ($this->_unMatchCount
) {
362 $headers = array_merge(array(ts('Line Number'),
368 $this->_misMatchFilemName
= self
::errorFileName(self
::NO_MATCH
);
369 self
::exportCSV($this->_misMatchFilemName
, $headers, $this->_unMatch
);
371 if ($this->_unparsedAddressCount
) {
372 $headers = array_merge(array(ts('Line Number'),
373 ts('Contact Edit URL'),
377 $this->_errorFileName
= self
::errorFileName(self
::UNPARSED_ADDRESS_WARNING
);
378 self
::exportCSV($this->_errorFileName
, $headers, $this->_unparsedAddresses
);
381 //echo "$this->_totalCount,$this->_invalidRowCount,$this->_conflictCount,$this->_duplicateCount";
382 return $this->fini();
386 * Given a list of the importable field keys that the user has selected
387 * set the active fields array to this list
389 * @param array mapped array of values
394 function setActiveFields($fieldKeys) {
395 $this->_activeFieldCount
= count($fieldKeys);
396 foreach ($fieldKeys as $key) {
397 if (empty($this->_fields
[$key])) {
398 $this->_activeFields
[] = new CRM_Contact_Import_Field('', ts('- do not import -'));
401 $this->_activeFields
[] = clone($this->_fields
[$key]);
406 function setActiveFieldLocationTypes($elements) {
407 for ($i = 0; $i < count($elements); $i++
) {
408 $this->_activeFields
[$i]->_hasLocationType
= $elements[$i];
412 function setActiveFieldPhoneTypes($elements) {
413 for ($i = 0; $i < count($elements); $i++
) {
414 $this->_activeFields
[$i]->_phoneType
= $elements[$i];
418 function setActiveFieldWebsiteTypes($elements) {
419 for ($i = 0; $i < count($elements); $i++
) {
420 $this->_activeFields
[$i]->_websiteType
= $elements[$i];
425 * Function to set IM Service Provider type fields
427 * @param array $elements IM service provider type ids
432 function setActiveFieldImProviders($elements) {
433 for ($i = 0; $i < count($elements); $i++
) {
434 $this->_activeFields
[$i]->_imProvider
= $elements[$i];
438 function setActiveFieldRelated($elements) {
439 for ($i = 0; $i < count($elements); $i++
) {
440 $this->_activeFields
[$i]->_related
= $elements[$i];
444 function setActiveFieldRelatedContactType($elements) {
445 for ($i = 0; $i < count($elements); $i++
) {
446 $this->_activeFields
[$i]->_relatedContactType
= $elements[$i];
450 function setActiveFieldRelatedContactDetails($elements) {
451 for ($i = 0; $i < count($elements); $i++
) {
452 $this->_activeFields
[$i]->_relatedContactDetails
= $elements[$i];
456 function setActiveFieldRelatedContactLocType($elements) {
457 for ($i = 0; $i < count($elements); $i++
) {
458 $this->_activeFields
[$i]->_relatedContactLocType
= $elements[$i];
462 function setActiveFieldRelatedContactPhoneType($elements) {
463 for ($i = 0; $i < count($elements); $i++
) {
464 $this->_activeFields
[$i]->_relatedContactPhoneType
= $elements[$i];
468 function setActiveFieldRelatedContactWebsiteType($elements) {
469 for ($i = 0; $i < count($elements); $i++
) {
470 $this->_activeFields
[$i]->_relatedContactWebsiteType
= $elements[$i];
475 * Function to set IM Service Provider type fields for related contacts
477 * @param array $elements IM service provider type ids of related contact
482 function setActiveFieldRelatedContactImProvider($elements) {
483 for ($i = 0; $i < count($elements); $i++
) {
484 $this->_activeFields
[$i]->_relatedContactImProvider
= $elements[$i];
489 * function to format the field values for input to the api
491 * @return array (reference ) associative array of name/value pairs
494 function &getActiveFieldParams() {
497 //CRM_Core_Error::debug( 'Count', $this->_activeFieldCount );
498 for ($i = 0; $i < $this->_activeFieldCount
; $i++
) {
499 if ($this->_activeFields
[$i]->_name
== 'do_not_import') {
503 if (isset($this->_activeFields
[$i]->_value
)) {
504 if (isset($this->_activeFields
[$i]->_hasLocationType
)) {
505 if (!isset($params[$this->_activeFields
[$i]->_name
])) {
506 $params[$this->_activeFields
[$i]->_name
] = array();
510 $this->_activeFields
[$i]->_name
=>
511 $this->_activeFields
[$i]->_value
,
512 'location_type_id' =>
513 $this->_activeFields
[$i]->_hasLocationType
,
516 if (isset($this->_activeFields
[$i]->_phoneType
)) {
517 $value['phone_type_id'] = $this->_activeFields
[$i]->_phoneType
;
520 // get IM service Provider type id
521 if (isset($this->_activeFields
[$i]->_imProvider
)) {
522 $value['provider_id'] = $this->_activeFields
[$i]->_imProvider
;
525 $params[$this->_activeFields
[$i]->_name
][] = $value;
527 elseif (isset($this->_activeFields
[$i]->_websiteType
)) {
529 $this->_activeFields
[$i]->_name
=> $this->_activeFields
[$i]->_value
,
530 'website_type_id' => $this->_activeFields
[$i]->_websiteType
,
533 $params[$this->_activeFields
[$i]->_name
][] = $value;
536 if (!isset($params[$this->_activeFields
[$i]->_name
])) {
537 if (!isset($this->_activeFields
[$i]->_related
)) {
538 $params[$this->_activeFields
[$i]->_name
] = $this->_activeFields
[$i]->_value
;
542 //minor fix for CRM-4062
543 if (isset($this->_activeFields
[$i]->_related
)) {
544 if (!isset($params[$this->_activeFields
[$i]->_related
])) {
545 $params[$this->_activeFields
[$i]->_related
] = array();
548 if (!isset($params[$this->_activeFields
[$i]->_related
]['contact_type']) && !empty($this->_activeFields
[$i]->_relatedContactType
)) {
549 $params[$this->_activeFields
[$i]->_related
]['contact_type'] = $this->_activeFields
[$i]->_relatedContactType
;
552 if (isset($this->_activeFields
[$i]->_relatedContactLocType
) && !empty($this->_activeFields
[$i]->_value
)) {
553 if (!is_array($params[$this->_activeFields
[$i]->_related
][$this->_activeFields
[$i]->_relatedContactDetails
])) {
554 $params[$this->_activeFields
[$i]->_related
][$this->_activeFields
[$i]->_relatedContactDetails
] = array();
557 $this->_activeFields
[$i]->_relatedContactDetails
=> $this->_activeFields
[$i]->_value
,
558 'location_type_id' => $this->_activeFields
[$i]->_relatedContactLocType
,
561 if (isset($this->_activeFields
[$i]->_relatedContactPhoneType
)) {
562 $value['phone_type_id'] = $this->_activeFields
[$i]->_relatedContactPhoneType
;
565 // get IM service Provider type id for related contact
566 if (isset($this->_activeFields
[$i]->_relatedContactImProvider
)) {
567 $value['provider_id'] = $this->_activeFields
[$i]->_relatedContactImProvider
;
570 $params[$this->_activeFields
[$i]->_related
][$this->_activeFields
[$i]->_relatedContactDetails
][] = $value;
572 elseif (isset($this->_activeFields
[$i]->_relatedContactWebsiteType
)) {
573 $params[$this->_activeFields
[$i]->_related
][$this->_activeFields
[$i]->_relatedContactDetails
][] = array(
574 'url' => $this->_activeFields
[$i]->_value
,
575 'website_type_id' => $this->_activeFields
[$i]->_relatedContactWebsiteType
,
579 $params[$this->_activeFields
[$i]->_related
][$this->_activeFields
[$i]->_relatedContactDetails
] = $this->_activeFields
[$i]->_value
;
588 function getColumnPatterns() {
590 foreach ($this->_fields
as $name => $field) {
591 $values[$name] = $field->_columnPattern
;
596 function addField($name, $title, $type = CRM_Utils_Type
::T_INT
,
597 $headerPattern = '//', $dataPattern = '//',
598 $hasLocationType = FALSE
600 $this->_fields
[$name] = new CRM_Contact_Import_Field($name, $title, $type, $headerPattern, $dataPattern, $hasLocationType);
602 $this->_fields
['doNotImport'] = new CRM_Contact_Import_Field($name, $title, $type, $headerPattern, $dataPattern, $hasLocationType);
607 * Store parser values
609 * @param CRM_Core_Session $store
614 function set($store, $mode = self
::MODE_SUMMARY
) {
615 $store->set('rowCount', $this->_rowCount
);
616 $store->set('fields', $this->getSelectValues());
617 $store->set('fieldTypes', $this->getSelectTypes());
619 $store->set('columnPatterns', $this->getColumnPatterns());
620 $store->set('dataPatterns', $this->getDataPatterns());
621 $store->set('columnCount', $this->_activeFieldCount
);
623 $store->set('totalRowCount', $this->_totalCount
);
624 $store->set('validRowCount', $this->_validCount
);
625 $store->set('invalidRowCount', $this->_invalidRowCount
);
626 $store->set('conflictRowCount', $this->_conflictCount
);
627 $store->set('unMatchCount', $this->_unMatchCount
);
629 switch ($this->_contactType
) {
631 $store->set('contactType', CRM_Import_Parser
::CONTACT_INDIVIDUAL
);
635 $store->set('contactType', CRM_Import_Parser
::CONTACT_HOUSEHOLD
);
639 $store->set('contactType', CRM_Import_Parser
::CONTACT_ORGANIZATION
);
642 if ($this->_invalidRowCount
) {
643 $store->set('errorsFileName', $this->_errorFileName
);
645 if ($this->_conflictCount
) {
646 $store->set('conflictsFileName', $this->_conflictFileName
);
648 if (isset($this->_rows
) && !empty($this->_rows
)) {
649 $store->set('dataValues', $this->_rows
);
652 if ($this->_unMatchCount
) {
653 $store->set('mismatchFileName', $this->_misMatchFilemName
);
656 if ($mode == self
::MODE_IMPORT
) {
657 $store->set('duplicateRowCount', $this->_duplicateCount
);
658 $store->set('unparsedAddressCount', $this->_unparsedAddressCount
);
659 if ($this->_duplicateCount
) {
660 $store->set('duplicatesFileName', $this->_duplicateFileName
);
662 if ($this->_unparsedAddressCount
) {
663 $store->set('errorsFileName', $this->_errorFileName
);
666 //echo "$this->_totalCount,$this->_invalidRowCount,$this->_conflictCount,$this->_duplicateCount";
670 * Export data to a CSV file
672 * @param string $filename
673 * @param array $header
679 static function exportCSV($fileName, $header, $data) {
681 if (file_exists($fileName) && !is_writable($fileName)) {
682 CRM_Core_Error
::movedSiteError($fileName);
684 //hack to remove '_status', '_statusMsg' and '_id' from error file
685 $errorValues = array();
686 $dbRecordStatus = array('IMPORTED', 'ERROR', 'DUPLICATE', 'INVALID', 'NEW');
687 foreach ($data as $rowCount => $rowValues) {
689 foreach ($rowValues as $key => $val) {
690 if (in_array($val, $dbRecordStatus) && $count == (count($rowValues) - 3)) {
693 $errorValues[$rowCount][$key] = $val;
697 $data = $errorValues;
700 $fd = fopen($fileName, 'w');
702 foreach ($header as $key => $value) {
703 $header[$key] = "\"$value\"";
705 $config = CRM_Core_Config
::singleton();
706 $output[] = implode($config->fieldSeparator
, $header);
708 foreach ($data as $datum) {
709 foreach ($datum as $key => $value) {
710 $datum[$key] = "\"$value\"";
712 $output[] = implode($config->fieldSeparator
, $datum);
714 fwrite($fd, implode("\n", $output));
719 * Update the record with PK $id in the import database table
722 * @param array $params
727 public function updateImportRecord($id, &$params) {
728 $statusFieldName = $this->_statusFieldName
;
729 $primaryKeyName = $this->_primaryKeyName
;
731 if ($statusFieldName && $primaryKeyName) {
732 $dao = new CRM_Core_DAO();
733 $db = $dao->getDatabaseConnection();
735 $query = "UPDATE $this->_tableName
736 SET $statusFieldName = ?,
737 ${statusFieldName}Msg = ?
738 WHERE $primaryKeyName = ?";
740 $params[$statusFieldName],
741 CRM_Utils_Array
::value("${statusFieldName}Msg", $params),
745 //print "Running query: $query<br/>With arguments: ".$params[$statusFieldName].", ".$params["${statusFieldName}Msg"].", $id<br/>";
747 $db->query($query, $args);