3 +--------------------------------------------------------------------+
4 | CiviCRM version 4.3 |
5 +--------------------------------------------------------------------+
6 | Copyright CiviCRM LLC (c) 2004-2013 |
7 +--------------------------------------------------------------------+
8 | This file is a part of CiviCRM. |
10 | CiviCRM is free software; you can copy, modify, and distribute it |
11 | under the terms of the GNU Affero General Public License |
12 | Version 3, 19 November 2007 and the CiviCRM Licensing Exception. |
14 | CiviCRM is distributed in the hope that it will be useful, but |
15 | WITHOUT ANY WARRANTY; without even the implied warranty of |
16 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. |
17 | See the GNU Affero General Public License for more details. |
19 | You should have received a copy of the GNU Affero General Public |
20 | License and the CiviCRM Licensing Exception along |
21 | with this program; if not, contact CiviCRM LLC |
22 | at info[AT]civicrm[DOT]org. If you have questions about the |
23 | GNU Affero General Public License or the licensing of CiviCRM, |
24 | see the CiviCRM license FAQ at http://civicrm.org/licensing |
25 +--------------------------------------------------------------------+
31 * @copyright CiviCRM LLC (c) 2004-2013
39 abstract class CRM_Member_Import_Parser
{
40 CONST MAX_ERRORS
= 250, MAX_WARNINGS
= 25, VALID
= 1, WARNING
= 2, ERROR
= 4, CONFLICT
= 8, STOP
= 16, DUPLICATE
= 32, MULTIPLE_DUPE
= 64, NO_MATCH
= 128;
43 * various parser modes
45 CONST MODE_MAPFIELD
= 1, MODE_PREVIEW
= 2, MODE_SUMMARY
= 4, MODE_IMPORT
= 8;
48 * codes for duplicate record handling
50 CONST DUPLICATE_SKIP
= 1, DUPLICATE_REPLACE
= 2, DUPLICATE_UPDATE
= 4, DUPLICATE_FILL
= 8, DUPLICATE_NOCHECK
= 16;
53 * various Contact types
55 CONST CONTACT_INDIVIDUAL
= 1, CONTACT_HOUSEHOLD
= 2, CONTACT_ORGANIZATION
= 4;
70 * seperator being used
72 protected $_seperator;
75 * total number of lines in file
77 protected $_lineCount;
80 * total number of non empty lines
82 protected $_totalCount;
85 * running total number of valid lines
87 protected $_validCount;
90 * running total number of invalid rows
92 protected $_invalidRowCount;
95 * maximum number of invalid rows to store
97 protected $_maxErrorCount;
100 * array of error lines, bounded by MAX_ERROR
105 * total number of conflict lines
107 protected $_conflictCount;
110 * array of conflict lines
112 protected $_conflicts;
115 * total number of duplicate (from database) lines
117 protected $_duplicateCount;
120 * array of duplicate lines
122 protected $_duplicates;
125 * running total number of warnings
127 protected $_warningCount;
130 * maximum number of warnings to store
132 protected $_maxWarningCount = self
::MAX_WARNINGS
;
135 * array of warning lines, bounded by MAX_WARNING
137 protected $_warnings;
140 * array of all the fields that could potentially be part
141 * of this import process
147 * array of the fields that are actually part of the import process
148 * the position in the array also dictates their position in the import
152 protected $_activeFields;
155 * cache the count of active fields
159 protected $_activeFieldCount;
162 * maximum number of non-empty/comment lines to process
166 protected $_maxLinesToProcess;
169 * cache of preview rows
176 * filename of error data
180 protected $_errorFileName;
183 * filename of conflict data
187 protected $_conflictFileName;
190 * filename of duplicate data
194 protected $_duplicateFileName;
197 * whether the file has a column header or not
201 protected $_haveColumnHeader;
209 public $_contactType;
210 function __construct() {
211 $this->_maxLinesToProcess
= 0;
212 $this->_maxErrorCount
= self
::MAX_ERRORS
;
215 abstract function init();
216 function run($fileName,
219 $skipColumnHeader = FALSE,
220 $mode = self
::MODE_PREVIEW
,
221 $contactType = self
::CONTACT_INDIVIDUAL
,
222 $onDuplicate = self
::DUPLICATE_SKIP
224 if (!is_array($fileName)) {
225 CRM_Core_Error
::fatal();
227 $fileName = $fileName['name'];
229 switch ($contactType) {
230 case self
::CONTACT_INDIVIDUAL
:
231 $this->_contactType
= 'Individual';
234 case self
::CONTACT_HOUSEHOLD
:
235 $this->_contactType
= 'Household';
238 case self
::CONTACT_ORGANIZATION
:
239 $this->_contactType
= 'Organization';
244 $this->_haveColumnHeader
= $skipColumnHeader;
246 $this->_seperator
= $seperator;
248 $fd = fopen($fileName, "r");
253 $this->_lineCount
= $this->_warningCount
= 0;
254 $this->_invalidRowCount
= $this->_validCount
= 0;
255 $this->_totalCount
= $this->_conflictCount
= 0;
257 $this->_errors
= array();
258 $this->_warnings
= array();
259 $this->_conflicts
= array();
261 $this->_fileSize
= number_format(filesize($fileName) / 1024.0, 2);
263 if ($mode == self
::MODE_MAPFIELD
) {
264 $this->_rows
= array();
267 $this->_activeFieldCount
= count($this->_activeFields
);
273 $values = fgetcsv($fd, 8192, $seperator);
278 self
::encloseScrub($values);
280 // skip column header if we're not in mapfield mode
281 if ($mode != self
::MODE_MAPFIELD
&& $skipColumnHeader) {
282 $skipColumnHeader = FALSE;
286 /* trim whitespace around the values */
289 foreach ($values as $k => $v) {
290 $values[$k] = trim($v, " \t\r\n");
292 if (CRM_Utils_System
::isNull($values)) {
296 $this->_totalCount++
;
298 if ($mode == self
::MODE_MAPFIELD
) {
299 $returnCode = $this->mapField($values);
301 elseif ($mode == self
::MODE_PREVIEW
) {
302 $returnCode = $this->preview($values);
304 elseif ($mode == self
::MODE_SUMMARY
) {
305 $returnCode = $this->summary($values);
307 elseif ($mode == self
::MODE_IMPORT
) {
308 $returnCode = $this->import($onDuplicate, $values);
311 $returnCode = self
::ERROR
;
314 // note that a line could be valid but still produce a warning
315 if ($returnCode & self
::VALID
) {
316 $this->_validCount++
;
317 if ($mode == self
::MODE_MAPFIELD
) {
318 $this->_rows
[] = $values;
319 $this->_activeFieldCount
= max($this->_activeFieldCount
, count($values));
323 if ($returnCode & self
::WARNING
) {
324 $this->_warningCount++
;
325 if ($this->_warningCount
< $this->_maxWarningCount
) {
326 $this->_warningCount
[] = $line;
330 if ($returnCode & self
::ERROR
) {
331 $this->_invalidRowCount++
;
332 if ($this->_invalidRowCount
< $this->_maxErrorCount
) {
333 $recordNumber = $this->_lineCount
;
334 array_unshift($values, $recordNumber);
335 $this->_errors
[] = $values;
339 if ($returnCode & self
::CONFLICT
) {
340 $this->_conflictCount++
;
341 $recordNumber = $this->_lineCount
;
342 array_unshift($values, $recordNumber);
343 $this->_conflicts
[] = $values;
346 if ($returnCode & self
::DUPLICATE
) {
347 if ($returnCode & self
::MULTIPLE_DUPE
) {
348 /* TODO: multi-dupes should be counted apart from singles
349 * on non-skip action */
351 $this->_duplicateCount++
;
352 $recordNumber = $this->_lineCount
;
353 array_unshift($values, $recordNumber);
354 $this->_duplicates
[] = $values;
355 if ($onDuplicate != self
::DUPLICATE_SKIP
) {
356 $this->_validCount++
;
360 // we give the derived class a way of aborting the process
361 // note that the return code could be multiple code or'ed together
362 if ($returnCode & self
::STOP
) {
366 // if we are done processing the maxNumber of lines, break
367 if ($this->_maxLinesToProcess
> 0 && $this->_validCount
>= $this->_maxLinesToProcess
) {
374 if ($mode == self
::MODE_PREVIEW ||
$mode == self
::MODE_IMPORT
) {
375 $customHeaders = $mapper;
377 $customfields = CRM_Core_BAO_CustomField
::getFields('Membership');
378 foreach ($customHeaders as $key => $value) {
379 if ($id = CRM_Core_BAO_CustomField
::getKeyID($value)) {
380 $customHeaders[$key] = $customfields[$id][0];
383 if ($this->_invalidRowCount
) {
384 // removed view url for invlaid contacts
385 $headers = array_merge(array(ts('Line Number'),
390 $this->_errorFileName
= self
::errorFileName(self
::ERROR
);
392 self
::exportCSV($this->_errorFileName
, $headers, $this->_errors
);
394 if ($this->_conflictCount
) {
395 $headers = array_merge(array(ts('Line Number'),
400 $this->_conflictFileName
= self
::errorFileName(self
::CONFLICT
);
401 self
::exportCSV($this->_conflictFileName
, $headers, $this->_conflicts
);
403 if ($this->_duplicateCount
) {
404 $headers = array_merge(array(ts('Line Number'),
405 ts('View Membership URL'),
410 $this->_duplicateFileName
= self
::errorFileName(self
::DUPLICATE
);
411 self
::exportCSV($this->_duplicateFileName
, $headers, $this->_duplicates
);
414 return $this->fini();
417 abstract function mapField(&$values);
418 abstract function preview(&$values);
419 abstract function summary(&$values);
420 abstract function import($onDuplicate, &$values);
422 abstract function fini();
425 * Given a list of the importable field keys that the user has selected
426 * set the active fields array to this list
428 * @param array mapped array of values
433 function setActiveFields($fieldKeys) {
434 $this->_activeFieldCount
= count($fieldKeys);
435 foreach ($fieldKeys as $key) {
436 if (empty($this->_fields
[$key])) {
437 $this->_activeFields
[] = new CRM_Member_Import_Field('', ts('- do not import -'));
440 $this->_activeFields
[] = clone($this->_fields
[$key]);
445 /*function setActiveFieldLocationTypes( $elements ) {
446 for ($i = 0; $i < count( $elements ); $i++) {
447 $this->_activeFields[$i]->_hasLocationType = $elements[$i];
451 function setActiveFieldPhoneTypes( $elements ) {
452 for ($i = 0; $i < count( $elements ); $i++) {
453 $this->_activeFields[$i]->_phoneType = $elements[$i];
456 function setActiveFieldValues($elements, &$erroneousField) {
457 $maxCount = count($elements) < $this->_activeFieldCount ?
count($elements) : $this->_activeFieldCount
;
458 for ($i = 0; $i < $maxCount; $i++
) {
459 $this->_activeFields
[$i]->setValue($elements[$i]);
462 // reset all the values that we did not have an equivalent import element
463 for (; $i < $this->_activeFieldCount
; $i++
) {
464 $this->_activeFields
[$i]->resetValue();
467 // now validate the fields and return false if error
468 $valid = self
::VALID
;
469 for ($i = 0; $i < $this->_activeFieldCount
; $i++
) {
470 if (!$this->_activeFields
[$i]->validate()) {
471 // no need to do any more validation
472 $erroneousField = $i;
473 $valid = self
::ERROR
;
481 * function to format the field values for input to the api
483 * @return array (reference ) associative array of name/value pairs
486 function &getActiveFieldParams() {
488 for ($i = 0; $i < $this->_activeFieldCount
; $i++
) {
489 if (isset($this->_activeFields
[$i]->_value
)
490 && !isset($params[$this->_activeFields
[$i]->_name
])
491 && !isset($this->_activeFields
[$i]->_related
)
494 $params[$this->_activeFields
[$i]->_name
] = $this->_activeFields
[$i]->_value
;
500 function getSelectValues() {
502 foreach ($this->_fields
as $name => $field) {
503 $values[$name] = $field->_title
;
508 function getSelectTypes() {
510 foreach ($this->_fields
as $name => $field) {
511 if (isset($field->_hasLocationType
)) {
512 $values[$name] = $field->_hasLocationType
;
518 function getHeaderPatterns() {
520 foreach ($this->_fields
as $name => $field) {
521 if (isset($field->_headerPattern
)) {
522 $values[$name] = $field->_headerPattern
;
528 function getDataPatterns() {
530 foreach ($this->_fields
as $name => $field) {
531 $values[$name] = $field->_dataPattern
;
536 function addField($name, $title, $type = CRM_Utils_Type
::T_INT
, $headerPattern = '//', $dataPattern = '//') {
538 $this->_fields
['doNotImport'] = new CRM_Member_Import_Field($name, $title, $type, $headerPattern, $dataPattern);
542 //$tempField = CRM_Contact_BAO_Contact::importableFields('Individual', null );
543 $tempField = CRM_Contact_BAO_Contact
::importableFields('All', NULL);
544 if (!array_key_exists($name, $tempField)) {
545 $this->_fields
[$name] = new CRM_Member_Import_Field($name, $title, $type, $headerPattern, $dataPattern);
548 $this->_fields
[$name] = new CRM_Import_Field($name, $title, $type, $headerPattern, $dataPattern,
549 CRM_Utils_Array
::value('hasLocationType', $tempField[$name])
563 function setMaxLinesToProcess($max) {
564 $this->_maxLinesToProcess
= $max;
568 * Store parser values
570 * @param CRM_Core_Session $store
575 function set($store, $mode = self
::MODE_SUMMARY
) {
576 $store->set('fileSize', $this->_fileSize
);
577 $store->set('lineCount', $this->_lineCount
);
578 $store->set('seperator', $this->_seperator
);
579 $store->set('fields', $this->getSelectValues());
580 $store->set('fieldTypes', $this->getSelectTypes());
582 $store->set('headerPatterns', $this->getHeaderPatterns());
583 $store->set('dataPatterns', $this->getDataPatterns());
584 $store->set('columnCount', $this->_activeFieldCount
);
586 $store->set('totalRowCount', $this->_totalCount
);
587 $store->set('validRowCount', $this->_validCount
);
588 $store->set('invalidRowCount', $this->_invalidRowCount
);
589 $store->set('conflictRowCount', $this->_conflictCount
);
591 switch ($this->_contactType
) {
593 $store->set('contactType', CRM_Member_Import_Parser
::CONTACT_INDIVIDUAL
);
597 $store->set('contactType', CRM_Member_Import_Parser
::CONTACT_HOUSEHOLD
);
601 $store->set('contactType', CRM_Member_Import_Parser
::CONTACT_ORGANIZATION
);
604 if ($this->_invalidRowCount
) {
605 $store->set('errorsFileName', $this->_errorFileName
);
607 if ($this->_conflictCount
) {
608 $store->set('conflictsFileName', $this->_conflictFileName
);
610 if (isset($this->_rows
) && !empty($this->_rows
)) {
611 $store->set('dataValues', $this->_rows
);
614 if ($mode == self
::MODE_IMPORT
) {
615 $store->set('duplicateRowCount', $this->_duplicateCount
);
616 if ($this->_duplicateCount
) {
617 $store->set('duplicatesFileName', $this->_duplicateFileName
);
623 * Export data to a CSV file
625 * @param string $filename
626 * @param array $header
632 static function exportCSV($fileName, $header, $data) {
634 $fd = fopen($fileName, 'w');
636 foreach ($header as $key => $value) {
637 $header[$key] = "\"$value\"";
639 $config = CRM_Core_Config
::singleton();
640 $output[] = implode($config->fieldSeparator
, $header);
642 foreach ($data as $datum) {
643 foreach ($datum as $key => $value) {
644 if (is_array($value)) {
645 foreach ($value[0] as $k1 => $v1) {
646 if ($k1 == 'location_type_id') {
653 $datum[$key] = "\"$value\"";
656 $output[] = implode($config->fieldSeparator
, $datum);
658 fwrite($fd, implode("\n", $output));
663 * Remove single-quote enclosures from a value array (row)
665 * @param array $values
666 * @param string $enclosure
672 static function encloseScrub(&$values, $enclosure = "'") {
673 if (empty($values)) {
677 foreach ($values as $k => $v) {
678 $values[$k] = preg_replace("/^$enclosure(.*)$enclosure$/", '$1', $v);
682 function errorFileName($type) {
683 $fileName = CRM_Import_Parser
::errorFileName($type);
687 function saveFileName($type) {
688 $fileName = CRM_Import_Parser
::saveFileName($type);