Commit | Line | Data |
---|---|---|
6a488035 TO |
1 | <?php |
2 | /* | |
3 | +--------------------------------------------------------------------+ | |
4 | | CiviCRM version 4.3 | | |
5 | +--------------------------------------------------------------------+ | |
6 | | Copyright CiviCRM LLC (c) 2004-2013 | | |
7 | +--------------------------------------------------------------------+ | |
8 | | This file is a part of CiviCRM. | | |
9 | | | | |
10 | | CiviCRM is free software; you can copy, modify, and distribute it | | |
11 | | under the terms of the GNU Affero General Public License | | |
12 | | Version 3, 19 November 2007 and the CiviCRM Licensing Exception. | | |
13 | | | | |
14 | | CiviCRM is distributed in the hope that it will be useful, but | | |
15 | | WITHOUT ANY WARRANTY; without even the implied warranty of | | |
16 | | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. | | |
17 | | See the GNU Affero General Public License for more details. | | |
18 | | | | |
19 | | You should have received a copy of the GNU Affero General Public | | |
20 | | License and the CiviCRM Licensing Exception along | | |
21 | | with this program; if not, contact CiviCRM LLC | | |
22 | | at info[AT]civicrm[DOT]org. If you have questions about the | | |
23 | | GNU Affero General Public License or the licensing of CiviCRM, | | |
24 | | see the CiviCRM license FAQ at http://civicrm.org/licensing | | |
25 | +--------------------------------------------------------------------+ | |
26 | */ | |
27 | ||
28 | /** | |
29 | * | |
30 | * @package CRM | |
31 | * @copyright CiviCRM LLC (c) 2004-2013 | |
32 | * $Id$ | |
33 | * | |
34 | */ | |
35 | ||
36 | ||
37 | ||
38 | abstract class CRM_Activity_Import_Parser { | |
39 | CONST MAX_ERRORS = 250, MAX_WARNINGS = 25, VALID = 1, WARNING = 2, ERROR = 4, CONFLICT = 8, STOP = 16, DUPLICATE = 32, MULTIPLE_DUPE = 64, NO_MATCH = 128; | |
40 | ||
41 | /** | |
42 | * various parser modes | |
43 | */ | |
44 | CONST MODE_MAPFIELD = 1, MODE_PREVIEW = 2, MODE_SUMMARY = 4, MODE_IMPORT = 8; | |
45 | ||
46 | /** | |
47 | * codes for duplicate record handling | |
48 | */ | |
49 | CONST DUPLICATE_SKIP = 1, DUPLICATE_REPLACE = 2, DUPLICATE_UPDATE = 4, DUPLICATE_FILL = 8, DUPLICATE_NOCHECK = 16; | |
50 | ||
51 | protected $_fileName; | |
52 | ||
53 | /**#@+ | |
54 | * @access protected | |
55 | * @var integer | |
56 | */ | |
57 | ||
58 | /** | |
59 | * imported file size | |
60 | */ | |
61 | protected $_fileSize; | |
62 | ||
63 | /** | |
64 | * seperator being used | |
65 | */ | |
66 | protected $_seperator; | |
67 | ||
68 | /** | |
69 | * total number of lines in file | |
70 | */ | |
71 | protected $_lineCount; | |
72 | ||
73 | /** | |
74 | * total number of non empty lines | |
75 | */ | |
76 | protected $_totalCount; | |
77 | ||
78 | /** | |
79 | * running total number of valid lines | |
80 | */ | |
81 | protected $_validCount; | |
82 | ||
83 | /** | |
84 | * running total number of invalid rows | |
85 | */ | |
86 | protected $_invalidRowCount; | |
87 | ||
88 | /** | |
89 | * maximum number of invalid rows to store | |
90 | */ | |
91 | protected $_maxErrorCount; | |
92 | ||
93 | /** | |
94 | * array of error lines, bounded by MAX_ERROR | |
95 | */ | |
96 | protected $_errors; | |
97 | ||
98 | /** | |
99 | * total number of conflict lines | |
100 | */ | |
101 | protected $_conflictCount; | |
102 | ||
103 | /** | |
104 | * array of conflict lines | |
105 | */ | |
106 | protected $_conflicts; | |
107 | ||
108 | /** | |
109 | * total number of duplicate (from database) lines | |
110 | */ | |
111 | protected $_duplicateCount; | |
112 | ||
113 | /** | |
114 | * array of duplicate lines | |
115 | */ | |
116 | protected $_duplicates; | |
117 | ||
118 | /** | |
119 | * running total number of warnings | |
120 | */ | |
121 | protected $_warningCount; | |
122 | ||
123 | /** | |
124 | * maximum number of warnings to store | |
125 | */ | |
126 | protected $_maxWarningCount = self::MAX_WARNINGS; | |
127 | ||
128 | /** | |
129 | * array of warning lines, bounded by MAX_WARNING | |
130 | */ | |
131 | protected $_warnings; | |
132 | ||
133 | /** | |
134 | * array of all the fields that could potentially be part | |
135 | * of this import process | |
136 | * @var array | |
137 | */ | |
138 | protected $_fields; | |
139 | ||
140 | /** | |
141 | * array of the fields that are actually part of the import process | |
142 | * the position in the array also dictates their position in the import | |
143 | * file | |
144 | * @var array | |
145 | */ | |
146 | protected $_activeFields; | |
147 | ||
148 | /** | |
149 | * cache the count of active fields | |
150 | * | |
151 | * @var int | |
152 | */ | |
153 | protected $_activeFieldCount; | |
154 | ||
155 | /** | |
156 | * maximum number of non-empty/comment lines to process | |
157 | * | |
158 | * @var int | |
159 | */ | |
160 | protected $_maxLinesToProcess; | |
161 | ||
162 | /** | |
163 | * cache of preview rows | |
164 | * | |
165 | * @var array | |
166 | */ | |
167 | protected $_rows; | |
168 | ||
169 | /** | |
170 | * filename of error data | |
171 | * | |
172 | * @var string | |
173 | */ | |
174 | protected $_errorFileName; | |
175 | ||
176 | /** | |
177 | * filename of conflict data | |
178 | * | |
179 | * @var string | |
180 | */ | |
181 | protected $_conflictFileName; | |
182 | ||
183 | /** | |
184 | * filename of duplicate data | |
185 | * | |
186 | * @var string | |
187 | */ | |
188 | protected $_duplicateFileName; | |
189 | ||
190 | /** | |
191 | * whether the file has a column header or not | |
192 | * | |
193 | * @var boolean | |
194 | */ | |
195 | protected $_haveColumnHeader; | |
196 | ||
197 | function __construct() { | |
198 | $this->_maxLinesToProcess = 0; | |
199 | $this->_maxErrorCount = self::MAX_ERRORS; | |
200 | } | |
201 | ||
202 | abstract function init(); | |
203 | function run($fileName, | |
204 | $seperator = ',', | |
205 | &$mapper, | |
206 | $skipColumnHeader = FALSE, | |
207 | $mode = self::MODE_PREVIEW, | |
208 | $onDuplicate = self::DUPLICATE_SKIP | |
209 | ) { | |
210 | if (!is_array($fileName)) { | |
211 | CRM_Core_Error::fatal(); | |
212 | } | |
213 | $fileName = $fileName['name']; | |
214 | ||
215 | $this->init(); | |
216 | ||
217 | $this->_haveColumnHeader = $skipColumnHeader; | |
218 | ||
219 | $this->_seperator = $seperator; | |
220 | ||
221 | $fd = fopen($fileName, "r"); | |
222 | if (!$fd) { | |
223 | return FALSE; | |
224 | } | |
225 | ||
226 | $this->_lineCount = $this->_warningCount = 0; | |
227 | $this->_invalidRowCount = $this->_validCount = 0; | |
228 | $this->_totalCount = $this->_conflictCount = 0; | |
229 | ||
230 | $this->_errors = array(); | |
231 | $this->_warnings = array(); | |
232 | $this->_conflicts = array(); | |
233 | ||
234 | $this->_fileSize = number_format(filesize($fileName) / 1024.0, 2); | |
235 | ||
236 | if ($mode == self::MODE_MAPFIELD) { | |
237 | $this->_rows = array(); | |
238 | } | |
239 | else { | |
240 | $this->_activeFieldCount = count($this->_activeFields); | |
241 | } | |
242 | ||
243 | while (!feof($fd)) { | |
244 | $this->_lineCount++; | |
245 | ||
246 | $values = fgetcsv($fd, 8192, $seperator); | |
247 | if (!$values) { | |
248 | continue; | |
249 | } | |
250 | ||
251 | self::encloseScrub($values); | |
252 | ||
253 | // skip column header if we're not in mapfield mode | |
254 | if ($mode != self::MODE_MAPFIELD && $skipColumnHeader) { | |
255 | $skipColumnHeader = FALSE; | |
256 | continue; | |
257 | } | |
258 | ||
259 | /* trim whitespace around the values */ | |
260 | ||
261 | $empty = TRUE; | |
262 | foreach ($values as $k => $v) { | |
263 | $values[$k] = trim($v, " \t\r\n"); | |
264 | } | |
265 | ||
266 | if (CRM_Utils_System::isNull($values)) { | |
267 | continue; | |
268 | } | |
269 | ||
270 | $this->_totalCount++; | |
271 | ||
272 | if ($mode == self::MODE_MAPFIELD) { | |
273 | $returnCode = $this->mapField($values); | |
274 | } | |
275 | elseif ($mode == self::MODE_PREVIEW) { | |
276 | $returnCode = $this->preview($values); | |
277 | } | |
278 | elseif ($mode == self::MODE_SUMMARY) { | |
279 | $returnCode = $this->summary($values); | |
280 | } | |
281 | elseif ($mode == self::MODE_IMPORT) { | |
282 | $returnCode = $this->import($onDuplicate, $values); | |
283 | } | |
284 | else { | |
285 | $returnCode = self::ERROR; | |
286 | } | |
287 | ||
288 | // note that a line could be valid but still produce a warning | |
289 | if ($returnCode & self::VALID) { | |
290 | $this->_validCount++; | |
291 | if ($mode == self::MODE_MAPFIELD) { | |
292 | $this->_rows[] = $values; | |
293 | $this->_activeFieldCount = max($this->_activeFieldCount, count($values)); | |
294 | } | |
295 | } | |
296 | ||
297 | if ($returnCode & self::WARNING) { | |
298 | $this->_warningCount++; | |
299 | if ($this->_warningCount < $this->_maxWarningCount) { | |
300 | $this->_warningCount[] = $line; | |
301 | } | |
302 | } | |
303 | ||
304 | if ($returnCode & self::ERROR) { | |
305 | $this->_invalidRowCount++; | |
306 | if ($this->_invalidRowCount < $this->_maxErrorCount) { | |
307 | $recordNumber = $this->_lineCount; | |
308 | if ($this->_haveColumnHeader) { | |
309 | $recordNumber--; | |
310 | } | |
311 | array_unshift($values, $recordNumber); | |
312 | $this->_errors[] = $values; | |
313 | } | |
314 | } | |
315 | ||
316 | if ($returnCode & self::CONFLICT) { | |
317 | $this->_conflictCount++; | |
318 | $recordNumber = $this->_lineCount; | |
319 | if ($this->_haveColumnHeader) { | |
320 | $recordNumber--; | |
321 | } | |
322 | array_unshift($values, $recordNumber); | |
323 | $this->_conflicts[] = $values; | |
324 | } | |
325 | ||
326 | if ($returnCode & self::DUPLICATE) { | |
327 | if ($returnCode & self::MULTIPLE_DUPE) { | |
328 | /* TODO: multi-dupes should be counted apart from singles | |
329 | * on non-skip action */ | |
330 | } | |
331 | $this->_duplicateCount++; | |
332 | $recordNumber = $this->_lineCount; | |
333 | if ($this->_haveColumnHeader) { | |
334 | $recordNumber--; | |
335 | } | |
336 | array_unshift($values, $recordNumber); | |
337 | $this->_duplicates[] = $values; | |
338 | if ($onDuplicate != self::DUPLICATE_SKIP) { | |
339 | $this->_validCount++; | |
340 | } | |
341 | } | |
342 | ||
343 | // we give the derived class a way of aborting the process | |
344 | // note that the return code could be multiple code or'ed together | |
345 | if ($returnCode & self::STOP) { | |
346 | break; | |
347 | } | |
348 | ||
349 | // if we are done processing the maxNumber of lines, break | |
350 | if ($this->_maxLinesToProcess > 0 && $this->_validCount >= $this->_maxLinesToProcess) { | |
351 | break; | |
352 | } | |
353 | } | |
354 | ||
355 | fclose($fd); | |
356 | ||
357 | ||
358 | if ($mode == self::MODE_PREVIEW || $mode == self::MODE_IMPORT) { | |
359 | $customHeaders = $mapper; | |
360 | ||
361 | $customfields = CRM_Core_BAO_CustomField::getFields('Activity'); | |
362 | foreach ($customHeaders as $key => $value) { | |
363 | if ($id = CRM_Core_BAO_CustomField::getKeyID($value)) { | |
364 | $customHeaders[$key] = $customfields[$id][0]; | |
365 | } | |
366 | } | |
367 | if ($this->_invalidRowCount) { | |
368 | // removed view url for invlaid contacts | |
369 | $headers = array_merge(array(ts('Line Number'), | |
370 | ts('Reason'), | |
371 | ), | |
372 | $customHeaders | |
373 | ); | |
374 | $this->_errorFileName = self::errorFileName(self::ERROR); | |
375 | self::exportCSV($this->_errorFileName, $headers, $this->_errors); | |
376 | } | |
377 | if ($this->_conflictCount) { | |
378 | $headers = array_merge(array(ts('Line Number'), | |
379 | ts('Reason'), | |
380 | ), | |
381 | $customHeaders | |
382 | ); | |
383 | $this->_conflictFileName = self::errorFileName(self::CONFLICT); | |
384 | self::exportCSV($this->_conflictFileName, $headers, $this->_conflicts); | |
385 | } | |
386 | if ($this->_duplicateCount) { | |
387 | $headers = array_merge(array(ts('Line Number'), | |
388 | ts('View Activity History URL'), | |
389 | ), | |
390 | $customHeaders | |
391 | ); | |
392 | ||
393 | $this->_duplicateFileName = self::errorFileName(self::DUPLICATE); | |
394 | self::exportCSV($this->_duplicateFileName, $headers, $this->_duplicates); | |
395 | } | |
396 | } | |
397 | //echo "$this->_totalCount,$this->_invalidRowCount,$this->_conflictCount,$this->_duplicateCount"; | |
398 | return $this->fini(); | |
399 | } | |
400 | ||
401 | abstract function mapField(&$values); | |
402 | abstract function preview(&$values); | |
403 | abstract function summary(&$values); | |
404 | abstract function import($onDuplicate, &$values); | |
405 | ||
406 | abstract function fini(); | |
407 | ||
408 | /** | |
409 | * Given a list of the importable field keys that the user has selected | |
410 | * set the active fields array to this list | |
411 | * | |
412 | * @param array mapped array of values | |
413 | * | |
414 | * @return void | |
415 | * @access public | |
416 | */ | |
417 | function setActiveFields($fieldKeys) { | |
418 | $this->_activeFieldCount = count($fieldKeys); | |
419 | foreach ($fieldKeys as $key) { | |
420 | if (empty($this->_fields[$key])) { | |
421 | $this->_activeFields[] = new CRM_Activity_Import_Field('', ts('- do not import -')); | |
422 | } | |
423 | else { | |
424 | $this->_activeFields[] = clone($this->_fields[$key]); | |
425 | } | |
426 | } | |
427 | } | |
428 | ||
c77a4757 | 429 | /*function setActiveFieldLocationTypes( $elements ) |
6a488035 TO |
430 | { |
431 | for ($i = 0; $i < count( $elements ); $i++) { | |
432 | $this->_activeFields[$i]->_hasLocationType = $elements[$i]; | |
433 | } | |
434 | } | |
c77a4757 DL |
435 | |
436 | function setActiveFieldPhoneTypes( $elements ) | |
6a488035 TO |
437 | { |
438 | for ($i = 0; $i < count( $elements ); $i++) { | |
439 | $this->_activeFields[$i]->_phoneType = $elements[$i]; | |
440 | } | |
441 | }*/ | |
442 | function setActiveFieldValues($elements, &$erroneousField) { | |
443 | $maxCount = count($elements) < $this->_activeFieldCount ? count($elements) : $this->_activeFieldCount; | |
444 | for ($i = 0; $i < $maxCount; $i++) { | |
445 | $this->_activeFields[$i]->setValue($elements[$i]); | |
446 | } | |
447 | ||
448 | // reset all the values that we did not have an equivalent import element | |
449 | for (; $i < $this->_activeFieldCount; $i++) { | |
450 | $this->_activeFields[$i]->resetValue(); | |
451 | } | |
452 | ||
453 | // now validate the fields and return false if error | |
454 | $valid = self::VALID; | |
455 | for ($i = 0; $i < $this->_activeFieldCount; $i++) { | |
456 | if (!$this->_activeFields[$i]->validate()) { | |
457 | // no need to do any more validation | |
458 | $erroneousField = $i; | |
459 | $valid = self::ERROR; | |
460 | break; | |
461 | } | |
462 | } | |
463 | return $valid; | |
464 | } | |
465 | ||
466 | /** | |
467 | * function to format the field values for input to the api | |
468 | * | |
469 | * @return array (reference ) associative array of name/value pairs | |
470 | * @access public | |
471 | */ | |
472 | function &getActiveFieldParams() { | |
473 | $params = array(); | |
474 | for ($i = 0; $i < $this->_activeFieldCount; $i++) { | |
475 | if (isset($this->_activeFields[$i]->_value) | |
476 | && !isset($params[$this->_activeFields[$i]->_name]) | |
477 | && !isset($this->_activeFields[$i]->_related) | |
478 | ) { | |
479 | ||
480 | $params[$this->_activeFields[$i]->_name] = $this->_activeFields[$i]->_value; | |
481 | } | |
482 | } | |
483 | return $params; | |
484 | } | |
485 | ||
486 | function getSelectValues() { | |
487 | $values = array(); | |
488 | foreach ($this->_fields as $name => $field) { | |
489 | $values[$name] = $field->_title; | |
490 | } | |
491 | return $values; | |
492 | } | |
493 | ||
494 | function getSelectTypes() { | |
495 | $values = array(); | |
496 | foreach ($this->_fields as $name => $field) { | |
497 | if (isset($field->_hasLocationType)) { | |
498 | $values[$name] = $field->_hasLocationType; | |
499 | } | |
500 | } | |
501 | return $values; | |
502 | } | |
503 | ||
504 | function getHeaderPatterns() { | |
505 | $values = array(); | |
506 | foreach ($this->_fields as $name => $field) { | |
507 | $values[$name] = $field->_headerPattern; | |
508 | } | |
509 | return $values; | |
510 | } | |
511 | ||
512 | function getDataPatterns() { | |
513 | $values = array(); | |
514 | foreach ($this->_fields as $name => $field) { | |
515 | $values[$name] = $field->_dataPattern; | |
516 | } | |
517 | return $values; | |
518 | } | |
519 | ||
520 | function addField($name, $title, $type = CRM_Utils_Type::T_INT, $headerPattern = '//', $dataPattern = '//') { | |
521 | if (empty($name)) { | |
522 | $this->_fields['doNotImport'] = new CRM_Activity_Import_Field($name, $title, $type, $headerPattern, $dataPattern); | |
523 | } | |
524 | else { | |
525 | ||
526 | $tempField = CRM_Contact_BAO_Contact::importableFields('Individual', NULL); | |
527 | if (!array_key_exists($name, $tempField)) { | |
528 | $this->_fields[$name] = new CRM_Activity_Import_Field($name, $title, $type, $headerPattern, $dataPattern); | |
529 | } | |
530 | else { | |
531 | $this->_fields[$name] = new CRM_Import_Field($name, $title, $type, $headerPattern, $dataPattern, CRM_Utils_Array::value('hasLocationType', $tempField[$name])); | |
532 | } | |
533 | } | |
534 | } | |
535 | ||
536 | /** | |
537 | * setter function | |
538 | * | |
539 | * @param int $max | |
540 | * | |
541 | * @return void | |
542 | * @access public | |
543 | */ | |
544 | function setMaxLinesToProcess($max) { | |
545 | $this->_maxLinesToProcess = $max; | |
546 | } | |
547 | ||
548 | /** | |
549 | * Store parser values | |
550 | * | |
551 | * @param CRM_Core_Session $store | |
552 | * | |
553 | * @return void | |
554 | * @access public | |
555 | */ | |
556 | function set($store, $mode = self::MODE_SUMMARY) { | |
557 | $store->set('fileSize', $this->_fileSize); | |
558 | $store->set('lineCount', $this->_lineCount); | |
559 | $store->set('seperator', $this->_seperator); | |
560 | $store->set('fields', $this->getSelectValues()); | |
561 | $store->set('fieldTypes', $this->getSelectTypes()); | |
562 | ||
563 | $store->set('headerPatterns', $this->getHeaderPatterns()); | |
564 | $store->set('dataPatterns', $this->getDataPatterns()); | |
565 | $store->set('columnCount', $this->_activeFieldCount); | |
566 | ||
567 | $store->set('totalRowCount', $this->_totalCount); | |
568 | $store->set('validRowCount', $this->_validCount); | |
569 | $store->set('invalidRowCount', $this->_invalidRowCount); | |
570 | $store->set('conflictRowCount', $this->_conflictCount); | |
571 | ||
572 | if ($this->_invalidRowCount) { | |
573 | $store->set('errorsFileName', $this->_errorFileName); | |
574 | } | |
575 | if ($this->_conflictCount) { | |
576 | $store->set('conflictsFileName', $this->_conflictFileName); | |
577 | } | |
578 | if (isset($this->_rows) && !empty($this->_rows)) { | |
579 | $store->set('dataValues', $this->_rows); | |
580 | } | |
581 | ||
582 | if ($mode == self::MODE_IMPORT) { | |
583 | $store->set('duplicateRowCount', $this->_duplicateCount); | |
584 | if ($this->_duplicateCount) { | |
585 | $store->set('duplicatesFileName', $this->_duplicateFileName); | |
586 | } | |
587 | } | |
588 | //echo "$this->_totalCount,$this->_invalidRowCount,$this->_conflictCount,$this->_duplicateCount"; | |
589 | } | |
590 | ||
591 | /** | |
592 | * Export data to a CSV file | |
593 | * | |
594 | * @param string $filename | |
595 | * @param array $header | |
596 | * @param data $data | |
597 | * | |
598 | * @return void | |
599 | * @access public | |
600 | */ | |
601 | static function exportCSV($fileName, $header, $data) { | |
602 | $output = array(); | |
603 | $fd = fopen($fileName, 'w'); | |
604 | ||
605 | foreach ($header as $key => $value) { | |
606 | $header[$key] = "\"$value\""; | |
607 | } | |
608 | $config = CRM_Core_Config::singleton(); | |
609 | $output[] = implode($config->fieldSeparator, $header); | |
610 | ||
611 | foreach ($data as $datum) { | |
612 | foreach ($datum as $key => $value) { | |
613 | $datum[$key] = "\"$value\""; | |
614 | } | |
615 | $output[] = implode($config->fieldSeparator, $datum); | |
616 | } | |
617 | fwrite($fd, implode("\n", $output)); | |
618 | fclose($fd); | |
619 | } | |
620 | ||
621 | /** | |
622 | * Remove single-quote enclosures from a value array (row) | |
623 | * | |
624 | * @param array $values | |
625 | * @param string $enclosure | |
626 | * | |
627 | * @return void | |
628 | * @static | |
629 | * @access public | |
630 | */ | |
631 | static function encloseScrub(&$values, $enclosure = "'") { | |
632 | if (empty($values)) { | |
633 | return; | |
634 | } | |
635 | ||
636 | foreach ($values as $k => $v) { | |
c77a4757 | 637 | $values[$k] = preg_replace("/^$enclosure(.*)$enclosure$/", '$1', $v); |
6a488035 TO |
638 | } |
639 | } | |
640 | ||
641 | function errorFileName($type) { | |
642 | $fileName = CRM_Import_Parser::errorFileName($type); | |
643 | return $fileName; | |
644 | } | |
645 | ||
646 | function saveFileName($type) { | |
647 | $fileName = CRM_Import_Parser::saveFileName($type); | |
648 | return $fileName; | |
649 | } | |
650 | } | |
651 |