Add progress bar for all import process
[civicrm-core.git] / CRM / Contact / Import / Form / Preview.php
1 <?php
2 /*
3 +--------------------------------------------------------------------+
4 | CiviCRM version 4.7 |
5 +--------------------------------------------------------------------+
6 | Copyright CiviCRM LLC (c) 2004-2017 |
7 +--------------------------------------------------------------------+
8 | This file is a part of CiviCRM. |
9 | |
10 | CiviCRM is free software; you can copy, modify, and distribute it |
11 | under the terms of the GNU Affero General Public License |
12 | Version 3, 19 November 2007 and the CiviCRM Licensing Exception. |
13 | |
14 | CiviCRM is distributed in the hope that it will be useful, but |
15 | WITHOUT ANY WARRANTY; without even the implied warranty of |
16 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. |
17 | See the GNU Affero General Public License for more details. |
18 | |
19 | You should have received a copy of the GNU Affero General Public |
20 | License and the CiviCRM Licensing Exception along |
21 | with this program; if not, contact CiviCRM LLC |
22 | at info[AT]civicrm[DOT]org. If you have questions about the |
23 | GNU Affero General Public License or the licensing of CiviCRM, |
24 | see the CiviCRM license FAQ at http://civicrm.org/licensing |
25 +--------------------------------------------------------------------+
26 */
27
28 /**
29 *
30 * @package CRM
31 * @copyright CiviCRM LLC (c) 2004-2017
32 */
33
34 /**
35 * This class previews the uploaded file and returns summary statistics.
36 */
37 class CRM_Contact_Import_Form_Preview extends CRM_Import_Form_Preview {
38
39 /**
40 * Whether USPS validation should be disabled during import.
41 *
42 * @var bool
43 */
44 protected $_disableUSPS;
45
46 /**
47 * Set variables up before form is built.
48 */
49 public function preProcess() {
50 //get the data from the session
51 $dataValues = $this->get('dataValues');
52 $mapper = $this->get('mapper');
53 $invalidRowCount = $this->get('invalidRowCount');
54 $conflictRowCount = $this->get('conflictRowCount');
55 $mismatchCount = $this->get('unMatchCount');
56 $columnNames = $this->get('columnNames');
57 $this->_disableUSPS = $this->get('disableUSPS');
58
59 //assign column names
60 $this->assign('columnNames', $columnNames);
61
62 //get the mapping name displayed if the mappingId is set
63 $mappingId = $this->get('loadMappingId');
64 if ($mappingId) {
65 $mapDAO = new CRM_Core_DAO_Mapping();
66 $mapDAO->id = $mappingId;
67 $mapDAO->find(TRUE);
68 $this->assign('loadedMapping', $mappingId);
69 $this->assign('savedName', $mapDAO->name);
70 }
71
72 $this->assign('rowDisplayCount', 2);
73
74 $groups = CRM_Core_PseudoConstant::nestedGroup();
75 $this->set('groups', $groups);
76
77 $tag = CRM_Core_PseudoConstant::get('CRM_Core_DAO_EntityTag', 'tag_id', array('onlyActive' => FALSE));
78 if ($tag) {
79 $this->set('tag', $tag);
80 }
81
82 if ($invalidRowCount) {
83 $urlParams = 'type=' . CRM_Import_Parser::ERROR . '&parser=CRM_Contact_Import_Parser';
84 $this->set('downloadErrorRecordsUrl', CRM_Utils_System::url('civicrm/export', $urlParams));
85 }
86
87 if ($conflictRowCount) {
88 $urlParams = 'type=' . CRM_Import_Parser::CONFLICT . '&parser=CRM_Contact_Import_Parser';
89 $this->set('downloadConflictRecordsUrl', CRM_Utils_System::url('civicrm/export', $urlParams));
90 }
91
92 if ($mismatchCount) {
93 $urlParams = 'type=' . CRM_Import_Parser::NO_MATCH . '&parser=CRM_Contact_Import_Parser';
94 $this->set('downloadMismatchRecordsUrl', CRM_Utils_System::url('civicrm/export', $urlParams));
95 }
96
97 $properties = array(
98 'mapper',
99 'locations',
100 'phones',
101 'ims',
102 'dataValues',
103 'columnCount',
104 'totalRowCount',
105 'validRowCount',
106 'invalidRowCount',
107 'conflictRowCount',
108 'downloadErrorRecordsUrl',
109 'downloadConflictRecordsUrl',
110 'downloadMismatchRecordsUrl',
111 'related',
112 'relatedContactDetails',
113 'relatedContactLocType',
114 'relatedContactPhoneType',
115 'relatedContactImProvider',
116 'websites',
117 'relatedContactWebsiteType',
118 );
119
120 foreach ($properties as $property) {
121 $this->assign($property, $this->get($property));
122 }
123
124 $this->setStatusUrl();
125
126 $showColNames = TRUE;
127 if ('CRM_Import_DataSource_CSV' == $this->get('dataSource') &&
128 !$this->get('skipColumnHeader')
129 ) {
130 $showColNames = FALSE;
131 }
132 $this->assign('showColNames', $showColNames);
133 }
134
135 /**
136 * Build the form object.
137 */
138 public function buildQuickForm() {
139 $this->addElement('text', 'newGroupName', ts('Name for new group'), CRM_Core_DAO::getAttribute('CRM_Contact_DAO_Group', 'title'));
140 $this->addElement('text', 'newGroupDesc', ts('Description of new group'));
141 $groupTypes = CRM_Core_OptionGroup::values('group_type', TRUE);
142 if (!empty($groupTypes)) {
143 $this->addCheckBox('newGroupType',
144 ts('Group Type'),
145 $groupTypes,
146 NULL, NULL, NULL, NULL, '&nbsp;&nbsp;&nbsp;'
147 );
148 }
149
150 $groups = $this->get('groups');
151
152 if (!empty($groups)) {
153 $this->addElement('select', 'groups', ts('Add imported records to existing group(s)'), $groups, array(
154 'multiple' => "multiple",
155 'class' => 'crm-select2',
156 ));
157 }
158
159 //display new tag
160 $this->addElement('text', 'newTagName', ts('Tag'), CRM_Core_DAO::getAttribute('CRM_Core_DAO_Tag', 'name'));
161 $this->addElement('text', 'newTagDesc', ts('Description'), CRM_Core_DAO::getAttribute('CRM_Core_DAO_Tag', 'description'));
162
163 $tag = $this->get('tag');
164 if (!empty($tag)) {
165 foreach ($tag as $tagID => $tagName) {
166 $this->addElement('checkbox', "tag[$tagID]", NULL, $tagName);
167 }
168 }
169
170 $path = "_qf_MapField_display=true";
171 $qfKey = CRM_Utils_Request::retrieve('qfKey', 'String', $form);
172 if (CRM_Utils_Rule::qfKey($qfKey)) {
173 $path .= "&qfKey=$qfKey";
174 }
175
176 $previousURL = CRM_Utils_System::url('civicrm/import/contact', $path, FALSE, NULL, FALSE);
177 $cancelURL = CRM_Utils_System::url('civicrm/import/contact', 'reset=1');
178
179 $buttons = array(
180 array(
181 'type' => 'back',
182 'name' => ts('Previous'),
183 'js' => array('onclick' => "location.href='{$previousURL}'; return false;"),
184 ),
185 array(
186 'type' => 'next',
187 'name' => ts('Import Now'),
188 'spacing' => '&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;',
189 'isDefault' => TRUE,
190 ),
191 array(
192 'type' => 'cancel',
193 'name' => ts('Cancel'),
194 'js' => array('onclick' => "location.href='{$cancelURL}'; return false;"),
195 ),
196 );
197
198 $this->addButtons($buttons);
199
200 $this->addFormRule(array('CRM_Contact_Import_Form_Preview', 'formRule'), $this);
201 }
202
203 /**
204 * Global validation rules for the form.
205 *
206 * @param array $fields
207 * Posted values of the form.
208 *
209 * @param $files
210 * @param $self
211 *
212 * @return array
213 * list of errors to be posted back to the form
214 */
215 public static function formRule($fields, $files, $self) {
216 $errors = array();
217 $invalidTagName = $invalidGroupName = FALSE;
218
219 if (!empty($fields['newTagName'])) {
220 if (!CRM_Utils_Rule::objectExists(trim($fields['newTagName']),
221 array('CRM_Core_DAO_Tag')
222 )
223 ) {
224 $errors['newTagName'] = ts('Tag \'%1\' already exists.',
225 array(1 => $fields['newTagName'])
226 );
227 $invalidTagName = TRUE;
228 }
229 }
230
231 if (!empty($fields['newGroupName'])) {
232 $title = trim($fields['newGroupName']);
233 $name = CRM_Utils_String::titleToVar($title);
234 $query = 'select count(*) from civicrm_group where name like %1 OR title like %2';
235 $grpCnt = CRM_Core_DAO::singleValueQuery(
236 $query,
237 array(
238 1 => array($name, 'String'),
239 2 => array($title, 'String'),
240 )
241 );
242 if ($grpCnt) {
243 $invalidGroupName = TRUE;
244 $errors['newGroupName'] = ts('Group \'%1\' already exists.', array(1 => $fields['newGroupName']));
245 }
246 }
247
248 $self->assign('invalidTagName', $invalidTagName);
249 $self->assign('invalidGroupName', $invalidGroupName);
250
251 return empty($errors) ? TRUE : $errors;
252 }
253
254 /**
255 * Process the mapped fields and map it into the uploaded file.
256 */
257 public function postProcess() {
258
259 $importJobParams = array(
260 'doGeocodeAddress' => $this->controller->exportValue('DataSource', 'doGeocodeAddress'),
261 'invalidRowCount' => $this->get('invalidRowCount'),
262 'conflictRowCount' => $this->get('conflictRowCount'),
263 'onDuplicate' => $this->get('onDuplicate'),
264 'dedupe' => $this->get('dedupe'),
265 'newGroupName' => $this->controller->exportValue($this->_name, 'newGroupName'),
266 'newGroupDesc' => $this->controller->exportValue($this->_name, 'newGroupDesc'),
267 'newGroupType' => $this->controller->exportValue($this->_name, 'newGroupType'),
268 'groups' => $this->controller->exportValue($this->_name, 'groups'),
269 'allGroups' => $this->get('groups'),
270 'newTagName' => $this->controller->exportValue($this->_name, 'newTagName'),
271 'newTagDesc' => $this->controller->exportValue($this->_name, 'newTagDesc'),
272 'tag' => $this->controller->exportValue($this->_name, 'tag'),
273 'allTags' => $this->get('tag'),
274 'mapper' => $this->controller->exportValue('MapField', 'mapper'),
275 'mapFields' => $this->get('fields'),
276 'contactType' => $this->get('contactType'),
277 'contactSubType' => $this->get('contactSubType'),
278 'primaryKeyName' => $this->get('primaryKeyName'),
279 'statusFieldName' => $this->get('statusFieldName'),
280 'statusID' => $this->get('statusID'),
281 'totalRowCount' => $this->get('totalRowCount'),
282 );
283
284 $tableName = $this->get('importTableName');
285 $importJob = new CRM_Contact_Import_ImportJob($tableName);
286 $importJob->setJobParams($importJobParams);
287
288 // If ACL applies to the current user, update cache before running the import.
289 if (!CRM_Core_Permission::check('view all contacts')) {
290 $session = CRM_Core_Session::singleton();
291 $userID = $session->get('userID');
292 CRM_ACL_BAO_Cache::updateEntry($userID);
293 }
294
295 CRM_Utils_Address_USPS::disable($this->_disableUSPS);
296
297 // run the import
298 $importJob->runImport($this);
299
300 // Clear all caches, forcing any searches to recheck the ACLs or group membership as the import
301 // may have changed it.
302 CRM_Contact_BAO_Contact_Utils::clearContactCaches(TRUE);
303
304 // add all the necessary variables to the form
305 $importJob->setFormVariables($this);
306
307 // check if there is any error occurred
308 $errorStack = CRM_Core_Error::singleton();
309 $errors = $errorStack->getErrors();
310 $errorMessage = array();
311
312 if (is_array($errors)) {
313 foreach ($errors as $key => $value) {
314 $errorMessage[] = $value['message'];
315 }
316
317 // there is no fileName since this is a sql import
318 // so fudge it
319 $config = CRM_Core_Config::singleton();
320 $errorFile = $config->uploadDir . "sqlImport.error.log";
321 if ($fd = fopen($errorFile, 'w')) {
322 fwrite($fd, implode('\n', $errorMessage));
323 }
324 fclose($fd);
325
326 $this->set('errorFile', $errorFile);
327
328 $urlParams = 'type=' . CRM_Import_Parser::ERROR . '&parser=CRM_Contact_Import_Parser';
329 $this->set('downloadErrorRecordsUrl', CRM_Utils_System::url('civicrm/export', $urlParams));
330
331 $urlParams = 'type=' . CRM_Import_Parser::CONFLICT . '&parser=CRM_Contact_Import_Parser';
332 $this->set('downloadConflictRecordsUrl', CRM_Utils_System::url('civicrm/export', $urlParams));
333
334 $urlParams = 'type=' . CRM_Import_Parser::NO_MATCH . '&parser=CRM_Contact_Import_Parser';
335 $this->set('downloadMismatchRecordsUrl', CRM_Utils_System::url('civicrm/export', $urlParams));
336 }
337
338 //hack to clean db
339 //if job complete drop table.
340 $importJob->isComplete(TRUE);
341 }
342
343 /**
344 * Process the mapped fields and map it into the uploaded file.
345 */
346 public function postProcessOld() {
347
348 $doGeocodeAddress = $this->controller->exportValue('DataSource', 'doGeocodeAddress');
349 $invalidRowCount = $this->get('invalidRowCount');
350 $conflictRowCount = $this->get('conflictRowCount');
351 $onDuplicate = $this->get('onDuplicate');
352 $newGroupName = $this->controller->exportValue($this->_name, 'newGroupName');
353 $newGroupDesc = $this->controller->exportValue($this->_name, 'newGroupDesc');
354 $newGroupType = $this->controller->exportValue($this->_name, 'newGroupType');
355 $groups = $this->controller->exportValue($this->_name, 'groups');
356 $allGroups = $this->get('groups');
357 $newTagName = $this->controller->exportValue($this->_name, 'newTagName');
358 $newTagDesc = $this->controller->exportValue($this->_name, 'newTagDesc');
359 $tag = $this->controller->exportValue($this->_name, 'tag');
360 $allTags = $this->get('tag');
361
362 $mapper = $this->controller->exportValue('MapField', 'mapper');
363
364 $mapperKeys = array();
365 $mapperLocTypes = array();
366 $mapperPhoneTypes = array();
367 $mapperRelated = array();
368 $mapperRelatedContactType = array();
369 $mapperRelatedContactDetails = array();
370 $mapperRelatedContactLocType = array();
371 $mapperRelatedContactPhoneType = array();
372
373 foreach ($mapper as $key => $value) {
374 $mapperKeys[$key] = $mapper[$key][0];
375 if (is_numeric($mapper[$key][1])) {
376 $mapperLocTypes[$key] = $mapper[$key][1];
377 }
378 else {
379 $mapperLocTypes[$key] = NULL;
380 }
381
382 if (CRM_Utils_Array::value($key, $mapperKeys) == 'phone') {
383 $mapperPhoneTypes[$key] = $mapper[$key][2];
384 }
385 else {
386 $mapperPhoneTypes[$key] = NULL;
387 }
388
389 list($id, $first, $second) = explode('_', $mapper[$key][0]);
390 if (($first == 'a' && $second == 'b') || ($first == 'b' && $second == 'a')) {
391 $relationType = new CRM_Contact_DAO_RelationshipType();
392 $relationType->id = $id;
393 $relationType->find(TRUE);
394 $fieldName = "contact_type_$second";
395 $mapperRelatedContactType[$key] = $relationType->$fieldName;
396 $mapperRelated[$key] = $mapper[$key][0];
397 $mapperRelatedContactDetails[$key] = $mapper[$key][1];
398 $mapperRelatedContactLocType[$key] = $mapper[$key][2];
399 $mapperRelatedContactPhoneType[$key] = $mapper[$key][3];
400 }
401 else {
402 $mapperRelated[$key] = NULL;
403 $mapperRelatedContactType[$key] = NULL;
404 $mapperRelatedContactDetails[$key] = NULL;
405 $mapperRelatedContactLocType[$key] = NULL;
406 $mapperRelatedContactPhoneType[$key] = NULL;
407 }
408 }
409
410 $parser = new CRM_Contact_Import_Parser_Contact($mapperKeys, $mapperLocTypes,
411 $mapperPhoneTypes, $mapperRelated, $mapperRelatedContactType,
412 $mapperRelatedContactDetails, $mapperRelatedContactLocType,
413 $mapperRelatedContactPhoneType
414 );
415
416 $mapFields = $this->get('fields');
417
418 $locationTypes = CRM_Core_PseudoConstant::get('CRM_Core_DAO_Address', 'location_type_id');
419 $phoneTypes = CRM_Core_PseudoConstant::get('CRM_Core_DAO_Phone', 'phone_type_id');
420
421 foreach ($mapper as $key => $value) {
422 $header = array();
423 list($id, $first, $second) = explode('_', $mapper[$key][0]);
424 if (($first == 'a' && $second == 'b') || ($first == 'b' && $second == 'a')) {
425 $relationType = new CRM_Contact_DAO_RelationshipType();
426 $relationType->id = $id;
427 $relationType->find(TRUE);
428
429 $header[] = $relationType->name_a_b;
430 $header[] = ucwords(str_replace("_", " ", $mapper[$key][1]));
431
432 if (isset($mapper[$key][2])) {
433 $header[] = $locationTypes[$mapper[$key][2]];
434 }
435 if (isset($mapper[$key][3])) {
436 $header[] = $phoneTypes[$mapper[$key][3]];
437 }
438 }
439 else {
440 if (isset($mapFields[$mapper[$key][0]])) {
441 $header[] = $mapFields[$mapper[$key][0]];
442 if (isset($mapper[$key][1])) {
443 $header[] = $locationTypes[$mapper[$key][1]];
444 }
445 if (isset($mapper[$key][2])) {
446 $header[] = $phoneTypes[$mapper[$key][2]];
447 }
448 }
449 }
450 $mapperFields[] = implode(' - ', $header);
451 }
452
453 $tableName = $this->get('importTableName');
454 //print "Running parser on table: $tableName<br/>";
455 $parser->run($tableName, $mapperFields,
456 CRM_Import_Parser::MODE_IMPORT,
457 $this->get('contactType'),
458 $this->get('primaryKeyName'),
459 $this->get('statusFieldName'),
460 $onDuplicate,
461 $this->get('statusID'),
462 $this->get('totalRowCount'),
463 $doGeocodeAddress,
464 CRM_Contact_Import_Parser::DEFAULT_TIMEOUT,
465 $this->get('contactSubType'),
466 $this->get('dedupe')
467 );
468
469 // add the new contacts to selected groups
470 $contactIds = &$parser->getImportedContacts();
471
472 // add the new related contacts to selected groups
473 $relatedContactIds = &$parser->getRelatedImportedContacts();
474
475 $this->set('relatedCount', count($relatedContactIds));
476 $newGroupId = NULL;
477
478 //changed below if-statement "if ($newGroup) {" to "if ($newGroupName) {"
479 if ($newGroupName) {
480 /* Create a new group */
481
482 $gParams = array(
483 'name' => $newGroupName,
484 'title' => $newGroupName,
485 'description' => $newGroupDesc,
486 'group_type' => $newGroupType,
487 'is_active' => TRUE,
488 );
489 $group = CRM_Contact_BAO_Group::create($gParams);
490 $groups[] = $newGroupId = $group->id;
491 }
492
493 if (is_array($groups)) {
494 $groupAdditions = array();
495 foreach ($groups as $groupId) {
496 $addCount = CRM_Contact_BAO_GroupContact::addContactsToGroup($contactIds, $groupId);
497 if (!empty($relatedContactIds)) {
498 $addRelCount = CRM_Contact_BAO_GroupContact::addContactsToGroup($relatedContactIds, $groupId);
499 }
500 $totalCount = $addCount[1] + $addRelCount[1];
501 if ($groupId == $newGroupId) {
502 $name = $newGroupName;
503 $new = TRUE;
504 }
505 else {
506 $name = $allGroups[$groupId];
507 $new = FALSE;
508 }
509 $groupAdditions[] = array(
510 'url' => CRM_Utils_System::url('civicrm/group/search',
511 'reset=1&force=1&context=smog&gid=' . $groupId
512 ),
513 'name' => $name,
514 'added' => $totalCount,
515 'notAdded' => $addCount[2] + $addRelCount[2],
516 'new' => $new,
517 );
518 }
519 $this->set('groupAdditions', $groupAdditions);
520 }
521
522 $newTagId = NULL;
523 if ($newTagName) {
524 /* Create a new Tag */
525
526 $tagParams = array(
527 'name' => $newTagName,
528 'title' => $newTagName,
529 'description' => $newTagDesc,
530 'is_active' => TRUE,
531 );
532 $id = array();
533 $addedTag = CRM_Core_BAO_Tag::add($tagParams, $id);
534 $tag[$addedTag->id] = 1;
535 }
536 //add Tag to Import
537
538 if (is_array($tag)) {
539
540 $tagAdditions = array();
541 foreach ($tag as $tagId => $val) {
542 $addTagCount = CRM_Core_BAO_EntityTag::addContactsToTag($contactIds, $tagId);
543 if (!empty($relatedContactIds)) {
544 $addRelTagCount = CRM_Core_BAO_EntityTag::addContactsToTag($relatedContactIds, $tagId);
545 }
546 $totalTagCount = $addTagCount[1] + $addRelTagCount[1];
547 if ($tagId == $addedTag->id) {
548 $tagName = $newTagName;
549 $new = TRUE;
550 }
551 else {
552 $tagName = $allTags[$tagId];
553 $new = FALSE;
554 }
555 $tagAdditions[] = array(
556 'url' => CRM_Utils_System::url('civicrm/contact/search',
557 'reset=1&force=1&context=smog&id=' . $tagId
558 ),
559 'name' => $tagName,
560 'added' => $totalTagCount,
561 'notAdded' => $addTagCount[2] + $addRelTagCount[2],
562 'new' => $new,
563 );
564 }
565 $this->set('tagAdditions', $tagAdditions);
566 }
567
568 // add all the necessary variables to the form
569 $parser->set($this, CRM_Import_Parser::MODE_IMPORT);
570
571 // check if there is any error occurred
572
573 $errorStack = CRM_Core_Error::singleton();
574 $errors = $errorStack->getErrors();
575 $errorMessage = array();
576
577 if (is_array($errors)) {
578 foreach ($errors as $key => $value) {
579 $errorMessage[] = $value['message'];
580 }
581
582 // there is no fileName since this is a sql import
583 // so fudge it
584 $config = CRM_Core_Config::singleton();
585 $errorFile = $config->uploadDir . "sqlImport.error.log";
586 if ($fd = fopen($errorFile, 'w')) {
587 fwrite($fd, implode('\n', $errorMessage));
588 }
589 fclose($fd);
590
591 $this->set('errorFile', $errorFile);
592
593 $urlParams = 'type=' . CRM_Import_Parser::ERROR . '&parser=CRM_Contact_Import_Parser';
594 $this->set('downloadErrorRecordsUrl', CRM_Utils_System::url('civicrm/export', $urlparams));
595
596 $urlParams = 'type=' . CRM_Import_Parser::CONFLICT . '&parser=CRM_Contact_Import_Parser';
597 $this->set('downloadConflictRecordsUrl', CRM_Utils_System::url('civicrm/export', $urlParams));
598
599 $urlParams = 'type=' . CRM_Import_Parser::NO_MATCH . '&parser=CRM_Contact_Import_Parser';
600 $this->set('downloadMismatchRecordsUrl', CRM_Utils_System::url('civicrm/export', $urlParams));
601 }
602 }
603
604 }