CRM/Import/Parser.php

   1 <?php
   2 /*
   3  +--------------------------------------------------------------------+
   4  | CiviCRM version 4.3                                                |
   5  +--------------------------------------------------------------------+
   6  | Copyright CiviCRM LLC (c) 2004-2013                                |
   7  +--------------------------------------------------------------------+
   8  | This file is a part of CiviCRM.                                    |
   9  |                                                                    |
  10  | CiviCRM is free software; you can copy, modify, and distribute it  |
  11  | under the terms of the GNU Affero General Public License           |
  12  | Version 3, 19 November 2007 and the CiviCRM Licensing Exception.   |
  13  |                                                                    |
  14  | CiviCRM is distributed in the hope that it will be useful, but     |
  15  | WITHOUT ANY WARRANTY; without even the implied warranty of         |
  16  | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.               |
  17  | See the GNU Affero General Public License for more details.        |
  18  |                                                                    |
  19  | You should have received a copy of the GNU Affero General Public   |
  20  | License and the CiviCRM Licensing Exception along                  |
  21  | with this program; if not, contact CiviCRM LLC                     |
  22  | at info[AT]civicrm[DOT]org. If you have questions about the        |
  23  | GNU Affero General Public License or the licensing of CiviCRM,     |
  24  | see the CiviCRM license FAQ at http://civicrm.org/licensing        |
  25  +--------------------------------------------------------------------+
  26 */
  27
  28 /**
  29  *
  30  * @package CRM
  31  * @copyright CiviCRM LLC (c) 2004-2013
  32  * $Id$
  33  *
  34  */
  35
  36
  37
  38 abstract class CRM_Import_Parser {
  39   CONST MAX_ERRORS = 250, MAX_WARNINGS = 25, VALID = 1, WARNING = 2, ERROR = 4, CONFLICT = 8, STOP = 16, DUPLICATE = 32, MULTIPLE_DUPE = 64, NO_MATCH = 128, UNPARSED_ADDRESS_WARNING = 256;
  40
  41   /**
  42    * various parser modes
  43    */
  44   CONST MODE_MAPFIELD = 1, MODE_PREVIEW = 2, MODE_SUMMARY = 4, MODE_IMPORT = 8;
  45
  46   /**
  47    * codes for duplicate record handling
  48    */
  49   CONST DUPLICATE_SKIP = 1, DUPLICATE_REPLACE = 2, DUPLICATE_UPDATE = 4, DUPLICATE_FILL = 8, DUPLICATE_NOCHECK = 16;
  50
  51   /**
  52    * various Contact types
  53    */
  54   CONST CONTACT_INDIVIDUAL = 1, CONTACT_HOUSEHOLD = 2, CONTACT_ORGANIZATION = 4;
  55   CONST DEFAULT_TIMEOUT = 30;
  56
  57   protected $_tableName;
  58
  59   /**#@+
  60    * @access protected
  61    * @var integer
  62    */
  63
  64   /**
  65    * total number of lines in file
  66    */
  67   protected $_rowCount;
  68
  69   /**
  70    * total number of non empty lines
  71    */
  72   protected $_totalCount;
  73
  74   /**
  75    * running total number of valid lines
  76    */
  77   protected $_validCount;
  78
  79   /**
  80    * running total number of invalid rows
  81    */
  82   protected $_invalidRowCount;
  83
  84   /**
  85    * maximum number of invalid rows to store
  86    */
  87   protected $_maxErrorCount;
  88
  89   /**
  90    * array of error lines, bounded by MAX_ERROR
  91    */
  92   protected $_errors;
  93
  94   /**
  95    * total number of conflict lines
  96    */
  97   protected $_conflictCount;
  98
  99   /**
 100    * array of conflict lines
 101    */
 102   protected $_conflicts;
 103
 104   /**
 105    * total number of duplicate (from database) lines
 106    */
 107   protected $_duplicateCount;
 108
 109   /**
 110    * array of duplicate lines
 111    */
 112   protected $_duplicates;
 113
 114   /**
 115    * running total number of warnings
 116    */
 117   protected $_warningCount;
 118
 119   /**
 120    * running total number of un matched Conact
 121    */
 122   protected $_unMatchCount;
 123
 124   /**
 125    * array of unmatched lines
 126    */
 127   protected $_unMatch;
 128
 129   /**
 130    * maximum number of warnings to store
 131    */
 132   protected $_maxWarningCount = self::MAX_WARNINGS;
 133
 134   /**
 135    * total number of contacts with unparsed addresses
 136    */
 137   protected $_unparsedAddressCount;
 138
 139   /**
 140    * array of warning lines, bounded by MAX_WARNING
 141    */
 142   protected $_warnings;
 143
 144   /**
 145    * array of all the fields that could potentially be part
 146    * of this import process
 147    * @var array
 148    */
 149   protected $_fields;
 150
 151   /**
 152    * array of the fields that are actually part of the import process
 153    * the position in the array also dictates their position in the import
 154    * file
 155    * @var array
 156    */
 157   protected $_activeFields;
 158
 159   /**
 160    * cache the count of active fields
 161    *
 162    * @var int
 163    */
 164   protected $_activeFieldCount;
 165
 166   /**
 167    * maximum number of non-empty/comment lines to process
 168    *
 169    * @var int
 170    */
 171   protected $_maxLinesToProcess;
 172
 173   /**
 174    * cache of preview rows
 175    *
 176    * @var array
 177    */
 178   protected $_rows;
 179
 180   /**
 181    * filename of error data
 182    *
 183    * @var string
 184    */
 185   protected $_errorFileName;
 186
 187   /**
 188    * filename of conflict data
 189    *
 190    * @var string
 191    */
 192   protected $_conflictFileName;
 193
 194   /**
 195    * filename of duplicate data
 196    *
 197    * @var string
 198    */
 199   protected $_duplicateFileName;
 200
 201   /**
 202    * filename of mismatch data
 203    *
 204    * @var string
 205    */
 206   protected $_misMatchFilemName;
 207
 208   protected $_primaryKeyName;
 209   protected $_statusFieldName;
 210
 211   /**
 212    * contact type
 213    *
 214    * @var int
 215    */
 216
 217   public $_contactType;
 218
 219   /**
 220    * on duplicate
 221    *
 222    * @var int
 223    */
 224   public $_onDuplicate;
 225
 226   /**
 227    * dedupe rule group id to use if set
 228    *
 229    * @var int
 230    */
 231   public $_dedupeRuleGroupID = NULL;
 232
 233   function __construct() {
 234     $this->_maxLinesToProcess = 0;
 235     $this->_maxErrorCount = self::MAX_ERRORS;
 236   }
 237
 238   abstract function init();
 239
 240   function run($tableName,
 241     &$mapper,
 242     $mode              = self::MODE_PREVIEW,
 243     $contactType       = self::CONTACT_INDIVIDUAL,
 244     $primaryKeyName    = '_id',
 245     $statusFieldName   = '_status',
 246     $onDuplicate       = self::DUPLICATE_SKIP,
 247     $statusID          = NULL,
 248     $totalRowCount     = NULL,
 249     $doGeocodeAddress  = FALSE,
 250     $timeout           = CRM_Import_Parser::DEFAULT_TIMEOUT,
 251     $contactSubType    = NULL,
 252     $dedupeRuleGroupID = NULL
 253   ) {
 254
 255     // TODO: Make the timeout actually work
 256     $this->_onDuplicate = $onDuplicate;
 257     $this->_dedupeRuleGroupID = $dedupeRuleGroupID;
 258
 259     switch ($contactType) {
 260       case CRM_Import_Parser::CONTACT_INDIVIDUAL:
 261         $this->_contactType = 'Individual';
 262         break;
 263
 264       case CRM_Import_Parser::CONTACT_HOUSEHOLD:
 265         $this->_contactType = 'Household';
 266         break;
 267
 268       case CRM_Import_Parser::CONTACT_ORGANIZATION:
 269         $this->_contactType = 'Organization';
 270     }
 271
 272     $this->_contactSubType = $contactSubType;
 273
 274     $this->init();
 275
 276     $this->_rowCount = $this->_warningCount = 0;
 277     $this->_invalidRowCount = $this->_validCount = 0;
 278     $this->_totalCount = $this->_conflictCount = 0;
 279
 280     $this->_errors = array();
 281     $this->_warnings = array();
 282     $this->_conflicts = array();
 283     $this->_unparsedAddresses = array();
 284
 285     $status = '';
 286
 287     $this->_tableName = $tableName;
 288     $this->_primaryKeyName = $primaryKeyName;
 289     $this->_statusFieldName = $statusFieldName;
 290
 291     if ($mode == self::MODE_MAPFIELD) {
 292       $this->_rows = array();
 293     }
 294     else {
 295       $this->_activeFieldCount = count($this->_activeFields);
 296     }
 297
 298     if ($mode == self::MODE_IMPORT) {
 299       //get the key of email field
 300       foreach ($mapper as $key => $value) {
 301         if (strtolower($value) == 'email') {
 302           $emailKey = $key;
 303           break;
 304         }
 305       }
 306     }
 307
 308     if ($statusID) {
 309       $skip = 50;
 310       // $skip = 1;
 311       $config     = CRM_Core_Config::singleton();
 312       $statusFile = "{$config->uploadDir}status_{$statusID}.txt";
 313       $status     = "<div class='description'>&nbsp; " . ts('No processing status reported yet.') . "</div>";
 314
 315       //do not force the browser to display the save dialog, CRM-7640
 316       $contents = json_encode(array(0, $status));
 317
 318       file_put_contents($statusFile, $contents);
 319
 320       $startTimestamp = $currTimestamp = $prevTimestamp = time();
 321     }
 322
 323     // get the contents of the temp. import table
 324     $query = "SELECT * FROM $tableName";
 325     if ($mode == self::MODE_IMPORT) {
 326       $query .= " WHERE $statusFieldName = 'NEW'";
 327     }
 328     $dao    = new CRM_Core_DAO();
 329     $db     = $dao->getDatabaseConnection();
 330     $result = $db->query($query);
 331
 332     while ($values = $result->fetchRow(DB_FETCHMODE_ORDERED)) {
 333       $this->_rowCount++;
 334
 335       /* trim whitespace around the values */
 336
 337       $empty = TRUE;
 338       foreach ($values as $k => $v) {
 339         $values[$k] = trim($v, " \t\r\n");
 340       }
 341       if (CRM_Utils_System::isNull($values)) {
 342         continue;
 343       }
 344
 345       $this->_totalCount++;
 346
 347       if ($mode == self::MODE_MAPFIELD) {
 348         $returnCode = $this->mapField($values);
 349       }
 350       elseif ($mode == self::MODE_PREVIEW) {
 351         $returnCode = $this->preview($values);
 352       }
 353       elseif ($mode == self::MODE_SUMMARY) {
 354         $returnCode = $this->summary($values);
 355       }
 356       elseif ($mode == self::MODE_IMPORT) {
 357         //print "Running parser in import mode<br/>\n";
 358         $returnCode = $this->import($onDuplicate, $values, $doGeocodeAddress);
 359         if ($statusID && (($this->_rowCount % $skip) == 0)) {
 360           $currTimestamp = time();
 361           $totalTime     = ($currTimestamp - $startTimestamp);
 362           $time          = ($currTimestamp - $prevTimestamp);
 363           $recordsLeft   = $totalRowCount - $this->_rowCount;
 364           if ($recordsLeft < 0) {
 365             $recordsLeft = 0;
 366           }
 367           $estimatedTime = ($recordsLeft / $skip) * $time;
 368           $estMinutes    = floor($estimatedTime / 60);
 369           $timeFormatted = '';
 370           if ($estMinutes > 1) {
 371             $timeFormatted = $estMinutes . ' ' . ts('minutes') . ' ';
 372             $estimatedTime = $estimatedTime - ($estMinutes * 60);
 373           }
 374           $timeFormatted .= round($estimatedTime) . ' ' . ts('seconds');
 375           $processedPercent = (int )(($this->_rowCount * 100) / $totalRowCount);
 376           $statusMsg = ts('%1 of %2 records - %3 remaining',
 377             array(1 => $this->_rowCount, 2 => $totalRowCount, 3 => $timeFormatted)
 378           );
 379           $status = "
 380 <div class=\"description\">
 381 &nbsp; <strong>{$statusMsg}</strong>
 382 </div>
 383 ";
 384
 385           $contents = json_encode (array($processedPercent, $status));
 386
 387           file_put_contents($statusFile, $contents);
 388
 389           $prevTimestamp = $currTimestamp;
 390         }
 391         // sleep(1);
 392       }
 393       else {
 394         $returnCode = self::ERROR;
 395       }
 396
 397       // note that a line could be valid but still produce a warning
 398       if ($returnCode & self::VALID) {
 399         $this->_validCount++;
 400         if ($mode == self::MODE_MAPFIELD) {
 401           $this->_rows[] = $values;
 402           $this->_activeFieldCount = max($this->_activeFieldCount, count($values));
 403         }
 404       }
 405
 406       if ($returnCode & self::WARNING) {
 407         $this->_warningCount++;
 408         if ($this->_warningCount < $this->_maxWarningCount) {
 409           $this->_warningCount[] = $line;
 410         }
 411       }
 412
 413       if ($returnCode & self::ERROR) {
 414         $this->_invalidRowCount++;
 415         if ($this->_invalidRowCount < $this->_maxErrorCount) {
 416           array_unshift($values, $this->_rowCount);
 417           $this->_errors[] = $values;
 418         }
 419       }
 420
 421       if ($returnCode & self::CONFLICT) {
 422         $this->_conflictCount++;
 423         array_unshift($values, $this->_rowCount);
 424         $this->_conflicts[] = $values;
 425       }
 426
 427       if ($returnCode & self::NO_MATCH) {
 428         $this->_unMatchCount++;
 429         array_unshift($values, $this->_rowCount);
 430         $this->_unMatch[] = $values;
 431       }
 432
 433       if ($returnCode & self::DUPLICATE) {
 434         if ($returnCode & self::MULTIPLE_DUPE) {
 435           /* TODO: multi-dupes should be counted apart from singles
 436                      * on non-skip action */
 437         }
 438         $this->_duplicateCount++;
 439         array_unshift($values, $this->_rowCount);
 440         $this->_duplicates[] = $values;
 441         if ($onDuplicate != self::DUPLICATE_SKIP) {
 442           $this->_validCount++;
 443         }
 444       }
 445
 446       if ($returnCode & self::UNPARSED_ADDRESS_WARNING) {
 447         $this->_unparsedAddressCount++;
 448         array_unshift($values, $this->_rowCount);
 449         $this->_unparsedAddresses[] = $values;
 450       }
 451       // we give the derived class a way of aborting the process
 452       // note that the return code could be multiple code or'ed together
 453       if ($returnCode & self::STOP) {
 454         break;
 455       }
 456
 457       // if we are done processing the maxNumber of lines, break
 458       if ($this->_maxLinesToProcess > 0 && $this->_validCount >= $this->_maxLinesToProcess) {
 459         break;
 460       }
 461
 462       // clean up memory from dao's
 463       CRM_Core_DAO::freeResult();
 464
 465       // see if we've hit our timeout yet
 466       /* if ( $the_thing_with_the_stuff ) {
 467                 do_something( );
 468             } */
 469     }
 470
 471
 472     if ($mode == self::MODE_PREVIEW || $mode == self::MODE_IMPORT) {
 473       $customHeaders = $mapper;
 474
 475       $customfields = CRM_Core_BAO_CustomField::getFields($this->_contactType);
 476       foreach ($customHeaders as $key => $value) {
 477         if ($id = CRM_Core_BAO_CustomField::getKeyID($value)) {
 478           $customHeaders[$key] = $customfields[$id][0];
 479         }
 480       }
 481
 482       if ($this->_invalidRowCount) {
 483         // removed view url for invlaid contacts
 484         $headers = array_merge(array(ts('Line Number'),
 485             ts('Reason'),
 486           ),
 487           $customHeaders
 488         );
 489         $this->_errorFileName = self::errorFileName(self::ERROR);
 490         self::exportCSV($this->_errorFileName, $headers, $this->_errors);
 491       }
 492       if ($this->_conflictCount) {
 493         $headers = array_merge(array(ts('Line Number'),
 494             ts('Reason'),
 495           ),
 496           $customHeaders
 497         );
 498         $this->_conflictFileName = self::errorFileName(self::CONFLICT);
 499         self::exportCSV($this->_conflictFileName, $headers, $this->_conflicts);
 500       }
 501       if ($this->_duplicateCount) {
 502         $headers = array_merge(array(ts('Line Number'),
 503             ts('View Contact URL'),
 504           ),
 505           $customHeaders
 506         );
 507
 508         $this->_duplicateFileName = self::errorFileName(self::DUPLICATE);
 509         self::exportCSV($this->_duplicateFileName, $headers, $this->_duplicates);
 510       }
 511       if ($this->_unMatchCount) {
 512         $headers = array_merge(array(ts('Line Number'),
 513             ts('Reason'),
 514           ),
 515           $customHeaders
 516         );
 517
 518         $this->_misMatchFilemName = self::errorFileName(self::NO_MATCH);
 519         self::exportCSV($this->_misMatchFilemName, $headers, $this->_unMatch);
 520       }
 521       if ($this->_unparsedAddressCount) {
 522         $headers = array_merge(array(ts('Line Number'),
 523             ts('Contact Edit URL'),
 524           ),
 525           $customHeaders
 526         );
 527         $this->_errorFileName = self::errorFileName(self::UNPARSED_ADDRESS_WARNING);
 528         self::exportCSV($this->_errorFileName, $headers, $this->_unparsedAddresses);
 529       }
 530     }
 531     //echo "$this->_totalCount,$this->_invalidRowCount,$this->_conflictCount,$this->_duplicateCount";
 532     return $this->fini();
 533   }
 534
 535   abstract function mapField(&$values);
 536   abstract function preview(&$values);
 537   abstract function summary(&$values);
 538   abstract function import($onDuplicate, &$values);
 539
 540   abstract function fini();
 541
 542   /**
 543    * Given a list of the importable field keys that the user has selected
 544    * set the active fields array to this list
 545    *
 546    * @param array mapped array of values
 547    *
 548    * @return void
 549    * @access public
 550    */
 551   function setActiveFields($fieldKeys) {
 552     $this->_activeFieldCount = count($fieldKeys);
 553     foreach ($fieldKeys as $key) {
 554       if (empty($this->_fields[$key])) {
 555         $this->_activeFields[] = new CRM_Import_Field('', ts('- do not import -'));
 556       }
 557       else {
 558         $this->_activeFields[] = clone($this->_fields[$key]);
 559       }
 560     }
 561   }
 562
 563   function setActiveFieldValues($elements) {
 564     $maxCount = count($elements) < $this->_activeFieldCount ? count($elements) : $this->_activeFieldCount;
 565     for ($i = 0; $i < $maxCount; $i++) {
 566       $this->_activeFields[$i]->setValue($elements[$i]);
 567     }
 568
 569     // reset all the values that we did not have an equivalent import element
 570     for (; $i < $this->_activeFieldCount; $i++) {
 571       $this->_activeFields[$i]->resetValue();
 572     }
 573
 574     // now validate the fields and return false if error
 575     $valid = self::VALID;
 576     for ($i = 0; $i < $this->_activeFieldCount; $i++) {
 577       if (!$this->_activeFields[$i]->validate()) {
 578         // no need to do any more validation
 579         $valid = self::ERROR;
 580         break;
 581       }
 582     }
 583     return $valid;
 584   }
 585
 586   function setActiveFieldLocationTypes($elements) {
 587     for ($i = 0; $i < count($elements); $i++) {
 588       $this->_activeFields[$i]->_hasLocationType = $elements[$i];
 589     }
 590   }
 591
 592   function setActiveFieldPhoneTypes($elements) {
 593     for ($i = 0; $i < count($elements); $i++) {
 594       $this->_activeFields[$i]->_phoneType = $elements[$i];
 595     }
 596   }
 597
 598   function setActiveFieldWebsiteTypes($elements) {
 599     for ($i = 0; $i < count($elements); $i++) {
 600       $this->_activeFields[$i]->_websiteType = $elements[$i];
 601     }
 602   }
 603
 604   /**
 605    * Function to set IM Service Provider type fields
 606    *
 607    * @param array $elements IM service provider type ids
 608    *
 609    * @return void
 610    * @access public
 611    */
 612   function setActiveFieldImProviders($elements) {
 613     for ($i = 0; $i < count($elements); $i++) {
 614       $this->_activeFields[$i]->_imProvider = $elements[$i];
 615     }
 616   }
 617
 618   function setActiveFieldRelated($elements) {
 619     for ($i = 0; $i < count($elements); $i++) {
 620       $this->_activeFields[$i]->_related = $elements[$i];
 621     }
 622   }
 623
 624   function setActiveFieldRelatedContactType($elements) {
 625     for ($i = 0; $i < count($elements); $i++) {
 626       $this->_activeFields[$i]->_relatedContactType = $elements[$i];
 627     }
 628   }
 629
 630   function setActiveFieldRelatedContactDetails($elements) {
 631     for ($i = 0; $i < count($elements); $i++) {
 632       $this->_activeFields[$i]->_relatedContactDetails = $elements[$i];
 633     }
 634   }
 635
 636   function setActiveFieldRelatedContactLocType($elements) {
 637     for ($i = 0; $i < count($elements); $i++) {
 638       $this->_activeFields[$i]->_relatedContactLocType = $elements[$i];
 639     }
 640   }
 641
 642   function setActiveFieldRelatedContactPhoneType($elements) {
 643     for ($i = 0; $i < count($elements); $i++) {
 644       $this->_activeFields[$i]->_relatedContactPhoneType = $elements[$i];
 645     }
 646   }
 647
 648   function setActiveFieldRelatedContactWebsiteType($elements) {
 649     for ($i = 0; $i < count($elements); $i++) {
 650       $this->_activeFields[$i]->_relatedContactWebsiteType = $elements[$i];
 651     }
 652   }
 653
 654   /**
 655    * Function to set IM Service Provider type fields for related contacts
 656    *
 657    * @param array $elements IM service provider type ids of related contact
 658    *
 659    * @return void
 660    * @access public
 661    */
 662   function setActiveFieldRelatedContactImProvider($elements) {
 663     for ($i = 0; $i < count($elements); $i++) {
 664       $this->_activeFields[$i]->_relatedContactImProvider = $elements[$i];
 665     }
 666   }
 667
 668   /**
 669    * function to format the field values for input to the api
 670    *
 671    * @return array (reference ) associative array of name/value pairs
 672    * @access public
 673    */
 674   function &getActiveFieldParams() {
 675     $params = array();
 676
 677     //CRM_Core_Error::debug( 'Count', $this->_activeFieldCount );
 678     for ($i = 0; $i < $this->_activeFieldCount; $i++) {
 679       if ($this->_activeFields[$i]->_name == 'do_not_import') {
 680         continue;
 681       }
 682
 683       if (isset($this->_activeFields[$i]->_value)) {
 684         if (isset($this->_activeFields[$i]->_hasLocationType)) {
 685           if (!isset($params[$this->_activeFields[$i]->_name])) {
 686             $params[$this->_activeFields[$i]->_name] = array();
 687           }
 688
 689           $value = array(
 690             $this->_activeFields[$i]->_name =>
 691             $this->_activeFields[$i]->_value,
 692             'location_type_id' =>
 693             $this->_activeFields[$i]->_hasLocationType,
 694           );
 695
 696           if (isset($this->_activeFields[$i]->_phoneType)) {
 697             $value['phone_type_id'] = $this->_activeFields[$i]->_phoneType;
 698           }
 699
 700           // get IM service Provider type id
 701           if (isset($this->_activeFields[$i]->_imProvider)) {
 702             $value['provider_id'] = $this->_activeFields[$i]->_imProvider;
 703           }
 704
 705           $params[$this->_activeFields[$i]->_name][] = $value;
 706         }
 707         elseif (isset($this->_activeFields[$i]->_websiteType)) {
 708           $value = array(
 709             $this->_activeFields[$i]->_name => $this->_activeFields[$i]->_value,
 710             'website_type_id' => $this->_activeFields[$i]->_websiteType,
 711           );
 712
 713           $params[$this->_activeFields[$i]->_name][] = $value;
 714         }
 715
 716         if (!isset($params[$this->_activeFields[$i]->_name])) {
 717           if (!isset($this->_activeFields[$i]->_related)) {
 718             $params[$this->_activeFields[$i]->_name] = $this->_activeFields[$i]->_value;
 719           }
 720         }
 721
 722         //minor fix for CRM-4062
 723         if (isset($this->_activeFields[$i]->_related)) {
 724           if (!isset($params[$this->_activeFields[$i]->_related])) {
 725             $params[$this->_activeFields[$i]->_related] = array();
 726           }
 727
 728           if (!isset($params[$this->_activeFields[$i]->_related]['contact_type']) && !empty($this->_activeFields[$i]->_relatedContactType)) {
 729             $params[$this->_activeFields[$i]->_related]['contact_type'] = $this->_activeFields[$i]->_relatedContactType;
 730           }
 731
 732           if (isset($this->_activeFields[$i]->_relatedContactLocType) && !empty($this->_activeFields[$i]->_value)) {
 733             if (!is_array($params[$this->_activeFields[$i]->_related][$this->_activeFields[$i]->_relatedContactDetails])) {
 734               $params[$this->_activeFields[$i]->_related][$this->_activeFields[$i]->_relatedContactDetails] = array();
 735             }
 736             $value = array(
 737               $this->_activeFields[$i]->_relatedContactDetails => $this->_activeFields[$i]->_value,
 738               'location_type_id' => $this->_activeFields[$i]->_relatedContactLocType,
 739             );
 740
 741             if (isset($this->_activeFields[$i]->_relatedContactPhoneType)) {
 742               $value['phone_type_id'] = $this->_activeFields[$i]->_relatedContactPhoneType;
 743             }
 744
 745             // get IM service Provider type id for related contact
 746             if (isset($this->_activeFields[$i]->_relatedContactImProvider)) {
 747               $value['provider_id'] = $this->_activeFields[$i]->_relatedContactImProvider;
 748             }
 749
 750             $params[$this->_activeFields[$i]->_related][$this->_activeFields[$i]->_relatedContactDetails][] = $value;
 751           }
 752           elseif (isset($this->_activeFields[$i]->_relatedContactWebsiteType)) {
 753             $params[$this->_activeFields[$i]->_related][$this->_activeFields[$i]->_relatedContactDetails][] = array(
 754               'url' => $this->_activeFields[$i]->_value,
 755               'website_type_id' => $this->_activeFields[$i]->_relatedContactWebsiteType,
 756             );
 757           }
 758           else {
 759             $params[$this->_activeFields[$i]->_related][$this->_activeFields[$i]->_relatedContactDetails] = $this->_activeFields[$i]->_value;
 760           }
 761         }
 762       }
 763     }
 764
 765     return $params;
 766   }
 767
 768   function getSelectValues() {
 769     $values = array();
 770     foreach ($this->_fields as $name => $field) {
 771       $values[$name] = $field->_title;
 772     }
 773     return $values;
 774   }
 775
 776   function getSelectTypes() {
 777     $values = array();
 778     foreach ($this->_fields as $name => $field) {
 779       $values[$name] = $field->_hasLocationType;
 780     }
 781     return $values;
 782   }
 783
 784   function getColumnPatterns() {
 785     $values = array();
 786     foreach ($this->_fields as $name => $field) {
 787       $values[$name] = $field->_columnPattern;
 788     }
 789     return $values;
 790   }
 791
 792   function getDataPatterns() {
 793     $values = array();
 794     foreach ($this->_fields as $name => $field) {
 795       $values[$name] = $field->_dataPattern;
 796     }
 797     return $values;
 798   }
 799
 800   function addField($name, $title, $type = CRM_Utils_Type::T_INT,
 801     $headerPattern = '//', $dataPattern = '//',
 802     $hasLocationType = FALSE
 803   ) {
 804     $this->_fields[$name] = new CRM_Import_Field($name, $title, $type, $headerPattern, $dataPattern, $hasLocationType);
 805     if (empty($name)) {
 806       $this->_fields['doNotImport'] = new CRM_Import_Field($name, $title, $type, $headerPattern, $dataPattern, $hasLocationType);
 807     }
 808   }
 809
 810   /**
 811    * setter function
 812    *
 813    * @param int $max
 814    *
 815    * @return void
 816    * @access public
 817    */
 818   function setMaxLinesToProcess($max) {
 819     $this->_maxLinesToProcess = $max;
 820   }
 821
 822   /**
 823    * Store parser values
 824    *
 825    * @param CRM_Core_Session $store
 826    *
 827    * @return void
 828    * @access public
 829    */
 830   function set($store, $mode = self::MODE_SUMMARY) {
 831     $store->set('rowCount', $this->_rowCount);
 832     $store->set('fields', $this->getSelectValues());
 833     $store->set('fieldTypes', $this->getSelectTypes());
 834
 835     $store->set('columnPatterns', $this->getColumnPatterns());
 836     $store->set('dataPatterns', $this->getDataPatterns());
 837     $store->set('columnCount', $this->_activeFieldCount);
 838
 839     $store->set('totalRowCount', $this->_totalCount);
 840     $store->set('validRowCount', $this->_validCount);
 841     $store->set('invalidRowCount', $this->_invalidRowCount);
 842     $store->set('conflictRowCount', $this->_conflictCount);
 843     $store->set('unMatchCount', $this->_unMatchCount);
 844
 845     switch ($this->_contactType) {
 846       case 'Individual':
 847         $store->set('contactType', CRM_Import_Parser::CONTACT_INDIVIDUAL);
 848         break;
 849
 850       case 'Household':
 851         $store->set('contactType', CRM_Import_Parser::CONTACT_HOUSEHOLD);
 852         break;
 853
 854       case 'Organization':
 855         $store->set('contactType', CRM_Import_Parser::CONTACT_ORGANIZATION);
 856     }
 857
 858     if ($this->_invalidRowCount) {
 859       $store->set('errorsFileName', $this->_errorFileName);
 860     }
 861     if ($this->_conflictCount) {
 862       $store->set('conflictsFileName', $this->_conflictFileName);
 863     }
 864     if (isset($this->_rows) && !empty($this->_rows)) {
 865       $store->set('dataValues', $this->_rows);
 866     }
 867
 868     if ($this->_unMatchCount) {
 869       $store->set('mismatchFileName', $this->_misMatchFilemName);
 870     }
 871
 872     if ($mode == self::MODE_IMPORT) {
 873       $store->set('duplicateRowCount', $this->_duplicateCount);
 874       $store->set('unparsedAddressCount', $this->_unparsedAddressCount);
 875       if ($this->_duplicateCount) {
 876         $store->set('duplicatesFileName', $this->_duplicateFileName);
 877       }
 878       if ($this->_unparsedAddressCount) {
 879         $store->set('errorsFileName', $this->_errorFileName);
 880       }
 881     }
 882     //echo "$this->_totalCount,$this->_invalidRowCount,$this->_conflictCount,$this->_duplicateCount";
 883   }
 884
 885   /**
 886    * Export data to a CSV file
 887    *
 888    * @param string $filename
 889    * @param array $header
 890    * @param data $data
 891    *
 892    * @return void
 893    * @access public
 894    */
 895   static function exportCSV($fileName, $header, $data) {
 896
 897     if (file_exists($fileName) && !is_writable($fileName)) {
 898       CRM_Core_Error::movedSiteError($fileName);
 899     }
 900     //hack to remove '_status', '_statusMsg' and '_id' from error file
 901     $errorValues = array();
 902     $dbRecordStatus = array('IMPORTED', 'ERROR', 'DUPLICATE', 'INVALID', 'NEW');
 903     foreach ($data as $rowCount => $rowValues) {
 904       $count = 0;
 905       foreach ($rowValues as $key => $val) {
 906         if (in_array($val, $dbRecordStatus) && $count == (count($rowValues) - 3)) {
 907           break;
 908         }
 909         $errorValues[$rowCount][$key] = $val;
 910         $count++;
 911       }
 912     }
 913     $data = $errorValues;
 914
 915     $output = array();
 916     $fd = fopen($fileName, 'w');
 917
 918     foreach ($header as $key => $value) {
 919       $header[$key] = "\"$value\"";
 920     }
 921     $config = CRM_Core_Config::singleton();
 922     $output[] = implode($config->fieldSeparator, $header);
 923
 924     foreach ($data as $datum) {
 925       foreach ($datum as $key => $value) {
 926         $datum[$key] = "\"$value\"";
 927       }
 928       $output[] = implode($config->fieldSeparator, $datum);
 929     }
 930     fwrite($fd, implode("\n", $output));
 931     fclose($fd);
 932   }
 933
 934   /**
 935    * Update the record with PK $id in the import database table
 936    *
 937    * @param int $id
 938    * @param array $params
 939    *
 940    * @return void
 941    * @access public
 942    */
 943   public function updateImportRecord($id, &$params) {
 944     $statusFieldName = $this->_statusFieldName;
 945     $primaryKeyName = $this->_primaryKeyName;
 946
 947     if ($statusFieldName && $primaryKeyName) {
 948       $dao = new CRM_Core_DAO();
 949       $db = $dao->getDatabaseConnection();
 950
 951       $query = "UPDATE $this->_tableName
 952                       SET    $statusFieldName      = ?,
 953                              ${statusFieldName}Msg = ?
 954                       WHERE  $primaryKeyName       = ?";
 955       $args = array(
 956         $params[$statusFieldName],
 957         CRM_Utils_Array::value("${statusFieldName}Msg", $params),
 958         $id,
 959       );
 960
 961       //print "Running query: $query<br/>With arguments: ".$params[$statusFieldName].", ".$params["${statusFieldName}Msg"].", $id<br/>";
 962
 963       $db->query($query, $args);
 964     }
 965   }
 966
 967   function errorFileName($type) {
 968     $fileName = NULL;
 969     if (empty($type)) {
 970       return $fileName;
 971     }
 972
 973     $config = CRM_Core_Config::singleton();
 974     $fileName = $config->uploadDir . "sqlImport";
 975     switch ($type) {
 976       case CRM_Import_Parser::ERROR:
 977         $fileName .= '.errors';
 978         break;
 979
 980       case CRM_Import_Parser::CONFLICT:
 981         $fileName .= '.conflicts';
 982         break;
 983
 984       case CRM_Import_Parser::DUPLICATE:
 985         $fileName .= '.duplicates';
 986         break;
 987
 988       case CRM_Import_Parser::NO_MATCH:
 989         $fileName .= '.mismatch';
 990         break;
 991
 992       case CRM_Import_Parser::UNPARSED_ADDRESS_WARNING:
 993         $fileName .= '.unparsedAddress';
 994         break;
 995     }
 996
 997     return $fileName;
 998   }
 999
1000   function saveFileName($type) {
1001     $fileName = NULL;
1002     if (empty($type)) {
1003       return $fileName;
1004     }
1005     switch ($type) {
1006       case CRM_Import_Parser::ERROR:
1007         $fileName = 'Import_Errors.csv';
1008         break;
1009
1010       case CRM_Import_Parser::CONFLICT:
1011         $fileName = 'Import_Conflicts.csv';
1012         break;
1013
1014       case CRM_Import_Parser::DUPLICATE:
1015         $fileName = 'Import_Duplicates.csv';
1016         break;
1017
1018       case CRM_Import_Parser::NO_MATCH:
1019         $fileName = 'Import_Mismatch.csv';
1020         break;
1021
1022       case CRM_Import_Parser::UNPARSED_ADDRESS_WARNING:
1023         $fileName = 'Import_Unparsed_Address.csv';
1024         break;
1025     }
1026
1027     return $fileName;
1028   }
1029 }
1030