[civicrm-core.git] / CRM / Custom / Import / Parser.php

<?php
/*
 +--------------------------------------------------------------------+
 | CiviCRM version 4.6                                                |
 +--------------------------------------------------------------------+
 | Copyright CiviCRM LLC (c) 2004-2014                                |
 +--------------------------------------------------------------------+
 | This file is a part of CiviCRM.                                    |
 |                                                                    |
 | CiviCRM is free software; you can copy, modify, and distribute it  |
 | under the terms of the GNU Affero General Public License           |
 | Version 3, 19 November 2007 and the CiviCRM Licensing Exception.   |
 |                                                                    |
 | CiviCRM is distributed in the hope that it will be useful, but     |
 | WITHOUT ANY WARRANTY; without even the implied warranty of         |
 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.               |
 | See the GNU Affero General Public License for more details.        |
 |                                                                    |
 | You should have received a copy of the GNU Affero General Public   |
 | License and the CiviCRM Licensing Exception along                  |
 | with this program; if not, contact CiviCRM LLC                     |
 | at info[AT]civicrm[DOT]org. If you have questions about the        |
 | GNU Affero General Public License or the licensing of CiviCRM,     |
 | see the CiviCRM license FAQ at http://civicrm.org/licensing        |
 +--------------------------------------------------------------------+
*/

/**
 *
 * @package CRM
 * @copyright CiviCRM LLC (c) 2004-2014
 * $Id$
 *
 */


abstract class CRM_Custom_Import_Parser extends CRM_Contact_Import_Parser {

  protected $_fileName;

  /**#@+
   * @var integer
   */

  /**
   * Imported file size
   */
  protected $_fileSize;

  /**
   * Separator being used
   */
  protected $_separator;

  /**
   * Total number of lines in file
   */
  protected $_lineCount;

  /**
   * Whether the file has a column header or not
   *
   * @var boolean
   */
  protected $_haveColumnHeader;

  /**
   * @param string $fileName
   * @param string $separator
   * @param int $mapper
   * @param bool $skipColumnHeader
   * @param int|string $mode
   * @param int|string $contactType
   * @param int $onDuplicate
   *
   * @return mixed
   * @throws Exception
   */
  function run($fileName,
    $separator = ',',
    &$mapper,
    $skipColumnHeader = FALSE,
    $mode = self::MODE_PREVIEW,
    $contactType = self::CONTACT_INDIVIDUAL,
    $onDuplicate = self::DUPLICATE_SKIP
  ) {
    if (!is_array($fileName)) {
      CRM_Core_Error::fatal();
    }
    $fileName = $fileName['name'];

    switch ($contactType) {
      case CRM_Import_Parser::CONTACT_INDIVIDUAL:
        $this->_contactType = 'Individual';
        break;

      case CRM_Import_Parser::CONTACT_HOUSEHOLD:
        $this->_contactType = 'Household';
        break;

      case CRM_Import_Parser::CONTACT_ORGANIZATION:
        $this->_contactType = 'Organization';
    }
    $this->init();

    $this->_haveColumnHeader = $skipColumnHeader;

    $this->_separator = $separator;

    $fd = fopen($fileName, "r");
    if (!$fd) {
      return FALSE;
    }

    $this->_lineCount = $this->_warningCount = 0;
    $this->_invalidRowCount = $this->_validCount = 0;
    $this->_totalCount = $this->_conflictCount = 0;

    $this->_errors = array();
    $this->_warnings = array();
    $this->_conflicts = array();

    $this->_fileSize = number_format(filesize($fileName) / 1024.0, 2);

    if ($mode == self::MODE_MAPFIELD) {
      $this->_rows = array();
    }
    else {
      $this->_activeFieldCount = count($this->_activeFields);
    }

    while (!feof($fd)) {
      $this->_lineCount++;

      $values = fgetcsv($fd, 8192, $separator);
      if (!$values) {
        continue;
      }

      self::encloseScrub($values);

      // skip column header if we're not in mapfield mode
      if ($mode != self::MODE_MAPFIELD && $skipColumnHeader) {
        $skipColumnHeader = FALSE;
        continue;
      }

      /* trim whitespace around the values */

      $empty = TRUE;
      foreach ($values as $k => $v) {
        $values[$k] = trim($v, " \t\r\n");
      }

      if (CRM_Utils_System::isNull($values)) {
        continue;
      }

      $this->_totalCount++;

      if ($mode == self::MODE_MAPFIELD) {
        $returnCode = $this->mapField($values);
      }
      elseif ($mode == self::MODE_PREVIEW) {
        $returnCode = $this->preview($values);
      }
      elseif ($mode == self::MODE_SUMMARY) {
        $returnCode = $this->summary($values);
      }
      elseif ($mode == self::MODE_IMPORT) {
        $returnCode = $this->import($onDuplicate, $values);
      }
      else {
        $returnCode = self::ERROR;
      }

      // note that a line could be valid but still produce a warning
      if ($returnCode & self::VALID) {
        $this->_validCount++;
        if ($mode == self::MODE_MAPFIELD) {
          $this->_rows[] = $values;
          $this->_activeFieldCount = max($this->_activeFieldCount, count($values));
        }
      }

      if ($returnCode & self::WARNING) {
        $this->_warningCount++;
        if ($this->_warningCount < $this->_maxWarningCount) {
          $this->_warningCount[] = $line;
        }
      }

      if ($returnCode & self::ERROR) {
        $this->_invalidRowCount++;
        if ($this->_invalidRowCount < $this->_maxErrorCount) {
          $recordNumber = $this->_lineCount;
          if ($this->_haveColumnHeader) {
            $recordNumber--;
          }
          array_unshift($values, $recordNumber);
          $this->_errors[] = $values;
        }
      }

      if ($returnCode & self::CONFLICT) {
        $this->_conflictCount++;
        $recordNumber = $this->_lineCount;
        if ($this->_haveColumnHeader) {
          $recordNumber--;
        }
        array_unshift($values, $recordNumber);
        $this->_conflicts[] = $values;
      }

      if ($returnCode & self::DUPLICATE) {
        if ($returnCode & self::MULTIPLE_DUPE) {
          /* TODO: multi-dupes should be counted apart from singles
                     * on non-skip action */
        }
        $this->_duplicateCount++;
        $recordNumber = $this->_lineCount;
        if ($this->_haveColumnHeader) {
          $recordNumber--;
        }
        array_unshift($values, $recordNumber);
        $this->_duplicates[] = $values;
        if ($onDuplicate != self::DUPLICATE_SKIP) {
          $this->_validCount++;
        }
      }

      // we give the derived class a way of aborting the process
      // note that the return code could be multiple code or'ed together
      if ($returnCode & self::STOP) {
        break;
      }

      // if we are done processing the maxNumber of lines, break
      if ($this->_maxLinesToProcess > 0 && $this->_validCount >= $this->_maxLinesToProcess) {
        break;
      }
    }

    fclose($fd);


    if ($mode == self::MODE_PREVIEW || $mode == self::MODE_IMPORT) {
      $customHeaders = $mapper;

      $customfields = CRM_Core_BAO_CustomField::getFields('Activity');
      foreach ($customHeaders as $key => $value) {
        if ($id = CRM_Core_BAO_CustomField::getKeyID($value)) {
          $customHeaders[$key] = $customfields[$id][0];
        }
      }
      if ($this->_invalidRowCount) {
        // removed view url for invlaid contacts
        $headers = array_merge(array(ts('Line Number'),
            ts('Reason'),
          ),
          $customHeaders
        );
        $this->_errorFileName = self::errorFileName(self::ERROR);
        self::exportCSV($this->_errorFileName, $headers, $this->_errors);
      }
      if ($this->_conflictCount) {
        $headers = array_merge(array(ts('Line Number'),
            ts('Reason'),
          ),
          $customHeaders
        );
        $this->_conflictFileName = self::errorFileName(self::CONFLICT);
        self::exportCSV($this->_conflictFileName, $headers, $this->_conflicts);
      }
      if ($this->_duplicateCount) {
        $headers = array_merge(array(ts('Line Number'),
            ts('View Activity History URL'),
          ),
          $customHeaders
        );

        $this->_duplicateFileName = self::errorFileName(self::DUPLICATE);
        self::exportCSV($this->_duplicateFileName, $headers, $this->_duplicates);
      }
    }
    return $this->fini();
  }

  /**
   * Given a list of the importable field keys that the user has selected
   * set the active fields array to this list
   *
   * @param array mapped array of values
   *
   * @return void
   */
  public function setActiveFields($fieldKeys) {
    $this->_activeFieldCount = count($fieldKeys);
    foreach ($fieldKeys as $key) {
      if (empty($this->_fields[$key])) {
        $this->_activeFields[] = new CRM_Custom_Import_Field('', ts('- do not import -'));
      }
      else {
        $this->_activeFields[] = clone($this->_fields[$key]);
      }
    }
  }

  /**
   * Format the field values for input to the api
   *
   * @return array (reference ) associative array of name/value pairs
   */
  public function &getActiveFieldParams() {
    $params = array();
    for ($i = 0; $i < $this->_activeFieldCount; $i++) {
      if (isset($this->_activeFields[$i]->_value)
        && !isset($params[$this->_activeFields[$i]->_name])
        && !isset($this->_activeFields[$i]->_related)
      ) {
        $params[$this->_activeFields[$i]->_name] = $this->_activeFields[$i]->_value;
      }
    }
    return $params;
  }

  /**
   * Store parser values
   *
   * @param CRM_Core_Session $store
   *
   * @param int $mode
   *
   * @return void
   */
  public function set($store, $mode = self::MODE_SUMMARY) {
    $store->set('fileSize', $this->_fileSize);
    $store->set('lineCount', $this->_lineCount);
    $store->set('seperator', $this->_separator);
    $store->set('fields', $this->getSelectValues());
    $store->set('fieldTypes', $this->getSelectTypes());

    $store->set('headerPatterns', $this->getHeaderPatterns());
    $store->set('dataPatterns', $this->getDataPatterns());
    $store->set('columnCount', $this->_activeFieldCount);
    $store->set('_entity', $this->_entity);
    $store->set('totalRowCount', $this->_totalCount);
    $store->set('validRowCount', $this->_validCount);
    $store->set('invalidRowCount', $this->_invalidRowCount);
    $store->set('conflictRowCount', $this->_conflictCount);

    switch ($this->_contactType) {
      case 'Individual':
        $store->set('contactType', CRM_Import_Parser::CONTACT_INDIVIDUAL);
        break;

      case 'Household':
        $store->set('contactType', CRM_Import_Parser::CONTACT_HOUSEHOLD);
        break;

      case 'Organization':
        $store->set('contactType', CRM_Import_Parser::CONTACT_ORGANIZATION);
    }

    if ($this->_invalidRowCount) {
      $store->set('errorsFileName', $this->_errorFileName);
    }
    if ($this->_conflictCount) {
      $store->set('conflictsFileName', $this->_conflictFileName);
    }
    if (isset($this->_rows) && !empty($this->_rows)) {
      $store->set('dataValues', $this->_rows);
    }

    if ($mode == self::MODE_IMPORT) {
      $store->set('duplicateRowCount', $this->_duplicateCount);
      if ($this->_duplicateCount) {
        $store->set('duplicatesFileName', $this->_duplicateFileName);
      }
    }
  }
}
Commit	Line	Data
9ff5f6c0 N	1	<?php
	2	/*
	3	+--------------------------------------------------------------------+
39de6fd5	4	\| CiviCRM version 4.6 \|
9ff5f6c0	5	+--------------------------------------------------------------------+
06b69b18	6	\| Copyright CiviCRM LLC (c) 2004-2014 \|
9ff5f6c0 N	7	+--------------------------------------------------------------------+
	8	\| This file is a part of CiviCRM. \|
	9	\| \|
	10	\| CiviCRM is free software; you can copy, modify, and distribute it \|
	11	\| under the terms of the GNU Affero General Public License \|
	12	\| Version 3, 19 November 2007 and the CiviCRM Licensing Exception. \|
	13	\| \|
	14	\| CiviCRM is distributed in the hope that it will be useful, but \|
	15	\| WITHOUT ANY WARRANTY; without even the implied warranty of \|
	16	\| MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. \|
	17	\| See the GNU Affero General Public License for more details. \|
	18	\| \|
	19	\| You should have received a copy of the GNU Affero General Public \|
	20	\| License and the CiviCRM Licensing Exception along \|
	21	\| with this program; if not, contact CiviCRM LLC \|
	22	\| at info[AT]civicrm[DOT]org. If you have questions about the \|
	23	\| GNU Affero General Public License or the licensing of CiviCRM, \|
	24	\| see the CiviCRM license FAQ at http://civicrm.org/licensing \|
	25	+--------------------------------------------------------------------+
	26	*/
	27
	28	/**
	29	*
	30	* @package CRM
06b69b18	31	* @copyright CiviCRM LLC (c) 2004-2014
9ff5f6c0 N	32	* $Id$
	33	*
	34	*/
	35
	36
	37	abstract class CRM_Custom_Import_Parser extends CRM_Contact_Import_Parser {
	38
	39	protected $_fileName;
	40
	41	/**#@+
9ff5f6c0 N	42	* @var integer
	43	*/
	44
	45	/**
100fef9d	46	* Imported file size
9ff5f6c0 N	47	*/
	48	protected $_fileSize;
	49
	50	/**
100fef9d	51	* Separator being used
9ff5f6c0 N	52	*/
	53	protected $_separator;
	54
	55	/**
100fef9d	56	* Total number of lines in file
9ff5f6c0 N	57	*/
	58	protected $_lineCount;
	59
	60	/**
100fef9d	61	* Whether the file has a column header or not
9ff5f6c0 N	62	*
	63	* @var boolean
	64	*/
	65	protected $_haveColumnHeader;
	66
e0ef6999	67	/**
100fef9d	68	* @param string $fileName
e0ef6999 EM	69	* @param string $separator
	70	* @param int $mapper
	71	* @param bool $skipColumnHeader
	72	* @param int\|string $mode
	73	* @param int\|string $contactType
	74	* @param int $onDuplicate
	75	*
	76	* @return mixed
	77	* @throws Exception
	78	*/
9ff5f6c0 N	79	function run($fileName,
	80	$separator = ',',
	81	&$mapper,
	82	$skipColumnHeader = FALSE,
	83	$mode = self::MODE_PREVIEW,
	84	$contactType = self::CONTACT_INDIVIDUAL,
	85	$onDuplicate = self::DUPLICATE_SKIP
	86	) {
	87	if (!is_array($fileName)) {
	88	CRM_Core_Error::fatal();
	89	}
	90	$fileName = $fileName['name'];
	91
	92	switch ($contactType) {
	93	case CRM_Import_Parser::CONTACT_INDIVIDUAL:
	94	$this->_contactType = 'Individual';
	95	break;
	96
	97	case CRM_Import_Parser::CONTACT_HOUSEHOLD:
	98	$this->_contactType = 'Household';
	99	break;
	100
	101	case CRM_Import_Parser::CONTACT_ORGANIZATION:
	102	$this->_contactType = 'Organization';
	103	}
	104	$this->init();
	105
	106	$this->_haveColumnHeader = $skipColumnHeader;
	107
	108	$this->_separator = $separator;
	109
	110	$fd = fopen($fileName, "r");
	111	if (!$fd) {
	112	return FALSE;
	113	}
	114
	115	$this->_lineCount = $this->_warningCount = 0;
	116	$this->_invalidRowCount = $this->_validCount = 0;
	117	$this->_totalCount = $this->_conflictCount = 0;
	118
	119	$this->_errors = array();
	120	$this->_warnings = array();
	121	$this->_conflicts = array();
	122
	123	$this->_fileSize = number_format(filesize($fileName) / 1024.0, 2);
	124
	125	if ($mode == self::MODE_MAPFIELD) {
	126	$this->_rows = array();
	127	}
	128	else {
	129	$this->_activeFieldCount = count($this->_activeFields);
	130	}
	131
	132	while (!feof($fd)) {
	133	$this->_lineCount++;
	134
	135	$values = fgetcsv($fd, 8192, $separator);
	136	if (!$values) {
	137	continue;
	138	}
	139
	140	self::encloseScrub($values);
	141
	142	// skip column header if we're not in mapfield mode
143	if ($mode != self::MODE_MAPFIELD && $skipColumnHeader) {
144	$skipColumnHeader = FALSE;
145	continue;
146	}
147
148	/* trim whitespace around the values */
149
150	$empty = TRUE;
151	foreach ($values as $k => $v) {
152	$values[$k] = trim($v, " \t\r\n");
153	}
154
155	if (CRM_Utils_System::isNull($values)) {
156	continue;
157	}
158
159	$this->_totalCount++;
160
161	if ($mode == self::MODE_MAPFIELD) {
162	$returnCode = $this->mapField($values);
163	}
164	elseif ($mode == self::MODE_PREVIEW) {
165	$returnCode = $this->preview($values);
166	}
167	elseif ($mode == self::MODE_SUMMARY) {
168	$returnCode = $this->summary($values);
169	}
170	elseif ($mode == self::MODE_IMPORT) {
171	$returnCode = $this->import($onDuplicate, $values);
172	}
173	else {
174	$returnCode = self::ERROR;
175	}
176
177	// note that a line could be valid but still produce a warning
178	if ($returnCode & self::VALID) {
179	$this->_validCount++;
180	if ($mode == self::MODE_MAPFIELD) {
181	$this->_rows[] = $values;
182	$this->_activeFieldCount = max($this->_activeFieldCount, count($values));
183	}
184	}
185
186	if ($returnCode & self::WARNING) {
187	$this->_warningCount++;
188	if ($this->_warningCount < $this->_maxWarningCount) {
189	$this->_warningCount[] = $line;
190	}
191	}
192
193	if ($returnCode & self::ERROR) {
194	$this->_invalidRowCount++;
195	if ($this->_invalidRowCount < $this->_maxErrorCount) {
196	$recordNumber = $this->_lineCount;
197	if ($this->_haveColumnHeader) {
198	$recordNumber--;
199	}
200	array_unshift($values, $recordNumber);
201	$this->_errors[] = $values;
202	}
203	}
204
205	if ($returnCode & self::CONFLICT) {
206	$this->_conflictCount++;
207	$recordNumber = $this->_lineCount;
208	if ($this->_haveColumnHeader) {
209	$recordNumber--;
210	}
211	array_unshift($values, $recordNumber);
212	$this->_conflicts[] = $values;
213	}
214
215	if ($returnCode & self::DUPLICATE) {
216	if ($returnCode & self::MULTIPLE_DUPE) {
217	/* TODO: multi-dupes should be counted apart from singles
218	* on non-skip action */
219	}
220	$this->_duplicateCount++;
221	$recordNumber = $this->_lineCount;
222	if ($this->_haveColumnHeader) {
223	$recordNumber--;
224	}
225	array_unshift($values, $recordNumber);
226	$this->_duplicates[] = $values;
227	if ($onDuplicate != self::DUPLICATE_SKIP) {
228	$this->_validCount++;
229	}
230	}
231
232	// we give the derived class a way of aborting the process
233	// note that the return code could be multiple code or'ed together
234	if ($returnCode & self::STOP) {
235	break;
236	}
237
238	// if we are done processing the maxNumber of lines, break
239	if ($this->_maxLinesToProcess > 0 && $this->_validCount >= $this->_maxLinesToProcess) {
240	break;
241	}
242	}
243
244	fclose($fd);
245
246
247	if ($mode == self::MODE_PREVIEW \|\| $mode == self::MODE_IMPORT) {
248	$customHeaders = $mapper;
249
250	$customfields = CRM_Core_BAO_CustomField::getFields('Activity');
251	foreach ($customHeaders as $key => $value) {
252	if ($id = CRM_Core_BAO_CustomField::getKeyID($value)) {
253	$customHeaders[$key] = $customfields[$id][0];
254	}
255	}
256	if ($this->_invalidRowCount) {
257	// removed view url for invlaid contacts
258	$headers = array_merge(array(ts('Line Number'),
259	ts('Reason'),
260	),
261	$customHeaders
262	);
263	$this->_errorFileName = self::errorFileName(self::ERROR);
264	self::exportCSV($this->_errorFileName, $headers, $this->_errors);
265	}
266	if ($this->_conflictCount) {
267	$headers = array_merge(array(ts('Line Number'),
268	ts('Reason'),
269	),
270	$customHeaders
271	);
272	$this->_conflictFileName = self::errorFileName(self::CONFLICT);
273	self::exportCSV($this->_conflictFileName, $headers, $this->_conflicts);
274	}
275	if ($this->_duplicateCount) {
276	$headers = array_merge(array(ts('Line Number'),
277	ts('View Activity History URL'),
278	),
279	$customHeaders
280	);
281
282	$this->_duplicateFileName = self::errorFileName(self::DUPLICATE);
283	self::exportCSV($this->_duplicateFileName, $headers, $this->_duplicates);
284	}
285	}
286	return $this->fini();
287	}
288
289	/**
290	* Given a list of the importable field keys that the user has selected
291	* set the active fields array to this list
292	*
293	* @param array mapped array of values
294	*
295	* @return void
9ff5f6c0	296	*/
00be9182	297	public function setActiveFields($fieldKeys) {
9ff5f6c0 N	298	$this->_activeFieldCount = count($fieldKeys);
	299	foreach ($fieldKeys as $key) {
	300	if (empty($this->_fields[$key])) {
	301	$this->_activeFields[] = new CRM_Custom_Import_Field('', ts('- do not import -'));
	302	}
	303	else {
	304	$this->_activeFields[] = clone($this->_fields[$key]);
	305	}
	306	}
	307	}
	308
	309	/**
100fef9d	310	* Format the field values for input to the api
9ff5f6c0 N	311	*
9ff5f6c0 N	312	* @return array (reference ) associative array of name/value pairs
9ff5f6c0	313	*/
00be9182	314	public function &getActiveFieldParams() {
9ff5f6c0 N	315	$params = array();
	316	for ($i = 0; $i < $this->_activeFieldCount; $i++) {
	317	if (isset($this->_activeFields[$i]->_value)
	318	&& !isset($params[$this->_activeFields[$i]->_name])
	319	&& !isset($this->_activeFields[$i]->_related)
	320	) {
	321	$params[$this->_activeFields[$i]->_name] = $this->_activeFields[$i]->_value;
	322	}
	323	}
	324	return $params;
	325	}
	326
	327	/**
	328	* Store parser values
	329	*
	330	* @param CRM_Core_Session $store
	331	*
da6b46f4 EM	332	* @param int $mode
da6b46f4 EM	333	*
9ff5f6c0	334	* @return void
9ff5f6c0	335	*/
00be9182	336	public function set($store, $mode = self::MODE_SUMMARY) {
9ff5f6c0 N	337	$store->set('fileSize', $this->_fileSize);
	338	$store->set('lineCount', $this->_lineCount);
	339	$store->set('seperator', $this->_separator);
	340	$store->set('fields', $this->getSelectValues());
	341	$store->set('fieldTypes', $this->getSelectTypes());
	342
	343	$store->set('headerPatterns', $this->getHeaderPatterns());
	344	$store->set('dataPatterns', $this->getDataPatterns());
	345	$store->set('columnCount', $this->_activeFieldCount);
	346	$store->set('_entity', $this->_entity);
	347	$store->set('totalRowCount', $this->_totalCount);
	348	$store->set('validRowCount', $this->_validCount);
	349	$store->set('invalidRowCount', $this->_invalidRowCount);
	350	$store->set('conflictRowCount', $this->_conflictCount);
	351
	352	switch ($this->_contactType) {
	353	case 'Individual':
	354	$store->set('contactType', CRM_Import_Parser::CONTACT_INDIVIDUAL);
	355	break;
	356
	357	case 'Household':
	358	$store->set('contactType', CRM_Import_Parser::CONTACT_HOUSEHOLD);
	359	break;
	360
	361	case 'Organization':
	362	$store->set('contactType', CRM_Import_Parser::CONTACT_ORGANIZATION);
	363	}
	364
	365	if ($this->_invalidRowCount) {
	366	$store->set('errorsFileName', $this->_errorFileName);
	367	}
	368	if ($this->_conflictCount) {
	369	$store->set('conflictsFileName', $this->_conflictFileName);
	370	}
	371	if (isset($this->_rows) && !empty($this->_rows)) {
	372	$store->set('dataValues', $this->_rows);
	373	}
	374
	375	if ($mode == self::MODE_IMPORT) {
	376	$store->set('duplicateRowCount', $this->_duplicateCount);
	377	if ($this->_duplicateCount) {
	378	$store->set('duplicatesFileName', $this->_duplicateFileName);
	379	}
	380	}
	381	}
	382	}