[civicrm-core.git] / CRM / Custom / Import / Parser.php

<?php
/*
 +--------------------------------------------------------------------+
 | CiviCRM version 4.5                                                |
 +--------------------------------------------------------------------+
 | Copyright CiviCRM LLC (c) 2004-2014                                |
 +--------------------------------------------------------------------+
 | This file is a part of CiviCRM.                                    |
 |                                                                    |
 | CiviCRM is free software; you can copy, modify, and distribute it  |
 | under the terms of the GNU Affero General Public License           |
 | Version 3, 19 November 2007 and the CiviCRM Licensing Exception.   |
 |                                                                    |
 | CiviCRM is distributed in the hope that it will be useful, but     |
 | WITHOUT ANY WARRANTY; without even the implied warranty of         |
 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.               |
 | See the GNU Affero General Public License for more details.        |
 |                                                                    |
 | You should have received a copy of the GNU Affero General Public   |
 | License and the CiviCRM Licensing Exception along                  |
 | with this program; if not, contact CiviCRM LLC                     |
 | at info[AT]civicrm[DOT]org. If you have questions about the        |
 | GNU Affero General Public License or the licensing of CiviCRM,     |
 | see the CiviCRM license FAQ at http://civicrm.org/licensing        |
 +--------------------------------------------------------------------+
*/

/**
 *
 * @package CRM
 * @copyright CiviCRM LLC (c) 2004-2014
 * $Id$
 *
 */


abstract class CRM_Custom_Import_Parser extends CRM_Contact_Import_Parser {

  protected $_fileName;

  /**#@+
   * @access protected
   * @var integer
   */

  /**
   * imported file size
   */
  protected $_fileSize;

  /**
   * separator being used
   */
  protected $_separator;

  /**
   * total number of lines in file
   */
  protected $_lineCount;

  /**
   * whether the file has a column header or not
   *
   * @var boolean
   */
  protected $_haveColumnHeader;

  /**
   * @param $fileName
   * @param string $separator
   * @param int $mapper
   * @param bool $skipColumnHeader
   * @param int|string $mode
   * @param int|string $contactType
   * @param int $onDuplicate
   *
   * @return mixed
   * @throws Exception
   */
  function run($fileName,
    $separator = ',',
    &$mapper,
    $skipColumnHeader = FALSE,
    $mode = self::MODE_PREVIEW,
    $contactType = self::CONTACT_INDIVIDUAL,
    $onDuplicate = self::DUPLICATE_SKIP
  ) {
    if (!is_array($fileName)) {
      CRM_Core_Error::fatal();
    }
    $fileName = $fileName['name'];

    switch ($contactType) {
      case CRM_Import_Parser::CONTACT_INDIVIDUAL:
        $this->_contactType = 'Individual';
        break;

      case CRM_Import_Parser::CONTACT_HOUSEHOLD:
        $this->_contactType = 'Household';
        break;

      case CRM_Import_Parser::CONTACT_ORGANIZATION:
        $this->_contactType = 'Organization';
    }
    $this->init();

    $this->_haveColumnHeader = $skipColumnHeader;

    $this->_separator = $separator;

    $fd = fopen($fileName, "r");
    if (!$fd) {
      return FALSE;
    }

    $this->_lineCount = $this->_warningCount = 0;
    $this->_invalidRowCount = $this->_validCount = 0;
    $this->_totalCount = $this->_conflictCount = 0;

    $this->_errors = array();
    $this->_warnings = array();
    $this->_conflicts = array();

    $this->_fileSize = number_format(filesize($fileName) / 1024.0, 2);

    if ($mode == self::MODE_MAPFIELD) {
      $this->_rows = array();
    }
    else {
      $this->_activeFieldCount = count($this->_activeFields);
    }

    while (!feof($fd)) {
      $this->_lineCount++;

      $values = fgetcsv($fd, 8192, $separator);
      if (!$values) {
        continue;
      }

      self::encloseScrub($values);

      // skip column header if we're not in mapfield mode
      if ($mode != self::MODE_MAPFIELD && $skipColumnHeader) {
        $skipColumnHeader = FALSE;
        continue;
      }

      /* trim whitespace around the values */

      $empty = TRUE;
      foreach ($values as $k => $v) {
        $values[$k] = trim($v, " \t\r\n");
      }

      if (CRM_Utils_System::isNull($values)) {
        continue;
      }

      $this->_totalCount++;

      if ($mode == self::MODE_MAPFIELD) {
        $returnCode = $this->mapField($values);
      }
      elseif ($mode == self::MODE_PREVIEW) {
        $returnCode = $this->preview($values);
      }
      elseif ($mode == self::MODE_SUMMARY) {
        $returnCode = $this->summary($values);
      }
      elseif ($mode == self::MODE_IMPORT) {
        $returnCode = $this->import($onDuplicate, $values);
      }
      else {
        $returnCode = self::ERROR;
      }

      // note that a line could be valid but still produce a warning
      if ($returnCode & self::VALID) {
        $this->_validCount++;
        if ($mode == self::MODE_MAPFIELD) {
          $this->_rows[] = $values;
          $this->_activeFieldCount = max($this->_activeFieldCount, count($values));
        }
      }

      if ($returnCode & self::WARNING) {
        $this->_warningCount++;
        if ($this->_warningCount < $this->_maxWarningCount) {
          $this->_warningCount[] = $line;
        }
      }

      if ($returnCode & self::ERROR) {
        $this->_invalidRowCount++;
        if ($this->_invalidRowCount < $this->_maxErrorCount) {
          $recordNumber = $this->_lineCount;
          if ($this->_haveColumnHeader) {
            $recordNumber--;
          }
          array_unshift($values, $recordNumber);
          $this->_errors[] = $values;
        }
      }

      if ($returnCode & self::CONFLICT) {
        $this->_conflictCount++;
        $recordNumber = $this->_lineCount;
        if ($this->_haveColumnHeader) {
          $recordNumber--;
        }
        array_unshift($values, $recordNumber);
        $this->_conflicts[] = $values;
      }

      if ($returnCode & self::DUPLICATE) {
        if ($returnCode & self::MULTIPLE_DUPE) {
          /* TODO: multi-dupes should be counted apart from singles
                     * on non-skip action */
        }
        $this->_duplicateCount++;
        $recordNumber = $this->_lineCount;
        if ($this->_haveColumnHeader) {
          $recordNumber--;
        }
        array_unshift($values, $recordNumber);
        $this->_duplicates[] = $values;
        if ($onDuplicate != self::DUPLICATE_SKIP) {
          $this->_validCount++;
        }
      }

      // we give the derived class a way of aborting the process
      // note that the return code could be multiple code or'ed together
      if ($returnCode & self::STOP) {
        break;
      }

      // if we are done processing the maxNumber of lines, break
      if ($this->_maxLinesToProcess > 0 && $this->_validCount >= $this->_maxLinesToProcess) {
        break;
      }
    }

    fclose($fd);


    if ($mode == self::MODE_PREVIEW || $mode == self::MODE_IMPORT) {
      $customHeaders = $mapper;

      $customfields = CRM_Core_BAO_CustomField::getFields('Activity');
      foreach ($customHeaders as $key => $value) {
        if ($id = CRM_Core_BAO_CustomField::getKeyID($value)) {
          $customHeaders[$key] = $customfields[$id][0];
        }
      }
      if ($this->_invalidRowCount) {
        // removed view url for invlaid contacts
        $headers = array_merge(array(ts('Line Number'),
            ts('Reason'),
          ),
          $customHeaders
        );
        $this->_errorFileName = self::errorFileName(self::ERROR);
        self::exportCSV($this->_errorFileName, $headers, $this->_errors);
      }
      if ($this->_conflictCount) {
        $headers = array_merge(array(ts('Line Number'),
            ts('Reason'),
          ),
          $customHeaders
        );
        $this->_conflictFileName = self::errorFileName(self::CONFLICT);
        self::exportCSV($this->_conflictFileName, $headers, $this->_conflicts);
      }
      if ($this->_duplicateCount) {
        $headers = array_merge(array(ts('Line Number'),
            ts('View Activity History URL'),
          ),
          $customHeaders
        );

        $this->_duplicateFileName = self::errorFileName(self::DUPLICATE);
        self::exportCSV($this->_duplicateFileName, $headers, $this->_duplicates);
      }
    }
    return $this->fini();
  }

  /**
   * Given a list of the importable field keys that the user has selected
   * set the active fields array to this list
   *
   * @param array mapped array of values
   *
   * @return void
   * @access public
   */
  function setActiveFields($fieldKeys) {
    $this->_activeFieldCount = count($fieldKeys);
    foreach ($fieldKeys as $key) {
      if (empty($this->_fields[$key])) {
        $this->_activeFields[] = new CRM_Custom_Import_Field('', ts('- do not import -'));
      }
      else {
        $this->_activeFields[] = clone($this->_fields[$key]);
      }
    }
  }

  /**
   * function to format the field values for input to the api
   *
   * @return array (reference ) associative array of name/value pairs
   * @access public
   */
  function &getActiveFieldParams() {
    $params = array();
    for ($i = 0; $i < $this->_activeFieldCount; $i++) {
      if (isset($this->_activeFields[$i]->_value)
        && !isset($params[$this->_activeFields[$i]->_name])
        && !isset($this->_activeFields[$i]->_related)
      ) {
        $params[$this->_activeFields[$i]->_name] = $this->_activeFields[$i]->_value;
      }
    }
    return $params;
  }

  /**
   * Store parser values
   *
   * @param CRM_Core_Session $store
   *
   * @param int $mode
   *
   * @return void
   * @access public
   */
  function set($store, $mode = self::MODE_SUMMARY) {
    $store->set('fileSize', $this->_fileSize);
    $store->set('lineCount', $this->_lineCount);
    $store->set('seperator', $this->_separator);
    $store->set('fields', $this->getSelectValues());
    $store->set('fieldTypes', $this->getSelectTypes());

    $store->set('headerPatterns', $this->getHeaderPatterns());
    $store->set('dataPatterns', $this->getDataPatterns());
    $store->set('columnCount', $this->_activeFieldCount);
    $store->set('_entity', $this->_entity);
    $store->set('totalRowCount', $this->_totalCount);
    $store->set('validRowCount', $this->_validCount);
    $store->set('invalidRowCount', $this->_invalidRowCount);
    $store->set('conflictRowCount', $this->_conflictCount);

    switch ($this->_contactType) {
      case 'Individual':
        $store->set('contactType', CRM_Import_Parser::CONTACT_INDIVIDUAL);
        break;

      case 'Household':
        $store->set('contactType', CRM_Import_Parser::CONTACT_HOUSEHOLD);
        break;

      case 'Organization':
        $store->set('contactType', CRM_Import_Parser::CONTACT_ORGANIZATION);
    }

    if ($this->_invalidRowCount) {
      $store->set('errorsFileName', $this->_errorFileName);
    }
    if ($this->_conflictCount) {
      $store->set('conflictsFileName', $this->_conflictFileName);
    }
    if (isset($this->_rows) && !empty($this->_rows)) {
      $store->set('dataValues', $this->_rows);
    }

    if ($mode == self::MODE_IMPORT) {
      $store->set('duplicateRowCount', $this->_duplicateCount);
      if ($this->_duplicateCount) {
        $store->set('duplicatesFileName', $this->_duplicateFileName);
      }
    }
  }
}
Commit	Line	Data
9ff5f6c0 N	1	<?php
	2	/*
	3	+--------------------------------------------------------------------+
06b69b18	4	\| CiviCRM version 4.5 \|
9ff5f6c0	5	+--------------------------------------------------------------------+
06b69b18	6	\| Copyright CiviCRM LLC (c) 2004-2014 \|
9ff5f6c0 N	7	+--------------------------------------------------------------------+
	8	\| This file is a part of CiviCRM. \|
	9	\| \|
	10	\| CiviCRM is free software; you can copy, modify, and distribute it \|
	11	\| under the terms of the GNU Affero General Public License \|
	12	\| Version 3, 19 November 2007 and the CiviCRM Licensing Exception. \|
	13	\| \|
	14	\| CiviCRM is distributed in the hope that it will be useful, but \|
	15	\| WITHOUT ANY WARRANTY; without even the implied warranty of \|
	16	\| MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. \|
	17	\| See the GNU Affero General Public License for more details. \|
	18	\| \|
	19	\| You should have received a copy of the GNU Affero General Public \|
	20	\| License and the CiviCRM Licensing Exception along \|
	21	\| with this program; if not, contact CiviCRM LLC \|
	22	\| at info[AT]civicrm[DOT]org. If you have questions about the \|
	23	\| GNU Affero General Public License or the licensing of CiviCRM, \|
	24	\| see the CiviCRM license FAQ at http://civicrm.org/licensing \|
	25	+--------------------------------------------------------------------+
	26	*/
	27
	28	/**
	29	*
	30	* @package CRM
06b69b18	31	* @copyright CiviCRM LLC (c) 2004-2014
9ff5f6c0 N	32	* $Id$
	33	*
	34	*/
	35
	36
	37	abstract class CRM_Custom_Import_Parser extends CRM_Contact_Import_Parser {
	38
	39	protected $_fileName;
	40
	41	/**#@+
	42	* @access protected
	43	* @var integer
	44	*/
	45
	46	/**
	47	* imported file size
	48	*/
	49	protected $_fileSize;
	50
	51	/**
	52	* separator being used
	53	*/
	54	protected $_separator;
	55
	56	/**
	57	* total number of lines in file
	58	*/
	59	protected $_lineCount;
	60
	61	/**
	62	* whether the file has a column header or not
	63	*
	64	* @var boolean
	65	*/
	66	protected $_haveColumnHeader;
	67
e0ef6999 EM	68	/**
	69	* @param $fileName
	70	* @param string $separator
	71	* @param int $mapper
	72	* @param bool $skipColumnHeader
	73	* @param int\|string $mode
	74	* @param int\|string $contactType
	75	* @param int $onDuplicate
	76	*
	77	* @return mixed
	78	* @throws Exception
	79	*/
9ff5f6c0 N	80	function run($fileName,
	81	$separator = ',',
	82	&$mapper,
	83	$skipColumnHeader = FALSE,
	84	$mode = self::MODE_PREVIEW,
	85	$contactType = self::CONTACT_INDIVIDUAL,
	86	$onDuplicate = self::DUPLICATE_SKIP
	87	) {
	88	if (!is_array($fileName)) {
	89	CRM_Core_Error::fatal();
	90	}
	91	$fileName = $fileName['name'];
	92
	93	switch ($contactType) {
	94	case CRM_Import_Parser::CONTACT_INDIVIDUAL:
	95	$this->_contactType = 'Individual';
	96	break;
	97
	98	case CRM_Import_Parser::CONTACT_HOUSEHOLD:
	99	$this->_contactType = 'Household';
	100	break;
	101
	102	case CRM_Import_Parser::CONTACT_ORGANIZATION:
	103	$this->_contactType = 'Organization';
	104	}
	105	$this->init();
	106
	107	$this->_haveColumnHeader = $skipColumnHeader;
	108
	109	$this->_separator = $separator;
	110
	111	$fd = fopen($fileName, "r");
	112	if (!$fd) {
	113	return FALSE;
	114	}
	115
	116	$this->_lineCount = $this->_warningCount = 0;
	117	$this->_invalidRowCount = $this->_validCount = 0;
	118	$this->_totalCount = $this->_conflictCount = 0;
	119
	120	$this->_errors = array();
	121	$this->_warnings = array();
	122	$this->_conflicts = array();
	123
	124	$this->_fileSize = number_format(filesize($fileName) / 1024.0, 2);
	125
	126	if ($mode == self::MODE_MAPFIELD) {
	127	$this->_rows = array();
	128	}
	129	else {
	130	$this->_activeFieldCount = count($this->_activeFields);
	131	}
	132
	133	while (!feof($fd)) {
	134	$this->_lineCount++;
	135
	136	$values = fgetcsv($fd, 8192, $separator);
	137	if (!$values) {
	138	continue;
	139	}
	140
	141	self::encloseScrub($values);
	142
	143	// skip column header if we're not in mapfield mode
144	if ($mode != self::MODE_MAPFIELD && $skipColumnHeader) {
145	$skipColumnHeader = FALSE;
146	continue;
147	}
148
149	/* trim whitespace around the values */
150
151	$empty = TRUE;
152	foreach ($values as $k => $v) {
153	$values[$k] = trim($v, " \t\r\n");
154	}
155
156	if (CRM_Utils_System::isNull($values)) {
157	continue;
158	}
159
160	$this->_totalCount++;
161
162	if ($mode == self::MODE_MAPFIELD) {
163	$returnCode = $this->mapField($values);
164	}
165	elseif ($mode == self::MODE_PREVIEW) {
166	$returnCode = $this->preview($values);
167	}
168	elseif ($mode == self::MODE_SUMMARY) {
169	$returnCode = $this->summary($values);
170	}
171	elseif ($mode == self::MODE_IMPORT) {
172	$returnCode = $this->import($onDuplicate, $values);
173	}
174	else {
175	$returnCode = self::ERROR;
176	}
177
178	// note that a line could be valid but still produce a warning
179	if ($returnCode & self::VALID) {
180	$this->_validCount++;
181	if ($mode == self::MODE_MAPFIELD) {
182	$this->_rows[] = $values;
183	$this->_activeFieldCount = max($this->_activeFieldCount, count($values));
184	}
185	}
186
187	if ($returnCode & self::WARNING) {
188	$this->_warningCount++;
189	if ($this->_warningCount < $this->_maxWarningCount) {
190	$this->_warningCount[] = $line;
191	}
192	}
193
194	if ($returnCode & self::ERROR) {
195	$this->_invalidRowCount++;
196	if ($this->_invalidRowCount < $this->_maxErrorCount) {
197	$recordNumber = $this->_lineCount;
198	if ($this->_haveColumnHeader) {
199	$recordNumber--;
200	}
201	array_unshift($values, $recordNumber);
202	$this->_errors[] = $values;
203	}
204	}
205
206	if ($returnCode & self::CONFLICT) {
207	$this->_conflictCount++;
208	$recordNumber = $this->_lineCount;
209	if ($this->_haveColumnHeader) {
210	$recordNumber--;
211	}
212	array_unshift($values, $recordNumber);
213	$this->_conflicts[] = $values;
214	}
215
216	if ($returnCode & self::DUPLICATE) {
217	if ($returnCode & self::MULTIPLE_DUPE) {
218	/* TODO: multi-dupes should be counted apart from singles
219	* on non-skip action */
220	}
221	$this->_duplicateCount++;
222	$recordNumber = $this->_lineCount;
223	if ($this->_haveColumnHeader) {
224	$recordNumber--;
225	}
226	array_unshift($values, $recordNumber);
227	$this->_duplicates[] = $values;
228	if ($onDuplicate != self::DUPLICATE_SKIP) {
229	$this->_validCount++;
230	}
231	}
232
233	// we give the derived class a way of aborting the process
234	// note that the return code could be multiple code or'ed together
235	if ($returnCode & self::STOP) {
236	break;
237	}
238
239	// if we are done processing the maxNumber of lines, break
240	if ($this->_maxLinesToProcess > 0 && $this->_validCount >= $this->_maxLinesToProcess) {
241	break;
242	}
243	}
244
245	fclose($fd);
246
247
248	if ($mode == self::MODE_PREVIEW \|\| $mode == self::MODE_IMPORT) {
249	$customHeaders = $mapper;
250
251	$customfields = CRM_Core_BAO_CustomField::getFields('Activity');
252	foreach ($customHeaders as $key => $value) {
253	if ($id = CRM_Core_BAO_CustomField::getKeyID($value)) {
254	$customHeaders[$key] = $customfields[$id][0];
255	}
256	}
257	if ($this->_invalidRowCount) {
258	// removed view url for invlaid contacts
259	$headers = array_merge(array(ts('Line Number'),
260	ts('Reason'),
261	),
262	$customHeaders
263	);
264	$this->_errorFileName = self::errorFileName(self::ERROR);
265	self::exportCSV($this->_errorFileName, $headers, $this->_errors);
266	}
267	if ($this->_conflictCount) {
268	$headers = array_merge(array(ts('Line Number'),
269	ts('Reason'),
270	),
271	$customHeaders
272	);
273	$this->_conflictFileName = self::errorFileName(self::CONFLICT);
274	self::exportCSV($this->_conflictFileName, $headers, $this->_conflicts);
275	}
276	if ($this->_duplicateCount) {
277	$headers = array_merge(array(ts('Line Number'),
278	ts('View Activity History URL'),
279	),
280	$customHeaders
281	);
282
283	$this->_duplicateFileName = self::errorFileName(self::DUPLICATE);
284	self::exportCSV($this->_duplicateFileName, $headers, $this->_duplicates);
285	}
286	}
287	return $this->fini();
288	}
289
290	/**
291	* Given a list of the importable field keys that the user has selected
292	* set the active fields array to this list
293	*
294	* @param array mapped array of values
295	*
296	* @return void
297	* @access public
298	*/
299	function setActiveFields($fieldKeys) {
300	$this->_activeFieldCount = count($fieldKeys);
301	foreach ($fieldKeys as $key) {
302	if (empty($this->_fields[$key])) {
303	$this->_activeFields[] = new CRM_Custom_Import_Field('', ts('- do not import -'));
304	}
305	else {
306	$this->_activeFields[] = clone($this->_fields[$key]);
307	}
308	}
309	}
310
311	/**
312	* function to format the field values for input to the api
313	*
314	* @return array (reference ) associative array of name/value pairs
315	* @access public
316	*/
317	function &getActiveFieldParams() {
318	$params = array();
319	for ($i = 0; $i < $this->_activeFieldCount; $i++) {
320	if (isset($this->_activeFields[$i]->_value)
321	&& !isset($params[$this->_activeFields[$i]->_name])
322	&& !isset($this->_activeFields[$i]->_related)
323	) {
324	$params[$this->_activeFields[$i]->_name] = $this->_activeFields[$i]->_value;
325	}
326	}
327	return $params;
328	}
329
330	/**
331	* Store parser values
332	*
333	* @param CRM_Core_Session $store
334	*
da6b46f4 EM	335	* @param int $mode
da6b46f4 EM	336	*
9ff5f6c0 N	337	* @return void
	338	* @access public
	339	*/
	340	function set($store, $mode = self::MODE_SUMMARY) {
	341	$store->set('fileSize', $this->_fileSize);
	342	$store->set('lineCount', $this->_lineCount);
	343	$store->set('seperator', $this->_separator);
	344	$store->set('fields', $this->getSelectValues());
	345	$store->set('fieldTypes', $this->getSelectTypes());
	346
	347	$store->set('headerPatterns', $this->getHeaderPatterns());
	348	$store->set('dataPatterns', $this->getDataPatterns());
	349	$store->set('columnCount', $this->_activeFieldCount);
	350	$store->set('_entity', $this->_entity);
	351	$store->set('totalRowCount', $this->_totalCount);
	352	$store->set('validRowCount', $this->_validCount);
	353	$store->set('invalidRowCount', $this->_invalidRowCount);
	354	$store->set('conflictRowCount', $this->_conflictCount);
	355
	356	switch ($this->_contactType) {
	357	case 'Individual':
	358	$store->set('contactType', CRM_Import_Parser::CONTACT_INDIVIDUAL);
	359	break;
	360
	361	case 'Household':
	362	$store->set('contactType', CRM_Import_Parser::CONTACT_HOUSEHOLD);
	363	break;
	364
	365	case 'Organization':
	366	$store->set('contactType', CRM_Import_Parser::CONTACT_ORGANIZATION);
	367	}
	368
	369	if ($this->_invalidRowCount) {
	370	$store->set('errorsFileName', $this->_errorFileName);
	371	}
	372	if ($this->_conflictCount) {
	373	$store->set('conflictsFileName', $this->_conflictFileName);
	374	}
	375	if (isset($this->_rows) && !empty($this->_rows)) {
	376	$store->set('dataValues', $this->_rows);
	377	}
	378
	379	if ($mode == self::MODE_IMPORT) {
	380	$store->set('duplicateRowCount', $this->_duplicateCount);
	381	if ($this->_duplicateCount) {
	382	$store->set('duplicatesFileName', $this->_duplicateFileName);
	383	}
	384	}
	385	}
	386	}
	387