From 8cebffb2608fb2f197bae07a6093fa8576b7dd14 Mon Sep 17 00:00:00 2001 From: Jitendra Purohit Date: Fri, 30 Jun 2017 14:05:31 +0530 Subject: [PATCH] Add progress bar for all import process --- CRM/Activity/Import/Form/Preview.php | 5 +- CRM/Activity/Import/Parser.php | 11 ++- CRM/Contact/Import/Form/Preview.php | 9 +-- CRM/Contact/Import/Parser.php | 47 +---------- CRM/Contribute/Import/Form/Preview.php | 5 +- CRM/Contribute/Import/Parser.php | 11 ++- CRM/Import/Form/Preview.php | 14 +++- CRM/Import/Parser.php | 52 ++++++++++++ CRM/Member/Import/Form/Preview.php | 6 +- CRM/Member/Import/Parser.php | 12 ++- .../CRM/Activity/Import/Form/Preview.tpl | 1 + templates/CRM/Contact/Import/Form/Preview.tpl | 54 +------------ .../CRM/Contribute/Import/Form/Preview.tpl | 1 + templates/CRM/Member/Import/Form/Preview.tpl | 1 + templates/CRM/common/importProgress.tpl | 81 +++++++++++++++++++ 15 files changed, 196 insertions(+), 114 deletions(-) create mode 100644 templates/CRM/common/importProgress.tpl diff --git a/CRM/Activity/Import/Form/Preview.php b/CRM/Activity/Import/Form/Preview.php index b74b0d6f13..b309805758 100644 --- a/CRM/Activity/Import/Form/Preview.php +++ b/CRM/Activity/Import/Form/Preview.php @@ -94,6 +94,7 @@ class CRM_Activity_Import_Form_Preview extends CRM_Import_Form_Preview { 'downloadConflictRecordsUrl', 'downloadMismatchRecordsUrl', ); + $this->setStatusUrl(); foreach ($properties as $property) { $this->assign($property, $this->get($property)); @@ -151,7 +152,9 @@ class CRM_Activity_Import_Form_Preview extends CRM_Import_Form_Preview { $mapperFields, $skipColumnHeader, CRM_Import_Parser::MODE_IMPORT, - $onDuplicate + $onDuplicate, + $this->get('statusID'), + $this->get('totalRowCount') ); // add all the necessary variables to the form diff --git a/CRM/Activity/Import/Parser.php b/CRM/Activity/Import/Parser.php index 62af0b7a35..4103c7cc02 100644 --- a/CRM/Activity/Import/Parser.php +++ b/CRM/Activity/Import/Parser.php @@ -73,7 +73,9 @@ abstract class CRM_Activity_Import_Parser extends CRM_Import_Parser { &$mapper, $skipColumnHeader = FALSE, $mode = self::MODE_PREVIEW, - $onDuplicate = self::DUPLICATE_SKIP + $onDuplicate = self::DUPLICATE_SKIP, + $statusID = NULL, + $totalRowCount = NULL ) { if (!is_array($fileName)) { CRM_Core_Error::fatal(); @@ -107,6 +109,10 @@ abstract class CRM_Activity_Import_Parser extends CRM_Import_Parser { else { $this->_activeFieldCount = count($this->_activeFields); } + if ($statusID) { + $this->progressImport($statusID); + $startTimestamp = $currTimestamp = $prevTimestamp = time(); + } while (!feof($fd)) { $this->_lineCount++; @@ -148,6 +154,9 @@ abstract class CRM_Activity_Import_Parser extends CRM_Import_Parser { } elseif ($mode == self::MODE_IMPORT) { $returnCode = $this->import($onDuplicate, $values); + if ($statusID && (($this->_lineCount % 50) == 0)) { + $prevTimestamp = $this->progressImport($statusID, FALSE, $startTimestamp, $prevTimestamp, $totalRowCount); + } } else { $returnCode = self::ERROR; diff --git a/CRM/Contact/Import/Form/Preview.php b/CRM/Contact/Import/Form/Preview.php index ca8aff5818..71aa3f6183 100644 --- a/CRM/Contact/Import/Form/Preview.php +++ b/CRM/Contact/Import/Form/Preview.php @@ -121,13 +121,7 @@ class CRM_Contact_Import_Form_Preview extends CRM_Import_Form_Preview { $this->assign($property, $this->get($property)); } - $statusID = $this->get('statusID'); - if (!$statusID) { - $statusID = md5(uniqid(rand(), TRUE)); - $this->set('statusID', $statusID); - } - $statusUrl = CRM_Utils_System::url('civicrm/ajax/status', "id={$statusID}", FALSE, NULL, FALSE); - $this->assign('statusUrl', $statusUrl); + $this->setStatusUrl(); $showColNames = TRUE; if ('CRM_Import_DataSource_CSV' == $this->get('dataSource') && @@ -193,7 +187,6 @@ class CRM_Contact_Import_Form_Preview extends CRM_Import_Form_Preview { 'name' => ts('Import Now'), 'spacing' => '          ', 'isDefault' => TRUE, - 'js' => array('onclick' => "return verify( );"), ), array( 'type' => 'cancel', diff --git a/CRM/Contact/Import/Parser.php b/CRM/Contact/Import/Parser.php index 0125b16256..6085e8a5a0 100644 --- a/CRM/Contact/Import/Parser.php +++ b/CRM/Contact/Import/Parser.php @@ -167,20 +167,9 @@ abstract class CRM_Contact_Import_Parser extends CRM_Import_Parser { } if ($statusID) { - $skip = 50; - // $skip = 1; - $config = CRM_Core_Config::singleton(); - $statusFile = "{$config->uploadDir}status_{$statusID}.txt"; - $status = "
  " . ts('No processing status reported yet.') . "
"; - - //do not force the browser to display the save dialog, CRM-7640 - $contents = json_encode(array(0, $status)); - - file_put_contents($statusFile, $contents); - + $this->progressImport($statusID); $startTimestamp = $currTimestamp = $prevTimestamp = time(); } - // get the contents of the temp. import table $query = "SELECT * FROM $tableName"; if ($mode == self::MODE_IMPORT) { @@ -215,39 +204,9 @@ abstract class CRM_Contact_Import_Parser extends CRM_Import_Parser { elseif ($mode == self::MODE_IMPORT) { //print "Running parser in import mode
\n"; $returnCode = $this->import($onDuplicate, $values, $doGeocodeAddress); - if ($statusID && (($this->_rowCount % $skip) == 0)) { - $currTimestamp = time(); - $totalTime = ($currTimestamp - $startTimestamp); - $time = ($currTimestamp - $prevTimestamp); - $recordsLeft = $totalRowCount - $this->_rowCount; - if ($recordsLeft < 0) { - $recordsLeft = 0; - } - $estimatedTime = ($recordsLeft / $skip) * $time; - $estMinutes = floor($estimatedTime / 60); - $timeFormatted = ''; - if ($estMinutes > 1) { - $timeFormatted = $estMinutes . ' ' . ts('minutes') . ' '; - $estimatedTime = $estimatedTime - ($estMinutes * 60); - } - $timeFormatted .= round($estimatedTime) . ' ' . ts('seconds'); - $processedPercent = (int ) (($this->_rowCount * 100) / $totalRowCount); - $statusMsg = ts('%1 of %2 records - %3 remaining', - array(1 => $this->_rowCount, 2 => $totalRowCount, 3 => $timeFormatted) - ); - $status = " -
-  {$statusMsg} -
-"; - - $contents = json_encode(array($processedPercent, $status)); - - file_put_contents($statusFile, $contents); - - $prevTimestamp = $currTimestamp; + if ($statusID && (($this->_rowCount % 50) == 0)) { + $prevTimestamp = $this->progressImport($statusID, FALSE, $startTimestamp, $prevTimestamp, $totalRowCount); } - // sleep(1); } else { $returnCode = self::ERROR; diff --git a/CRM/Contribute/Import/Form/Preview.php b/CRM/Contribute/Import/Form/Preview.php index c5bdf920c9..3b25cfc1c0 100644 --- a/CRM/Contribute/Import/Form/Preview.php +++ b/CRM/Contribute/Import/Form/Preview.php @@ -98,6 +98,7 @@ class CRM_Contribute_Import_Form_Preview extends CRM_Import_Form_Preview { 'downloadConflictRecordsUrl', 'downloadMismatchRecordsUrl', ); + $this->setStatusUrl(); foreach ($properties as $property) { $this->assign($property, $this->get($property)); @@ -148,7 +149,9 @@ class CRM_Contribute_Import_Form_Preview extends CRM_Import_Form_Preview { $skipColumnHeader, CRM_Import_Parser::MODE_IMPORT, $this->get('contactType'), - $onDuplicate + $onDuplicate, + $this->get('statusID'), + $this->get('totalRowCount') ); // Add all the necessary variables to the form. diff --git a/CRM/Contribute/Import/Parser.php b/CRM/Contribute/Import/Parser.php index 46472f9e1e..603943a5db 100644 --- a/CRM/Contribute/Import/Parser.php +++ b/CRM/Contribute/Import/Parser.php @@ -125,7 +125,9 @@ abstract class CRM_Contribute_Import_Parser extends CRM_Import_Parser { $skipColumnHeader = FALSE, $mode = self::MODE_PREVIEW, $contactType = self::CONTACT_INDIVIDUAL, - $onDuplicate = self::DUPLICATE_SKIP + $onDuplicate = self::DUPLICATE_SKIP, + $statusID = NULL, + $totalRowCount = NULL ) { if (!is_array($fileName)) { CRM_Core_Error::fatal(); @@ -165,6 +167,10 @@ abstract class CRM_Contribute_Import_Parser extends CRM_Import_Parser { $this->_conflicts = array(); $this->_pledgePaymentErrors = array(); $this->_softCreditErrors = array(); + if ($statusID) { + $this->progressImport($statusID); + $startTimestamp = $currTimestamp = $prevTimestamp = time(); + } $this->_fileSize = number_format(filesize($fileName) / 1024.0, 2); @@ -215,6 +221,9 @@ abstract class CRM_Contribute_Import_Parser extends CRM_Import_Parser { } elseif ($mode == self::MODE_IMPORT) { $returnCode = $this->import($onDuplicate, $values); + if ($statusID && (($this->_lineCount % 50) == 0)) { + $prevTimestamp = $this->progressImport($statusID, FALSE, $startTimestamp, $prevTimestamp, $totalRowCount); + } } else { $returnCode = self::ERROR; diff --git a/CRM/Import/Form/Preview.php b/CRM/Import/Form/Preview.php index 1ed08fda51..0d08e83592 100644 --- a/CRM/Import/Form/Preview.php +++ b/CRM/Import/Form/Preview.php @@ -51,7 +51,6 @@ abstract class CRM_Import_Form_Preview extends CRM_Core_Form { * Build the form object. */ public function buildQuickForm() { - $this->addButtons(array( array( 'type' => 'back', @@ -71,4 +70,17 @@ abstract class CRM_Import_Form_Preview extends CRM_Core_Form { ); } + /** + * Set status url for ajax. + */ + public function setStatusUrl() { + $statusID = $this->get('statusID'); + if (!$statusID) { + $statusID = md5(uniqid(rand(), TRUE)); + $this->set('statusID', $statusID); + } + $statusUrl = CRM_Utils_System::url('civicrm/ajax/status', "id={$statusID}", FALSE, NULL, FALSE); + $this->assign('statusUrl', $statusUrl); + } + } diff --git a/CRM/Import/Parser.php b/CRM/Import/Parser.php index 738254e5c1..b17821f0e4 100644 --- a/CRM/Import/Parser.php +++ b/CRM/Import/Parser.php @@ -286,6 +286,58 @@ abstract class CRM_Import_Parser { return $params; } + /** + * @param $statusID + * @param bool $startImport + * True when progress bar is to be initiated. + * @param $startTimestamp + * Initial timstamp when the import was started. + * @param $prevTimestamp + * Previous timestamp when this function was last called. + * @param $totalRowCount + * Total number of rows in the import file. + * + * @return NULL|$currTimestamp + */ + public function progressImport($statusID, $startImport = TRUE, $startTimestamp = NULL, $prevTimestamp = NULL, $totalRowCount = NULL) { + $config = CRM_Core_Config::singleton(); + $statusFile = "{$config->uploadDir}status_{$statusID}.txt"; + + if ($startImport) { + $status = "
  " . ts('No processing status reported yet.') . "
"; + //do not force the browser to display the save dialog, CRM-7640 + $contents = json_encode(array(0, $status)); + file_put_contents($statusFile, $contents); + } + else { + $rowCount = isset($this->_rowCount) ? $this->_rowCount : $this->_lineCount; + $currTimestamp = time(); + $totalTime = ($currTimestamp - $startTimestamp); + $time = ($currTimestamp - $prevTimestamp); + $recordsLeft = $totalRowCount - $rowCount; + if ($recordsLeft < 0) { + $recordsLeft = 0; + } + $estimatedTime = ($recordsLeft / 50) * $time; + $estMinutes = floor($estimatedTime / 60); + $timeFormatted = ''; + if ($estMinutes > 1) { + $timeFormatted = $estMinutes . ' ' . ts('minutes') . ' '; + $estimatedTime = $estimatedTime - ($estMinutes * 60); + } + $timeFormatted .= round($estimatedTime) . ' ' . ts('seconds'); + $processedPercent = (int ) (($rowCount * 100) / $totalRowCount); + $statusMsg = ts('%1 of %2 records - %3 remaining', + array(1 => $rowCount, 2 => $totalRowCount, 3 => $timeFormatted) + ); + $status = "
  {$statusMsg}
"; + $contents = json_encode(array($processedPercent, $status)); + + file_put_contents($statusFile, $contents); + return $currTimestamp; + } + } + /** * @return array */ diff --git a/CRM/Member/Import/Form/Preview.php b/CRM/Member/Import/Form/Preview.php index a595b0a28d..3114d0271f 100644 --- a/CRM/Member/Import/Form/Preview.php +++ b/CRM/Member/Import/Form/Preview.php @@ -99,6 +99,7 @@ class CRM_Member_Import_Form_Preview extends CRM_Import_Form_Preview { 'downloadConflictRecordsUrl', 'downloadMismatchRecordsUrl', ); + $this->setStatusUrl(); foreach ($properties as $property) { $this->assign($property, $this->get($property)); @@ -161,14 +162,15 @@ class CRM_Member_Import_Form_Preview extends CRM_Import_Form_Preview { $skipColumnHeader, CRM_Import_Parser::MODE_IMPORT, $this->get('contactType'), - $onDuplicate + $onDuplicate, + $this->get('statusID'), + $this->get('totalRowCount') ); // add all the necessary variables to the form $parser->set($this, CRM_Import_Parser::MODE_IMPORT); // check if there is any error occurred - $errorStack = CRM_Core_Error::singleton(); $errors = $errorStack->getErrors(); $errorMessage = array(); diff --git a/CRM/Member/Import/Parser.php b/CRM/Member/Import/Parser.php index fdf5014a5c..b8ffc0f91e 100644 --- a/CRM/Member/Import/Parser.php +++ b/CRM/Member/Import/Parser.php @@ -81,7 +81,9 @@ abstract class CRM_Member_Import_Parser extends CRM_Import_Parser { $skipColumnHeader = FALSE, $mode = self::MODE_PREVIEW, $contactType = self::CONTACT_INDIVIDUAL, - $onDuplicate = self::DUPLICATE_SKIP + $onDuplicate = self::DUPLICATE_SKIP, + $statusID = NULL, + $totalRowCount = NULL ) { if (!is_array($fileName)) { CRM_Core_Error::fatal(); @@ -128,6 +130,10 @@ abstract class CRM_Member_Import_Parser extends CRM_Import_Parser { else { $this->_activeFieldCount = count($this->_activeFields); } + if ($statusID) { + $this->progressImport($statusID); + $startTimestamp = $currTimestamp = $prevTimestamp = time(); + } while (!feof($fd)) { $this->_lineCount++; @@ -146,7 +152,6 @@ abstract class CRM_Member_Import_Parser extends CRM_Import_Parser { } /* trim whitespace around the values */ - $empty = TRUE; foreach ($values as $k => $v) { $values[$k] = trim($v, " \t\r\n"); @@ -168,6 +173,9 @@ abstract class CRM_Member_Import_Parser extends CRM_Import_Parser { } elseif ($mode == self::MODE_IMPORT) { $returnCode = $this->import($onDuplicate, $values); + if ($statusID && (($this->_lineCount % 50) == 0)) { + $prevTimestamp = $this->progressImport($statusID, FALSE, $startTimestamp, $prevTimestamp, $totalRowCount); + } } else { $returnCode = self::ERROR; diff --git a/templates/CRM/Activity/Import/Form/Preview.tpl b/templates/CRM/Activity/Import/Form/Preview.tpl index deab87eabc..a59bb6caae 100644 --- a/templates/CRM/Activity/Import/Form/Preview.tpl +++ b/templates/CRM/Activity/Import/Form/Preview.tpl @@ -51,6 +51,7 @@

{ts}Click 'Import Now' if you are ready to proceed.{/ts}

{include file="CRM/common/formButtons.tpl" location="top"}
+ {include file="CRM/common/importProgress.tpl"} {* Summary Preview (record counts) *} diff --git a/templates/CRM/Contact/Import/Form/Preview.tpl b/templates/CRM/Contact/Import/Form/Preview.tpl index faac0e0fba..dc0fb0afbc 100644 --- a/templates/CRM/Contact/Import/Form/Preview.tpl +++ b/templates/CRM/Contact/Import/Form/Preview.tpl @@ -24,51 +24,6 @@ +--------------------------------------------------------------------+ *}
- -{literal} - -{/literal} - {* Import Wizard - Step 3 (preview import results prior to actual data loading) *} {* @var $form Contains the array for the form elements and other form associated information assigned to the template by the controller *} @@ -94,14 +49,7 @@ function verify( ) {

{ts}Click 'Import Now' if you are ready to proceed.{/ts}

{include file="CRM/common/formButtons.tpl" location="top"}
-{* Import Progress Bar and Info *} -
-

Importing records...


-
- -
-
-
+{include file="CRM/common/importProgress.tpl"}
{* Summary Preview (record counts) *} diff --git a/templates/CRM/Contribute/Import/Form/Preview.tpl b/templates/CRM/Contribute/Import/Form/Preview.tpl index 97a559e291..9031b3d72f 100644 --- a/templates/CRM/Contribute/Import/Form/Preview.tpl +++ b/templates/CRM/Contribute/Import/Form/Preview.tpl @@ -50,6 +50,7 @@

{ts}Click 'Import Now' if you are ready to proceed.{/ts}

{include file="CRM/common/formButtons.tpl" location="top"}
+ {include file="CRM/common/importProgress.tpl"} {* Summary Preview (record counts) *}
{ts}Total Rows{/ts}
diff --git a/templates/CRM/Member/Import/Form/Preview.tpl b/templates/CRM/Member/Import/Form/Preview.tpl index f8ccde6c3c..22178c0fad 100644 --- a/templates/CRM/Member/Import/Form/Preview.tpl +++ b/templates/CRM/Member/Import/Form/Preview.tpl @@ -51,6 +51,7 @@

{ts}Click 'Import Now' if you are ready to proceed.{/ts}

{include file="CRM/common/formButtons.tpl" location="top"}
+ {include file="CRM/common/importProgress.tpl"} {* Summary Preview (record counts) *}
{ts}Total Rows{/ts}
diff --git a/templates/CRM/common/importProgress.tpl b/templates/CRM/common/importProgress.tpl new file mode 100644 index 0000000000..9323fd72a7 --- /dev/null +++ b/templates/CRM/common/importProgress.tpl @@ -0,0 +1,81 @@ +{* + +--------------------------------------------------------------------+ + | CiviCRM version 4.7 | + +--------------------------------------------------------------------+ + | Copyright CiviCRM LLC (c) 2004-2017 | + +--------------------------------------------------------------------+ + | This file is a part of CiviCRM. | + | | + | CiviCRM is free software; you can copy, modify, and distribute it | + | under the terms of the GNU Affero General Public License | + | Version 3, 19 November 2007 and the CiviCRM Licensing Exception. | + | | + | CiviCRM is distributed in the hope that it will be useful, but | + | WITHOUT ANY WARRANTY; without even the implied warranty of | + | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. | + | See the GNU Affero General Public License for more details. | + | | + | You should have received a copy of the GNU Affero General Public | + | License and the CiviCRM Licensing Exception along | + | with this program; if not, contact CiviCRM LLC | + | at info[AT]civicrm[DOT]org. If you have questions about the | + | GNU Affero General Public License or the licensing of CiviCRM, | + | see the CiviCRM license FAQ at http://civicrm.org/licensing | + +--------------------------------------------------------------------+ +*} +{literal} + +{/literal} + +{* Import Progress Bar and Info *} +
+

Importing records...


+
+ +
+
+
-- 2.25.1
{ts}Total Rows{/ts}