From 66dc6009c16733565469e4b031e9aa381eb2c7cd Mon Sep 17 00:00:00 2001 From: atif-shaikh Date: Mon, 30 Jun 2014 19:30:01 +0530 Subject: [PATCH] CRM-14199 - Incorrect Math in Import routines https://issues.civicrm.org/jira/browse/CRM-14199 --- CRM/Activity/Import/Form/DataSource.php | 17 ++++++++--------- CRM/Admin/Form/Setting/Miscellaneous.php | 22 ++++++++++++++++++++++ CRM/Contact/Import/Form/DataSource.php | 3 +++ CRM/Contribute/Import/Form/DataSource.php | 16 ++++++++-------- CRM/Core/Config/Defaults.php | 22 +++++++++++++++++++++- CRM/Custom/Import/Form/DataSource.php | 17 +++++++++-------- CRM/Event/Import/Form/DataSource.php | 16 ++++++++-------- CRM/Import/DataSource/CSV.php | 7 +++---- CRM/Member/Import/Form/DataSource.php | 16 ++++++++-------- 9 files changed, 90 insertions(+), 46 deletions(-) diff --git a/CRM/Activity/Import/Form/DataSource.php b/CRM/Activity/Import/Form/DataSource.php index d3fd88ac70..9c048cc215 100644 --- a/CRM/Activity/Import/Form/DataSource.php +++ b/CRM/Activity/Import/Form/DataSource.php @@ -47,6 +47,8 @@ class CRM_Activity_Import_Form_DataSource extends CRM_Core_Form { public function preProcess() { $session = CRM_Core_Session::singleton(); $session->pushUserContext(CRM_Utils_System::url('civicrm/import/activity', 'reset=1')); + // check for post max size + CRM_Core_Config_Defaults::formatUnitSize(ini_get('post_max_size'), TRUE); } /** @@ -58,21 +60,18 @@ class CRM_Activity_Import_Form_DataSource extends CRM_Core_Form { public function buildQuickForm() { //Setting Upload File Size $config = CRM_Core_Config::singleton(); - if ($config->maxImportFileSize >= 8388608) { + + $uploadFileSize = CRM_Core_Config_Defaults::formatUnitSize($config->maxFileSize.'m'); + if ($uploadFileSize >= 8388608) { $uploadFileSize = 8388608; } - else { - $uploadFileSize = $config->maxImportFileSize; - } $uploadSize = round(($uploadFileSize / (1024 * 1024)), 2); $this->assign('uploadSize', $uploadSize); - - $this->add('file', 'uploadFile', ts('Import Data File'), 'size=30 maxlength=255', TRUE); - - $this->addRule('uploadFile', ts('A valid file must be uploaded.'), 'uploadedfile'); - $this->addRule('uploadFile', ts('File size should be less than %1 MBytes (%2 bytes)', array(1 => $uploadSize, 2 => $uploadFileSize)), 'maxfilesize', $uploadFileSize); $this->setMaxFileSize($uploadFileSize); + $this->add('File', 'uploadFile', ts('Import Data File'), 'size=30 maxlength=255', TRUE); + $this->addRule('uploadFile', ts('File size should be less than %1 MBytes (%2 bytes)', array(1 => $uploadSize, 2 => $uploadFileSize)), 'maxfilesize', $uploadFileSize); + $this->addRule('uploadFile', ts('A valid file must be uploaded.'), 'uploadedfile'); $this->addRule('uploadFile', ts('Input file must be in CSV format'), 'utf8File'); $this->addElement('checkbox', 'skipColumnHeader', ts('First row contains column headers')); diff --git a/CRM/Admin/Form/Setting/Miscellaneous.php b/CRM/Admin/Form/Setting/Miscellaneous.php index 57bdb4b616..d96f9a63cb 100644 --- a/CRM/Admin/Form/Setting/Miscellaneous.php +++ b/CRM/Admin/Form/Setting/Miscellaneous.php @@ -51,6 +51,19 @@ class CRM_Admin_Form_Setting_Miscellaneous extends CRM_Admin_Form_Setting { 'checksumTimeout' => CRM_Core_BAO_Setting::SYSTEM_PREFERENCES_NAME, ); + public $_uploadMaxSize; + + /** + * Basic setup + */ + + public function preProcess() { + $config = CRM_Core_Config::singleton(); + $this->_uploadMaxSize = (int) ini_get('upload_max_filesize'); + // check for post max size + CRM_Core_Config_Defaults::formatUnitSize(ini_get('post_max_size'), TRUE); + } + /** * Function to build the form * @@ -117,6 +130,11 @@ class CRM_Admin_Form_Setting_Miscellaneous extends CRM_Admin_Form_Setting { static function formRule($fields, $files, $options) { $errors = array(); + // validate max file size + if ($fields['maxFileSize'] > $options->_uploadMaxSize) { + $errors['maxFileSize'] = ts("Maximum file size cannot exceed Upload max size ('upload_max_filesize') as defined in PHP.ini."); + } + if (!empty($fields['wkhtmltopdfPath'])) { // check and ensure that thi leads to the wkhtmltopdf binary // and it is a valid executable binary @@ -140,6 +158,10 @@ class CRM_Admin_Form_Setting_Miscellaneous extends CRM_Admin_Form_Setting { $config = CRM_Core_Config::singleton(); $params = $this->controller->exportValues($this->_name); + // update upload max size in DB + $params['maxImportFileSize'] = CRM_Core_Config_Defaults::formatUnitSize(ini_get('upload_max_filesize')); + CRM_Core_BAO_ConfigSetting::create($params); + // get current logging status $values = $this->exportValues(); diff --git a/CRM/Contact/Import/Form/DataSource.php b/CRM/Contact/Import/Form/DataSource.php index 93196eb548..a2a414d199 100644 --- a/CRM/Contact/Import/Form/DataSource.php +++ b/CRM/Contact/Import/Form/DataSource.php @@ -72,6 +72,9 @@ class CRM_Contact_Import_Form_DataSource extends CRM_Core_Form { $handler = opendir($config->uploadDir); $errorFiles = array('sqlImport.errors', 'sqlImport.conflicts', 'sqlImport.duplicates', 'sqlImport.mismatch'); + // check for post max size + CRM_Core_Config_Defaults::formatUnitSize(ini_get('post_max_size'), TRUE); + while ($file = readdir($handler)) { if ($file != '.' && $file != '..' && in_array($file, $errorFiles) && !is_writable($config->uploadDir . $file) diff --git a/CRM/Contribute/Import/Form/DataSource.php b/CRM/Contribute/Import/Form/DataSource.php index 843ac6fa02..1c23dd9ed4 100644 --- a/CRM/Contribute/Import/Form/DataSource.php +++ b/CRM/Contribute/Import/Form/DataSource.php @@ -47,6 +47,8 @@ class CRM_Contribute_Import_Form_DataSource extends CRM_Core_Form { public function preProcess() { $session = CRM_Core_Session::singleton(); $session->pushUserContext(CRM_Utils_System::url('civicrm/contribute/import', 'reset=1')); + // check for post max size + CRM_Core_Config_Defaults::formatUnitSize(ini_get('post_max_size'), TRUE); } /** @@ -58,21 +60,19 @@ class CRM_Contribute_Import_Form_DataSource extends CRM_Core_Form { public function buildQuickForm() { //Setting Upload File Size $config = CRM_Core_Config::singleton(); - if ($config->maxImportFileSize >= 8388608) { + + $uploadFileSize = CRM_Core_Config_Defaults::formatUnitSize($config->maxFileSize.'m'); + if ($uploadFileSize >= 8388608) { $uploadFileSize = 8388608; } - else { - $uploadFileSize = $config->maxImportFileSize; - } $uploadSize = round(($uploadFileSize / (1024 * 1024)), 2); $this->assign('uploadSize', $uploadSize); - $this->add('file', 'uploadFile', ts('Import Data File'), 'size=30 maxlength=255', TRUE); - - $this->addRule('uploadFile', ts('A valid file must be uploaded.'), 'uploadedfile'); - $this->addRule('uploadFile', ts('File size should be less than %1 MBytes (%2 bytes)', array(1 => $uploadSize, 2 => $uploadFileSize)), 'maxfilesize', $uploadFileSize); + $this->add('File', 'uploadFile', ts('Import Data File'), 'size=30 maxlength=255', TRUE); $this->setMaxFileSize($uploadFileSize); + $this->addRule('uploadFile', ts('File size should be less than %1 MBytes (%2 bytes)', array(1 => $uploadSize, 2 => $uploadFileSize)), 'maxfilesize', $uploadFileSize); + $this->addRule('uploadFile', ts('A valid file must be uploaded.'), 'uploadedfile'); $this->addRule('uploadFile', ts('Input file must be in CSV format'), 'utf8File'); $this->addElement('checkbox', 'skipColumnHeader', ts('First row contains column headers')); diff --git a/CRM/Core/Config/Defaults.php b/CRM/Core/Config/Defaults.php index 3cc1480ed7..b942b907a6 100644 --- a/CRM/Core/Config/Defaults.php +++ b/CRM/Core/Config/Defaults.php @@ -69,6 +69,19 @@ class CRM_Core_Config_Defaults { $this->revampPages = array(); $size = trim(ini_get('upload_max_filesize')); + if ($size) { + $this->maxImportFileSize = self::formatUnitSize($size); + } + } + + /** + * Function to format size + * + * @access public + * @static + */ + + public static function formatUnitSize($size, $checkForPostMax = FALSE) { if ($size) { $last = strtolower($size{strlen($size) - 1}); switch ($last) { @@ -81,7 +94,14 @@ class CRM_Core_Config_Defaults { case 'k': $size *= 1024; } - $this->maxImportFileSize = $size; + + if ($checkForPostMax) { + $config = CRM_Core_Config::singleton(); + if($config->maxImportFileSize > $size) { + CRM_Core_Session::setStatus(ts("Note: Upload max filesize ('upload_max_filesize') should not exceed Post max size ('post_max_size') as defined in PHP.ini, please check with your system administrator."), ts("Warning"), "alert"); + } + } + return $size; } } diff --git a/CRM/Custom/Import/Form/DataSource.php b/CRM/Custom/Import/Form/DataSource.php index b98c846e93..a0770da02c 100644 --- a/CRM/Custom/Import/Form/DataSource.php +++ b/CRM/Custom/Import/Form/DataSource.php @@ -54,6 +54,9 @@ class CRM_Custom_Import_Form_DataSource extends CRM_Core_Form { $params = "reset=1"; } $session->pushUserContext(CRM_Utils_System::url('civicrm/import/custom', $params)); + + // check for post max size + CRM_Core_Config_Defaults::formatUnitSize(ini_get('post_max_size'), TRUE); } /** @@ -87,20 +90,18 @@ class CRM_Custom_Import_Form_DataSource extends CRM_Core_Form { //Setting Upload File Size $config = CRM_Core_Config::singleton(); - if ($config->maxImportFileSize >= 8388608) { + + $uploadFileSize = CRM_Core_Config_Defaults::formatUnitSize($config->maxFileSize.'m'); + if ($uploadFileSize >= 8388608) { $uploadFileSize = 8388608; } - else { - $uploadFileSize = $config->maxImportFileSize; - } $uploadSize = round(($uploadFileSize / (1024 * 1024)), 2); $this->assign('uploadSize', $uploadSize); - $this->add('file', 'uploadFile', ts('Import Data File'), 'size=30 maxlength=255', TRUE); - - $this->addRule('uploadFile', ts('A valid file must be uploaded.'), 'uploadedfile'); - $this->addRule('uploadFile', ts('File size should be less than %1 MBytes (%2 bytes)', array(1 => $uploadSize, 2 => $uploadFileSize)), 'maxfilesize', $uploadFileSize); + $this->add('File', 'uploadFile', ts('Import Data File'), 'size=30 maxlength=255', TRUE); $this->setMaxFileSize($uploadFileSize); + $this->addRule('uploadFile', ts('File size should be less than %1 MBytes (%2 bytes)', array(1 => $uploadSize, 2 => $uploadFileSize)), 'maxfilesize', $uploadFileSize); + $this->addRule('uploadFile', ts('A valid file must be uploaded.'), 'uploadedfile'); $this->addRule('uploadFile', ts('Input file must be in CSV format'), 'utf8File'); $this->addElement('checkbox', 'skipColumnHeader', ts('First row contains column headers')); diff --git a/CRM/Event/Import/Form/DataSource.php b/CRM/Event/Import/Form/DataSource.php index 844ea05e1b..e93bb83ee6 100644 --- a/CRM/Event/Import/Form/DataSource.php +++ b/CRM/Event/Import/Form/DataSource.php @@ -47,6 +47,8 @@ class CRM_Event_Import_Form_DataSource extends CRM_Core_Form { public function preProcess() { $session = CRM_Core_Session::singleton(); $session->pushUserContext(CRM_Utils_System::url('civicrm/event/import', 'reset=1')); + // check for post max size + CRM_Core_Config_Defaults::formatUnitSize(ini_get('post_max_size'), TRUE); } /** @@ -58,21 +60,19 @@ class CRM_Event_Import_Form_DataSource extends CRM_Core_Form { public function buildQuickForm() { //Setting Upload File Size $config = CRM_Core_Config::singleton(); - if ($config->maxImportFileSize >= 8388608) { + + $uploadFileSize = CRM_Core_Config_Defaults::formatUnitSize($config->maxFileSize.'m'); + if ($uploadFileSize >= 8388608) { $uploadFileSize = 8388608; } - else { - $uploadFileSize = $config->maxImportFileSize; - } $uploadSize = round(($uploadFileSize / (1024 * 1024)), 2); $this->assign('uploadSize', $uploadSize); - $this->add('file', 'uploadFile', ts('Import Data File'), 'size=30 maxlength=255', TRUE); - - $this->addRule('uploadFile', ts('A valid file must be uploaded.'), 'uploadedfile'); - $this->addRule('uploadFile', ts('File size should be less than %1 MBytes (%2 bytes)', array(1 => $uploadSize, 2 => $uploadFileSize)), 'maxfilesize', $uploadFileSize); + $this->add('File', 'uploadFile', ts('Import Data File'), 'size=30 maxlength=255', TRUE); $this->setMaxFileSize($uploadFileSize); + $this->addRule('uploadFile', ts('File size should be less than %1 MBytes (%2 bytes)', array(1 => $uploadSize, 2 => $uploadFileSize)), 'maxfilesize', $uploadFileSize); + $this->addRule('uploadFile', ts('A valid file must be uploaded.'), 'uploadedfile'); $this->addRule('uploadFile', ts('Input file must be in CSV format'), 'utf8File'); $this->addElement('checkbox', 'skipColumnHeader', ts('First row contains column headers')); diff --git a/CRM/Import/DataSource/CSV.php b/CRM/Import/DataSource/CSV.php index bf4e102ef9..c9cee1e0f3 100644 --- a/CRM/Import/DataSource/CSV.php +++ b/CRM/Import/DataSource/CSV.php @@ -71,11 +71,11 @@ class CRM_Import_DataSource_CSV extends CRM_Import_DataSource { $config = CRM_Core_Config::singleton(); // FIXME: why do we limit the file size to 8 MiB if it's larger in config? - $uploadFileSize = $config->maxImportFileSize >= 8388608 ? 8388608 : $config->maxImportFileSize; + $sizeInBytes = CRM_Core_Config_Defaults::formatUnitSize($config->maxFileSize.'m'); + $uploadFileSize = $sizeInBytes >= 8388608 ? 8388608 : $sizeInBytes; $uploadSize = round(($uploadFileSize / (1024 * 1024)), 2); $form->assign('uploadSize', $uploadSize); - $form->add('file', 'uploadFile', ts('Import Data File'), 'size=30 maxlength=255', TRUE); - + $form->add('File', 'uploadFile', ts('Import Data File'), 'size=30 maxlength=255', TRUE); $form->setMaxFileSize($uploadFileSize); $form->addRule('uploadFile', ts('File size should be less than %1 MBytes (%2 bytes)', array(1 => $uploadSize, 2 => $uploadFileSize)), 'maxfilesize', $uploadFileSize); $form->addRule('uploadFile', ts('Input file must be in CSV format'), 'utf8File'); @@ -91,7 +91,6 @@ class CRM_Import_DataSource_CSV extends CRM_Import_DataSource { */ function postProcess(&$params, &$db, &$form) { $file = $params['uploadFile']['name']; - $result = self::_CsvToTable($db, $file, CRM_Utils_Array::value('skipColumnHeader', $params, FALSE), diff --git a/CRM/Member/Import/Form/DataSource.php b/CRM/Member/Import/Form/DataSource.php index 466427f811..57aa30ef29 100644 --- a/CRM/Member/Import/Form/DataSource.php +++ b/CRM/Member/Import/Form/DataSource.php @@ -47,6 +47,8 @@ class CRM_Member_Import_Form_DataSource extends CRM_Core_Form { public function preProcess() { $session = CRM_Core_Session::singleton(); $session->pushUserContext(CRM_Utils_System::url('civicrm/member/import', 'reset=1')); + // check for post max size + CRM_Core_Config_Defaults::formatUnitSize(ini_get('post_max_size'), TRUE); } /** @@ -58,21 +60,19 @@ class CRM_Member_Import_Form_DataSource extends CRM_Core_Form { public function buildQuickForm() { //Setting Upload File Size $config = CRM_Core_Config::singleton(); - if ($config->maxImportFileSize >= 8388608) { + + $uploadFileSize = CRM_Core_Config_Defaults::formatUnitSize($config->maxFileSize.'m'); + if ($uploadFileSize >= 8388608) { $uploadFileSize = 8388608; } - else { - $uploadFileSize = $config->maxImportFileSize; - } $uploadSize = round(($uploadFileSize / (1024 * 1024)), 2); $this->assign('uploadSize', $uploadSize); - $this->add('file', 'uploadFile', ts('Import Data File'), 'size=30 maxlength=255', TRUE); - - $this->addRule('uploadFile', ts('A valid file must be uploaded.'), 'uploadedfile'); - $this->addRule('uploadFile', ts('File size should be less than %1 MBytes (%2 bytes)', array(1 => $uploadSize, 2 => $uploadFileSize)), 'maxfilesize', $uploadFileSize); + $this->add('File', 'uploadFile', ts('Import Data File'), 'size=30 maxlength=255', TRUE); $this->setMaxFileSize($uploadFileSize); + $this->addRule('uploadFile', ts('File size should be less than %1 MBytes (%2 bytes)', array(1 => $uploadSize, 2 => $uploadFileSize)), 'maxfilesize', $uploadFileSize); + $this->addRule('uploadFile', ts('A valid file must be uploaded.'), 'uploadedfile'); $this->addRule('uploadFile', ts('Input file must be in CSV format'), 'utf8File'); $this->addElement('checkbox', 'skipColumnHeader', ts('First row contains column headers')); -- 2.25.1