Fix Contribution import with pledge handling, add test
authorEileen McNaughton <emcnaughton@wikimedia.org>
Wed, 8 Jun 2022 00:50:50 +0000 (12:50 +1200)
committerEileen McNaughton <emcnaughton@wikimedia.org>
Wed, 8 Jun 2022 00:54:58 +0000 (12:54 +1200)
CRM/Contribute/Import/Form/MapField.php
CRM/Contribute/Import/Form/Preview.php
CRM/Contribute/Import/Parser/Contribution.php
CRM/Import/Parser.php
tests/phpunit/CRM/Contribute/Import/Parser/ContributionTest.php
tests/phpunit/CRM/Contribute/Import/Parser/data/pledge.csv

index abd920c1866c801797fbd9e9def8a015e99a46a6..9026f66c5c6c344d39852f27682b279ae3575d2d 100644 (file)
@@ -376,49 +376,12 @@ class CRM_Contribute_Import_Form_MapField extends CRM_Import_Form_MapField {
   }
 
   /**
-   * Process the mapped fields and map it into the uploaded file preview the file and extract some summary statistics.
+   * Get the mapping name per the civicrm_mapping_field.type_id option group.
+   *
+   * @return string
    */
-  public function postProcess() {
-    $params = $this->controller->exportValues('MapField');
-
-    //reload the mapfield if load mapping is pressed
-    if (!empty($params['savedMapping'])) {
-      $this->set('savedMapping', $params['savedMapping']);
-      $this->controller->resetPage($this->_name);
-      return;
-    }
-    $this->updateUserJobMetadata('submitted_values', $this->getSubmittedValues());
-
-    $mapper = $mapperKeysMain = $mapperSoftCredit = $softCreditFields = $mapperPhoneType = $mapperSoftCreditType = [];
-    $mapperKeys = $this->controller->exportValue($this->_name, 'mapper');
-
-    $softCreditTypes = CRM_Core_OptionGroup::values('soft_credit_type');
-
-    for ($i = 0; $i < $this->_columnCount; $i++) {
-      $mapper[$i] = $this->_mapperFields[$mapperKeys[$i][0]];
-      $mapperKeysMain[$i] = $mapperKeys[$i][0];
-    }
-
-    $this->set('mapper', $mapper);
-
-    // store mapping Id to display it in the preview page
-    $this->set('loadMappingId', CRM_Utils_Array::value('mappingId', $params));
-
-    $mappingType = 'Import Contribution';
-    $this->saveMapping($mappingType);
-
-    $parser = new CRM_Contribute_Import_Parser_Contribution($mapperKeysMain);
-    $parser->setUserJobID($this->getUserJobID());
-    $parser->run(
-      $this->getSubmittedValue('uploadFile'),
-      $this->getSubmittedValue('fieldSeparator'),
-      $mapper,
-      $this->getSubmittedValue('skipColumnHeader'),
-      CRM_Import_Parser::MODE_PREVIEW
-    );
-
-    // add all the necessary variables to the form
-    $parser->set($this);
+  public function getMappingTypeName(): string {
+    return 'Import Contribution';
   }
 
   /**
index bce405c8e8a7da9697328b37ecd224b1b6787bb9..3bfc113f7b4ad277c869d6aacb95b86966f36147 100644 (file)
@@ -56,66 +56,6 @@ class CRM_Contribute_Import_Form_Preview extends CRM_Import_Form_Preview {
     return $mapper;
   }
 
-  /**
-   * Process the mapped fields and map it into the uploaded file preview the file and extract some summary statistics.
-   */
-  public function postProcess() {
-    $fileName = $this->controller->exportValue('DataSource', 'uploadFile');
-    $onDuplicate = $this->get('onDuplicate');
-    $this->updateUserJobMetadata('submitted_values', $this->getSubmittedValues());
-    $mapper = $this->controller->exportValue('MapField', 'mapper');
-
-    $parser = new CRM_Contribute_Import_Parser_Contribution();
-    $parser->setUserJobID($this->getUserJobID());
-
-    $mapFields = $this->get('fields');
-
-    foreach ($mapper as $key => $value) {
-      $header = [];
-      if (isset($mapFields[$mapper[$key][0]])) {
-        $header[] = $mapFields[$mapper[$key][0]];
-      }
-      $mapperFields[] = implode(' - ', $header);
-    }
-    $parser->run(
-      $this->getSubmittedValue('uploadFile'),
-      $this->getSubmittedValue('fieldSeparator'),
-      $mapperFields,
-      $this->getSubmittedValue('skipColumnHeader'),
-      CRM_Import_Parser::MODE_IMPORT,
-      $this->getSubmittedValue('contactType'),
-      $onDuplicate,
-      $this->get('statusID'),
-      $this->get('totalRowCount')
-    );
-
-    // Add all the necessary variables to the form.
-    $parser->set($this, CRM_Import_Parser::MODE_IMPORT);
-
-    // Check if there is any error occurred.
-
-    $errorStack = CRM_Core_Error::singleton();
-    $errors = $errorStack->getErrors();
-    $errorMessage = [];
-
-    if (is_array($errors)) {
-      foreach ($errors as $key => $value) {
-        $errorMessage[] = $value['message'];
-      }
-
-      $errorFile = $fileName['name'] . '.error.log';
-
-      if ($fd = fopen($errorFile, 'w')) {
-        fwrite($fd, implode('\n', $errorMessage));
-      }
-      fclose($fd);
-
-      $this->set('errorFile', $errorFile);
-      $urlParams = 'type=' . CRM_Import_Parser::ERROR . '&parser=CRM_Contribute_Import_Parser_Contribution';
-      $this->set('downloadErrorRecordsUrl', CRM_Utils_System::url('civicrm/export', $urlParams));
-    }
-  }
-
   /**
    * @return \CRM_Contribute_Import_Parser_Contribution
    */
index 3e087ad68e5632ac8de80911ae4a98ba01d2e228..c69cd37002805f0cf92f2be23292b2945a8fb3d2 100644 (file)
@@ -177,141 +177,14 @@ class CRM_Contribute_Import_Parser_Contribution extends CRM_Import_Parser {
         $returnCode = $this->summary($values);
       }
       elseif ($mode == self::MODE_IMPORT) {
-        $returnCode = $this->import($values);
+        $this->import($values);
         if ($statusID && (($this->_lineCount % 50) == 0)) {
           $prevTimestamp = $this->progressImport($statusID, FALSE, $startTimestamp, $prevTimestamp, $totalRowCount);
         }
       }
-      else {
-        $returnCode = self::ERROR;
-      }
-
-      // note that a line could be valid but still produce a warning
-      if ($returnCode == self::VALID) {
-        $this->_validCount++;
-        if ($mode == self::MODE_MAPFIELD) {
-          $this->_rows[] = $values;
-          $this->_activeFieldCount = max($this->_activeFieldCount, count($values));
-        }
-      }
-
-      if ($returnCode == self::SOFT_CREDIT) {
-        $this->_validSoftCreditRowCount++;
-        $this->_validCount++;
-        if ($mode == self::MODE_MAPFIELD) {
-          $this->_rows[] = $values;
-          $this->_activeFieldCount = max($this->_activeFieldCount, count($values));
-        }
-      }
-
-      if ($returnCode == self::PLEDGE_PAYMENT) {
-        $this->_validPledgePaymentRowCount++;
-        $this->_validCount++;
-        if ($mode == self::MODE_MAPFIELD) {
-          $this->_rows[] = $values;
-          $this->_activeFieldCount = max($this->_activeFieldCount, count($values));
-        }
-      }
 
-      if ($returnCode == self::ERROR) {
-        $this->_invalidRowCount++;
-        $recordNumber = $this->_lineCount;
-        array_unshift($values, $recordNumber);
-        $this->_errors[] = $values;
-      }
-
-      if ($returnCode == self::PLEDGE_PAYMENT_ERROR) {
-        $this->_invalidPledgePaymentRowCount++;
-        $recordNumber = $this->_lineCount;
-        array_unshift($values, $recordNumber);
-        $this->_pledgePaymentErrors[] = $values;
-      }
-
-      if ($returnCode == self::SOFT_CREDIT_ERROR) {
-        $this->_invalidSoftCreditRowCount++;
-        $recordNumber = $this->_lineCount;
-        array_unshift($values, $recordNumber);
-        $this->_softCreditErrors[] = $values;
-      }
-
-      if ($returnCode == self::DUPLICATE) {
-        $this->_duplicateCount++;
-        $recordNumber = $this->_lineCount;
-        array_unshift($values, $recordNumber);
-        $this->_duplicates[] = $values;
-        if (!$this->isSkipDuplicates()) {
-          $this->_validCount++;
-        }
-      }
     }
 
-    if ($mode == self::MODE_PREVIEW || $mode == self::MODE_IMPORT) {
-      $customHeaders = $mapper;
-
-      $customfields = CRM_Core_BAO_CustomField::getFields('Contribution');
-      foreach ($customHeaders as $key => $value) {
-        if ($id = CRM_Core_BAO_CustomField::getKeyID($value)) {
-          $customHeaders[$key] = $customfields[$id][0];
-        }
-      }
-      if ($this->_invalidRowCount) {
-        // removed view url for invlaid contacts
-        $headers = array_merge([
-          ts('Line Number'),
-          ts('Reason'),
-        ], $customHeaders);
-        $this->_errorFileName = self::errorFileName(self::ERROR);
-        self::exportCSV($this->_errorFileName, $headers, $this->_errors);
-      }
-
-      if ($this->_invalidPledgePaymentRowCount) {
-        // removed view url for invlaid contacts
-        $headers = array_merge([
-          ts('Line Number'),
-          ts('Reason'),
-        ], $customHeaders);
-        $this->_pledgePaymentErrorsFileName = self::errorFileName(self::PLEDGE_PAYMENT_ERROR);
-        self::exportCSV($this->_pledgePaymentErrorsFileName, $headers, $this->_pledgePaymentErrors);
-      }
-
-      if ($this->_invalidSoftCreditRowCount) {
-        // removed view url for invlaid contacts
-        $headers = array_merge([
-          ts('Line Number'),
-          ts('Reason'),
-        ], $customHeaders);
-        $this->_softCreditErrorsFileName = self::errorFileName(self::SOFT_CREDIT_ERROR);
-        self::exportCSV($this->_softCreditErrorsFileName, $headers, $this->_softCreditErrors);
-      }
-
-      if ($this->_duplicateCount) {
-        $headers = array_merge([
-          ts('Line Number'),
-          ts('View Contribution URL'),
-        ], $customHeaders);
-
-        $this->_duplicateFileName = self::errorFileName(self::DUPLICATE);
-        self::exportCSV($this->_duplicateFileName, $headers, $this->_duplicates);
-      }
-    }
-  }
-
-  /**
-   * Given a list of the importable field keys that the user has selected
-   * set the active fields array to this list
-   *
-   * @param array $fieldKeys mapped array of values
-   */
-  public function setActiveFields($fieldKeys) {
-    $this->_activeFieldCount = count($fieldKeys);
-    foreach ($fieldKeys as $key) {
-      if (empty($this->_fields[$key])) {
-        $this->_activeFields[] = new CRM_Contribute_Import_Field('', ts('- do not import -'));
-      }
-      else {
-        $this->_activeFields[] = clone($this->_fields[$key]);
-      }
-    }
   }
 
   /**
@@ -335,6 +208,15 @@ class CRM_Contribute_Import_Parser_Contribution extends CRM_Import_Parser {
     return $mappedFields;
   }
 
+  /**
+   * Get the required fields.
+   *
+   * @return array
+   */
+  public function getRequiredFields(): array {
+    return ['id' => ts('Contribution ID'), ['financial_type_id' => ts('Financial Type'), 'total_amount' => ts('Total Amount')]];
+  }
+
   /**
    * Transform the input parameters into the form handled by the input routine.
    *
@@ -555,10 +437,6 @@ class CRM_Contribute_Import_Parser_Contribution extends CRM_Import_Parser {
     foreach ($this->getImportableFieldsMetadata() as $name => $field) {
       $this->addField($name, $field['title'], $field['type'], $field['headerPattern'], $field['dataPattern']);
     }
-
-    $this->_newContributions = [];
-
-    $this->setActiveFields($this->_mapperKeys);
   }
 
   /**
@@ -581,13 +459,13 @@ class CRM_Contribute_Import_Parser_Contribution extends CRM_Import_Parser {
       // add pledge fields only if its is enabled
       if (CRM_Core_Permission::access('CiviPledge')) {
         $pledgeFields = [
-          'pledge_payment' => [
-            'title' => ts('Pledge Payment'),
-            'headerPattern' => '/Pledge Payment/i',
-          ],
           'pledge_id' => [
             'title' => ts('Pledge ID'),
             'headerPattern' => '/Pledge ID/i',
+            'name' => 'pledge_id',
+            'entity' => 'Pledge',
+            'type' => CRM_Utils_Type::T_INT,
+            'options' => FALSE,
           ],
         ];
 
@@ -620,9 +498,7 @@ class CRM_Contribute_Import_Parser_Contribution extends CRM_Import_Parser {
     $params['contact_type'] = 'Contribution';
 
     if ($errorMessage) {
-      $tempMsg = "Invalid value for field(s) : $errorMessage";
-      array_unshift($values, $tempMsg);
-      $this->setImportStatus($rowNumber, 'ERROR', $tempMsg);
+      $this->setImportStatus($rowNumber, 'ERROR', "Invalid value for field(s) : $errorMessage");
       return CRM_Import_Parser::ERROR;
     }
 
@@ -634,18 +510,8 @@ class CRM_Contribute_Import_Parser_Contribution extends CRM_Import_Parser {
    *
    * @param array $values
    *   The array of values belonging to this line.
-   *
-   * @return int
-   *   the result of this processing - one of
-   *   - CRM_Import_Parser::VALID
-   *   - CRM_Import_Parser::ERROR
-   *   - CRM_Import_Parser::SOFT_CREDIT_ERROR
-   *   - CRM_Import_Parser::PLEDGE_PAYMENT_ERROR
-   *   - CRM_Import_Parser::DUPLICATE
-   *   - CRM_Import_Parser::SOFT_CREDIT (successful creation)
-   *   - CRM_Import_Parser::PLEDGE_PAYMENT (successful creation)
    */
-  public function import(&$values) {
+  public function import($values): void {
     $rowNumber = (int) ($values[array_key_last($values)]);
     try {
       $params = $this->getMappedRow($values);
@@ -682,12 +548,11 @@ class CRM_Contribute_Import_Parser_Contribution extends CRM_Import_Parser {
       $formatError = $this->deprecatedFormatParams($paramValues, $formatted);
 
       if ($formatError) {
-        array_unshift($values, $formatError['error_message']);
         if (CRM_Utils_Array::value('error_data', $formatError) == 'soft_credit') {
-          return self::SOFT_CREDIT_ERROR;
+          throw new CRM_Core_Exception('', self::SOFT_CREDIT_ERROR);
         }
         if (CRM_Utils_Array::value('error_data', $formatError) == 'pledge_payment') {
-          return self::PLEDGE_PAYMENT_ERROR;
+          throw new CRM_Core_Exception('', self::PLEDGE_PAYMENT_ERROR);
         }
         throw new CRM_Core_Exception('', CRM_Import_Parser::ERROR);
       }
@@ -754,11 +619,14 @@ class CRM_Contribute_Import_Parser_Contribution extends CRM_Import_Parser {
 
             //return soft valid since we need to show how soft credits were added
             if (!empty($formatted['soft_credit'])) {
-              return self::SOFT_CREDIT;
+              $this->setImportStatus($rowNumber, $this->getStatus(self::SOFT_CREDIT));
+              return;
             }
 
             // process pledge payment assoc w/ the contribution
-            return $this->processPledgePayments($formatted);
+            $this->processPledgePayments($formatted);
+            $this->setImportStatus($rowNumber, $this->getStatus(self::PLEDGE_PAYMENT));
+            return;
           }
           $labels = [
             'id' => 'Contribution ID',
@@ -804,11 +672,13 @@ class CRM_Contribute_Import_Parser_Contribution extends CRM_Import_Parser {
 
           //return soft valid since we need to show how soft credits were added
           if (!empty($formatted['soft_credit'])) {
-            return self::SOFT_CREDIT;
+            $this->setImportStatus($rowNumber, $this->getStatus(self::SOFT_CREDIT));
+            return;
           }
 
-          // process pledge payment assoc w/ the contribution
-          return $this->processPledgePayments($formatted);
+          $this->processPledgePayments($formatted);
+          $this->setImportStatus($rowNumber, $this->getStatus(self::PLEDGE_PAYMENT));
+          return;
         }
 
         // Using new Dedupe rule.
@@ -868,26 +738,44 @@ class CRM_Contribute_Import_Parser_Contribution extends CRM_Import_Parser {
 
       //return soft valid since we need to show how soft credits were added
       if (!empty($formatted['soft_credit'])) {
-        return self::SOFT_CREDIT;
+        $this->setImportStatus($rowNumber, $this->getStatus(self::SOFT_CREDIT), '');
+        return;
       }
 
       // process pledge payment assoc w/ the contribution
-      return $this->processPledgePayments($formatted);
+      $this->processPledgePayments($formatted);
+      $this->setImportStatus($rowNumber, $this->getStatus(self::PLEDGE_PAYMENT));
+      return;
+
     }
     catch (CRM_Core_Exception $e) {
-      array_unshift($values, $e->getMessage());
-      $errorMapping = [self::SOFT_CREDIT_ERROR => 'soft_credit_error', self::PLEDGE_PAYMENT_ERROR => 'pledge_payment_error', CRM_Import_Parser::DUPLICATE => 'DUPLICATE'];
-      $this->setImportStatus($rowNumber, $errorMapping[$e->getErrorCode()] ?? 'ERROR', $e->getMessage());
-      return $errorMapping[$e->getErrorCode()] ?? CRM_Import_Parser::ERROR;
+      $this->setImportStatus($rowNumber, $this->getStatus($e->getErrorCode()), $e->getMessage());
     }
   }
 
+  /**
+   * Get the status to record.
+   *
+   * @param int|null $code
+   *
+   * @return string
+   */
+  protected function getStatus(?int $code): string {
+    $errorMapping = [
+      self::SOFT_CREDIT_ERROR => 'soft_credit_error',
+      self::PLEDGE_PAYMENT_ERROR => 'pledge_payment_error',
+      self::SOFT_CREDIT => 'soft_credit_imported',
+      self::PLEDGE_PAYMENT => 'pledge_payment_imported',
+      CRM_Import_Parser::DUPLICATE => 'DUPLICATE',
+      CRM_Import_Parser::VALID => 'IMPORTED',
+    ];
+    return $errorMapping[$code] ?? 'ERROR';
+  }
+
   /**
    * Process pledge payments.
    *
    * @param array $formatted
-   *
-   * @return int
    */
   private function processPledgePayments(array $formatted) {
     if (!empty($formatted['pledge_payment_id']) && !empty($formatted['pledge_id'])) {
@@ -905,8 +793,6 @@ class CRM_Contribute_Import_Parser_Contribution extends CRM_Import_Parser {
         NULL,
         $formatted['total_amount']
       );
-
-      return self::PLEDGE_PAYMENT;
     }
   }
 
@@ -1069,25 +955,15 @@ class CRM_Contribute_Import_Parser_Contribution extends CRM_Import_Parser {
           }
           break;
 
-        case 'pledge_payment':
         case 'pledge_id':
-
-          // giving respect to pledge_payment flag.
-          if (empty($params['pledge_payment'])) {
-            break;
-          }
-
           // get total amount of from import fields
           $totalAmount = $params['total_amount'] ?? NULL;
-
-          $onDuplicate = $params['onDuplicate'] ?? NULL;
-
           // we need to get contact id $contributionContactID to
           // retrieve pledge details as well as to validate pledge ID
 
           // first need to check for update mode
           if ($this->isUpdateExisting() &&
-            ($params['contribution_id'] || $params['trxn_id'] || $params['invoice_id'])
+            ($params['id'] || $params['trxn_id'] || $params['invoice_id'])
           ) {
             $contribution = new CRM_Contribute_DAO_Contribution();
             if ($params['contribution_id']) {
@@ -1107,7 +983,7 @@ class CRM_Contribute_Import_Parser_Contribution extends CRM_Import_Parser {
               }
             }
             else {
-              return civicrm_api3_create_error('No match found for specified contact in pledge payment data. Row was skipped.');
+              throw new CRM_Core_Exception('No match found for specified contact in pledge payment data. Row was skipped.');
             }
           }
           else {
index bd0a0d9e1f47b7888dbb7c62c53d69321a477669..7295afdc07cda7fdbdbdf80dd8c1221929dedab7 100644 (file)
@@ -1630,7 +1630,9 @@ abstract class CRM_Import_Parser {
    * @throws \CRM_Core_Exception
    */
   protected function validateParams(array $params): void {
-    $this->validateRequiredFields($this->getRequiredFields(), $params);
+    if (empty($params['id'])) {
+      $this->validateRequiredFields($this->getRequiredFields(), $params);
+    }
     $errors = [];
     foreach ($params as $key => $value) {
       $errors = array_merge($this->getInvalidValues($value, $key), $errors);
@@ -1941,10 +1943,8 @@ abstract class CRM_Import_Parser {
    *
    * @noinspection PhpDocMissingThrowsInspection
    * @noinspection PhpUnhandledExceptionInspection
-   * @throws \API_Exception
-   * @throws \CRM_Core_Exception
    */
-  protected function setImportStatus(int $id, string $status, string $message, ?int $entityID = NULL, $additionalFields = []): void {
+  protected function setImportStatus(int $id, string $status, string $message = '', ?int $entityID = NULL, $additionalFields = []): void {
     $this->getDataSourceObject()->updateStatus($id, $status, $message, $entityID, $additionalFields);
   }
 
index 49c3282e8f25f1d57f53f72367af5323bea4bac2..bc8d41ad57b73b777d75b67777252678af5939bc 100644 (file)
@@ -88,6 +88,10 @@ class CRM_Contribute_Import_Parser_ContributionTest extends CiviUnitTestCase {
 
     $contributionsOfSoftContact = ContributionSoft::get()->addWhere('contact_id', '=', $contact2Id)->execute();
     $this->assertCount(1, $contributionsOfSoftContact, 'Contribution Soft not added for primary contact');
+    $dataSource = new CRM_Import_DataSource_CSV($this->userJobID);
+    $this->assertEquals(1, $dataSource->getRowCount([CRM_Import_Parser::ERROR]));
+    $this->assertEquals(1, $dataSource->getRowCount([CRM_Contribute_Import_Parser_Contribution::SOFT_CREDIT]));
+    $this->assertEquals(1, $dataSource->getRowCount([CRM_Import_Parser::VALID]));
   }
 
   /**
@@ -174,6 +178,26 @@ class CRM_Contribute_Import_Parser_ContributionTest extends CiviUnitTestCase {
     $this->callAPISuccess('CustomGroup', 'delete', ['id' => $this->ids['CustomGroup']['Custom Group']]);
   }
 
+  /**
+   * Test importing to a pledge.
+   */
+  public function testPledgeImport(): void {
+    $contactID = $this->individualCreate(['email' => 'mum@example.com']);
+    $pledgeID = $this->pledgeCreate(['contact_id' => $contactID]);
+    $this->importCSV('pledge.csv', [
+      ['name' => 'email'],
+      ['name' => 'total_amount'],
+      ['name' => 'pledge_id'],
+      ['name' => 'receive_date'],
+      ['name' => 'financial_type_id'],
+    ], ['onDuplicate' => CRM_Import_Parser::NO_MATCH]);
+    $dataSource = new CRM_Import_DataSource_CSV($this->userJobID);
+    $this->assertEquals(1, $dataSource->getRowCount([CRM_Contribute_Import_Parser_Contribution::PLEDGE_PAYMENT]));
+    $this->assertEquals(1, $dataSource->getRowCount([CRM_Import_Parser::VALID]));
+    $contribution = $this->callAPISuccessGetSingle('Contribution', ['contact_id' => $contactID]);
+    $this->callAPISuccessGetSingle('PledgePayment', ['pledge_id' => $pledgeID, 'contribution_id' => $contribution['id']]);
+  }
+
   /**
    * Import the csv file values.
    *
index 1ce47a44349e0ac8ed9f9eaa093febb659d0bc17..181620197aec037b72b01fc58578875019779e0f 100644 (file)
@@ -1,2 +1,2 @@
 Email,Amount,Pledge ID,Receive date,Financial Type
-mum@example.com,5,1,2022-01-07,Donation
+mum@example.com,20,1,2022-01-07,Donation