dev/core#3505, dev/core#3506 Import fixes for unicode url, Côte d’Ivoire

diff --git a/CRM/Import/Parser.php b/CRM/Import/Parser.php
index 7ec35b7c9aa7095af8dc53486a7f44d0d8af5da8..753f0a5077d51450b39b9da6ec5284f468aa258d 100644 (file)
--- a/CRM/Import/Parser.php
+++ b/CRM/Import/Parser.php
@@ -1364,7 +1364,7 @@ abstract class CRM_Import_Parser {
         return $importedValue;
       }
 
-      $comparisonValue = is_numeric($importedValue) ? $importedValue : mb_strtolower($importedValue);
+      $comparisonValue = $this->getComparisonValue($importedValue);
       return $options[$comparisonValue] ?? 'invalid_import_value';
     }
     if (!empty($fieldMetadata['FKClassName']) || !empty($fieldMetadata['pseudoconstant']['prefetch'])) {
@@ -1468,10 +1468,10 @@ abstract class CRM_Import_Parser {
         // name AND label as either might be used. We also lower case before checking
         $values = [];
         foreach ($options as $option) {
-          $idKey = is_numeric($option['id']) ? $option['id'] : mb_strtolower($option['id']);
+          $idKey = $this->getComparisonValue($option['id']);
           $values[$idKey] = $option['id'];
           foreach (['name', 'label', 'abbr'] as $key) {
-            $optionValue = mb_strtolower($option[$key] ?? '');
+            $optionValue = $this->getComparisonValue($option[$key] ?? '');
             if ($optionValue !== '') {
               if (isset($values[$optionValue]) && $values[$optionValue] !== $option['id']) {
                 if (!isset($this->ambiguousOptions[$fieldName][$optionValue])) {
@@ -1775,7 +1775,7 @@ abstract class CRM_Import_Parser {
    * @param string $importedValue
    */
   protected function isAmbiguous(string $fieldName, $importedValue): bool {
-    return !empty($this->ambiguousOptions[$fieldName][mb_strtolower($importedValue)]);
+    return !empty($this->ambiguousOptions[$fieldName][$this->getComparisonValue($importedValue)]);
   }
 
   /**
@@ -2020,4 +2020,20 @@ abstract class CRM_Import_Parser {
     $formatted[$dateParam] = CRM_Utils_Date::processDate($params[$dateParam]);
   }
 
+  /**
+   * Get the value to use for option comparison purposes.
+   *
+   * We do a case-insensitive comparison, also swapping ’ for '
+   * which has at least one known usage (Côte d’Ivoire).
+   *
+   * Note we do this to both sides of the comparison.
+   *
+   * @param int|string|false|null $importedValue
+   *
+   * @return false|int|string|null
+   */
+  protected function getComparisonValue($importedValue) {
+    return is_numeric($importedValue) ? $importedValue : mb_strtolower(str_replace('’', "'", $importedValue));
+  }
+
 }

diff --git a/CRM/Utils/Rule.php b/CRM/Utils/Rule.php
index 4ba6855887494cf6c8b9f6e4ca66be39da629e92..611f2281043a65394f3d7210ffeaae3434880d07 100644 (file)
--- a/CRM/Utils/Rule.php
+++ b/CRM/Utils/Rule.php
@@ -221,7 +221,7 @@ class CRM_Utils_Rule {
       // allow relative URL's (CRM-15598)
       $url = 'http://' . $_SERVER['HTTP_HOST'] . $url;
     }
-    return (bool) filter_var($url, FILTER_VALIDATE_URL);
+    return (bool) filter_var(self::idnToAsci($url), FILTER_VALIDATE_URL);
   }
 
   /**

diff --git a/tests/phpunit/CRM/Contact/Import/Form/data/individual_unicode.csv b/tests/phpunit/CRM/Contact/Import/Form/data/individual_unicode.csv
new file mode 100644 (file)
index 0000000..576915b
--- /dev/null
+++ b/tests/phpunit/CRM/Contact/Import/Form/data/individual_unicode.csv
@@ -0,0 +1,2 @@
+first_name,Last Name,Website,Country
+Iron,Man,https://কানাডা.com,Côte d'Ivoire

diff --git a/tests/phpunit/CRM/Contact/Import/Parser/ContactTest.php b/tests/phpunit/CRM/Contact/Import/Parser/ContactTest.php
index 00c907789639ce3fcc695e0e8607fd26fc5fbe68..5f70b8b6e4c837b88a9831b64076d2bb9d16ba70 100644 (file)
--- a/tests/phpunit/CRM/Contact/Import/Parser/ContactTest.php
+++ b/tests/phpunit/CRM/Contact/Import/Parser/ContactTest.php
@@ -1075,6 +1075,12 @@ class CRM_Contact_Import_Parser_ContactTest extends CiviUnitTestCase {
    */
   public function importDataProvider(): array {
     return [
+      'individual_unicode.csv' => [
+        'csv' => 'individual_unicode.csv',
+        'mapper' => [['first_name'], ['last_name'], ['url', 1], ['country', 1]],
+        'expected_error' => '',
+        'expected_outcomes' => [CRM_Import_Parser::VALID => 1],
+      ],
       'individual_invalid_sub_type' => [
         'csv' => 'individual_invalid_contact_sub_type.csv',
         'mapper' => [['first_name'], ['last_name'], ['contact_sub_type']],