tests/phpunit/CRM/Contact/Import/Parser/ContactTest.php

   1 <?php
   2 /*
   3  +--------------------------------------------------------------------+
   4  | Copyright CiviCRM LLC. All rights reserved.                        |
   5  |                                                                    |
   6  | This code is published under the GNU AGPLv3 license with some      |
   7  | permitted exceptions and without any warranty. For full license    |
   8  | and copyright information, see https://civicrm.org/licensing       |
   9  +--------------------------------------------------------------------+
  10  */
  11
  12 /**
  13  * @file
  14  * File for the CRM_Contact_Imports_Parser_ContactTest class.
  15  */
  16
  17 use Civi\Api4\Address;
  18 use Civi\Api4\Contact;
  19 use Civi\Api4\ContactType;
  20 use Civi\Api4\Email;
  21 use Civi\Api4\IM;
  22 use Civi\Api4\LocationType;
  23 use Civi\Api4\OpenID;
  24 use Civi\Api4\Phone;
  25 use Civi\Api4\RelationshipType;
  26 use Civi\Api4\UserJob;
  27 use Civi\Api4\Website;
  28
  29 /**
  30  *  Test contact import parser.
  31  *
  32  * @package CiviCRM
  33  * @group headless
  34  */
  35 class CRM_Contact_Import_Parser_ContactTest extends CiviUnitTestCase {
  36   use CRMTraits_Custom_CustomDataTrait;
  37
  38   /**
  39    * Main entity for the class.
  40    *
  41    * @var string
  42    */
  43   protected $entity = 'Contact';
  44
  45   /**
  46    * Array of existing relationships.
  47    *
  48    * @var array
  49    */
  50   private $relationships = [];
  51
  52   /**
  53    * Tear down after test.
  54    */
  55   public function tearDown(): void {
  56     $this->quickCleanup(['civicrm_address', 'civicrm_phone', 'civicrm_openid', 'civicrm_email', 'civicrm_user_job', 'civicrm_relationship', 'civicrm_im', 'civicrm_website'], TRUE);
  57     RelationshipType::delete()->addWhere('name_a_b', '=', 'Dad to')->execute();
  58     ContactType::delete()->addWhere('name', '=', 'baby')->execute();
  59     parent::tearDown();
  60   }
  61
  62   /**
  63    * Test that import parser will add contact with employee of relationship.
  64    *
  65    * @throws \API_Exception
  66    * @throws \CRM_Core_Exception
  67    * @throws \CiviCRM_API3_Exception
  68    */
  69   public function testImportParserWithEmployeeOfRelationship(): void {
  70     $this->organizationCreate([
  71       'organization_name' => 'Agileware',
  72       'legal_name'        => 'Agileware',
  73     ]);
  74     $contactImportValues = [
  75       'first_name' => 'Alok',
  76       'last_name' => 'Patel',
  77       'Employee of' => 'Agileware',
  78     ];
  79
  80     $fields = array_keys($contactImportValues);
  81     $values = array_values($contactImportValues);
  82     $userJobID = $this->getUserJobID([
  83       'mapper' => [['first_name'], ['last_name'], ['5_a_b', 'organization_name']],
  84       'onDuplicate' => CRM_Import_Parser::DUPLICATE_UPDATE,
  85     ]);
  86
  87     $parser = new CRM_Contact_Import_Parser_Contact($fields);
  88     $parser->setUserJobID($userJobID);
  89     $parser->init();
  90
  91     $this->assertEquals(CRM_Import_Parser::VALID, $parser->import(CRM_Import_Parser::DUPLICATE_UPDATE, $values), 'Return code from parser import was not as expected');
  92     $this->callAPISuccessGetSingle('Contact', [
  93       'first_name' => 'Alok',
  94       'last_name' => 'Patel',
  95       'organization_name' => 'Agileware',
  96     ]);
  97   }
  98
  99   /**
 100    * Test that import parser will not fail when same external_identifier found
 101    * of deleted contact.
 102    *
 103    * @throws \API_Exception
 104    * @throws \CRM_Core_Exception
 105    * @throws \CiviCRM_API3_Exception
 106    */
 107   public function testImportParserWithDeletedContactExternalIdentifier(): void {
 108     $contactId = $this->individualCreate([
 109       'external_identifier' => 'ext-1',
 110     ]);
 111     $this->callAPISuccess('Contact', 'delete', ['id' => $contactId]);
 112     [$originalValues, $result] = $this->setUpBaseContact([
 113       'external_identifier' => 'ext-1',
 114     ]);
 115     $originalValues['nick_name'] = 'Old Bill';
 116     $this->runImport($originalValues, CRM_Import_Parser::DUPLICATE_UPDATE, CRM_Import_Parser::VALID);
 117     $originalValues['id'] = $result['id'];
 118     $this->assertEquals('ext-1', $this->callAPISuccessGetValue('Contact', ['id' => $result['id'], 'return' => 'external_identifier']));
 119     $this->callAPISuccessGetSingle('Contact', $originalValues);
 120   }
 121
 122   /**
 123    * Test import parser will update based on a rule match.
 124    *
 125    * In this case the contact has no external identifier.
 126    *
 127    * @throws \API_Exception
 128    * @throws \CRM_Core_Exception
 129    * @throws \CiviCRM_API3_Exception
 130    */
 131   public function testImportParserWithUpdateWithoutExternalIdentifier(): void {
 132     [$originalValues, $result] = $this->setUpBaseContact();
 133     $originalValues['nick_name'] = 'Old Bill';
 134     $this->runImport($originalValues, CRM_Import_Parser::DUPLICATE_UPDATE, CRM_Import_Parser::VALID);
 135     $originalValues['id'] = $result['id'];
 136     $this->assertEquals('Old Bill', $this->callAPISuccessGetValue('Contact', ['id' => $result['id'], 'return' => 'nick_name']));
 137     $this->callAPISuccessGetSingle('Contact', $originalValues);
 138   }
 139
 140   /**
 141    * Test import parser will update based on a custom rule match.
 142    *
 143    * In this case the contact has no external identifier.
 144    *
 145    * @throws \API_Exception
 146    * @throws \CRM_Core_Exception
 147    * @throws \CiviCRM_API3_Exception
 148    */
 149   public function testImportParserWithUpdateWithCustomRule(): void {
 150     $this->createCustomGroupWithFieldsOfAllTypes();
 151
 152     $ruleGroup = $this->callAPISuccess('RuleGroup', 'create', [
 153       'contact_type' => 'Individual',
 154       'threshold' => 10,
 155       'used' => 'General',
 156       'name' => 'TestRule',
 157       'title' => 'TestRule',
 158       'is_reserved' => 0,
 159     ]);
 160     $this->callAPISuccess('Rule', 'create', [
 161       'dedupe_rule_group_id' => $ruleGroup['id'],
 162       'rule_table' => $this->getCustomGroupTable(),
 163       'rule_weight' => 10,
 164       'rule_field' => $this->getCustomFieldColumnName('text'),
 165     ]);
 166
 167     $extra = [
 168       $this->getCustomFieldName('select_string') => 'Yellow',
 169       $this->getCustomFieldName('text') => 'Duplicate',
 170     ];
 171
 172     [$originalValues, $result] = $this->setUpBaseContact($extra);
 173
 174     $contactValues = [
 175       'first_name' => 'Tim',
 176       'last_name' => 'Cook',
 177       'email' => 'tim.cook@apple.com',
 178       'nick_name' => 'Steve',
 179       $this->getCustomFieldName('select_string') => 'Red',
 180       $this->getCustomFieldName('text') => 'Duplicate',
 181     ];
 182
 183     $this->runImport($contactValues, CRM_Import_Parser::DUPLICATE_UPDATE, CRM_Import_Parser::VALID, [], NULL, $ruleGroup['id']);
 184     $contactValues['id'] = $result['id'];
 185     $this->assertEquals('R', $this->callAPISuccessGetValue('Contact', ['id' => $result['id'], 'return' => $this->getCustomFieldName('select_string')]));
 186     $this->callAPISuccessGetSingle('Contact', $contactValues);
 187
 188     $foundDupes = CRM_Dedupe_Finder::dupes($ruleGroup['id']);
 189     $this->assertCount(0, $foundDupes);
 190   }
 191
 192   /**
 193    * Test import parser will update based on a custom rule match.
 194    *
 195    * In this case the contact has no external identifier.
 196    *
 197    * @throws \API_Exception
 198    * @throws \CRM_Core_Exception
 199    * @throws \CiviCRM_API3_Exception
 200    */
 201   public function testImportParserWithUpdateWithCustomRuleNoExternalIDMatch(): void {
 202     $this->createCustomGroupWithFieldsOfAllTypes();
 203
 204     $ruleGroup = $this->callAPISuccess('RuleGroup', 'create', [
 205       'contact_type' => 'Individual',
 206       'threshold' => 10,
 207       'used' => 'General',
 208       'name' => 'TestRule',
 209       'title' => 'TestRule',
 210       'is_reserved' => 0,
 211     ]);
 212     $this->callAPISuccess('Rule', 'create', [
 213       'dedupe_rule_group_id' => $ruleGroup['id'],
 214       'rule_table' => $this->getCustomGroupTable(),
 215       'rule_weight' => 10,
 216       'rule_field' => $this->getCustomFieldColumnName('text'),
 217     ]);
 218
 219     $extra = [
 220       $this->getCustomFieldName('select_string') => 'Yellow',
 221       $this->getCustomFieldName('text') => 'Duplicate',
 222       'external_identifier' => 'ext-2',
 223     ];
 224
 225     [$originalValues, $result] = $this->setUpBaseContact($extra);
 226
 227     $contactValues = [
 228       'first_name' => 'Tim',
 229       'last_name' => 'Cook',
 230       'email' => 'tim.cook@apple.com',
 231       'nick_name' => 'Steve',
 232       'external_identifier' => 'ext-1',
 233       $this->getCustomFieldName('select_string') => 'Red',
 234       $this->getCustomFieldName('text') => 'Duplicate',
 235     ];
 236
 237     $this->runImport($contactValues, CRM_Import_Parser::DUPLICATE_UPDATE, CRM_Import_Parser::VALID, [], NULL, $ruleGroup['id']);
 238     $contactValues['id'] = $result['id'];
 239     $this->assertEquals('R', $this->callAPISuccessGetValue('Contact', ['id' => $result['id'], 'return' => $this->getCustomFieldName('select_string')]));
 240     $this->callAPISuccessGetSingle('Contact', $contactValues);
 241
 242     $foundDupes = CRM_Dedupe_Finder::dupes($ruleGroup['id']);
 243     $this->assertCount(0, $foundDupes);
 244   }
 245
 246   /**
 247    * Test import parser will update contacts with an external identifier.
 248    *
 249    * This is the basic test where the identifier matches the import parameters.
 250    *
 251    * @throws \Exception
 252    */
 253   public function testImportParserWithUpdateWithExternalIdentifier(): void {
 254     [$originalValues, $result] = $this->setUpBaseContact(['external_identifier' => 'windows']);
 255
 256     $this->assertEquals($result['id'], CRM_Core_DAO::getFieldValue('CRM_Contact_DAO_Contact', 'windows', 'id', 'external_identifier', TRUE));
 257     $this->assertEquals('windows', $result['external_identifier']);
 258
 259     $originalValues['nick_name'] = 'Old Bill';
 260     $this->runImport($originalValues, CRM_Import_Parser::DUPLICATE_UPDATE, CRM_Import_Parser::VALID);
 261     $originalValues['id'] = $result['id'];
 262
 263     $this->assertEquals('Old Bill', $this->callAPISuccessGetValue('Contact', ['id' => $result['id'], 'return' => 'nick_name']));
 264     $this->callAPISuccessGetSingle('Contact', $originalValues);
 265   }
 266
 267   /**
 268    * Test updating an existing contact with external_identifier match but subtype mismatch.
 269    *
 270    * The subtype is updated, as there is no conflicting contact data.
 271    *
 272    * @throws \Exception
 273    */
 274   public function testImportParserWithUpdateWithExternalIdentifierSubtypeChange(): void {
 275     $contactID = $this->individualCreate(['external_identifier' => 'billy', 'first_name' => 'William', 'contact_sub_type' => 'Parent']);
 276     $this->runImport([
 277       'external_identifier' => 'billy',
 278       'nick_name' => 'Old Bill',
 279       'contact_sub_type' => 'Staff',
 280     ], CRM_Import_Parser::DUPLICATE_UPDATE, CRM_Import_Parser::VALID);
 281     $contact = $this->callAPISuccessGetSingle('Contact', ['id' => $contactID]);
 282     $this->assertEquals('Old Bill', $contact['nick_name']);
 283     $this->assertEquals('William', $contact['first_name']);
 284     $this->assertEquals('billy', $contact['external_identifier']);
 285     $this->assertEquals(['Staff'], $contact['contact_sub_type']);
 286   }
 287
 288   /**
 289    * Test updating an existing contact with external_identifier match but subtype mismatch.
 290    *
 291    * The subtype is not updated, as there is conflicting contact data.
 292    *
 293    * @throws \Exception
 294    */
 295   public function testImportParserUpdateWithExternalIdentifierSubtypeChangeFail(): void {
 296     $contactID = $this->individualCreate(['external_identifier' => 'billy', 'first_name' => 'William', 'contact_sub_type' => 'Parent']);
 297     $this->addChild($contactID);
 298
 299     $this->runImport([
 300       'external_identifier' => 'billy',
 301       'nick_name' => 'Old Bill',
 302       'contact_sub_type' => 'Staff',
 303     ], CRM_Import_Parser::DUPLICATE_UPDATE, FALSE);
 304     $contact = $this->callAPISuccessGetSingle('Contact', ['id' => $contactID]);
 305     $this->assertEquals('', $contact['nick_name']);
 306     $this->assertEquals(['Parent'], $contact['contact_sub_type']);
 307   }
 308
 309   /**
 310    * Test updating an existing contact with external_identifier match but subtype mismatch.
 311    *
 312    * @throws \Exception
 313    */
 314   public function testImportParserWithUpdateWithTypeMismatch(): void {
 315     $contactID = $this->organizationCreate(['external_identifier' => 'billy']);
 316     $this->runImport([
 317       'external_identifier' => 'billy',
 318       'nick_name' => 'Old Bill',
 319     ], CRM_Import_Parser::DUPLICATE_UPDATE, FALSE);
 320     $contact = $this->callAPISuccessGetSingle('Contact', ['id' => $contactID]);
 321     $this->assertEquals('', $contact['nick_name']);
 322     $this->assertEquals('billy', $contact['external_identifier']);
 323     $this->assertEquals('Organization', $contact['contact_type']);
 324
 325     $this->runImport([
 326       'id' => $contactID,
 327       'nick_name' => 'Old Bill',
 328     ], CRM_Import_Parser::DUPLICATE_UPDATE, FALSE);
 329     $contact = $this->callAPISuccessGetSingle('Contact', ['id' => $contactID]);
 330     $this->assertEquals('', $contact['nick_name']);
 331     $this->assertEquals('billy', $contact['external_identifier']);
 332     $this->assertEquals('Organization', $contact['contact_type']);
 333
 334   }
 335
 336   /**
 337    * Test import parser will fallback to external identifier.
 338    *
 339    * In this case no primary match exists (e.g the details are not supplied) so it falls back on external identifier.
 340    *
 341    * @see https://issues.civicrm.org/jira/browse/CRM-17275
 342    *
 343    * @throws \Exception
 344    */
 345   public function testImportParserWithUpdateWithExternalIdentifierButNoPrimaryMatch(): void {
 346     [$originalValues, $result] = $this->setUpBaseContact([
 347       'external_identifier' => 'windows',
 348       'email' => NULL,
 349     ]);
 350
 351     $this->assertEquals('windows', $result['external_identifier']);
 352
 353     $originalValues['nick_name'] = 'Old Bill';
 354     $this->runImport($originalValues, CRM_Import_Parser::DUPLICATE_UPDATE, CRM_Import_Parser::VALID);
 355     $originalValues['id'] = $result['id'];
 356
 357     $this->assertEquals('Old Bill', $this->callAPISuccessGetValue('Contact', ['id' => $result['id'], 'return' => 'nick_name']));
 358     $this->callAPISuccessGetSingle('Contact', $originalValues);
 359   }
 360
 361   /**
 362    * Test import parser will fallback to external identifier.
 363    *
 364    * In this case no primary match exists (e.g the details are not supplied) so it falls back on external identifier.
 365    *
 366    * @see https://issues.civicrm.org/jira/browse/CRM-17275
 367    *
 368    * @throws \Exception
 369    */
 370   public function testImportParserWithUpdateWithContactID(): void {
 371     [$originalValues, $result] = $this->setUpBaseContact([
 372       'external_identifier' => '',
 373       'email' => NULL,
 374     ]);
 375     $updateValues = ['id' => $result['id'], 'email' => 'bill@example.com'];
 376     // This is some deep weirdness - this sets a flag for updatingBlankLocinfo - allowing input to be blanked
 377     // (which IS a good thing but it's pretty weird & all to do with legacy profile stuff).
 378     CRM_Core_Session::singleton()->set('authSrc', CRM_Core_Permission::AUTH_SRC_CHECKSUM);
 379     $this->runImport($updateValues, CRM_Import_Parser::DUPLICATE_UPDATE, CRM_Import_Parser::VALID);
 380     $originalValues['id'] = $result['id'];
 381     $this->callAPISuccessGetSingle('Email', ['contact_id' => $originalValues['id'], 'is_primary' => 1]);
 382     $this->callAPISuccessGetSingle('Contact', $originalValues);
 383   }
 384
 385   /**
 386    * Test that the import parser adds the external identifier where none is set.
 387    *
 388    * @throws \Exception
 389    */
 390   public function testImportParserWithUpdateWithNoExternalIdentifier(): void {
 391     [$originalValues, $result] = $this->setUpBaseContact();
 392     $originalValues['nick_name'] = 'Old Bill';
 393     $originalValues['external_identifier'] = 'windows';
 394     $this->runImport($originalValues, CRM_Import_Parser::DUPLICATE_UPDATE, CRM_Import_Parser::VALID);
 395     $originalValues['id'] = $result['id'];
 396     $this->assertEquals('Old Bill', $this->callAPISuccessGetValue('Contact', ['id' => $result['id'], 'return' => 'nick_name']));
 397     $this->callAPISuccessGetSingle('Contact', $originalValues);
 398   }
 399
 400   /**
 401    * Test that the import parser changes the external identifier when there is a dedupe match.
 402    *
 403    * @throws \Exception
 404    */
 405   public function testImportParserWithUpdateWithChangedExternalIdentifier(): void {
 406     [$contactValues, $result] = $this->setUpBaseContact(['external_identifier' => 'windows']);
 407     $contact_id = $result['id'];
 408     $contactValues['nick_name'] = 'Old Bill';
 409     $contactValues['external_identifier'] = 'android';
 410     $this->runImport($contactValues, CRM_Import_Parser::DUPLICATE_UPDATE, CRM_Import_Parser::VALID);
 411     $contactValues['id'] = $contact_id;
 412     $this->assertEquals('Old Bill', $this->callAPISuccessGetValue('Contact', ['id' => $contact_id, 'return' => 'nick_name']));
 413     $this->callAPISuccessGetSingle('Contact', $contactValues);
 414   }
 415
 416   /**
 417    * Test that the import parser adds the address to the right location.
 418    *
 419    * @throws \API_Exception
 420    * @throws \CRM_Core_Exception
 421    * @throws \CiviCRM_API3_Exception
 422    */
 423   public function testImportBillingAddress(): void {
 424     [$contactValues] = $this->setUpBaseContact();
 425     $contactValues['nick_name'] = 'Old Bill';
 426     $contactValues['external_identifier'] = 'android';
 427     $contactValues['street_address'] = 'Big Mansion';
 428     $contactValues['phone'] = '911';
 429     $mapper = $this->getFieldMappingFromInput($contactValues, 2);
 430     $this->runImport($contactValues, CRM_Import_Parser::DUPLICATE_UPDATE, CRM_Import_Parser::VALID, $mapper);
 431     $address = $this->callAPISuccessGetSingle('Address', ['street_address' => 'Big Mansion']);
 432     $this->assertEquals(2, $address['location_type_id']);
 433
 434     $phone = $this->callAPISuccessGetSingle('Phone', ['phone' => '911']);
 435     $this->assertEquals(2, $phone['location_type_id']);
 436
 437     $contact = $this->callAPISuccessGetSingle('Contact', $contactValues);
 438     $this->callAPISuccess('Contact', 'delete', ['id' => $contact['id']]);
 439   }
 440
 441   /**
 442    * Test that the not-really-encouraged way of creating locations via contact.create doesn't mess up primaries.
 443    */
 444   public function testContactLocationBlockHandling(): void {
 445     $id = $this->individualCreate([
 446       'phone' => [
 447         1 => [
 448           'location_type_id' => 1,
 449           'phone' => '987654321',
 450         ],
 451         2 => [
 452           'location_type_id' => 2,
 453           'phone' => '456-7890',
 454         ],
 455       ],
 456       'im' => [
 457         1 => [
 458           'location_type_id' => 1,
 459           'name' => 'bob',
 460         ],
 461         2 => [
 462           'location_type_id' => 2,
 463           'name' => 'fred',
 464         ],
 465       ],
 466       'openid' => [
 467         1 => [
 468           'location_type_id' => 1,
 469           'openid' => 'bob',
 470         ],
 471         2 => [
 472           'location_type_id' => 2,
 473           'openid' => 'fred',
 474         ],
 475       ],
 476       'email' => [
 477         1 => [
 478           'location_type_id' => 1,
 479           'email' => 'bob@example.com',
 480         ],
 481         2 => [
 482           'location_type_id' => 2,
 483           'email' => 'fred@example.com',
 484         ],
 485       ],
 486     ]);
 487     $phones = $this->callAPISuccess('Phone', 'get', ['contact_id' => $id])['values'];
 488     $emails = $this->callAPISuccess('Email', 'get', ['contact_id' => $id])['values'];
 489     $openIDs = $this->callAPISuccess('OpenID', 'get', ['contact_id' => $id])['values'];
 490     $ims = $this->callAPISuccess('IM', 'get', ['contact_id' => $id])['values'];
 491     $this->assertCount(2, $phones);
 492     $this->assertCount(2, $emails);
 493     $this->assertCount(2, $ims);
 494     $this->assertCount(2, $openIDs);
 495
 496     $this->assertLocationValidity();
 497     $this->callAPISuccess('Contact', 'create', [
 498       'id' => $id,
 499       // This is secret code for 'delete this phone'.
 500       'updateBlankLocInfo' => TRUE,
 501       'phone' => [
 502         1 => [
 503           'id' => key($phones),
 504         ],
 505       ],
 506       'email' => [
 507         1 => [
 508           'id' => key($emails),
 509         ],
 510       ],
 511       'im' => [
 512         1 => [
 513           'id' => key($ims),
 514         ],
 515       ],
 516       'openid' => [
 517         1 => [
 518           'id' => key($openIDs),
 519         ],
 520       ],
 521     ]);
 522     $this->assertLocationValidity();
 523     $this->callAPISuccessGetCount('Phone', ['contact_id' => $id], 1);
 524     $this->callAPISuccessGetCount('Email', ['contact_id' => $id], 1);
 525     $this->callAPISuccessGetCount('OpenID', ['contact_id' => $id], 1);
 526     $this->callAPISuccessGetCount('IM', ['contact_id' => $id], 1);
 527   }
 528
 529   /**
 530    * Test that the import parser adds the address to the primary location.
 531    *
 532    * @throws \Exception
 533    */
 534   public function testImportPrimaryAddress(): void {
 535     [$contactValues] = $this->setUpBaseContact();
 536     $contactValues['nick_name'] = 'Old Bill';
 537     $contactValues['external_identifier'] = 'android';
 538     $contactValues['street_address'] = 'Big Mansion';
 539     $contactValues['phone'] = 12334;
 540     $mapper = $this->getFieldMappingFromInput($contactValues);
 541     $this->runImport($contactValues, CRM_Import_Parser::DUPLICATE_UPDATE, CRM_Import_Parser::VALID, $mapper);
 542     $address = $this->callAPISuccessGetSingle('Address', ['street_address' => 'Big Mansion']);
 543     $this->assertEquals(1, $address['location_type_id']);
 544     $this->assertEquals(1, $address['is_primary']);
 545
 546     $phone = $this->callAPISuccessGetSingle('Phone', ['phone' => '12334']);
 547     $this->assertEquals(1, $phone['location_type_id']);
 548
 549     $this->callAPISuccessGetSingle('Email', ['email' => 'bill.gates@microsoft.com']);
 550
 551     $contact = $this->callAPISuccessGetSingle('Contact', $contactValues);
 552     $this->callAPISuccess('Contact', 'delete', ['id' => $contact['id']]);
 553   }
 554
 555   /**
 556    * Test that address location type id is ignored for dedupe purposes on import.
 557    *
 558    * @throws \Exception
 559    */
 560   public function testIgnoreLocationTypeId(): void {
 561     // Create a rule that matches on last name and street address.
 562     $rgid = $this->createRuleGroup()['id'];
 563     $this->callAPISuccess('Rule', 'create', [
 564       'dedupe_rule_group_id' => $rgid,
 565       'rule_field' => 'last_name',
 566       'rule_table' => 'civicrm_contact',
 567       'rule_weight' => 4,
 568     ]);
 569     $this->callAPISuccess('Rule', 'create', [
 570       'dedupe_rule_group_id' => $rgid,
 571       'rule_field' => 'street_address',
 572       'rule_table' => 'civicrm_address',
 573       'rule_weight' => 4,
 574     ]);
 575     // Create a contact with an address of location_type_id 1.
 576     $contact1Params = [
 577       'contact_type' => 'Individual',
 578       'first_name' => 'Original',
 579       'last_name' => 'Smith',
 580     ];
 581     $contact1 = $this->callAPISuccess('Contact', 'create', $contact1Params);
 582     $this->callAPISuccess('Address', 'create', [
 583       'contact_id' => $contact1['id'],
 584       'location_type_id' => 1,
 585       'street_address' => 'Big Mansion',
 586     ]);
 587
 588     $contactValues = [
 589       'first_name' => 'New',
 590       'last_name' => 'Smith',
 591       'street_address' => 'Big Mansion',
 592     ];
 593
 594     // We want to import with a location_type_id of 4.
 595     $fieldMapping = $this->getFieldMappingFromInput($contactValues, 4);
 596     $this->runImport($contactValues, CRM_Import_Parser::DUPLICATE_SKIP, CRM_Import_Parser::DUPLICATE, $fieldMapping, NULL, $rgid);
 597     $address = $this->callAPISuccessGetSingle('Address', ['street_address' => 'Big Mansion']);
 598     $this->assertEquals(1, $address['location_type_id']);
 599     $contact = $this->callAPISuccessGetSingle('Contact', $contact1Params);
 600     $this->callAPISuccess('Contact', 'delete', ['id' => $contact['id']]);
 601   }
 602
 603   /**
 604    * Test that address custom fields can be imported
 605    * FIXME: Api4
 606    *
 607    * @throws \CRM_Core_Exception
 608    */
 609   public function testAddressWithCustomData(): void {
 610     $ids = $this->entityCustomGroupWithSingleFieldCreate('Address', 'AddressTest.php');
 611     [$contactValues] = $this->setUpBaseContact();
 612     $contactValues['nick_name'] = 'Old Bill';
 613     $contactValues['external_identifier'] = 'android';
 614     $contactValues['street_address'] = 'Big Mansion';
 615     $contactValues['custom_' . $ids['custom_field_id']] = 'Update';
 616     $this->runImport($contactValues, CRM_Import_Parser::DUPLICATE_UPDATE, CRM_Import_Parser::VALID);
 617     $address = $this->callAPISuccessGetSingle('Address', ['street_address' => 'Big Mansion', 'return' => 'custom_' . $ids['custom_field_id']]);
 618     $this->assertEquals('Update', $address['custom_' . $ids['custom_field_id']]);
 619   }
 620
 621   /**
 622    * Test gender works when you specify the label.
 623    *
 624    * There is an expectation that you can import by label here.
 625    *
 626    * @throws \CRM_Core_Exception
 627    */
 628   public function testGenderLabel() {
 629     $contactValues = [
 630       'first_name' => 'Bill',
 631       'last_name' => 'Gates',
 632       'email' => 'bill.gates@microsoft.com',
 633       'nick_name' => 'Billy-boy',
 634       'gender_id' => 'Female',
 635     ];
 636     $this->runImport($contactValues, CRM_Import_Parser::DUPLICATE_UPDATE, CRM_Import_Parser::VALID);
 637     $this->callAPISuccessGetSingle('Contact', $contactValues);
 638   }
 639
 640   /**
 641    * Test greeting imports.
 642    *
 643    * @throws \API_Exception
 644    * @throws \CRM_Core_Exception
 645    * @throws \CiviCRM_API3_Exception
 646    */
 647   public function testGreetings(): void {
 648     $contactValues = [
 649       'first_name' => 'Bill',
 650       'last_name' => 'Gates',
 651       // id = 2
 652       'email_greeting' => 'Dear {contact.prefix_id:label} {contact.first_name} {contact.last_name}',
 653       // id = 3
 654       'postal_greeting' => 'Dear {contact.prefix_id:label} {contact.last_name}',
 655       // id = 1
 656       'addressee' => '{contact.prefix_id:label}{ }{contact.first_name}{ }{contact.middle_name}{ }{contact.last_name}{ }{contact.suffix_id:label}',
 657       5 => 1,
 658     ];
 659     $userJobID = $this->getUserJobID(['mapper' => [['first_name'], ['last_name'], ['email_greeting'], ['postal_greeting'], ['addressee']]]);
 660     $parser = new CRM_Contact_Import_Parser_Contact(array_keys($contactValues));
 661     $parser->setUserJobID($userJobID);
 662     $values = array_values($contactValues);
 663     $parser->import(CRM_Import_Parser::DUPLICATE_UPDATE, $values);
 664     $contact = Contact::get(FALSE)->addWhere('last_name', '=', 'Gates')->addSelect('email_greeting_id', 'postal_greeting_id', 'addressee_id')->execute()->first();
 665     $this->assertEquals(2, $contact['email_greeting_id']);
 666     $this->assertEquals(3, $contact['postal_greeting_id']);
 667     $this->assertEquals(1, $contact['addressee_id']);
 668
 669     Contact::delete()->addWhere('id', '=', $contact['id'])->setUseTrash(TRUE)->execute();
 670
 671     // Now try again with numbers.
 672     $values[2] = 2;
 673     $values[3] = 3;
 674     $values[4] = 1;
 675     $parser->import(CRM_Import_Parser::DUPLICATE_UPDATE, $values);
 676     $contact = Contact::get(FALSE)->addWhere('last_name', '=', 'Gates')->addSelect('email_greeting_id', 'postal_greeting_id', 'addressee_id')->execute()->first();
 677     $this->assertEquals(2, $contact['email_greeting_id']);
 678     $this->assertEquals(3, $contact['postal_greeting_id']);
 679     $this->assertEquals(1, $contact['addressee_id']);
 680
 681   }
 682
 683   /**
 684    * Test prefix & suffix work when you specify the label.
 685    *
 686    * There is an expectation that you can import by label here.
 687    *
 688    * @throws \API_Exception
 689    * @throws \CRM_Core_Exception
 690    * @throws \CiviCRM_API3_Exception
 691    */
 692   public function testPrefixLabel(): void {
 693     $this->callAPISuccess('OptionValue', 'create', ['option_group_id' => 'individual_prefix', 'name' => 'new_one', 'label' => 'special', 'value' => 70]);
 694     $mapping = [
 695       ['name' => 'first_name', 'column_number' => 0],
 696       ['name' => 'last_name', 'column_number' => 1],
 697       ['name' => 'email', 'column_number' => 2, 'location_type_id' => CRM_Core_PseudoConstant::getKey('CRM_Core_BAO_Email', 'location_type_id', 'Home')],
 698       ['name' => 'prefix_id', 'column_number' => 3],
 699       ['name' => 'suffix_id', 'column_number' => 4],
 700     ];
 701     $mapperInput = [['first_name'], ['last_name'], ['email', CRM_Core_PseudoConstant::getKey('CRM_Core_BAO_Email', 'location_type_id', 'Home')], ['prefix_id'], ['suffix_id']];
 702
 703     $processor = new CRM_Import_ImportProcessor();
 704     $processor->setMappingFields($mapping);
 705     $userJobID = $this->getUserJobID(['mapper' => $mapperInput]);
 706     $processor->setUserJobID($userJobID);
 707     $importer = $processor->getImporterObject();
 708
 709     $contactValues = [
 710       'Bill',
 711       'Gates',
 712       'bill.gates@microsoft.com',
 713       'special',
 714       'III',
 715     ];
 716     $importer->import(CRM_Import_Parser::DUPLICATE_NOCHECK, $contactValues);
 717
 718     $contact = $this->callAPISuccessGetSingle('Contact', ['first_name' => 'Bill', 'prefix_id' => 'new_one', 'suffix_id' => 'III']);
 719     $this->assertEquals('special Bill Gates III', $contact['display_name']);
 720   }
 721
 722   /**
 723    * Test that labels work for importing custom data.
 724    *
 725    * @throws \API_Exception
 726    * @throws \CRM_Core_Exception
 727    * @throws \CiviCRM_API3_Exception
 728    */
 729   public function testCustomDataLabel(): void {
 730     $this->createCustomGroupWithFieldOfType([], 'select');
 731     $contactValues = [
 732       'first_name' => 'Bill',
 733       'last_name' => 'Gates',
 734       'email' => 'bill.gates@microsoft.com',
 735       'nick_name' => 'Billy-boy',
 736       $this->getCustomFieldName('select') => 'Yellow',
 737     ];
 738     $this->runImport($contactValues, CRM_Import_Parser::DUPLICATE_UPDATE, CRM_Import_Parser::VALID);
 739     $contact = $this->callAPISuccessGetSingle('Contact', array_merge($contactValues, ['return' => $this->getCustomFieldName('select')]));
 740     $this->assertEquals('Y', $contact[$this->getCustomFieldName('select')]);
 741   }
 742
 743   /**
 744    * Test that names work for importing custom data.
 745    *
 746    * @throws \CRM_Core_Exception
 747    */
 748   public function testCustomDataName() {
 749     $this->createCustomGroupWithFieldOfType([], 'select');
 750     $contactValues = [
 751       'first_name' => 'Bill',
 752       'last_name' => 'Gates',
 753       'email' => 'bill.gates@microsoft.com',
 754       'nick_name' => 'Billy-boy',
 755       $this->getCustomFieldName('select') => 'Y',
 756     ];
 757     $this->runImport($contactValues, CRM_Import_Parser::DUPLICATE_UPDATE, CRM_Import_Parser::VALID);
 758     $contact = $this->callAPISuccessGetSingle('Contact', array_merge($contactValues, ['return' => $this->getCustomFieldName('select')]));
 759     $this->assertEquals('Y', $contact[$this->getCustomFieldName('select')]);
 760   }
 761
 762   /**
 763    * Test importing in the Preferred Language Field
 764    *
 765    * @throws \CRM_Core_Exception
 766    */
 767   public function testPreferredLanguageImport() {
 768     $contactValues = [
 769       'first_name' => 'Bill',
 770       'last_name' => 'Gates',
 771       'email' => 'bill.gates@microsoft.com',
 772       'nick_name' => 'Billy-boy',
 773       'preferred_language' => 'English (Australia)',
 774     ];
 775     $this->runImport($contactValues, CRM_Import_Parser::DUPLICATE_UPDATE, CRM_Import_Parser::VALID);
 776   }
 777
 778   /**
 779    * Test that the import parser adds the address to the primary location.
 780    *
 781    * @throws \Exception
 782    */
 783   public function testImportDeceased() {
 784     [$contactValues] = $this->setUpBaseContact();
 785     CRM_Core_Session::singleton()->set("dateTypes", 1);
 786     $contactValues['birth_date'] = '1910-12-17';
 787     $contactValues['deceased_date'] = '2010-12-17';
 788     $this->runImport($contactValues, CRM_Import_Parser::DUPLICATE_UPDATE, CRM_Import_Parser::VALID);
 789     $contact = $this->callAPISuccessGetSingle('Contact', $contactValues);
 790     $this->assertEquals('1910-12-17', $contact['birth_date']);
 791     $this->assertEquals('2010-12-17', $contact['deceased_date']);
 792     $this->assertEquals(1, $contact['is_deceased']);
 793     $this->callAPISuccess('Contact', 'delete', ['id' => $contact['id']]);
 794   }
 795
 796   /**
 797    * Test that the import parser adds the address to the primary location.
 798    *
 799    * @throws \Exception
 800    */
 801   public function testImportTwoAddressFirstPrimary(): void {
 802     [$contactValues] = $this->setUpBaseContact();
 803     $contactValues['nick_name'] = 'Old Bill';
 804     $contactValues['external_identifier'] = 'android';
 805
 806     $contactValues['street_address'] = 'Big Mansion';
 807     $contactValues['phone'] = 12334;
 808
 809     $fieldMapping = $this->getFieldMappingFromInput($contactValues);
 810     $contactValues['street_address_2'] = 'Teeny Mansion';
 811     $fieldMapping[] = ['name' => 'street_address', 'location_type_id' => 3];
 812     $contactValues['phone_2'] = 4444;
 813     $fieldMapping[] = ['name' => 'phone', 'location_type_id' => 3, 'phone_type_id' => 1];
 814
 815     $this->runImport($contactValues, CRM_Import_Parser::DUPLICATE_UPDATE, CRM_Import_Parser::VALID, $fieldMapping);
 816     $contact = $this->callAPISuccessGetSingle('Contact', ['external_identifier' => 'android']);
 817     $address = $this->callAPISuccess('Address', 'get', ['contact_id' => $contact['id'], 'sequential' => 1]);
 818
 819     $this->assertEquals(3, $address['values'][0]['location_type_id']);
 820     $this->assertEquals(0, $address['values'][0]['is_primary']);
 821     $this->assertEquals('Teeny Mansion', $address['values'][0]['street_address']);
 822
 823     $this->assertEquals(1, $address['values'][1]['location_type_id']);
 824     $this->assertEquals(1, $address['values'][1]['is_primary']);
 825     $this->assertEquals('Big Mansion', $address['values'][1]['street_address']);
 826
 827     $phone = $this->callAPISuccess('Phone', 'get', ['contact_id' => $contact['id'], 'sequential' => 1]);
 828     $this->assertEquals(1, $phone['values'][0]['location_type_id']);
 829     $this->assertEquals(1, $phone['values'][0]['is_primary']);
 830     $this->assertEquals(12334, $phone['values'][0]['phone']);
 831     $this->assertEquals(3, $phone['values'][1]['location_type_id']);
 832     $this->assertEquals(0, $phone['values'][1]['is_primary']);
 833     $this->assertEquals(4444, $phone['values'][1]['phone']);
 834
 835     $this->callAPISuccess('Contact', 'delete', ['id' => $contact['id']]);
 836   }
 837
 838   /**
 839    * Test importing 2 phones of different types.
 840    *
 841    * @throws \API_Exception
 842    * @throws \CRM_Core_Exception
 843    * @throws \CiviCRM_API3_Exception
 844    */
 845   public function testImportTwoPhonesDifferentTypes(): void {
 846     $processor = new CRM_Import_ImportProcessor();
 847     $processor->setUserJobID($this->getUserJobID([
 848       'mapper' => [['first_name'], ['last_name'], ['email'], ['phone', 1, 2], ['phone', 1, 1]],
 849     ]));
 850     $processor->setMappingFields(
 851       [
 852         ['name' => 'first_name'],
 853         ['name' => 'last_name'],
 854         ['name' => 'email'],
 855         ['name' => 'phone', 'location_type_id' => 1, 'phone_type_id' => 2],
 856         ['name' => 'phone', 'location_type_id' => 1, 'phone_type_id' => 1],
 857       ]
 858     );
 859     $importer = $processor->getImporterObject();
 860     $fields = ['First Name', 'new last name', 'bob@example.com', '1234', '5678'];
 861     $importer->import(CRM_Import_Parser::DUPLICATE_UPDATE, $fields);
 862     $contact = $this->callAPISuccessGetSingle('Contact', ['last_name' => 'new last name']);
 863     $phones = $this->callAPISuccess('Phone', 'get', ['contact_id' => $contact['id']])['values'];
 864     $this->assertCount(2, $phones);
 865   }
 866
 867   /**
 868    * Test that the import parser adds the address to the primary location.
 869    *
 870    * @throws \Exception
 871    */
 872   public function testImportTwoAddressSecondPrimary(): void {
 873     [$contactValues] = $this->setUpBaseContact();
 874     $contactValues['nick_name'] = 'Old Bill';
 875     $contactValues['external_identifier'] = 'android';
 876     $contactValues['street_address'] = 'Big Mansion';
 877     $contactValues['phone'] = 12334;
 878
 879     $fieldMapping = $this->getFieldMappingFromInput($contactValues, 3);
 880
 881     $contactValues['street_address_2'] = 'Teeny Mansion';
 882     $fieldMapping[] = ['name' => 'street_address', 'location_type_id' => 'Primary'];
 883     $contactValues['phone_2'] = 4444;
 884     $fieldMapping[] = ['name' => 'phone', 'location_type_id' => 'Primary', 'phone_type_id' => 1];
 885
 886     $this->runImport($contactValues, CRM_Import_Parser::DUPLICATE_UPDATE, CRM_Import_Parser::VALID, $fieldMapping);
 887     $contact = $this->callAPISuccessGetSingle('Contact', ['external_identifier' => 'android']);
 888     $address = $this->callAPISuccess('Address', 'get', ['contact_id' => $contact['id'], 'sequential' => 1])['values'];
 889
 890     $this->assertEquals(1, $address[1]['location_type_id']);
 891     $this->assertEquals(1, $address[1]['is_primary']);
 892     $this->assertEquals('Teeny Mansion', $address[1]['street_address']);
 893
 894     $this->assertEquals(3, $address[0]['location_type_id']);
 895     $this->assertEquals(0, $address[0]['is_primary']);
 896     $this->assertEquals('Big Mansion', $address[0]['street_address']);
 897
 898     $phone = $this->callAPISuccess('Phone', 'get', ['contact_id' => $contact['id'], 'sequential' => 1, 'options' => ['sort' => 'is_primary DESC']])['values'];
 899     $this->assertEquals(3, $phone[1]['location_type_id']);
 900     $this->assertEquals(0, $phone[1]['is_primary']);
 901     $this->assertEquals(12334, $phone[1]['phone']);
 902     $this->assertEquals(1, $phone[0]['location_type_id']);
 903     $this->assertEquals(1, $phone[0]['is_primary']);
 904     $this->assertEquals(4444, $phone[0]['phone']);
 905
 906     $this->callAPISuccess('Contact', 'delete', ['id' => $contact['id']]);
 907   }
 908
 909   /**
 910    * Test that the import parser updates the address on the existing primary location.
 911    *
 912    * @throws \Exception
 913    */
 914   public function testImportPrimaryAddressUpdate(): void {
 915     [$contactValues] = $this->setUpBaseContact(['external_identifier' => 'android']);
 916     $contactValues['email'] = 'melinda.gates@microsoft.com';
 917     $contactValues['phone'] = '98765';
 918     $contactValues['external_identifier'] = 'android';
 919     $contactValues['street_address'] = 'Big Mansion';
 920     $contactValues['city'] = 'Big City';
 921     $contactID = $this->callAPISuccessGetValue('Contact', ['external_identifier' => 'android', 'return' => 'id']);
 922     $originalAddress = $this->callAPISuccess('Address', 'create', ['location_type_id' => 2, 'street_address' => 'small house', 'contact_id' => $contactID]);
 923     $originalPhone = $this->callAPISuccess('phone', 'create', ['location_type_id' => 2, 'phone' => '1234', 'contact_id' => $contactID, 'phone_type_id' => 1]);
 924     $this->runImport($contactValues, CRM_Import_Parser::DUPLICATE_UPDATE, CRM_Import_Parser::VALID, []);
 925     $phone = $this->callAPISuccessGetSingle('Phone', ['phone' => '98765']);
 926     $this->assertEquals(2, $phone['location_type_id']);
 927     $this->assertEquals($originalPhone['id'], $phone['id']);
 928     $email = $this->callAPISuccess('Email', 'getsingle', ['contact_id' => $contactID]);
 929     $address = $this->callAPISuccessGetSingle('Address', ['street_address' => 'Big Mansion']);
 930     $this->assertEquals(2, $address['location_type_id']);
 931     $this->assertEquals($originalAddress['id'], $address['id']);
 932     $this->assertEquals('Big City', $address['city']);
 933     $this->callAPISuccessGetSingle('Contact', $contactValues);
 934   }
 935
 936   /**
 937    * Test the determination of whether a custom field is valid.
 938    */
 939   public function testCustomFieldValidation(): void {
 940     $errorMessage = '';
 941     $customGroup = $this->customGroupCreate([
 942       'extends' => 'Contact',
 943       'title' => 'ABC',
 944     ]);
 945     $customField = $this->customFieldOptionValueCreate($customGroup, 'fieldABC', ['html_type' => 'Select', 'serialize' => 1]);
 946     $params = [
 947       'custom_' . $customField['id'] => 'Label1|Label2',
 948     ];
 949     CRM_Contact_Import_Parser_Contact::isErrorInCustomData($params, $errorMessage);
 950     $this->assertEquals(NULL, $errorMessage);
 951   }
 952
 953   /**
 954    * Test the import validation.
 955    *
 956    * @dataProvider validateDataProvider
 957    *
 958    * @param string $csv
 959    * @param array $mapper Mapping as entered on MapField form.
 960    *   e.g [['first_name']['email', 1]].
 961    *   {@see \CRM_Contact_Import_Parser_Contact::getMappingFieldFromMapperInput}
 962    * @param string $expectedError
 963    * @param array $submittedValues
 964    *
 965    *
 966    * @throws \API_Exception
 967    */
 968   public function testValidation(string $csv, array $mapper, string $expectedError = '', $submittedValues = []): void {
 969     try {
 970       $this->validateCSV($csv, $mapper, $submittedValues);
 971     }
 972     catch (CRM_Core_Exception $e) {
 973       $this->assertSame($expectedError, $e->getMessage());
 974       return;
 975     }
 976     if ($expectedError) {
 977       $this->fail('expected error :' . $expectedError);
 978     }
 979   }
 980
 981   /**
 982    * Get combinations to test for validation.
 983    *
 984    * @return array[]
 985    */
 986   public function validateDataProvider(): array {
 987     return [
 988       'individual_required' => [
 989         'csv' => 'individual_invalid_missing_name.csv',
 990         'mapper' => [['last_name']],
 991         'expected_error' => 'Missing required fields: First Name OR Email Address',
 992       ],
 993       'individual_related_required_met' => [
 994         'csv' => 'individual_valid_with_related_email.csv',
 995         'mapper' => [['first_name'], ['last_name'], ['1_a_b', 'email']],
 996         'expected_error' => '',
 997       ],
 998       'individual_related_required_not_met' => [
 999         'csv' => 'individual_invalid_with_related_phone.csv',
1000         'mapper' => [['first_name'], ['last_name'], ['1_a_b', 'phone', 1, 2]],
1001         'expected_error' => '(Child of) Missing required fields: First Name and Last Name OR Email Address OR External Identifier',
1002       ],
1003       'individual_bad_email' => [
1004         'csv' => 'individual_invalid_email.csv',
1005         'mapper' => [['email', 1], ['first_name'], ['last_name']],
1006         'expected_error' => 'Invalid value for field(s) : Email',
1007       ],
1008       'individual_related_bad_email' => [
1009         'csv' => 'individual_invalid_related_email.csv',
1010         'mapper' => [['1_a_b', 'email', 1], ['first_name'], ['last_name']],
1011         'expected_error' => 'Invalid value for field(s) : (Child of) Email',
1012       ],
1013       'individual_invalid_external_identifier_only' => [
1014         // External identifier is only enough in upgrade mode.
1015         'csv' => 'individual_invalid_external_identifier_only.csv',
1016         'mapper' => [['external_identifier'], ['gender_id']],
1017         'expected_error' => 'Missing required fields: First Name and Last Name OR Email Address',
1018       ],
1019       'individual_invalid_external_identifier_only_update_mode' => [
1020         // External identifier only enough in upgrade mode, so no error here.
1021         'csv' => 'individual_invalid_external_identifier_only.csv',
1022         'mapper' => [['external_identifier'], ['gender_id']],
1023         'expected_error' => '',
1024         'submitted_values' => ['onDuplicate' => CRM_Import_Parser::DUPLICATE_UPDATE],
1025       ],
1026       'organization_email_no_organization_name' => [
1027         // Email is only enough in upgrade mode.
1028         'csv' => 'organization_email_no_organization_name.csv',
1029         'mapper' => [['email'], ['phone', 1, 1]],
1030         'expected_error' => 'Missing required fields: Organization Name',
1031         'submitted_values' => ['onDuplicate' => CRM_Import_Parser::DUPLICATE_SKIP, 'contactType' => CRM_Import_Parser::CONTACT_ORGANIZATION],
1032       ],
1033       'organization_email_no_organization_name_update_mode' => [
1034         // Email is enough in upgrade mode (at least to pass validate).
1035         'csv' => 'organization_email_no_organization_name.csv',
1036         'mapper' => [['email'], ['phone', 1, 1]],
1037         'expected_error' => '',
1038         'submitted_values' => ['onDuplicate' => CRM_Import_Parser::DUPLICATE_UPDATE, 'contactType' => CRM_Import_Parser::CONTACT_ORGANIZATION],
1039       ],
1040     ];
1041   }
1042
1043   /**
1044    * Test the import.
1045    *
1046    * @dataProvider importDataProvider
1047    *
1048    * @throws \API_Exception
1049    * @throws \CRM_Core_Exception
1050    */
1051   public function testImport($csv, $mapper, $expectedError, $expectedOutcomes = []): void {
1052     try {
1053       $this->importCSV($csv, $mapper);
1054     }
1055     catch (CRM_Core_Exception $e) {
1056       $this->assertSame($expectedError, $e->getMessage());
1057       return;
1058     }
1059     if ($expectedError) {
1060       $this->fail('expected error :' . $expectedError);
1061     }
1062     $dataSource = new CRM_Import_DataSource_CSV(UserJob::get(FALSE)->setSelect(['id'])->execute()->first()['id']);
1063     foreach ($expectedOutcomes as $outcome => $count) {
1064       $this->assertEquals($dataSource->getRowCount([$outcome]), $count);
1065     }
1066   }
1067
1068   /**
1069    * Get combinations to test for validation.
1070    *
1071    * @return array[]
1072    */
1073   public function importDataProvider(): array {
1074     return [
1075       'individual_invalid_sub_type' => [
1076         'csv' => 'individual_invalid_contact_sub_type.csv',
1077         'mapper' => [['first_name'], ['last_name'], ['contact_sub_type']],
1078         'expected_error' => '',
1079         'expected_outcomes' => [CRM_Import_Parser::ERROR => 1],
1080       ],
1081     ];
1082   }
1083
1084   /**
1085    * Test the handling of validation when importing genders.
1086    *
1087    * If it's not gonna import it should fail at the validation stage...
1088    *
1089    * @throws \API_Exception
1090    * @throws \CRM_Core_Exception
1091    */
1092   public function testImportGenders(): void {
1093     $mapper = [
1094       ['first_name'],
1095       ['last_name'],
1096       ['gender_id'],
1097       ['1_a_b', 'first_name'],
1098       ['1_a_b', 'last_name'],
1099       ['1_a_b', 'gender_id'],
1100       ['do_not_import'],
1101     ];
1102     $csv = 'individual_genders.csv';
1103     $this->validateMultiRowCsv($csv, $mapper, 'gender');
1104
1105     $this->importCSV($csv, $mapper);
1106     $contacts = Contact::get()
1107       ->addWhere('first_name', '=', 'Madame')
1108       ->addSelect('gender_id:name')->execute();
1109     foreach ($contacts as $contact) {
1110       $this->assertEquals('Female', $contact['gender_id:name']);
1111     }
1112     $this->assertCount(8, $contacts);
1113   }
1114
1115   /**
1116    * Test importing state country & county.
1117    *
1118    * @throws \API_Exception
1119    * @throws \CRM_Core_Exception
1120    */
1121   public function testImportCountryStateCounty(): void {
1122     $childKey = $this->getRelationships()['Child of']['id'] . '_a_b';
1123     // @todo - rows that don't work yet are set to do_not_import.
1124     $addressCustomGroupID = $this->createCustomGroup(['extends' => 'Address', 'name' => 'Address']);
1125     $contactCustomGroupID = $this->createCustomGroup(['extends' => 'Contact', 'name' => 'Contact']);
1126     $addressCustomFieldID = $this->createCountryCustomField(['custom_group_id' => $addressCustomGroupID])['id'];
1127     $contactCustomFieldID = $this->createMultiCountryCustomField(['custom_group_id' => $contactCustomGroupID])['id'];
1128     $contactStateCustomFieldID = $this->createStateCustomField(['custom_group_id' => $contactCustomGroupID])['id'];
1129     $customField = 'custom_' . $contactCustomFieldID;
1130     $addressCustomField = 'custom_' . $addressCustomFieldID;
1131     $contactStateCustomField = 'custom_' . $contactStateCustomFieldID;
1132
1133     $mapper = [
1134       ['first_name'],
1135       ['last_name'],
1136       ['email'],
1137       ['county'],
1138       ['country'],
1139       ['state_province'],
1140       [$contactStateCustomField],
1141       [$customField],
1142       [$addressCustomField],
1143       // [$addressCustomField, 'state_province'],
1144       ['do_not_import'],
1145       [$childKey, 'first_name'],
1146       [$childKey, 'last_name'],
1147       [$childKey, 'email'],
1148       [$childKey, 'state_province'],
1149       [$childKey, 'country'],
1150       [$childKey, 'county'],
1151       // [$childKey, $addressCustomField, 'country'],
1152       ['do_not_import'],
1153       // [$childKey, $addressCustomField, 'state_province'],
1154       ['do_not_import'],
1155       // [$childKey, $customField, 'country'],
1156       ['do_not_import'],
1157       // [$childKey, $customField, 'state_province'],
1158       ['do_not_import'],
1159       // mapField Form expects all fields to be mapped.
1160       ['do_not_import'],
1161       ['do_not_import'],
1162     ];
1163     $csv = 'individual_country_state_county_with_related.csv';
1164     $this->validateMultiRowCsv($csv, $mapper, 'error_value');
1165
1166     $this->importCSV($csv, $mapper);
1167     $contacts = $this->getImportedContacts();
1168     foreach ($contacts as $contact) {
1169       $this->assertEquals(1013, $contact['address'][0]['country_id']);
1170       $this->assertEquals(1640, $contact['address'][0]['state_province_id']);
1171     }
1172     $this->assertCount(2, $contacts);
1173   }
1174
1175   /**
1176    * Test date validation.
1177    *
1178    * @dataProvider dateDataProvider
1179    *
1180    * @param string $csv
1181    * @param int $dateType
1182    *
1183    * @throws \API_Exception
1184    * @throws \CRM_Core_Exception
1185    */
1186   public function testValidateDateData($csv, $dateType): void {
1187     $addressCustomGroupID = $this->createCustomGroup(['extends' => 'Address', 'name' => 'Address']);
1188     $contactCustomGroupID = $this->createCustomGroup(['extends' => 'Contact', 'name' => 'Contact']);
1189     $addressCustomFieldID = $this->createDateCustomField(['custom_group_id' => $addressCustomGroupID])['id'];
1190     $contactCustomFieldID = $this->createDateCustomField(['custom_group_id' => $contactCustomGroupID])['id'];
1191     $mapper = [
1192       ['first_name'],
1193       ['last_name'],
1194       ['birth_date'],
1195       ['deceased_date'],
1196       ['custom_' . $contactCustomFieldID],
1197       ['custom_' . $addressCustomFieldID, 1],
1198       ['street_address', 1],
1199       ['do_not_import'],
1200     ];
1201     // Date types should be picked up from submitted values but still some clean up to do.
1202     CRM_Core_Session::singleton()->set('dateTypes', $dateType);
1203     $this->validateMultiRowCsv($csv, $mapper, 'custom_date_one', ['dateFormats' => $dateType]);
1204     $fields = [
1205       'contact_id.birth_date',
1206       'contact_id.deceased_date',
1207       'contact_id.is_deceased',
1208       'contact_id.custom_' . $contactCustomFieldID,
1209       $addressCustomFieldID,
1210     ];
1211     $contacts = Address::get()->addWhere('contact_id.first_name', '=', 'Joe')->setSelect($fields)->execute();
1212     foreach ($contacts as $contact) {
1213       foreach ($fields as $field) {
1214         if ($field === 'contact_is_deceased') {
1215           $this->assertTrue($contact[$field]);
1216         }
1217         else {
1218           $this->assertEquals('2008-09-01', $contact[$field]);
1219         }
1220       }
1221     }
1222   }
1223
1224   /**
1225    * @throws \API_Exception
1226    */
1227   public function testImportContactSubTypes(): void {
1228     ContactType::create()->setValues([
1229       'name' => 'baby',
1230       'label' => 'Infant',
1231       'parent_id:name' => 'Individual',
1232     ])->execute();
1233     $mapper = [
1234       ['first_name'],
1235       ['last_name'],
1236       ['5_a_b', 'organization_name'],
1237       ['contact_sub_type'],
1238       ['5_a_b', 'contact_sub_type'],
1239       // mapField Form expects all fields to be mapped.
1240       ['do_not_import'],
1241       ['do_not_import'],
1242       ['do_not_import'],
1243     ];
1244     $csv = 'individual_contact_sub_types.csv';
1245     $field = 'contact_sub_type';
1246
1247     $this->validateMultiRowCsv($csv, $mapper, $field);
1248     $this->importCSV($csv, $mapper);
1249     $contacts = Contact::get()
1250       ->addWhere('last_name', '=', 'Green')
1251       ->addSelect('contact_sub_type:name')->execute();
1252     foreach ($contacts as $contact) {
1253       $this->assertEquals(['baby'], $contact['contact_sub_type:name']);
1254     }
1255     $this->assertCount(3, $contacts);
1256   }
1257
1258   /**
1259    * Data provider for date tests.
1260    *
1261    * @return array[]
1262    */
1263   public function dateDataProvider(): array {
1264     return [
1265       'type_1' => ['csv' => 'individual_dates_type1.csv', 'dateType' => CRM_Core_Form_Date::DATE_yyyy_mm_dd],
1266       'type_2' => ['csv' => 'individual_dates_type2.csv', 'dateType' => CRM_Core_Form_Date::DATE_mm_dd_yy],
1267       'type_4' => ['csv' => 'individual_dates_type4.csv', 'dateType' => CRM_Core_Form_Date::DATE_mm_dd_yyyy],
1268       'type_8' => ['csv' => 'individual_dates_type8.csv', 'dateType' => CRM_Core_Form_Date::DATE_Month_dd_yyyy],
1269       'type_16' => ['csv' => 'individual_dates_type16.csv', 'dateType' => CRM_Core_Form_Date::DATE_dd_mon_yy],
1270       'type_32' => ['csv' => 'individual_dates_type32.csv', 'dateType' => CRM_Core_Form_Date::DATE_dd_mm_yyyy],
1271     ];
1272   }
1273
1274   /**
1275    * Test location importing, including for related contacts.
1276    *
1277    * @throws \API_Exception
1278    */
1279   public function testImportLocations(): void {
1280     $csv = 'individual_locations_with_related.csv';
1281     $relationships = $this->getRelationships();
1282
1283     $childKey = $relationships['Child of']['id'] . '_a_b';
1284     $siblingKey = $relationships['Sibling of']['id'] . '_a_b';
1285     $employeeKey = $relationships['Employee of']['id'] . '_a_b';
1286     $locations = LocationType::get()->execute()->indexBy('name');
1287     $phoneTypeID = CRM_Core_PseudoConstant::getKey('CRM_Core_BAO_Phone', 'phone_type_id', 'Phone');
1288     $mobileTypeID = CRM_Core_PseudoConstant::getKey('CRM_Core_BAO_Phone', 'phone_type_id', 'Mobile');
1289     $skypeTypeID = CRM_Core_PseudoConstant::getKey('CRM_Core_BAO_IM', 'provider_id', 'Skype');
1290     $mainWebsiteTypeID = CRM_Core_PseudoConstant::getKey('CRM_Core_BAO_Website', 'website_type_id', 'Main');
1291     $linkedInTypeID = CRM_Core_PseudoConstant::getKey('CRM_Core_BAO_Website', 'website_type_id', 'LinkedIn');
1292     $homeID = $locations['Home']['id'];
1293     $workID = $locations['Work']['id'];
1294     $mapper = [
1295       ['first_name'],
1296       ['last_name'],
1297       ['birth_date'],
1298       ['street_address', $homeID],
1299       ['city', $homeID],
1300       ['postal_code', $homeID],
1301       ['country', $homeID],
1302       ['state_province', $homeID],
1303       // No location type ID means 'Primary'
1304       ['email'],
1305       ['signature_text'],
1306       ['im', NULL, $skypeTypeID],
1307       ['url', $mainWebsiteTypeID],
1308       ['phone', $homeID, $phoneTypeID],
1309       ['phone_ext', $homeID, $phoneTypeID],
1310       [$childKey, 'first_name'],
1311       [$childKey, 'last_name'],
1312       [$childKey, 'street_address'],
1313       [$childKey, 'city'],
1314       [$childKey, 'country'],
1315       [$childKey, 'state_province'],
1316       [$childKey, 'email', $homeID],
1317       [$childKey, 'signature_text', $homeID],
1318       [$childKey, 'im', $homeID, $skypeTypeID],
1319       [$childKey, 'url', $linkedInTypeID],
1320       // Same location type, different phone typ in these phones
1321       [$childKey, 'phone', $homeID, $phoneTypeID],
1322       [$childKey, 'phone_ext', $homeID, $phoneTypeID],
1323       [$childKey, 'phone', $homeID, $mobileTypeID],
1324       [$childKey, 'phone_ext', $homeID, $mobileTypeID],
1325       [$siblingKey, 'street_address', $homeID],
1326       [$siblingKey, 'city', $homeID],
1327       [$siblingKey, 'country', $homeID],
1328       [$siblingKey, 'state_province', $homeID],
1329       [$siblingKey, 'email', $homeID],
1330       [$siblingKey, 'signature_text', $homeID],
1331       [$siblingKey, 'im', $homeID, $skypeTypeID],
1332       // The 2 is website_type_id (yes, small hard-coding cheat)
1333       [$siblingKey, 'url', $linkedInTypeID],
1334       [$siblingKey, 'phone', $workID, $phoneTypeID],
1335       [$siblingKey, 'phone_ext', $workID, $phoneTypeID],
1336       [$employeeKey, 'organization_name'],
1337       [$employeeKey, 'url', $mainWebsiteTypeID],
1338       [$employeeKey, 'email', $homeID],
1339       [$employeeKey, 'do_not_import'],
1340       [$employeeKey, 'street_address', $homeID],
1341       [$employeeKey, 'supplemental_address_1', $homeID],
1342       [$employeeKey, 'do_not_import'],
1343       // Second website, different type.
1344       [$employeeKey, 'url', $linkedInTypeID],
1345       ['openid'],
1346     ];
1347     $this->validateCSV($csv, $mapper);
1348
1349     $this->importCSV($csv, $mapper);
1350     $contacts = $this->getImportedContacts();
1351     $this->assertCount(4, $contacts);
1352     $this->assertCount(1, $contacts['Susie Jones']['phone']);
1353     $this->assertEquals('123', $contacts['Susie Jones']['phone'][0]['phone_ext']);
1354     $this->assertCount(2, $contacts['Mum Jones']['phone']);
1355     $this->assertCount(1, $contacts['sis@example.com']['phone']);
1356     $this->assertCount(0, $contacts['Soccer Superstars']['phone']);
1357     $this->assertCount(1, $contacts['Susie Jones']['website']);
1358     $this->assertCount(1, $contacts['Mum Jones']['website']);
1359     $this->assertCount(0, $contacts['sis@example.com']['website']);
1360     $this->assertCount(2, $contacts['Soccer Superstars']['website']);
1361     $this->assertCount(1, $contacts['Susie Jones']['email']);
1362     $this->assertEquals('Regards', $contacts['Susie Jones']['email'][0]['signature_text']);
1363     $this->assertCount(1, $contacts['Mum Jones']['email']);
1364     $this->assertCount(1, $contacts['sis@example.com']['email']);
1365     $this->assertCount(1, $contacts['Soccer Superstars']['email']);
1366     $this->assertCount(1, $contacts['Susie Jones']['im']);
1367     $this->assertCount(1, $contacts['Mum Jones']['im']);
1368     $this->assertCount(0, $contacts['sis@example.com']['im']);
1369     $this->assertCount(0, $contacts['Soccer Superstars']['im']);
1370     $this->assertCount(1, $contacts['Susie Jones']['address']);
1371     $this->assertCount(1, $contacts['Mum Jones']['address']);
1372     $this->assertCount(1, $contacts['sis@example.com']['address']);
1373     $this->assertCount(1, $contacts['Soccer Superstars']['address']);
1374     $this->assertCount(1, $contacts['Susie Jones']['openid']);
1375   }
1376
1377   /**
1378    * Test that setting duplicate action to fill doesn't blow away data
1379    * that exists, but does fill in where it's empty.
1380    *
1381    * @throw \Exception
1382    */
1383   public function testImportFill() {
1384     // Create a custom field group for testing.
1385     $this->createCustomGroup([
1386       'title' => 'importFillGroup',
1387       'extends' => 'Individual',
1388       'is_active' => TRUE,
1389     ]);
1390     $customGroupID = $this->ids['CustomGroup']['importFillGroup'];
1391
1392     // Add two custom fields.
1393     $api_params = [
1394       'custom_group_id' => $customGroupID,
1395       'label' => 'importFillField1',
1396       'html_type' => 'Select',
1397       'data_type' => 'String',
1398       'option_values' => [
1399         'foo' => 'Foo',
1400         'bar' => 'Bar',
1401       ],
1402     ];
1403     $result = $this->callAPISuccess('custom_field', 'create', $api_params);
1404     $customField1 = $result['id'];
1405
1406     $api_params = [
1407       'custom_group_id' => $customGroupID,
1408       'label' => 'importFillField2',
1409       'html_type' => 'Select',
1410       'data_type' => 'String',
1411       'option_values' => [
1412         'baz' => 'Baz',
1413         'boo' => 'Boo',
1414       ],
1415     ];
1416     $result = $this->callAPISuccess('custom_field', 'create', $api_params);
1417     $customField2 = $result['id'];
1418
1419     // Now set up values.
1420     $original_gender = 'Male';
1421     $original_custom1 = 'foo';
1422     $original_email = 'test-import-fill@example.org';
1423
1424     $import_gender = 'Female';
1425     $import_custom1 = 'bar';
1426     $import_job_title = 'Chief data importer';
1427     $import_custom2 = 'baz';
1428
1429     // Create contact with both one known core field and one custom
1430     // field filled in.
1431     $api_params = [
1432       'contact_type' => 'Individual',
1433       'email' => $original_email,
1434       'gender' => $original_gender,
1435       'custom_' . $customField1 => $original_custom1,
1436     ];
1437     $result = $this->callAPISuccess('contact', 'create', $api_params);
1438     $contact_id = $result['id'];
1439
1440     // Run an import.
1441     $import = [
1442       'email' => $original_email,
1443       'gender_id' => $import_gender,
1444       'custom_' . $customField1 => $import_custom1,
1445       'job_title' => $import_job_title,
1446       'custom_' . $customField2 => $import_custom2,
1447     ];
1448
1449     $this->runImport($import, CRM_Import_Parser::DUPLICATE_FILL, CRM_Import_Parser::VALID);
1450
1451     $expected = [
1452       'gender' => $original_gender,
1453       'custom_' . $customField1 => $original_custom1,
1454       'job_title' => $import_job_title,
1455       'custom_' . $customField2 => $import_custom2,
1456     ];
1457
1458     $params = [
1459       'id' => $contact_id,
1460       'return' => [
1461         'gender',
1462         'custom_' . $customField1,
1463         'job_title',
1464         'custom_' . $customField2,
1465       ],
1466     ];
1467     $result = civicrm_api3('Contact', 'get', $params);
1468     $values = array_pop($result['values']);
1469     foreach ($expected as $field => $expected_value) {
1470       if (!isset($values[$field])) {
1471         $given_value = NULL;
1472       }
1473       else {
1474         $given_value = $values[$field];
1475       }
1476       // We expect:
1477       //   gender: Male
1478       //   job_title: Chief Data Importer
1479       //   importFillField1: foo
1480       //   importFillField2: baz
1481       $this->assertEquals($expected_value, $given_value, "$field properly handled during Fill import");
1482     }
1483   }
1484
1485   /**
1486    * CRM-19888 default country should be used if ambiguous.
1487    *
1488    * @throws \API_Exception
1489    * @throws \CRM_Core_Exception
1490    * @throws \CiviCRM_API3_Exception
1491    */
1492   public function testImportAmbiguousStateCountry(): void {
1493     $this->callAPISuccess('Setting', 'create', ['defaultContactCountry' => 1228]);
1494     $countries = CRM_Core_PseudoConstant::country(FALSE, FALSE);
1495     $this->callAPISuccess('Setting', 'create', ['countryLimit' => [array_search('United States', $countries, TRUE), array_search('Guyana', $countries, TRUE), array_search('Netherlands', $countries, TRUE)]]);
1496     $this->callAPISuccess('Setting', 'create', ['provinceLimit' => [array_search('United States', $countries, TRUE), array_search('Guyana', $countries, TRUE), array_search('Netherlands', $countries, TRUE)]]);
1497     [$contactValues] = $this->setUpBaseContact();
1498
1499     // Set up the field mapping  - this looks like an array per mapping as saved in
1500     // civicrm_mapping_field - eg ['name' => 'street_address', 'location_type_id' => 1],
1501     $fieldMapping = [];
1502     foreach (array_keys($contactValues) as $fieldName) {
1503       $fieldMapping[] = ['name' => $fieldName];
1504     }
1505
1506     $addressValues = [
1507       'street_address' => 'PO Box 2716',
1508       'city' => 'Midway',
1509       'state_province' => 'UT',
1510       'postal_code' => 84049,
1511       'country' => 'United States',
1512     ];
1513
1514     $homeLocationTypeID = CRM_Core_PseudoConstant::getKey('CRM_Core_BAO_Address', 'location_type_id', 'Home');
1515     $workLocationTypeID = CRM_Core_PseudoConstant::getKey('CRM_Core_BAO_Address', 'location_type_id', 'Work');
1516     foreach ($addressValues as $field => $value) {
1517       $contactValues['home_' . $field] = $value;
1518       $contactValues['work_' . $field] = $value;
1519       $fieldMapping[] = ['name' => $field, 'location_type_id' => $homeLocationTypeID];
1520       $fieldMapping[] = ['name' => $field, 'location_type_id' => $workLocationTypeID];
1521     }
1522     // The value is set to nothing to show it will be calculated.
1523     $contactValues['work_country'] = '';
1524
1525     $this->runImport($contactValues, CRM_Import_Parser::DUPLICATE_UPDATE, CRM_Import_Parser::VALID, $fieldMapping);
1526     $addresses = $this->callAPISuccess('Address', 'get', ['contact_id' => ['>' => 2], 'sequential' => 1]);
1527     $this->assertEquals(2, $addresses['count']);
1528     $this->assertEquals(array_search('United States', $countries, TRUE), $addresses['values'][0]['country_id']);
1529     $this->assertEquals(array_search('United States', $countries, TRUE), $addresses['values'][1]['country_id']);
1530   }
1531
1532   /**
1533    * Test importing fields with various options.
1534    *
1535    * Ensure we can import multiple preferred_communication_methods, single
1536    * gender, and single preferred language using both labels and values.
1537    *
1538    * @throws \API_Exception
1539    * @throws \CRM_Core_Exception
1540    * @throws \CiviCRM_API3_Exception
1541    */
1542   public function testImportFieldsWithVariousOptions(): void {
1543     $processor = new CRM_Import_ImportProcessor();
1544     $processor->setUserJobID($this->getUserJobID([
1545       'mapper' => [['first_name'], ['last_name'], ['preferred_communication_method'], ['gender_id'], ['preferred_language']],
1546     ]));
1547     $processor->setMappingFields(
1548       [
1549         ['name' => 'first_name'],
1550         ['name' => 'last_name'],
1551         ['name' => 'preferred_communication_method'],
1552         ['name' => 'gender_id'],
1553         ['name' => 'preferred_language'],
1554       ]
1555     );
1556     $importer = $processor->getImporterObject();
1557     $fields = ['Ima', 'Texter', 'SMS,Phone', 'Female', 'Danish'];
1558     $importer->import(CRM_Import_Parser::DUPLICATE_NOCHECK, $fields);
1559     $contact = $this->callAPISuccessGetSingle('Contact', ['last_name' => 'Texter']);
1560
1561     $this->assertEquals([4, 1], $contact['preferred_communication_method'], 'Import multiple preferred communication methods using labels.');
1562     $this->assertEquals(1, $contact['gender_id'], 'Import gender with label.');
1563     $this->assertEquals('da_DK', $contact['preferred_language'], 'Import preferred language with label.');
1564     $this->callAPISuccess('Contact', 'delete', ['id' => $contact['id']]);
1565
1566     $importer = $processor->getImporterObject();
1567     $fields = ['Ima', 'Texter', '4,1', '1', 'da_DK'];
1568     $importer->import(CRM_Import_Parser::DUPLICATE_NOCHECK, $fields);
1569     $contact = $this->callAPISuccessGetSingle('Contact', ['last_name' => 'Texter']);
1570
1571     $this->assertEquals([4, 1], $contact['preferred_communication_method'], 'Import multiple preferred communication methods using values.');
1572     $this->assertEquals(1, $contact['gender_id'], 'Import gender with id.');
1573     $this->assertEquals('da_DK', $contact['preferred_language'], 'Import preferred language with value.');
1574   }
1575
1576   /**
1577    * Run the import parser.
1578    *
1579    * @param array $originalValues
1580    *
1581    * @param int $onDuplicateAction
1582    * @param int $expectedResult
1583    * @param array|null $fieldMapping
1584    *   Array of field mappings in the format used in civicrm_mapping_field.
1585    * @param array|null $fields
1586    *   Array of field names. Will be calculated from $originalValues if not passed in, but
1587    *   that method does not cope with duplicates.
1588    * @param int|null $ruleGroupId
1589    *   To test against a specific dedupe rule group, pass its ID as this argument.
1590    *
1591    * @throws \API_Exception
1592    * @throws \CRM_Core_Exception
1593    * @throws \CiviCRM_API3_Exception
1594    */
1595   protected function runImport(array $originalValues, $onDuplicateAction, $expectedResult, $fieldMapping = [], $fields = NULL, int $ruleGroupId = NULL): void {
1596     $values = array_values($originalValues);
1597     // Stand in for row number.
1598     $values[] = 1;
1599
1600     if ($fieldMapping) {
1601       $fields = [];
1602       foreach ($fieldMapping as $mappedField) {
1603         $fields[] = $mappedField['name'];
1604       }
1605       $mapper = $this->getMapperFromFieldMappingFormat($fieldMapping);
1606     }
1607     else {
1608       if (!$fields) {
1609         $fields = array_keys($originalValues);
1610       }
1611       $mapper = [];
1612       foreach ($fields as $field) {
1613         $mapper[] = [
1614           $field,
1615           in_array($field, ['phone', 'email'], TRUE) ? 'Primary' : NULL,
1616           $field === 'phone' ? 1 : NULL,
1617         ];
1618       }
1619     }
1620     $userJobID = $this->getUserJobID(['mapper' => $mapper, 'onDuplicate' => $onDuplicateAction, 'dedupe_rule_id' => $ruleGroupId]);
1621     $parser = new CRM_Contact_Import_Parser_Contact($fields);
1622     $parser->setUserJobID($userJobID);
1623     $parser->_dedupeRuleGroupID = $ruleGroupId;
1624     $parser->init();
1625     $result = $parser->import($onDuplicateAction, $values);
1626     $dataSource = new CRM_Import_DataSource_CSV($userJobID);
1627     if ($result === FALSE && $expectedResult !== FALSE) {
1628       // Import is moving away from returning a status - this is a better way to check
1629       $this->assertGreaterThan(0, $dataSource->getRowCount([$expectedResult]));
1630       return;
1631     }
1632     $this->assertEquals($expectedResult, $result, 'Return code from parser import was not as expected');
1633   }
1634
1635   /**
1636    * @param string $csv
1637    * @param array $mapper Mapping as entered on MapField form.
1638    *   e.g [['first_name']['email', 1]].
1639    *   {@see \CRM_Contact_Import_Parser_Contact::getMappingFieldFromMapperInput}
1640    * @param array $submittedValues
1641    *
1642    * @return array
1643    * @throws \API_Exception
1644    * @throws \Civi\API\Exception\UnauthorizedException
1645    */
1646   protected function getDataSourceAndParser(string $csv, array $mapper, array $submittedValues): array {
1647     $userJobID = $this->getUserJobID(array_merge([
1648       'uploadFile' => ['name' => __DIR__ . '/../Form/data/' . $csv],
1649       'skipColumnHeader' => TRUE,
1650       'fieldSeparator' => ',',
1651       'onDuplicate' => CRM_Import_Parser::DUPLICATE_SKIP,
1652       'contactType' => CRM_Import_Parser::CONTACT_INDIVIDUAL,
1653       'mapper' => $mapper,
1654       'dataSource' => 'CRM_Import_DataSource_CSV',
1655     ], $submittedValues));
1656
1657     $dataSource = new CRM_Import_DataSource_CSV($userJobID);
1658     $parser = new CRM_Contact_Import_Parser_Contact();
1659     $parser->setUserJobID($userJobID);
1660     $parser->init();
1661     return [$dataSource, $parser];
1662   }
1663
1664   /**
1665    * @param int $contactID
1666    *
1667    * @throws \API_Exception
1668    * @throws \Civi\API\Exception\UnauthorizedException
1669    */
1670   protected function addChild(int $contactID): void {
1671     $relatedContactID = $this->individualCreate();
1672     $relationshipTypeID = RelationshipType::create()->setValues([
1673       'name_a_b' => 'Dad to',
1674       'name_b_a' => 'Sleep destroyer of',
1675       'contact_type_a' => 'Individual',
1676       'contact_type_b' => 'Individual',
1677       'contact_sub_type_a' => 'Parent',
1678     ])->execute()->first()['id'];
1679     \Civi\Api4\Relationship::create()->setValues([
1680       'relationship_type_id' => $relationshipTypeID,
1681       'contact_id_a' => $contactID,
1682       'contact_id_b' => $relatedContactID,
1683     ])->execute();
1684   }
1685
1686   /**
1687    * @return array
1688    * @throws \API_Exception
1689    * @throws \Civi\API\Exception\UnauthorizedException
1690    */
1691   private function getRelationships(): array {
1692     if (empty($this->relationships)) {
1693       $this->relationships = (array) RelationshipType::get()
1694         ->addSelect('name_a_b', 'id')
1695         ->execute()
1696         ->indexBy('name_a_b');
1697     }
1698     return $this->relationships;
1699   }
1700
1701   /**
1702    * Get the mapper array from the field mapping array format.
1703    *
1704    * The fieldMapping format is the same as the civicrm_mapping_field
1705    * table and is readable  - eg ['name' => 'street_address', 'location_type_id' => 1].
1706    *
1707    * The mapper format is converted to the array that would be submitted by the form
1708    * and is keyed by row number with the meaning of the fields depending on
1709    * the selection.
1710    *
1711    * @param array $fieldMapping
1712    *
1713    * @return array
1714    */
1715   protected function getMapperFromFieldMappingFormat($fieldMapping): array {
1716     $mapper = [];
1717     foreach ($fieldMapping as $mapping) {
1718       $mappedRow = [];
1719       if (!empty($mapping['relationship_type_id'])) {
1720         $mappedRow[] = $mapping['relationship_type_id'] . $mapping['relationship_direction'];
1721       }
1722       $mappedRow[] = $mapping['name'];
1723       if (!empty($mapping['location_type_id'])) {
1724         $mappedRow[] = $mapping['location_type_id'];
1725       }
1726       elseif (in_array($mapping['name'], ['email', 'phone'], TRUE)) {
1727         // Lets make it easy on test writers by assuming primary if not specified.
1728         $mappedRow[] = 'Primary';
1729       }
1730       if (!empty($mapping['im_provider_id'])) {
1731         $mappedRow[] = $mapping['im_provider_id'];
1732       }
1733       if (!empty($mapping['phone_type_id'])) {
1734         $mappedRow[] = $mapping['phone_type_id'];
1735       }
1736       if (!empty($mapping['website_type_id'])) {
1737         $mappedRow[] = $mapping['website_type_id'];
1738       }
1739       $mapper[] = $mappedRow;
1740     }
1741     return $mapper;
1742   }
1743
1744   /**
1745    * Get a suitable mapper for the array with location defaults.
1746    *
1747    * This function is designed for when 'good assumptions' are required rather
1748    * than careful mapping.
1749    *
1750    * @param array $contactValues
1751    * @param string|int $defaultLocationType
1752    *
1753    * @return array
1754    */
1755   protected function getFieldMappingFromInput(array $contactValues, $defaultLocationType = 'Primary'): array {
1756     $mapper = [];
1757     foreach (array_keys($contactValues) as $fieldName) {
1758       $mapping = ['name' => $fieldName];
1759       $addressFields = $this->callAPISuccess('Address', 'getfields', [])['values'];
1760       unset($addressFields['contact_id'], $addressFields['id'], $addressFields['location_type_id']);
1761       $locationFields = array_merge(['email', 'phone', 'im', 'openid'], array_keys($addressFields));
1762       if (in_array($fieldName, $locationFields, TRUE)) {
1763         $mapping['location_type_id'] = $defaultLocationType;
1764       }
1765       if ($fieldName === 'phone') {
1766         $mapping['phone_type_id'] = 1;
1767       }
1768       $mapper[] = $mapping;
1769     }
1770     return $mapper;
1771   }
1772
1773   /**
1774    * @param array $fields Array of fields to be imported
1775    * @param array $allfields Array of all fields which can be part of import
1776    */
1777   private function mapRelationshipFields(&$fields, $allfields) {
1778     foreach ($allfields as $key => $fieldtocheck) {
1779       $elementIndex = array_search($fieldtocheck->_title, $fields);
1780       if ($elementIndex !== FALSE) {
1781         $fields[$elementIndex] = $key;
1782       }
1783     }
1784   }
1785
1786   /**
1787    * Test mapping fields within the Parser class.
1788    *
1789    * @throws \API_Exception
1790    * @throws \Civi\API\Exception\UnauthorizedException
1791    */
1792   public function testMapFields(): void {
1793     $parser = new CRM_Contact_Import_Parser_Contact(
1794       // Array of field names
1795       ['first_name', 'phone', NULL, 'im', NULL],
1796       // Array of location types, ie columns 2 & 4 have types.
1797       [NULL, 1, NULL, 1, NULL],
1798       // Array of phone types
1799       [NULL, 1, NULL, NULL, NULL],
1800       // Array of im provider types
1801       [NULL, NULL, NULL, 1, NULL],
1802       // Array of filled in relationship values.
1803       [NULL, NULL, '5_a_b', NULL, '5_a_b'],
1804       // Array of the contact type to map to - note this can be determined from ^^
1805       [NULL, NULL, 'Organization', NULL, 'Organization'],
1806       // Related contact field names
1807       [NULL, NULL, 'url', NULL, 'phone'],
1808       // Related contact location types
1809       [NULL, NULL, NULL, NULL, 1],
1810       // Related contact phone types
1811       [NULL, NULL, NULL, NULL, 1],
1812       // Related contact im provider types
1813       [NULL, NULL, NULL, NULL, NULL],
1814       // Website types
1815       [NULL, NULL, NULL, NULL, NULL],
1816       // Related contact website types
1817       [NULL, NULL, 1, NULL, NULL]
1818     );
1819     $parser->setUserJobID($this->getUserJobID([
1820       'mapper' => [
1821         ['first_name'],
1822         ['phone', 1, 1],
1823         ['5_a_b', 'url', 1],
1824         ['im', 1, 1],
1825         ['5_a_b', 'phone', 1, 1],
1826       ],
1827     ]));
1828     $parser->init();
1829     $params = $parser->getMappedRow(
1830       ['Bob', '123', 'https://example.org', 'my-handle', '456']
1831     );
1832     $this->assertEquals([
1833       'first_name' => 'Bob',
1834       'phone' => [
1835         '1_1' => [
1836           'phone' => '123',
1837           'location_type_id' => 1,
1838           'phone_type_id' => 1,
1839         ],
1840       ],
1841       '5_a_b' => [
1842         'contact_type' => 'Organization',
1843         'contact_sub_type' => NULL,
1844         'website' => [
1845           'https://example.org' => [
1846             'url' => 'https://example.org',
1847             'website_type_id' => 1,
1848           ],
1849         ],
1850         'phone' => [
1851           '1_1' => [
1852             'phone' => '456',
1853             'location_type_id' => 1,
1854             'phone_type_id' => 1,
1855           ],
1856         ],
1857       ],
1858       'im' => [
1859         '1_1' => [
1860           'name' => 'my-handle',
1861           'location_type_id' => 1,
1862           'provider_id' => 1,
1863         ],
1864       ],
1865       'contact_type' => 'Individual',
1866     ], $params);
1867   }
1868
1869   /**
1870    * Test that import parser will not match the imported primary to
1871    * an existing contact via the related contacts fields.
1872    *
1873    * Currently fails because CRM_Dedupe_Finder::formatParams($input, $contactType);
1874    * called in getDuplicateContacts flattens the contact array adding the
1875    * related contacts values to the primary contact.
1876    *
1877    * https://github.com/civicrm/civicrm-core/blob/ca13ec46eae2042604e4e106c6cb3dc0439db3e2/CRM/Dedupe/Finder.php#L238
1878    *
1879    * @throws \API_Exception
1880    * @throws \CRM_Core_Exception
1881    * @throws \CiviCRM_API3_Exception
1882    * @throws \Civi\API\Exception\UnauthorizedException
1883    */
1884   public function testImportParserDoesNotMatchPrimaryToRelated(): void {
1885     $this->individualCreate([
1886       'first_name' => 'Bob',
1887       'last_name' => 'Dobbs',
1888       'email' => 'tim.cook@apple.com',
1889     ]);
1890
1891     $contactImportValues = [
1892       'first_name' => 'Alok',
1893       'last_name' => 'Patel',
1894       'Employee of' => 'email',
1895     ];
1896
1897     $mapper = [
1898       ['first_name'],
1899       ['last_name'],
1900       ['5_a_b', 'email'],
1901     ];
1902     $fields = array_keys($contactImportValues);
1903     $values = array_values($contactImportValues);
1904     $values[] = 'tim.cook@apple.com';
1905     // Stand in for row number.
1906     $values[] = 1;
1907
1908     $userJobID = $this->getUserJobID([
1909       'mapper' => $mapper,
1910       'onDuplicate' => CRM_Import_Parser::DUPLICATE_UPDATE,
1911     ]);
1912
1913     $parser = new CRM_Contact_Import_Parser_Contact($fields);
1914     $parser->setUserJobID($userJobID);
1915     $dataSource = new CRM_Import_DataSource_CSV($userJobID);
1916
1917     $parser->init();
1918     $parser->import(CRM_Import_Parser::DUPLICATE_UPDATE, $values);
1919     $this->assertEquals(1, $dataSource->getRowCount([CRM_Import_Parser::ERROR]));
1920     $this->callAPISuccessGetSingle('Contact', [
1921       'first_name' => 'Bob',
1922       'last_name' => 'Dobbs',
1923       'email' => 'tim.cook@apple.com',
1924     ]);
1925   }
1926
1927   /**
1928    * Set up the underlying contact.
1929    *
1930    * @param array $params
1931    *   Optional extra parameters to set.
1932    *
1933    * @return array
1934    * @throws \CRM_Core_Exception
1935    */
1936   protected function setUpBaseContact($params = []) {
1937     $originalValues = array_merge([
1938       'first_name' => 'Bill',
1939       'last_name' => 'Gates',
1940       'email' => 'bill.gates@microsoft.com',
1941       'nick_name' => 'Billy-boy',
1942     ], $params);
1943     $this->runImport($originalValues, CRM_Import_Parser::DUPLICATE_UPDATE, CRM_Import_Parser::VALID);
1944     $result = $this->callAPISuccessGetSingle('Contact', $originalValues);
1945     return [$originalValues, $result];
1946   }
1947
1948   /**
1949    * @return mixed
1950    * @throws \API_Exception
1951    * @throws \Civi\API\Exception\UnauthorizedException
1952    */
1953   protected function getUserJobID($submittedValues = []) {
1954     $userJobID = UserJob::create()->setValues([
1955       'metadata' => [
1956         'submitted_values' => array_merge([
1957           'contactType' => CRM_Import_Parser::CONTACT_INDIVIDUAL,
1958           'contactSubType' => '',
1959           'doGeocodeAddress' => 0,
1960           'dataSource' => 'CRM_Import_DataSource_SQL',
1961           'sqlQuery' => 'SELECT first_name FROM civicrm_contact',
1962           'onDuplicate' => CRM_Import_Parser::DUPLICATE_SKIP,
1963           'dedupe_rule_id' => NULL,
1964           'dateFormats' => CRM_Core_Form_Date::DATE_yyyy_mm_dd,
1965         ], $submittedValues),
1966       ],
1967       'status_id:name' => 'draft',
1968       'type_id:name' => 'contact_import',
1969     ])->execute()->first()['id'];
1970     if ($submittedValues['dataSource'] ?? NULL === 'CRM_Import_DataSource') {
1971       $dataSource = new CRM_Import_DataSource_CSV($userJobID);
1972     }
1973     else {
1974       $dataSource = new CRM_Import_DataSource_SQL($userJobID);
1975     }
1976     $dataSource->initialize();
1977     return $userJobID;
1978   }
1979
1980   /**
1981    * Test geocode validation.
1982    *
1983    * @throws \API_Exception
1984    * @throws \CRM_Core_Exception
1985    */
1986   public function testImportGeocodes(): void {
1987     $mapper = [
1988       ['first_name'],
1989       ['last_name'],
1990       ['geo_code_1', 1],
1991       ['geo_code_2', 1],
1992     ];
1993     $csv = 'individual_geocode.csv';
1994     $this->validateMultiRowCsv($csv, $mapper, 'GeoCode2');
1995   }
1996
1997   /**
1998    * Validate the csv file values.
1999    *
2000    * @param string $csv Name of csv file.
2001    * @param array $mapper Mapping as entered on MapField form.
2002    *   e.g [['first_name']['email', 1]].
2003    *   {@see \CRM_Contact_Import_Parser_Contact::getMappingFieldFromMapperInput}
2004    * @param array $submittedValues
2005    *   Any submitted values overrides.
2006    *
2007    * @throws \API_Exception
2008    */
2009   protected function validateCSV(string $csv, array $mapper, array $submittedValues = []): void {
2010     [$dataSource, $parser] = $this->getDataSourceAndParser($csv, $mapper, $submittedValues);
2011     $parser->validateValues(array_values($dataSource->getRow()));
2012   }
2013
2014   /**
2015    * Import the csv file values.
2016    *
2017    * This function uses a flow that mimics the UI flow.
2018    *
2019    * @param string $csv Name of csv file.
2020    * @param array $mapper Mapping as entered on MapField form.
2021    *   e.g [['first_name']['email', 1]].
2022    *   {@see \CRM_Contact_Import_Parser_Contact::getMappingFieldFromMapperInput}
2023    * @param array $submittedValues
2024    */
2025   protected function importCSV(string $csv, array $mapper, array $submittedValues = []): void {
2026     $submittedValues = array_merge([
2027       'uploadFile' => ['name' => __DIR__ . '/../Form/data/' . $csv],
2028       'skipColumnHeader' => TRUE,
2029       'fieldSeparator' => ',',
2030       'contactType' => CRM_Import_Parser::CONTACT_INDIVIDUAL,
2031       'mapper' => $mapper,
2032       'dataSource' => 'CRM_Import_DataSource_CSV',
2033       'file' => ['name' => $csv],
2034       'dateFormats' => CRM_Core_Form_Date::DATE_yyyy_mm_dd,
2035       'onDuplicate' => CRM_Import_Parser::DUPLICATE_UPDATE,
2036       'groups' => [],
2037     ], $submittedValues);
2038     $form = $this->getFormObject('CRM_Contact_Import_Form_DataSource', $submittedValues);
2039     $form->buildForm();
2040     $form->postProcess();
2041     $userJobID = $form->getUserJobID();
2042     /* @var CRM_Contact_Import_Form_MapField $form */
2043     $form = $this->getFormObject('CRM_Contact_Import_Form_MapField', $submittedValues);
2044     $form->setUserJobID($userJobID);
2045     $form->buildForm();
2046     $form->postProcess();
2047     /* @var CRM_Contact_Import_Form_MapField $form */
2048     $form = $this->getFormObject('CRM_Contact_Import_Form_Preview', $submittedValues);
2049     $form->setUserJobID($userJobID);
2050     $form->buildForm();
2051     $form->postProcess();
2052   }
2053
2054   /**
2055    * Validate a csv with multiple rows in it.
2056    *
2057    * @param string $csv
2058    * @param array $mapper Mapping as entered on MapField form.
2059    *   e.g [['first_name']['email', 1]].
2060    * @param string $field
2061    *   Name of the field whose data should be output in the error message.
2062    * @param array $submittedValues
2063    *   Values submitted in the form process.
2064    *
2065    * @throws \API_Exception
2066    * @throws \CRM_Core_Exception
2067    * @throws \Civi\API\Exception\UnauthorizedException
2068    */
2069   private function validateMultiRowCsv(string $csv, array $mapper, string $field, $submittedValues = []): void {
2070     /* @var CRM_Import_DataSource_CSV $dataSource */
2071     /* @var \CRM_Contact_Import_Parser_Contact $parser */
2072     [$dataSource, $parser] = $this->getDataSourceAndParser($csv, $mapper, $submittedValues);
2073     while ($values = $dataSource->getRow()) {
2074       try {
2075         $parser->validateValues(array_values($values));
2076         if ($values['expected'] !== 'Valid') {
2077           $this->fail($values[$field] . ' should not have been valid');
2078         }
2079       }
2080       catch (CRM_Core_Exception $e) {
2081         if ($values['expected'] !== 'Invalid') {
2082           $this->fail($values[$field] . ' should have been valid');
2083         }
2084       }
2085     }
2086   }
2087
2088   /**
2089    * Get the contacts we imported (Susie Jones & family).
2090    *
2091    * @return array
2092    * @throws \API_Exception
2093    */
2094   public function getImportedContacts(): array {
2095     return (array) Contact::get()
2096       ->addWhere('display_name', 'IN', [
2097         'Susie Jones',
2098         'Mum Jones',
2099         'sis@example.com',
2100         'Soccer Superstars',
2101       ])
2102       ->addChain('phone', Phone::get()->addWhere('contact_id', '=', '$id'))
2103       ->addChain('address', Address::get()->addWhere('contact_id', '=', '$id'))
2104       ->addChain('website', Website::get()->addWhere('contact_id', '=', '$id'))
2105       ->addChain('im', IM::get()->addWhere('contact_id', '=', '$id'))
2106       ->addChain('email', Email::get()->addWhere('contact_id', '=', '$id'))
2107       ->addChain('openid', OpenID::get()->addWhere('contact_id', '=', '$id'))
2108       ->execute()->indexBy('display_name');
2109   }
2110
2111   /**
2112    * Test that import parser will not throw error if Related Contact is not found via passed in External ID.
2113    *
2114    * Currently fails because validation assumes the Related contact will be found.
2115    * When it is later not found creating the contact via the API throws an
2116    * error for missing required fields.
2117    *
2118    * @throws \API_Exception
2119    * @throws \CRM_Core_Exception
2120    * @throws \CiviCRM_API3_Exception
2121    */
2122   public function testImportParserWithExternalIdForRelationship(): void {
2123     $contactImportValues = [
2124       'first_name' => 'Alok',
2125       'last_name' => 'Patel',
2126       'Employee of' => 'related external identifier',
2127     ];
2128
2129     $mapper = [
2130       ['first_name'],
2131       ['last_name'],
2132       ['5_a_b', 'external_identifier'],
2133     ];
2134     $fields = array_keys($contactImportValues);
2135     $values = array_values($contactImportValues);
2136     $userJobID = $this->getUserJobID([
2137       'mapper' => $mapper,
2138     ]);
2139
2140     $parser = new CRM_Contact_Import_Parser_Contact($fields);
2141     $parser->setUserJobID($userJobID);
2142     $parser->init();
2143
2144     $parser->import(CRM_Import_Parser::DUPLICATE_UPDATE, $values);
2145   }
2146
2147 }